From a49c06b1789b398ac4c9207cc5bd0cd369342e85 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 13 Jun 2019 18:57:49 +0300 Subject: [PATCH 001/542] server: drop unused dep on tls from config_validation (#7260) Signed-off-by: Dmitry Rozhkov --- source/server/config_validation/BUILD | 1 - source/server/config_validation/server.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/source/server/config_validation/BUILD b/source/server/config_validation/BUILD index fb2b4d08cf8a..a649fdd6bbb2 100644 --- a/source/server/config_validation/BUILD +++ b/source/server/config_validation/BUILD @@ -104,7 +104,6 @@ envoy_cc_library( "//source/common/runtime:runtime_lib", "//source/common/stats:stats_lib", "//source/common/thread_local:thread_local_lib", - "//source/extensions/transport_sockets/tls:context_lib", "//source/server:configuration_lib", "//source/server:server_lib", "//source/server/http:admin_lib", diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index 0af5c6db6d64..bc772e940574 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -25,8 +25,6 @@ #include "server/listener_manager_impl.h" #include "server/server.h" -#include "extensions/transport_sockets/tls/context_manager_impl.h" - #include "absl/types/optional.h" namespace Envoy { From fd7c70b3f3739554974769f1d0e72b706eb5c5a3 Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Thu, 13 Jun 2019 09:58:56 -0700 Subject: [PATCH 002/542] cleanup: {}->default, typedef->using, virtual->override (#7249) Signed-off-by: Fred Douglas --- STYLE.md | 6 +- bazel/DEVELOPER.md | 2 +- include/envoy/access_log/access_log.h | 12 +-- include/envoy/api/api.h | 4 +- include/envoy/api/io_error.h | 8 +- include/envoy/api/os_sys_calls.h | 4 +- include/envoy/api/os_sys_calls_common.h | 10 +-- include/envoy/api/os_sys_calls_hot_restart.h | 4 +- include/envoy/api/os_sys_calls_linux.h | 4 +- include/envoy/buffer/buffer.h | 10 +-- include/envoy/common/backoff_strategy.h | 4 +- include/envoy/common/callback.h | 2 +- include/envoy/common/interval_set.h | 4 +- include/envoy/common/mutex_tracer.h | 2 +- include/envoy/common/time.h | 4 +- include/envoy/common/token_bucket.h | 2 +- include/envoy/compressor/compressor.h | 2 +- include/envoy/config/grpc_mux.h | 10 +-- include/envoy/config/typed_metadata.h | 6 +- include/envoy/config/xds_grpc_context.h | 2 +- include/envoy/decompressor/decompressor.h | 2 +- include/envoy/event/deferred_deletable.h | 4 +- include/envoy/event/dispatcher.h | 6 +- include/envoy/event/file_event.h | 6 +- include/envoy/event/signal.h | 6 +- include/envoy/event/timer.h | 10 +-- include/envoy/filesystem/filesystem.h | 6 +- include/envoy/filesystem/watcher.h | 4 +- include/envoy/grpc/async_client.h | 12 +-- include/envoy/grpc/async_client_manager.h | 8 +- include/envoy/grpc/google_grpc_creds.h | 2 +- include/envoy/http/async_client.h | 12 +-- include/envoy/http/codec.h | 18 ++--- include/envoy/http/conn_pool.h | 10 +-- include/envoy/http/filter.h | 18 ++--- include/envoy/http/header_map.h | 14 ++-- include/envoy/http/message.h | 4 +- include/envoy/http/query_params.h | 2 +- include/envoy/json/json_object.h | 6 +- include/envoy/local_info/local_info.h | 4 +- include/envoy/network/address.h | 10 +-- include/envoy/network/connection.h | 10 +-- include/envoy/network/connection_handler.h | 4 +- include/envoy/network/dns.h | 8 +- include/envoy/network/drain_decision.h | 2 +- include/envoy/network/filter.h | 51 +++++++------ include/envoy/network/io_handle.h | 4 +- include/envoy/network/listen_socket.h | 20 ++--- include/envoy/network/listener.h | 10 +-- include/envoy/network/resolver.h | 2 +- include/envoy/network/transport_socket.h | 14 ++-- include/envoy/router/rds.h | 4 +- .../router/route_config_provider_manager.h | 2 +- include/envoy/router/router.h | 63 ++++++++-------- include/envoy/router/router_ratelimit.h | 8 +- include/envoy/router/shadow_writer.h | 4 +- include/envoy/runtime/runtime.h | 14 ++-- include/envoy/secret/secret_callbacks.h | 2 +- include/envoy/secret/secret_manager.h | 2 +- include/envoy/secret/secret_provider.h | 20 ++--- include/envoy/server/access_log_config.h | 2 +- include/envoy/server/admin.h | 12 ++- include/envoy/server/config_tracker.h | 12 +-- include/envoy/server/configuration.h | 6 +- include/envoy/server/drain_manager.h | 2 +- include/envoy/server/filter_config.h | 14 ++-- include/envoy/server/guarddog.h | 2 +- include/envoy/server/health_checker_config.h | 4 +- include/envoy/server/hot_restart.h | 2 +- include/envoy/server/instance.h | 2 +- include/envoy/server/listener_manager.h | 8 +- include/envoy/server/options.h | 4 +- include/envoy/server/overload_manager.h | 6 +- include/envoy/server/process_context.h | 4 +- include/envoy/server/resource_monitor.h | 6 +- .../envoy/server/resource_monitor_config.h | 4 +- include/envoy/server/tracer_config.h | 2 +- .../envoy/server/transport_socket_config.h | 4 +- include/envoy/server/watchdog.h | 4 +- include/envoy/server/worker.h | 8 +- include/envoy/singleton/instance.h | 4 +- include/envoy/singleton/manager.h | 8 +- .../certificate_validation_context_config.h | 4 +- include/envoy/ssl/connection.h | 2 +- include/envoy/ssl/context.h | 10 +-- include/envoy/ssl/context_config.h | 7 +- include/envoy/ssl/context_manager.h | 2 +- include/envoy/ssl/tls_certificate_config.h | 4 +- include/envoy/stats/histogram.h | 10 +-- include/envoy/stats/scope.h | 6 +- include/envoy/stats/stat_data_allocator.h | 2 +- include/envoy/stats/stats.h | 10 +-- include/envoy/stats/stats_matcher.h | 4 +- include/envoy/stats/store.h | 6 +- include/envoy/stats/tag_extractor.h | 4 +- include/envoy/stats/tag_producer.h | 4 +- include/envoy/stats/timespan.h | 8 +- include/envoy/stream_info/filter_state.h | 4 +- include/envoy/stream_info/stream_info.h | 4 +- include/envoy/tcp/conn_pool.h | 20 ++--- include/envoy/thread/thread.h | 12 +-- include/envoy/thread_local/thread_local.h | 12 +-- include/envoy/tracing/http_tracer.h | 14 ++-- include/envoy/upstream/cluster_manager.h | 24 +++--- .../upstream/health_check_host_monitor.h | 4 +- include/envoy/upstream/health_checker.h | 12 +-- include/envoy/upstream/host_description.h | 4 +- include/envoy/upstream/load_balancer.h | 14 ++-- include/envoy/upstream/load_balancer_type.h | 2 +- include/envoy/upstream/outlier_detection.h | 18 ++--- include/envoy/upstream/resource_manager.h | 4 +- include/envoy/upstream/retry.h | 12 +-- include/envoy/upstream/thread_local_cluster.h | 2 +- include/envoy/upstream/types.h | 4 +- include/envoy/upstream/upstream.h | 73 +++++++++---------- source/common/api/os_sys_calls_impl.h | 2 +- .../api/os_sys_calls_impl_hot_restart.h | 2 +- source/common/api/os_sys_calls_impl_linux.h | 2 +- source/common/buffer/watermark_buffer.h | 2 +- source/common/chromium_url/url_canon.h | 4 +- source/common/common/assert.h | 4 +- source/common/common/callback_impl.h | 2 +- source/common/common/linked_object.h | 2 +- source/common/common/logger.h | 2 +- source/common/common/logger_delegates.h | 2 +- source/common/common/matchers.h | 4 +- source/common/common/utility.cc | 2 +- source/common/common/utility.h | 8 +- source/common/config/resources.h | 2 +- source/common/config/utility.h | 2 +- source/common/config/well_known_names.h | 8 +- source/common/crypto/utility.h | 2 +- source/common/event/libevent.h | 8 +- source/common/filesystem/file_shared_impl.h | 6 +- .../common/filesystem/kqueue/watcher_impl.h | 2 +- source/common/grpc/google_async_client_impl.h | 4 +- source/common/grpc/typed_async_client.h | 6 +- source/common/http/codec_client.h | 6 +- source/common/http/conn_manager_config.h | 6 +- source/common/http/conn_manager_impl.h | 6 +- source/common/http/conn_pool_base.h | 2 +- source/common/http/date_provider.h | 2 +- source/common/http/header_map_impl.h | 2 +- source/common/http/headers.h | 2 +- source/common/http/http1/conn_pool.h | 4 +- source/common/http/http2/codec_impl.h | 2 +- source/common/http/http2/conn_pool.h | 2 +- source/common/http/http2/metadata_decoder.h | 2 +- source/common/http/http2/metadata_encoder.h | 2 +- source/common/json/json_loader.cc | 2 +- source/common/network/filter_manager_impl.h | 12 +-- source/common/network/lc_trie.h | 10 +-- source/common/network/listen_socket_impl.h | 4 +- source/common/network/utility.h | 2 +- source/common/protobuf/protobuf.h | 6 +- source/common/protobuf/utility.h | 2 +- source/common/router/config_impl.h | 19 +++-- source/common/router/header_formatter.h | 4 +- source/common/router/header_parser.h | 2 +- .../router/metadatamatchcriteria_impl.h | 2 +- source/common/router/router.h | 4 +- source/common/secret/sds_api.h | 6 +- source/common/stats/thread_local_store.h | 2 +- source/common/tcp/conn_pool.h | 8 +- source/common/tcp_proxy/tcp_proxy.h | 8 +- source/common/tracing/http_tracer_impl.h | 6 +- source/common/upstream/cluster_manager_impl.h | 14 ++-- .../upstream/health_checker_base_impl.h | 2 +- source/common/upstream/health_checker_impl.h | 6 +- .../upstream/health_discovery_service.h | 4 +- source/common/upstream/load_balancer_impl.h | 2 +- source/common/upstream/load_stats_reporter.h | 2 +- .../common/upstream/resource_manager_impl.h | 2 +- source/common/upstream/ring_hash_lb.h | 2 +- source/common/upstream/strict_dns_cluster.h | 2 +- source/common/upstream/subset_lb.h | 14 ++-- source/common/upstream/thread_aware_lb_impl.h | 8 +- source/common/upstream/upstream_impl.h | 6 +- .../http_grpc/grpc_access_log_impl.h | 6 +- .../access_loggers/well_known_names.h | 2 +- .../extensions/clusters/redis/redis_cluster.h | 4 +- .../clusters/redis/redis_cluster_lb.h | 6 +- .../filters/common/ext_authz/ext_authz.h | 8 +- .../common/ext_authz/ext_authz_grpc_impl.h | 4 +- .../common/ext_authz/ext_authz_http_impl.h | 6 +- .../filters/common/fault/fault_config.h | 2 +- source/extensions/filters/common/lua/lua.h | 6 +- .../filters/common/ratelimit/ratelimit.h | 6 +- .../filters/common/ratelimit/ratelimit_impl.h | 6 +- .../extensions/filters/common/rbac/engine.h | 2 +- .../extensions/filters/common/rbac/matchers.h | 4 +- .../extensions/filters/common/rbac/utility.h | 2 +- .../filters/http/buffer/buffer_filter.h | 2 +- .../http/common/aws/credentials_provider.h | 2 +- .../filters/http/common/aws/signer.h | 2 +- .../filters/http/common/aws/signer_impl.h | 4 +- .../filters/http/common/jwks_fetcher.h | 6 +- .../filters/http/cors/cors_filter.h | 2 +- .../filters/http/csrf/csrf_filter.cc | 2 +- .../filters/http/csrf/csrf_filter.h | 2 +- .../filters/http/ext_authz/ext_authz.cc | 2 +- .../filters/http/ext_authz/ext_authz.h | 2 +- .../filters/http/fault/fault_filter.cc | 2 +- .../filters/http/fault/fault_filter.h | 2 +- .../http/grpc_http1_reverse_bridge/filter.cc | 2 +- .../json_transcoder_filter.cc | 2 +- .../json_transcoder_filter.h | 2 +- .../filters/http/grpc_web/grpc_web_filter.cc | 2 +- .../filters/http/grpc_web/grpc_web_filter.h | 2 +- .../filters/http/gzip/gzip_filter.h | 2 +- .../header_to_metadata_filter.h | 12 +-- .../filters/http/health_check/health_check.cc | 2 +- .../filters/http/health_check/health_check.h | 10 +-- .../http/ip_tagging/ip_tagging_filter.h | 2 +- .../filters/http/jwt_authn/authenticator.h | 12 +-- .../filters/http/jwt_authn/extractor.cc | 4 +- .../filters/http/jwt_authn/extractor.h | 8 +- .../filters/http/jwt_authn/filter.cc | 5 +- .../filters/http/jwt_authn/filter_config.h | 12 +-- .../filters/http/jwt_authn/jwks_cache.h | 6 +- .../filters/http/jwt_authn/matcher.h | 4 +- .../filters/http/jwt_authn/verifier.h | 12 +-- .../extensions/filters/http/lua/lua_filter.h | 6 +- source/extensions/filters/http/lua/wrappers.h | 2 +- .../filters/http/ratelimit/ratelimit.cc | 2 +- .../filters/http/ratelimit/ratelimit.h | 2 +- .../filters/http/rbac/rbac_filter.cc | 2 +- .../filters/http/rbac/rbac_filter.h | 4 +- .../filters/http/squash/squash_filter.h | 2 +- .../filters/http/well_known_names.h | 2 +- .../listener/proxy_protocol/proxy_protocol.h | 2 +- .../listener/tls_inspector/tls_inspector.h | 2 +- .../filters/listener/well_known_names.h | 2 +- .../network/client_ssl_auth/client_ssl_auth.h | 4 +- .../filters/network/common/redis/client.h | 12 +-- .../network/common/redis/client_impl.h | 2 +- .../filters/network/common/redis/codec.h | 16 ++-- .../filters/network/dubbo_proxy/BUILD | 1 + .../network/dubbo_proxy/active_message.h | 6 +- .../filters/network/dubbo_proxy/config.cc | 4 +- .../filters/network/dubbo_proxy/decoder.cc | 2 +- .../filters/network/dubbo_proxy/decoder.h | 4 +- .../dubbo_proxy/decoder_event_handler.h | 2 +- .../network/dubbo_proxy/deserializer.h | 14 ++-- .../network/dubbo_proxy/filters/filter.h | 6 +- .../dubbo_proxy/filters/well_known_names.h | 2 +- .../filters/network/dubbo_proxy/message.h | 10 +-- .../filters/network/dubbo_proxy/metadata.h | 8 +- .../filters/network/dubbo_proxy/protocol.h | 6 +- .../dubbo_proxy/router/route_matcher.h | 8 +- .../network/dubbo_proxy/router/router.h | 4 +- .../filters/network/ext_authz/ext_authz.h | 2 +- .../network/http_connection_manager/config.cc | 4 +- .../network/http_connection_manager/config.h | 2 +- .../filters/network/kafka/kafka_request.h | 4 +- .../network/kafka/kafka_request_parser.h | 4 +- .../filters/network/kafka/kafka_types.h | 6 +- .../filters/network/kafka/request_codec.h | 2 +- .../serialization_composite_test_cc.j2 | 2 +- .../filters/network/mongo_proxy/bson.h | 10 +-- .../filters/network/mongo_proxy/codec.h | 24 +++--- .../filters/network/mongo_proxy/proxy.cc | 2 +- .../filters/network/mongo_proxy/proxy.h | 6 +- .../filters/network/mysql_proxy/mysql_codec.h | 2 +- .../network/mysql_proxy/mysql_decoder.h | 6 +- .../filters/network/ratelimit/ratelimit.h | 2 +- .../filters/network/rbac/rbac_filter.h | 4 +- .../network/redis_proxy/command_splitter.h | 2 +- .../redis_proxy/command_splitter_impl.h | 4 +- .../filters/network/redis_proxy/conn_pool.h | 4 +- .../network/redis_proxy/conn_pool_impl.h | 2 +- .../network/redis_proxy/proxy_filter.h | 2 +- .../filters/network/redis_proxy/router.h | 8 +- .../filters/network/redis_proxy/router_impl.h | 4 +- .../filters/network/thrift_proxy/config.cc | 9 +-- .../network/thrift_proxy/conn_manager.h | 10 +-- .../filters/network/thrift_proxy/decoder.h | 8 +- .../network/thrift_proxy/decoder_events.h | 4 +- .../network/thrift_proxy/filters/filter.h | 12 +-- .../thrift_proxy/filters/filter_config.h | 2 +- .../filters/ratelimit/ratelimit.h | 70 +++++++----------- .../thrift_proxy/filters/well_known_names.h | 2 +- .../filters/network/thrift_proxy/metadata.h | 4 +- .../filters/network/thrift_proxy/protocol.h | 10 +-- .../network/thrift_proxy/protocol_converter.h | 4 +- .../network/thrift_proxy/router/router.h | 10 +-- .../network/thrift_proxy/router/router_impl.h | 4 +- .../thrift_proxy/router/router_ratelimit.h | 8 +- .../filters/network/thrift_proxy/thrift.h | 4 +- .../network/thrift_proxy/thrift_object.h | 26 +++---- .../filters/network/thrift_proxy/tracing.h | 6 +- .../filters/network/thrift_proxy/transport.h | 6 +- .../thrift_proxy/twitter_protocol_impl.cc | 24 +++--- .../filters/network/well_known_names.h | 2 +- .../filters/network/zookeeper_proxy/decoder.h | 6 +- .../grpc_credentials/well_known_names.h | 2 +- .../extensions/health_checkers/redis/redis.h | 2 +- .../health_checkers/well_known_names.h | 2 +- .../fixed_heap/fixed_heap_monitor.h | 4 +- .../resource_monitors/well_known_names.h | 2 +- .../extensions/retry/host/well_known_names.h | 2 +- .../retry/priority/well_known_names.h | 2 +- .../extensions/stat_sinks/hystrix/hystrix.h | 8 +- .../grpc_metrics_service_impl.h | 4 +- .../extensions/stat_sinks/well_known_names.h | 2 +- .../common/ot/opentracing_driver_impl.cc | 5 +- .../tracers/datadog/datadog_tracer_impl.h | 4 +- .../opencensus/opencensus_tracer_impl.cc | 2 +- source/extensions/tracers/well_known_names.h | 2 +- source/extensions/tracers/zipkin/tracer.h | 6 +- .../tracers/zipkin/tracer_interface.h | 2 +- .../tracers/zipkin/zipkin_core_constants.h | 2 +- .../tracers/zipkin/zipkin_core_types.h | 6 +- .../tracers/zipkin/zipkin_json_field_names.h | 2 +- .../tracers/zipkin/zipkin_tracer_impl.h | 2 +- .../transport_sockets/alts/config.cc | 4 +- .../transport_sockets/alts/grpc_tsi.h | 6 +- .../alts/noop_transport_socket_callbacks.h | 2 +- .../alts/tsi_frame_protector.h | 2 +- .../transport_sockets/alts/tsi_handshaker.h | 6 +- .../transport_sockets/alts/tsi_socket.h | 7 +- .../transport_sockets/raw_buffer/config.h | 2 +- .../extensions/transport_sockets/tap/config.h | 2 +- .../extensions/transport_sockets/tls/config.h | 2 +- .../transport_sockets/tls/context_impl.h | 2 +- .../transport_sockets/well_known_names.h | 2 +- source/server/configuration_impl.h | 2 +- source/server/connection_handler_impl.h | 12 +-- source/server/listener_hooks.h | 2 +- source/server/listener_manager_impl.h | 4 +- source/server/options_impl.h | 2 +- source/server/overload_manager_impl.h | 8 +- source/server/server.h | 2 +- test/common/buffer/buffer_fuzz.cc | 2 +- test/common/buffer/utility.h | 2 +- test/common/common/phantom_test.cc | 4 +- test/common/common/utility_test.cc | 2 +- .../common/config/subscription_test_harness.h | 2 +- test/common/filesystem/directory_test.cc | 2 +- test/common/grpc/grpc_client_integration.h | 2 +- .../grpc_client_integration_test_harness.h | 2 +- test/common/http/codec_impl_fuzz_test.cc | 2 +- test/common/http/common.h | 2 +- .../http/conn_manager_impl_fuzz_test.cc | 2 +- test/common/http/header_map_impl_test.cc | 8 +- test/common/http/http2/codec_impl_test.cc | 4 +- test/common/http/http2/frame_replay.h | 2 +- test/common/network/dns_impl_test.cc | 6 +- test/common/router/header_formatter_test.cc | 2 +- test/common/secret/sds_api_test.cc | 2 +- .../singleton/threadsafe_singleton_test.cc | 2 +- .../upstream/health_checker_impl_test.cc | 9 +-- .../upstream/load_balancer_impl_test.cc | 2 +- .../upstream/logical_dns_cluster_test.cc | 4 +- test/common/upstream/ring_hash_lb_test.cc | 2 +- test/common/upstream/subset_lb_test.cc | 6 +- test/common/upstream/upstream_impl_test.cc | 4 +- test/config/utility.h | 7 +- .../clusters/redis/redis_cluster_test.cc | 5 +- .../filters/common/ext_authz/test_common.h | 6 +- .../buffer/buffer_filter_integration_test.cc | 2 +- .../http/jwt_authn/filter_integration_test.cc | 2 +- .../http/jwt_authn/group_verifier_test.cc | 2 +- .../http/rbac/rbac_filter_integration_test.cc | 2 +- .../network/dubbo_proxy/decoder_test.cc | 2 +- .../network/kafka/request_codec_unit_test.cc | 4 +- .../network/kafka/serialization_utilities.h | 2 +- .../network/thrift_proxy/decoder_test.cc | 2 +- .../thrift_proxy/thrift_object_impl_test.cc | 2 +- .../health_checkers/redis/config_test.cc | 2 +- .../tracers/zipkin/zipkin_tracer_impl_test.cc | 2 +- test/integration/autonomous_upstream.h | 2 +- test/integration/fake_upstream.h | 14 ++-- .../filter_manager_integration_test.cc | 2 +- test/integration/http_integration.h | 4 +- test/integration/http_protocol_integration.h | 2 +- test/integration/integration.h | 6 +- test/integration/protocol_integration_test.cc | 2 +- test/integration/server.h | 2 +- test/integration/server_stats.h | 2 +- test/integration/tcp_dump.h | 2 +- test/integration/utility.h | 4 +- test/integration/xds_integration_test.cc | 2 +- test/mocks/buffer/mocks.h | 2 +- test/server/connection_handler_test.cc | 2 +- test/test_common/environment.h | 4 +- test/test_common/logging.h | 4 +- test/test_common/printers.h | 2 +- 388 files changed, 1095 insertions(+), 1118 deletions(-) diff --git a/STYLE.md b/STYLE.md index 1a31a95c2cc5..c06d6d5282d7 100644 --- a/STYLE.md +++ b/STYLE.md @@ -39,9 +39,9 @@ * 100 columns is the line limit. * Use your GitHub name in TODO comments, e.g. `TODO(foobar): blah`. * Smart pointers are type aliased: - * `typedef std::unique_ptr FooPtr;` - * `typedef std::shared_ptr BarSharedPtr;` - * `typedef std::shared_ptr BlahConstSharedPtr;` + * `using FooPtr = std::unique_ptr;` + * `using BarSharedPtr = std::shared_ptr;` + * `using BlahConstSharedPtr = std::shared_ptr;` * Regular pointers (e.g. `int* foo`) should not be type aliased. * If move semantics are intended, prefer specifying function arguments with `&&`. E.g., `void onHeaders(Http::HeaderMapPtr&& headers, ...)`. The rationale for this is that it diff --git a/bazel/DEVELOPER.md b/bazel/DEVELOPER.md index e54d30ea9e62..41a3d5046e04 100644 --- a/bazel/DEVELOPER.md +++ b/bazel/DEVELOPER.md @@ -37,7 +37,7 @@ As an example, consider adding the following interface in `include/envoy/foo/bar class Bar { public: - virtual ~Bar() {} + virtual ~Bar() = default; virtual void someThing() PURE; ... diff --git a/include/envoy/access_log/access_log.h b/include/envoy/access_log/access_log.h index 952265933065..3648a6e44a67 100644 --- a/include/envoy/access_log/access_log.h +++ b/include/envoy/access_log/access_log.h @@ -12,7 +12,7 @@ namespace AccessLog { class AccessLogFile { public: - virtual ~AccessLogFile() {} + virtual ~AccessLogFile() = default; /** * Write data to the file. @@ -34,7 +34,7 @@ using AccessLogFileSharedPtr = std::shared_ptr; class AccessLogManager { public: - virtual ~AccessLogManager() {} + virtual ~AccessLogManager() = default; /** * Reopen all of the access log files. @@ -56,7 +56,7 @@ using AccessLogManagerPtr = std::unique_ptr; */ class Filter { public: - virtual ~Filter() {} + virtual ~Filter() = default; /** * Evaluate whether an access log should be written based on request and response data. @@ -74,7 +74,7 @@ using FilterPtr = std::unique_ptr; */ class Instance { public: - virtual ~Instance() {} + virtual ~Instance() = default; /** * Log a completed request. @@ -97,7 +97,7 @@ using InstanceSharedPtr = std::shared_ptr; */ class Formatter { public: - virtual ~Formatter() {} + virtual ~Formatter() = default; /** * Return a formatted access log line. @@ -121,7 +121,7 @@ using FormatterPtr = std::unique_ptr; */ class FormatterProvider { public: - virtual ~FormatterProvider() {} + virtual ~FormatterProvider() = default; /** * Extract a value from the provided headers/trailers/stream. diff --git a/include/envoy/api/api.h b/include/envoy/api/api.h index 6f397cae0102..8e2a019777b5 100644 --- a/include/envoy/api/api.h +++ b/include/envoy/api/api.h @@ -17,7 +17,7 @@ namespace Api { */ class Api { public: - virtual ~Api() {} + virtual ~Api() = default; /** * Allocate a dispatcher. @@ -54,7 +54,7 @@ class Api { virtual const Stats::Scope& rootScope() PURE; }; -typedef std::unique_ptr ApiPtr; +using ApiPtr = std::unique_ptr; } // namespace Api } // namespace Envoy diff --git a/include/envoy/api/io_error.h b/include/envoy/api/io_error.h index fb5dda090efc..dd80d36d58f5 100644 --- a/include/envoy/api/io_error.h +++ b/include/envoy/api/io_error.h @@ -28,7 +28,7 @@ class IoError { // Other error codes cannot be mapped to any one above in getErrorCode(). UnknownError }; - virtual ~IoError() {} + virtual ~IoError() = default; virtual IoErrorCode getErrorCode() const PURE; virtual std::string getErrorDetails() const PURE; @@ -47,12 +47,12 @@ using IoErrorPtr = std::unique_ptr; template struct IoCallResult { IoCallResult(ReturnValue rc, IoErrorPtr err) : rc_(rc), err_(std::move(err)) {} - IoCallResult(IoCallResult&& result) + IoCallResult(IoCallResult&& result) noexcept : rc_(result.rc_), err_(std::move(result.err_)) {} - virtual ~IoCallResult() {} + virtual ~IoCallResult() = default; - IoCallResult& operator=(IoCallResult&& result) { + IoCallResult& operator=(IoCallResult&& result) noexcept { rc_ = result.rc_; err_ = std::move(result.err_); return *this; diff --git a/include/envoy/api/os_sys_calls.h b/include/envoy/api/os_sys_calls.h index ba324f3472c2..bdece6810d02 100644 --- a/include/envoy/api/os_sys_calls.h +++ b/include/envoy/api/os_sys_calls.h @@ -21,7 +21,7 @@ namespace Api { class OsSysCalls { public: - virtual ~OsSysCalls() {} + virtual ~OsSysCalls() = default; /** * @see bind (man 2 bind) @@ -109,7 +109,7 @@ class OsSysCalls { virtual SysCallIntResult getsockname(int sockfd, sockaddr* addr, socklen_t* addrlen) PURE; }; -typedef std::unique_ptr OsSysCallsPtr; +using OsSysCallsPtr = std::unique_ptr; } // namespace Api } // namespace Envoy diff --git a/include/envoy/api/os_sys_calls_common.h b/include/envoy/api/os_sys_calls_common.h index 3c283e064bbf..d85aed3407b2 100644 --- a/include/envoy/api/os_sys_calls_common.h +++ b/include/envoy/api/os_sys_calls_common.h @@ -21,11 +21,11 @@ template struct SysCallResult { int errno_; }; -typedef SysCallResult SysCallIntResult; -typedef SysCallResult SysCallSizeResult; -typedef SysCallResult SysCallPtrResult; -typedef SysCallResult SysCallStringResult; -typedef SysCallResult SysCallBoolResult; +using SysCallIntResult = SysCallResult; +using SysCallSizeResult = SysCallResult; +using SysCallPtrResult = SysCallResult; +using SysCallStringResult = SysCallResult; +using SysCallBoolResult = SysCallResult; } // namespace Api } // namespace Envoy diff --git a/include/envoy/api/os_sys_calls_hot_restart.h b/include/envoy/api/os_sys_calls_hot_restart.h index 7e557cf4d295..d5302bee1de3 100644 --- a/include/envoy/api/os_sys_calls_hot_restart.h +++ b/include/envoy/api/os_sys_calls_hot_restart.h @@ -13,7 +13,7 @@ namespace Api { class HotRestartOsSysCalls { public: - virtual ~HotRestartOsSysCalls() {} + virtual ~HotRestartOsSysCalls() = default; /** * @see shm_open (man 3 shm_open) @@ -26,7 +26,7 @@ class HotRestartOsSysCalls { virtual SysCallIntResult shmUnlink(const char* name) PURE; }; -typedef std::unique_ptr HotRestartOsSysCallsPtr; +using HotRestartOsSysCallsPtr = std::unique_ptr; } // namespace Api } // namespace Envoy diff --git a/include/envoy/api/os_sys_calls_linux.h b/include/envoy/api/os_sys_calls_linux.h index cd90daea538d..76a2cdd7a3c4 100644 --- a/include/envoy/api/os_sys_calls_linux.h +++ b/include/envoy/api/os_sys_calls_linux.h @@ -14,7 +14,7 @@ namespace Api { class LinuxOsSysCalls { public: - virtual ~LinuxOsSysCalls() {} + virtual ~LinuxOsSysCalls() = default; /** * @see sched_getaffinity (man 2 sched_getaffinity) @@ -22,7 +22,7 @@ class LinuxOsSysCalls { virtual SysCallIntResult sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t* mask) PURE; }; -typedef std::unique_ptr LinuxOsSysCallsPtr; +using LinuxOsSysCallsPtr = std::unique_ptr; } // namespace Api } // namespace Envoy diff --git a/include/envoy/buffer/buffer.h b/include/envoy/buffer/buffer.h index 48f5c1842a0f..1f19d94d6601 100644 --- a/include/envoy/buffer/buffer.h +++ b/include/envoy/buffer/buffer.h @@ -49,7 +49,7 @@ class BufferFragment { virtual void done() PURE; protected: - virtual ~BufferFragment() {} + virtual ~BufferFragment() = default; }; /** @@ -57,7 +57,7 @@ class BufferFragment { */ class Instance { public: - virtual ~Instance() {} + virtual ~Instance() = default; /** * Copy data into the buffer (deprecated, use absl::string_view variant @@ -357,14 +357,14 @@ class Instance { } }; -typedef std::unique_ptr InstancePtr; +using InstancePtr = std::unique_ptr; /** * A factory for creating buffers which call callbacks when reaching high and low watermarks. */ class WatermarkFactory { public: - virtual ~WatermarkFactory() {} + virtual ~WatermarkFactory() = default; /** * Creates and returns a unique pointer to a new buffer. @@ -378,7 +378,7 @@ class WatermarkFactory { std::function above_high_watermark) PURE; }; -typedef std::unique_ptr WatermarkFactoryPtr; +using WatermarkFactoryPtr = std::unique_ptr; } // namespace Buffer } // namespace Envoy diff --git a/include/envoy/common/backoff_strategy.h b/include/envoy/common/backoff_strategy.h index 1f1862d2a603..b51880a2e2e0 100644 --- a/include/envoy/common/backoff_strategy.h +++ b/include/envoy/common/backoff_strategy.h @@ -11,7 +11,7 @@ namespace Envoy { */ class BackOffStrategy { public: - virtual ~BackOffStrategy() {} + virtual ~BackOffStrategy() = default; /** * @return the next backoff interval in milli seconds. @@ -24,5 +24,5 @@ class BackOffStrategy { virtual void reset() PURE; }; -typedef std::unique_ptr BackOffStrategyPtr; +using BackOffStrategyPtr = std::unique_ptr; } // namespace Envoy diff --git a/include/envoy/common/callback.h b/include/envoy/common/callback.h index f7693794a2a1..bc5ac59ed244 100644 --- a/include/envoy/common/callback.h +++ b/include/envoy/common/callback.h @@ -11,7 +11,7 @@ namespace Common { */ class CallbackHandle { public: - virtual ~CallbackHandle() {} + virtual ~CallbackHandle() = default; /** * Remove the callback. After this routine returns the callback will no longer be called. diff --git a/include/envoy/common/interval_set.h b/include/envoy/common/interval_set.h index 541994541e00..51ca2068f309 100644 --- a/include/envoy/common/interval_set.h +++ b/include/envoy/common/interval_set.h @@ -15,9 +15,9 @@ namespace Envoy { */ template class IntervalSet { public: - virtual ~IntervalSet() {} + virtual ~IntervalSet() = default; - typedef std::pair Interval; + using Interval = std::pair; /** * Inserts a new interval into the set, merging any overlaps. The intervals are in diff --git a/include/envoy/common/mutex_tracer.h b/include/envoy/common/mutex_tracer.h index 98620c968679..aec6758872fb 100644 --- a/include/envoy/common/mutex_tracer.h +++ b/include/envoy/common/mutex_tracer.h @@ -13,7 +13,7 @@ namespace Envoy { */ class MutexTracer { public: - virtual ~MutexTracer() {} + virtual ~MutexTracer() = default; /** * @return resets the captured statistics. diff --git a/include/envoy/common/time.h b/include/envoy/common/time.h index f48ca72f7981..4c5694fabef6 100644 --- a/include/envoy/common/time.h +++ b/include/envoy/common/time.h @@ -12,8 +12,8 @@ namespace Envoy { * SystemTime should be used when getting a time to present to the user, e.g. for logging. * MonotonicTime should be used when tracking time for computing an interval. */ -typedef std::chrono::time_point SystemTime; -typedef std::chrono::time_point MonotonicTime; +using SystemTime = std::chrono::time_point; +using MonotonicTime = std::chrono::time_point; /** * Captures a system-time source, capable of computing both monotonically increasing diff --git a/include/envoy/common/token_bucket.h b/include/envoy/common/token_bucket.h index 2f04c7c8c461..f3db6a884154 100644 --- a/include/envoy/common/token_bucket.h +++ b/include/envoy/common/token_bucket.h @@ -40,6 +40,6 @@ class TokenBucket { virtual void reset(uint64_t num_tokens) PURE; }; -typedef std::unique_ptr TokenBucketPtr; +using TokenBucketPtr = std::unique_ptr; }; // namespace Envoy diff --git a/include/envoy/compressor/compressor.h b/include/envoy/compressor/compressor.h index a755c513115a..d25204a3ead4 100644 --- a/include/envoy/compressor/compressor.h +++ b/include/envoy/compressor/compressor.h @@ -15,7 +15,7 @@ enum class State { Flush, Finish }; */ class Compressor { public: - virtual ~Compressor() {} + virtual ~Compressor() = default; /** * Compresses data buffer. diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index b82bbbb9abe6..1d66e2ddda77 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -26,7 +26,7 @@ struct ControlPlaneStats { class GrpcMuxCallbacks { public: - virtual ~GrpcMuxCallbacks() {} + virtual ~GrpcMuxCallbacks() = default; /** * Called when a configuration update is received. @@ -58,10 +58,10 @@ class GrpcMuxCallbacks { */ class GrpcMuxWatch { public: - virtual ~GrpcMuxWatch() {} + virtual ~GrpcMuxWatch() = default; }; -typedef std::unique_ptr GrpcMuxWatchPtr; +using GrpcMuxWatchPtr = std::unique_ptr; /** * Manage one or more gRPC subscriptions on a single stream to management server. This can be used @@ -69,7 +69,7 @@ typedef std::unique_ptr GrpcMuxWatchPtr; */ class GrpcMux { public: - virtual ~GrpcMux() {} + virtual ~GrpcMux() = default; /** * Initiate stream with management server. @@ -109,7 +109,7 @@ class GrpcMux { virtual void resume(const std::string& type_url) PURE; }; -typedef std::unique_ptr GrpcMuxPtr; +using GrpcMuxPtr = std::unique_ptr; } // namespace Config } // namespace Envoy diff --git a/include/envoy/config/typed_metadata.h b/include/envoy/config/typed_metadata.h index b316ef4737f9..47abbcdeeebc 100644 --- a/include/envoy/config/typed_metadata.h +++ b/include/envoy/config/typed_metadata.h @@ -17,10 +17,10 @@ class TypedMetadata { public: class Object { public: - virtual ~Object() {} + virtual ~Object() = default; }; - virtual ~TypedMetadata() {} + virtual ~TypedMetadata() = default; /** * @return a T instance by key. If the conversion is not able to complete, or @@ -52,7 +52,7 @@ class TypedMetadata { */ class TypedMetadataFactory { public: - virtual ~TypedMetadataFactory() {} + virtual ~TypedMetadataFactory() = default; /** * Name of the factory, a reversed DNS name is encouraged to avoid cross-org conflict. diff --git a/include/envoy/config/xds_grpc_context.h b/include/envoy/config/xds_grpc_context.h index aba3a824a67c..312d3e0650f7 100644 --- a/include/envoy/config/xds_grpc_context.h +++ b/include/envoy/config/xds_grpc_context.h @@ -13,7 +13,7 @@ namespace Config { */ template class GrpcStreamCallbacks { public: - virtual ~GrpcStreamCallbacks() {} + virtual ~GrpcStreamCallbacks() = default; /** * For the GrpcStream to prompt the context to take appropriate action in response to the diff --git a/include/envoy/decompressor/decompressor.h b/include/envoy/decompressor/decompressor.h index 1a4da7c66353..d694aa50ca1c 100644 --- a/include/envoy/decompressor/decompressor.h +++ b/include/envoy/decompressor/decompressor.h @@ -10,7 +10,7 @@ namespace Decompressor { */ class Decompressor { public: - virtual ~Decompressor() {} + virtual ~Decompressor() = default; /** * Decompresses data from one buffer into another buffer. diff --git a/include/envoy/event/deferred_deletable.h b/include/envoy/event/deferred_deletable.h index f92137314aa2..c0e3dfee2835 100644 --- a/include/envoy/event/deferred_deletable.h +++ b/include/envoy/event/deferred_deletable.h @@ -12,10 +12,10 @@ namespace Event { */ class DeferredDeletable { public: - virtual ~DeferredDeletable() {} + virtual ~DeferredDeletable() = default; }; -typedef std::unique_ptr DeferredDeletablePtr; +using DeferredDeletablePtr = std::unique_ptr; } // namespace Event } // namespace Envoy diff --git a/include/envoy/event/dispatcher.h b/include/envoy/event/dispatcher.h index 0024b3a2e779..50a6ddbadb86 100644 --- a/include/envoy/event/dispatcher.h +++ b/include/envoy/event/dispatcher.h @@ -43,14 +43,14 @@ struct DispatcherStats { /** * Callback invoked when a dispatcher post() runs. */ -typedef std::function PostCb; +using PostCb = std::function; /** * Abstract event dispatching loop. */ class Dispatcher { public: - virtual ~Dispatcher() {} + virtual ~Dispatcher() = default; /** * Returns a time-source to use with this dispatcher. @@ -201,7 +201,7 @@ class Dispatcher { virtual Buffer::WatermarkFactory& getWatermarkFactory() PURE; }; -typedef std::unique_ptr DispatcherPtr; +using DispatcherPtr = std::unique_ptr; } // namespace Event } // namespace Envoy diff --git a/include/envoy/event/file_event.h b/include/envoy/event/file_event.h index 8c25a904e955..72f618097ec5 100644 --- a/include/envoy/event/file_event.h +++ b/include/envoy/event/file_event.h @@ -23,14 +23,14 @@ enum class FileTriggerType { Level, Edge }; /** * Callback invoked when a FileEvent is ready for reading or writing. */ -typedef std::function FileReadyCb; +using FileReadyCb = std::function; /** * Wrapper for file based (read/write) event notifications. */ class FileEvent { public: - virtual ~FileEvent() {} + virtual ~FileEvent() = default; /** * Activate the file event explicitly for a set of events. Should be a logical OR of FileReadyType @@ -47,7 +47,7 @@ class FileEvent { virtual void setEnabled(uint32_t events) PURE; }; -typedef std::unique_ptr FileEventPtr; +using FileEventPtr = std::unique_ptr; } // namespace Event } // namespace Envoy diff --git a/include/envoy/event/signal.h b/include/envoy/event/signal.h index 4a2b4634c674..a502689d8a9c 100644 --- a/include/envoy/event/signal.h +++ b/include/envoy/event/signal.h @@ -9,17 +9,17 @@ namespace Event { /** * Callback invoked when a signal event fires. */ -typedef std::function SignalCb; +using SignalCb = std::function; /** * An abstract signal event. Free the event to stop listening on the signal. */ class SignalEvent { public: - virtual ~SignalEvent() {} + virtual ~SignalEvent() = default; }; -typedef std::unique_ptr SignalEventPtr; +using SignalEventPtr = std::unique_ptr; } // namespace Event } // namespace Envoy diff --git a/include/envoy/event/timer.h b/include/envoy/event/timer.h index f26d58c22f2c..8166a1308ad8 100644 --- a/include/envoy/event/timer.h +++ b/include/envoy/event/timer.h @@ -13,14 +13,14 @@ namespace Event { /** * Callback invoked when a timer event fires. */ -typedef std::function TimerCb; +using TimerCb = std::function; /** * An abstract timer event. Free the timer to unregister any pending timeouts. */ class Timer { public: - virtual ~Timer() {} + virtual ~Timer() = default; /** * Disable a pending timeout without destroying the underlying timer. @@ -38,11 +38,11 @@ class Timer { virtual bool enabled() PURE; }; -typedef std::unique_ptr TimerPtr; +using TimerPtr = std::unique_ptr; class Scheduler { public: - virtual ~Scheduler() {} + virtual ~Scheduler() = default; /** * Creates a timer. @@ -50,7 +50,7 @@ class Scheduler { virtual TimerPtr createTimer(const TimerCb& cb) PURE; }; -typedef std::unique_ptr SchedulerPtr; +using SchedulerPtr = std::unique_ptr; /** * Interface providing a mechanism to measure time and set timers that run callbacks diff --git a/include/envoy/filesystem/filesystem.h b/include/envoy/filesystem/filesystem.h index a866cff3afb7..64923d508544 100644 --- a/include/envoy/filesystem/filesystem.h +++ b/include/envoy/filesystem/filesystem.h @@ -18,7 +18,7 @@ namespace Filesystem { */ class File { public: - virtual ~File() {} + virtual ~File() = default; /** * Open the file with O_RDWR | O_APPEND | O_CREAT @@ -60,7 +60,7 @@ using FilePtr = std::unique_ptr; */ class Instance { public: - virtual ~Instance() {} + virtual ~Instance() = default; /** * @param path The path of the File @@ -128,7 +128,7 @@ class DirectoryIteratorImpl; class DirectoryIterator { public: DirectoryIterator() : entry_({"", FileType::Other}) {} - virtual ~DirectoryIterator() {} + virtual ~DirectoryIterator() = default; const DirectoryEntry& operator*() const { return entry_; } diff --git a/include/envoy/filesystem/watcher.h b/include/envoy/filesystem/watcher.h index d50264254e4a..2d8e729292aa 100644 --- a/include/envoy/filesystem/watcher.h +++ b/include/envoy/filesystem/watcher.h @@ -16,14 +16,14 @@ namespace Filesystem { */ class Watcher { public: - typedef std::function OnChangedCb; + using OnChangedCb = std::function; struct Events { static const uint32_t MovedTo = 0x1; static const uint32_t Modified = 0x2; }; - virtual ~Watcher() {} + virtual ~Watcher() = default; /** * Add a file watch. diff --git a/include/envoy/grpc/async_client.h b/include/envoy/grpc/async_client.h index e2ce86e1a766..4a2e9e0c451c 100644 --- a/include/envoy/grpc/async_client.h +++ b/include/envoy/grpc/async_client.h @@ -21,7 +21,7 @@ namespace Grpc { */ class AsyncRequest { public: - virtual ~AsyncRequest() {} + virtual ~AsyncRequest() = default; /** * Signals that the request should be cancelled. No further callbacks will be invoked. @@ -34,7 +34,7 @@ class AsyncRequest { */ class RawAsyncStream { public: - virtual ~RawAsyncStream() {} + virtual ~RawAsyncStream() = default; /** * Send request message to the stream. @@ -61,7 +61,7 @@ class RawAsyncStream { class RawAsyncRequestCallbacks { public: - virtual ~RawAsyncRequestCallbacks() {} + virtual ~RawAsyncRequestCallbacks() = default; /** * Called when populating the headers to send with initial metadata. @@ -95,7 +95,7 @@ class RawAsyncRequestCallbacks { */ class RawAsyncStreamCallbacks { public: - virtual ~RawAsyncStreamCallbacks() {} + virtual ~RawAsyncStreamCallbacks() = default; /** * Called when populating the headers to send with initial metadata. @@ -141,7 +141,7 @@ class RawAsyncStreamCallbacks { */ class RawAsyncClient { public: - virtual ~RawAsyncClient() {} + virtual ~RawAsyncClient() = default; /** * Start a gRPC unary RPC asynchronously. @@ -177,7 +177,7 @@ class RawAsyncClient { RawAsyncStreamCallbacks& callbacks) PURE; }; -typedef std::unique_ptr RawAsyncClientPtr; +using RawAsyncClientPtr = std::unique_ptr; } // namespace Grpc } // namespace Envoy diff --git a/include/envoy/grpc/async_client_manager.h b/include/envoy/grpc/async_client_manager.h index e7580712d1b1..18c3e3923eab 100644 --- a/include/envoy/grpc/async_client_manager.h +++ b/include/envoy/grpc/async_client_manager.h @@ -11,7 +11,7 @@ namespace Grpc { // with thread local state. Clients will use ThreadLocal::Instance::dispatcher() for event handling. class AsyncClientFactory { public: - virtual ~AsyncClientFactory() {} + virtual ~AsyncClientFactory() = default; /** * Create a gRPC::RawAsyncClient. @@ -20,14 +20,14 @@ class AsyncClientFactory { virtual RawAsyncClientPtr create() PURE; }; -typedef std::unique_ptr AsyncClientFactoryPtr; +using AsyncClientFactoryPtr = std::unique_ptr; // Singleton gRPC client manager. Grpc::AsyncClientManager can be used to create per-service // Grpc::AsyncClientFactory instances. All manufactured Grpc::AsyncClients must // be destroyed before the AsyncClientManager can be safely destructed. class AsyncClientManager { public: - virtual ~AsyncClientManager() {} + virtual ~AsyncClientManager() = default; /** * Create a Grpc::AsyncClients factory for a service. Validation of the service is performed and @@ -44,7 +44,7 @@ class AsyncClientManager { bool skip_cluster_check) PURE; }; -typedef std::unique_ptr AsyncClientManagerPtr; +using AsyncClientManagerPtr = std::unique_ptr; } // namespace Grpc } // namespace Envoy diff --git a/include/envoy/grpc/google_grpc_creds.h b/include/envoy/grpc/google_grpc_creds.h index dd43d9ab8f3e..c01cf1e8875a 100644 --- a/include/envoy/grpc/google_grpc_creds.h +++ b/include/envoy/grpc/google_grpc_creds.h @@ -16,7 +16,7 @@ namespace Grpc { */ class GoogleGrpcCredentialsFactory { public: - virtual ~GoogleGrpcCredentialsFactory() {} + virtual ~GoogleGrpcCredentialsFactory() = default; /** * Get a ChannelCredentials to be used for authentication of a gRPC channel. diff --git a/include/envoy/http/async_client.h b/include/envoy/http/async_client.h index c9794d12cd53..d3bf280198c7 100644 --- a/include/envoy/http/async_client.h +++ b/include/envoy/http/async_client.h @@ -29,7 +29,7 @@ class AsyncClient { */ class Callbacks { public: - virtual ~Callbacks() {} + virtual ~Callbacks() = default; /** * Called when the async HTTP request succeeds. @@ -52,7 +52,7 @@ class AsyncClient { */ class StreamCallbacks { public: - virtual ~StreamCallbacks() {} + virtual ~StreamCallbacks() = default; /** * Called when all headers get received on the async HTTP stream. @@ -86,7 +86,7 @@ class AsyncClient { */ class Request { public: - virtual ~Request() {} + virtual ~Request() = default; /** * Signals that the request should be cancelled. @@ -99,7 +99,7 @@ class AsyncClient { */ class Stream { public: - virtual ~Stream() {} + virtual ~Stream() = default; /*** * Send headers to the stream. This method cannot be invoked more than once and @@ -129,7 +129,7 @@ class AsyncClient { virtual void reset() PURE; }; - virtual ~AsyncClient() {} + virtual ~AsyncClient() = default; /** * A structure to hold the options for AsyncStream object. @@ -226,7 +226,7 @@ class AsyncClient { virtual Event::Dispatcher& dispatcher() PURE; }; -typedef std::unique_ptr AsyncClientPtr; +using AsyncClientPtr = std::unique_ptr; } // namespace Http } // namespace Envoy diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 4bcceeaca978..4ac313f7d6f3 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -23,7 +23,7 @@ class Stream; */ class StreamEncoder { public: - virtual ~StreamEncoder() {} + virtual ~StreamEncoder() = default; /** * Encode 100-Continue headers. @@ -69,7 +69,7 @@ class StreamEncoder { */ class StreamDecoder { public: - virtual ~StreamDecoder() {} + virtual ~StreamDecoder() = default; /** * Called with decoded 100-Continue headers. @@ -129,7 +129,7 @@ enum class StreamResetReason { */ class StreamCallbacks { public: - virtual ~StreamCallbacks() {} + virtual ~StreamCallbacks() = default; /** * Fires when a stream has been remote reset. @@ -156,7 +156,7 @@ class StreamCallbacks { */ class Stream { public: - virtual ~Stream() {} + virtual ~Stream() = default; /** * Add stream callbacks. @@ -203,7 +203,7 @@ class Stream { */ class ConnectionCallbacks { public: - virtual ~ConnectionCallbacks() {} + virtual ~ConnectionCallbacks() = default; /** * Fires when the remote indicates "go away." No new streams should be created. @@ -279,7 +279,7 @@ struct Http2Settings { */ class Connection { public: - virtual ~Connection() {} + virtual ~Connection() = default; /** * Dispatch incoming connection data. @@ -327,7 +327,7 @@ class Connection { */ class DownstreamWatermarkCallbacks { public: - virtual ~DownstreamWatermarkCallbacks() {} + virtual ~DownstreamWatermarkCallbacks() = default; /** * Called when the downstream connection or stream goes over its high watermark. Note that this @@ -369,7 +369,7 @@ class ServerConnectionCallbacks : public virtual ConnectionCallbacks { */ class ServerConnection : public virtual Connection {}; -typedef std::unique_ptr ServerConnectionPtr; +using ServerConnectionPtr = std::unique_ptr; /** * A client side HTTP connection. @@ -384,7 +384,7 @@ class ClientConnection : public virtual Connection { virtual StreamEncoder& newStream(StreamDecoder& response_decoder) PURE; }; -typedef std::unique_ptr ClientConnectionPtr; +using ClientConnectionPtr = std::unique_ptr; } // namespace Http } // namespace Envoy diff --git a/include/envoy/http/conn_pool.h b/include/envoy/http/conn_pool.h index 5bd5a2390bb8..7dc23a943a79 100644 --- a/include/envoy/http/conn_pool.h +++ b/include/envoy/http/conn_pool.h @@ -17,7 +17,7 @@ namespace ConnectionPool { */ class Cancellable { public: - virtual ~Cancellable() {} + virtual ~Cancellable() = default; /** * Cancel the pending request. @@ -41,7 +41,7 @@ enum class PoolFailureReason { */ class Callbacks { public: - virtual ~Callbacks() {} + virtual ~Callbacks() = default; /** * Called when a pool error occurred and no connection could be acquired for making the request. @@ -68,7 +68,7 @@ class Callbacks { */ class Instance : public Event::DeferredDeletable { public: - virtual ~Instance() {} + ~Instance() override = default; /** * @return Http::Protocol Reports the protocol in use by this connection pool. @@ -79,7 +79,7 @@ class Instance : public Event::DeferredDeletable { * Called when a connection pool has been drained of pending requests, busy connections, and * ready connections. */ - typedef std::function DrainedCb; + using DrainedCb = std::function; /** * Register a callback that gets called when the connection pool is fully drained. No actual @@ -123,7 +123,7 @@ class Instance : public Event::DeferredDeletable { virtual Upstream::HostDescriptionConstSharedPtr host() const PURE; }; -typedef std::unique_ptr InstancePtr; +using InstancePtr = std::unique_ptr; } // namespace ConnectionPool } // namespace Http diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index 7feb8482fa92..1e1bacaab3fc 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -124,7 +124,7 @@ enum class FilterMetadataStatus { */ class StreamFilterCallbacks { public: - virtual ~StreamFilterCallbacks() {} + virtual ~StreamFilterCallbacks() = default; /** * @return const Network::Connection* the originating connection, or nullptr if there is none. @@ -427,7 +427,7 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { */ class StreamFilterBase { public: - virtual ~StreamFilterBase() {} + virtual ~StreamFilterBase() = default; /** * This routine is called prior to a filter being destroyed. This may happen after normal stream @@ -480,7 +480,7 @@ class StreamDecoderFilter : public StreamFilterBase { virtual void decodeComplete() {} }; -typedef std::shared_ptr StreamDecoderFilterSharedPtr; +using StreamDecoderFilterSharedPtr = std::shared_ptr; /** * Stream encoder filter callbacks add additional callbacks that allow a encoding filter to restart @@ -666,14 +666,14 @@ class StreamEncoderFilter : public StreamFilterBase { virtual void encodeComplete() {} }; -typedef std::shared_ptr StreamEncoderFilterSharedPtr; +using StreamEncoderFilterSharedPtr = std::shared_ptr; /** * A filter that handles both encoding and decoding. */ class StreamFilter : public virtual StreamDecoderFilter, public virtual StreamEncoderFilter {}; -typedef std::shared_ptr StreamFilterSharedPtr; +using StreamFilterSharedPtr = std::shared_ptr; /** * These callbacks are provided by the connection manager to the factory so that the factory can @@ -681,7 +681,7 @@ typedef std::shared_ptr StreamFilterSharedPtr; */ class FilterChainFactoryCallbacks { public: - virtual ~FilterChainFactoryCallbacks() {} + virtual ~FilterChainFactoryCallbacks() = default; /** * Add a decoder filter that is used when reading stream data. @@ -716,7 +716,7 @@ class FilterChainFactoryCallbacks { * function will install a single filter, but it's technically possibly to install more than one * if desired. */ -typedef std::function FilterFactoryCb; +using FilterFactoryCb = std::function; /** * A FilterChainFactory is used by a connection manager to create an HTTP level filter chain when a @@ -726,7 +726,7 @@ typedef std::function FilterFactor */ class FilterChainFactory { public: - virtual ~FilterChainFactory() {} + virtual ~FilterChainFactory() = default; /** * Called when a new HTTP stream is created on the connection. @@ -744,7 +744,7 @@ class FilterChainFactory { * @return true if upgrades of this type are allowed and the filter chain has been created. * returns false if this upgrade type is not configured, and no filter chain is created. */ - typedef std::map UpgradeMap; + using UpgradeMap = std::map; virtual bool createUpgradeFilterChain(absl::string_view upgrade, const UpgradeMap* per_route_upgrade_map, FilterChainFactoryCallbacks& callbacks) PURE; diff --git a/include/envoy/http/header_map.h b/include/envoy/http/header_map.h index 2b92fa573ba2..ee202a2fccc2 100644 --- a/include/envoy/http/header_map.h +++ b/include/envoy/http/header_map.h @@ -76,13 +76,13 @@ struct LowerCaseStringHash { /** * Convenient type for unordered set of lower case string. */ -typedef std::unordered_set LowerCaseStrUnorderedSet; +using LowerCaseStrUnorderedSet = std::unordered_set; /** * Convenient type for a vector of lower case string and string pair. */ -typedef std::vector> - LowerCaseStrPairVector; +using LowerCaseStrPairVector = + std::vector>; /** * This is a string implementation for use in header processing. It is heavily optimized for @@ -214,7 +214,7 @@ class HeaderString { */ class HeaderEntry { public: - virtual ~HeaderEntry() {} + virtual ~HeaderEntry() = default; /** * @return the header key. @@ -359,7 +359,7 @@ class HeaderEntry { */ class HeaderMap { public: - virtual ~HeaderMap() {} + virtual ~HeaderMap() = default; ALL_INLINE_HEADERS(DEFINE_INLINE_HEADER) @@ -545,12 +545,12 @@ class HeaderMap { } }; -typedef std::unique_ptr HeaderMapPtr; +using HeaderMapPtr = std::unique_ptr; /** * Convenient container type for storing Http::LowerCaseString and std::string key/value pairs. */ -typedef std::vector> HeaderVector; +using HeaderVector = std::vector>; } // namespace Http } // namespace Envoy diff --git a/include/envoy/http/message.h b/include/envoy/http/message.h index 167fcd6e492c..3d36842e72a8 100644 --- a/include/envoy/http/message.h +++ b/include/envoy/http/message.h @@ -14,7 +14,7 @@ namespace Http { */ class Message { public: - virtual ~Message() {} + virtual ~Message() = default; /** * @return HeaderMap& the message headers. @@ -44,7 +44,7 @@ class Message { virtual std::string bodyAsString() const PURE; }; -typedef std::unique_ptr MessagePtr; +using MessagePtr = std::unique_ptr; } // namespace Http } // namespace Envoy diff --git a/include/envoy/http/query_params.h b/include/envoy/http/query_params.h index 40c25f4f3853..d30ae58b1ab3 100644 --- a/include/envoy/http/query_params.h +++ b/include/envoy/http/query_params.h @@ -11,7 +11,7 @@ namespace Utility { // using proper formatting. Perhaps similar to // https://github.com/apache/incubator-pagespeed-mod/blob/master/pagespeed/kernel/http/query_params.h -typedef std::map QueryParams; +using QueryParams = std::map; } // namespace Utility } // namespace Http diff --git a/include/envoy/json/json_object.h b/include/envoy/json/json_object.h index 40124ab56796..a5161ccfa91b 100644 --- a/include/envoy/json/json_object.h +++ b/include/envoy/json/json_object.h @@ -13,10 +13,10 @@ namespace Envoy { namespace Json { class Object; -typedef std::shared_ptr ObjectSharedPtr; +using ObjectSharedPtr = std::shared_ptr; // @return false if immediate exit from iteration required. -typedef std::function ObjectCallback; +using ObjectCallback = std::function; /** * Exception thrown when a JSON error occurs. @@ -31,7 +31,7 @@ class Exception : public EnvoyException { */ class Object { public: - virtual ~Object() {} + virtual ~Object() = default; /** * Convert a generic object into an array of objects. This is useful for dealing diff --git a/include/envoy/local_info/local_info.h b/include/envoy/local_info/local_info.h index 1c9c152c8c46..54f1ea556ae5 100644 --- a/include/envoy/local_info/local_info.h +++ b/include/envoy/local_info/local_info.h @@ -14,7 +14,7 @@ namespace LocalInfo { */ class LocalInfo { public: - virtual ~LocalInfo() {} + virtual ~LocalInfo() = default; /** * @return the local (non-loopback) address of the server. @@ -42,7 +42,7 @@ class LocalInfo { virtual const envoy::api::v2::core::Node& node() const PURE; }; -typedef std::unique_ptr LocalInfoPtr; +using LocalInfoPtr = std::unique_ptr; } // namespace LocalInfo } // namespace Envoy diff --git a/include/envoy/network/address.h b/include/envoy/network/address.h index 38d5b673a3f5..c439cecc27af 100644 --- a/include/envoy/network/address.h +++ b/include/envoy/network/address.h @@ -23,7 +23,7 @@ namespace Address { */ class Ipv4 { public: - virtual ~Ipv4() {} + virtual ~Ipv4() = default; /** * @return the 32-bit IPv4 address in network byte order. @@ -36,7 +36,7 @@ class Ipv4 { */ class Ipv6 { public: - virtual ~Ipv6() {} + virtual ~Ipv6() = default; /** * @return the absl::uint128 IPv6 address in network byte order. @@ -51,7 +51,7 @@ enum class IpVersion { v4, v6 }; */ class Ip { public: - virtual ~Ip() {} + virtual ~Ip() = default; /** * @return the address as a string. E.g., "1.2.3.4" for an IPv4 address. @@ -100,7 +100,7 @@ enum class SocketType { Stream, Datagram }; */ class Instance { public: - virtual ~Instance() {} + virtual ~Instance() = default; virtual bool operator==(const Instance& rhs) const PURE; bool operator!=(const Instance& rhs) const { return !operator==(rhs); } @@ -163,7 +163,7 @@ class Instance { virtual Type type() const PURE; }; -typedef std::shared_ptr InstanceConstSharedPtr; +using InstanceConstSharedPtr = std::shared_ptr; } // namespace Address } // namespace Network diff --git a/include/envoy/network/connection.h b/include/envoy/network/connection.h index 745e6e3e9ba9..0eeaa2855d04 100644 --- a/include/envoy/network/connection.h +++ b/include/envoy/network/connection.h @@ -39,7 +39,7 @@ enum class ConnectionBufferType { Read, Write }; */ class ConnectionCallbacks { public: - virtual ~ConnectionCallbacks() {} + virtual ~ConnectionCallbacks() = default; /** * Callback for connection events. @@ -80,7 +80,7 @@ class Connection : public Event::DeferredDeletable, public FilterManager { * Callback function for when bytes have been sent by a connection. * @param bytes_sent supplies the number of bytes written to the connection. */ - typedef std::function BytesSentCb; + using BytesSentCb = std::function; struct ConnectionStats { Stats::Counter& read_total_; @@ -93,7 +93,7 @@ class Connection : public Event::DeferredDeletable, public FilterManager { Stats::Counter* delayed_close_timeouts_; }; - virtual ~Connection() {} + ~Connection() override = default; /** * Register callbacks that fire when connection events occur. @@ -301,7 +301,7 @@ class Connection : public Event::DeferredDeletable, public FilterManager { virtual absl::string_view transportFailureReason() const PURE; }; -typedef std::unique_ptr ConnectionPtr; +using ConnectionPtr = std::unique_ptr; /** * Connections capable of outbound connects. @@ -315,7 +315,7 @@ class ClientConnection : public virtual Connection { virtual void connect() PURE; }; -typedef std::unique_ptr ClientConnectionPtr; +using ClientConnectionPtr = std::unique_ptr; } // namespace Network } // namespace Envoy diff --git a/include/envoy/network/connection_handler.h b/include/envoy/network/connection_handler.h index 7ca8649cd99d..9a36aed1cc49 100644 --- a/include/envoy/network/connection_handler.h +++ b/include/envoy/network/connection_handler.h @@ -17,7 +17,7 @@ namespace Network { */ class ConnectionHandler { public: - virtual ~ConnectionHandler() {} + virtual ~ConnectionHandler() = default; /** * @return uint64_t the number of active connections owned by the handler. @@ -70,7 +70,7 @@ class ConnectionHandler { virtual void enableListeners() PURE; }; -typedef std::unique_ptr ConnectionHandlerPtr; +using ConnectionHandlerPtr = std::unique_ptr; } // namespace Network } // namespace Envoy diff --git a/include/envoy/network/dns.h b/include/envoy/network/dns.h index f89cac1ef461..6e898ee9e4e5 100644 --- a/include/envoy/network/dns.h +++ b/include/envoy/network/dns.h @@ -16,7 +16,7 @@ namespace Network { */ class ActiveDnsQuery { public: - virtual ~ActiveDnsQuery() {} + virtual ~ActiveDnsQuery() = default; /** * Cancel an outstanding DNS request. @@ -31,14 +31,14 @@ enum class DnsLookupFamily { V4Only, V6Only, Auto }; */ class DnsResolver { public: - virtual ~DnsResolver() {} + virtual ~DnsResolver() = default; /** * Called when a resolution attempt is complete. * @param address_list supplies the list of resolved IP addresses. The list will be empty if * the resolution failed. */ - typedef std::function&& address_list)> ResolveCb; + using ResolveCb = std::function&& address_list)>; /** * Initiate an async DNS resolution. @@ -52,7 +52,7 @@ class DnsResolver { ResolveCb callback) PURE; }; -typedef std::shared_ptr DnsResolverSharedPtr; +using DnsResolverSharedPtr = std::shared_ptr; } // namespace Network } // namespace Envoy diff --git a/include/envoy/network/drain_decision.h b/include/envoy/network/drain_decision.h index 454bef1254cd..e071dfdc8480 100644 --- a/include/envoy/network/drain_decision.h +++ b/include/envoy/network/drain_decision.h @@ -7,7 +7,7 @@ namespace Network { class DrainDecision { public: - virtual ~DrainDecision() {} + virtual ~DrainDecision() = default; /** * @return TRUE if a connection should be drained and closed. It is up to individual network diff --git a/include/envoy/network/filter.h b/include/envoy/network/filter.h index 7f1a673ea07b..e446b1bf7a2c 100644 --- a/include/envoy/network/filter.h +++ b/include/envoy/network/filter.h @@ -35,7 +35,7 @@ enum class FilterStatus { */ class NetworkFilterCallbacks { public: - virtual ~NetworkFilterCallbacks() {} + virtual ~NetworkFilterCallbacks() = default; /** * @return the connection that owns this filter. @@ -48,7 +48,7 @@ class NetworkFilterCallbacks { */ class WriteFilterCallbacks : public virtual NetworkFilterCallbacks { public: - virtual ~WriteFilterCallbacks() {} + ~WriteFilterCallbacks() override = default; /** * Pass data directly to subsequent filters in the filter chain. This method is used in @@ -75,7 +75,7 @@ class WriteFilterCallbacks : public virtual NetworkFilterCallbacks { */ class WriteFilter { public: - virtual ~WriteFilter() {} + virtual ~WriteFilter() = default; /** * Called when data is to be written on the connection. @@ -99,14 +99,14 @@ class WriteFilter { virtual void initializeWriteFilterCallbacks(WriteFilterCallbacks&) {} }; -typedef std::shared_ptr WriteFilterSharedPtr; +using WriteFilterSharedPtr = std::shared_ptr; /** * Callbacks used by individual read filter instances to communicate with the filter manager. */ class ReadFilterCallbacks : public virtual NetworkFilterCallbacks { public: - virtual ~ReadFilterCallbacks() {} + ~ReadFilterCallbacks() override = default; /** * If a read filter stopped filter iteration, continueReading() can be called to continue the @@ -158,7 +158,7 @@ class ReadFilterCallbacks : public virtual NetworkFilterCallbacks { */ class ReadFilter { public: - virtual ~ReadFilter() {} + virtual ~ReadFilter() = default; /** * Called when data is read on the connection. @@ -191,21 +191,21 @@ class ReadFilter { virtual void initializeReadFilterCallbacks(ReadFilterCallbacks& callbacks) PURE; }; -typedef std::shared_ptr ReadFilterSharedPtr; +using ReadFilterSharedPtr = std::shared_ptr; /** * A combination read and write filter. This allows a single filter instance to cover * both the read and write paths. */ class Filter : public WriteFilter, public ReadFilter {}; -typedef std::shared_ptr FilterSharedPtr; +using FilterSharedPtr = std::shared_ptr; /** * Interface for adding individual network filters to a manager. */ class FilterManager { public: - virtual ~FilterManager() {} + virtual ~FilterManager() = default; /** * Add a write filter to the connection. Filters are invoked in LIFO order (the last added @@ -241,7 +241,7 @@ class FilterManager { * to. Typically the function will install a single filter, but it's technically possibly to * install more than one if desired. */ -typedef std::function FilterFactoryCb; +using FilterFactoryCb = std::function; /** * Callbacks used by individual listener filter instances to communicate with the listener filter @@ -249,7 +249,7 @@ typedef std::function FilterFactoryCb; */ class ListenerFilterCallbacks { public: - virtual ~ListenerFilterCallbacks() {} + virtual ~ListenerFilterCallbacks() = default; /** * @return ConnectionSocket the socket the filter is operating on. @@ -276,7 +276,7 @@ class ListenerFilterCallbacks { */ class ListenerFilter { public: - virtual ~ListenerFilter() {} + virtual ~ListenerFilter() = default; /** * Called when a new connection is accepted, but before a Connection is created. @@ -287,14 +287,14 @@ class ListenerFilter { virtual FilterStatus onAccept(ListenerFilterCallbacks& cb) PURE; }; -typedef std::unique_ptr ListenerFilterPtr; +using ListenerFilterPtr = std::unique_ptr; /** * Interface for filter callbacks and adding listener filters to a manager. */ class ListenerFilterManager { public: - virtual ~ListenerFilterManager() {} + virtual ~ListenerFilterManager() = default; /** * Add a filter to the listener. Filters are invoked in FIFO order (the filter added @@ -312,14 +312,14 @@ class ListenerFilterManager { * Typically the function will install a single filter, but it's technically possibly to install * more than one if desired. */ -typedef std::function ListenerFilterFactoryCb; +using ListenerFilterFactoryCb = std::function; /** * Interface representing a single filter chain. */ class FilterChain { public: - virtual ~FilterChain() {} + virtual ~FilterChain() = default; /** * @return const TransportSocketFactory& a transport socket factory to be used by the new @@ -333,14 +333,14 @@ class FilterChain { virtual const std::vector& networkFilterFactories() const PURE; }; -typedef std::shared_ptr FilterChainSharedPtr; +using FilterChainSharedPtr = std::shared_ptr; /** * Interface for searching through configured filter chains. */ class FilterChainManager { public: - virtual ~FilterChainManager() {} + virtual ~FilterChainManager() = default; /** * Find filter chain that's matching metadata from the new connection. @@ -357,7 +357,7 @@ class FilterChainManager { */ class UdpReadFilterCallbacks { public: - virtual ~UdpReadFilterCallbacks() {} + virtual ~UdpReadFilterCallbacks() = default; /** * @return the udp listener that owns this read filter. @@ -370,7 +370,7 @@ class UdpReadFilterCallbacks { */ class UdpListenerReadFilter { public: - virtual ~UdpListenerReadFilter() {} + virtual ~UdpListenerReadFilter() = default; /** * Called when a new data packet is received on a UDP listener. @@ -387,14 +387,14 @@ class UdpListenerReadFilter { UdpReadFilterCallbacks* read_callbacks_{}; }; -typedef std::unique_ptr UdpListenerReadFilterPtr; +using UdpListenerReadFilterPtr = std::unique_ptr; /** * Interface for adding UDP listener filters to a manager. */ class UdpListenerFilterManager { public: - virtual ~UdpListenerFilterManager() {} + virtual ~UdpListenerFilterManager() = default; /** * Add a read filter to the udp listener. Filters are invoked in FIFO order (the @@ -404,16 +404,15 @@ class UdpListenerFilterManager { virtual void addReadFilter(UdpListenerReadFilterPtr&& filter) PURE; }; -typedef std::function - UdpListenerFilterFactoryCb; +using UdpListenerFilterFactoryCb = std::function; /** * Creates a chain of network filters for a new connection. */ class FilterChainFactory { public: - virtual ~FilterChainFactory() {} + virtual ~FilterChainFactory() = default; /** * Called to create the network filter chain. diff --git a/include/envoy/network/io_handle.h b/include/envoy/network/io_handle.h index 13dba086af40..49cb81e561e4 100644 --- a/include/envoy/network/io_handle.h +++ b/include/envoy/network/io_handle.h @@ -18,7 +18,7 @@ class Instance; */ class IoHandle { public: - virtual ~IoHandle() {} + virtual ~IoHandle() = default; /** * Return data associated with IoHandle. @@ -79,7 +79,7 @@ class IoHandle { int flags, const Address::Instance& address) PURE; }; -typedef std::unique_ptr IoHandlePtr; +using IoHandlePtr = std::unique_ptr; } // namespace Network } // namespace Envoy diff --git a/include/envoy/network/listen_socket.h b/include/envoy/network/listen_socket.h index bcb6b31f6d5f..b2c3dc472cec 100644 --- a/include/envoy/network/listen_socket.h +++ b/include/envoy/network/listen_socket.h @@ -17,14 +17,14 @@ namespace Network { // Optional variant of setsockopt(2) optname. The idea here is that if the option is not supported // on a platform, we can make this the empty value. This allows us to avoid proliferation of #ifdef. -typedef absl::optional> SocketOptionName; +using SocketOptionName = absl::optional>; /** * Base class for Sockets */ class Socket { public: - virtual ~Socket() {} + virtual ~Socket() = default; /** * @return the local address of the socket. @@ -65,7 +65,7 @@ class Socket { */ class Option { public: - virtual ~Option() {} + virtual ~Option() = default; /** * @param socket the socket on which to apply options. @@ -105,9 +105,9 @@ class Socket { envoy::api::v2::core::SocketOption::SocketState state) const PURE; }; - typedef std::shared_ptr OptionConstSharedPtr; - typedef std::vector Options; - typedef std::shared_ptr OptionsSharedPtr; + using OptionConstSharedPtr = std::shared_ptr; + using Options = std::vector; + using OptionsSharedPtr = std::shared_ptr; static OptionsSharedPtr& appendOptions(OptionsSharedPtr& to, const OptionsSharedPtr& from) { to->insert(to->end(), from->begin(), from->end()); @@ -143,8 +143,8 @@ class Socket { virtual const OptionsSharedPtr& options() const PURE; }; -typedef std::unique_ptr SocketPtr; -typedef std::shared_ptr SocketSharedPtr; +using SocketPtr = std::unique_ptr; +using SocketSharedPtr = std::shared_ptr; /** * A socket passed to a connection. For server connections this represents the accepted socket, and @@ -155,7 +155,7 @@ typedef std::shared_ptr SocketSharedPtr; */ class ConnectionSocket : public virtual Socket { public: - virtual ~ConnectionSocket() {} + ~ConnectionSocket() override = default; /** * @return the remote address of the socket. @@ -217,7 +217,7 @@ class ConnectionSocket : public virtual Socket { virtual absl::string_view requestedServerName() const PURE; }; -typedef std::unique_ptr ConnectionSocketPtr; +using ConnectionSocketPtr = std::unique_ptr; /** * Thrown when there is a runtime error binding a socket. diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index d4cb51dfefba..0e61cd4ced4e 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -20,7 +20,7 @@ class UdpListenerFilterManager; */ class ListenerConfig { public: - virtual ~ListenerConfig() {} + virtual ~ListenerConfig() = default; /** * @return FilterChainManager& the factory for adding and searching through configured @@ -90,7 +90,7 @@ class ListenerConfig { */ class ListenerCallbacks { public: - virtual ~ListenerCallbacks() {} + virtual ~ListenerCallbacks() = default; /** * Called when a new connection is accepted. @@ -179,7 +179,7 @@ class UdpListenerCallbacks { */ class Listener { public: - virtual ~Listener() {} + virtual ~Listener() = default; /** * Temporarily disable accepting new connections. @@ -192,14 +192,14 @@ class Listener { virtual void enable() PURE; }; -typedef std::unique_ptr ListenerPtr; +using ListenerPtr = std::unique_ptr; /** * A UDP listener interface. */ class UdpListener : public virtual Listener { public: - virtual ~UdpListener() {} + ~UdpListener() override = default; /** * @return Event::Dispatcher& the dispatcher backing this listener. diff --git a/include/envoy/network/resolver.h b/include/envoy/network/resolver.h index 5e1c6809829e..6544a3c20f3d 100644 --- a/include/envoy/network/resolver.h +++ b/include/envoy/network/resolver.h @@ -18,7 +18,7 @@ namespace Address { */ class Resolver { public: - virtual ~Resolver() {} + virtual ~Resolver() = default; /** * Resolve a custom address string and port to an Address::Instance. diff --git a/include/envoy/network/transport_socket.h b/include/envoy/network/transport_socket.h index 5d8430126949..dab4a2054c31 100644 --- a/include/envoy/network/transport_socket.h +++ b/include/envoy/network/transport_socket.h @@ -46,7 +46,7 @@ struct IoResult { */ class TransportSocketCallbacks { public: - virtual ~TransportSocketCallbacks() {} + virtual ~TransportSocketCallbacks() = default; /** * @return reference to the IoHandle associated with the connection. @@ -89,7 +89,7 @@ class TransportSocketCallbacks { */ class TransportSocket { public: - virtual ~TransportSocket() {} + virtual ~TransportSocket() = default; /** * Called by connection once to initialize the transport socket callbacks that the transport @@ -147,14 +147,14 @@ class TransportSocket { virtual const Ssl::ConnectionInfo* ssl() const PURE; }; -typedef std::unique_ptr TransportSocketPtr; +using TransportSocketPtr = std::unique_ptr; /** * Options for creating transport sockets. */ class TransportSocketOptions { public: - virtual ~TransportSocketOptions() {} + virtual ~TransportSocketOptions() = default; /** * @return the const optional server name to set in the transport socket, for example SNI for @@ -173,14 +173,14 @@ class TransportSocketOptions { virtual void hashKey(std::vector& key) const PURE; }; -typedef std::shared_ptr TransportSocketOptionsSharedPtr; +using TransportSocketOptionsSharedPtr = std::shared_ptr; /** * A factory for creating transport socket. It will be associated to filter chains and clusters. */ class TransportSocketFactory { public: - virtual ~TransportSocketFactory() {} + virtual ~TransportSocketFactory() = default; /** * @return bool whether the transport socket implements secure transport. @@ -195,7 +195,7 @@ class TransportSocketFactory { createTransportSocket(TransportSocketOptionsSharedPtr options) const PURE; }; -typedef std::unique_ptr TransportSocketFactoryPtr; +using TransportSocketFactoryPtr = std::unique_ptr; } // namespace Network } // namespace Envoy diff --git a/include/envoy/router/rds.h b/include/envoy/router/rds.h index 67a82fd45140..9dcd4c3f64e5 100644 --- a/include/envoy/router/rds.h +++ b/include/envoy/router/rds.h @@ -23,7 +23,7 @@ class RouteConfigProvider { std::string version_; }; - virtual ~RouteConfigProvider() {} + virtual ~RouteConfigProvider() = default; /** * @return Router::ConfigConstSharedPtr a route configuration for use during a single request. The @@ -50,7 +50,7 @@ class RouteConfigProvider { virtual void onConfigUpdate() PURE; }; -typedef std::unique_ptr RouteConfigProviderPtr; +using RouteConfigProviderPtr = std::unique_ptr; } // namespace Router } // namespace Envoy diff --git a/include/envoy/router/route_config_provider_manager.h b/include/envoy/router/route_config_provider_manager.h index 9bf02066a600..ffd9922bd1a7 100644 --- a/include/envoy/router/route_config_provider_manager.h +++ b/include/envoy/router/route_config_provider_manager.h @@ -22,7 +22,7 @@ namespace Router { */ class RouteConfigProviderManager { public: - virtual ~RouteConfigProviderManager() {} + virtual ~RouteConfigProviderManager() = default; /** * Get a RouteConfigProviderPtr for a route from RDS. Ownership of the RouteConfigProvider is the diff --git a/include/envoy/router/router.h b/include/envoy/router/router.h index f0ad23da97db..c705a6499300 100644 --- a/include/envoy/router/router.h +++ b/include/envoy/router/router.h @@ -36,7 +36,7 @@ namespace Router { */ class ResponseEntry { public: - virtual ~ResponseEntry() {} + virtual ~ResponseEntry() = default; /** * Do potentially destructive header transforms on response headers prior to forwarding. For @@ -54,7 +54,7 @@ class ResponseEntry { */ class DirectResponseEntry : public ResponseEntry { public: - virtual ~DirectResponseEntry() {} + ~DirectResponseEntry() override = default; /** * Returns the HTTP status code to return. @@ -98,7 +98,7 @@ class DirectResponseEntry : public ResponseEntry { */ class CorsPolicy { public: - virtual ~CorsPolicy() {} + virtual ~CorsPolicy() = default; /** * @return std::list& access-control-allow-origin values. @@ -165,7 +165,7 @@ class RetryPolicy { static const uint32_t RETRY_ON_RETRIABLE_STATUS_CODES = 0x400; // clang-format on - virtual ~RetryPolicy() {} + virtual ~RetryPolicy() = default; /** * @return std::chrono::milliseconds timeout per retry attempt. @@ -233,9 +233,9 @@ enum class InternalRedirectAction { PassThrough, Handle }; */ class RetryState { public: - typedef std::function DoRetryCallback; + using DoRetryCallback = std::function; - virtual ~RetryState() {} + virtual ~RetryState() = default; /** * @return true if a policy is in place for the active request that allows retries. @@ -321,14 +321,14 @@ class RetryState { virtual uint32_t hostSelectionMaxAttempts() const PURE; }; -typedef std::unique_ptr RetryStatePtr; +using RetryStatePtr = std::unique_ptr; /** * Per route policy for request shadowing. */ class ShadowPolicy { public: - virtual ~ShadowPolicy() {} + virtual ~ShadowPolicy() = default; /** * @return the name of the cluster that a matching request should be shadowed to. Returns empty @@ -357,7 +357,7 @@ class ShadowPolicy { */ class VirtualCluster { public: - virtual ~VirtualCluster() {} + virtual ~VirtualCluster() = default; /** * @return the stat-name of the virtual cluster. @@ -375,16 +375,16 @@ class Config; */ class RouteSpecificFilterConfig { public: - virtual ~RouteSpecificFilterConfig() {} + virtual ~RouteSpecificFilterConfig() = default; }; -typedef std::shared_ptr RouteSpecificFilterConfigConstSharedPtr; +using RouteSpecificFilterConfigConstSharedPtr = std::shared_ptr; /** * Virtual host definition. */ class VirtualHost { public: - virtual ~VirtualHost() {} + virtual ~VirtualHost() = default; /** * @return const CorsPolicy* the CORS policy for this virtual host. @@ -433,7 +433,7 @@ class VirtualHost { */ class HashPolicy { public: - virtual ~HashPolicy() {} + virtual ~HashPolicy() = default; /** * A callback used for requesting that a cookie be set with the given lifetime. @@ -442,9 +442,8 @@ class HashPolicy { * @param ttl the lifetime of the cookie * @return std::string the opaque value of the cookie that will be set */ - typedef std::function - AddCookieCallback; + using AddCookieCallback = std::function; /** * @param downstream_address is the address of the connected client host, or nullptr if the @@ -465,7 +464,7 @@ class HashPolicy { */ class HedgePolicy { public: - virtual ~HedgePolicy() {} + virtual ~HedgePolicy() = default; /** * @return number of upstream requests that should be sent initially. @@ -488,7 +487,7 @@ class HedgePolicy { class MetadataMatchCriterion { public: - virtual ~MetadataMatchCriterion() {} + virtual ~MetadataMatchCriterion() = default; /* * @return const std::string& the name of the metadata key @@ -501,14 +500,14 @@ class MetadataMatchCriterion { virtual const HashedValue& value() const PURE; }; -typedef std::shared_ptr MetadataMatchCriterionConstSharedPtr; +using MetadataMatchCriterionConstSharedPtr = std::shared_ptr; class MetadataMatchCriteria; -typedef std::unique_ptr MetadataMatchCriteriaConstPtr; +using MetadataMatchCriteriaConstPtr = std::unique_ptr; class MetadataMatchCriteria { public: - virtual ~MetadataMatchCriteria() {} + virtual ~MetadataMatchCriteria() = default; /* * @return std::vector& a vector of @@ -545,7 +544,7 @@ enum class PathMatchType { */ class PathMatchCriterion { public: - virtual ~PathMatchCriterion() {} + virtual ~PathMatchCriterion() = default; /** * @return PathMatchType type of path match. @@ -568,7 +567,7 @@ class HttpRouteTypedMetadataFactory : public Envoy::Config::TypedMetadataFactory */ class RouteEntry : public ResponseEntry { public: - virtual ~RouteEntry() {} + ~RouteEntry() override = default; /** * @return const std::string& the upstream cluster that owns the route. @@ -729,7 +728,7 @@ class RouteEntry : public ResponseEntry { */ virtual bool includeAttemptCount() const PURE; - typedef std::map UpgradeMap; + using UpgradeMap = std::map; /** * @return a map of route-specific upgrades to their enabled/disabled status. */ @@ -751,7 +750,7 @@ class RouteEntry : public ResponseEntry { */ class Decorator { public: - virtual ~Decorator() {} + virtual ~Decorator() = default; /** * This method decorates the supplied span. @@ -766,14 +765,14 @@ class Decorator { virtual const std::string& getOperation() const PURE; }; -typedef std::unique_ptr DecoratorConstPtr; +using DecoratorConstPtr = std::unique_ptr; /** * An interface representing the Tracing for the route configuration. */ class RouteTracing { public: - virtual ~RouteTracing() {} + virtual ~RouteTracing() = default; /** * This method returns the client sampling percentage. @@ -794,14 +793,14 @@ class RouteTracing { virtual const envoy::type::FractionalPercent& getOverallSampling() const PURE; }; -typedef std::unique_ptr RouteTracingConstPtr; +using RouteTracingConstPtr = std::unique_ptr; /** * An interface that holds a DirectResponseEntry or RouteEntry for a request. */ class Route { public: - virtual ~Route() {} + virtual ~Route() = default; /** * @return the direct response entry or nullptr if there is no direct response for the request. @@ -839,14 +838,14 @@ class Route { } }; -typedef std::shared_ptr RouteConstSharedPtr; +using RouteConstSharedPtr = std::shared_ptr; /** * The router configuration. */ class Config { public: - virtual ~Config() {} + virtual ~Config() = default; /** * Based on the incoming HTTP request headers, determine the target route (containing either a @@ -876,7 +875,7 @@ class Config { virtual bool usesVhds() const PURE; }; -typedef std::shared_ptr ConfigConstSharedPtr; +using ConfigConstSharedPtr = std::shared_ptr; } // namespace Router } // namespace Envoy diff --git a/include/envoy/router/router_ratelimit.h b/include/envoy/router/router_ratelimit.h index 9908b3af13d7..246c177bd47b 100644 --- a/include/envoy/router/router_ratelimit.h +++ b/include/envoy/router/router_ratelimit.h @@ -16,7 +16,7 @@ namespace Router { */ class RateLimitAction { public: - virtual ~RateLimitAction() {} + virtual ~RateLimitAction() = default; /** * Potentially append a descriptor entry to the end of descriptor. @@ -33,14 +33,14 @@ class RateLimitAction { const Network::Address::Instance& remote_address) const PURE; }; -typedef std::unique_ptr RateLimitActionPtr; +using RateLimitActionPtr = std::unique_ptr; /** * Rate limit configuration. */ class RateLimitPolicyEntry { public: - virtual ~RateLimitPolicyEntry() {} + virtual ~RateLimitPolicyEntry() = default; /** * @return the stage value that the configuration is applicable to. @@ -72,7 +72,7 @@ class RateLimitPolicyEntry { */ class RateLimitPolicy { public: - virtual ~RateLimitPolicy() {} + virtual ~RateLimitPolicy() = default; /** * @return true if there is no rate limit policy for all stage settings. diff --git a/include/envoy/router/shadow_writer.h b/include/envoy/router/shadow_writer.h index 9db8d2ba3eb6..3631ff5ec722 100644 --- a/include/envoy/router/shadow_writer.h +++ b/include/envoy/router/shadow_writer.h @@ -16,7 +16,7 @@ namespace Router { */ class ShadowWriter { public: - virtual ~ShadowWriter() {} + virtual ~ShadowWriter() = default; /** * Shadow a request. @@ -28,7 +28,7 @@ class ShadowWriter { std::chrono::milliseconds timeout) PURE; }; -typedef std::unique_ptr ShadowWriterPtr; +using ShadowWriterPtr = std::unique_ptr; } // namespace Router } // namespace Envoy diff --git a/include/envoy/runtime/runtime.h b/include/envoy/runtime/runtime.h index b5e308b14d4a..5f1686570bf1 100644 --- a/include/envoy/runtime/runtime.h +++ b/include/envoy/runtime/runtime.h @@ -23,7 +23,7 @@ namespace Runtime { */ class RandomGenerator { public: - virtual ~RandomGenerator() {} + virtual ~RandomGenerator() = default; /** * @return uint64_t a new random number. @@ -37,14 +37,14 @@ class RandomGenerator { virtual std::string uuid() PURE; }; -typedef std::unique_ptr RandomGeneratorPtr; +using RandomGeneratorPtr = std::unique_ptr; /** * A snapshot of runtime data. */ class Snapshot { public: - virtual ~Snapshot() {} + virtual ~Snapshot() = default; struct Entry { std::string raw_string_value_; @@ -53,7 +53,7 @@ class Snapshot { absl::optional bool_value_; }; - typedef absl::flat_hash_map EntryMap; + using EntryMap = absl::flat_hash_map; /** * A provider of runtime values. One or more of these compose the snapshot's source of values, @@ -61,7 +61,7 @@ class Snapshot { */ class OverrideLayer { public: - virtual ~OverrideLayer() {} + virtual ~OverrideLayer() = default; /** * @return const absl::flat_hash_map& the values in this layer. @@ -74,7 +74,7 @@ class Snapshot { virtual const std::string& name() const PURE; }; - typedef std::unique_ptr OverrideLayerConstPtr; + using OverrideLayerConstPtr = std::unique_ptr; // Returns true if a deprecated feature is allowed. // @@ -204,7 +204,7 @@ class Snapshot { */ class Loader { public: - virtual ~Loader() {} + virtual ~Loader() = default; /** * @return Snapshot& the current snapshot. This reference is safe to use for the duration of diff --git a/include/envoy/secret/secret_callbacks.h b/include/envoy/secret/secret_callbacks.h index b4ee4b814837..afdbcdccca75 100644 --- a/include/envoy/secret/secret_callbacks.h +++ b/include/envoy/secret/secret_callbacks.h @@ -10,7 +10,7 @@ namespace Secret { */ class SecretCallbacks { public: - virtual ~SecretCallbacks() {} + virtual ~SecretCallbacks() = default; virtual void onAddOrUpdateSecret() PURE; }; diff --git a/include/envoy/secret/secret_manager.h b/include/envoy/secret/secret_manager.h index 93205243f046..d37ea920d502 100644 --- a/include/envoy/secret/secret_manager.h +++ b/include/envoy/secret/secret_manager.h @@ -20,7 +20,7 @@ namespace Secret { */ class SecretManager { public: - virtual ~SecretManager() {} + virtual ~SecretManager() = default; /** * @param add a static secret from envoy::api::v2::auth::Secret. diff --git a/include/envoy/secret/secret_provider.h b/include/envoy/secret/secret_provider.h index 93f3004d66ad..4ec8375d5840 100644 --- a/include/envoy/secret/secret_provider.h +++ b/include/envoy/secret/secret_provider.h @@ -16,7 +16,7 @@ namespace Secret { */ template class SecretProvider { public: - virtual ~SecretProvider() {} + virtual ~SecretProvider() = default; /** * @return the secret. Returns nullptr if the secret is not ready. @@ -33,17 +33,17 @@ template class SecretProvider { virtual Common::CallbackHandle* addUpdateCallback(std::function callback) PURE; }; -typedef std::unique_ptr TlsCertificatePtr; -typedef std::unique_ptr - CertificateValidationContextPtr; +using TlsCertificatePtr = std::unique_ptr; +using CertificateValidationContextPtr = + std::unique_ptr; -typedef SecretProvider TlsCertificateConfigProvider; -typedef std::shared_ptr TlsCertificateConfigProviderSharedPtr; +using TlsCertificateConfigProvider = SecretProvider; +using TlsCertificateConfigProviderSharedPtr = std::shared_ptr; -typedef SecretProvider - CertificateValidationContextConfigProvider; -typedef std::shared_ptr - CertificateValidationContextConfigProviderSharedPtr; +using CertificateValidationContextConfigProvider = + SecretProvider; +using CertificateValidationContextConfigProviderSharedPtr = + std::shared_ptr; } // namespace Secret } // namespace Envoy diff --git a/include/envoy/server/access_log_config.h b/include/envoy/server/access_log_config.h index 2b7d02c2229b..032c81e1542c 100644 --- a/include/envoy/server/access_log_config.h +++ b/include/envoy/server/access_log_config.h @@ -17,7 +17,7 @@ namespace Configuration { */ class AccessLogInstanceFactory { public: - virtual ~AccessLogInstanceFactory() {} + virtual ~AccessLogInstanceFactory() = default; /** * Create a particular AccessLog::Instance implementation from a config proto. If the diff --git a/include/envoy/server/admin.h b/include/envoy/server/admin.h index 03500ac84aa3..7e5a4b5f33c2 100644 --- a/include/envoy/server/admin.h +++ b/include/envoy/server/admin.h @@ -19,7 +19,7 @@ namespace Server { class AdminStream { public: - virtual ~AdminStream() {} + virtual ~AdminStream() = default; /** * @param end_stream set to false for streaming response. Default is true, which will @@ -68,7 +68,7 @@ class AdminStream { */ class Admin { public: - virtual ~Admin() {} + virtual ~Admin() = default; /** * Callback for admin URL handlers. @@ -80,11 +80,9 @@ class Admin { * its data. * @return Http::Code the response code. */ - typedef std::function - - HandlerCb; + using HandlerCb = + std::function; /** * Add an admin handler. diff --git a/include/envoy/server/config_tracker.h b/include/envoy/server/config_tracker.h index b893bd2d62ef..53bcb11121e9 100644 --- a/include/envoy/server/config_tracker.h +++ b/include/envoy/server/config_tracker.h @@ -21,8 +21,8 @@ namespace Server { */ class ConfigTracker { public: - typedef std::function Cb; - typedef std::map CbsMap; + using Cb = std::function; + using CbsMap = std::map; /** * EntryOwner supplies RAII semantics for entries in the map. @@ -32,14 +32,14 @@ class ConfigTracker { */ class EntryOwner { public: - virtual ~EntryOwner() {} + virtual ~EntryOwner() = default; protected: - EntryOwner(){}; // A sly way to make this class "abstract." + EntryOwner() = default; // A sly way to make this class "abstract." }; - typedef std::unique_ptr EntryOwnerPtr; + using EntryOwnerPtr = std::unique_ptr; - virtual ~ConfigTracker(){}; + virtual ~ConfigTracker() = default; /** * @return const CbsMap& The map of string keys to tracked callbacks. diff --git a/include/envoy/server/configuration.h b/include/envoy/server/configuration.h index 87020f525297..df2022ba9cee 100644 --- a/include/envoy/server/configuration.h +++ b/include/envoy/server/configuration.h @@ -22,7 +22,7 @@ namespace Configuration { */ class Main { public: - virtual ~Main() {} + virtual ~Main() = default; /** * @return Upstream::ClusterManager* singleton for use by the entire server. @@ -76,7 +76,7 @@ class Main { */ class Admin { public: - virtual ~Admin() {} + virtual ~Admin() = default; /** * @return const std::string& the admin access log path. @@ -99,7 +99,7 @@ class Admin { */ class Initial { public: - virtual ~Initial() {} + virtual ~Initial() = default; /** * @return Admin& the admin config. diff --git a/include/envoy/server/drain_manager.h b/include/envoy/server/drain_manager.h index 61aa7dd377a9..214ed65c0f93 100644 --- a/include/envoy/server/drain_manager.h +++ b/include/envoy/server/drain_manager.h @@ -28,7 +28,7 @@ class DrainManager : public Network::DrainDecision { virtual void startParentShutdownSequence() PURE; }; -typedef std::unique_ptr DrainManagerPtr; +using DrainManagerPtr = std::unique_ptr; } // namespace Server } // namespace Envoy diff --git a/include/envoy/server/filter_config.h b/include/envoy/server/filter_config.h index 4fec1c04d0d3..cf99d734ad49 100644 --- a/include/envoy/server/filter_config.h +++ b/include/envoy/server/filter_config.h @@ -38,7 +38,7 @@ namespace Configuration { */ class FactoryContext { public: - virtual ~FactoryContext() {} + virtual ~FactoryContext() = default; /** * @return AccessLogManager for use by the entire server. @@ -191,7 +191,7 @@ class ListenerFactoryContext : public virtual FactoryContext { */ class ListenerFilterConfigFactoryBase { public: - virtual ~ListenerFilterConfigFactoryBase() {} + virtual ~ListenerFilterConfigFactoryBase() = default; /** * @return ProtobufTypes::MessagePtr create empty config proto message. The filter @@ -212,7 +212,7 @@ class ListenerFilterConfigFactoryBase { */ class NamedListenerFilterConfigFactory : public ListenerFilterConfigFactoryBase { public: - virtual ~NamedListenerFilterConfigFactory() {} + ~NamedListenerFilterConfigFactory() override = default; /** * Create a particular listener filter factory implementation. If the implementation is unable to @@ -234,7 +234,7 @@ class NamedListenerFilterConfigFactory : public ListenerFilterConfigFactoryBase */ class NamedUdpListenerFilterConfigFactory : public ListenerFilterConfigFactoryBase { public: - virtual ~NamedUdpListenerFilterConfigFactory() {} + ~NamedUdpListenerFilterConfigFactory() override = default; /** * Create a particular UDP listener filter factory implementation. If the implementation is unable @@ -255,7 +255,7 @@ class NamedUdpListenerFilterConfigFactory : public ListenerFilterConfigFactoryBa */ class ProtocolOptionsFactory { public: - virtual ~ProtocolOptionsFactory() {} + virtual ~ProtocolOptionsFactory() = default; /** * Create a particular filter's protocol specific options implementation. If the factory @@ -283,7 +283,7 @@ class ProtocolOptionsFactory { */ class NamedNetworkFilterConfigFactory : public ProtocolOptionsFactory { public: - virtual ~NamedNetworkFilterConfigFactory() {} + ~NamedNetworkFilterConfigFactory() override = default; /** * Create a particular network filter factory implementation. If the implementation is unable to @@ -329,7 +329,7 @@ class NamedNetworkFilterConfigFactory : public ProtocolOptionsFactory { */ class NamedHttpFilterConfigFactory : public ProtocolOptionsFactory { public: - virtual ~NamedHttpFilterConfigFactory() {} + ~NamedHttpFilterConfigFactory() override = default; /** * Create a particular http filter factory implementation. If the implementation is unable to diff --git a/include/envoy/server/guarddog.h b/include/envoy/server/guarddog.h index ce5a37c4b2bc..b17c55379aac 100644 --- a/include/envoy/server/guarddog.h +++ b/include/envoy/server/guarddog.h @@ -17,7 +17,7 @@ namespace Server { */ class GuardDog { public: - virtual ~GuardDog() {} + virtual ~GuardDog() = default; /** * Get a WatchDog object pointer to a new WatchDog. diff --git a/include/envoy/server/health_checker_config.h b/include/envoy/server/health_checker_config.h index 6e6194a9aa53..163a1d7c2801 100644 --- a/include/envoy/server/health_checker_config.h +++ b/include/envoy/server/health_checker_config.h @@ -9,7 +9,7 @@ namespace Configuration { class HealthCheckerFactoryContext { public: - virtual ~HealthCheckerFactoryContext() {} + virtual ~HealthCheckerFactoryContext() = default; /** * @return Upstream::Cluster& the owning cluster. @@ -51,7 +51,7 @@ class HealthCheckerFactoryContext { */ class CustomHealthCheckerFactory { public: - virtual ~CustomHealthCheckerFactory() {} + virtual ~CustomHealthCheckerFactory() = default; /** * Creates a particular custom health checker factory implementation. diff --git a/include/envoy/server/hot_restart.h b/include/envoy/server/hot_restart.h index f9d80729bf2b..916646f5c087 100644 --- a/include/envoy/server/hot_restart.h +++ b/include/envoy/server/hot_restart.h @@ -28,7 +28,7 @@ class HotRestart { uint64_t parent_connections_ = 0; }; - virtual ~HotRestart() {} + virtual ~HotRestart() = default; /** * Shutdown listeners in the parent process if applicable. Listeners will begin draining to diff --git a/include/envoy/server/instance.h b/include/envoy/server/instance.h index 3659c67973b3..dbb964eb3df0 100644 --- a/include/envoy/server/instance.h +++ b/include/envoy/server/instance.h @@ -35,7 +35,7 @@ namespace Server { */ class Instance { public: - virtual ~Instance() {} + virtual ~Instance() = default; /** * @return Admin& the global HTTP admin endpoint for the server. diff --git a/include/envoy/server/listener_manager.h b/include/envoy/server/listener_manager.h index 513e7b179380..ce2a015decf3 100644 --- a/include/envoy/server/listener_manager.h +++ b/include/envoy/server/listener_manager.h @@ -18,7 +18,7 @@ namespace Server { */ class LdsApi { public: - virtual ~LdsApi() {} + virtual ~LdsApi() = default; /** * @return std::string the last received version by the xDS API for LDS. @@ -26,14 +26,14 @@ class LdsApi { virtual std::string versionInfo() const PURE; }; -typedef std::unique_ptr LdsApiPtr; +using LdsApiPtr = std::unique_ptr; /** * Factory for creating listener components. */ class ListenerComponentFactory { public: - virtual ~ListenerComponentFactory() {} + virtual ~ListenerComponentFactory() = default; /** * @return an LDS API provider. @@ -101,7 +101,7 @@ class ListenerComponentFactory { */ class ListenerManager { public: - virtual ~ListenerManager() {} + virtual ~ListenerManager() = default; /** * Add or update a listener. Listeners are referenced by a unique name. If no name is provided, diff --git a/include/envoy/server/options.h b/include/envoy/server/options.h index 63cc06503e72..e78b499a7f3e 100644 --- a/include/envoy/server/options.h +++ b/include/envoy/server/options.h @@ -40,14 +40,14 @@ enum class Mode { // to be validated in a non-prod environment. }; -typedef std::unique_ptr CommandLineOptionsPtr; +using CommandLineOptionsPtr = std::unique_ptr; /** * General options for the server. */ class Options { public: - virtual ~Options() {} + virtual ~Options() = default; /** * @return uint64_t the base ID for the server. This is required for system-wide things like diff --git a/include/envoy/server/overload_manager.h b/include/envoy/server/overload_manager.h index 8ee920548e20..127b5f71bb52 100644 --- a/include/envoy/server/overload_manager.h +++ b/include/envoy/server/overload_manager.h @@ -26,7 +26,7 @@ enum class OverloadActionState { /** * Callback invoked when an overload action changes state. */ -typedef std::function OverloadActionCb; +using OverloadActionCb = std::function; /** * Thread-local copy of the state of each configured overload action. @@ -72,7 +72,7 @@ class OverloadActionNameValues { const std::string ShrinkHeap = "envoy.overload_actions.shrink_heap"; }; -typedef ConstSingleton OverloadActionNames; +using OverloadActionNames = ConstSingleton; /** * The OverloadManager protects the Envoy instance from being overwhelmed by client @@ -81,7 +81,7 @@ typedef ConstSingleton OverloadActionNames; */ class OverloadManager { public: - virtual ~OverloadManager() {} + virtual ~OverloadManager() = default; /** * Start a recurring timer to monitor resources and notify listeners when overload actions diff --git a/include/envoy/server/process_context.h b/include/envoy/server/process_context.h index 7a4ceeddb782..cbba5ea1592b 100644 --- a/include/envoy/server/process_context.h +++ b/include/envoy/server/process_context.h @@ -9,7 +9,7 @@ namespace Envoy { */ class ProcessObject { public: - virtual ~ProcessObject() {} + virtual ~ProcessObject() = default; }; /** @@ -18,7 +18,7 @@ class ProcessObject { */ class ProcessContext { public: - virtual ~ProcessContext() {} + virtual ~ProcessContext() = default; /** * @return the ProcessObject for this context. diff --git a/include/envoy/server/resource_monitor.h b/include/envoy/server/resource_monitor.h index 98b219d48cc2..4eb947527ade 100644 --- a/include/envoy/server/resource_monitor.h +++ b/include/envoy/server/resource_monitor.h @@ -20,14 +20,14 @@ struct ResourceUsage { class ResourceMonitor { public: - virtual ~ResourceMonitor() {} + virtual ~ResourceMonitor() = default; /** * Notifies caller of updated resource usage. */ class Callbacks { public: - virtual ~Callbacks() {} + virtual ~Callbacks() = default; /** * Called when the request for updated resource usage succeeds. @@ -50,7 +50,7 @@ class ResourceMonitor { virtual void updateResourceUsage(Callbacks& callbacks) PURE; }; -typedef std::unique_ptr ResourceMonitorPtr; +using ResourceMonitorPtr = std::unique_ptr; } // namespace Server } // namespace Envoy diff --git a/include/envoy/server/resource_monitor_config.h b/include/envoy/server/resource_monitor_config.h index 3c280f3543d8..aef576b8e4bf 100644 --- a/include/envoy/server/resource_monitor_config.h +++ b/include/envoy/server/resource_monitor_config.h @@ -13,7 +13,7 @@ namespace Configuration { class ResourceMonitorFactoryContext { public: - virtual ~ResourceMonitorFactoryContext() {} + virtual ~ResourceMonitorFactoryContext() = default; /** * @return Event::Dispatcher& the main thread's dispatcher. This dispatcher should be used @@ -33,7 +33,7 @@ class ResourceMonitorFactoryContext { */ class ResourceMonitorFactory { public: - virtual ~ResourceMonitorFactory() {} + virtual ~ResourceMonitorFactory() = default; /** * Create a particular resource monitor implementation. diff --git a/include/envoy/server/tracer_config.h b/include/envoy/server/tracer_config.h index f9ee2a55509d..fcbe8bb7f1e3 100644 --- a/include/envoy/server/tracer_config.h +++ b/include/envoy/server/tracer_config.h @@ -16,7 +16,7 @@ namespace Configuration { */ class TracerFactory { public: - virtual ~TracerFactory() {} + virtual ~TracerFactory() = default; /** * Create a particular HttpTracer implementation. If the implementation is unable to produce an diff --git a/include/envoy/server/transport_socket_config.h b/include/envoy/server/transport_socket_config.h index cd8c6d8643af..ec568e3045b1 100644 --- a/include/envoy/server/transport_socket_config.h +++ b/include/envoy/server/transport_socket_config.h @@ -25,7 +25,7 @@ namespace Configuration { */ class TransportSocketFactoryContext { public: - virtual ~TransportSocketFactoryContext() {} + virtual ~TransportSocketFactoryContext() = default; /** * @return Server::Admin& the server's admin interface. @@ -108,7 +108,7 @@ class TransportSocketFactoryContext { class TransportSocketConfigFactory { public: - virtual ~TransportSocketConfigFactory() {} + virtual ~TransportSocketConfigFactory() = default; /** * @return ProtobufTypes::MessagePtr create empty config proto message. The transport socket diff --git a/include/envoy/server/watchdog.h b/include/envoy/server/watchdog.h index a571326f6ad4..f60755784582 100644 --- a/include/envoy/server/watchdog.h +++ b/include/envoy/server/watchdog.h @@ -17,7 +17,7 @@ namespace Server { */ class WatchDog { public: - virtual ~WatchDog() {} + virtual ~WatchDog() = default; /** * Start a recurring touch timer in the dispatcher passed as argument. @@ -40,7 +40,7 @@ class WatchDog { virtual MonotonicTime lastTouchTime() const PURE; }; -typedef std::shared_ptr WatchDogSharedPtr; +using WatchDogSharedPtr = std::shared_ptr; } // namespace Server } // namespace Envoy diff --git a/include/envoy/server/worker.h b/include/envoy/server/worker.h index 255209531f57..f412dc922235 100644 --- a/include/envoy/server/worker.h +++ b/include/envoy/server/worker.h @@ -13,7 +13,7 @@ namespace Server { */ class Worker { public: - virtual ~Worker() {} + virtual ~Worker() = default; /** * Completion called when a listener has been added on a worker and is listening for new @@ -21,7 +21,7 @@ class Worker { * @param success supplies whether the addition was successful or not. FALSE can be returned * when there is a race condition between bind() and listen(). */ - typedef std::function AddListenerCompletion; + using AddListenerCompletion = std::function; /** * Add a listener to the worker. @@ -81,14 +81,14 @@ class Worker { virtual void stopListeners() PURE; }; -typedef std::unique_ptr WorkerPtr; +using WorkerPtr = std::unique_ptr; /** * Factory for creating workers. */ class WorkerFactory { public: - virtual ~WorkerFactory() {} + virtual ~WorkerFactory() = default; /** * @return WorkerPtr a new worker. diff --git a/include/envoy/singleton/instance.h b/include/envoy/singleton/instance.h index 26d77800984e..f22af8c489c0 100644 --- a/include/envoy/singleton/instance.h +++ b/include/envoy/singleton/instance.h @@ -10,10 +10,10 @@ namespace Singleton { */ class Instance { public: - virtual ~Instance() {} + virtual ~Instance() = default; }; -typedef std::shared_ptr InstanceSharedPtr; +using InstanceSharedPtr = std::shared_ptr; } // namespace Singleton } // namespace Envoy diff --git a/include/envoy/singleton/manager.h b/include/envoy/singleton/manager.h index 9721d9318ff8..1cfba8d7b3ae 100644 --- a/include/envoy/singleton/manager.h +++ b/include/envoy/singleton/manager.h @@ -16,7 +16,7 @@ namespace Singleton { */ class Registration { public: - virtual ~Registration() {} + virtual ~Registration() = default; virtual std::string name() PURE; }; @@ -54,14 +54,14 @@ template class RegistrationImpl : public Registration { /** * Callback function used to create a singleton. */ -typedef std::function SingletonFactoryCb; +using SingletonFactoryCb = std::function; /** * A manager for all server-side singletons. */ class Manager { public: - virtual ~Manager() {} + virtual ~Manager() = default; /** * This is a helper on top of get() that casts the object stored to the specified type. Since the @@ -84,7 +84,7 @@ class Manager { virtual InstanceSharedPtr get(const std::string& name, SingletonFactoryCb) PURE; }; -typedef std::unique_ptr ManagerPtr; +using ManagerPtr = std::unique_ptr; } // namespace Singleton } // namespace Envoy diff --git a/include/envoy/ssl/certificate_validation_context_config.h b/include/envoy/ssl/certificate_validation_context_config.h index 60d223eebc16..58d26bac5eff 100644 --- a/include/envoy/ssl/certificate_validation_context_config.h +++ b/include/envoy/ssl/certificate_validation_context_config.h @@ -11,7 +11,7 @@ namespace Ssl { class CertificateValidationContextConfig { public: - virtual ~CertificateValidationContextConfig() {} + virtual ~CertificateValidationContextConfig() = default; /** * @return The CA certificate to use for peer validation. @@ -56,7 +56,7 @@ class CertificateValidationContextConfig { virtual bool allowExpiredCertificate() const PURE; }; -typedef std::unique_ptr CertificateValidationContextConfigPtr; +using CertificateValidationContextConfigPtr = std::unique_ptr; } // namespace Ssl } // namespace Envoy diff --git a/include/envoy/ssl/connection.h b/include/envoy/ssl/connection.h index 0302159cba14..203bd9a4c812 100644 --- a/include/envoy/ssl/connection.h +++ b/include/envoy/ssl/connection.h @@ -16,7 +16,7 @@ namespace Ssl { */ class ConnectionInfo { public: - virtual ~ConnectionInfo() {} + virtual ~ConnectionInfo() = default; /** * @return bool whether the peer certificate is presented. diff --git a/include/envoy/ssl/context.h b/include/envoy/ssl/context.h index eb105b39fa85..6b7a3600331f 100644 --- a/include/envoy/ssl/context.h +++ b/include/envoy/ssl/context.h @@ -9,14 +9,14 @@ namespace Envoy { namespace Ssl { -typedef std::unique_ptr CertificateDetailsPtr; +using CertificateDetailsPtr = std::unique_ptr; /** * SSL Context is used as a template for SSL connection configuration. */ class Context { public: - virtual ~Context() {} + virtual ~Context() = default; /** * @return the number of days in this context until the next certificate will expire @@ -33,13 +33,13 @@ class Context { */ virtual std::vector getCertChainInformation() const PURE; }; -typedef std::shared_ptr ContextSharedPtr; +using ContextSharedPtr = std::shared_ptr; class ClientContext : public virtual Context {}; -typedef std::shared_ptr ClientContextSharedPtr; +using ClientContextSharedPtr = std::shared_ptr; class ServerContext : public virtual Context {}; -typedef std::shared_ptr ServerContextSharedPtr; +using ServerContextSharedPtr = std::shared_ptr; } // namespace Ssl } // namespace Envoy diff --git a/include/envoy/ssl/context_config.h b/include/envoy/ssl/context_config.h index 5f9063e85d0c..41b41c37d3ef 100644 --- a/include/envoy/ssl/context_config.h +++ b/include/envoy/ssl/context_config.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -16,7 +17,7 @@ namespace Ssl { */ class ContextConfig { public: - virtual ~ContextConfig() {} + virtual ~ContextConfig() = default; /** * The list of supported protocols exposed via ALPN. Client connections will send these @@ -98,7 +99,7 @@ class ClientContextConfig : public virtual ContextConfig { virtual const std::string& signingAlgorithmsForTest() const PURE; }; -typedef std::unique_ptr ClientContextConfigPtr; +using ClientContextConfigPtr = std::unique_ptr; class ServerContextConfig : public virtual ContextConfig { public: @@ -121,7 +122,7 @@ class ServerContextConfig : public virtual ContextConfig { virtual const std::vector& sessionTicketKeys() const PURE; }; -typedef std::unique_ptr ServerContextConfigPtr; +using ServerContextConfigPtr = std::unique_ptr; } // namespace Ssl } // namespace Envoy diff --git a/include/envoy/ssl/context_manager.h b/include/envoy/ssl/context_manager.h index 6f55d3477d96..928e2bf9a54b 100644 --- a/include/envoy/ssl/context_manager.h +++ b/include/envoy/ssl/context_manager.h @@ -14,7 +14,7 @@ namespace Ssl { */ class ContextManager { public: - virtual ~ContextManager() {} + virtual ~ContextManager() = default; /** * Builds a ClientContext from a ClientContextConfig. diff --git a/include/envoy/ssl/tls_certificate_config.h b/include/envoy/ssl/tls_certificate_config.h index d72a92c31f82..f934e5654a7a 100644 --- a/include/envoy/ssl/tls_certificate_config.h +++ b/include/envoy/ssl/tls_certificate_config.h @@ -10,7 +10,7 @@ namespace Ssl { class TlsCertificateConfig { public: - virtual ~TlsCertificateConfig() {} + virtual ~TlsCertificateConfig() = default; /** * @return a string of certificate chain. @@ -46,7 +46,7 @@ class TlsCertificateConfig { virtual const std::string& passwordPath() const PURE; }; -typedef std::unique_ptr TlsCertificateConfigPtr; +using TlsCertificateConfigPtr = std::unique_ptr; } // namespace Ssl } // namespace Envoy diff --git a/include/envoy/stats/histogram.h b/include/envoy/stats/histogram.h index 0cd2d6a474f3..6aa312a62572 100644 --- a/include/envoy/stats/histogram.h +++ b/include/envoy/stats/histogram.h @@ -15,7 +15,7 @@ namespace Stats { */ class HistogramStatistics { public: - virtual ~HistogramStatistics() {} + virtual ~HistogramStatistics() = default; /** * Returns quantile summary representation of the histogram. @@ -73,7 +73,7 @@ class HistogramStatistics { */ class Histogram : public virtual Metric { public: - virtual ~Histogram() {} + ~Histogram() override = default; /** * Records an unsigned value. If a timer, values are in units of milliseconds. @@ -81,14 +81,14 @@ class Histogram : public virtual Metric { virtual void recordValue(uint64_t value) PURE; }; -typedef std::shared_ptr HistogramSharedPtr; +using HistogramSharedPtr = std::shared_ptr; /** * A histogram that is stored in main thread and provides summary view of the histogram. */ class ParentHistogram : public virtual Histogram { public: - virtual ~ParentHistogram() {} + ~ParentHistogram() override = default; /** * This method is called during the main stats flush process for each of the histograms and used @@ -117,7 +117,7 @@ class ParentHistogram : public virtual Histogram { virtual const std::string bucketSummary() const PURE; }; -typedef std::shared_ptr ParentHistogramSharedPtr; +using ParentHistogramSharedPtr = std::shared_ptr; } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stats/scope.h b/include/envoy/stats/scope.h index 865c027553d6..ef01045374f1 100644 --- a/include/envoy/stats/scope.h +++ b/include/envoy/stats/scope.h @@ -19,8 +19,8 @@ class Histogram; class Scope; class NullGaugeImpl; -typedef std::unique_ptr ScopePtr; -typedef std::shared_ptr ScopeSharedPtr; +using ScopePtr = std::unique_ptr; +using ScopeSharedPtr = std::shared_ptr; /** * A named scope for stats. Scopes are a grouping of stats that can be acted on as a unit if needed @@ -28,7 +28,7 @@ typedef std::shared_ptr ScopeSharedPtr; */ class Scope { public: - virtual ~Scope() {} + virtual ~Scope() = default; /** * Allocate a new scope. NOTE: The implementation should correctly handle overlapping scopes diff --git a/include/envoy/stats/stat_data_allocator.h b/include/envoy/stats/stat_data_allocator.h index fed33d927e92..2fc94861828d 100644 --- a/include/envoy/stats/stat_data_allocator.h +++ b/include/envoy/stats/stat_data_allocator.h @@ -27,7 +27,7 @@ namespace Stats { */ class StatDataAllocator { public: - virtual ~StatDataAllocator() {} + virtual ~StatDataAllocator() = default; /** * @param name the full name of the stat. diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index 8a8cfd7ab423..7dca08d43da4 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -22,7 +22,7 @@ struct Tag; */ class Metric { public: - virtual ~Metric() {} + virtual ~Metric() = default; /** * Returns the full name of the Metric. This is intended for most uses, such * as streaming out the name to a stats sink or admin request, or comparing @@ -114,7 +114,7 @@ class Metric { */ class Counter : public virtual Metric { public: - virtual ~Counter() {} + ~Counter() override = default; virtual void add(uint64_t amount) PURE; virtual void inc() PURE; virtual uint64_t latch() PURE; @@ -122,7 +122,7 @@ class Counter : public virtual Metric { virtual uint64_t value() const PURE; }; -typedef std::shared_ptr CounterSharedPtr; +using CounterSharedPtr = std::shared_ptr; /** * A gauge that can both increment and decrement. @@ -135,7 +135,7 @@ class Gauge : public virtual Metric { Accumulate, // Transfers gauge state on hot-restart. }; - virtual ~Gauge() {} + ~Gauge() override = default; virtual void add(uint64_t amount) PURE; virtual void dec() PURE; @@ -161,7 +161,7 @@ class Gauge : public virtual Metric { virtual void mergeImportMode(ImportMode import_mode) PURE; }; -typedef std::shared_ptr GaugeSharedPtr; +using GaugeSharedPtr = std::shared_ptr; } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stats/stats_matcher.h b/include/envoy/stats/stats_matcher.h index 2e42cb3289d6..fc0c7c1cc84e 100644 --- a/include/envoy/stats/stats_matcher.h +++ b/include/envoy/stats/stats_matcher.h @@ -11,7 +11,7 @@ namespace Stats { class StatsMatcher { public: - virtual ~StatsMatcher() {} + virtual ~StatsMatcher() = default; /** * Take a metric name and report whether or not it should be instantiated. @@ -41,7 +41,7 @@ class StatsMatcher { virtual bool rejectsAll() const PURE; }; -typedef std::unique_ptr StatsMatcherPtr; +using StatsMatcherPtr = std::unique_ptr; } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stats/store.h b/include/envoy/stats/store.h index 36f827467d8a..d959a58b01d9 100644 --- a/include/envoy/stats/store.h +++ b/include/envoy/stats/store.h @@ -44,12 +44,12 @@ class Store : public Scope { virtual std::vector histograms() const PURE; }; -typedef std::unique_ptr StorePtr; +using StorePtr = std::unique_ptr; /** * Callback invoked when a store's mergeHistogram() runs. */ -typedef std::function PostMergeCb; +using PostMergeCb = std::function; /** * The root of the stat store. @@ -96,7 +96,7 @@ class StoreRoot : public Store { virtual void mergeHistograms(PostMergeCb merge_complete_cb) PURE; }; -typedef std::unique_ptr StoreRootPtr; +using StoreRootPtr = std::unique_ptr; } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stats/tag_extractor.h b/include/envoy/stats/tag_extractor.h index 361cda3e2266..1b6f10d32ea1 100644 --- a/include/envoy/stats/tag_extractor.h +++ b/include/envoy/stats/tag_extractor.h @@ -18,7 +18,7 @@ namespace Stats { */ class TagExtractor { public: - virtual ~TagExtractor() {} + virtual ~TagExtractor() = default; /** * Identifier for the tag extracted by this object. @@ -58,7 +58,7 @@ class TagExtractor { virtual absl::string_view prefixToken() const PURE; }; -typedef std::unique_ptr TagExtractorPtr; +using TagExtractorPtr = std::unique_ptr; } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stats/tag_producer.h b/include/envoy/stats/tag_producer.h index 5ec72877981b..0edfe110bfc6 100644 --- a/include/envoy/stats/tag_producer.h +++ b/include/envoy/stats/tag_producer.h @@ -14,7 +14,7 @@ namespace Stats { class TagProducer { public: - virtual ~TagProducer() {} + virtual ~TagProducer() = default; /** * Take a metric name and a vector then add proper tags into the vector and @@ -32,7 +32,7 @@ class TagProducer { virtual std::string produceTags(absl::string_view metric_name, std::vector& tags) const PURE; }; -typedef std::unique_ptr TagProducerPtr; +using TagProducerPtr = std::unique_ptr; } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stats/timespan.h b/include/envoy/stats/timespan.h index 20910847db28..8fad46bfb2ce 100644 --- a/include/envoy/stats/timespan.h +++ b/include/envoy/stats/timespan.h @@ -15,7 +15,7 @@ namespace Stats { */ class CompletableTimespan { public: - virtual ~CompletableTimespan() {} + virtual ~CompletableTimespan() = default; /** * Complete the timespan. @@ -52,9 +52,9 @@ template class TimespanWithUnit : public CompletableTimespan { const MonotonicTime start_; }; -typedef TimespanWithUnit Timespan; -typedef std::unique_ptr TimespanPtr; -typedef std::unique_ptr CompletableTimespanPtr; +using Timespan = TimespanWithUnit; +using TimespanPtr = std::unique_ptr; +using CompletableTimespanPtr = std::unique_ptr; } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stream_info/filter_state.h b/include/envoy/stream_info/filter_state.h index d52a64584bfb..28cec7f01d78 100644 --- a/include/envoy/stream_info/filter_state.h +++ b/include/envoy/stream_info/filter_state.h @@ -24,10 +24,10 @@ class FilterState { class Object { public: - virtual ~Object(){}; + virtual ~Object() = default; }; - virtual ~FilterState(){}; + virtual ~FilterState() = default; /** * @param data_name the name of the data being set. diff --git a/include/envoy/stream_info/stream_info.h b/include/envoy/stream_info/stream_info.h index 34f4e26c6d3a..fa2a0ffd92ed 100644 --- a/include/envoy/stream_info/stream_info.h +++ b/include/envoy/stream_info/stream_info.h @@ -130,7 +130,7 @@ struct ResponseCodeDetailValues { const std::string LateUpstreamReset = "upstream_reset_after_response_started"; }; -typedef ConstSingleton ResponseCodeDetails; +using ResponseCodeDetails = ConstSingleton; struct UpstreamTiming { /** @@ -176,7 +176,7 @@ struct UpstreamTiming { */ class StreamInfo { public: - virtual ~StreamInfo() {} + virtual ~StreamInfo() = default; /** * @param response_flag the response flag. Each filter can set independent response flags. The diff --git a/include/envoy/tcp/conn_pool.h b/include/envoy/tcp/conn_pool.h index 6c8515c6ac94..43eba8bddae7 100644 --- a/include/envoy/tcp/conn_pool.h +++ b/include/envoy/tcp/conn_pool.h @@ -32,7 +32,7 @@ enum class CancelPolicy { */ class Cancellable { public: - virtual ~Cancellable() {} + virtual ~Cancellable() = default; /** * Cancel the pending connection request. @@ -63,7 +63,7 @@ enum class PoolFailureReason { */ class UpstreamCallbacks : public Network::ConnectionCallbacks { public: - virtual ~UpstreamCallbacks() {} + ~UpstreamCallbacks() override = default; /* * Invoked when data is delivered from the upstream connection while the connection is owned by a @@ -85,10 +85,10 @@ class UpstreamCallbacks : public Network::ConnectionCallbacks { */ class ConnectionState { public: - virtual ~ConnectionState() {} + virtual ~ConnectionState() = default; }; -typedef std::unique_ptr ConnectionStatePtr; +using ConnectionStatePtr = std::unique_ptr; /* * ConnectionData wraps a ClientConnection allocated to a caller. Open ClientConnections are @@ -96,7 +96,7 @@ typedef std::unique_ptr ConnectionStatePtr; */ class ConnectionData { public: - virtual ~ConnectionData() {} + virtual ~ConnectionData() = default; /** * @return the ClientConnection for the connection. @@ -130,7 +130,7 @@ class ConnectionData { virtual ConnectionState* connectionState() PURE; }; -typedef std::unique_ptr ConnectionDataPtr; +using ConnectionDataPtr = std::unique_ptr; /** * Pool callbacks invoked in the context of a newConnection() call, either synchronously or @@ -138,7 +138,7 @@ typedef std::unique_ptr ConnectionDataPtr; */ class Callbacks { public: - virtual ~Callbacks() {} + virtual ~Callbacks() = default; /** * Called when a pool error occurred and no connection could be acquired for making the request. @@ -168,13 +168,13 @@ class Callbacks { */ class Instance : public Event::DeferredDeletable { public: - virtual ~Instance() {} + ~Instance() override = default; /** * Called when a connection pool has been drained of pending requests, busy connections, and * ready connections. */ - typedef std::function DrainedCb; + using DrainedCb = std::function; /** * Register a callback that gets called when the connection pool is fully drained. No actual @@ -205,7 +205,7 @@ class Instance : public Event::DeferredDeletable { virtual Cancellable* newConnection(Callbacks& callbacks) PURE; }; -typedef std::unique_ptr InstancePtr; +using InstancePtr = std::unique_ptr; } // namespace ConnectionPool } // namespace Tcp diff --git a/include/envoy/thread/thread.h b/include/envoy/thread/thread.h index e9078afa476b..41dd7291ee07 100644 --- a/include/envoy/thread/thread.h +++ b/include/envoy/thread/thread.h @@ -12,17 +12,17 @@ namespace Thread { class ThreadId { public: - virtual ~ThreadId() {} + virtual ~ThreadId() = default; virtual std::string debugString() const PURE; virtual bool isCurrentThreadId() const PURE; }; -typedef std::unique_ptr ThreadIdPtr; +using ThreadIdPtr = std::unique_ptr; class Thread { public: - virtual ~Thread() {} + virtual ~Thread() = default; /** * Join on thread exit. @@ -30,14 +30,14 @@ class Thread { virtual void join() PURE; }; -typedef std::unique_ptr ThreadPtr; +using ThreadPtr = std::unique_ptr; /** * Interface providing a mechanism for creating threads. */ class ThreadFactory { public: - virtual ~ThreadFactory() {} + virtual ~ThreadFactory() = default; /** * Create a thread. @@ -57,7 +57,7 @@ class ThreadFactory { */ class LOCKABLE BasicLockable { public: - virtual ~BasicLockable() {} + virtual ~BasicLockable() = default; virtual void lock() EXCLUSIVE_LOCK_FUNCTION() PURE; virtual bool tryLock() EXCLUSIVE_TRYLOCK_FUNCTION(true) PURE; diff --git a/include/envoy/thread_local/thread_local.h b/include/envoy/thread_local/thread_local.h index c6eb0b54bb4a..14fac5cd5e22 100644 --- a/include/envoy/thread_local/thread_local.h +++ b/include/envoy/thread_local/thread_local.h @@ -15,10 +15,10 @@ namespace ThreadLocal { */ class ThreadLocalObject { public: - virtual ~ThreadLocalObject() {} + virtual ~ThreadLocalObject() = default; }; -typedef std::shared_ptr ThreadLocalObjectSharedPtr; +using ThreadLocalObjectSharedPtr = std::shared_ptr; /** * An individual allocated TLS slot. When the slot is destroyed the stored thread local will @@ -26,7 +26,7 @@ typedef std::shared_ptr ThreadLocalObjectSharedPtr; */ class Slot { public: - virtual ~Slot() {} + virtual ~Slot() = default; /** * @return ThreadLocalObjectSharedPtr a thread local object stored in the slot. @@ -62,18 +62,18 @@ class Slot { * a shared_ptr. Thus, this is a flexible mechanism that can be used to share * the same data across all threads or to share different data on each thread. */ - typedef std::function InitializeCb; + using InitializeCb = std::function; virtual void set(InitializeCb cb) PURE; }; -typedef std::unique_ptr SlotPtr; +using SlotPtr = std::unique_ptr; /** * Interface used to allocate thread local slots. */ class SlotAllocator { public: - virtual ~SlotAllocator() {} + virtual ~SlotAllocator() = default; /** * @return SlotPtr a dedicated slot for use in further calls to get(), set(), etc. diff --git a/include/envoy/tracing/http_tracer.h b/include/envoy/tracing/http_tracer.h index efb81375dc0e..eb6f4960b62a 100644 --- a/include/envoy/tracing/http_tracer.h +++ b/include/envoy/tracing/http_tracer.h @@ -42,7 +42,7 @@ struct Decision { */ class Config { public: - virtual ~Config() {} + virtual ~Config() = default; /** * @return operation name for tracing, e.g., ingress. @@ -61,14 +61,14 @@ class Config { }; class Span; -typedef std::unique_ptr SpanPtr; +using SpanPtr = std::unique_ptr; /** * Basic abstraction for span. */ class Span { public: - virtual ~Span() {} + virtual ~Span() = default; /** * Set the operation name. @@ -126,7 +126,7 @@ class Span { */ class Driver { public: - virtual ~Driver() {} + virtual ~Driver() = default; /** * Start driver specific span. @@ -136,7 +136,7 @@ class Driver { const Tracing::Decision tracing_decision) PURE; }; -typedef std::unique_ptr DriverPtr; +using DriverPtr = std::unique_ptr; /** * HttpTracer is responsible for handling traces and delegate actions to the @@ -144,14 +144,14 @@ typedef std::unique_ptr DriverPtr; */ class HttpTracer { public: - virtual ~HttpTracer() {} + virtual ~HttpTracer() = default; virtual SpanPtr startSpan(const Config& config, Http::HeaderMap& request_headers, const StreamInfo::StreamInfo& stream_info, const Tracing::Decision tracing_decision) PURE; }; -typedef std::unique_ptr HttpTracerPtr; +using HttpTracerPtr = std::unique_ptr; } // namespace Tracing } // namespace Envoy diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 478f11c5cdb6..59f1716d4150 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -38,7 +38,7 @@ namespace Upstream { */ class ClusterUpdateCallbacks { public: - virtual ~ClusterUpdateCallbacks() {} + virtual ~ClusterUpdateCallbacks() = default; /** * onClusterAddOrUpdate is called when a new cluster is added or an existing cluster @@ -61,10 +61,10 @@ class ClusterUpdateCallbacks { */ class ClusterUpdateCallbacksHandle { public: - virtual ~ClusterUpdateCallbacksHandle() {} + virtual ~ClusterUpdateCallbacksHandle() = default; }; -typedef std::unique_ptr ClusterUpdateCallbacksHandlePtr; +using ClusterUpdateCallbacksHandlePtr = std::unique_ptr; class ClusterManagerFactory; @@ -74,7 +74,7 @@ class ClusterManagerFactory; */ class ClusterManager { public: - virtual ~ClusterManager() {} + virtual ~ClusterManager() = default; /** * Warming state a cluster is currently in. Used as an argument for the ClusterWarmingCallback. @@ -92,8 +92,8 @@ class ClusterManager { * @param cluster_name name of the cluster. * @param warming_state state the cluster transitioned to. */ - typedef std::function - ClusterWarmingCallback; + using ClusterWarmingCallback = + std::function; /** * Add or update a cluster via API. The semantics of this API are: @@ -114,7 +114,7 @@ class ClusterManager { */ virtual void setInitializedCb(std::function callback) PURE; - typedef std::unordered_map> ClusterInfoMap; + using ClusterInfoMap = std::unordered_map>; /** * @return ClusterInfoMap all current clusters. These are the primary (not thread local) @@ -247,14 +247,14 @@ class ClusterManager { virtual std::size_t warmingClusterCount() const PURE; }; -typedef std::unique_ptr ClusterManagerPtr; +using ClusterManagerPtr = std::unique_ptr; /** * Abstract interface for a CDS API provider. */ class CdsApi { public: - virtual ~CdsApi() {} + virtual ~CdsApi() = default; /** * Start the first fetch of CDS data. @@ -273,14 +273,14 @@ class CdsApi { virtual const std::string versionInfo() const PURE; }; -typedef std::unique_ptr CdsApiPtr; +using CdsApiPtr = std::unique_ptr; /** * Factory for objects needed during cluster manager operation. */ class ClusterManagerFactory { public: - virtual ~ClusterManagerFactory() {} + virtual ~ClusterManagerFactory() = default; /** * Allocate a cluster manager from configuration proto. @@ -331,7 +331,7 @@ class ClusterManagerFactory { */ class ClusterInfoFactory { public: - virtual ~ClusterInfoFactory() {} + virtual ~ClusterInfoFactory() = default; /** * Parameters for createClusterInfo(). diff --git a/include/envoy/upstream/health_check_host_monitor.h b/include/envoy/upstream/health_check_host_monitor.h index 135876055096..b32dcfa23d65 100644 --- a/include/envoy/upstream/health_check_host_monitor.h +++ b/include/envoy/upstream/health_check_host_monitor.h @@ -15,7 +15,7 @@ namespace Upstream { */ class HealthCheckHostMonitor { public: - virtual ~HealthCheckHostMonitor() {} + virtual ~HealthCheckHostMonitor() = default; /** * Mark the host as unhealthy. Note that this may not be immediate as events may need to be @@ -24,7 +24,7 @@ class HealthCheckHostMonitor { virtual void setUnhealthy() PURE; }; -typedef std::unique_ptr HealthCheckHostMonitorPtr; +using HealthCheckHostMonitorPtr = std::unique_ptr; } // namespace Upstream } // namespace Envoy diff --git a/include/envoy/upstream/health_checker.h b/include/envoy/upstream/health_checker.h index 9d8041235366..b9f7e9cbb837 100644 --- a/include/envoy/upstream/health_checker.h +++ b/include/envoy/upstream/health_checker.h @@ -32,7 +32,7 @@ enum class HealthTransition { */ class HealthChecker { public: - virtual ~HealthChecker() {} + virtual ~HealthChecker() = default; /** * Called when a host has been health checked. @@ -40,8 +40,8 @@ class HealthChecker { * @param changed_state supplies whether the health check resulted in a host moving from healthy * to not healthy or vice versa. */ - typedef std::function - HostStatusCb; + using HostStatusCb = + std::function; /** * Install a callback that will be invoked every time a health check round is completed for @@ -56,7 +56,7 @@ class HealthChecker { virtual void start() PURE; }; -typedef std::shared_ptr HealthCheckerSharedPtr; +using HealthCheckerSharedPtr = std::shared_ptr; std::ostream& operator<<(std::ostream& out, HealthState state); std::ostream& operator<<(std::ostream& out, HealthTransition changed_state); @@ -66,7 +66,7 @@ std::ostream& operator<<(std::ostream& out, HealthTransition changed_state); */ class HealthCheckEventLogger { public: - virtual ~HealthCheckEventLogger() {} + virtual ~HealthCheckEventLogger() = default; /** * Log an unhealthy host ejection event. @@ -118,7 +118,7 @@ class HealthCheckEventLogger { const HostDescriptionConstSharedPtr& host) PURE; }; -typedef std::unique_ptr HealthCheckEventLoggerPtr; +using HealthCheckEventLoggerPtr = std::unique_ptr; } // namespace Upstream } // namespace Envoy diff --git a/include/envoy/upstream/host_description.h b/include/envoy/upstream/host_description.h index 7a7ef277b41c..02c9f97888c8 100644 --- a/include/envoy/upstream/host_description.h +++ b/include/envoy/upstream/host_description.h @@ -43,7 +43,7 @@ class ClusterInfo; */ class HostDescription { public: - virtual ~HostDescription() {} + virtual ~HostDescription() = default; /** * @return whether the host is a canary. @@ -123,7 +123,7 @@ class HostDescription { virtual void priority(uint32_t) PURE; }; -typedef std::shared_ptr HostDescriptionConstSharedPtr; +using HostDescriptionConstSharedPtr = std::shared_ptr; } // namespace Upstream } // namespace Envoy diff --git a/include/envoy/upstream/load_balancer.h b/include/envoy/upstream/load_balancer.h index a62d2ac3e309..c631997b4323 100644 --- a/include/envoy/upstream/load_balancer.h +++ b/include/envoy/upstream/load_balancer.h @@ -17,7 +17,7 @@ namespace Upstream { */ class LoadBalancerContext { public: - virtual ~LoadBalancerContext() {} + virtual ~LoadBalancerContext() = default; /** * Compute and return an optional hash key to use during load balancing. This @@ -83,7 +83,7 @@ class LoadBalancerContext { */ class LoadBalancer { public: - virtual ~LoadBalancer() {} + virtual ~LoadBalancer() = default; /** * Ask the load balancer for the next host to use depending on the underlying LB algorithm. @@ -94,14 +94,14 @@ class LoadBalancer { virtual HostConstSharedPtr chooseHost(LoadBalancerContext* context) PURE; }; -typedef std::unique_ptr LoadBalancerPtr; +using LoadBalancerPtr = std::unique_ptr; /** * Factory for load balancers. */ class LoadBalancerFactory { public: - virtual ~LoadBalancerFactory() {} + virtual ~LoadBalancerFactory() = default; /** * @return LoadBalancerPtr a new load balancer. @@ -109,7 +109,7 @@ class LoadBalancerFactory { virtual LoadBalancerPtr create() PURE; }; -typedef std::shared_ptr LoadBalancerFactorySharedPtr; +using LoadBalancerFactorySharedPtr = std::shared_ptr; /** * A thread aware load balancer is a load balancer that is global to all workers on behalf of a @@ -139,7 +139,7 @@ typedef std::shared_ptr LoadBalancerFactorySharedPtr; */ class ThreadAwareLoadBalancer { public: - virtual ~ThreadAwareLoadBalancer() {} + virtual ~ThreadAwareLoadBalancer() = default; /** * @return LoadBalancerFactorySharedPtr the shared factory to use for creating new worker local @@ -156,7 +156,7 @@ class ThreadAwareLoadBalancer { virtual void initialize() PURE; }; -typedef std::unique_ptr ThreadAwareLoadBalancerPtr; +using ThreadAwareLoadBalancerPtr = std::unique_ptr; } // namespace Upstream } // namespace Envoy diff --git a/include/envoy/upstream/load_balancer_type.h b/include/envoy/upstream/load_balancer_type.h index 23c648918edf..5c508aa4edc3 100644 --- a/include/envoy/upstream/load_balancer_type.h +++ b/include/envoy/upstream/load_balancer_type.h @@ -30,7 +30,7 @@ enum class LoadBalancerType { */ class LoadBalancerSubsetInfo { public: - virtual ~LoadBalancerSubsetInfo() {} + virtual ~LoadBalancerSubsetInfo() = default; /** * @return bool true if load balancer subsets are configured. diff --git a/include/envoy/upstream/outlier_detection.h b/include/envoy/upstream/outlier_detection.h index d5ee2fbcd938..d8ffa2badc0a 100644 --- a/include/envoy/upstream/outlier_detection.h +++ b/include/envoy/upstream/outlier_detection.h @@ -15,10 +15,10 @@ namespace Envoy { namespace Upstream { class Host; -typedef std::shared_ptr HostSharedPtr; +using HostSharedPtr = std::shared_ptr; class HostDescription; -typedef std::shared_ptr HostDescriptionConstSharedPtr; +using HostDescriptionConstSharedPtr = std::shared_ptr; namespace Outlier { @@ -42,7 +42,7 @@ enum class Result { */ class DetectorHostMonitor { public: - virtual ~DetectorHostMonitor() {} + virtual ~DetectorHostMonitor() = default; /** * @return the number of times this host has been ejected. @@ -85,7 +85,7 @@ class DetectorHostMonitor { virtual double successRate() const PURE; }; -typedef std::unique_ptr DetectorHostMonitorPtr; +using DetectorHostMonitorPtr = std::unique_ptr; /** * Interface for an outlier detection engine. Uses per host data to determine which hosts in a @@ -93,12 +93,12 @@ typedef std::unique_ptr DetectorHostMonitorPtr; */ class Detector { public: - virtual ~Detector() {} + virtual ~Detector() = default; /** * Outlier detection change state callback. */ - typedef std::function ChangeStateCb; + using ChangeStateCb = std::function; /** * Add a changed state callback to the detector. The callback will be called whenever any host @@ -123,7 +123,7 @@ class Detector { virtual double successRateEjectionThreshold() const PURE; }; -typedef std::shared_ptr DetectorSharedPtr; +using DetectorSharedPtr = std::shared_ptr; enum class EjectionType { Consecutive5xx, SuccessRate, ConsecutiveGatewayFailure }; @@ -132,7 +132,7 @@ enum class EjectionType { Consecutive5xx, SuccessRate, ConsecutiveGatewayFailure */ class EventLogger { public: - virtual ~EventLogger() {} + virtual ~EventLogger() = default; /** * Log an ejection event. @@ -152,7 +152,7 @@ class EventLogger { virtual void logUneject(const HostDescriptionConstSharedPtr& host) PURE; }; -typedef std::shared_ptr EventLoggerSharedPtr; +using EventLoggerSharedPtr = std::shared_ptr; } // namespace Outlier } // namespace Upstream diff --git a/include/envoy/upstream/resource_manager.h b/include/envoy/upstream/resource_manager.h index 902ce75955f6..e439a3af7294 100644 --- a/include/envoy/upstream/resource_manager.h +++ b/include/envoy/upstream/resource_manager.h @@ -20,7 +20,7 @@ const size_t NumResourcePriorities = 2; */ class Resource { public: - virtual ~Resource() {} + virtual ~Resource() = default; /** * @return true if the resource can be created. @@ -55,7 +55,7 @@ class Resource { */ class ResourceManager { public: - virtual ~ResourceManager() {} + virtual ~ResourceManager() = default; /** * @return Resource& active TCP connections. diff --git a/include/envoy/upstream/retry.h b/include/envoy/upstream/retry.h index 7555901bf68c..dd518eb13ea5 100644 --- a/include/envoy/upstream/retry.h +++ b/include/envoy/upstream/retry.h @@ -15,7 +15,7 @@ namespace Upstream { */ class RetryPriority { public: - virtual ~RetryPriority() {} + virtual ~RetryPriority() = default; /** * Determines what PriorityLoad to use. @@ -39,7 +39,7 @@ class RetryPriority { virtual void onHostAttempted(HostDescriptionConstSharedPtr attempted_host) PURE; }; -typedef std::shared_ptr RetryPrioritySharedPtr; +using RetryPrioritySharedPtr = std::shared_ptr; /** * Used to decide whether a selected host should be rejected during retries. Host selection will be @@ -51,7 +51,7 @@ typedef std::shared_ptr RetryPrioritySharedPtr; */ class RetryHostPredicate { public: - virtual ~RetryHostPredicate() {} + virtual ~RetryHostPredicate() = default; /** * Determines whether a host should be rejected during host selection. @@ -70,14 +70,14 @@ class RetryHostPredicate { virtual void onHostAttempted(HostDescriptionConstSharedPtr attempted_host) PURE; }; -typedef std::shared_ptr RetryHostPredicateSharedPtr; +using RetryHostPredicateSharedPtr = std::shared_ptr; /** * Factory for RetryPriority. */ class RetryPriorityFactory { public: - virtual ~RetryPriorityFactory() {} + virtual ~RetryPriorityFactory() = default; virtual RetryPrioritySharedPtr createRetryPriority(const Protobuf::Message& config, uint32_t retry_count) PURE; @@ -92,7 +92,7 @@ class RetryPriorityFactory { */ class RetryHostPredicateFactory { public: - virtual ~RetryHostPredicateFactory() {} + virtual ~RetryHostPredicateFactory() = default; virtual RetryHostPredicateSharedPtr createHostPredicate(const Protobuf::Message& config, uint32_t retry_count) PURE; diff --git a/include/envoy/upstream/thread_local_cluster.h b/include/envoy/upstream/thread_local_cluster.h index 247f8ed462af..2a9dafb9af6a 100644 --- a/include/envoy/upstream/thread_local_cluster.h +++ b/include/envoy/upstream/thread_local_cluster.h @@ -15,7 +15,7 @@ namespace Upstream { */ class ThreadLocalCluster { public: - virtual ~ThreadLocalCluster() {} + virtual ~ThreadLocalCluster() = default; /** * @return const PrioritySet& the backing priority set. diff --git a/include/envoy/upstream/types.h b/include/envoy/upstream/types.h index 202a4d136a17..322392aadff3 100644 --- a/include/envoy/upstream/types.h +++ b/include/envoy/upstream/types.h @@ -17,7 +17,7 @@ struct Load {}; // and 20% to P2. // // This should either sum to 100 or consist of all zeros. -typedef Phantom, Load> PriorityLoad; +using PriorityLoad = Phantom, Load>; // PriorityLoad specific to degraded hosts. struct DegradedLoad : PriorityLoad { @@ -39,7 +39,7 @@ struct Availability {}; // Mapping from a priority how available the given priority is, eg. the ratio of healthy host to // total hosts. -typedef Phantom, Availability> PriorityAvailability; +using PriorityAvailability = Phantom, Availability>; // Availability specific to degraded hosts. struct DegradedAvailability : PriorityAvailability { diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 612538c0063d..567466c18ebc 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -194,31 +194,31 @@ class Host : virtual public HostDescription { virtual void used(bool new_used) PURE; }; -typedef std::shared_ptr HostConstSharedPtr; +using HostConstSharedPtr = std::shared_ptr; -typedef std::vector HostVector; -typedef Phantom HealthyHostVector; -typedef Phantom DegradedHostVector; -typedef Phantom ExcludedHostVector; -typedef std::unordered_map HostMap; -typedef std::shared_ptr HostVectorSharedPtr; -typedef std::shared_ptr HostVectorConstSharedPtr; +using HostVector = std::vector; +using HealthyHostVector = Phantom; +using DegradedHostVector = Phantom; +using ExcludedHostVector = Phantom; +using HostMap = std::unordered_map; +using HostVectorSharedPtr = std::shared_ptr; +using HostVectorConstSharedPtr = std::shared_ptr; -typedef std::shared_ptr HealthyHostVectorConstSharedPtr; -typedef std::shared_ptr DegradedHostVectorConstSharedPtr; -typedef std::shared_ptr ExcludedHostVectorConstSharedPtr; +using HealthyHostVectorConstSharedPtr = std::shared_ptr; +using DegradedHostVectorConstSharedPtr = std::shared_ptr; +using ExcludedHostVectorConstSharedPtr = std::shared_ptr; -typedef std::unique_ptr HostListPtr; -typedef std::unordered_map - LocalityWeightsMap; -typedef std::vector> PriorityState; +using HostListPtr = std::unique_ptr; +using LocalityWeightsMap = + std::unordered_map; +using PriorityState = std::vector>; /** * Bucket hosts by locality. */ class HostsPerLocality { public: - virtual ~HostsPerLocality() {} + virtual ~HostsPerLocality() = default; /** * @return bool is local locality one of the locality buckets? If so, the @@ -252,13 +252,13 @@ class HostsPerLocality { } }; -typedef std::shared_ptr HostsPerLocalitySharedPtr; -typedef std::shared_ptr HostsPerLocalityConstSharedPtr; +using HostsPerLocalitySharedPtr = std::shared_ptr; +using HostsPerLocalityConstSharedPtr = std::shared_ptr; // Weight for each locality index in HostsPerLocality. -typedef std::vector LocalityWeights; -typedef std::shared_ptr LocalityWeightsSharedPtr; -typedef std::shared_ptr LocalityWeightsConstSharedPtr; +using LocalityWeights = std::vector; +using LocalityWeightsSharedPtr = std::shared_ptr; +using LocalityWeightsConstSharedPtr = std::shared_ptr; /** * Base host set interface. This contains all of the endpoints for a given LocalityLbEndpoints @@ -267,7 +267,7 @@ typedef std::shared_ptr LocalityWeightsConstSharedPtr; // TODO(snowp): Remove the const ref accessors in favor of the shared_ptr ones. class HostSet { public: - virtual ~HostSet() {} + virtual ~HostSet() = default; /** * @return all hosts that make up the set at the current time. @@ -384,7 +384,7 @@ class HostSet { virtual uint32_t overprovisioningFactor() const PURE; }; -typedef std::unique_ptr HostSetPtr; +using HostSetPtr = std::unique_ptr; /** * This class contains all of the HostSets for a given cluster grouped by priority, for @@ -392,14 +392,13 @@ typedef std::unique_ptr HostSetPtr; */ class PrioritySet { public: - typedef std::function - MemberUpdateCb; + using MemberUpdateCb = + std::function; - typedef std::function - PriorityUpdateCb; + using PriorityUpdateCb = std::function; - virtual ~PrioritySet() {} + virtual ~PrioritySet() = default; /** * Install a callback that will be invoked when any of the HostSets in the PrioritySet changes. @@ -461,7 +460,7 @@ class PrioritySet { */ class HostUpdateCb { public: - virtual ~HostUpdateCb() {} + virtual ~HostUpdateCb() = default; /** * Updates the hosts in a given host set. * @@ -483,7 +482,7 @@ class PrioritySet { */ class BatchUpdateCb { public: - virtual ~BatchUpdateCb() {} + virtual ~BatchUpdateCb() = default; /** * Performs a batch host update. Implementors should use the provided callback to update hosts @@ -638,9 +637,9 @@ struct ClusterCircuitBreakersStats { */ class ProtocolOptionsConfig { public: - virtual ~ProtocolOptionsConfig() {} + virtual ~ProtocolOptionsConfig() = default; }; -typedef std::shared_ptr ProtocolOptionsConfigConstSharedPtr; +using ProtocolOptionsConfigConstSharedPtr = std::shared_ptr; /** * Base class for all cluster typed metadata factory. @@ -662,7 +661,7 @@ class ClusterInfo { static const uint64_t CLOSE_CONNECTIONS_ON_HOST_HEALTH_FAILURE = 0x4; }; - virtual ~ClusterInfo() {} + virtual ~ClusterInfo() = default; /** * @return bool whether the cluster was added via API (if false the cluster was present in the @@ -856,7 +855,7 @@ class ClusterInfo { extensionProtocolOptions(const std::string& name) const PURE; }; -typedef std::shared_ptr ClusterInfoConstSharedPtr; +using ClusterInfoConstSharedPtr = std::shared_ptr; class HealthChecker; @@ -866,7 +865,7 @@ class HealthChecker; */ class Cluster { public: - virtual ~Cluster() {} + virtual ~Cluster() = default; enum class InitializePhase { Primary, Secondary }; @@ -915,7 +914,7 @@ class Cluster { virtual const PrioritySet& prioritySet() const PURE; }; -typedef std::shared_ptr ClusterSharedPtr; +using ClusterSharedPtr = std::shared_ptr; } // namespace Upstream } // namespace Envoy diff --git a/source/common/api/os_sys_calls_impl.h b/source/common/api/os_sys_calls_impl.h index 0246f5b3aedb..1e2706778c72 100644 --- a/source/common/api/os_sys_calls_impl.h +++ b/source/common/api/os_sys_calls_impl.h @@ -33,7 +33,7 @@ class OsSysCallsImpl : public OsSysCalls { SysCallIntResult getsockname(int sockfd, sockaddr* addr, socklen_t* addrlen) override; }; -typedef ThreadSafeSingleton OsSysCallsSingleton; +using OsSysCallsSingleton = ThreadSafeSingleton; } // namespace Api } // namespace Envoy diff --git a/source/common/api/os_sys_calls_impl_hot_restart.h b/source/common/api/os_sys_calls_impl_hot_restart.h index bfce5dd9bc18..d46833d17700 100644 --- a/source/common/api/os_sys_calls_impl_hot_restart.h +++ b/source/common/api/os_sys_calls_impl_hot_restart.h @@ -14,7 +14,7 @@ class HotRestartOsSysCallsImpl : public HotRestartOsSysCalls { SysCallIntResult shmUnlink(const char* name) override; }; -typedef ThreadSafeSingleton HotRestartOsSysCallsSingleton; +using HotRestartOsSysCallsSingleton = ThreadSafeSingleton; } // namespace Api } // namespace Envoy \ No newline at end of file diff --git a/source/common/api/os_sys_calls_impl_linux.h b/source/common/api/os_sys_calls_impl_linux.h index d3b08fe427d9..7a250943c7ac 100644 --- a/source/common/api/os_sys_calls_impl_linux.h +++ b/source/common/api/os_sys_calls_impl_linux.h @@ -17,7 +17,7 @@ class LinuxOsSysCallsImpl : public LinuxOsSysCalls { SysCallIntResult sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t* mask) override; }; -typedef ThreadSafeSingleton LinuxOsSysCallsSingleton; +using LinuxOsSysCallsSingleton = ThreadSafeSingleton; } // namespace Api } // namespace Envoy diff --git a/source/common/buffer/watermark_buffer.h b/source/common/buffer/watermark_buffer.h index 9dd12d3d8aff..827d1a51bccf 100644 --- a/source/common/buffer/watermark_buffer.h +++ b/source/common/buffer/watermark_buffer.h @@ -56,7 +56,7 @@ class WatermarkBuffer : public OwnedImpl { bool above_high_watermark_called_{false}; }; -typedef std::unique_ptr WatermarkBufferPtr; +using WatermarkBufferPtr = std::unique_ptr; class WatermarkBufferFactory : public WatermarkFactory { public: diff --git a/source/common/chromium_url/url_canon.h b/source/common/chromium_url/url_canon.h index 89a11bb0418b..0280de643ac8 100644 --- a/source/common/chromium_url/url_canon.h +++ b/source/common/chromium_url/url_canon.h @@ -29,7 +29,7 @@ namespace chromium_url { template class CanonOutputT { public: CanonOutputT() : buffer_(NULL), buffer_len_(0), cur_len_(0) {} - virtual ~CanonOutputT() {} + virtual ~CanonOutputT() = default; // Implemented to resize the buffer. This function should update the buffer // pointer to point to the new buffer, and any old data up to |cur_len_| in @@ -163,7 +163,7 @@ extern template class EXPORT_TEMPLATE_DECLARE(COMPONENT_EXPORT(URL)) CanonOutput // Normally, all canonicalization output is in narrow characters. We support // the templates so it can also be used internally if a wide buffer is // required. -typedef CanonOutputT CanonOutput; +using CanonOutput = CanonOutputT; template class RawCanonOutput : public RawCanonOutputT {}; diff --git a/source/common/common/assert.h b/source/common/common/assert.h index d99ce8f754f8..3bd13367ac5e 100644 --- a/source/common/common/assert.h +++ b/source/common/common/assert.h @@ -9,9 +9,9 @@ namespace Assert { class ActionRegistration { public: - virtual ~ActionRegistration() {} + virtual ~ActionRegistration() = default; }; -typedef std::unique_ptr ActionRegistrationPtr; +using ActionRegistrationPtr = std::unique_ptr; /** * Sets an action to be invoked when a debug assertion failure is detected diff --git a/source/common/common/callback_impl.h b/source/common/common/callback_impl.h index 6e90b52860c5..3e9b554df7e2 100644 --- a/source/common/common/callback_impl.h +++ b/source/common/common/callback_impl.h @@ -15,7 +15,7 @@ namespace Common { */ template class CallbackManager { public: - typedef std::function Callback; + using Callback = std::function; /** * Add a callback. diff --git a/source/common/common/linked_object.h b/source/common/common/linked_object.h index b2ed96bbdbe5..ee0ab8275989 100644 --- a/source/common/common/linked_object.h +++ b/source/common/common/linked_object.h @@ -12,7 +12,7 @@ namespace Envoy { */ template class LinkedObject { public: - typedef std::list> ListType; + using ListType = std::list>; /** * @return the list iterator for the object. diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 019a714ef2d8..735aafa18f98 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -100,7 +100,7 @@ class Logger { }; class DelegatingLogSink; -typedef std::shared_ptr DelegatingLogSinkPtr; +using DelegatingLogSinkPtr = std::shared_ptr; /** * Captures a logging sink that can be delegated to for a bounded amount of time. diff --git a/source/common/common/logger_delegates.h b/source/common/common/logger_delegates.h index 67264d29c3a3..fb189b2973e3 100644 --- a/source/common/common/logger_delegates.h +++ b/source/common/common/logger_delegates.h @@ -15,7 +15,7 @@ namespace Envoy { namespace Logger { class DelegatingLogSink; -typedef std::shared_ptr DelegatingLogSinkPtr; +using DelegatingLogSinkPtr = std::shared_ptr; /** * SinkDelegate that writes log messages to a file. diff --git a/source/common/common/matchers.h b/source/common/common/matchers.h index 24902c5e329d..ca021f56b461 100644 --- a/source/common/common/matchers.h +++ b/source/common/common/matchers.h @@ -15,11 +15,11 @@ namespace Envoy { namespace Matchers { class ValueMatcher; -typedef std::shared_ptr ValueMatcherConstSharedPtr; +using ValueMatcherConstSharedPtr = std::shared_ptr; class ValueMatcher { public: - virtual ~ValueMatcher() {} + virtual ~ValueMatcher() = default; /** * Check whether the value is matched to the matcher. diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index 190cfc40b598..ed6483795814 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -32,7 +32,7 @@ class SpecifierConstantValues { const std::regex PATTERN{"(%([1-9])?f)|(%s)", std::regex::optimize}; }; -typedef ConstSingleton SpecifierConstants; +using SpecifierConstants = ConstSingleton; } // namespace diff --git a/source/common/common/utility.h b/source/common/common/utility.h index 5f9a99a5e109..3697c9fb0542 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -47,7 +47,7 @@ class DateFormatter { private: void parse(const std::string& format_string); - typedef std::vector SpecifierOffsets; + using SpecifierOffsets = std::vector; std::string fromTimeAndPrepareSpecifierOffsets(time_t time, SpecifierOffsets& specifier_offsets, const std::string& seconds_str) const; @@ -359,8 +359,8 @@ class StringUtil { /** * Definition of unordered set of case-insensitive std::string. */ - typedef absl::flat_hash_set - CaseUnorderedSet; + using CaseUnorderedSet = + absl::flat_hash_set; /** * Removes all the character indices from str contained in the interval-set. @@ -458,7 +458,7 @@ class WeightedClusterUtil { template class IntervalSetImpl : public IntervalSet { public: // Interval is a pair of Values. - typedef typename IntervalSet::Interval Interval; + using Interval = typename IntervalSet::Interval; void insert(Value left, Value right) override { if (left == right) { diff --git a/source/common/config/resources.h b/source/common/config/resources.h index cb1a343bea81..d4baae1622f1 100644 --- a/source/common/config/resources.h +++ b/source/common/config/resources.h @@ -22,7 +22,7 @@ class TypeUrlValues { "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration"}; }; -typedef ConstSingleton TypeUrl; +using TypeUrl = ConstSingleton; } // namespace Config } // namespace Envoy diff --git a/source/common/config/utility.h b/source/common/config/utility.h index 8181e2022773..140f7f41e896 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -50,7 +50,7 @@ struct RateLimitSettings { bool enabled_{false}; }; -typedef ConstSingleton ApiType; +using ApiType = ConstSingleton; /** * General config API utilities. diff --git a/source/common/config/well_known_names.h b/source/common/config/well_known_names.h index 748400b84dd6..255b86927ae1 100644 --- a/source/common/config/well_known_names.h +++ b/source/common/config/well_known_names.h @@ -52,7 +52,7 @@ class AddressResolverNameValues { const std::string IP = "envoy.ip"; }; -typedef ConstSingleton AddressResolverNames; +using AddressResolverNames = ConstSingleton; /** * Well-known metadata filter namespaces. @@ -63,7 +63,7 @@ class MetadataFilterValues { const std::string ENVOY_LB = "envoy.lb"; }; -typedef ConstSingleton MetadataFilters; +using MetadataFilters = ConstSingleton; /** * Keys for MetadataFilterValues::ENVOY_LB metadata. @@ -74,7 +74,7 @@ class MetadataEnvoyLbKeyValues { const std::string CANARY = "canary"; }; -typedef ConstSingleton MetadataEnvoyLbKeys; +using MetadataEnvoyLbKeys = ConstSingleton; /** * Well known tags values and a mapping from these names to the regexes they @@ -163,7 +163,7 @@ class TagNameValues { std::vector descriptor_vec_; }; -typedef ConstSingleton TagNames; +using TagNames = ConstSingleton; } // namespace Config } // namespace Envoy diff --git a/source/common/crypto/utility.h b/source/common/crypto/utility.h index 9649f74b4519..ef1a1c68fcd5 100644 --- a/source/common/crypto/utility.h +++ b/source/common/crypto/utility.h @@ -25,7 +25,7 @@ struct VerificationOutput { std::string error_message_; }; -typedef bssl::UniquePtr PublicKeyPtr; +using PublicKeyPtr = bssl::UniquePtr; class Utility { public: diff --git a/source/common/event/libevent.h b/source/common/event/libevent.h index a07a8816e9e2..20fdfefcbcd6 100644 --- a/source/common/event/libevent.h +++ b/source/common/event/libevent.h @@ -43,10 +43,10 @@ class Global { static bool initialized_; }; -typedef CSmartPtr BasePtr; -typedef CSmartPtr BufferPtr; -typedef CSmartPtr BufferEventPtr; -typedef CSmartPtr ListenerPtr; +using BasePtr = CSmartPtr; +using BufferPtr = CSmartPtr; +using BufferEventPtr = CSmartPtr; +using ListenerPtr = CSmartPtr; } // namespace Libevent } // namespace Event diff --git a/source/common/filesystem/file_shared_impl.h b/source/common/filesystem/file_shared_impl.h index d018ed0314ae..42d4ddb0f939 100644 --- a/source/common/filesystem/file_shared_impl.h +++ b/source/common/filesystem/file_shared_impl.h @@ -37,9 +37,9 @@ template Api::IoCallResult resultSuccess(T result) { class FileSharedImpl : public File { public: - FileSharedImpl(const std::string& path) : fd_(-1), path_(path) {} + FileSharedImpl(std::string path) : fd_(-1), path_(std::move(path)) {} - virtual ~FileSharedImpl() {} + ~FileSharedImpl() override = default; // Filesystem::File Api::IoCallBoolResult open() override; @@ -58,4 +58,4 @@ class FileSharedImpl : public File { }; } // namespace Filesystem -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/common/filesystem/kqueue/watcher_impl.h b/source/common/filesystem/kqueue/watcher_impl.h index d3916a1ce288..7922d62ca391 100644 --- a/source/common/filesystem/kqueue/watcher_impl.h +++ b/source/common/filesystem/kqueue/watcher_impl.h @@ -37,7 +37,7 @@ class WatcherImpl : public Watcher, Logger::Loggable { bool watching_dir_; }; - typedef std::shared_ptr FileWatchPtr; + using FileWatchPtr = std::shared_ptr; void onKqueueEvent(); FileWatchPtr addWatch(const std::string& path, uint32_t events, Watcher::OnChangedCb cb, diff --git a/source/common/grpc/google_async_client_impl.h b/source/common/grpc/google_async_client_impl.h index 7feeb48076c6..b14456235b21 100644 --- a/source/common/grpc/google_async_client_impl.h +++ b/source/common/grpc/google_async_client_impl.h @@ -114,7 +114,7 @@ struct GoogleAsyncClientStats { // Interface to allow the gRPC stub to be mocked out by tests. class GoogleStub { public: - virtual ~GoogleStub() {} + virtual ~GoogleStub() = default; // See grpc::PrepareCall(). virtual std::unique_ptr @@ -139,7 +139,7 @@ class GoogleGenericStub : public GoogleStub { // Interface to allow the gRPC stub creation to be mocked out by tests. class GoogleStubFactory { public: - virtual ~GoogleStubFactory() {} + virtual ~GoogleStubFactory() = default; // Create a stub from a given channel. virtual std::shared_ptr createStub(std::shared_ptr channel) PURE; diff --git a/source/common/grpc/typed_async_client.h b/source/common/grpc/typed_async_client.h index 6f8531367afe..8b3df8efc3b8 100644 --- a/source/common/grpc/typed_async_client.h +++ b/source/common/grpc/typed_async_client.h @@ -54,7 +54,7 @@ template class AsyncStream /* : public RawAsyncStream */ { */ template class AsyncRequestCallbacks : public RawAsyncRequestCallbacks { public: - virtual ~AsyncRequestCallbacks() {} + ~AsyncRequestCallbacks() override = default; virtual void onSuccess(std::unique_ptr&& response, Tracing::Span& span) PURE; private: @@ -75,11 +75,11 @@ template class AsyncRequestCallbacks : public RawAsyncReques */ template class AsyncStreamCallbacks : public RawAsyncStreamCallbacks { public: - virtual ~AsyncStreamCallbacks() {} + ~AsyncStreamCallbacks() override = default; virtual void onReceiveMessage(std::unique_ptr&& message) PURE; private: - bool onReceiveMessageRaw(Buffer::InstancePtr&& response) { + bool onReceiveMessageRaw(Buffer::InstancePtr&& response) override { auto message = std::unique_ptr(dynamic_cast( Internal::parseMessageUntyped(std::make_unique(), std::move(response)) .release())); diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index caae81485fcc..4fa085d2e4fc 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -25,7 +25,7 @@ namespace Http { */ class CodecClientCallbacks { public: - virtual ~CodecClientCallbacks() {} + virtual ~CodecClientCallbacks() = default; /** * Called every time an owned stream is destroyed, whether complete or not. @@ -203,7 +203,7 @@ class CodecClient : Logger::Loggable, CodecClient& parent_; }; - typedef std::unique_ptr ActiveRequestPtr; + using ActiveRequestPtr = std::unique_ptr; /** * Called when a response finishes decoding. This is called *before* forwarding on to the @@ -233,7 +233,7 @@ class CodecClient : Logger::Loggable, bool remote_closed_{}; }; -typedef std::unique_ptr CodecClientPtr; +using CodecClientPtr = std::unique_ptr; /** * Production implementation that installs a real codec. diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 251d4bd53902..5e08e267d5b9 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -109,7 +109,7 @@ struct TracingConnectionManagerConfig { bool verbose_; }; -typedef std::unique_ptr TracingConnectionManagerConfigPtr; +using TracingConnectionManagerConfigPtr = std::unique_ptr; /** * Connection manager per listener stats. @see stats_macros.h @@ -151,7 +151,7 @@ enum class ClientCertDetailsType { Cert, Chain, Subject, URI, DNS }; */ class InternalAddressConfig { public: - virtual ~InternalAddressConfig() {} + virtual ~InternalAddressConfig() = default; virtual bool isInternalAddress(const Network::Address::Instance& address) const PURE; }; @@ -170,7 +170,7 @@ class DefaultInternalAddressConfig : public Http::InternalAddressConfig { */ class ConnectionManagerConfig { public: - virtual ~ConnectionManagerConfig() {} + virtual ~ConnectionManagerConfig() = default; /** * @return const std::list& the access logs to write to. diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index dc23f0810b8a..c6e54e889522 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -270,7 +270,7 @@ class ConnectionManagerImpl : Logger::Loggable, bool is_grpc_request_{}; }; - typedef std::unique_ptr ActiveStreamDecoderFilterPtr; + using ActiveStreamDecoderFilterPtr = std::unique_ptr; /** * Wrapper for a stream encoder filter. @@ -323,7 +323,7 @@ class ConnectionManagerImpl : Logger::Loggable, StreamEncoderFilterSharedPtr handle_; }; - typedef std::unique_ptr ActiveStreamEncoderFilterPtr; + using ActiveStreamEncoderFilterPtr = std::unique_ptr; /** * Wraps a single active stream on the connection. These are either full request/response pairs @@ -536,7 +536,7 @@ class ConnectionManagerImpl : Logger::Loggable, Network::Socket::OptionsSharedPtr upstream_options_; }; - typedef std::unique_ptr ActiveStreamPtr; + using ActiveStreamPtr = std::unique_ptr; /** * Check to see if the connection can be closed after gracefully waiting to send pending codec diff --git a/source/common/http/conn_pool_base.h b/source/common/http/conn_pool_base.h index 2c0884228194..7317acde39ec 100644 --- a/source/common/http/conn_pool_base.h +++ b/source/common/http/conn_pool_base.h @@ -29,7 +29,7 @@ class ConnPoolImplBase : protected Logger::Loggable { ConnectionPool::Callbacks& callbacks_; }; - typedef std::unique_ptr PendingRequestPtr; + using PendingRequestPtr = std::unique_ptr; // Creates a new PendingRequest and enqueues it into the request queue. ConnectionPool::Cancellable* newPendingRequest(StreamDecoder& decoder, diff --git a/source/common/http/date_provider.h b/source/common/http/date_provider.h index 8ad7770a361b..d4e0a4b7118c 100644 --- a/source/common/http/date_provider.h +++ b/source/common/http/date_provider.h @@ -11,7 +11,7 @@ namespace Http { */ class DateProvider { public: - virtual ~DateProvider() {} + virtual ~DateProvider() = default; /** * Set the Date header potentially using a cached value. diff --git a/source/common/http/header_map_impl.h b/source/common/http/header_map_impl.h index ffa2e069f33d..7dcb665922a8 100644 --- a/source/common/http/header_map_impl.h +++ b/source/common/http/header_map_impl.h @@ -201,7 +201,7 @@ class HeaderMapImpl : public HeaderMap, NonCopyable { ALL_INLINE_HEADERS(DEFINE_INLINE_HEADER_FUNCS) }; -typedef std::unique_ptr HeaderMapImplPtr; +using HeaderMapImplPtr = std::unique_ptr; } // namespace Http } // namespace Envoy diff --git a/source/common/http/headers.h b/source/common/http/headers.h index 3a74c1eb277e..2b9247747e56 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -243,7 +243,7 @@ class HeaderValues { } AccessControlAllowOriginValue; }; -typedef ConstSingleton Headers; +using Headers = ConstSingleton; } // namespace Http } // namespace Envoy diff --git a/source/common/http/http1/conn_pool.h b/source/common/http/http1/conn_pool.h index fba5b83268e1..40228d27515d 100644 --- a/source/common/http/http1/conn_pool.h +++ b/source/common/http/http1/conn_pool.h @@ -78,7 +78,7 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase { bool decode_complete_{}; }; - typedef std::unique_ptr StreamWrapperPtr; + using StreamWrapperPtr = std::unique_ptr; struct ActiveClient : LinkedObject, public Network::ConnectionCallbacks, @@ -104,7 +104,7 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase { uint64_t remaining_requests_; }; - typedef std::unique_ptr ActiveClientPtr; + using ActiveClientPtr = std::unique_ptr; void attachRequestToClient(ActiveClient& client, StreamDecoder& response_decoder, ConnectionPool::Callbacks& callbacks); diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 1277e217e891..9bf50ad9b9e0 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -227,7 +227,7 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable StreamImplPtr; + using StreamImplPtr = std::unique_ptr; /** * Client side stream (request). diff --git a/source/common/http/http2/conn_pool.h b/source/common/http/http2/conn_pool.h index 134f38750a6e..774f654816ac 100644 --- a/source/common/http/http2/conn_pool.h +++ b/source/common/http/http2/conn_pool.h @@ -74,7 +74,7 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase { bool closed_with_active_rq_{}; }; - typedef std::unique_ptr ActiveClientPtr; + using ActiveClientPtr = std::unique_ptr; // Http::ConnPoolImplBase void checkForDrained() override; diff --git a/source/common/http/http2/metadata_decoder.h b/source/common/http/http2/metadata_decoder.h index 2cac52559ced..23331b67700b 100644 --- a/source/common/http/http2/metadata_decoder.h +++ b/source/common/http/http2/metadata_decoder.h @@ -77,7 +77,7 @@ class MetadataDecoder : Logger::Loggable { // TODO(soya3129): consider sharing the inflater with all streams in a connection. Caveat: // inflater failure on one stream can impact other streams. - typedef CSmartPtr Inflater; + using Inflater = CSmartPtr; Inflater inflater_; }; diff --git a/source/common/http/http2/metadata_encoder.h b/source/common/http/http2/metadata_encoder.h index 7e2744420446..7a6e477506ff 100644 --- a/source/common/http/http2/metadata_encoder.h +++ b/source/common/http/http2/metadata_encoder.h @@ -82,7 +82,7 @@ class MetadataEncoder : Logger::Loggable { // TODO(soya3129): share deflater among all encoders in the same connection. The benefit is less // memory, and the caveat is encoding error on one stream can impact other streams. - typedef CSmartPtr Deflater; + using Deflater = CSmartPtr; Deflater deflater_; // Stores the remaining payload size of each metadata_map to be packed. The payload size is needed diff --git a/source/common/json/json_loader.cc b/source/common/json/json_loader.cc index b645512464f3..835f7b26f08d 100644 --- a/source/common/json/json_loader.cc +++ b/source/common/json/json_loader.cc @@ -34,7 +34,7 @@ namespace { * Internal representation of Object. */ class Field; -typedef std::shared_ptr FieldSharedPtr; +using FieldSharedPtr = std::shared_ptr; class Field : public Object { public: diff --git a/source/common/network/filter_manager_impl.h b/source/common/network/filter_manager_impl.h index 6db1332c6174..0975c2ecd7ed 100644 --- a/source/common/network/filter_manager_impl.h +++ b/source/common/network/filter_manager_impl.h @@ -21,7 +21,7 @@ struct StreamBuffer { */ class ReadBufferSource { public: - virtual ~ReadBufferSource() {} + virtual ~ReadBufferSource() = default; /** * Fetch the read buffer for the source. @@ -34,7 +34,7 @@ class ReadBufferSource { */ class WriteBufferSource { public: - virtual ~WriteBufferSource() {} + virtual ~WriteBufferSource() = default; /** * Fetch the write buffer for the source. @@ -81,7 +81,7 @@ class FilterManagerConnection : public virtual Connection, public ReadBufferSource, public WriteBufferSource { public: - virtual ~FilterManagerConnection() {} + ~FilterManagerConnection() override = default; /** * Write data to the connection bypassing filter chain. @@ -132,11 +132,11 @@ class FilterManagerImpl { bool initialized_{}; }; - typedef std::unique_ptr ActiveReadFilterPtr; + using ActiveReadFilterPtr = std::unique_ptr; struct ActiveWriteFilter : public WriteFilterCallbacks, LinkedObject { ActiveWriteFilter(FilterManagerImpl& parent, WriteFilterSharedPtr filter) - : parent_(parent), filter_(filter) {} + : parent_(parent), filter_(std::move(filter)) {} Connection& connection() override { return parent_.connection_; } void injectWriteDataToFilterChain(Buffer::Instance& data, bool end_stream) override { @@ -148,7 +148,7 @@ class FilterManagerImpl { WriteFilterSharedPtr filter_; }; - typedef std::unique_ptr ActiveWriteFilterPtr; + using ActiveWriteFilterPtr = std::unique_ptr; void onContinueReading(ActiveReadFilter* filter, ReadBufferSource& buffer_source); diff --git a/source/common/network/lc_trie.h b/source/common/network/lc_trie.h index 0eca74ea0f0f..059cb038d412 100644 --- a/source/common/network/lc_trie.h +++ b/source/common/network/lc_trie.h @@ -227,11 +227,11 @@ template class LcTrie { } // IP addresses are stored in host byte order to simplify - typedef uint32_t Ipv4; - typedef absl::uint128 Ipv6; + using Ipv4 = uint32_t; + using Ipv6 = absl::uint128; - typedef std::unordered_set DataSet; - typedef std::shared_ptr DataSetSharedPtr; + using DataSet = std::unordered_set; + using DataSetSharedPtr = std::shared_ptr; /** * Structure to hold a CIDR range and the data associated with it. @@ -392,7 +392,7 @@ template class LcTrie { std::unique_ptr children[2]; DataSetSharedPtr data; }; - typedef std::unique_ptr NodePtr; + using NodePtr = std::unique_ptr; NodePtr root_; bool exclusive_; }; diff --git a/source/common/network/listen_socket_impl.h b/source/common/network/listen_socket_impl.h index 9e7403159c3c..5ff5a35267ab 100644 --- a/source/common/network/listen_socket_impl.h +++ b/source/common/network/listen_socket_impl.h @@ -103,10 +103,10 @@ template class NetworkListenSocket : public ListenSocketImpl { }; using TcpListenSocket = NetworkListenSocket>; -typedef std::unique_ptr TcpListenSocketPtr; +using TcpListenSocketPtr = std::unique_ptr; using UdpListenSocket = NetworkListenSocket>; -typedef std::unique_ptr UdpListenSocketPtr; +using UdpListenSocketPtr = std::unique_ptr; class UdsListenSocket : public ListenSocketImpl { public: diff --git a/source/common/network/utility.h b/source/common/network/utility.h index 2eceaaf7f24a..24eb526b6a6e 100644 --- a/source/common/network/utility.h +++ b/source/common/network/utility.h @@ -26,7 +26,7 @@ class PortRange { const uint32_t max_; }; -typedef std::list PortRangeList; +using PortRangeList = std::list; /** * Common network utility routines. diff --git a/source/common/protobuf/protobuf.h b/source/common/protobuf/protobuf.h index 4f5e7de88c2a..ff6da52694f3 100644 --- a/source/common/protobuf/protobuf.h +++ b/source/common/protobuf/protobuf.h @@ -44,10 +44,10 @@ namespace ProtobufWkt = google::protobuf; // Below we provide wrappers to facilitate remapping of the type during import. namespace ProtobufTypes { -typedef std::unique_ptr MessagePtr; -typedef std::vector> ConstMessagePtrVector; +using MessagePtr = std::unique_ptr; +using ConstMessagePtrVector = std::vector>; -typedef int64_t Int64; +using Int64 = int64_t; } // namespace ProtobufTypes } // namespace Envoy diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 3df86caefa2d..fa60e8ede2f4 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -185,7 +185,7 @@ class MessageUtil { const std::string Yaml = ".yaml"; }; - typedef ConstSingleton FileExtensions; + using FileExtensions = ConstSingleton; static std::size_t hash(const Protobuf::Message& message) { // Use Protobuf::io::CodedOutputStream to force deterministic serialization, so that the same diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 5976e214a0d6..24fbf3d73644 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -36,7 +36,7 @@ namespace Router { */ class Matchable { public: - virtual ~Matchable() {} + virtual ~Matchable() = default; /** * See if this object matches the incoming headers. @@ -62,7 +62,7 @@ class PerFilterConfigs { }; class RouteEntryImplBase; -typedef std::shared_ptr RouteEntryImplBaseConstSharedPtr; +using RouteEntryImplBaseConstSharedPtr = std::shared_ptr; /** * Direct response entry that does an SSL redirect. @@ -218,7 +218,7 @@ class VirtualHostImpl : public VirtualHost { const CatchAllVirtualCluster virtual_cluster_catch_all_; }; -typedef std::shared_ptr VirtualHostSharedPtr; +using VirtualHostSharedPtr = std::shared_ptr; /** * Implementation of RetryPolicy that reads from the proto route or virtual host config. @@ -294,7 +294,7 @@ class HashPolicyImpl : public HashPolicy { class HashMethod { public: - virtual ~HashMethod() {} + virtual ~HashMethod() = default; virtual absl::optional evaluate(const Network::Address::Instance* downstream_addr, const Http::HeaderMap& headers, const AddCookieCallback add_cookie) const PURE; @@ -303,7 +303,7 @@ class HashPolicyImpl : public HashPolicy { virtual bool terminal() const PURE; }; - typedef std::unique_ptr HashMethodPtr; + using HashMethodPtr = std::unique_ptr; private: std::vector hash_impls_; @@ -617,7 +617,7 @@ class RouteEntryImplBase : public RouteEntry, PerFilterConfigs per_filter_configs_; }; - typedef std::shared_ptr WeightedClusterEntrySharedPtr; + using WeightedClusterEntrySharedPtr = std::shared_ptr; absl::optional loadRuntimeData(const envoy::api::v2::route::RouteMatch& route); @@ -776,10 +776,9 @@ class RouteMatcher { private: const VirtualHostImpl* findVirtualHost(const Http::HeaderMap& headers) const; - typedef std::map, - std::greater> - WildcardVirtualHosts; - typedef std::function SubstringFunction; + using WildcardVirtualHosts = + std::map, std::greater<>>; + using SubstringFunction = std::function; const VirtualHostImpl* findWildcardVirtualHost(const std::string& host, const WildcardVirtualHosts& wildcard_virtual_hosts, SubstringFunction substring_function) const; diff --git a/source/common/router/header_formatter.h b/source/common/router/header_formatter.h index e1b86b3912d8..eb9f8766f548 100644 --- a/source/common/router/header_formatter.h +++ b/source/common/router/header_formatter.h @@ -16,7 +16,7 @@ namespace Router { */ class HeaderFormatter { public: - virtual ~HeaderFormatter() {} + virtual ~HeaderFormatter() = default; virtual const std::string format(const Envoy::StreamInfo::StreamInfo& stream_info) const PURE; @@ -27,7 +27,7 @@ class HeaderFormatter { virtual bool append() const PURE; }; -typedef std::unique_ptr HeaderFormatterPtr; +using HeaderFormatterPtr = std::unique_ptr; /** * A formatter that expands the request header variable to a value based on info in StreamInfo. diff --git a/source/common/router/header_parser.h b/source/common/router/header_parser.h index 147574c43abd..b64ebd169c4b 100644 --- a/source/common/router/header_parser.h +++ b/source/common/router/header_parser.h @@ -14,7 +14,7 @@ namespace Envoy { namespace Router { class HeaderParser; -typedef std::unique_ptr HeaderParserPtr; +using HeaderParserPtr = std::unique_ptr; /** * HeaderParser manipulates Http::HeaderMap instances. Headers to be added are pre-parsed to select diff --git a/source/common/router/metadatamatchcriteria_impl.h b/source/common/router/metadatamatchcriteria_impl.h index df469750f6c4..357447945aa4 100644 --- a/source/common/router/metadatamatchcriteria_impl.h +++ b/source/common/router/metadatamatchcriteria_impl.h @@ -6,7 +6,7 @@ namespace Envoy { namespace Router { class MetadataMatchCriteriaImpl; -typedef std::unique_ptr MetadataMatchCriteriaImplConstPtr; +using MetadataMatchCriteriaImplConstPtr = std::unique_ptr; class MetadataMatchCriteriaImpl : public MetadataMatchCriteria { public: diff --git a/source/common/router/router.h b/source/common/router/router.h index a4b93e13ef44..0fc6f816332e 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -162,7 +162,7 @@ class FilterConfig { TimeSource& time_source_; }; -typedef std::shared_ptr FilterConfigSharedPtr; +using FilterConfigSharedPtr = std::shared_ptr; /** * Service routing filter. @@ -414,7 +414,7 @@ class Filter : Logger::Loggable, bool create_per_try_timeout_on_request_complete_ : 1; }; - typedef std::unique_ptr UpstreamRequestPtr; + using UpstreamRequestPtr = std::unique_ptr; StreamInfo::ResponseFlag streamResetReasonToResponseFlag(Http::StreamResetReason reset_reason); diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index 710402930621..e05e3e5c0302 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -72,9 +72,9 @@ class SdsApi : public Config::SubscriptionCallbacks { class TlsCertificateSdsApi; class CertificateValidationContextSdsApi; -typedef std::shared_ptr TlsCertificateSdsApiSharedPtr; -typedef std::shared_ptr - CertificateValidationContextSdsApiSharedPtr; +using TlsCertificateSdsApiSharedPtr = std::shared_ptr; +using CertificateValidationContextSdsApiSharedPtr = + std::shared_ptr; /** * TlsCertificateSdsApi implementation maintains and updates dynamic TLS certificate secrets. diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index c4fbd0101226..7867a3bcd87c 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -120,7 +120,7 @@ using ParentHistogramImplSharedPtr = std::shared_ptr; */ class TlsScope : public Scope { public: - virtual ~TlsScope() {} + ~TlsScope() override = default; // TODO(ramaraochavali): Allow direct TLS access for the advanced consumers. /** diff --git a/source/common/tcp/conn_pool.h b/source/common/tcp/conn_pool.h index faf206726a55..ff28802476cf 100644 --- a/source/common/tcp/conn_pool.h +++ b/source/common/tcp/conn_pool.h @@ -55,10 +55,10 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: bool conn_valid_{true}; }; - typedef std::shared_ptr ConnectionWrapperSharedPtr; + using ConnectionWrapperSharedPtr = std::shared_ptr; struct ConnectionDataImpl : public ConnectionPool::ConnectionData { - ConnectionDataImpl(ConnectionWrapperSharedPtr wrapper) : wrapper_(wrapper) {} + ConnectionDataImpl(ConnectionWrapperSharedPtr wrapper) : wrapper_(std::move(wrapper)) {} ~ConnectionDataImpl() { wrapper_->release(false); } // ConnectionPool::ConnectionData @@ -118,7 +118,7 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: bool timed_out_; }; - typedef std::unique_ptr ActiveConnPtr; + using ActiveConnPtr = std::unique_ptr; struct PendingRequest : LinkedObject, public ConnectionPool::Cancellable { PendingRequest(ConnPoolImpl& parent, ConnectionPool::Callbacks& callbacks); @@ -133,7 +133,7 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: ConnectionPool::Callbacks& callbacks_; }; - typedef std::unique_ptr PendingRequestPtr; + using PendingRequestPtr = std::unique_ptr; void assignConnection(ActiveConn& conn, ConnectionPool::Callbacks& callbacks); void createNewConnection(); diff --git a/source/common/tcp_proxy/tcp_proxy.h b/source/common/tcp_proxy/tcp_proxy.h index 9b5fe91d3f28..44cdd35aeacf 100644 --- a/source/common/tcp_proxy/tcp_proxy.h +++ b/source/common/tcp_proxy/tcp_proxy.h @@ -86,7 +86,7 @@ class Config { absl::optional idle_timeout_; }; - typedef std::shared_ptr SharedConfigSharedPtr; + using SharedConfigSharedPtr = std::shared_ptr; Config(const envoy::config::filter::network::tcp_proxy::v2::TcpProxy& config, Server::Configuration::FactoryContext& context); @@ -138,7 +138,7 @@ class Config { const std::string cluster_name_; const uint64_t cluster_weight_; }; - typedef std::unique_ptr WeightedClusterEntrySharedPtr; + using WeightedClusterEntrySharedPtr = std::unique_ptr; std::vector routes_; std::vector weighted_clusters_; @@ -151,7 +151,7 @@ class Config { Runtime::RandomGenerator& random_generator_; }; -typedef std::shared_ptr ConfigSharedPtr; +using ConfigSharedPtr = std::shared_ptr; /** * Per-connection TCP Proxy Cluster configuration. @@ -311,7 +311,7 @@ class Drainer : public Event::DeferredDeletable { Config::SharedConfigSharedPtr config_; }; -typedef std::unique_ptr DrainerPtr; +using DrainerPtr = std::unique_ptr; class UpstreamDrainManager : public ThreadLocal::ThreadLocalObject { public: diff --git a/source/common/tracing/http_tracer_impl.h b/source/common/tracing/http_tracer_impl.h index 2c557d41cace..50a782146043 100644 --- a/source/common/tracing/http_tracer_impl.h +++ b/source/common/tracing/http_tracer_impl.h @@ -59,7 +59,7 @@ class TracingTagValues { const std::string True = "true"; }; -typedef ConstSingleton Tags; +using Tags = ConstSingleton; class TracingLogValues { public: @@ -76,7 +76,7 @@ class TracingLogValues { const std::string LastDownstreamTxByteSent = "last_downstream_tx_byte_sent"; }; -typedef ConstSingleton Logs; +using Logs = ConstSingleton; class HttpTracerUtility { public: @@ -120,7 +120,7 @@ class EgressConfigImpl : public Config { const std::vector request_headers_for_tags_{}; }; -typedef ConstSingleton EgressConfig; +using EgressConfig = ConstSingleton; class NullSpan : public Span { public: diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index b4545acbe59c..d096e10afaff 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -248,7 +248,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable(dispatcher, host)} {} - typedef PriorityConnPoolMap, Http::ConnectionPool::Instance> ConnPools; + using ConnPools = PriorityConnPoolMap, Http::ConnectionPool::Instance>; // This is a shared_ptr so we can keep it alive while cleaning up. std::shared_ptr pools_; @@ -257,7 +257,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable, Tcp::ConnectionPool::InstancePtr> ConnPools; + using ConnPools = std::map, Tcp::ConnectionPool::InstancePtr>; ConnPools pools_; uint64_t drains_remaining_{}; @@ -287,8 +287,8 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable> - TcpConnectionsMap; + using TcpConnectionsMap = + std::unordered_map>; struct ClusterEntry : public ThreadLocalCluster { ClusterEntry(ThreadLocalClusterManagerImpl& parent, ClusterInfoConstSharedPtr cluster, @@ -319,7 +319,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable ClusterEntryPtr; + using ClusterEntryPtr = std::unique_ptr; ThreadLocalClusterManagerImpl(ClusterManagerImpl& parent, Event::Dispatcher& dispatcher, const absl::optional& local_cluster_name); @@ -391,9 +391,9 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable(parent, &cb) {} }; - typedef std::unique_ptr ClusterDataPtr; + using ClusterDataPtr = std::unique_ptr; // This map is ordered so that config dumping is consistent. - typedef std::map ClusterMap; + using ClusterMap = std::map; struct PendingUpdates { ~PendingUpdates() { disableTimer(); } diff --git a/source/common/upstream/health_checker_base_impl.h b/source/common/upstream/health_checker_base_impl.h index 3e3281aa766a..5920462adf8a 100644 --- a/source/common/upstream/health_checker_base_impl.h +++ b/source/common/upstream/health_checker_base_impl.h @@ -80,7 +80,7 @@ class HealthCheckerImplBase : public HealthChecker, bool first_check_{true}; }; - typedef std::unique_ptr ActiveHealthCheckSessionPtr; + using ActiveHealthCheckSessionPtr = std::unique_ptr; HealthCheckerImplBase(const Cluster& cluster, const envoy::api::v2::core::HealthCheck& config, Event::Dispatcher& dispatcher, Runtime::Loader& runtime, diff --git a/source/common/upstream/health_checker_impl.h b/source/common/upstream/health_checker_impl.h index 052f2d1bb9df..f47ddac7467c 100644 --- a/source/common/upstream/health_checker_impl.h +++ b/source/common/upstream/health_checker_impl.h @@ -121,7 +121,7 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase { bool expect_reset_{}; }; - typedef std::unique_ptr HttpActiveHealthCheckSessionPtr; + using HttpActiveHealthCheckSessionPtr = std::unique_ptr; virtual Http::CodecClient* createCodecClient(Upstream::Host::CreateConnectionData& data) PURE; @@ -201,7 +201,7 @@ class ProdHttpHealthCheckerImpl : public HttpHealthCheckerImpl { */ class TcpHealthCheckMatcher { public: - typedef std::list> MatchSegments; + using MatchSegments = std::list>; static MatchSegments loadProtoBytes( const Protobuf::RepeatedPtrField& byte_array); @@ -259,7 +259,7 @@ class TcpHealthCheckerImpl : public HealthCheckerImplBase { bool expect_close_{}; }; - typedef std::unique_ptr TcpActiveHealthCheckSessionPtr; + using TcpActiveHealthCheckSessionPtr = std::unique_ptr; // HealthCheckerImplBase ActiveHealthCheckSessionPtr makeSession(HostSharedPtr host) override { diff --git a/source/common/upstream/health_discovery_service.h b/source/common/upstream/health_discovery_service.h index e035568d402e..b1c889a30c23 100644 --- a/source/common/upstream/health_discovery_service.h +++ b/source/common/upstream/health_discovery_service.h @@ -84,7 +84,7 @@ class HdsCluster : public Cluster, Logger::Loggable { ProtobufMessage::ValidationVisitor& validation_visitor_; }; -typedef std::shared_ptr HdsClusterPtr; +using HdsClusterPtr = std::shared_ptr; /** * All hds stats. @see stats_macros.h @@ -199,7 +199,7 @@ class HdsDelegate : Grpc::AsyncStreamCallbacks HdsDelegatePtr; +using HdsDelegatePtr = std::unique_ptr; } // namespace Upstream } // namespace Envoy diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 3f0dca76df6a..7b26302c35cd 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -311,7 +311,7 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase { // routed where. std::vector residual_capacity_; }; - typedef std::unique_ptr PerPriorityStatePtr; + using PerPriorityStatePtr = std::unique_ptr; // Routing state broken out for each priority level in priority_set_. std::vector per_priority_state_; Common::CallbackHandle* local_priority_set_member_update_cb_handle_{}; diff --git a/source/common/upstream/load_stats_reporter.h b/source/common/upstream/load_stats_reporter.h index 669b41e5df12..348d46aff1a0 100644 --- a/source/common/upstream/load_stats_reporter.h +++ b/source/common/upstream/load_stats_reporter.h @@ -69,7 +69,7 @@ class LoadStatsReporter TimeSource& time_source_; }; -typedef std::unique_ptr LoadStatsReporterPtr; +using LoadStatsReporterPtr = std::unique_ptr; } // namespace Upstream } // namespace Envoy diff --git a/source/common/upstream/resource_manager_impl.h b/source/common/upstream/resource_manager_impl.h index a887182c6398..48ae3bb85969 100644 --- a/source/common/upstream/resource_manager_impl.h +++ b/source/common/upstream/resource_manager_impl.h @@ -112,7 +112,7 @@ class ResourceManagerImpl : public ResourceManager { ResourceImpl connection_pools_; }; -typedef std::unique_ptr ResourceManagerImplPtr; +using ResourceManagerImplPtr = std::unique_ptr; } // namespace Upstream } // namespace Envoy diff --git a/source/common/upstream/ring_hash_lb.h b/source/common/upstream/ring_hash_lb.h index f0a69a1a12ff..76bb1923b07b 100644 --- a/source/common/upstream/ring_hash_lb.h +++ b/source/common/upstream/ring_hash_lb.h @@ -66,7 +66,7 @@ class RingHashLoadBalancer : public ThreadAwareLoadBalancerBase, RingHashLoadBalancerStats& stats_; }; - typedef std::shared_ptr RingConstSharedPtr; + using RingConstSharedPtr = std::shared_ptr; // ThreadAwareLoadBalancerBase HashingLoadBalancerSharedPtr diff --git a/source/common/upstream/strict_dns_cluster.h b/source/common/upstream/strict_dns_cluster.h index 63f1a4bd647f..27b3fc5f3062 100644 --- a/source/common/upstream/strict_dns_cluster.h +++ b/source/common/upstream/strict_dns_cluster.h @@ -40,7 +40,7 @@ class StrictDnsClusterImpl : public BaseDynamicClusterImpl { HostMap all_hosts_; }; - typedef std::unique_ptr ResolveTargetPtr; + using ResolveTargetPtr = std::unique_ptr; void updateAllHosts(const HostVector& hosts_added, const HostVector& hosts_removed, uint32_t priority); diff --git a/source/common/upstream/subset_lb.h b/source/common/upstream/subset_lb.h index 3980cce1330a..08499782ca26 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/common/upstream/subset_lb.h @@ -35,7 +35,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable HostPredicate; + using HostPredicate = std::function; // Represents a subset of an original HostSet. class HostSubsetImpl : public HostSetImpl { @@ -105,15 +105,15 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable HostSubsetImplPtr; - typedef std::shared_ptr PrioritySubsetImplPtr; + using HostSubsetImplPtr = std::shared_ptr; + using PrioritySubsetImplPtr = std::shared_ptr; - typedef std::vector> SubsetMetadata; + using SubsetMetadata = std::vector>; class LbSubsetEntry; - typedef std::shared_ptr LbSubsetEntryPtr; - typedef std::unordered_map ValueSubsetMap; - typedef std::unordered_map LbSubsetMap; + using LbSubsetEntryPtr = std::shared_ptr; + using ValueSubsetMap = std::unordered_map; + using LbSubsetMap = std::unordered_map; // Entry in the subset hierarchy. class LbSubsetEntry { diff --git a/source/common/upstream/thread_aware_lb_impl.h b/source/common/upstream/thread_aware_lb_impl.h index d8e7de1be87f..2cf47f4e7552 100644 --- a/source/common/upstream/thread_aware_lb_impl.h +++ b/source/common/upstream/thread_aware_lb_impl.h @@ -7,7 +7,7 @@ namespace Envoy { namespace Upstream { -typedef std::vector> NormalizedHostWeightVector; +using NormalizedHostWeightVector = std::vector>; class ThreadAwareLoadBalancerBase : public LoadBalancerBase, public ThreadAwareLoadBalancer { public: @@ -21,10 +21,10 @@ class ThreadAwareLoadBalancerBase : public LoadBalancerBase, public ThreadAwareL */ class HashingLoadBalancer { public: - virtual ~HashingLoadBalancer() {} + virtual ~HashingLoadBalancer() = default; virtual HostConstSharedPtr chooseHost(uint64_t hash) const PURE; }; - typedef std::shared_ptr HashingLoadBalancerSharedPtr; + using HashingLoadBalancerSharedPtr = std::shared_ptr; // Upstream::ThreadAwareLoadBalancer LoadBalancerFactorySharedPtr factory() override { return factory_; } @@ -47,7 +47,7 @@ class ThreadAwareLoadBalancerBase : public LoadBalancerBase, public ThreadAwareL std::shared_ptr current_lb_; bool global_panic_{}; }; - typedef std::unique_ptr PerPriorityStatePtr; + using PerPriorityStatePtr = std::unique_ptr; struct LoadBalancerImpl : public LoadBalancer { LoadBalancerImpl(ClusterStats& stats, Runtime::RandomGenerator& random) diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index f4040cae85f1..fe87258f5910 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -417,7 +417,7 @@ class HostSetImpl : public HostSet { std::unique_ptr> degraded_locality_scheduler_; }; -typedef std::unique_ptr HostSetImplPtr; +using HostSetImplPtr = std::unique_ptr; /** * A class for management of the set of hosts in a given cluster. @@ -584,7 +584,7 @@ class ClusterInfoImpl : public ClusterInfo { const std::string& cluster_name, Stats::Scope& stats_scope, const envoy::api::v2::core::RoutingPriority& priority); - typedef std::array Managers; + using Managers = std::array; Managers managers_; }; @@ -789,7 +789,7 @@ class PriorityStateManager : protected Logger::Loggable { PrioritySet::HostUpdateCb* update_cb_; }; -typedef std::unique_ptr PriorityStateManagerPtr; +using PriorityStateManagerPtr = std::unique_ptr; /** * Base for all dynamic cluster types. diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h index 3625108a86fd..e8c6acde5e19 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h @@ -28,7 +28,7 @@ namespace HttpGrpc { */ class GrpcAccessLogStreamer { public: - virtual ~GrpcAccessLogStreamer() {} + virtual ~GrpcAccessLogStreamer() = default; /** * Send an access log. @@ -39,7 +39,7 @@ class GrpcAccessLogStreamer { const std::string& log_name) PURE; }; -typedef std::shared_ptr GrpcAccessLogStreamerSharedPtr; +using GrpcAccessLogStreamerSharedPtr = std::shared_ptr; /** * Production implementation of GrpcAccessLogStreamer that supports per-thread and per-log @@ -69,7 +69,7 @@ class GrpcAccessLogStreamerImpl : public Singleton::Instance, public GrpcAccessL const LocalInfo::LocalInfo& local_info_; }; - typedef std::shared_ptr SharedStateSharedPtr; + using SharedStateSharedPtr = std::shared_ptr; struct ThreadLocalStreamer; diff --git a/source/extensions/access_loggers/well_known_names.h b/source/extensions/access_loggers/well_known_names.h index f74f726a50b3..7c5e7576c1df 100644 --- a/source/extensions/access_loggers/well_known_names.h +++ b/source/extensions/access_loggers/well_known_names.h @@ -20,7 +20,7 @@ class AccessLogNameValues { const std::string HttpGrpc = "envoy.http_grpc_access_log"; }; -typedef ConstSingleton AccessLogNames; +using AccessLogNames = ConstSingleton; } // namespace AccessLoggers } // namespace Extensions diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index 0f1edd816575..d21b69a7e176 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -168,7 +168,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { const uint32_t port_; }; - typedef std::unique_ptr DnsDiscoveryResolveTargetPtr; + using DnsDiscoveryResolveTargetPtr = std::unique_ptr; struct RedisDiscoverySession; @@ -185,7 +185,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { Extensions::NetworkFilters::Common::Redis::Client::ClientPtr client_; }; - typedef std::unique_ptr RedisDiscoveryClientPtr; + using RedisDiscoveryClientPtr = std::unique_ptr; struct RedisDiscoverySession : public Extensions::NetworkFilters::Common::Redis::Client::Config, diff --git a/source/extensions/clusters/redis/redis_cluster_lb.h b/source/extensions/clusters/redis/redis_cluster_lb.h index e0dd2985499e..c34112352bfa 100644 --- a/source/extensions/clusters/redis/redis_cluster_lb.h +++ b/source/extensions/clusters/redis/redis_cluster_lb.h @@ -24,9 +24,9 @@ namespace Redis { static const uint64_t MaxSlot = 16384; -typedef std::array SlotArray; +using SlotArray = std::array; -typedef std::shared_ptr SlotArraySharedPtr; +using SlotArraySharedPtr = std::shared_ptr; class ClusterSlot { public: @@ -93,7 +93,7 @@ class ClusterSlotUpdateCallBack { Upstream::HostMap all_hosts) PURE; }; -typedef std::shared_ptr ClusterSlotUpdateCallBackSharedPtr; +using ClusterSlotUpdateCallBackSharedPtr = std::shared_ptr; /** * This factory is created and returned by RedisCluster's factory() method, the create() method will diff --git a/source/extensions/filters/common/ext_authz/ext_authz.h b/source/extensions/filters/common/ext_authz/ext_authz.h index f5f751c9ed96..f79df739884c 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz.h +++ b/source/extensions/filters/common/ext_authz/ext_authz.h @@ -44,14 +44,14 @@ struct Response { Http::Code status_code{}; }; -typedef std::unique_ptr ResponsePtr; +using ResponsePtr = std::unique_ptr; /** * Async callbacks used during check() calls. */ class RequestCallbacks { public: - virtual ~RequestCallbacks() {} + virtual ~RequestCallbacks() = default; /** * Called when a check request is complete. The resulting ResponsePtr is supplied. @@ -62,7 +62,7 @@ class RequestCallbacks { class Client { public: // Destructor - virtual ~Client() {} + virtual ~Client() = default; /** * Cancel an inflight Check request. @@ -83,7 +83,7 @@ class Client { Tracing::Span& parent_span) PURE; }; -typedef std::unique_ptr ClientPtr; +using ClientPtr = std::unique_ptr; } // namespace ExtAuthz } // namespace Common diff --git a/source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h b/source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h index 37adadb7b783..6fa44003f50e 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h +++ b/source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h @@ -28,7 +28,7 @@ namespace Filters { namespace Common { namespace ExtAuthz { -typedef Grpc::AsyncRequestCallbacks ExtAuthzAsyncCallbacks; +using ExtAuthzAsyncCallbacks = Grpc::AsyncRequestCallbacks; struct ConstantValues { const std::string TraceStatus = "ext_authz_status"; @@ -36,7 +36,7 @@ struct ConstantValues { const std::string TraceOk = "ext_authz_ok"; }; -typedef ConstSingleton Constants; +using Constants = ConstSingleton; /* * This client implementation is used when the Ext_Authz filter needs to communicate with an gRPC diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h index 39d1666a7f26..376ccdc64c70 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h @@ -15,14 +15,14 @@ namespace Common { namespace ExtAuthz { class Matcher; -typedef std::shared_ptr MatcherSharedPtr; +using MatcherSharedPtr = std::shared_ptr; /** * Matchers describe the rules for matching authorization request and response headers. */ class Matcher { public: - virtual ~Matcher() {} + virtual ~Matcher() = default; /** * Returns whether or not the header key matches the rules of the matcher. @@ -115,7 +115,7 @@ class ClientConfig { const std::string path_prefix_; }; -typedef std::shared_ptr ClientConfigSharedPtr; +using ClientConfigSharedPtr = std::shared_ptr; /** * This client implementation is used when the Ext_Authz filter needs to communicate with an diff --git a/source/extensions/filters/common/fault/fault_config.h b/source/extensions/filters/common/fault/fault_config.h index 61b3ada9eda7..64e966d87d79 100644 --- a/source/extensions/filters/common/fault/fault_config.h +++ b/source/extensions/filters/common/fault/fault_config.h @@ -17,7 +17,7 @@ class HeaderNameValues { const Http::LowerCaseString ThroughputResponse{"x-envoy-fault-throughput-response"}; }; -typedef ConstSingleton HeaderNames; +using HeaderNames = ConstSingleton; /** * Generic configuration for a delay fault. diff --git a/source/extensions/filters/common/lua/lua.h b/source/extensions/filters/common/lua/lua.h index 872e0dba2e14..f5eaff56950d 100644 --- a/source/extensions/filters/common/lua/lua.h +++ b/source/extensions/filters/common/lua/lua.h @@ -103,9 +103,9 @@ template inline T* allocateLuaUserData(lua_State* state) { */ template class BaseLuaObject : protected Logger::Loggable { public: - typedef std::vector> ExportedFunctions; + using ExportedFunctions = std::vector>; - virtual ~BaseLuaObject() {} + virtual ~BaseLuaObject() = default; /** * Create a new object of this type, owned by Lua. This type must have previously been registered @@ -351,7 +351,7 @@ class Coroutine : Logger::Loggable { State state_{State::NotStarted}; }; -typedef std::unique_ptr CoroutinePtr; +using CoroutinePtr = std::unique_ptr; /** * This class wraps a Lua state that can be used safely across threads. The model is that every diff --git a/source/extensions/filters/common/ratelimit/ratelimit.h b/source/extensions/filters/common/ratelimit/ratelimit.h index 0da01c1f250e..24223d0aa1dd 100644 --- a/source/extensions/filters/common/ratelimit/ratelimit.h +++ b/source/extensions/filters/common/ratelimit/ratelimit.h @@ -36,7 +36,7 @@ enum class LimitStatus { */ class RequestCallbacks { public: - virtual ~RequestCallbacks() {} + virtual ~RequestCallbacks() = default; /** * Called when a limit request is complete. The resulting status and @@ -50,7 +50,7 @@ class RequestCallbacks { */ class Client { public: - virtual ~Client() {} + virtual ~Client() = default; /** * Cancel an inflight limit request. @@ -74,7 +74,7 @@ class Client { Tracing::Span& parent_span) PURE; }; -typedef std::unique_ptr ClientPtr; +using ClientPtr = std::unique_ptr; } // namespace RateLimit } // namespace Common diff --git a/source/extensions/filters/common/ratelimit/ratelimit_impl.h b/source/extensions/filters/common/ratelimit/ratelimit_impl.h index 3ea040f342cf..11ca4c4396b0 100644 --- a/source/extensions/filters/common/ratelimit/ratelimit_impl.h +++ b/source/extensions/filters/common/ratelimit/ratelimit_impl.h @@ -27,8 +27,8 @@ namespace Filters { namespace Common { namespace RateLimit { -typedef Grpc::AsyncRequestCallbacks - RateLimitAsyncCallbacks; +using RateLimitAsyncCallbacks = + Grpc::AsyncRequestCallbacks; struct ConstantValues { const std::string TraceStatus = "ratelimit_status"; @@ -36,7 +36,7 @@ struct ConstantValues { const std::string TraceOk = "ok"; }; -typedef ConstSingleton Constants; +using Constants = ConstSingleton; // TODO(htuch): We should have only one client per thread, but today we create one per filter stack. // This will require support for more than one outstanding request per client (limit() assumes only diff --git a/source/extensions/filters/common/rbac/engine.h b/source/extensions/filters/common/rbac/engine.h index b4f4c78d0914..4a07c03d2a31 100644 --- a/source/extensions/filters/common/rbac/engine.h +++ b/source/extensions/filters/common/rbac/engine.h @@ -16,7 +16,7 @@ namespace RBAC { */ class RoleBasedAccessControlEngine { public: - virtual ~RoleBasedAccessControlEngine() {} + virtual ~RoleBasedAccessControlEngine() = default; /** * Returns whether or not the current action is permitted. diff --git a/source/extensions/filters/common/rbac/matchers.h b/source/extensions/filters/common/rbac/matchers.h index d62dc6267ee7..28b81846e114 100644 --- a/source/extensions/filters/common/rbac/matchers.h +++ b/source/extensions/filters/common/rbac/matchers.h @@ -18,14 +18,14 @@ namespace Common { namespace RBAC { class Matcher; -typedef std::shared_ptr MatcherConstSharedPtr; +using MatcherConstSharedPtr = std::shared_ptr; /** * Matchers describe the rules for matching either a permission action or principal. */ class Matcher { public: - virtual ~Matcher() {} + virtual ~Matcher() = default; /** * Returns whether or not the permission/principal matches the rules of the matcher. diff --git a/source/extensions/filters/common/rbac/utility.h b/source/extensions/filters/common/rbac/utility.h index 5cd04c52f303..bcf934f41feb 100644 --- a/source/extensions/filters/common/rbac/utility.h +++ b/source/extensions/filters/common/rbac/utility.h @@ -22,7 +22,7 @@ class DynamicMetadataKeys { const std::string EngineResultDenied{"denied"}; }; -typedef ConstSingleton DynamicMetadataKeysSingleton; +using DynamicMetadataKeysSingleton = ConstSingleton; /** * All stats for the RBAC filter. @see stats_macros.h diff --git a/source/extensions/filters/http/buffer/buffer_filter.h b/source/extensions/filters/http/buffer/buffer_filter.h index 4839189ef6e1..85754ae33f1c 100644 --- a/source/extensions/filters/http/buffer/buffer_filter.h +++ b/source/extensions/filters/http/buffer/buffer_filter.h @@ -41,7 +41,7 @@ class BufferFilterConfig { const BufferFilterSettings settings_; }; -typedef std::shared_ptr BufferFilterConfigSharedPtr; +using BufferFilterConfigSharedPtr = std::shared_ptr; /** * A filter that is capable of buffering an entire request before dispatching it upstream. diff --git a/source/extensions/filters/http/common/aws/credentials_provider.h b/source/extensions/filters/http/common/aws/credentials_provider.h index 1afc551f0bf3..e761b915ba4e 100644 --- a/source/extensions/filters/http/common/aws/credentials_provider.h +++ b/source/extensions/filters/http/common/aws/credentials_provider.h @@ -44,7 +44,7 @@ class CredentialsProvider { virtual Credentials getCredentials() PURE; }; -typedef std::shared_ptr CredentialsProviderSharedPtr; +using CredentialsProviderSharedPtr = std::shared_ptr; } // namespace Aws } // namespace Common diff --git a/source/extensions/filters/http/common/aws/signer.h b/source/extensions/filters/http/common/aws/signer.h index bf504bc4356b..672c233ceaa0 100644 --- a/source/extensions/filters/http/common/aws/signer.h +++ b/source/extensions/filters/http/common/aws/signer.h @@ -23,7 +23,7 @@ class Signer { virtual void sign(Http::Message& message, bool sign_body) PURE; }; -typedef std::unique_ptr SignerPtr; +using SignerPtr = std::unique_ptr; } // namespace Aws } // namespace Common diff --git a/source/extensions/filters/http/common/aws/signer_impl.h b/source/extensions/filters/http/common/aws/signer_impl.h index 8a7dabfdcf76..3b4baedd98a0 100644 --- a/source/extensions/filters/http/common/aws/signer_impl.h +++ b/source/extensions/filters/http/common/aws/signer_impl.h @@ -20,7 +20,7 @@ class SignatureHeaderValues { const Http::LowerCaseString SecurityToken{"x-amz-security-token"}; }; -typedef ConstSingleton SignatureHeaders; +using SignatureHeaders = ConstSingleton; class SignatureConstantValues { public: @@ -37,7 +37,7 @@ class SignatureConstantValues { const std::string ShortDateFormat{"%Y%m%d"}; }; -typedef ConstSingleton SignatureConstants; +using SignatureConstants = ConstSingleton; /** * Implementation of the Signature V4 signing process. diff --git a/source/extensions/filters/http/common/jwks_fetcher.h b/source/extensions/filters/http/common/jwks_fetcher.h index ec948bbc4df2..08344a39f935 100644 --- a/source/extensions/filters/http/common/jwks_fetcher.h +++ b/source/extensions/filters/http/common/jwks_fetcher.h @@ -12,7 +12,7 @@ namespace HttpFilters { namespace Common { class JwksFetcher; -typedef std::unique_ptr JwksFetcherPtr; +using JwksFetcherPtr = std::unique_ptr; /** * JwksFetcher interface can be used to retrieve remote JWKS * (https://tools.ietf.org/html/rfc7517) data structures returning a concrete, @@ -30,7 +30,7 @@ class JwksFetcher { InvalidJwks, }; - virtual ~JwksReceiver(){}; + virtual ~JwksReceiver() = default; /* * Successful retrieval callback. * of the returned JWKS object. @@ -44,7 +44,7 @@ class JwksFetcher { virtual void onJwksError(Failure reason) PURE; }; - virtual ~JwksFetcher(){}; + virtual ~JwksFetcher() = default; /* * Cancel any in-flight request. diff --git a/source/extensions/filters/http/cors/cors_filter.h b/source/extensions/filters/http/cors/cors_filter.h index edac209bee13..b70b382a2177 100644 --- a/source/extensions/filters/http/cors/cors_filter.h +++ b/source/extensions/filters/http/cors/cors_filter.h @@ -42,7 +42,7 @@ class CorsFilterConfig { CorsStats stats_; }; -typedef std::shared_ptr CorsFilterConfigSharedPtr; +using CorsFilterConfigSharedPtr = std::shared_ptr; class CorsFilter : public Http::StreamFilter { public: diff --git a/source/extensions/filters/http/csrf/csrf_filter.cc b/source/extensions/filters/http/csrf/csrf_filter.cc index b536ef914a2e..f8065b6d58ed 100644 --- a/source/extensions/filters/http/csrf/csrf_filter.cc +++ b/source/extensions/filters/http/csrf/csrf_filter.cc @@ -17,7 +17,7 @@ namespace Csrf { struct RcDetailsValues { const std::string OriginMismatch = "csrf_origin_mismatch"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; namespace { bool isModifyMethod(const Http::HeaderMap& headers) { diff --git a/source/extensions/filters/http/csrf/csrf_filter.h b/source/extensions/filters/http/csrf/csrf_filter.h index 392e5e2a95a6..4ee3af9d1fa7 100644 --- a/source/extensions/filters/http/csrf/csrf_filter.h +++ b/source/extensions/filters/http/csrf/csrf_filter.h @@ -74,7 +74,7 @@ class CsrfFilterConfig { CsrfStats stats_; const CsrfPolicy policy_; }; -typedef std::shared_ptr CsrfFilterConfigSharedPtr; +using CsrfFilterConfigSharedPtr = std::shared_ptr; class CsrfFilter : public Http::StreamDecoderFilter { public: diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index a7a577308460..3f1a1535f8aa 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -18,7 +18,7 @@ struct RcDetailsValues { // The ext_authz filter encountered a failure, and was configured to fail-closed. const std::string AuthzError = "ext_authz_error"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; void FilterConfigPerRoute::merge(const FilterConfigPerRoute& other) { disabled_ = other.disabled_; diff --git a/source/extensions/filters/http/ext_authz/ext_authz.h b/source/extensions/filters/http/ext_authz/ext_authz.h index 427616f6b8ab..75831ed9fc5e 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.h +++ b/source/extensions/filters/http/ext_authz/ext_authz.h @@ -88,7 +88,7 @@ class FilterConfig { Http::Context& http_context_; }; -typedef std::shared_ptr FilterConfigSharedPtr; +using FilterConfigSharedPtr = std::shared_ptr; /** * Per route settings for ExtAuth. Allows customizing the CheckRequest on a diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index ee491a73f695..2d5ad0c9d62a 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -30,7 +30,7 @@ struct RcDetailsValues { // The fault filter injected an abort for this request. const std::string FaultAbort = "fault_filter_abort"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; FaultSettings::FaultSettings(const envoy::config::filter::http::fault::v2::HTTPFault& fault) { if (fault.has_abort()) { diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index 4d130d46a0ee..8185133e73ab 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -101,7 +101,7 @@ class FaultFilterConfig { TimeSource& time_source_; }; -typedef std::shared_ptr FaultFilterConfigSharedPtr; +using FaultFilterConfigSharedPtr = std::shared_ptr; /** * An HTTP stream rate limiter. Split out for ease of testing and potential code reuse elsewhere. diff --git a/source/extensions/filters/http/grpc_http1_reverse_bridge/filter.cc b/source/extensions/filters/http/grpc_http1_reverse_bridge/filter.cc index 54b1fca59efc..865c2cb08b6a 100644 --- a/source/extensions/filters/http/grpc_http1_reverse_bridge/filter.cc +++ b/source/extensions/filters/http/grpc_http1_reverse_bridge/filter.cc @@ -21,7 +21,7 @@ struct RcDetailsValues { // The gRPC HTTP/1 bridge encountered an unsupported content type. const std::string GrpcBridgeFailedContentType = "grpc_bridge_content_type_wrong"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; namespace { Grpc::Status::GrpcStatus grpcStatusFromHeaders(Http::HeaderMap& headers) { diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc index 85f227845cb6..e21977ce5a80 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc @@ -51,7 +51,7 @@ struct RcDetailsValues { // This will generally be accompanied by details about the transcoder failure. const std::string GrpcTranscodeFailed = "grpc_json_transcode_failure"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; namespace { // Transcoder: diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h index 48b91b922bda..0b7f3b7ff301 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h @@ -91,7 +91,7 @@ class JsonTranscoderConfig : public Logger::Loggable { bool match_incoming_request_route_{false}; }; -typedef std::shared_ptr JsonTranscoderConfigSharedPtr; +using JsonTranscoderConfigSharedPtr = std::shared_ptr; /** * The filter instance for gRPC JSON transcoder. diff --git a/source/extensions/filters/http/grpc_web/grpc_web_filter.cc b/source/extensions/filters/http/grpc_web/grpc_web_filter.cc index 423baa890bca..67113d6d73b5 100644 --- a/source/extensions/filters/http/grpc_web/grpc_web_filter.cc +++ b/source/extensions/filters/http/grpc_web/grpc_web_filter.cc @@ -21,7 +21,7 @@ struct RcDetailsValues { // The grpc web filter couldn't decode the data provided. const std::string GrpcDecodeFailedDueToData = "grpc_base_64_decode_failed"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; // Bit mask denotes a trailers frame of gRPC-Web. const uint8_t GrpcWebFilter::GRPC_WEB_TRAILER = 0b10000000; diff --git a/source/extensions/filters/http/grpc_web/grpc_web_filter.h b/source/extensions/filters/http/grpc_web/grpc_web_filter.h index 79c1c562ae7e..3b2312006e50 100644 --- a/source/extensions/filters/http/grpc_web/grpc_web_filter.h +++ b/source/extensions/filters/http/grpc_web/grpc_web_filter.h @@ -21,7 +21,7 @@ namespace GrpcWeb { class GrpcWebFilter : public Http::StreamFilter, NonCopyable { public: explicit GrpcWebFilter(Grpc::Context& context) : context_(context) {} - virtual ~GrpcWebFilter() {} + ~GrpcWebFilter() override = default; // Http::StreamFilterBase void onDestroy() override {} diff --git a/source/extensions/filters/http/gzip/gzip_filter.h b/source/extensions/filters/http/gzip/gzip_filter.h index 6ef81b1e9559..2346157070bb 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.h +++ b/source/extensions/filters/http/gzip/gzip_filter.h @@ -107,7 +107,7 @@ class GzipFilterConfig { GzipStats stats_; Runtime::Loader& runtime_; }; -typedef std::shared_ptr GzipFilterConfigSharedPtr; +using GzipFilterConfigSharedPtr = std::shared_ptr; /** * A filter that compresses data dispatched from the upstream upon client request. diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h index c292886e8b1a..188124e9d16f 100644 --- a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h +++ b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h @@ -16,9 +16,9 @@ namespace Extensions { namespace HttpFilters { namespace HeaderToMetadataFilter { -typedef envoy::config::filter::http::header_to_metadata::v2::Config::Rule Rule; -typedef envoy::config::filter::http::header_to_metadata::v2::Config::ValueType ValueType; -typedef std::vector> HeaderToMetadataRules; +using Rule = envoy::config::filter::http::header_to_metadata::v2::Config::Rule; +using ValueType = envoy::config::filter::http::header_to_metadata::v2::Config::ValueType; +using HeaderToMetadataRules = std::vector>; /** * Encapsulates the filter configuration with STL containers and provides an area for any custom @@ -34,7 +34,7 @@ class Config : public Logger::Loggable { bool doRequest() const { return request_set_; } private: - typedef Protobuf::RepeatedPtrField ProtobufRepeatedRule; + using ProtobufRepeatedRule = Protobuf::RepeatedPtrField; HeaderToMetadataRules request_rules_; HeaderToMetadataRules response_rules_; @@ -56,7 +56,7 @@ class Config : public Logger::Loggable { const std::string& decideNamespace(const std::string& nspace) const; }; -typedef std::shared_ptr ConfigSharedPtr; +using ConfigSharedPtr = std::shared_ptr; /** * Header-To-Metadata examines request/response headers and either copies or @@ -98,7 +98,7 @@ class HeaderToMetadataFilter : public Http::StreamFilter, void setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks& callbacks) override; private: - typedef std::map StructMap; + using StructMap = std::map; const ConfigSharedPtr config_; Http::StreamDecoderFilterCallbacks* decoder_callbacks_{}; diff --git a/source/extensions/filters/http/health_check/health_check.cc b/source/extensions/filters/http/health_check/health_check.cc index aa704773706f..351e7ae9b046 100644 --- a/source/extensions/filters/http/health_check/health_check.cc +++ b/source/extensions/filters/http/health_check/health_check.cc @@ -36,7 +36,7 @@ struct RcDetailsValues { // The health check filter failed given the cluster health was not sufficient. const std::string HealthCheckClusterUnhealthy = "health_check_failed_cluster_unhealthy"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; HealthCheckCacheManager::HealthCheckCacheManager(Event::Dispatcher& dispatcher, std::chrono::milliseconds timeout) diff --git a/source/extensions/filters/http/health_check/health_check.h b/source/extensions/filters/http/health_check/health_check.h index 28175e7816e3..5515e52e0435 100644 --- a/source/extensions/filters/http/health_check/health_check.h +++ b/source/extensions/filters/http/health_check/health_check.h @@ -47,13 +47,13 @@ class HealthCheckCacheManager { std::atomic last_response_degraded_{}; }; -typedef std::shared_ptr HealthCheckCacheManagerSharedPtr; +using HealthCheckCacheManagerSharedPtr = std::shared_ptr; -typedef std::map ClusterMinHealthyPercentages; -typedef std::shared_ptr - ClusterMinHealthyPercentagesConstSharedPtr; +using ClusterMinHealthyPercentages = std::map; +using ClusterMinHealthyPercentagesConstSharedPtr = + std::shared_ptr; -typedef std::shared_ptr> HeaderDataVectorSharedPtr; +using HeaderDataVectorSharedPtr = std::shared_ptr>; /** * Health check responder filter. diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 7b63afbb6eb5..14a35a9a914b 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -98,7 +98,7 @@ class IpTaggingFilterConfig { std::unique_ptr> trie_; }; -typedef std::shared_ptr IpTaggingFilterConfigSharedPtr; +using IpTaggingFilterConfigSharedPtr = std::shared_ptr; /** * A filter that gets all tags associated with a request's downstream remote address and diff --git a/source/extensions/filters/http/jwt_authn/authenticator.h b/source/extensions/filters/http/jwt_authn/authenticator.h index 83bbd05684c2..74414c54aea0 100644 --- a/source/extensions/filters/http/jwt_authn/authenticator.h +++ b/source/extensions/filters/http/jwt_authn/authenticator.h @@ -15,23 +15,23 @@ namespace HttpFilters { namespace JwtAuthn { class Authenticator; -typedef std::unique_ptr AuthenticatorPtr; +using AuthenticatorPtr = std::unique_ptr; -typedef std::function AuthenticatorCallback; +using AuthenticatorCallback = std::function; -typedef std::function SetPayloadCallback; +using SetPayloadCallback = std::function; /** * CreateJwksFetcherCb is a callback interface for creating a JwksFetcher instance. */ -typedef std::function CreateJwksFetcherCb; +using CreateJwksFetcherCb = std::function; /** * Authenticator object to handle all JWT authentication flow. */ class Authenticator { public: - virtual ~Authenticator() {} + virtual ~Authenticator() = default; // Verify if headers satisfies the JWT requirements. Can be limited to single provider with // extract_param. @@ -54,7 +54,7 @@ class Authenticator { */ class AuthFactory { public: - virtual ~AuthFactory() {} + virtual ~AuthFactory() = default; // Factory method for creating authenticator, and populate it with provider config. virtual AuthenticatorPtr create(const ::google::jwt_verify::CheckAudience* check_audience, diff --git a/source/extensions/filters/http/jwt_authn/extractor.cc b/source/extensions/filters/http/jwt_authn/extractor.cc index b589c2c8bb71..08103d852207 100644 --- a/source/extensions/filters/http/jwt_authn/extractor.cc +++ b/source/extensions/filters/http/jwt_authn/extractor.cc @@ -30,7 +30,7 @@ struct JwtConstValueStruct { // The default query parameter name to extract JWT token const std::string AccessTokenParam{"access_token"}; }; -typedef ConstSingleton JwtConstValues; +using JwtConstValues = ConstSingleton; // A base JwtLocation object to store token and specified_issuers. class JwtLocationBase : public JwtLocation { @@ -118,7 +118,7 @@ class ExtractorImpl : public Extractor { // Issuers that specified this header. std::unordered_set specified_issuers_; }; - typedef std::unique_ptr HeaderLocationSpecPtr; + using HeaderLocationSpecPtr = std::unique_ptr; // The map of (header + value_prefix) to HeaderLocationSpecPtr std::map header_locations_; diff --git a/source/extensions/filters/http/jwt_authn/extractor.h b/source/extensions/filters/http/jwt_authn/extractor.h index d79fb65592ab..185976202cae 100644 --- a/source/extensions/filters/http/jwt_authn/extractor.h +++ b/source/extensions/filters/http/jwt_authn/extractor.h @@ -24,7 +24,7 @@ namespace JwtAuthn { */ class JwtLocation { public: - virtual ~JwtLocation() {} + virtual ~JwtLocation() = default; // Get the token string virtual const std::string& token() const PURE; @@ -36,10 +36,10 @@ class JwtLocation { virtual void removeJwt(Http::HeaderMap& headers) const PURE; }; -typedef std::unique_ptr JwtLocationConstPtr; +using JwtLocationConstPtr = std::unique_ptr; class Extractor; -typedef std::unique_ptr ExtractorConstPtr; +using ExtractorConstPtr = std::unique_ptr; /** * Extracts JWT from locations specified in the config. @@ -63,7 +63,7 @@ typedef std::unique_ptr ExtractorConstPtr; */ class Extractor { public: - virtual ~Extractor() {} + virtual ~Extractor() = default; /** * Extract all JWT tokens from the headers. If set of header_keys or param_keys diff --git a/source/extensions/filters/http/jwt_authn/filter.cc b/source/extensions/filters/http/jwt_authn/filter.cc index bae13d7825ec..02576892a517 100644 --- a/source/extensions/filters/http/jwt_authn/filter.cc +++ b/source/extensions/filters/http/jwt_authn/filter.cc @@ -15,9 +15,10 @@ struct RcDetailsValues { // The jwt_authn filter rejected the request const std::string JwtAuthnAccessDenied = "jwt_authn_access_denied"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; -Filter::Filter(FilterConfigSharedPtr config) : stats_(config->stats()), config_(config) {} +Filter::Filter(FilterConfigSharedPtr config) + : stats_(config->stats()), config_(std::move(config)) {} void Filter::onDestroy() { ENVOY_LOG(debug, "Called Filter : {}", __func__); diff --git a/source/extensions/filters/http/jwt_authn/filter_config.h b/source/extensions/filters/http/jwt_authn/filter_config.h index 622a1e50f5c4..454fa9b1314c 100644 --- a/source/extensions/filters/http/jwt_authn/filter_config.h +++ b/source/extensions/filters/http/jwt_authn/filter_config.h @@ -61,12 +61,12 @@ struct JwtAuthnFilterStats { */ class FilterConfig : public Logger::Loggable, public AuthFactory { public: - virtual ~FilterConfig() {} + ~FilterConfig() override = default; - FilterConfig( - const ::envoy::config::filter::http::jwt_authn::v2alpha::JwtAuthentication& proto_config, - const std::string& stats_prefix, Server::Configuration::FactoryContext& context) - : proto_config_(proto_config), stats_(generateStats(stats_prefix, context.scope())), + FilterConfig(::envoy::config::filter::http::jwt_authn::v2alpha::JwtAuthentication proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) + : proto_config_(std::move(proto_config)), + stats_(generateStats(stats_prefix, context.scope())), tls_(context.threadLocal().allocateSlot()), cm_(context.clusterManager()), time_source_(context.dispatcher().timeSource()), api_(context.api()) { ENVOY_LOG(info, "Loaded JwtAuthConfig: {}", proto_config_.DebugString()); @@ -168,7 +168,7 @@ class FilterConfig : public Logger::Loggable, public AuthFac TimeSource& time_source_; Api::Api& api_; }; -typedef std::shared_ptr FilterConfigSharedPtr; +using FilterConfigSharedPtr = std::shared_ptr; } // namespace JwtAuthn } // namespace HttpFilters diff --git a/source/extensions/filters/http/jwt_authn/jwks_cache.h b/source/extensions/filters/http/jwt_authn/jwks_cache.h index 9011fa4d8d88..a27cd5ccd380 100644 --- a/source/extensions/filters/http/jwt_authn/jwks_cache.h +++ b/source/extensions/filters/http/jwt_authn/jwks_cache.h @@ -13,7 +13,7 @@ namespace HttpFilters { namespace JwtAuthn { class JwksCache; -typedef std::unique_ptr JwksCachePtr; +using JwksCachePtr = std::unique_ptr; /** * Interface to access all configured Jwt rules and their cached Jwks objects. @@ -35,12 +35,12 @@ typedef std::unique_ptr JwksCachePtr; class JwksCache { public: - virtual ~JwksCache() {} + virtual ~JwksCache() = default; // Interface to access a Jwks config rule and its cached Jwks object. class JwksData { public: - virtual ~JwksData() {} + virtual ~JwksData() = default; // Check if a list of audiences are allowed. virtual bool areAudiencesAllowed(const std::vector& audiences) const PURE; diff --git a/source/extensions/filters/http/jwt_authn/matcher.h b/source/extensions/filters/http/jwt_authn/matcher.h index af7104729bb8..77095382202e 100644 --- a/source/extensions/filters/http/jwt_authn/matcher.h +++ b/source/extensions/filters/http/jwt_authn/matcher.h @@ -9,14 +9,14 @@ namespace HttpFilters { namespace JwtAuthn { class Matcher; -typedef std::unique_ptr MatcherConstPtr; +using MatcherConstPtr = std::unique_ptr; /** * Supports matching a HTTP requests with JWT requirements. */ class Matcher { public: - virtual ~Matcher() {} + virtual ~Matcher() = default; /** * Returns if a HTTP request matches with the rules of the matcher. diff --git a/source/extensions/filters/http/jwt_authn/verifier.h b/source/extensions/filters/http/jwt_authn/verifier.h index d3a9575ad40e..7f6999a9c370 100644 --- a/source/extensions/filters/http/jwt_authn/verifier.h +++ b/source/extensions/filters/http/jwt_authn/verifier.h @@ -8,21 +8,21 @@ namespace HttpFilters { namespace JwtAuthn { class Verifier; -typedef std::unique_ptr VerifierConstPtr; +using VerifierConstPtr = std::unique_ptr; /** * Supports verification of JWTs with configured requirements. */ class Verifier { public: - virtual ~Verifier() {} + virtual ~Verifier() = default; /** * Handle for notifying Verifier callers of request completion. */ class Callbacks { public: - virtual ~Callbacks() {} + virtual ~Callbacks() = default; /** * Successfully verified JWT payload are stored in the struct with its @@ -43,7 +43,7 @@ class Verifier { // Context object to hold data needed for verifier. class Context { public: - virtual ~Context() {} + virtual ~Context() = default; /** * Returns the request headers wrapped in this context. @@ -65,7 +65,7 @@ class Verifier { virtual void cancel() PURE; }; - typedef std::shared_ptr ContextSharedPtr; + using ContextSharedPtr = std::shared_ptr; // Verify all tokens on headers, and signal the caller with callback. virtual void verify(ContextSharedPtr context) const PURE; @@ -81,7 +81,7 @@ class Verifier { static ContextSharedPtr createContext(Http::HeaderMap& headers, Callbacks* callback); }; -typedef std::shared_ptr ContextSharedPtr; +using ContextSharedPtr = std::shared_ptr; } // namespace JwtAuthn } // namespace HttpFilters diff --git a/source/extensions/filters/http/lua/lua_filter.h b/source/extensions/filters/http/lua/lua_filter.h index 84da9e013648..6b562cfbbcb2 100644 --- a/source/extensions/filters/http/lua/lua_filter.h +++ b/source/extensions/filters/http/lua/lua_filter.h @@ -31,7 +31,7 @@ const ProtobufWkt::Struct& getMetadata(Http::StreamFilterCallbacks* callbacks) { */ class FilterCallbacks { public: - virtual ~FilterCallbacks() {} + virtual ~FilterCallbacks() = default; /** * Add data to the connection manager buffer. @@ -305,7 +305,7 @@ class FilterConfig : Logger::Loggable { uint64_t response_function_slot_; }; -typedef std::shared_ptr FilterConfigConstSharedPtr; +using FilterConfigConstSharedPtr = std::shared_ptr; // TODO(mattklein123): Filter stats. @@ -400,7 +400,7 @@ class Filter : public Http::StreamFilter, Logger::Loggable { Http::StreamEncoderFilterCallbacks* callbacks_{}; }; - typedef Filters::Common::Lua::LuaDeathRef StreamHandleRef; + using StreamHandleRef = Filters::Common::Lua::LuaDeathRef; Http::FilterHeadersStatus doHeaders(StreamHandleRef& handle, Filters::Common::Lua::CoroutinePtr& coroutine, diff --git a/source/extensions/filters/http/lua/wrappers.h b/source/extensions/filters/http/lua/wrappers.h index e5bc0041d0d8..1e6d3dcf15a5 100644 --- a/source/extensions/filters/http/lua/wrappers.h +++ b/source/extensions/filters/http/lua/wrappers.h @@ -37,7 +37,7 @@ class HeaderMapIterator : public Filters::Common::Lua::BaseLuaObject { public: - typedef std::function CheckModifiableCb; + using CheckModifiableCb = std::function; HeaderMapWrapper(Http::HeaderMap& headers, CheckModifiableCb cb) : headers_(headers), cb_(cb) {} diff --git a/source/extensions/filters/http/ratelimit/ratelimit.cc b/source/extensions/filters/http/ratelimit/ratelimit.cc index ed267a1fd3e2..2a5761c6b612 100644 --- a/source/extensions/filters/http/ratelimit/ratelimit.cc +++ b/source/extensions/filters/http/ratelimit/ratelimit.cc @@ -23,7 +23,7 @@ struct RcDetailsValues { // The rate limiter encountered a failure, and was configured to fail-closed. const std::string RateLimitError = "rate_limiter_error"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; void Filter::initiateCall(const Http::HeaderMap& headers) { bool is_internal_request = diff --git a/source/extensions/filters/http/ratelimit/ratelimit.h b/source/extensions/filters/http/ratelimit/ratelimit.h index 66c6fc5494c1..1103f9e9763b 100644 --- a/source/extensions/filters/http/ratelimit/ratelimit.h +++ b/source/extensions/filters/http/ratelimit/ratelimit.h @@ -82,7 +82,7 @@ class FilterConfig { Http::Context& http_context_; }; -typedef std::shared_ptr FilterConfigSharedPtr; +using FilterConfigSharedPtr = std::shared_ptr; /** * HTTP rate limit filter. Depending on the route configuration, this filter calls the global diff --git a/source/extensions/filters/http/rbac/rbac_filter.cc b/source/extensions/filters/http/rbac/rbac_filter.cc index ce82d3415b6d..965888fd3105 100644 --- a/source/extensions/filters/http/rbac/rbac_filter.cc +++ b/source/extensions/filters/http/rbac/rbac_filter.cc @@ -17,7 +17,7 @@ struct RcDetailsValues { // The rbac filter rejected the request const std::string RbacAccessDenied = "rbac_access_denied"; }; -typedef ConstSingleton RcDetails; +using RcDetails = ConstSingleton; RoleBasedAccessControlFilterConfig::RoleBasedAccessControlFilterConfig( const envoy::config::filter::http::rbac::v2::RBAC& proto_config, diff --git a/source/extensions/filters/http/rbac/rbac_filter.h b/source/extensions/filters/http/rbac/rbac_filter.h index 50d916814749..5397a3c31c58 100644 --- a/source/extensions/filters/http/rbac/rbac_filter.h +++ b/source/extensions/filters/http/rbac/rbac_filter.h @@ -59,8 +59,8 @@ class RoleBasedAccessControlFilterConfig { const absl::optional shadow_engine_; }; -typedef std::shared_ptr - RoleBasedAccessControlFilterConfigSharedPtr; +using RoleBasedAccessControlFilterConfigSharedPtr = + std::shared_ptr; /** * A filter that provides role-based access control authorization for HTTP requests. diff --git a/source/extensions/filters/http/squash/squash_filter.h b/source/extensions/filters/http/squash/squash_filter.h index b09e23092547..4dc2ea532712 100644 --- a/source/extensions/filters/http/squash/squash_filter.h +++ b/source/extensions/filters/http/squash/squash_filter.h @@ -51,7 +51,7 @@ class SquashFilterConfig : protected Logger::Loggable { const static std::regex ENV_REGEX; }; -typedef std::shared_ptr SquashFilterConfigSharedPtr; +using SquashFilterConfigSharedPtr = std::shared_ptr; class AsyncClientCallbackShim : public Http::AsyncClient::Callbacks { public: diff --git a/source/extensions/filters/http/well_known_names.h b/source/extensions/filters/http/well_known_names.h index ef9e140acc76..d62f3886d5bc 100644 --- a/source/extensions/filters/http/well_known_names.h +++ b/source/extensions/filters/http/well_known_names.h @@ -67,7 +67,7 @@ class HttpFilterNameValues { ExtAuthorization}) {} }; -typedef ConstSingleton HttpFilterNames; +using HttpFilterNames = ConstSingleton; } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.h b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.h index 42bf9161b447..1a4b41cde883 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.h +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.h @@ -39,7 +39,7 @@ class Config { ProxyProtocolStats stats_; }; -typedef std::shared_ptr ConfigSharedPtr; +using ConfigSharedPtr = std::shared_ptr; enum ProxyProtocolVersion { Unknown = -1, InProgress = -2, V1 = 1, V2 = 2 }; diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.h b/source/extensions/filters/listener/tls_inspector/tls_inspector.h index 52d45dcd108a..cf75fe87e954 100644 --- a/source/extensions/filters/listener/tls_inspector/tls_inspector.h +++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.h @@ -55,7 +55,7 @@ class Config { const uint32_t max_client_hello_size_; }; -typedef std::shared_ptr ConfigSharedPtr; +using ConfigSharedPtr = std::shared_ptr; /** * TLS inspector listener filter. diff --git a/source/extensions/filters/listener/well_known_names.h b/source/extensions/filters/listener/well_known_names.h index 7b350351ef2d..f7745ed2bba8 100644 --- a/source/extensions/filters/listener/well_known_names.h +++ b/source/extensions/filters/listener/well_known_names.h @@ -24,7 +24,7 @@ class ListenerFilterNameValues { const std::string TlsInspector = "envoy.listener.tls_inspector"; }; -typedef ConstSingleton ListenerFilterNames; +using ListenerFilterNames = ConstSingleton; } // namespace ListenerFilters } // namespace Extensions diff --git a/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.h b/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.h index 013051253bf7..54ad916fe8dd 100644 --- a/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.h +++ b/source/extensions/filters/network/client_ssl_auth/client_ssl_auth.h @@ -61,10 +61,10 @@ class AllowedPrincipals : public ThreadLocal::ThreadLocalObject { std::unordered_set allowed_sha256_digests_; }; -typedef std::shared_ptr AllowedPrincipalsSharedPtr; +using AllowedPrincipalsSharedPtr = std::shared_ptr; class ClientSslAuthConfig; -typedef std::shared_ptr ClientSslAuthConfigSharedPtr; +using ClientSslAuthConfigSharedPtr = std::shared_ptr; /** * Global configuration for client SSL authentication. The config contacts a JSON API to fetch the diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 24170c127b9c..e6db95d75c25 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -18,7 +18,7 @@ namespace Client { */ class PoolRequest { public: - virtual ~PoolRequest() {} + virtual ~PoolRequest() = default; /** * Cancel the request. No further request callbacks will be called. @@ -31,7 +31,7 @@ class PoolRequest { */ class PoolCallbacks { public: - virtual ~PoolCallbacks() {} + virtual ~PoolCallbacks() = default; /** * Called when a pipelined response is received. @@ -69,7 +69,7 @@ class DoNothingPoolCallbacks : public PoolCallbacks { */ class Client : public Event::DeferredDeletable { public: - virtual ~Client() {} + ~Client() override = default; /** * Adds network connection callbacks to the underlying network connection. @@ -91,14 +91,14 @@ class Client : public Event::DeferredDeletable { virtual PoolRequest* makeRequest(const RespValue& request, PoolCallbacks& callbacks) PURE; }; -typedef std::unique_ptr ClientPtr; +using ClientPtr = std::unique_ptr; /** * Configuration for a redis connection pool. */ class Config { public: - virtual ~Config() {} + virtual ~Config() = default; /** * @return std::chrono::milliseconds the timeout for an individual redis operation. Currently, @@ -141,7 +141,7 @@ class Config { */ class ClientFactory { public: - virtual ~ClientFactory() {} + virtual ~ClientFactory() = default; /** * Create a client given an upstream host. diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 377d54268d83..20a7adcd0cc5 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -30,7 +30,7 @@ struct RedirectionValues { const std::string MOVED = "MOVED"; }; -typedef ConstSingleton RedirectionResponse; +using RedirectionResponse = ConstSingleton; class ConfigImpl : public Config { public: diff --git a/source/extensions/filters/network/common/redis/codec.h b/source/extensions/filters/network/common/redis/codec.h index dda00888c800..47b4b6da2002 100644 --- a/source/extensions/filters/network/common/redis/codec.h +++ b/source/extensions/filters/network/common/redis/codec.h @@ -64,17 +64,17 @@ class RespValue { void cleanup(); - RespType type_; + RespType type_{}; }; -typedef std::unique_ptr RespValuePtr; +using RespValuePtr = std::unique_ptr; /** * Callbacks that the decoder fires. */ class DecoderCallbacks { public: - virtual ~DecoderCallbacks() {} + virtual ~DecoderCallbacks() = default; /** * Called when a new top level RESP value has been decoded. This value may include multiple @@ -89,7 +89,7 @@ class DecoderCallbacks { */ class Decoder { public: - virtual ~Decoder() {} + virtual ~Decoder() = default; /** * Decode redis protocol bytes. @@ -99,14 +99,14 @@ class Decoder { virtual void decode(Buffer::Instance& data) PURE; }; -typedef std::unique_ptr DecoderPtr; +using DecoderPtr = std::unique_ptr; /** * A factory for a redis decoder. */ class DecoderFactory { public: - virtual ~DecoderFactory() {} + virtual ~DecoderFactory() = default; /** * Create a decoder given a set of decoder callbacks. @@ -119,7 +119,7 @@ class DecoderFactory { */ class Encoder { public: - virtual ~Encoder() {} + virtual ~Encoder() = default; /** * Encode a RESP value to a buffer. @@ -129,7 +129,7 @@ class Encoder { virtual void encode(const RespValue& value, Buffer::Instance& out) PURE; }; -typedef std::unique_ptr EncoderPtr; +using EncoderPtr = std::unique_ptr; /** * A redis protocol error. diff --git a/source/extensions/filters/network/dubbo_proxy/BUILD b/source/extensions/filters/network/dubbo_proxy/BUILD index 46159802a947..88c33334bca7 100644 --- a/source/extensions/filters/network/dubbo_proxy/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/BUILD @@ -141,6 +141,7 @@ envoy_cc_library( hdrs = ["decoder_event_handler.h"], deps = [ ":metadata_lib", + "//include/envoy/network:connection_interface", "//include/envoy/network:filter_interface", "//source/common/buffer:buffer_lib", ], diff --git a/source/extensions/filters/network/dubbo_proxy/active_message.h b/source/extensions/filters/network/dubbo_proxy/active_message.h index 8ecda9ea74aa..712651ce9a77 100644 --- a/source/extensions/filters/network/dubbo_proxy/active_message.h +++ b/source/extensions/filters/network/dubbo_proxy/active_message.h @@ -58,7 +58,7 @@ class ResponseDecoder : public DecoderCallbacks, bool complete_ : 1; }; -typedef std::unique_ptr ResponseDecoderPtr; +using ResponseDecoderPtr = std::unique_ptr; // Wraps a DecoderFilter and acts as the DecoderFilterCallbacks for the filter, enabling filter // chain continuation. @@ -90,7 +90,7 @@ class ActiveMessageDecoderFilter : public DubboFilters::DecoderFilterCallbacks, DubboFilters::DecoderFilterSharedPtr handle_; }; -typedef std::unique_ptr ActiveMessageDecoderFilterPtr; +using ActiveMessageDecoderFilterPtr = std::unique_ptr; // ActiveMessage tracks downstream requests for which no response has been received. class ActiveMessage : public LinkedObject, @@ -162,7 +162,7 @@ class ActiveMessage : public LinkedObject, bool local_response_sent_ : 1; }; -typedef std::unique_ptr ActiveMessagePtr; +using ActiveMessagePtr = std::unique_ptr; } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/config.cc b/source/extensions/filters/network/dubbo_proxy/config.cc index 5324025e6c02..1c145d590acd 100644 --- a/source/extensions/filters/network/dubbo_proxy/config.cc +++ b/source/extensions/filters/network/dubbo_proxy/config.cc @@ -34,7 +34,7 @@ REGISTER_FACTORY(DubboProxyFilterConfigFactory, class ProtocolTypeMapper { public: using ConfigProtocolType = envoy::config::filter::network::dubbo_proxy::v2alpha1::ProtocolType; - typedef absl::flat_hash_map ProtocolTypeMap; + using ProtocolTypeMap = absl::flat_hash_map; static ProtocolType lookupProtocolType(ConfigProtocolType config_type) { const auto& iter = protocolTypeMap().find(config_type); @@ -54,7 +54,7 @@ class SerializationTypeMapper { public: using ConfigSerializationType = envoy::config::filter::network::dubbo_proxy::v2alpha1::SerializationType; - typedef absl::flat_hash_map SerializationTypeMap; + using SerializationTypeMap = absl::flat_hash_map; static SerializationType lookupSerializationType(ConfigSerializationType type) { const auto& iter = serializationTypeMap().find(type); diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.cc b/source/extensions/filters/network/dubbo_proxy/decoder.cc index c5be03feabbe..c47433e373d2 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.cc +++ b/source/extensions/filters/network/dubbo_proxy/decoder.cc @@ -124,7 +124,7 @@ ProtocolState DecoderStateMachine::run(Buffer::Instance& buffer) { return state_; } -typedef std::unique_ptr DecoderStateMachinePtr; +using DecoderStateMachinePtr = std::unique_ptr; Decoder::Decoder(Protocol& protocol, Deserializer& deserializer, DecoderCallbacks& decoder_callbacks) diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.h b/source/extensions/filters/network/dubbo_proxy/decoder.h index 71bde4016c66..6a320fb2a411 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.h +++ b/source/extensions/filters/network/dubbo_proxy/decoder.h @@ -108,7 +108,7 @@ class DecoderStateMachine : public Logger::Loggable { DecoderEventHandler* handler_; }; -typedef std::unique_ptr DecoderStateMachinePtr; +using DecoderStateMachinePtr = std::unique_ptr; /** * Decoder encapsulates a configured and ProtocolPtr and SerializationPtr. @@ -140,7 +140,7 @@ class Decoder : public Logger::Loggable { DecoderCallbacks& decoder_callbacks_; }; -typedef std::unique_ptr DecoderPtr; +using DecoderPtr = std::unique_ptr; } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/decoder_event_handler.h b/source/extensions/filters/network/dubbo_proxy/decoder_event_handler.h index 00858ae61a3c..35b2caed3b01 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder_event_handler.h +++ b/source/extensions/filters/network/dubbo_proxy/decoder_event_handler.h @@ -108,7 +108,7 @@ class DecoderCallbacks { virtual void onHeartbeat(MessageMetadataSharedPtr) {} }; -typedef std::shared_ptr DecoderEventHandlerSharedPtr; +using DecoderEventHandlerSharedPtr = std::shared_ptr; } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/deserializer.h b/source/extensions/filters/network/dubbo_proxy/deserializer.h index 2a153ecc23f3..95f2f8e5bc44 100644 --- a/source/extensions/filters/network/dubbo_proxy/deserializer.h +++ b/source/extensions/filters/network/dubbo_proxy/deserializer.h @@ -26,8 +26,8 @@ class DeserializerNameValues { template std::size_t operator()(T t) const { return static_cast(t); } }; - typedef std::unordered_map - DeserializerTypeNameMap; + using DeserializerTypeNameMap = + std::unordered_map; const DeserializerTypeNameMap deserializerTypeNameMap = { {SerializationType::Hessian, "hessian"}, @@ -43,7 +43,7 @@ class DeserializerNameValues { } }; -typedef ConstSingleton DeserializerNames; +using DeserializerNames = ConstSingleton; /** * RpcInvocation represent an rpc call @@ -58,7 +58,7 @@ class RpcInvocation { virtual const std::string& getServiceVersion() const PURE; }; -typedef std::unique_ptr RpcInvocationPtr; +using RpcInvocationPtr = std::unique_ptr; /** * RpcResult represent the result of an rpc call @@ -71,7 +71,7 @@ class RpcResult { virtual bool hasException() const PURE; }; -typedef std::unique_ptr RpcResultPtr; +using RpcResultPtr = std::unique_ptr; class Deserializer { public: @@ -121,7 +121,7 @@ class Deserializer { RpcResponseType type) PURE; }; -typedef std::unique_ptr DeserializerPtr; +using DeserializerPtr = std::unique_ptr; /** * Implemented by each Dubbo deserialize and registered via Registry::registerFactory or the @@ -174,4 +174,4 @@ class DeserializerFactoryBase : public NamedDeserializerConfigFactory { } // namespace DubboProxy } // namespace NetworkFilters } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/extensions/filters/network/dubbo_proxy/filters/filter.h b/source/extensions/filters/network/dubbo_proxy/filters/filter.h index 28e1fed8d37f..5d73624d1a0a 100644 --- a/source/extensions/filters/network/dubbo_proxy/filters/filter.h +++ b/source/extensions/filters/network/dubbo_proxy/filters/filter.h @@ -54,7 +54,7 @@ class DirectResponse { Deserializer& deserializer, Buffer::Instance& buffer) const PURE; }; -typedef std::unique_ptr DirectResponsePtr; +using DirectResponsePtr = std::unique_ptr; /** * Decoder filter callbacks add additional callbacks. @@ -164,7 +164,7 @@ class DecoderFilter : public DecoderEventHandler { virtual void setDecoderFilterCallbacks(DecoderFilterCallbacks& callbacks) PURE; }; -typedef std::shared_ptr DecoderFilterSharedPtr; +using DecoderFilterSharedPtr = std::shared_ptr; /** * These callbacks are provided by the connection manager to the factory so that the factory can @@ -189,7 +189,7 @@ class FilterChainFactoryCallbacks { * function will install a single filter, but it's technically possibly to install more than one * if desired. */ -typedef std::function FilterFactoryCb; +using FilterFactoryCb = std::function; /** * A FilterChainFactory is used by a connection manager to create a Dubbo level filter chain when diff --git a/source/extensions/filters/network/dubbo_proxy/filters/well_known_names.h b/source/extensions/filters/network/dubbo_proxy/filters/well_known_names.h index af4f021ac538..45528e57e373 100644 --- a/source/extensions/filters/network/dubbo_proxy/filters/well_known_names.h +++ b/source/extensions/filters/network/dubbo_proxy/filters/well_known_names.h @@ -20,7 +20,7 @@ class DubboFilterNameValues { const std::string ROUTER = "envoy.filters.dubbo.router"; }; -typedef ConstSingleton DubboFilterNames; +using DubboFilterNames = ConstSingleton; } // namespace DubboFilters } // namespace DubboProxy diff --git a/source/extensions/filters/network/dubbo_proxy/message.h b/source/extensions/filters/network/dubbo_proxy/message.h index 81ecdae6b2f2..30c5673fe309 100644 --- a/source/extensions/filters/network/dubbo_proxy/message.h +++ b/source/extensions/filters/network/dubbo_proxy/message.h @@ -55,7 +55,7 @@ enum class RpcResponseType : uint8_t { class Message { public: - virtual ~Message() {} + virtual ~Message() = default; virtual MessageType messageType() const PURE; virtual int32_t bodySize() const PURE; virtual bool isEvent() const PURE; @@ -65,20 +65,20 @@ class Message { class RequestMessage : public virtual Message { public: - virtual ~RequestMessage() {} + ~RequestMessage() override = default; virtual SerializationType serializationType() const PURE; virtual bool isTwoWay() const PURE; }; -typedef std::unique_ptr RequestMessagePtr; +using RequestMessagePtr = std::unique_ptr; class ResponseMessage : public virtual Message { public: - virtual ~ResponseMessage() {} + ~ResponseMessage() override = default; virtual ResponseStatus responseStatus() const PURE; }; -typedef std::unique_ptr ResponseMessagePtr; +using ResponseMessagePtr = std::unique_ptr; } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/metadata.h b/source/extensions/filters/network/dubbo_proxy/metadata.h index d67cd21a4a6f..a8251c75c021 100644 --- a/source/extensions/filters/network/dubbo_proxy/metadata.h +++ b/source/extensions/filters/network/dubbo_proxy/metadata.h @@ -20,10 +20,10 @@ namespace DubboProxy { class MessageMetadata { public: // TODO(gengleilei) Add parameter data types and implement Dubbo data type mapping. - typedef std::unordered_map ParameterValueMap; - typedef std::unique_ptr ParameterValueMapPtr; + using ParameterValueMap = std::unordered_map; + using ParameterValueMapPtr = std::unique_ptr; - typedef std::unique_ptr HeaderMapPtr; + using HeaderMapPtr = std::unique_ptr; void setServiceName(const std::string& name) { service_name_ = name; } const std::string& service_name() const { return service_name_; } @@ -119,7 +119,7 @@ class MessageMetadata { HeaderMapPtr headers_; // attachment }; -typedef std::shared_ptr MessageMetadataSharedPtr; +using MessageMetadataSharedPtr = std::shared_ptr; } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/protocol.h b/source/extensions/filters/network/dubbo_proxy/protocol.h index 26ae6f9c58d3..85efff09249d 100644 --- a/source/extensions/filters/network/dubbo_proxy/protocol.h +++ b/source/extensions/filters/network/dubbo_proxy/protocol.h @@ -34,7 +34,7 @@ class ProtocolNameValues { template std::size_t operator()(T t) const { return static_cast(t); } }; - typedef std::unordered_map ProtocolTypeNameMap; + using ProtocolTypeNameMap = std::unordered_map; const ProtocolTypeNameMap protocolTypeNameMap = { {ProtocolType::Dubbo, "dubbo"}, @@ -50,7 +50,7 @@ class ProtocolNameValues { } }; -typedef ConstSingleton ProtocolNames; +using ProtocolNames = ConstSingleton; /** * ProtocolCallbacks are Dubbo protocol-level callbacks. @@ -107,7 +107,7 @@ class Protocol { const MessageMetadata& metadata) PURE; }; -typedef std::unique_ptr ProtocolPtr; +using ProtocolPtr = std::unique_ptr; /** * Implemented by each Dubbo protocol and registered via Registry::registerFactory or the diff --git a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h index 0cb6a2724173..cf01e91c9e28 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h +++ b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h @@ -72,7 +72,7 @@ class RouteEntryImplBase : public RouteEntry, Envoy::Router::MetadataMatchCriteriaConstPtr metadata_match_criteria_; }; - typedef std::shared_ptr WeightedClusterEntrySharedPtr; + using WeightedClusterEntrySharedPtr = std::shared_ptr; uint64_t total_cluster_weight_; const std::string cluster_name_; @@ -83,7 +83,7 @@ class RouteEntryImplBase : public RouteEntry, Envoy::Router::MetadataMatchCriteriaConstPtr metadata_match_criteria_; }; -typedef std::shared_ptr RouteEntryImplBaseConstSharedPtr; +using RouteEntryImplBaseConstSharedPtr = std::shared_ptr; class ParameterRouteEntryImpl : public RouteEntryImplBase { public: @@ -140,8 +140,8 @@ class RouteMatcher : public Logger::Loggable { const absl::optional version_; }; -typedef std::shared_ptr RouteMatcherConstSharedPtr; -typedef std::unique_ptr RouteMatcherPtr; +using RouteMatcherConstSharedPtr = std::shared_ptr; +using RouteMatcherPtr = std::unique_ptr; class MultiRouteMatcher : public Logger::Loggable { public: diff --git a/source/extensions/filters/network/dubbo_proxy/router/router.h b/source/extensions/filters/network/dubbo_proxy/router/router.h index 8dd0292c98b2..790abe825d5e 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/router.h +++ b/source/extensions/filters/network/dubbo_proxy/router/router.h @@ -45,7 +45,7 @@ class Route { virtual const RouteEntry* routeEntry() const PURE; }; -typedef std::shared_ptr RouteConstSharedPtr; +using RouteConstSharedPtr = std::shared_ptr; /** * The router configuration. @@ -65,7 +65,7 @@ class Config { uint64_t random_value) const PURE; }; -typedef std::shared_ptr ConfigConstSharedPtr; +using ConfigConstSharedPtr = std::shared_ptr; } // namespace Router } // namespace DubboProxy diff --git a/source/extensions/filters/network/ext_authz/ext_authz.h b/source/extensions/filters/network/ext_authz/ext_authz.h index 0283e1d6364e..b259c62f2f3d 100644 --- a/source/extensions/filters/network/ext_authz/ext_authz.h +++ b/source/extensions/filters/network/ext_authz/ext_authz.h @@ -59,7 +59,7 @@ class Config { bool failure_mode_allow_; }; -typedef std::shared_ptr ConfigSharedPtr; +using ConfigSharedPtr = std::shared_ptr; /** * ExtAuthz filter instance. This filter will call the Authorization service with the given diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 16a4a6d43c0b..07c604ace6b6 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -30,8 +30,8 @@ namespace NetworkFilters { namespace HttpConnectionManager { namespace { -typedef std::list FilterFactoriesList; -typedef std::map FilterFactoryMap; +using FilterFactoriesList = std::list; +using FilterFactoryMap = std::map; HttpConnectionManagerConfig::UpgradeMap::const_iterator findUpgradeBoolCaseInsensitive(const HttpConnectionManagerConfig::UpgradeMap& upgrade_map, diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index ca858adcabe6..796b67fc9728 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -86,7 +86,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, // Http::FilterChainFactory void createFilterChain(Http::FilterChainFactoryCallbacks& callbacks) override; - typedef std::list FilterFactoriesList; + using FilterFactoriesList = std::list; struct FilterConfig { std::unique_ptr filter_factories; bool allow_upgrade; diff --git a/source/extensions/filters/network/kafka/kafka_request.h b/source/extensions/filters/network/kafka/kafka_request.h index 258012cfb1ec..0c21543cc214 100644 --- a/source/extensions/filters/network/kafka/kafka_request.h +++ b/source/extensions/filters/network/kafka/kafka_request.h @@ -39,7 +39,7 @@ class RequestParseFailure { const RequestHeader request_header_; }; -typedef std::shared_ptr RequestParseFailureSharedPtr; +using RequestParseFailureSharedPtr = std::shared_ptr; /** * Abstract Kafka request. @@ -74,7 +74,7 @@ class AbstractRequest { const RequestHeader request_header_; }; -typedef std::shared_ptr AbstractRequestSharedPtr; +using AbstractRequestSharedPtr = std::shared_ptr; /** * Concrete request that carries data particular to given request type. diff --git a/source/extensions/filters/network/kafka/kafka_request_parser.h b/source/extensions/filters/network/kafka/kafka_request_parser.h index 861d4dc4a3a9..3171c5e46197 100644 --- a/source/extensions/filters/network/kafka/kafka_request_parser.h +++ b/source/extensions/filters/network/kafka/kafka_request_parser.h @@ -26,7 +26,7 @@ struct RequestContext { RequestHeader request_header_{}; }; -typedef std::shared_ptr RequestContextSharedPtr; +using RequestContextSharedPtr = std::shared_ptr; /** * Request decoder configuration object. @@ -86,7 +86,7 @@ class RequestHeaderDeserializer Int16Deserializer, Int32Deserializer, NullableStringDeserializer> {}; -typedef std::unique_ptr RequestHeaderDeserializerPtr; +using RequestHeaderDeserializerPtr = std::unique_ptr; /** * Parser responsible for extracting the request header and putting it into context. diff --git a/source/extensions/filters/network/kafka/kafka_types.h b/source/extensions/filters/network/kafka/kafka_types.h index 71d1ce920a82..3240b9a9c2d6 100644 --- a/source/extensions/filters/network/kafka/kafka_types.h +++ b/source/extensions/filters/network/kafka/kafka_types.h @@ -14,17 +14,17 @@ namespace Kafka { /** * Nullable string used by Kafka. */ -typedef absl::optional NullableString; +using NullableString = absl::optional; /** * Bytes array used by Kafka. */ -typedef std::vector Bytes; +using Bytes = std::vector; /** * Nullable bytes array used by Kafka. */ -typedef absl::optional NullableBytes; +using NullableBytes = absl::optional; /** * Kafka array of elements of type T. diff --git a/source/extensions/filters/network/kafka/request_codec.h b/source/extensions/filters/network/kafka/request_codec.h index c8a6b69f8797..b33befb6ad9d 100644 --- a/source/extensions/filters/network/kafka/request_codec.h +++ b/source/extensions/filters/network/kafka/request_codec.h @@ -33,7 +33,7 @@ class RequestCallback { virtual void onFailedParse(RequestParseFailureSharedPtr failure_data) PURE; }; -typedef std::shared_ptr RequestCallbackSharedPtr; +using RequestCallbackSharedPtr = std::shared_ptr; /** * Provides initial parser for messages (class extracted to allow injecting test factories). diff --git a/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 b/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 index 1b52340a6cda..04e1a55beb85 100644 --- a/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 +++ b/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 @@ -26,7 +26,7 @@ struct CompositeResultWith0Fields { bool operator==(const CompositeResultWith0Fields&) const { return true; } }; -typedef CompositeDeserializerWith0Delegates TestCompositeDeserializer0; +using TestCompositeDeserializer0 = CompositeDeserializerWith0Delegates; // Composite with 0 delegates is special case: it's always ready. TEST(CompositeDeserializerWith0Delegates, EmptyBufferShouldBeReady) { diff --git a/source/extensions/filters/network/mongo_proxy/bson.h b/source/extensions/filters/network/mongo_proxy/bson.h index 12080e09284c..3098838fe254 100644 --- a/source/extensions/filters/network/mongo_proxy/bson.h +++ b/source/extensions/filters/network/mongo_proxy/bson.h @@ -18,7 +18,7 @@ namespace Bson { * Implementation of http://bsonspec.org/spec.html */ class Document; -typedef std::shared_ptr DocumentSharedPtr; +using DocumentSharedPtr = std::shared_ptr; /** * A BSON document field. This is essentially a variably typed parameter that can be "cast" to @@ -49,7 +49,7 @@ class Field { /** * 12 byte ObjectId type. */ - typedef std::array ObjectId; + using ObjectId = std::array; /** * Regex type. @@ -63,7 +63,7 @@ class Field { std::string options_; }; - virtual ~Field() {} + virtual ~Field() = default; virtual double asDouble() const PURE; virtual const std::string& asString() const PURE; @@ -87,14 +87,14 @@ class Field { virtual Type type() const PURE; }; -typedef std::unique_ptr FieldPtr; +using FieldPtr = std::unique_ptr; /** * A BSON document. add*() is used to add strongly typed fields. */ class Document { public: - virtual ~Document() {} + virtual ~Document() = default; virtual DocumentSharedPtr addDouble(const std::string& key, double value) PURE; virtual DocumentSharedPtr addString(const std::string& key, std::string&& value) PURE; diff --git a/source/extensions/filters/network/mongo_proxy/codec.h b/source/extensions/filters/network/mongo_proxy/codec.h index 55f37c9e354e..a6a5508d7f1f 100644 --- a/source/extensions/filters/network/mongo_proxy/codec.h +++ b/source/extensions/filters/network/mongo_proxy/codec.h @@ -34,7 +34,7 @@ class Message { OP_COMMANDREPLY = 2011 }; - virtual ~Message(){}; + virtual ~Message() = default; virtual int32_t requestId() const PURE; virtual int32_t responseTo() const PURE; @@ -62,7 +62,7 @@ class GetMoreMessage : public virtual Message { virtual void cursorId(int64_t cursor_id) PURE; }; -typedef std::unique_ptr GetMoreMessagePtr; +using GetMoreMessagePtr = std::unique_ptr; /** * Mongo OP_INSERT message. @@ -79,7 +79,7 @@ class InsertMessage : public virtual Message { virtual std::list& documents() PURE; }; -typedef std::unique_ptr InsertMessagePtr; +using InsertMessagePtr = std::unique_ptr; /** * Mongo OP_KILL_CURSORS message. @@ -94,7 +94,7 @@ class KillCursorsMessage : public virtual Message { virtual void cursorIds(std::vector&& cursors_ids) PURE; }; -typedef std::unique_ptr KillCursorsMessagePtr; +using KillCursorsMessagePtr = std::unique_ptr; /** * Mongo OP_QUERY message. @@ -126,7 +126,7 @@ class QueryMessage : public virtual Message { virtual void returnFieldsSelector(Bson::DocumentSharedPtr&& fields) PURE; }; -typedef std::unique_ptr QueryMessagePtr; +using QueryMessagePtr = std::unique_ptr; /** * Mongo OP_REPLY @@ -154,7 +154,7 @@ class ReplyMessage : public virtual Message { virtual std::list& documents() PURE; }; -typedef std::unique_ptr ReplyMessagePtr; +using ReplyMessagePtr = std::unique_ptr; class CommandMessage : public virtual Message { public: @@ -172,7 +172,7 @@ class CommandMessage : public virtual Message { virtual std::list& inputDocs() PURE; }; -typedef std::unique_ptr CommandMessagePtr; +using CommandMessagePtr = std::unique_ptr; class CommandReplyMessage : public virtual Message { public: @@ -185,14 +185,14 @@ class CommandReplyMessage : public virtual Message { virtual std::list& outputDocs() PURE; }; -typedef std::unique_ptr CommandReplyMessagePtr; +using CommandReplyMessagePtr = std::unique_ptr; /** * General callbacks for dispatching decoded mongo messages to a sink. */ class DecoderCallbacks { public: - virtual ~DecoderCallbacks() {} + virtual ~DecoderCallbacks() = default; virtual void decodeGetMore(GetMoreMessagePtr&& message) PURE; virtual void decodeInsert(InsertMessagePtr&& message) PURE; @@ -208,19 +208,19 @@ class DecoderCallbacks { */ class Decoder { public: - virtual ~Decoder() {} + virtual ~Decoder() = default; virtual void onData(Buffer::Instance& data) PURE; }; -typedef std::unique_ptr DecoderPtr; +using DecoderPtr = std::unique_ptr; /** * Mongo message encoder. */ class Encoder { public: - virtual ~Encoder() {} + virtual ~Encoder() = default; virtual void encodeGetMore(const GetMoreMessage& message) PURE; virtual void encodeInsert(const InsertMessage& message) PURE; diff --git a/source/extensions/filters/network/mongo_proxy/proxy.cc b/source/extensions/filters/network/mongo_proxy/proxy.cc index 59621b337928..be08a04fc638 100644 --- a/source/extensions/filters/network/mongo_proxy/proxy.cc +++ b/source/extensions/filters/network/mongo_proxy/proxy.cc @@ -33,7 +33,7 @@ class DynamicMetadataKeys { const std::string OperationDelete{"delete"}; }; -typedef ConstSingleton DynamicMetadataKeysSingleton; +using DynamicMetadataKeysSingleton = ConstSingleton; AccessLog::AccessLog(const std::string& file_name, Envoy::AccessLog::AccessLogManager& log_manager, TimeSource& time_source) diff --git a/source/extensions/filters/network/mongo_proxy/proxy.h b/source/extensions/filters/network/mongo_proxy/proxy.h index f744d3da3981..35cdc29c17d1 100644 --- a/source/extensions/filters/network/mongo_proxy/proxy.h +++ b/source/extensions/filters/network/mongo_proxy/proxy.h @@ -42,7 +42,7 @@ class MongoRuntimeConfigKeys { const std::string DrainCloseEnabled{"mongo.drain_close_enabled"}; }; -typedef ConstSingleton MongoRuntimeConfig; +using MongoRuntimeConfig = ConstSingleton; /** * All mongo proxy stats. @see stats_macros.h @@ -95,7 +95,7 @@ class AccessLog { Envoy::AccessLog::AccessLogFileSharedPtr file_; }; -typedef std::shared_ptr AccessLogSharedPtr; +using AccessLogSharedPtr = std::shared_ptr; /** * A sniffing filter for mongo traffic. The current implementation makes a copy of read/written @@ -156,7 +156,7 @@ class ProxyFilter : public Network::Filter, MonotonicTime start_time_; }; - typedef std::unique_ptr ActiveQueryPtr; + using ActiveQueryPtr = std::unique_ptr; MongoProxyStats generateStats(const std::string& prefix, Stats::Scope& scope) { return MongoProxyStats{ALL_MONGO_PROXY_STATS(POOL_COUNTER_PREFIX(scope, prefix), diff --git a/source/extensions/filters/network/mysql_proxy/mysql_codec.h b/source/extensions/filters/network/mysql_proxy/mysql_codec.h index d11a509ecc36..3d03cb1d03ea 100644 --- a/source/extensions/filters/network/mysql_proxy/mysql_codec.h +++ b/source/extensions/filters/network/mysql_proxy/mysql_codec.h @@ -107,7 +107,7 @@ class MySQLCodec : public Logger::Loggable { uint32_t bits_; }; - virtual ~MySQLCodec() {} + virtual ~MySQLCodec() = default; int decode(Buffer::Instance& data, uint8_t seq, uint32_t len) { seq_ = seq; diff --git a/source/extensions/filters/network/mysql_proxy/mysql_decoder.h b/source/extensions/filters/network/mysql_proxy/mysql_decoder.h index 3aa9a7224fa2..ff11a613f87b 100644 --- a/source/extensions/filters/network/mysql_proxy/mysql_decoder.h +++ b/source/extensions/filters/network/mysql_proxy/mysql_decoder.h @@ -23,7 +23,7 @@ namespace MySQLProxy { */ class DecoderCallbacks { public: - virtual ~DecoderCallbacks() {} + virtual ~DecoderCallbacks() = default; virtual void onProtocolError() PURE; virtual void onNewMessage(MySQLSession::State) PURE; @@ -41,13 +41,13 @@ class DecoderCallbacks { */ class Decoder { public: - virtual ~Decoder() {} + virtual ~Decoder() = default; virtual void onData(Buffer::Instance& data) PURE; virtual MySQLSession& getSession() PURE; }; -typedef std::unique_ptr DecoderPtr; +using DecoderPtr = std::unique_ptr; class DecoderImpl : public Decoder, Logger::Loggable { public: diff --git a/source/extensions/filters/network/ratelimit/ratelimit.h b/source/extensions/filters/network/ratelimit/ratelimit.h index dbf7a309bc06..9757c72865ae 100644 --- a/source/extensions/filters/network/ratelimit/ratelimit.h +++ b/source/extensions/filters/network/ratelimit/ratelimit.h @@ -62,7 +62,7 @@ class Config { const bool failure_mode_deny_; }; -typedef std::shared_ptr ConfigSharedPtr; +using ConfigSharedPtr = std::shared_ptr; /** * TCP rate limit filter instance. This filter will call the rate limit service with the given diff --git a/source/extensions/filters/network/rbac/rbac_filter.h b/source/extensions/filters/network/rbac/rbac_filter.h index bbb09708b787..8d9402e99680 100644 --- a/source/extensions/filters/network/rbac/rbac_filter.h +++ b/source/extensions/filters/network/rbac/rbac_filter.h @@ -43,8 +43,8 @@ class RoleBasedAccessControlFilterConfig { const envoy::config::filter::network::rbac::v2::RBAC::EnforcementType enforcement_type_; }; -typedef std::shared_ptr - RoleBasedAccessControlFilterConfigSharedPtr; +using RoleBasedAccessControlFilterConfigSharedPtr = + std::shared_ptr; /** * Implementation of a basic RBAC network filter. diff --git a/source/extensions/filters/network/redis_proxy/command_splitter.h b/source/extensions/filters/network/redis_proxy/command_splitter.h index 53bb929a2dd8..5e1248d3b500 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter.h +++ b/source/extensions/filters/network/redis_proxy/command_splitter.h @@ -25,7 +25,7 @@ class SplitRequest { virtual void cancel() PURE; }; -typedef std::unique_ptr SplitRequestPtr; +using SplitRequestPtr = std::unique_ptr; /** * Split request callbacks. diff --git a/source/extensions/filters/network/redis_proxy/command_splitter_impl.h b/source/extensions/filters/network/redis_proxy/command_splitter_impl.h index 6c2233fdc964..e530fe8d3fdf 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter_impl.h +++ b/source/extensions/filters/network/redis_proxy/command_splitter_impl.h @@ -61,7 +61,7 @@ struct CommandStats { class CommandHandler { public: - virtual ~CommandHandler() {} + virtual ~CommandHandler() = default; virtual SplitRequestPtr startRequest(Common::Redis::RespValuePtr&& request, SplitCallbacks& callbacks, CommandStats& command_stats, @@ -314,7 +314,7 @@ class InstanceImpl : public Instance, Logger::Loggable { std::reference_wrapper handler_; }; - typedef std::shared_ptr HandlerDataPtr; + using HandlerDataPtr = std::shared_ptr; void addHandler(Stats::Scope& scope, const std::string& stat_prefix, const std::string& name, CommandHandler& handler); diff --git a/source/extensions/filters/network/redis_proxy/conn_pool.h b/source/extensions/filters/network/redis_proxy/conn_pool.h index a926f568f062..937b35ddc36f 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool.h @@ -21,7 +21,7 @@ namespace ConnPool { */ class Instance { public: - virtual ~Instance() {} + virtual ~Instance() = default; /** * Makes a redis request. @@ -50,7 +50,7 @@ class Instance { Common::Redis::Client::PoolCallbacks& callbacks) PURE; }; -typedef std::shared_ptr InstanceSharedPtr; +using InstanceSharedPtr = std::shared_ptr; } // namespace ConnPool } // namespace RedisProxy diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index fbc8f0a6eac8..34f86c5bb78e 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -71,7 +71,7 @@ class InstanceImpl : public Instance { Common::Redis::Client::ClientPtr redis_client_; }; - typedef std::unique_ptr ThreadLocalActiveClientPtr; + using ThreadLocalActiveClientPtr = std::unique_ptr; struct ThreadLocalPool : public ThreadLocal::ThreadLocalObject, public Upstream::ClusterUpdateCallbacks { diff --git a/source/extensions/filters/network/redis_proxy/proxy_filter.h b/source/extensions/filters/network/redis_proxy/proxy_filter.h index dc00a43a7c07..d916a97f6fad 100644 --- a/source/extensions/filters/network/redis_proxy/proxy_filter.h +++ b/source/extensions/filters/network/redis_proxy/proxy_filter.h @@ -63,7 +63,7 @@ class ProxyFilterConfig { static ProxyStats generateStats(const std::string& prefix, Stats::Scope& scope); }; -typedef std::shared_ptr ProxyFilterConfigSharedPtr; +using ProxyFilterConfigSharedPtr = std::shared_ptr; /** * A redis multiplexing proxy filter. This filter will take incoming redis pipelined commands, and diff --git a/source/extensions/filters/network/redis_proxy/router.h b/source/extensions/filters/network/redis_proxy/router.h index d2072ab1b995..337261945beb 100644 --- a/source/extensions/filters/network/redis_proxy/router.h +++ b/source/extensions/filters/network/redis_proxy/router.h @@ -35,9 +35,9 @@ class MirrorPolicy { virtual bool shouldMirror(const std::string& command) const PURE; }; -typedef std::shared_ptr MirrorPolicyConstSharedPtr; +using MirrorPolicyConstSharedPtr = std::shared_ptr; -typedef std::vector MirrorPolicies; +using MirrorPolicies = std::vector; /** * An resolved route that wraps an upstream connection pool and list of mirror policies @@ -51,7 +51,7 @@ class Route { virtual const MirrorPolicies& mirrorPolicies() const PURE; }; -typedef std::shared_ptr RouteSharedPtr; +using RouteSharedPtr = std::shared_ptr; /* * Decorator of a connection pool in order to enable key based routing. @@ -69,7 +69,7 @@ class Router { virtual RouteSharedPtr upstreamPool(std::string& key) PURE; }; -typedef std::unique_ptr RouterPtr; +using RouterPtr = std::unique_ptr; } // namespace RedisProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/redis_proxy/router_impl.h b/source/extensions/filters/network/redis_proxy/router_impl.h index b4811aff1bf7..26963adb1898 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.h +++ b/source/extensions/filters/network/redis_proxy/router_impl.h @@ -23,7 +23,7 @@ namespace Extensions { namespace NetworkFilters { namespace RedisProxy { -typedef std::map Upstreams; +using Upstreams = std::map; class MirrorPolicyImpl : public MirrorPolicy { public: @@ -61,7 +61,7 @@ class Prefix : public Route { MirrorPolicies mirror_policies_; }; -typedef std::shared_ptr PrefixSharedPtr; +using PrefixSharedPtr = std::shared_ptr; class PrefixRoutes : public Router { public: diff --git a/source/extensions/filters/network/thrift_proxy/config.cc b/source/extensions/filters/network/thrift_proxy/config.cc index 4917ffb3c51d..845fadc92e71 100644 --- a/source/extensions/filters/network/thrift_proxy/config.cc +++ b/source/extensions/filters/network/thrift_proxy/config.cc @@ -25,9 +25,8 @@ namespace NetworkFilters { namespace ThriftProxy { namespace { -typedef std::map - TransportTypeMap; +using TransportTypeMap = + std::map; static const TransportTypeMap& transportTypeMap() { CONSTRUCT_ON_FIRST_USE( @@ -44,8 +43,8 @@ static const TransportTypeMap& transportTypeMap() { }); } -typedef std::map - ProtocolTypeMap; +using ProtocolTypeMap = + std::map; static const ProtocolTypeMap& protocolTypeMap() { CONSTRUCT_ON_FIRST_USE( diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.h b/source/extensions/filters/network/thrift_proxy/conn_manager.h index 0cea4a628153..afdcbcc8ff34 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.h +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.h @@ -31,7 +31,7 @@ namespace ThriftProxy { */ class Config { public: - virtual ~Config() {} + virtual ~Config() = default; virtual ThriftFilters::FilterChainFactory& filterFactory() PURE; virtual ThriftFilterStats& stats() PURE; @@ -45,7 +45,7 @@ class Config { */ class ProtocolOptionsConfig : public Upstream::ProtocolOptionsConfig { public: - virtual ~ProtocolOptionsConfig() {} + ~ProtocolOptionsConfig() override = default; virtual TransportType transport(TransportType downstream_transport) const PURE; virtual ProtocolType protocol(ProtocolType downstream_protocol) const PURE; @@ -109,7 +109,7 @@ class ConnectionManager : public Network::ReadFilter, bool complete_ : 1; bool first_reply_field_ : 1; }; - typedef std::unique_ptr ResponseDecoderPtr; + using ResponseDecoderPtr = std::unique_ptr; // Wraps a DecoderFilter and acts as the DecoderFilterCallbacks for the filter, enabling filter // chain continuation. @@ -144,7 +144,7 @@ class ConnectionManager : public Network::ReadFilter, ActiveRpc& parent_; ThriftFilters::DecoderFilterSharedPtr handle_; }; - typedef std::unique_ptr ActiveRpcDecoderFilterPtr; + using ActiveRpcDecoderFilterPtr = std::unique_ptr; // ActiveRpc tracks request/response pairs. struct ActiveRpc : LinkedObject, @@ -246,7 +246,7 @@ class ConnectionManager : public Network::ReadFilter, bool pending_transport_end_ : 1; }; - typedef std::unique_ptr ActiveRpcPtr; + using ActiveRpcPtr = std::unique_ptr; void continueDecoding(); void dispatch(); diff --git a/source/extensions/filters/network/thrift_proxy/decoder.h b/source/extensions/filters/network/thrift_proxy/decoder.h index e2886aedebd6..7e5678c556b2 100644 --- a/source/extensions/filters/network/thrift_proxy/decoder.h +++ b/source/extensions/filters/network/thrift_proxy/decoder.h @@ -169,11 +169,11 @@ class DecoderStateMachine : public Logger::Loggable { std::vector stack_; }; -typedef std::unique_ptr DecoderStateMachinePtr; +using DecoderStateMachinePtr = std::unique_ptr; class DecoderCallbacks { public: - virtual ~DecoderCallbacks() {} + virtual ~DecoderCallbacks() = default; /** * @return DecoderEventHandler& a new DecoderEventHandler for a message. @@ -209,7 +209,7 @@ class Decoder : public Logger::Loggable { DecoderEventHandler& handler_; }; - typedef std::unique_ptr ActiveRequestPtr; + using ActiveRequestPtr = std::unique_ptr; void complete(); @@ -223,7 +223,7 @@ class Decoder : public Logger::Loggable { bool frame_ended_{false}; }; -typedef std::unique_ptr DecoderPtr; +using DecoderPtr = std::unique_ptr; } // namespace ThriftProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/thrift_proxy/decoder_events.h b/source/extensions/filters/network/thrift_proxy/decoder_events.h index 7f40b187e24b..c69db94c0d26 100644 --- a/source/extensions/filters/network/thrift_proxy/decoder_events.h +++ b/source/extensions/filters/network/thrift_proxy/decoder_events.h @@ -19,7 +19,7 @@ enum class FilterStatus { class DecoderEventHandler { public: - virtual ~DecoderEventHandler() {} + virtual ~DecoderEventHandler() = default; /** * Indicates the start of a Thrift transport frame was detected. Unframed transports generate @@ -134,7 +134,7 @@ class DecoderEventHandler { virtual FilterStatus setEnd() PURE; }; -typedef std::shared_ptr DecoderEventHandlerSharedPtr; +using DecoderEventHandlerSharedPtr = std::shared_ptr; } // namespace ThriftProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/thrift_proxy/filters/filter.h b/source/extensions/filters/network/thrift_proxy/filters/filter.h index 142d22e3f8d8..c2ef1a895061 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/filter.h +++ b/source/extensions/filters/network/thrift_proxy/filters/filter.h @@ -31,7 +31,7 @@ enum class ResponseStatus { */ class DecoderFilterCallbacks { public: - virtual ~DecoderFilterCallbacks() {} + virtual ~DecoderFilterCallbacks() = default; /** * @return uint64_t the ID of the originating stream for logging purposes. @@ -105,7 +105,7 @@ class DecoderFilterCallbacks { */ class DecoderFilter : public virtual DecoderEventHandler { public: - virtual ~DecoderFilter() {} + ~DecoderFilter() override = default; /** * This routine is called prior to a filter being destroyed. This may happen after normal stream @@ -125,7 +125,7 @@ class DecoderFilter : public virtual DecoderEventHandler { virtual void setDecoderFilterCallbacks(DecoderFilterCallbacks& callbacks) PURE; }; -typedef std::shared_ptr DecoderFilterSharedPtr; +using DecoderFilterSharedPtr = std::shared_ptr; /** * These callbacks are provided by the connection manager to the factory so that the factory can @@ -133,7 +133,7 @@ typedef std::shared_ptr DecoderFilterSharedPtr; */ class FilterChainFactoryCallbacks { public: - virtual ~FilterChainFactoryCallbacks() {} + virtual ~FilterChainFactoryCallbacks() = default; /** * Add a decoder filter that is used when reading connection data. @@ -150,7 +150,7 @@ class FilterChainFactoryCallbacks { * function will install a single filter, but it's technically possibly to install more than one * if desired. */ -typedef std::function FilterFactoryCb; +using FilterFactoryCb = std::function; /** * A FilterChainFactory is used by a connection manager to create a Thrift level filter chain when @@ -160,7 +160,7 @@ typedef std::function FilterFactor */ class FilterChainFactory { public: - virtual ~FilterChainFactory() {} + virtual ~FilterChainFactory() = default; /** * Called when a new Thrift stream is created on the connection. diff --git a/source/extensions/filters/network/thrift_proxy/filters/filter_config.h b/source/extensions/filters/network/thrift_proxy/filters/filter_config.h index 86f4b7730517..caffd9f76936 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/filter_config.h +++ b/source/extensions/filters/network/thrift_proxy/filters/filter_config.h @@ -19,7 +19,7 @@ namespace ThriftFilters { */ class NamedThriftFilterConfigFactory { public: - virtual ~NamedThriftFilterConfigFactory() {} + virtual ~NamedThriftFilterConfigFactory() = default; /** * Create a particular thrift filter factory implementation. If the implementation is unable to diff --git a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.h b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.h index 797b1db089fe..eb9e60dbb5f0 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.h +++ b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.h @@ -49,7 +49,7 @@ class Config { const bool failure_mode_deny_; }; -typedef std::shared_ptr ConfigSharedPtr; +using ConfigSharedPtr = std::shared_ptr; /** * Thrift rate limit filter instance. Calls the rate limit service with the given configuration @@ -63,79 +63,65 @@ class Filter : public ThriftProxy::ThriftFilters::DecoderFilter, public Filters::Common::RateLimit::RequestCallbacks { public: Filter(ConfigSharedPtr config, Filters::Common::RateLimit::ClientPtr&& client) - : config_(config), client_(std::move(client)) {} - virtual ~Filter() {} + : config_(std::move(config)), client_(std::move(client)) {} + ~Filter() override = default; // ThriftFilters::ThriftDecoderFilter - virtual void onDestroy() override; - virtual void setDecoderFilterCallbacks( + void onDestroy() override; + void setDecoderFilterCallbacks( ThriftProxy::ThriftFilters::DecoderFilterCallbacks& callbacks) override { callbacks_ = &callbacks; }; - virtual ThriftProxy::FilterStatus + ThriftProxy::FilterStatus transportBegin(NetworkFilters::ThriftProxy::MessageMetadataSharedPtr) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus transportEnd() override { + ThriftProxy::FilterStatus transportEnd() override { return ThriftProxy::FilterStatus::Continue; } + ThriftProxy::FilterStatus messageBegin(ThriftProxy::MessageMetadataSharedPtr) override; + ThriftProxy::FilterStatus messageEnd() override { return ThriftProxy::FilterStatus::Continue; } + ThriftProxy::FilterStatus structBegin(absl::string_view) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus messageBegin(ThriftProxy::MessageMetadataSharedPtr) override; - virtual ThriftProxy::FilterStatus messageEnd() override { + ThriftProxy::FilterStatus structEnd() override { return ThriftProxy::FilterStatus::Continue; } + ThriftProxy::FilterStatus fieldBegin(absl::string_view, ThriftProxy::FieldType&, + int16_t&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus structBegin(absl::string_view) override { + ThriftProxy::FilterStatus fieldEnd() override { return ThriftProxy::FilterStatus::Continue; } + ThriftProxy::FilterStatus boolValue(bool&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus structEnd() override { + ThriftProxy::FilterStatus byteValue(uint8_t&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus fieldBegin(absl::string_view, ThriftProxy::FieldType&, - int16_t&) override { + ThriftProxy::FilterStatus int16Value(int16_t&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus fieldEnd() override { + ThriftProxy::FilterStatus int32Value(int32_t&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus boolValue(bool&) override { + ThriftProxy::FilterStatus int64Value(int64_t&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus byteValue(uint8_t&) override { + ThriftProxy::FilterStatus doubleValue(double&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus int16Value(int16_t&) override { + ThriftProxy::FilterStatus stringValue(absl::string_view) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus int32Value(int32_t&) override { + ThriftProxy::FilterStatus mapBegin(ThriftProxy::FieldType&, ThriftProxy::FieldType&, + uint32_t&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus int64Value(int64_t&) override { + ThriftProxy::FilterStatus mapEnd() override { return ThriftProxy::FilterStatus::Continue; } + ThriftProxy::FilterStatus listBegin(ThriftProxy::FieldType&, uint32_t&) override { return ThriftProxy::FilterStatus::Continue; } - virtual ThriftProxy::FilterStatus doubleValue(double&) override { - return ThriftProxy::FilterStatus::Continue; - } - virtual ThriftProxy::FilterStatus stringValue(absl::string_view) override { - return ThriftProxy::FilterStatus::Continue; - } - virtual ThriftProxy::FilterStatus mapBegin(ThriftProxy::FieldType&, ThriftProxy::FieldType&, - uint32_t&) override { - return ThriftProxy::FilterStatus::Continue; - } - virtual ThriftProxy::FilterStatus mapEnd() override { - return ThriftProxy::FilterStatus::Continue; - } - virtual ThriftProxy::FilterStatus listBegin(ThriftProxy::FieldType&, uint32_t&) override { - return ThriftProxy::FilterStatus::Continue; - } - virtual ThriftProxy::FilterStatus listEnd() override { - return ThriftProxy::FilterStatus::Continue; - } - virtual ThriftProxy::FilterStatus setBegin(ThriftProxy::FieldType&, uint32_t&) override { - return ThriftProxy::FilterStatus::Continue; - } - virtual ThriftProxy::FilterStatus setEnd() override { + ThriftProxy::FilterStatus listEnd() override { return ThriftProxy::FilterStatus::Continue; } + ThriftProxy::FilterStatus setBegin(ThriftProxy::FieldType&, uint32_t&) override { return ThriftProxy::FilterStatus::Continue; } + ThriftProxy::FilterStatus setEnd() override { return ThriftProxy::FilterStatus::Continue; } // RateLimit::RequestCallbacks void complete(Filters::Common::RateLimit::LimitStatus status, diff --git a/source/extensions/filters/network/thrift_proxy/filters/well_known_names.h b/source/extensions/filters/network/thrift_proxy/filters/well_known_names.h index 6fec45aa997b..83f672c524f1 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/well_known_names.h +++ b/source/extensions/filters/network/thrift_proxy/filters/well_known_names.h @@ -23,7 +23,7 @@ class ThriftFilterNameValues { const std::string ROUTER = "envoy.filters.thrift.router"; }; -typedef ConstSingleton ThriftFilterNames; +using ThriftFilterNames = ConstSingleton; } // namespace ThriftFilters } // namespace ThriftProxy diff --git a/source/extensions/filters/network/thrift_proxy/metadata.h b/source/extensions/filters/network/thrift_proxy/metadata.h index 755c669b37b7..5c1469e86e3f 100644 --- a/source/extensions/filters/network/thrift_proxy/metadata.h +++ b/source/extensions/filters/network/thrift_proxy/metadata.h @@ -118,7 +118,7 @@ class MessageMetadata { absl::optional sampled_; }; -typedef std::shared_ptr MessageMetadataSharedPtr; +using MessageMetadataSharedPtr = std::shared_ptr; /** * Constant Thrift headers. All lower case. @@ -129,7 +129,7 @@ class HeaderValues { const Http::LowerCaseString Dest{":dest"}; const Http::LowerCaseString MethodName{":method-name"}; }; -typedef ConstSingleton Headers; +using Headers = ConstSingleton; } // namespace ThriftProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/thrift_proxy/protocol.h b/source/extensions/filters/network/thrift_proxy/protocol.h index 50cc877be752..e2cd608d91b1 100644 --- a/source/extensions/filters/network/thrift_proxy/protocol.h +++ b/source/extensions/filters/network/thrift_proxy/protocol.h @@ -26,7 +26,7 @@ namespace NetworkFilters { namespace ThriftProxy { class DirectResponse; -typedef std::unique_ptr DirectResponsePtr; +using DirectResponsePtr = std::unique_ptr; /** * Protocol represents the operations necessary to implement the a generic Thrift protocol. @@ -34,7 +34,7 @@ typedef std::unique_ptr DirectResponsePtr; */ class Protocol { public: - virtual ~Protocol() {} + virtual ~Protocol() = default; /** * @return const std::string& the human-readable name of the protocol @@ -462,14 +462,14 @@ class Protocol { } }; -typedef std::unique_ptr ProtocolPtr; +using ProtocolPtr = std::unique_ptr; /** * A DirectResponse manipulates a Protocol to directly create a Thrift response message. */ class DirectResponse { public: - virtual ~DirectResponse() {} + virtual ~DirectResponse() = default; enum class ResponseType { // DirectResponse encodes MessageType::Reply with success payload @@ -500,7 +500,7 @@ class DirectResponse { */ class NamedProtocolConfigFactory { public: - virtual ~NamedProtocolConfigFactory() {} + virtual ~NamedProtocolConfigFactory() = default; /** * Create a particular Thrift protocol diff --git a/source/extensions/filters/network/thrift_proxy/protocol_converter.h b/source/extensions/filters/network/thrift_proxy/protocol_converter.h index a7c0854a4f2f..47ab48ac8204 100644 --- a/source/extensions/filters/network/thrift_proxy/protocol_converter.h +++ b/source/extensions/filters/network/thrift_proxy/protocol_converter.h @@ -16,8 +16,8 @@ namespace ThriftProxy { */ class ProtocolConverter : public virtual DecoderEventHandler { public: - ProtocolConverter() {} - virtual ~ProtocolConverter() {} + ProtocolConverter() = default; + ~ProtocolConverter() override = default; void initProtocolConverter(Protocol& proto, Buffer::Instance& buffer) { proto_ = &proto; diff --git a/source/extensions/filters/network/thrift_proxy/router/router.h b/source/extensions/filters/network/thrift_proxy/router/router.h index 3f412e361b91..d2819fc49acb 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router.h +++ b/source/extensions/filters/network/thrift_proxy/router/router.h @@ -20,7 +20,7 @@ class RateLimitPolicy; */ class RouteEntry { public: - virtual ~RouteEntry() {} + virtual ~RouteEntry() = default; /** * @return const std::string& the upstream cluster that owns the route. @@ -44,7 +44,7 @@ class RouteEntry { */ class Route { public: - virtual ~Route() {} + virtual ~Route() = default; /** * @return the route entry or nullptr if there is no matching route for the request. @@ -52,14 +52,14 @@ class Route { virtual const RouteEntry* routeEntry() const PURE; }; -typedef std::shared_ptr RouteConstSharedPtr; +using RouteConstSharedPtr = std::shared_ptr; /** * The router configuration. */ class Config { public: - virtual ~Config() {} + virtual ~Config() = default; /** * Based on the incoming Thrift request transport and/or protocol data, determine the target @@ -72,7 +72,7 @@ class Config { uint64_t random_value) const PURE; }; -typedef std::shared_ptr ConfigConstSharedPtr; +using ConfigConstSharedPtr = std::shared_ptr; } // namespace Router } // namespace ThriftProxy diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.h b/source/extensions/filters/network/thrift_proxy/router/router_impl.h index 9e77da8d03fd..85468dd1702a 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.h +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.h @@ -80,7 +80,7 @@ class RouteEntryImplBase : public RouteEntry, const uint64_t cluster_weight_; Envoy::Router::MetadataMatchCriteriaConstPtr metadata_match_criteria_; }; - typedef std::shared_ptr WeightedClusterEntrySharedPtr; + using WeightedClusterEntrySharedPtr = std::shared_ptr; const std::string cluster_name_; std::vector config_headers_; @@ -90,7 +90,7 @@ class RouteEntryImplBase : public RouteEntry, const RateLimitPolicyImpl rate_limit_policy_; }; -typedef std::shared_ptr RouteEntryImplBaseConstSharedPtr; +using RouteEntryImplBaseConstSharedPtr = std::shared_ptr; class MethodNameRouteEntryImpl : public RouteEntryImplBase { public: diff --git a/source/extensions/filters/network/thrift_proxy/router/router_ratelimit.h b/source/extensions/filters/network/thrift_proxy/router/router_ratelimit.h index ba1449bd381a..aca46cd33036 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_ratelimit.h +++ b/source/extensions/filters/network/thrift_proxy/router/router_ratelimit.h @@ -22,7 +22,7 @@ namespace Router { */ class RateLimitAction { public: - virtual ~RateLimitAction() {} + virtual ~RateLimitAction() = default; /** * Potentially append a descriptor entry to the end of descriptor. @@ -39,14 +39,14 @@ class RateLimitAction { const Network::Address::Instance& remote_address) const PURE; }; -typedef std::unique_ptr RateLimitActionPtr; +using RateLimitActionPtr = std::unique_ptr; /** * Rate limit configuration. */ class RateLimitPolicyEntry { public: - virtual ~RateLimitPolicyEntry() {} + virtual ~RateLimitPolicyEntry() = default; /** * @return the stage value that the configuration is applicable to. @@ -78,7 +78,7 @@ class RateLimitPolicyEntry { */ class RateLimitPolicy { public: - virtual ~RateLimitPolicy() {} + virtual ~RateLimitPolicy() = default; /** * @return true if there is no rate limit policy for all stage settings. diff --git a/source/extensions/filters/network/thrift_proxy/thrift.h b/source/extensions/filters/network/thrift_proxy/thrift.h index 963a55daf345..d16e074561a4 100644 --- a/source/extensions/filters/network/thrift_proxy/thrift.h +++ b/source/extensions/filters/network/thrift_proxy/thrift.h @@ -51,7 +51,7 @@ class TransportNameValues { } }; -typedef ConstSingleton TransportNames; +using TransportNames = ConstSingleton; enum class ProtocolType { Binary, @@ -102,7 +102,7 @@ class ProtocolNameValues { } }; -typedef ConstSingleton ProtocolNames; +using ProtocolNames = ConstSingleton; /** * Thrift protocol message types. diff --git a/source/extensions/filters/network/thrift_proxy/thrift_object.h b/source/extensions/filters/network/thrift_proxy/thrift_object.h index f1d5324c5fdb..4f99d23bf322 100644 --- a/source/extensions/filters/network/thrift_proxy/thrift_object.h +++ b/source/extensions/filters/network/thrift_proxy/thrift_object.h @@ -20,7 +20,7 @@ class ThriftBase; */ class ThriftValue { public: - virtual ~ThriftValue() {} + virtual ~ThriftValue() = default; /** * @return FieldType the type of this value @@ -99,16 +99,16 @@ template <> class ThriftValue::Traits { static FieldType getFieldType() { return FieldType::String; } }; -typedef std::unique_ptr ThriftValuePtr; -typedef std::list ThriftValuePtrList; -typedef std::list> ThriftValuePtrPairList; +using ThriftValuePtr = std::unique_ptr; +using ThriftValuePtrList = std::list; +using ThriftValuePtrPairList = std::list>; /** * ThriftField is a field within a ThriftStruct. */ class ThriftField { public: - virtual ~ThriftField() {} + virtual ~ThriftField() = default; /** * @return FieldType this field's type @@ -126,15 +126,15 @@ class ThriftField { virtual const ThriftValue& getValue() const PURE; }; -typedef std::unique_ptr ThriftFieldPtr; -typedef std::list ThriftFieldPtrList; +using ThriftFieldPtr = std::unique_ptr; +using ThriftFieldPtrList = std::list; /** * ThriftListValue is an ordered list of ThriftValues. */ class ThriftListValue { public: - virtual ~ThriftListValue() {} + virtual ~ThriftListValue() = default; /** * @return const ThriftValuePtrList& containing the ThriftValues that comprise the list @@ -157,7 +157,7 @@ class ThriftListValue { */ class ThriftSetValue { public: - virtual ~ThriftSetValue() {} + virtual ~ThriftSetValue() = default; /** * @return const ThriftValuePtrList& containing the ThriftValues that comprise the set @@ -180,7 +180,7 @@ class ThriftSetValue { */ class ThriftMapValue { public: - virtual ~ThriftMapValue() {} + virtual ~ThriftMapValue() = default; /** * @return const ThriftValuePtrPairList& containing the ThriftValue key-value pairs that comprise @@ -209,7 +209,7 @@ class ThriftMapValue { */ class ThriftStructValue { public: - virtual ~ThriftStructValue() {} + virtual ~ThriftStructValue() = default; /** * @return const ThriftFieldPtrList& containing the ThriftFields that comprise the struct. @@ -227,7 +227,7 @@ class ThriftStructValue { */ class ThriftObject : public ThriftStructValue { public: - virtual ~ThriftObject() {} + ~ThriftObject() override = default; /* * Consumes bytes from the buffer until a single complete Thrift struct has been consumed. @@ -239,7 +239,7 @@ class ThriftObject : public ThriftStructValue { virtual bool onData(Buffer::Instance& buffer) PURE; }; -typedef std::unique_ptr ThriftObjectPtr; +using ThriftObjectPtr = std::unique_ptr; } // namespace ThriftProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/thrift_proxy/tracing.h b/source/extensions/filters/network/thrift_proxy/tracing.h index 2024ae473cd0..da0f1d52fc5c 100644 --- a/source/extensions/filters/network/thrift_proxy/tracing.h +++ b/source/extensions/filters/network/thrift_proxy/tracing.h @@ -37,7 +37,7 @@ class Annotation { std::string value_; absl::optional host_; }; -typedef std::list AnnotationList; +using AnnotationList = std::list; /** * AnnotationType represents a BinaryAnnotation's type. @@ -67,7 +67,7 @@ class BinaryAnnotation { AnnotationType annotation_type_{AnnotationType::Bool}; absl::optional host_; }; -typedef std::list BinaryAnnotationList; +using BinaryAnnotationList = std::list; /** * Span is a single, annotated span in a trace. @@ -90,7 +90,7 @@ class Span { BinaryAnnotationList binary_annotations_; bool debug_{false}; }; -typedef std::list SpanList; +using SpanList = std::list; } // namespace ThriftProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/thrift_proxy/transport.h b/source/extensions/filters/network/thrift_proxy/transport.h index b7c0cac458db..f1cd74cf4baf 100644 --- a/source/extensions/filters/network/thrift_proxy/transport.h +++ b/source/extensions/filters/network/thrift_proxy/transport.h @@ -24,7 +24,7 @@ namespace ThriftProxy { */ class Transport { public: - virtual ~Transport() {} + virtual ~Transport() = default; /* * Returns this transport's name. @@ -76,7 +76,7 @@ class Transport { Buffer::Instance& message) PURE; }; -typedef std::unique_ptr TransportPtr; +using TransportPtr = std::unique_ptr; /** * Implemented by each Thrift transport and registered via Registry::registerFactory or the @@ -84,7 +84,7 @@ typedef std::unique_ptr TransportPtr; */ class NamedTransportConfigFactory { public: - virtual ~NamedTransportConfigFactory() {} + virtual ~NamedTransportConfigFactory() = default; /** * Create a particular Thrift transport. diff --git a/source/extensions/filters/network/thrift_proxy/twitter_protocol_impl.cc b/source/extensions/filters/network/thrift_proxy/twitter_protocol_impl.cc index 614e98de363a..77e6862dac14 100644 --- a/source/extensions/filters/network/thrift_proxy/twitter_protocol_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/twitter_protocol_impl.cc @@ -27,7 +27,7 @@ struct StructNameValues { const std::string endpointStruct = "Endpoint"; const std::string upgradeReplyStruct = "UpgradeReply"; }; -typedef ConstSingleton StructNames; +using StructNames = ConstSingleton; struct RequestHeaderFieldNameValues { const std::string traceIdField = "trace_id"; @@ -41,30 +41,30 @@ struct RequestHeaderFieldNameValues { const std::string delegationsField = "delegations"; const std::string traceIdHighField = "trace_id_high"; }; -typedef ConstSingleton RequestHeaderFieldNames; +using RequestHeaderFieldNames = ConstSingleton; struct ClientIdFieldNameValues { const std::string nameField = "name"; }; -typedef ConstSingleton ClientIdFieldNames; +using ClientIdFieldNames = ConstSingleton; struct DelegationFieldNameValues { const std::string srcField = "src"; const std::string dstField = "dst"; }; -typedef ConstSingleton DelegationFieldNames; +using DelegationFieldNames = ConstSingleton; struct RequestContextFieldNameValues { const std::string keyField = "key"; const std::string valueField = "value"; }; -typedef ConstSingleton RequestContextFieldNames; +using RequestContextFieldNames = ConstSingleton; struct ResponseHeaderFieldNameValues { const std::string spansField = "spans"; const std::string contextsField = "contexts"; }; -typedef ConstSingleton ResponseHeaderFieldNames; +using ResponseHeaderFieldNames = ConstSingleton; struct SpanFieldNameValues { const std::string traceIdField = "trace_id"; @@ -75,14 +75,14 @@ struct SpanFieldNameValues { const std::string binaryAnnotationsField = "binary_annotations"; const std::string debugField = "debug"; }; -typedef ConstSingleton SpanFieldNames; +using SpanFieldNames = ConstSingleton; struct AnnotationFieldNameValues { const std::string timestampField = "timestamp"; const std::string valueField = "value"; const std::string hostField = "host"; }; -typedef ConstSingleton AnnotationFieldNames; +using AnnotationFieldNames = ConstSingleton; struct BinaryAnnotationFieldNameValues { const std::string keyField = "key"; @@ -90,14 +90,14 @@ struct BinaryAnnotationFieldNameValues { const std::string annotationTypeField = "annotation_type"; const std::string hostField = "host"; }; -typedef ConstSingleton BinaryAnnotationFieldNames; +using BinaryAnnotationFieldNames = ConstSingleton; struct EndpointFieldNameValues { const std::string ipv4Field = "ipv4"; const std::string portField = "port"; const std::string serviceNameField = "service_name"; }; -typedef ConstSingleton EndpointFieldNames; +using EndpointFieldNames = ConstSingleton; const std::string& emptyString() { CONSTRUCT_ON_FIRST_USE(std::string, ""); } @@ -262,7 +262,7 @@ class RequestContext { std::string key_; std::string value_; }; -typedef std::list RequestContextList; +using RequestContextList = std::list; /** * Delegation is Twitter protocol delegation table entry. @@ -312,7 +312,7 @@ class Delegation { std::string src_; std::string dst_; }; -typedef std::list DelegationList; +using DelegationList = std::list; /** * RequestHeader is a Twitter protocol request header, inserted between the transport start and diff --git a/source/extensions/filters/network/well_known_names.h b/source/extensions/filters/network/well_known_names.h index a1d435f4e7b2..a397ee552719 100644 --- a/source/extensions/filters/network/well_known_names.h +++ b/source/extensions/filters/network/well_known_names.h @@ -50,7 +50,7 @@ class NetworkFilterNameValues { RedisProxy, TcpProxy, ExtAuthorization}) {} }; -typedef ConstSingleton NetworkFilterNames; +using NetworkFilterNames = ConstSingleton; } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.h b/source/extensions/filters/network/zookeeper_proxy/decoder.h index 46efb96fe65e..4e8db8c9361c 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.h @@ -70,7 +70,7 @@ const char* createFlagsToString(CreateFlags flags); */ class DecoderCallbacks { public: - virtual ~DecoderCallbacks() {} + virtual ~DecoderCallbacks() = default; virtual void onDecodeError() PURE; virtual void onRequestBytes(uint64_t bytes) PURE; @@ -102,12 +102,12 @@ class DecoderCallbacks { */ class Decoder { public: - virtual ~Decoder() {} + virtual ~Decoder() = default; virtual void onData(Buffer::Instance& data) PURE; }; -typedef std::unique_ptr DecoderPtr; +using DecoderPtr = std::unique_ptr; class DecoderImpl : public Decoder, Logger::Loggable { public: diff --git a/source/extensions/grpc_credentials/well_known_names.h b/source/extensions/grpc_credentials/well_known_names.h index 92dbcaf7c226..bfb145770045 100644 --- a/source/extensions/grpc_credentials/well_known_names.h +++ b/source/extensions/grpc_credentials/well_known_names.h @@ -20,7 +20,7 @@ class GrpcCredentialsNameValues { const std::string FileBasedMetadata = "envoy.grpc_credentials.file_based_metadata"; }; -typedef ConstSingleton GrpcCredentialsNames; +using GrpcCredentialsNames = ConstSingleton; } // namespace GrpcCredentials } // namespace Extensions diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index f0be06351adf..b046da958931 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -101,7 +101,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { NetworkFilters::Common::Redis::RespValue request_; }; - typedef std::unique_ptr RedisActiveHealthCheckSessionPtr; + using RedisActiveHealthCheckSessionPtr = std::unique_ptr; // HealthCheckerImplBase ActiveHealthCheckSessionPtr makeSession(Upstream::HostSharedPtr host) override { diff --git a/source/extensions/health_checkers/well_known_names.h b/source/extensions/health_checkers/well_known_names.h index f7514e8d8fa4..e17e4a2349a4 100644 --- a/source/extensions/health_checkers/well_known_names.h +++ b/source/extensions/health_checkers/well_known_names.h @@ -18,7 +18,7 @@ class HealthCheckerNameValues { const std::string RedisHealthChecker = "envoy.health_checkers.redis"; }; -typedef ConstSingleton HealthCheckerNames; +using HealthCheckerNames = ConstSingleton; } // namespace HealthCheckers } // namespace Extensions diff --git a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h index 9bc4bb2697c0..9576695e44d1 100644 --- a/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h +++ b/source/extensions/resource_monitors/fixed_heap/fixed_heap_monitor.h @@ -13,8 +13,8 @@ namespace FixedHeapMonitor { */ class MemoryStatsReader { public: - MemoryStatsReader() {} - virtual ~MemoryStatsReader() {} + MemoryStatsReader() = default; + virtual ~MemoryStatsReader() = default; // Memory reserved for the process by the heap. virtual uint64_t reservedHeapBytes(); diff --git a/source/extensions/resource_monitors/well_known_names.h b/source/extensions/resource_monitors/well_known_names.h index e3ddfde7866d..7828997ed782 100644 --- a/source/extensions/resource_monitors/well_known_names.h +++ b/source/extensions/resource_monitors/well_known_names.h @@ -20,7 +20,7 @@ class ResourceMonitorNameValues { const std::string InjectedResource = "envoy.resource_monitors.injected_resource"; }; -typedef ConstSingleton ResourceMonitorNames; +using ResourceMonitorNames = ConstSingleton; } // namespace ResourceMonitors } // namespace Extensions diff --git a/source/extensions/retry/host/well_known_names.h b/source/extensions/retry/host/well_known_names.h index de1807e7901d..a79002befab3 100644 --- a/source/extensions/retry/host/well_known_names.h +++ b/source/extensions/retry/host/well_known_names.h @@ -18,7 +18,7 @@ class RetryHostPredicatesNameValues { const std::string PreviousHostsPredicate = "envoy.retry_host_predicates.previous_hosts"; }; -typedef ConstSingleton RetryHostPredicateValues; +using RetryHostPredicateValues = ConstSingleton; } // namespace Host } // namespace Retry diff --git a/source/extensions/retry/priority/well_known_names.h b/source/extensions/retry/priority/well_known_names.h index 18df4fcda546..5e20524e8df6 100644 --- a/source/extensions/retry/priority/well_known_names.h +++ b/source/extensions/retry/priority/well_known_names.h @@ -18,7 +18,7 @@ class RetryPriorityNameValues { const std::string PreviousPrioritiesRetryPriority = "envoy.retry_priorities.previous_priorities"; }; -typedef ConstSingleton RetryPriorityValues; +using RetryPriorityValues = ConstSingleton; } // namespace Priority } // namespace Retry diff --git a/source/extensions/stat_sinks/hystrix/hystrix.h b/source/extensions/stat_sinks/hystrix/hystrix.h index 31c1df700b02..2a7fac6f88cc 100644 --- a/source/extensions/stat_sinks/hystrix/hystrix.h +++ b/source/extensions/stat_sinks/hystrix/hystrix.h @@ -16,8 +16,8 @@ namespace Extensions { namespace StatSinks { namespace Hystrix { -typedef std::vector RollingWindow; -typedef std::map RollingStatsMap; +using RollingWindow = std::vector; +using RollingStatsMap = std::map; using QuantileLatencyMap = std::unordered_map; static const std::vector hystrix_quantiles = {0, 0.25, 0.5, 0.75, 0.90, @@ -43,7 +43,7 @@ struct ClusterStatsCache { RollingWindow rejected_; }; -typedef std::unique_ptr ClusterStatsCachePtr; +using ClusterStatsCachePtr = std::unique_ptr; class HystrixSink : public Stats::Sink, public Logger::Loggable { public: @@ -169,7 +169,7 @@ class HystrixSink : public Stats::Sink, public Logger::Loggable HystrixSinkPtr; +using HystrixSinkPtr = std::unique_ptr; } // namespace Hystrix } // namespace StatSinks diff --git a/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.h b/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.h index f68cc5e8885f..14a99e92ac07 100644 --- a/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.h +++ b/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.h @@ -25,7 +25,7 @@ namespace MetricsService { class GrpcMetricsStreamer : public Grpc::AsyncStreamCallbacks { public: - virtual ~GrpcMetricsStreamer() {} + ~GrpcMetricsStreamer() override = default; /** * Send Metrics Message. @@ -43,7 +43,7 @@ class GrpcMetricsStreamer void onRemoteClose(Grpc::Status::GrpcStatus, const std::string&) override{}; }; -typedef std::shared_ptr GrpcMetricsStreamerSharedPtr; +using GrpcMetricsStreamerSharedPtr = std::shared_ptr; /** * Production implementation of GrpcMetricsStreamer diff --git a/source/extensions/stat_sinks/well_known_names.h b/source/extensions/stat_sinks/well_known_names.h index c6dfd4a5816b..a564d168d636 100644 --- a/source/extensions/stat_sinks/well_known_names.h +++ b/source/extensions/stat_sinks/well_known_names.h @@ -24,7 +24,7 @@ class StatsSinkNameValues { const std::string Hystrix = "envoy.stat_sinks.hystrix"; }; -typedef ConstSingleton StatsSinkNames; +using StatsSinkNames = ConstSingleton; } // namespace StatSinks } // namespace Extensions diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc index 4b5f5f5b3627..c1e40522ec39 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc @@ -39,9 +39,8 @@ class OpenTracingHTTPHeadersReader : public opentracing::HTTPHeadersReader { explicit OpenTracingHTTPHeadersReader(const Http::HeaderMap& request_headers) : request_headers_(request_headers) {} - typedef std::function(opentracing::string_view, - opentracing::string_view)> - OpenTracingCb; + using OpenTracingCb = std::function(opentracing::string_view, + opentracing::string_view)>; // opentracing::HTTPHeadersReader opentracing::expected diff --git a/source/extensions/tracers/datadog/datadog_tracer_impl.h b/source/extensions/tracers/datadog/datadog_tracer_impl.h index 54f8379b84da..41b962848716 100644 --- a/source/extensions/tracers/datadog/datadog_tracer_impl.h +++ b/source/extensions/tracers/datadog/datadog_tracer_impl.h @@ -30,8 +30,8 @@ struct DatadogTracerStats { }; class TraceReporter; -typedef std::unique_ptr TraceReporterPtr; -typedef std::shared_ptr TraceEncoderSharedPtr; +using TraceReporterPtr = std::unique_ptr; +using TraceEncoderSharedPtr = std::shared_ptr; /** * Class for a Datadog-specific Driver. diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 67e291a8cae3..5badb14f4faf 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -31,7 +31,7 @@ class ConstantValues { const Http::LowerCaseString X_CLOUD_TRACE_CONTEXT{"x-cloud-trace-context"}; }; -typedef ConstSingleton Constants; +using Constants = ConstSingleton; /** * OpenCensus tracing implementation of the Envoy Span object. diff --git a/source/extensions/tracers/well_known_names.h b/source/extensions/tracers/well_known_names.h index e3e1b3f556ff..a756df8de089 100644 --- a/source/extensions/tracers/well_known_names.h +++ b/source/extensions/tracers/well_known_names.h @@ -25,7 +25,7 @@ class TracerNameValues { const std::string OpenCensus = "envoy.tracers.opencensus"; }; -typedef ConstSingleton TracerNames; +using TracerNames = ConstSingleton; } // namespace Tracers } // namespace Extensions diff --git a/source/extensions/tracers/zipkin/tracer.h b/source/extensions/tracers/zipkin/tracer.h index 0ee24934a351..190b68631b63 100644 --- a/source/extensions/tracers/zipkin/tracer.h +++ b/source/extensions/tracers/zipkin/tracer.h @@ -25,7 +25,7 @@ class Reporter { /** * Destructor. */ - virtual ~Reporter() {} + virtual ~Reporter() = default; /** * Method that a concrete Reporter class must implement to handle finished spans. @@ -36,7 +36,7 @@ class Reporter { virtual void reportSpan(const Span& span) PURE; }; -typedef std::unique_ptr ReporterPtr; +using ReporterPtr = std::unique_ptr; /** * This class implements the Zipkin tracer. It has methods to create the appropriate Zipkin span @@ -121,7 +121,7 @@ class Tracer : public TracerInterface { TimeSource& time_source_; }; -typedef std::unique_ptr TracerPtr; +using TracerPtr = std::unique_ptr; } // namespace Zipkin } // namespace Tracers diff --git a/source/extensions/tracers/zipkin/tracer_interface.h b/source/extensions/tracers/zipkin/tracer_interface.h index 7e593c657d0e..4c55ab90980b 100644 --- a/source/extensions/tracers/zipkin/tracer_interface.h +++ b/source/extensions/tracers/zipkin/tracer_interface.h @@ -17,7 +17,7 @@ class TracerInterface { /** * Destructor. */ - virtual ~TracerInterface() {} + virtual ~TracerInterface() = default; /** * A Zipkin tracer must implement this method. Its implementation must perform whatever diff --git a/source/extensions/tracers/zipkin/zipkin_core_constants.h b/source/extensions/tracers/zipkin/zipkin_core_constants.h index f31a2a43ef6d..7384df786a19 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_constants.h +++ b/source/extensions/tracers/zipkin/zipkin_core_constants.h @@ -48,7 +48,7 @@ class ZipkinCoreConstantValues { const bool DEFAULT_SHARED_SPAN_CONTEXT = true; }; -typedef ConstSingleton ZipkinCoreConstants; +using ZipkinCoreConstants = ConstSingleton; } // namespace Zipkin } // namespace Tracers diff --git a/source/extensions/tracers/zipkin/zipkin_core_types.h b/source/extensions/tracers/zipkin/zipkin_core_types.h index 9060432c25a5..165ae642c840 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_types.h +++ b/source/extensions/tracers/zipkin/zipkin_core_types.h @@ -28,7 +28,7 @@ class ZipkinBase { /** * Destructor. */ - virtual ~ZipkinBase() {} + virtual ~ZipkinBase() = default; /** * All classes defining Zipkin abstractions need to implement this method to convert @@ -288,10 +288,10 @@ class BinaryAnnotation : public ZipkinBase { std::string key_; std::string value_; absl::optional endpoint_; - AnnotationType annotation_type_; + AnnotationType annotation_type_{}; }; -typedef std::unique_ptr SpanPtr; +using SpanPtr = std::unique_ptr; /** * Represents a Zipkin span. This class is based on Zipkin's Thrift definition of a span. diff --git a/source/extensions/tracers/zipkin/zipkin_json_field_names.h b/source/extensions/tracers/zipkin/zipkin_json_field_names.h index ff3f6068511e..d904eb73700a 100644 --- a/source/extensions/tracers/zipkin/zipkin_json_field_names.h +++ b/source/extensions/tracers/zipkin/zipkin_json_field_names.h @@ -34,7 +34,7 @@ class ZipkinJsonFieldNameValues { const std::string ENDPOINT_IPV6 = "ipv6"; }; -typedef ConstSingleton ZipkinJsonFieldNames; +using ZipkinJsonFieldNames = ConstSingleton; } // namespace Zipkin } // namespace Tracers diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h index daa0bb70a1e9..048b37cd18dc 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h @@ -80,7 +80,7 @@ class ZipkinSpan : public Tracing::Span { Zipkin::Tracer& tracer_; }; -typedef std::unique_ptr ZipkinSpanPtr; +using ZipkinSpanPtr = std::unique_ptr; /** * Class for a Zipkin-specific Driver. diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index e0c728919332..dd41e6fefc17 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -20,8 +20,8 @@ namespace TransportSockets { namespace Alts { // smart pointer for grpc_alts_credentials_options that will be automatically freed. -typedef CSmartPtr - GrpcAltsCredentialsOptionsPtr; +using GrpcAltsCredentialsOptionsPtr = + CSmartPtr; namespace { diff --git a/source/extensions/transport_sockets/alts/grpc_tsi.h b/source/extensions/transport_sockets/alts/grpc_tsi.h index ac7265de6f3c..825f3d495d93 100644 --- a/source/extensions/transport_sockets/alts/grpc_tsi.h +++ b/source/extensions/transport_sockets/alts/grpc_tsi.h @@ -22,10 +22,10 @@ namespace Extensions { namespace TransportSockets { namespace Alts { -typedef CSmartPtr CFrameProtectorPtr; +using CFrameProtectorPtr = CSmartPtr; -typedef CSmartPtr CHandshakerResultPtr; -typedef CSmartPtr CHandshakerPtr; +using CHandshakerResultPtr = CSmartPtr; +using CHandshakerPtr = CSmartPtr; } // namespace Alts } // namespace TransportSockets diff --git a/source/extensions/transport_sockets/alts/noop_transport_socket_callbacks.h b/source/extensions/transport_sockets/alts/noop_transport_socket_callbacks.h index a10bc149b0b3..5c897ece015a 100644 --- a/source/extensions/transport_sockets/alts/noop_transport_socket_callbacks.h +++ b/source/extensions/transport_sockets/alts/noop_transport_socket_callbacks.h @@ -32,7 +32,7 @@ class NoOpTransportSocketCallbacks : public Network::TransportSocketCallbacks { Network::TransportSocketCallbacks& parent_; }; -typedef std::unique_ptr NoOpTransportSocketCallbacksPtr; +using NoOpTransportSocketCallbacksPtr = std::unique_ptr; } // namespace Alts } // namespace TransportSockets diff --git a/source/extensions/transport_sockets/alts/tsi_frame_protector.h b/source/extensions/transport_sockets/alts/tsi_frame_protector.h index ac2fe1fc8f7f..7867770e810e 100644 --- a/source/extensions/transport_sockets/alts/tsi_frame_protector.h +++ b/source/extensions/transport_sockets/alts/tsi_frame_protector.h @@ -41,7 +41,7 @@ class TsiFrameProtector final { CFrameProtectorPtr frame_protector_; }; -typedef std::unique_ptr TsiFrameProtectorPtr; +using TsiFrameProtectorPtr = std::unique_ptr; } // namespace Alts } // namespace TransportSockets diff --git a/source/extensions/transport_sockets/alts/tsi_handshaker.h b/source/extensions/transport_sockets/alts/tsi_handshaker.h index e751f6aab381..d578493573ec 100644 --- a/source/extensions/transport_sockets/alts/tsi_handshaker.h +++ b/source/extensions/transport_sockets/alts/tsi_handshaker.h @@ -20,7 +20,7 @@ namespace Alts { */ class TsiHandshakerCallbacks { public: - virtual ~TsiHandshakerCallbacks() {} + virtual ~TsiHandshakerCallbacks() = default; struct NextResult { // A enum of the result. @@ -33,7 +33,7 @@ class TsiHandshakerCallbacks { CHandshakerResultPtr result_; }; - typedef std::unique_ptr NextResultPtr; + using NextResultPtr = std::unique_ptr; /** * Called when `next` is done, this may be called inline in `next` if the handshaker is not @@ -93,7 +93,7 @@ class TsiHandshaker final : public Event::DeferredDeletable { Event::Dispatcher& dispatcher_; }; -typedef std::unique_ptr TsiHandshakerPtr; +using TsiHandshakerPtr = std::unique_ptr; } // namespace Alts } // namespace TransportSockets diff --git a/source/extensions/transport_sockets/alts/tsi_socket.h b/source/extensions/transport_sockets/alts/tsi_socket.h index b304b5580725..31b4571db096 100644 --- a/source/extensions/transport_sockets/alts/tsi_socket.h +++ b/source/extensions/transport_sockets/alts/tsi_socket.h @@ -20,10 +20,9 @@ namespace Alts { * @param local_address the local address of the connection. * @param remote_address the remote address of the connection. */ -typedef std::function - HandshakerFactory; + const Network::Address::InstanceConstSharedPtr& remote_address)>; /** * A function to validate the peer of the connection. @@ -32,7 +31,7 @@ typedef std::function HandshakeValidator; +using HandshakeValidator = std::function; /** * A implementation of Network::TransportSocket based on gRPC TSI diff --git a/source/extensions/transport_sockets/raw_buffer/config.h b/source/extensions/transport_sockets/raw_buffer/config.h index 50d2dc714cbc..a7c68d6875a7 100644 --- a/source/extensions/transport_sockets/raw_buffer/config.h +++ b/source/extensions/transport_sockets/raw_buffer/config.h @@ -16,7 +16,7 @@ namespace RawBuffer { */ class RawBufferSocketFactory : public virtual Server::Configuration::TransportSocketConfigFactory { public: - virtual ~RawBufferSocketFactory() {} + ~RawBufferSocketFactory() override = default; std::string name() const override { return TransportSocketNames::get().RawBuffer; } ProtobufTypes::MessagePtr createEmptyConfigProto() override; }; diff --git a/source/extensions/transport_sockets/tap/config.h b/source/extensions/transport_sockets/tap/config.h index 20bca58dd38a..8068779ada01 100644 --- a/source/extensions/transport_sockets/tap/config.h +++ b/source/extensions/transport_sockets/tap/config.h @@ -15,7 +15,7 @@ namespace Tap { */ class TapSocketConfigFactory : public virtual Server::Configuration::TransportSocketConfigFactory { public: - virtual ~TapSocketConfigFactory() {} + ~TapSocketConfigFactory() override = default; std::string name() const override { return TransportSocketNames::get().Tap; } ProtobufTypes::MessagePtr createEmptyConfigProto() override; }; diff --git a/source/extensions/transport_sockets/tls/config.h b/source/extensions/transport_sockets/tls/config.h index 6ec0f436679d..2455f7448f93 100644 --- a/source/extensions/transport_sockets/tls/config.h +++ b/source/extensions/transport_sockets/tls/config.h @@ -16,7 +16,7 @@ namespace Tls { */ class SslSocketConfigFactory : public virtual Server::Configuration::TransportSocketConfigFactory { public: - virtual ~SslSocketConfigFactory() {} + ~SslSocketConfigFactory() override = default; std::string name() const override { return TransportSocketNames::get().Tls; } }; diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index b8f074a4b880..57c8ffbdec35 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -161,7 +161,7 @@ class ContextImpl : public virtual Envoy::Ssl::Context { const unsigned tls_max_version_; }; -typedef std::shared_ptr ContextImplSharedPtr; +using ContextImplSharedPtr = std::shared_ptr; class ClientContextImpl : public ContextImpl, public Envoy::Ssl::ClientContext { public: diff --git a/source/extensions/transport_sockets/well_known_names.h b/source/extensions/transport_sockets/well_known_names.h index b65060155e1a..3d2270a63230 100644 --- a/source/extensions/transport_sockets/well_known_names.h +++ b/source/extensions/transport_sockets/well_known_names.h @@ -20,7 +20,7 @@ class TransportSocketNameValues { const std::string Tls = "tls"; }; -typedef ConstSingleton TransportSocketNames; +using TransportSocketNames = ConstSingleton; } // namespace TransportSockets } // namespace Extensions diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h index 1e9fb60a26c2..772c3c332546 100644 --- a/source/server/configuration_impl.h +++ b/source/server/configuration_impl.h @@ -31,7 +31,7 @@ namespace Configuration { */ class StatsSinkFactory { public: - virtual ~StatsSinkFactory() {} + virtual ~StatsSinkFactory() = default; /** * Create a particular Stats::Sink implementation. If the implementation is unable to produce a diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index ad3f4a0e81ec..3c5de7f99852 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -61,20 +61,20 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { private: struct ActiveListenerBase; - typedef std::unique_ptr ActiveListenerBasePtr; + using ActiveListenerBasePtr = std::unique_ptr; struct ActiveTcpListener; - typedef std::unique_ptr ActiveTcpListenerPtr; + using ActiveTcpListenerPtr = std::unique_ptr; struct ActiveUdpListener; - typedef std::unique_ptr ActiveUdpListenerPtr; + using ActiveUdpListenerPtr = std::unique_ptr; ActiveListenerBase* findActiveListenerByAddress(const Network::Address::Instance& address); struct ActiveConnection; - typedef std::unique_ptr ActiveConnectionPtr; + using ActiveConnectionPtr = std::unique_ptr; struct ActiveSocket; - typedef std::unique_ptr ActiveSocketPtr; + using ActiveSocketPtr = std::unique_ptr; /** * Wrapper for an active listener owned by this handler. @@ -83,7 +83,7 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { ActiveListenerBase(ConnectionHandlerImpl& parent, Network::ListenerPtr&& listener, Network::ListenerConfig& config); - virtual ~ActiveListenerBase() {} + virtual ~ActiveListenerBase() = default; ConnectionHandlerImpl& parent_; Network::ListenerPtr listener_; diff --git a/source/server/listener_hooks.h b/source/server/listener_hooks.h index 6293a80dacd0..1b3de394ab13 100644 --- a/source/server/listener_hooks.h +++ b/source/server/listener_hooks.h @@ -10,7 +10,7 @@ namespace Envoy { */ class ListenerHooks { public: - virtual ~ListenerHooks() {} + virtual ~ListenerHooks() = default; /** * Called when a worker has added a listener and it is listening. diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 883777ec935f..80cb5ae4b356 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -86,7 +86,7 @@ class ProdListenerComponentFactory : public ListenerComponentFactory, }; class ListenerImpl; -typedef std::unique_ptr ListenerImplPtr; +using ListenerImplPtr = std::unique_ptr; /** * All listener manager stats. @see stats_macros.h @@ -137,7 +137,7 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable ListenerList; + using ListenerList = std::list; struct DrainingListener { DrainingListener(ListenerImplPtr&& listener, uint64_t workers_pending_removal) diff --git a/source/server/options_impl.h b/source/server/options_impl.h index ec25f5ac164f..cddf660f79f4 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -20,7 +20,7 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable HotRestartVersionCb; + using HotRestartVersionCb = std::function; /** * @throw NoServingException if Envoy has already done everything specified by the argv (e.g. diff --git a/source/server/overload_manager_impl.h b/source/server/overload_manager_impl.h index 1f84854ca6aa..8b55786ecdfd 100644 --- a/source/server/overload_manager_impl.h +++ b/source/server/overload_manager_impl.h @@ -34,7 +34,7 @@ class OverloadAction { class Trigger { public: - virtual ~Trigger() {} + virtual ~Trigger() = default; // Updates the current value of the metric and returns whether the trigger has changed state. virtual bool updateValue(double value) PURE; @@ -42,7 +42,7 @@ class OverloadAction { // Returns whether the trigger is currently fired or not. virtual bool isFired() const PURE; }; - typedef std::unique_ptr TriggerPtr; + using TriggerPtr = std::unique_ptr; private: std::unordered_map triggers_; @@ -107,10 +107,10 @@ class OverloadManagerImpl : Logger::Loggable, public OverloadM std::unordered_map resources_; std::unordered_map actions_; - typedef std::unordered_multimap ResourceToActionMap; + using ResourceToActionMap = std::unordered_multimap; ResourceToActionMap resource_to_actions_; - typedef std::unordered_multimap ActionToCallbackMap; + using ActionToCallbackMap = std::unordered_multimap; ActionToCallbackMap action_to_callbacks_; }; diff --git a/source/server/server.h b/source/server/server.h index 75eff17064c5..871c71ca0355 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -71,7 +71,7 @@ struct ServerStats { */ class ComponentFactory { public: - virtual ~ComponentFactory() {} + virtual ~ComponentFactory() = default; /** * @return DrainManagerPtr a new drain manager for the server. diff --git a/test/common/buffer/buffer_fuzz.cc b/test/common/buffer/buffer_fuzz.cc index 0ac318f66483..1cf63847da3f 100644 --- a/test/common/buffer/buffer_fuzz.cc +++ b/test/common/buffer/buffer_fuzz.cc @@ -141,7 +141,7 @@ class StringBuffer : public Buffer::Instance { std::unique_ptr tmp_buf_{new char[MaxAllocation]}; }; -typedef std::vector> BufferList; +using BufferList = std::vector>; // Process a single buffer operation. uint32_t bufferAction(Context& ctxt, char insert_value, uint32_t max_alloc, BufferList& buffers, diff --git a/test/common/buffer/utility.h b/test/common/buffer/utility.h index 30723ccbe8b6..daf67d85c1d2 100644 --- a/test/common/buffer/utility.h +++ b/test/common/buffer/utility.h @@ -25,7 +25,7 @@ class BufferImplementationParamTest : public testing::TestWithParam PhantomIntTest; -typedef Phantom PhantomIntTest2; +using PhantomIntTest = Phantom; +using PhantomIntTest2 = Phantom; TEST(PhantomTest, TypeBehavior) { // Should not be possible to implicitly convert from two phantoms with different markers. diff --git a/test/common/common/utility_test.cc b/test/common/common/utility_test.cc index a90272214509..3ad89010ac49 100644 --- a/test/common/common/utility_test.cc +++ b/test/common/common/utility_test.cc @@ -429,7 +429,7 @@ class WeightedClusterEntry { const std::string name_; const uint64_t weight_; }; -typedef std::shared_ptr WeightedClusterEntrySharedPtr; +using WeightedClusterEntrySharedPtr = std::shared_ptr; TEST(WeightedClusterUtil, pickCluster) { std::vector clusters; diff --git a/test/common/config/subscription_test_harness.h b/test/common/config/subscription_test_harness.h index acb1c2caacfc..5e12ab8c4180 100644 --- a/test/common/config/subscription_test_harness.h +++ b/test/common/config/subscription_test_harness.h @@ -18,7 +18,7 @@ namespace Config { class SubscriptionTestHarness { public: SubscriptionTestHarness() : stats_(Utility::generateStats(stats_store_)) {} - virtual ~SubscriptionTestHarness() {} + virtual ~SubscriptionTestHarness() = default; /** * Start subscription and set related expectations. diff --git a/test/common/filesystem/directory_test.cc b/test/common/filesystem/directory_test.cc index 5536f77d7286..4c63a0d68d8c 100644 --- a/test/common/filesystem/directory_test.cc +++ b/test/common/filesystem/directory_test.cc @@ -70,7 +70,7 @@ struct EntryHash { } }; -typedef std::unordered_set EntrySet; +using EntrySet = std::unordered_set; EntrySet getDirectoryContents(const std::string& dir_path, bool recursive) { Directory directory(dir_path); diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index 9a7178892055..7585a14fd1e9 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -16,7 +16,7 @@ enum class SotwOrDelta { Sotw, Delta }; class BaseGrpcClientIntegrationParamTest { public: - virtual ~BaseGrpcClientIntegrationParamTest(){}; + virtual ~BaseGrpcClientIntegrationParamTest() = default; virtual Network::Address::IpVersion ipVersion() const PURE; virtual ClientType clientType() const PURE; diff --git a/test/common/grpc/grpc_client_integration_test_harness.h b/test/common/grpc/grpc_client_integration_test_harness.h index 12351a8d8b8f..ad8a997d770b 100644 --- a/test/common/grpc/grpc_client_integration_test_harness.h +++ b/test/common/grpc/grpc_client_integration_test_harness.h @@ -51,7 +51,7 @@ const char HELLO_REPLY[] = "DEFG"; MATCHER_P(HelloworldReplyEq, rhs, "") { return arg.message() == rhs; } -typedef std::vector> TestMetadata; +using TestMetadata = std::vector>; // Use in EXPECT_CALL(foo, bar(_)).WillExitIfNeeded() to exit dispatcher loop if // there are no longer any pending events in DispatcherHelper. diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index f9f18d2e16cd..7d51f36665e9 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -338,7 +338,7 @@ class ReorderBuffer { std::deque bufs_; }; -typedef std::unique_ptr HttpStreamPtr; +using HttpStreamPtr = std::unique_ptr; namespace { diff --git a/test/common/http/common.h b/test/common/http/common.h index fd3431911dbb..e2c4d4a5e12c 100644 --- a/test/common/http/common.h +++ b/test/common/http/common.h @@ -16,7 +16,7 @@ namespace Envoy { */ class CodecClientForTest : public Http::CodecClient { public: - typedef std::function DestroyCb; + using DestroyCb = std::function; CodecClientForTest(Network::ClientConnectionPtr&& connection, Http::ClientConnection* codec, DestroyCb destroy_cb, Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher) diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index cb4dedf9a12b..3c11424b5964 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -375,7 +375,7 @@ class FuzzStream { StreamState response_state_; }; -typedef std::unique_ptr FuzzStreamPtr; +using FuzzStreamPtr = std::unique_ptr; DEFINE_PROTO_FUZZER(const test::common::http::ConnManagerImplTestCase& input) { FuzzConfig config; diff --git a/test/common/http/header_map_impl_test.cc b/test/common/http/header_map_impl_test.cc index 5a5e676f55ae..da9dc1cccf7b 100644 --- a/test/common/http/header_map_impl_test.cc +++ b/test/common/http/header_map_impl_test.cc @@ -484,7 +484,7 @@ TEST(HeaderMapImplTest, SetRemovesAllValues) { headers.addReference(key1, ref_value3); headers.addReference(key1, ref_value4); - typedef testing::MockFunction MockCb; + using MockCb = testing::MockFunction; { MockCb cb; @@ -682,7 +682,7 @@ TEST(HeaderMapImplTest, Iterate) { LowerCaseString foo_key("foo"); headers.setReferenceKey(foo_key, "bar"); // set moves key to end - typedef testing::MockFunction MockCb; + using MockCb = testing::MockFunction; MockCb cb; InSequence seq; @@ -705,7 +705,7 @@ TEST(HeaderMapImplTest, IterateReverse) { LowerCaseString world_key("world"); headers.setReferenceKey(world_key, "hello"); - typedef testing::MockFunction MockCb; + using MockCb = testing::MockFunction; MockCb cb; InSequence seq; @@ -819,7 +819,7 @@ TEST(HeaderMapImplDeathTest, TestHeaderLengthChecks) { } TEST(HeaderMapImplTest, PseudoHeaderOrder) { - typedef testing::MockFunction MockCb; + using MockCb = testing::MockFunction; MockCb cb; { diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 441bf5fdd09a..28b4732f3bd0 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -54,7 +54,7 @@ class Http2CodecImplTestFixture { Http2CodecImplTestFixture(Http2SettingsTuple client_settings, Http2SettingsTuple server_settings) : client_settings_(client_settings), server_settings_(server_settings) {} - virtual ~Http2CodecImplTestFixture() {} + virtual ~Http2CodecImplTestFixture() = default; virtual void initialize() { Http2SettingsFromTuple(client_http2settings_, client_settings_); @@ -850,7 +850,7 @@ INSTANTIATE_TEST_SUITE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTest, // Make sure we have coverage for high and low values for various combinations and permutations // of HTTP settings in at least one test fixture. // Use with caution as any test using this runs 255 times. -typedef Http2CodecImplTest Http2CodecImplTestAll; +using Http2CodecImplTestAll = Http2CodecImplTest; INSTANTIATE_TEST_SUITE_P(Http2CodecImplTestDefaultSettings, Http2CodecImplTestAll, ::testing::Combine(HTTP2SETTINGS_DEFAULT_COMBINE, diff --git a/test/common/http/http2/frame_replay.h b/test/common/http/http2/frame_replay.h index fcd750225fcf..c5b426771c34 100644 --- a/test/common/http/http2/frame_replay.h +++ b/test/common/http/http2/frame_replay.h @@ -17,7 +17,7 @@ namespace Http { namespace Http2 { // A byte vector representation of an HTTP/2 frame. -typedef std::vector Frame; +using Frame = std::vector; // An HTTP/2 frame derived from a file location. class FileFrame { diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index 728d14439263..bc2727363c95 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -43,11 +43,11 @@ namespace Network { namespace { // List of IP address (in human readable format). -typedef std::list IpList; +using IpList = std::list; // Map from hostname to IpList. -typedef std::unordered_map HostMap; +using HostMap = std::unordered_map; // Map from hostname to CNAME -typedef std::unordered_map CNameMap; +using CNameMap = std::unordered_map; // Represents a single TestDnsServer query state and lifecycle. This implements // just enough of RFC 1035 to handle queries we generate in the tests below. enum record_type { A, AAAA }; diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 967254bd5848..0d05981537e2 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -1008,7 +1008,7 @@ match: { prefix: "/new_endpoint" } EXPECT_EQ("123456000, 1, 12, 123, 1234, 12345, 123456, 1234560, 12345600, 123456000", header_map.get_("x-request-start-range")); - typedef absl::flat_hash_map CountMap; + using CountMap = absl::flat_hash_map; CountMap counts; header_map.iterate( [](const Http::HeaderEntry& header, void* cb_v) -> Http::HeaderMap::Iterate { diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index f85edd205787..7a2d0abe50d9 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -234,7 +234,7 @@ TEST_F(SdsApiTest, DynamicCertificateValidationContextUpdateSuccess) { class CvcValidationCallback { public: - virtual ~CvcValidationCallback() {} + virtual ~CvcValidationCallback() = default; virtual void validateCvc(const envoy::api::v2::auth::CertificateValidationContext&) PURE; }; diff --git a/test/common/singleton/threadsafe_singleton_test.cc b/test/common/singleton/threadsafe_singleton_test.cc index a9172aceade7..4877425735c5 100644 --- a/test/common/singleton/threadsafe_singleton_test.cc +++ b/test/common/singleton/threadsafe_singleton_test.cc @@ -14,7 +14,7 @@ namespace Envoy { class TestSingleton { public: - virtual ~TestSingleton() {} + virtual ~TestSingleton() = default; virtual void addOne() { Thread::LockGuard lock(lock_); diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index 58a01d06b0f9..f1d28d67d9b6 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -117,10 +117,9 @@ class HttpHealthCheckerImplTest : public testing::Test { Http::StreamDecoder* stream_response_callbacks_{}; }; - typedef std::unique_ptr TestSessionPtr; - typedef std::unordered_map - HostWithHealthCheckMap; + using TestSessionPtr = std::unique_ptr; + using HostWithHealthCheckMap = + std::unordered_map; HttpHealthCheckerImplTest() : cluster_(new NiceMock()), @@ -2845,7 +2844,7 @@ class GrpcHealthCheckerImplTestBase { CodecClientForTest* codec_client_{}; }; - typedef std::unique_ptr TestSessionPtr; + using TestSessionPtr = std::unique_ptr; struct ResponseSpec { struct ChunkSpec { diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 4c9328550256..1d13d1e9644c 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -477,7 +477,7 @@ class RoundRobinLoadBalancerTest : public LoadBalancerTestBase { // For the tests which mutate primary and failover host sets explicitly, only // run once. -typedef RoundRobinLoadBalancerTest FailoverTest; +using FailoverTest = RoundRobinLoadBalancerTest; // Ensure if all the hosts with priority 0 unhealthy, the next priority hosts are used. TEST_P(FailoverTest, BasicFailover) { diff --git a/test/common/upstream/logical_dns_cluster_test.cc b/test/common/upstream/logical_dns_cluster_test.cc index 6e1a9d7119ab..367d7afa55aa 100644 --- a/test/common/upstream/logical_dns_cluster_test.cc +++ b/test/common/upstream/logical_dns_cluster_test.cc @@ -219,8 +219,8 @@ class LogicalDnsClusterTest : public testing::Test { Api::ApiPtr api_; }; -typedef std::tuple> - LogicalDnsConfigTuple; +using LogicalDnsConfigTuple = + std::tuple>; std::vector generateLogicalDnsParams() { std::vector dns_config; { diff --git a/test/common/upstream/ring_hash_lb_test.cc b/test/common/upstream/ring_hash_lb_test.cc index 35d7942ac60f..243f297bf178 100644 --- a/test/common/upstream/ring_hash_lb_test.cc +++ b/test/common/upstream/ring_hash_lb_test.cc @@ -63,7 +63,7 @@ class RingHashLoadBalancerTest : public testing::TestWithParam { }; // For tests which don't need to be run in both primary and failover modes. -typedef RingHashLoadBalancerTest RingHashFailoverTest; +using RingHashFailoverTest = RingHashLoadBalancerTest; INSTANTIATE_TEST_SUITE_P(RingHashPrimaryOrFailover, RingHashLoadBalancerTest, ::testing::Values(true, false)); diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index 002191906dad..b9119ea16cf2 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -35,7 +35,7 @@ class SubsetLoadBalancerDescribeMetadataTester { public: SubsetLoadBalancerDescribeMetadataTester(std::shared_ptr lb) : lb_(lb) {} - typedef std::vector> MetadataVector; + using MetadataVector = std::vector>; void test(std::string expected, const MetadataVector& metadata) { SubsetLoadBalancer::SubsetMetadata subset_metadata(metadata); @@ -113,8 +113,8 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { least_request_lb_config_.mutable_choice_count()->set_value(2); } - typedef std::map HostMetadata; - typedef std::map HostURLMetadataMap; + using HostMetadata = std::map; + using HostURLMetadataMap = std::map; void init() { init({ diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index dd20fc41b22d..e2b058264d2f 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -108,8 +108,8 @@ struct ResolverData { Network::MockActiveDnsQuery active_dns_query_; }; -typedef std::tuple> - StrictDnsConfigTuple; +using StrictDnsConfigTuple = + std::tuple>; std::vector generateStrictDnsParams() { std::vector dns_config; { diff --git a/test/config/utility.h b/test/config/utility.h index e131955425ee..5be493a51b9f 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -64,10 +64,9 @@ class ConfigHelper { static void initializeTls(const ServerSslOptions& options, envoy::api::v2::auth::CommonTlsContext& common_context); - typedef std::function ConfigModifierFunction; - typedef std::function - HttpModifierFunction; + using ConfigModifierFunction = std::function; + using HttpModifierFunction = std::function; // A basic configuration (admin port, cluster_0, one listener) with no network filters. static const std::string BASE_CONFIG; diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 8e6b25b2724f..7e69602477bc 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -469,9 +469,8 @@ class RedisClusterTest : public testing::Test, Network::MockActiveDnsQuery active_dns_query_; }; -typedef std::tuple, - std::list> - RedisDnsConfigTuple; +using RedisDnsConfigTuple = std::tuple, std::list>; std::vector generateRedisDnsParams() { std::vector dns_config; { diff --git a/test/extensions/filters/common/ext_authz/test_common.h b/test/extensions/filters/common/ext_authz/test_common.h index 52f08b2b9383..17d75d7e89f3 100644 --- a/test/extensions/filters/common/ext_authz/test_common.h +++ b/test/extensions/filters/common/ext_authz/test_common.h @@ -20,9 +20,9 @@ struct KeyValueOption { bool append; }; -typedef std::vector KeyValueOptionVector; -typedef std::vector HeaderValueOptionVector; -typedef std::unique_ptr CheckResponsePtr; +using KeyValueOptionVector = std::vector; +using HeaderValueOptionVector = std::vector; +using CheckResponsePtr = std::unique_ptr; class TestCommon { public: diff --git a/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc b/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc index aaf560bfbbfc..895712fb2382 100644 --- a/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc +++ b/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc @@ -5,7 +5,7 @@ namespace Envoy { namespace { -typedef HttpProtocolIntegrationTest BufferIntegrationTest; +using BufferIntegrationTest = HttpProtocolIntegrationTest; INSTANTIATE_TEST_SUITE_P(Protocols, BufferIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams()), diff --git a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc index 7997949a8bb0..f65637c7fcfb 100644 --- a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc @@ -80,7 +80,7 @@ std::string getFilterConfig(bool use_local_jwks) { return getAuthFilterConfig(ExampleConfig, use_local_jwks); } -typedef HttpProtocolIntegrationTest LocalJwksIntegrationTest; +using LocalJwksIntegrationTest = HttpProtocolIntegrationTest; INSTANTIATE_TEST_SUITE_P(Protocols, LocalJwksIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams()), diff --git a/test/extensions/filters/http/jwt_authn/group_verifier_test.cc b/test/extensions/filters/http/jwt_authn/group_verifier_test.cc index 424b8840b52f..0a9ca7f0a6bb 100644 --- a/test/extensions/filters/http/jwt_authn/group_verifier_test.cc +++ b/test/extensions/filters/http/jwt_authn/group_verifier_test.cc @@ -61,7 +61,7 @@ const char AnyWithAll[] = R"( - provider_name: "provider_4" )"; -typedef std::unordered_map StatusMap; +using StatusMap = std::unordered_map; constexpr auto allowfailed = "_allow_failed_"; diff --git a/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc b/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc index f27a1c2c9082..5e47c7bf86a7 100644 --- a/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc +++ b/test/extensions/filters/http/rbac/rbac_filter_integration_test.cc @@ -31,7 +31,7 @@ name: envoy.filters.http.rbac - any: true )EOF"; -typedef HttpProtocolIntegrationTest RBACIntegrationTest; +using RBACIntegrationTest = HttpProtocolIntegrationTest; INSTANTIATE_TEST_SUITE_P(Protocols, RBACIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams()), diff --git a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc index 96f5d313d442..f33943c94100 100644 --- a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc @@ -25,7 +25,7 @@ class DecoderStateMachineTestBase { DecoderStateMachineTestBase() : metadata_(std::make_shared()) { context_.header_size_ = 16; } - virtual ~DecoderStateMachineTestBase() {} + virtual ~DecoderStateMachineTestBase() = default; void initHandler() { EXPECT_CALL(decoder_callback_, newDecoderEventHandler()) diff --git a/test/extensions/filters/network/kafka/request_codec_unit_test.cc b/test/extensions/filters/network/kafka/request_codec_unit_test.cc index 3685f019759e..d26c0dfa08a5 100644 --- a/test/extensions/filters/network/kafka/request_codec_unit_test.cc +++ b/test/extensions/filters/network/kafka/request_codec_unit_test.cc @@ -28,7 +28,7 @@ class MockParser : public RequestParser { MOCK_METHOD1(parse, RequestParseResponse(absl::string_view&)); }; -typedef std::shared_ptr MockParserSharedPtr; +using MockParserSharedPtr = std::shared_ptr; class MockRequestParserResolver : public RequestParserResolver { public: @@ -43,7 +43,7 @@ class MockRequestCallback : public RequestCallback { MOCK_METHOD1(onFailedParse, void(RequestParseFailureSharedPtr)); }; -typedef std::shared_ptr MockRequestCallbackSharedPtr; +using MockRequestCallbackSharedPtr = std::shared_ptr; class RequestCodecUnitTest : public testing::Test { protected: diff --git a/test/extensions/filters/network/kafka/serialization_utilities.h b/test/extensions/filters/network/kafka/serialization_utilities.h index e7f28ba24779..3063bda95525 100644 --- a/test/extensions/filters/network/kafka/serialization_utilities.h +++ b/test/extensions/filters/network/kafka/serialization_utilities.h @@ -137,7 +137,7 @@ class CapturingRequestCallback : public RequestCallback { std::vector parse_failures_; }; -typedef std::shared_ptr CapturingRequestCallbackSharedPtr; +using CapturingRequestCallbackSharedPtr = std::shared_ptr; } // namespace Kafka } // namespace NetworkFilters diff --git a/test/extensions/filters/network/thrift_proxy/decoder_test.cc b/test/extensions/filters/network/thrift_proxy/decoder_test.cc index 6b61c020747f..74d706a5416e 100644 --- a/test/extensions/filters/network/thrift_proxy/decoder_test.cc +++ b/test/extensions/filters/network/thrift_proxy/decoder_test.cc @@ -181,7 +181,7 @@ ExpectationSet expectContainerEnd(MockProtocol& proto, MockDecoderEventHandler& class DecoderStateMachineTestBase { public: DecoderStateMachineTestBase() : metadata_(std::make_shared()) {} - virtual ~DecoderStateMachineTestBase() {} + virtual ~DecoderStateMachineTestBase() = default; NiceMock proto_; MessageMetadataSharedPtr metadata_; diff --git a/test/extensions/filters/network/thrift_proxy/thrift_object_impl_test.cc b/test/extensions/filters/network/thrift_proxy/thrift_object_impl_test.cc index bd973f88cbb0..7c4bad9614e1 100644 --- a/test/extensions/filters/network/thrift_proxy/thrift_object_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/thrift_object_impl_test.cc @@ -27,7 +27,7 @@ namespace ThriftProxy { class ThriftObjectImplTestBase { public: - virtual ~ThriftObjectImplTestBase() {} + virtual ~ThriftObjectImplTestBase() = default; Expectation expectValue(FieldType field_type) { switch (field_type) { diff --git a/test/extensions/health_checkers/redis/config_test.cc b/test/extensions/health_checkers/redis/config_test.cc index fe759c9354ef..2dcb9a439127 100644 --- a/test/extensions/health_checkers/redis/config_test.cc +++ b/test/extensions/health_checkers/redis/config_test.cc @@ -14,7 +14,7 @@ namespace HealthCheckers { namespace RedisHealthChecker { namespace { -typedef Extensions::HealthCheckers::RedisHealthChecker::RedisHealthChecker CustomRedisHealthChecker; +using CustomRedisHealthChecker = Extensions::HealthCheckers::RedisHealthChecker::RedisHealthChecker; TEST(HealthCheckerFactoryTest, CreateRedis) { const std::string yaml = R"EOF( diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index 5810778974f0..69f46e25cf75 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -636,7 +636,7 @@ TEST_F(ZipkinDriverTest, DuplicatedHeader) { Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, start_time_, {Tracing::Reason::Sampling, false}); - typedef std::function DupCallback; + using DupCallback = std::function; DupCallback dup_callback = [](absl::string_view key) -> bool { static absl::flat_hash_map dup; if (dup.find(key) == dup.end()) { diff --git a/test/integration/autonomous_upstream.h b/test/integration/autonomous_upstream.h index d79f8012c0fb..bdfc8a8eaa73 100644 --- a/test/integration/autonomous_upstream.h +++ b/test/integration/autonomous_upstream.h @@ -44,7 +44,7 @@ class AutonomousHttpConnection : public FakeHttpConnection { std::vector streams_; }; -typedef std::unique_ptr AutonomousHttpConnectionPtr; +using AutonomousHttpConnectionPtr = std::unique_ptr; // An upstream which creates AutonomousHttpConnection for new incoming connections. class AutonomousUpstream : public FakeUpstream { diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 4dc386076038..7fa348ee5a52 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -180,7 +180,7 @@ class FakeStream : public Http::StreamDecoder, Event::TestTimeSystem& time_system_; }; -typedef std::unique_ptr FakeStreamPtr; +using FakeStreamPtr = std::unique_ptr; // Encapsulates various state and functionality related to sharing a Connection object across // threads. With FakeUpstream fabricated objects, we have a Connection that is associated with a @@ -289,10 +289,10 @@ class SharedConnectionWrapper : public Network::ConnectionCallbacks { const bool allow_unexpected_disconnects_; }; -typedef std::unique_ptr SharedConnectionWrapperPtr; +using SharedConnectionWrapperPtr = std::unique_ptr; class QueuedConnectionWrapper; -typedef std::unique_ptr QueuedConnectionWrapperPtr; +using QueuedConnectionWrapperPtr = std::unique_ptr; /** * Wraps a raw Network::Connection in a safe way, such that the connection can @@ -435,7 +435,7 @@ class FakeHttpConnection : public Http::ServerConnectionCallbacks, public FakeCo std::list new_streams_; }; -typedef std::unique_ptr FakeHttpConnectionPtr; +using FakeHttpConnectionPtr = std::unique_ptr; /** * Fake raw connection for integration testing. @@ -444,7 +444,7 @@ class FakeRawConnection : public FakeConnectionBase { public: FakeRawConnection(SharedConnectionWrapper& shared_connection, Event::TestTimeSystem& time_system) : FakeConnectionBase(shared_connection, time_system) {} - typedef const std::function ValidatorFunction; + using ValidatorFunction = const std::function; // Writes to data. If data is nullptr, discards the received data. ABSL_MUST_USE_RESULT @@ -500,7 +500,7 @@ class FakeRawConnection : public FakeConnectionBase { std::string data_; }; -typedef std::unique_ptr FakeRawConnectionPtr; +using FakeRawConnectionPtr = std::unique_ptr; /** * Provides a fake upstream server for integration testing. @@ -624,6 +624,6 @@ class FakeUpstream : Logger::Loggable, const Network::FilterChainSharedPtr filter_chain_; }; -typedef std::unique_ptr FakeUpstreamPtr; +using FakeUpstreamPtr = std::unique_ptr; } // namespace Envoy diff --git a/test/integration/filter_manager_integration_test.cc b/test/integration/filter_manager_integration_test.cc index c2ac8c0ee221..4a0501e09624 100644 --- a/test/integration/filter_manager_integration_test.cc +++ b/test/integration/filter_manager_integration_test.cc @@ -47,7 +47,7 @@ class TestWithAuxiliaryFilter { explicit TestWithAuxiliaryFilter(const std::string& auxiliary_filter_name) : auxiliary_filter_name_(auxiliary_filter_name) {} - virtual ~TestWithAuxiliaryFilter() {} + virtual ~TestWithAuxiliaryFilter() = default; protected: /** diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index 00beabdd0114..5db6086cf572 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -73,7 +73,7 @@ class IntegrationCodecClient : public Http::CodecClientProd { Network::ConnectionEvent last_connection_event_; }; -typedef std::unique_ptr IntegrationCodecClientPtr; +using IntegrationCodecClientPtr = std::unique_ptr; /** * Test fixture for HTTP and HTTP/2 integration tests. @@ -142,7 +142,7 @@ class HttpIntegrationTest : public BaseIntegrationTest { void checkSimpleRequestSuccess(uint64_t expected_request_size, uint64_t expected_response_size, IntegrationStreamDecoder* response); - typedef std::function ConnectionCreationFunction; + using ConnectionCreationFunction = std::function; // Sends a simple header-only HTTP request, and waits for a response. IntegrationStreamDecoderPtr makeHeaderOnlyRequest(ConnectionCreationFunction* create_connection, int upstream_index, diff --git a/test/integration/http_protocol_integration.h b/test/integration/http_protocol_integration.h index c9429a4bd332..6233e54865f1 100644 --- a/test/integration/http_protocol_integration.h +++ b/test/integration/http_protocol_integration.h @@ -16,7 +16,7 @@ struct HttpProtocolTestParams { // // Usage: // -// typedef HttpProtocolIntegrationTest MyTest +// using MyTest = HttpProtocolIntegrationTest; // // INSTANTIATE_TEST_SUITE_P(Protocols, MyTest, // testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams()), diff --git a/test/integration/integration.h b/test/integration/integration.h index 20ef89b818ed..72f00790a9c6 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -80,7 +80,7 @@ class IntegrationStreamDecoder : public Http::StreamDecoder, public Http::Stream Http::StreamResetReason reset_reason_{}; }; -typedef std::unique_ptr IntegrationStreamDecoderPtr; +using IntegrationStreamDecoderPtr = std::unique_ptr; /** * TCP client used during integration testing. @@ -119,7 +119,7 @@ class IntegrationTcpClient { MockWatermarkBuffer* client_write_buffer_; }; -typedef std::unique_ptr IntegrationTcpClientPtr; +using IntegrationTcpClientPtr = std::unique_ptr; struct ApiFilesystemConfig { std::string bootstrap_path_; @@ -150,7 +150,7 @@ class BaseIntegrationTest : Logger::Loggable { Network::Address::IpVersion version, const std::string& config = ConfigHelper::HTTP_PROXY_CONFIG); - virtual ~BaseIntegrationTest() {} + virtual ~BaseIntegrationTest() = default; // TODO(jmarantz): Remove this once // https://github.com/envoyproxy/envoy-filter-example/pull/69 is reverted. diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 41b7451acea1..3c302f3c8e33 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -83,7 +83,7 @@ class DownstreamProtocolIntegrationTest : public HttpProtocolIntegrationTest { // Tests for ProtocolIntegrationTest will be run with the full mesh of H1/H2 // downstream and H1/H2 upstreams. -typedef HttpProtocolIntegrationTest ProtocolIntegrationTest; +using ProtocolIntegrationTest = HttpProtocolIntegrationTest; TEST_P(ProtocolIntegrationTest, ShutdownWithActiveConnPoolConnections) { auto response = makeHeaderOnlyRequest(nullptr, 0); diff --git a/test/integration/server.h b/test/integration/server.h index ac8877d9dc0d..801b4ac75865 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -215,7 +215,7 @@ class TestIsolatedStoreImpl : public StoreRoot { } // namespace Stats class IntegrationTestServer; -typedef std::unique_ptr IntegrationTestServerPtr; +using IntegrationTestServerPtr = std::unique_ptr; /** * Wrapper for running the real server for the purpose of integration tests. diff --git a/test/integration/server_stats.h b/test/integration/server_stats.h index 34ca6839a585..859363aa0f11 100644 --- a/test/integration/server_stats.h +++ b/test/integration/server_stats.h @@ -7,7 +7,7 @@ namespace Envoy { // Abstract interface for IntegrationTestServer stats methods. class IntegrationTestServerStats { public: - virtual ~IntegrationTestServerStats() {} + virtual ~IntegrationTestServerStats() = default; /** * Wait for a counter to == a given value. diff --git a/test/integration/tcp_dump.h b/test/integration/tcp_dump.h index 902242cc86b5..6f9028855de6 100644 --- a/test/integration/tcp_dump.h +++ b/test/integration/tcp_dump.h @@ -20,6 +20,6 @@ class TcpDump { int tcpdump_pid_; }; -typedef std::unique_ptr TcpDumpPtr; +using TcpDumpPtr = std::unique_ptr; } // namespace Envoy diff --git a/test/integration/utility.h b/test/integration/utility.h index 2a3a3f1c016b..7d3dc1b2fcdc 100644 --- a/test/integration/utility.h +++ b/test/integration/utility.h @@ -52,14 +52,14 @@ class BufferingStreamDecoder : public Http::StreamDecoder, public Http::StreamCa std::function on_complete_cb_; }; -typedef std::unique_ptr BufferingStreamDecoderPtr; +using BufferingStreamDecoderPtr = std::unique_ptr; /** * Basic driver for a raw connection. */ class RawConnectionDriver { public: - typedef std::function ReadCallback; + using ReadCallback = std::function; RawConnectionDriver(uint32_t port, Buffer::Instance& initial_data, ReadCallback data_callback, Network::Address::IpVersion version); diff --git a/test/integration/xds_integration_test.cc b/test/integration/xds_integration_test.cc index c02698a45b4c..218ea948dc78 100644 --- a/test/integration/xds_integration_test.cc +++ b/test/integration/xds_integration_test.cc @@ -40,7 +40,7 @@ TEST_P(XdsIntegrationTest, RouterRequestAndResponseWithBodyNoBuffer) { testRouterRequestAndResponseWithBody(1024, 512, false); } -typedef HttpProtocolIntegrationTest LdsIntegrationTest; +using LdsIntegrationTest = HttpProtocolIntegrationTest; INSTANTIATE_TEST_SUITE_P(Protocols, LdsIntegrationTest, testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( diff --git a/test/mocks/buffer/mocks.h b/test/mocks/buffer/mocks.h index 36ce8dd0afe3..6713259969f7 100644 --- a/test/mocks/buffer/mocks.h +++ b/test/mocks/buffer/mocks.h @@ -76,7 +76,7 @@ class MockBuffer : public MockBufferBase { class MockWatermarkBuffer : public MockBufferBase { public: - typedef MockBufferBase BaseClass; + using BaseClass = MockBufferBase; MockWatermarkBuffer(std::function below_low, std::function above_high) : BaseClass(below_low, above_high) { diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 5fdf7f72cafc..1aca75fb3158 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -71,7 +71,7 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable TestListenerPtr; + using TestListenerPtr = std::unique_ptr; TestListener* addListener( uint64_t tag, bool bind_to_port, bool hand_off_restored_destination_connections, diff --git a/test/test_common/environment.h b/test/test_common/environment.h index 43fae4c923bb..f77d54226dcb 100644 --- a/test/test_common/environment.h +++ b/test/test_common/environment.h @@ -15,9 +15,9 @@ namespace Envoy { class TestEnvironment { public: - typedef std::unordered_map PortMap; + using PortMap = std::unordered_map; - typedef std::unordered_map ParamMap; + using ParamMap = std::unordered_map; /** * Initialize command-line options for later access by tests in getOptions(). diff --git a/test/test_common/logging.h b/test/test_common/logging.h index c4aa391ca90c..b56161c96979 100644 --- a/test/test_common/logging.h +++ b/test/test_common/logging.h @@ -61,9 +61,9 @@ class LogRecordingSink : public Logger::SinkDelegate { std::vector messages_; }; -typedef std::pair StringPair; +using StringPair = std::pair; -typedef std::vector ExpectedLogMessages; +using ExpectedLogMessages = std::vector; // Below macros specify Envoy:: before class names so that the macro can be used outside of // namespace Envoy. diff --git a/test/test_common/printers.h b/test/test_common/printers.h index 945a056cb5a8..ca73b6f4c375 100644 --- a/test/test_common/printers.h +++ b/test/test_common/printers.h @@ -17,7 +17,7 @@ void PrintTo(const HeaderMapImpl& headers, std::ostream* os); * Pretty print const HeaderMapPtr& */ class HeaderMap; -typedef std::unique_ptr HeaderMapPtr; +using HeaderMapPtr = std::unique_ptr; void PrintTo(const HeaderMap& headers, std::ostream* os); void PrintTo(const HeaderMapPtr& headers, std::ostream* os); } // namespace Http From 294d0d73665e6a3f329c54ed641ee1f45bd9fabf Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Thu, 13 Jun 2019 10:06:00 -0700 Subject: [PATCH 003/542] router: add UPSTREAM_REMOTE_ADDRESS header formatter (#7223) Implements %UPSTREAM_REMOTE_ADDRESS% http header formatting. If used before an upstream has been selected, the values are empty strings. Risk Level: low Testing: unit tests Doc Changes: added documentation Release Notes: added Signed-off-by: Stephan Zuercher --- docs/root/configuration/http_conn_man/headers.rst | 4 ++++ docs/root/intro/version_history.rst | 4 +++- source/common/router/header_formatter.cc | 7 +++++++ test/common/router/header_formatter_test.cc | 9 +++++++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/root/configuration/http_conn_man/headers.rst b/docs/root/configuration/http_conn_man/headers.rst index 492dc850d0ac..be47a4e45232 100644 --- a/docs/root/configuration/http_conn_man/headers.rst +++ b/docs/root/configuration/http_conn_man/headers.rst @@ -602,6 +602,10 @@ Supported variable names are: namespace and key(s) are specified as a JSON array of strings. Finally, percent symbols in the parameters **do not** need to be escaped by doubling them. +%UPSTREAM_REMOTE_ADDRESS% + Remote address of the upstream host. If the address is an IP address it includes both address + and port. + %PER_REQUEST_STATE(reverse.dns.data.name)% Populates the header with values set on the stream info filterState() object. To be usable in custom request/response headers, these values must be of type diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b82a86a72e05..cb231b444dd3 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -49,12 +49,14 @@ Version history * router: add support for configuring a :ref:`grpc timeout offset ` on incoming requests. * router: added ability to control retry back-off intervals via :ref:`retry policy `. * router: added ability to issue a hedged retry in response to a per try timeout via a :ref:`hedge policy `. -* router: added a route name field to each http route in route.Route list +* router: added a route name field to each http route in route.Route list * router: added several new variables for exposing information about the downstream TLS connection via :ref:`header formatters `. * router: per try timeouts will no longer start before the downstream request has been received in full by the router. This ensures that the per try timeout does not account for slow downstreams and that will not start before the global timeout. +* router: added support for UPSTREAM_REMOTE_ADDRESS :ref:`header formatter + `. * runtime: added support for :ref:`flexible layering configuration `. * runtime: added support for statically :ref:`specifying the runtime in the bootstrap configuration diff --git a/source/common/router/header_formatter.cc b/source/common/router/header_formatter.cc index 06742da1c5c9..706fd282b10b 100644 --- a/source/common/router/header_formatter.cc +++ b/source/common/router/header_formatter.cc @@ -273,6 +273,13 @@ StreamInfoHeaderFormatter::StreamInfoHeaderFormatter(absl::string_view field_nam sslConnectionInfoStringTimeHeaderExtractor([](const Ssl::ConnectionInfo& connection_info) { return connection_info.expirationPeerCertificate(); }); + } else if (field_name == "UPSTREAM_REMOTE_ADDRESS") { + field_extractor_ = [](const Envoy::StreamInfo::StreamInfo& stream_info) -> std::string { + if (stream_info.upstreamHost()) { + return stream_info.upstreamHost()->address()->asString(); + } + return ""; + }; } else if (field_name.find("START_TIME") == 0) { const std::string pattern = fmt::format("%{}%", field_name); if (start_time_formatters_.find(pattern) == start_time_formatters_.end()) { diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 0d05981537e2..e0820fedf79e 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -71,6 +71,14 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalAddressWithou testFormatting("DOWNSTREAM_LOCAL_ADDRESS_WITHOUT_PORT", "127.0.0.2"); } +TEST_F(StreamInfoHeaderFormatterTest, TestformatWithUpstreamRemoteAddressVariable) { + testFormatting("UPSTREAM_REMOTE_ADDRESS", "10.0.0.1:443"); + + NiceMock stream_info; + stream_info.host_.reset(); + testFormatting(stream_info, "UPSTREAM_REMOTE_ADDRESS", ""); +} + TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithProtocolVariable) { NiceMock stream_info; absl::optional protocol = Envoy::Http::Protocol::Http11; @@ -668,6 +676,7 @@ TEST(HeaderParserTest, TestParseInternal) { {"%UPSTREAM_METADATA([\"ns\", \t \"key\"])%", {"value"}, {}}, {"%UPSTREAM_METADATA([\"ns\", \n \"key\"])%", {"value"}, {}}, {"%UPSTREAM_METADATA( \t [ \t \"ns\" \t , \t \"key\" \t ] \t )%", {"value"}, {}}, + {"%UPSTREAM_REMOTE_ADDRESS%", {"10.0.0.1:443"}, {}}, {"%PER_REQUEST_STATE(testing)%", {"test_value"}, {}}, {"%START_TIME%", {"2018-04-03T23:06:09.123Z"}, {}}, From 715a448156da4df7da72d5927fc890dfb1ed3930 Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 13 Jun 2019 17:19:00 -0400 Subject: [PATCH 004/542] fuzz: fix abrt from missing mock action (#7259) Signed-off-by: Asra Ali --- ...uzz-testcase-header_parser_fuzz_test-5710655463620608 | 9 +++++++++ test/fuzz/BUILD | 1 + test/fuzz/utility.h | 3 +++ test/mocks/ssl/mocks.h | 6 +++--- 4 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 test/common/router/header_parser_corpus/clusterfuzz-testcase-header_parser_fuzz_test-5710655463620608 diff --git a/test/common/router/header_parser_corpus/clusterfuzz-testcase-header_parser_fuzz_test-5710655463620608 b/test/common/router/header_parser_corpus/clusterfuzz-testcase-header_parser_fuzz_test-5710655463620608 new file mode 100644 index 000000000000..062b08993e46 --- /dev/null +++ b/test/common/router/header_parser_corpus/clusterfuzz-testcase-header_parser_fuzz_test-5710655463620608 @@ -0,0 +1,9 @@ +headers_to_add { + header { + key: "P\000\000\000\000\006b|H" + value: "%START_TIMEt_in(%?f,%%%% %5f%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%UPSTREAM_HOST%%%%%%%%%%[ZZZZZZ_%START_TIMEt_in(%?f,%%%% %5f%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%UPSTREAM_HOST%%%%%%%%DOWNSTREAM_PEER_CERT%%[ZZZZZZ_%START_TIMEt_in(%?f,%%%% %5f%%%%%%%%%%\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177%DOWNSTREAM_PEER_URI_SAN%\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177%DOWNSTREAM_LOCAL_URI_SAN%\177\177\177%DOWNSTREAM_PEER_CERT_V_START%\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%UPSTREAM_HOST%%%%%%%%%%[ZZZZZZ_%START_TIME(%)%" + } +} +stream_info { + start_time: 63 +} diff --git a/test/fuzz/BUILD b/test/fuzz/BUILD index 5cb9b78c9bf0..07273e9d721b 100644 --- a/test/fuzz/BUILD +++ b/test/fuzz/BUILD @@ -49,6 +49,7 @@ envoy_cc_test_library( hdrs = ["utility.h"], deps = [ ":common_proto_cc", + "//source/common/common:empty_string", "//source/common/network:utility_lib", "//test/common/stream_info:test_util", "//test/mocks/ssl:ssl_mocks", diff --git a/test/fuzz/utility.h b/test/fuzz/utility.h index c81473ce27a1..0f869149e2bf 100644 --- a/test/fuzz/utility.h +++ b/test/fuzz/utility.h @@ -1,5 +1,6 @@ #pragma once +#include "common/common/empty_string.h" #include "common/network/utility.h" #include "test/common/stream_info/test_util.h" @@ -87,6 +88,8 @@ inline test::fuzz::Headers toHeaders(const Http::HeaderMap& headers) { inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info, const Ssl::MockConnectionInfo* connection_info) { + // Set mocks' default string return value to be an empty string. + testing::DefaultValue::Set(EMPTY_STRING); TestStreamInfo test_stream_info; test_stream_info.metadata_ = stream_info.dynamic_metadata(); // libc++ clocks don't track at nanosecond on macOS. diff --git a/test/mocks/ssl/mocks.h b/test/mocks/ssl/mocks.h index 628becf70bb4..faf709bb6ef0 100644 --- a/test/mocks/ssl/mocks.h +++ b/test/mocks/ssl/mocks.h @@ -37,14 +37,14 @@ class MockConnectionInfo : public ConnectionInfo { MOCK_CONST_METHOD0(peerCertificatePresented, bool()); MOCK_CONST_METHOD0(uriSanLocalCertificate, std::vector()); - MOCK_CONST_METHOD0(sha256PeerCertificateDigest, std::string&()); + MOCK_CONST_METHOD0(sha256PeerCertificateDigest, const std::string&()); MOCK_CONST_METHOD0(serialNumberPeerCertificate, std::string()); MOCK_CONST_METHOD0(issuerPeerCertificate, std::string()); MOCK_CONST_METHOD0(subjectPeerCertificate, std::string()); MOCK_CONST_METHOD0(uriSanPeerCertificate, std::vector()); MOCK_CONST_METHOD0(subjectLocalCertificate, std::string()); - MOCK_CONST_METHOD0(urlEncodedPemEncodedPeerCertificate, std::string&()); - MOCK_CONST_METHOD0(urlEncodedPemEncodedPeerCertificateChain, std::string&()); + MOCK_CONST_METHOD0(urlEncodedPemEncodedPeerCertificate, const std::string&()); + MOCK_CONST_METHOD0(urlEncodedPemEncodedPeerCertificateChain, const std::string&()); MOCK_CONST_METHOD0(dnsSansPeerCertificate, std::vector()); MOCK_CONST_METHOD0(dnsSansLocalCertificate, std::vector()); MOCK_CONST_METHOD0(validFromPeerCertificate, absl::optional()); From de18671ab2f89ddb52adaa82e50cb592b4f69298 Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Thu, 13 Jun 2019 15:48:44 -0700 Subject: [PATCH 005/542] upstream: simplify warming cluster CDS pausing (#7236) Signed-off-by: Fred Douglas --- include/envoy/config/grpc_mux.h | 11 +- include/envoy/upstream/cluster_manager.h | 22 +- source/common/config/grpc_mux_impl.cc | 8 + source/common/config/grpc_mux_impl.h | 2 + source/common/upstream/cds_api_impl.cc | 33 +-- .../common/upstream/cluster_manager_impl.cc | 39 ++- source/common/upstream/cluster_manager_impl.h | 9 +- test/common/upstream/cds_api_impl_test.cc | 232 ++---------------- .../upstream/cluster_manager_impl_test.cc | 50 ++-- test/integration/ads_integration_test.cc | 8 + test/mocks/config/mocks.h | 1 + test/mocks/upstream/mocks.h | 5 +- 12 files changed, 103 insertions(+), 317 deletions(-) diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 1d66e2ddda77..3d5bf2c17c70 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -103,10 +103,17 @@ class GrpcMux { /** * Resume discovery requests for a given API type. This will send a discovery request if one would * have been sent during the pause. - * @param type_url type URL corresponding to xDS API, - * e.g.type.googleapis.com/envoy.api.v2.Cluster. + * @param type_url type URL corresponding to xDS API e.g. type.googleapis.com/envoy.api.v2.Cluster */ virtual void resume(const std::string& type_url) PURE; + + /** + * Retrieves the current pause state as set by pause()/resume(). + * @param type_url type URL corresponding to xDS API, e.g. + * type.googleapis.com/envoy.api.v2.Cluster + * @return bool whether the API is paused. + */ + virtual bool paused(const std::string& type_url) const PURE; }; using GrpcMuxPtr = std::unique_ptr; diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 59f1716d4150..7dfde10cc03c 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -76,25 +76,6 @@ class ClusterManager { public: virtual ~ClusterManager() = default; - /** - * Warming state a cluster is currently in. Used as an argument for the ClusterWarmingCallback. - */ - enum class ClusterWarmingState { - // Sent after cluster warming has finished. - Finished = 0, - // Sent just before cluster warming is about to start. - Starting = 1, - }; - - /** - * Called by the ClusterManager when cluster's warming state changes - * - * @param cluster_name name of the cluster. - * @param warming_state state the cluster transitioned to. - */ - using ClusterWarmingCallback = - std::function; - /** * Add or update a cluster via API. The semantics of this API are: * 1) The hash of the config is used to determine if an already existing cluster has changed. @@ -106,8 +87,7 @@ class ClusterManager { * @return true if the action results in an add/update of a cluster. */ virtual bool addOrUpdateCluster(const envoy::api::v2::Cluster& cluster, - const std::string& version_info, - ClusterWarmingCallback cluster_warming_cb) PURE; + const std::string& version_info) PURE; /** * Set a callback that will be invoked when all owned clusters have been initialized. diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 20d376df5264..4d9b32975bf0 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -114,6 +114,14 @@ void GrpcMuxImpl::resume(const std::string& type_url) { } } +bool GrpcMuxImpl::paused(const std::string& type_url) const { + auto entry = api_state_.find(type_url); + if (entry == api_state_.end()) { + return false; + } + return entry->second.paused_; +} + void GrpcMuxImpl::onDiscoveryResponse( std::unique_ptr&& message) { const std::string& type_url = message->type_url(); diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 473179b69501..1181ef0dfec5 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -36,6 +36,7 @@ class GrpcMuxImpl : public GrpcMux, GrpcMuxCallbacks& callbacks) override; void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; + bool paused(const std::string& type_url) const override; void sendDiscoveryRequest(const std::string& type_url); @@ -122,6 +123,7 @@ class NullGrpcMuxImpl : public GrpcMux { } void pause(const std::string&) override {} void resume(const std::string&) override {} + bool paused(const std::string&) const override { NOT_REACHED_GCOVR_EXCL_LINE; } }; } // namespace Config diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index ff375bc5ff50..483a5b145d16 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -70,39 +70,10 @@ void CdsApiImpl::onConfigUpdate( validation_visitor_); MessageUtil::validate(cluster); if (!cluster_names.insert(cluster.name()).second) { - // NOTE: at this point, the first of these duplicates has already been successfully - // applied. + // NOTE: at this point, the first of these duplicates has already been successfully applied. throw EnvoyException(fmt::format("duplicate cluster {} found", cluster.name())); } - if (cm_.addOrUpdateCluster( - cluster, resource.version(), - [this](const std::string&, ClusterManager::ClusterWarmingState state) { - // Following if/else block implements a control flow mechanism that can be used - // by an ADS implementation to properly sequence CDS and RDS update. It is not - // enforcing on ADS. ADS can use it to detect when a previously sent cluster - // becomes warm before sending routes that depend on it. This can improve - // incidence of HTTP 503 responses from Envoy when a route is used before it's - // supporting cluster is ready. - // - // We achieve that by leaving CDS in the paused state as long as there is at - // least one cluster in the warming state. This prevents CDS ACK from being sent - // to ADS. Once cluster is warmed up, CDS is resumed, and ACK is sent to ADS, - // providing a signal to ADS to proceed with RDS updates. - // - // Major concern with this approach is CDS being left in the paused state - // forever. As long as ClusterManager::removeCluster() is not called on a - // warming cluster this is not an issue. CdsApiImpl takes care of doing this - // properly, and there is no other component removing clusters from the - // ClusterManagerImpl. If this ever changes, we would need to correct the - // following logic. - if (state == ClusterManager::ClusterWarmingState::Starting && - cm_.warmingClusterCount() == 1) { - cm_.adsMux().pause(Config::TypeUrl::get().Cluster); - } else if (state == ClusterManager::ClusterWarmingState::Finished && - cm_.warmingClusterCount() == 0) { - cm_.adsMux().resume(Config::TypeUrl::get().Cluster); - } - })) { + if (cm_.addOrUpdateCluster(cluster, resource.version())) { any_applied = true; ENVOY_LOG(debug, "cds: add/update cluster '{}'", cluster.name()); } diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index b4d17fc8e128..e8d23e95b2c8 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -19,6 +19,7 @@ #include "common/common/fmt.h" #include "common/common/utility.h" #include "common/config/cds_json.h" +#include "common/config/resources.h" #include "common/config/utility.h" #include "common/grpc/async_client_manager_impl.h" #include "common/http/async_client_impl.h" @@ -242,7 +243,7 @@ ClusterManagerImpl::ClusterManagerImpl( } cm_stats_.cluster_added_.add(bootstrap.static_resources().clusters().size()); - updateGauges(); + updateClusterCounts(); absl::optional local_cluster_name; if (!cm_config.local_cluster_name().empty()) { @@ -444,8 +445,7 @@ void ClusterManagerImpl::applyUpdates(const Cluster& cluster, uint32_t priority, } bool ClusterManagerImpl::addOrUpdateCluster(const envoy::api::v2::Cluster& cluster, - const std::string& version_info, - ClusterWarmingCallback cluster_warming_cb) { + const std::string& version_info) { // First we need to see if this new config is new or an update to an existing dynamic cluster. // We don't allow updates to statically configured clusters in the main configuration. We check // both the warming clusters and the active clusters to see if we need an update or the update @@ -493,8 +493,7 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::api::v2::Cluster& clust } else { auto& cluster_entry = warming_clusters_.at(cluster_name); ENVOY_LOG(info, "add/update cluster {} starting warming", cluster_name); - cluster_warming_cb(cluster_name, ClusterWarmingState::Starting); - cluster_entry->cluster_->initialize([this, cluster_name, cluster_warming_cb] { + cluster_entry->cluster_->initialize([this, cluster_name] { auto warming_it = warming_clusters_.find(cluster_name); auto& cluster_entry = *warming_it->second; @@ -508,12 +507,11 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::api::v2::Cluster& clust ENVOY_LOG(info, "warming cluster {} complete", cluster_name); createOrUpdateThreadLocalCluster(cluster_entry); onClusterInit(*cluster_entry.cluster_); - cluster_warming_cb(cluster_name, ClusterWarmingState::Finished); - updateGauges(); + updateClusterCounts(); }); } - updateGauges(); + updateClusterCounts(); return true; } @@ -571,7 +569,7 @@ bool ClusterManagerImpl::removeCluster(const std::string& cluster_name) { if (removed) { cm_stats_.cluster_removed_.inc(); - updateGauges(); + updateClusterCounts(); // Cancel any pending merged updates. updates_map_.erase(cluster_name); } @@ -647,10 +645,29 @@ void ClusterManagerImpl::loadCluster(const envoy::api::v2::Cluster& cluster, cluster_entry_it->second->thread_aware_lb_ = std::move(new_cluster_pair.second); } - updateGauges(); + updateClusterCounts(); } -void ClusterManagerImpl::updateGauges() { +void ClusterManagerImpl::updateClusterCounts() { + // This if/else block implements a control flow mechanism that can be used by an ADS + // implementation to properly sequence CDS and RDS updates. It is not enforcing on ADS. ADS can + // use it to detect when a previously sent cluster becomes warm before sending routes that depend + // on it. This can improve incidence of HTTP 503 responses from Envoy when a route is used before + // it's supporting cluster is ready. + // + // We achieve that by leaving CDS in the paused state as long as there is at least + // one cluster in the warming state. This prevents CDS ACK from being sent to ADS. + // Once cluster is warmed up, CDS is resumed, and ACK is sent to ADS, providing a + // signal to ADS to proceed with RDS updates. + // If we're in the middle of shutting down (ads_mux_ already gone) then this is irrelevant. + if (ads_mux_) { + const uint64_t previous_warming = cm_stats_.warming_clusters_.value(); + if (previous_warming == 0 && !warming_clusters_.empty()) { + ads_mux_->pause(Config::TypeUrl::get().Cluster); + } else if (previous_warming > 0 && warming_clusters_.empty()) { + ads_mux_->resume(Config::TypeUrl::get().Cluster); + } + } cm_stats_.active_clusters_.set(active_clusters_.size()); cm_stats_.warming_clusters_.set(warming_clusters_.size()); } diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index d096e10afaff..abe595033747 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -177,8 +177,8 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable callback) override { init_helper_.setInitializedCb(callback); } @@ -212,7 +212,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable; using PendingUpdatesByPriorityMap = std::unordered_map; using PendingUpdatesByPriorityMapPtr = std::unique_ptr; @@ -439,7 +440,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::LoggablesetInitializedCb([this]() -> void { initialized_.ready(); }); EXPECT_CALL(*cm_.subscription_factory_.subscription_, start(_)); cds_->initialize(); cds_callbacks_ = cm_.subscription_factory_.callbacks_; } - void resetCdsInitializedCb() { - cds_->setInitializedCb([this]() -> void { - initialized_.ready(); - cm_.finishClusterWarming(); - }); + void expectAdd(const std::string& cluster_name, const std::string& version = std::string("")) { + EXPECT_CALL(cm_, addOrUpdateCluster(WithName(cluster_name), version)).WillOnce(Return(true)); + } + + void expectAddToThrow(const std::string& cluster_name, const std::string& exception_msg) { + EXPECT_CALL(cm_, addOrUpdateCluster(WithName(cluster_name), _)) + .WillOnce(Throw(EnvoyException(exception_msg))); } ClusterManager::ClusterInfoMap makeClusterMap(const std::vector& clusters) { @@ -60,66 +62,7 @@ class CdsApiImplTest : public testing::Test { return map; } - class MockWarmingClusterManager : public MockClusterManager { - public: - explicit MockWarmingClusterManager(TimeSource& time_source) : MockClusterManager(time_source) {} - - MockWarmingClusterManager() {} - - void expectAdd(const std::string& cluster_name, const std::string& version = std::string("")) { - EXPECT_CALL(*this, addOrUpdateCluster(WithName(cluster_name), version, _)) - .WillOnce(Return(true)); - } - - void expectAddToThrow(const std::string& cluster_name, const std::string& exception_msg) { - EXPECT_CALL(*this, addOrUpdateCluster(WithName(cluster_name), _, _)) - .WillOnce(Throw(EnvoyException(exception_msg))); - } - - void expectAddWithWarming(const std::string& cluster_name, const std::string& version, - bool immediately_warm_up = false) { - EXPECT_CALL(*this, addOrUpdateCluster(_, version, _)) - .WillOnce(Invoke([this, cluster_name, - immediately_warm_up](const envoy::api::v2::Cluster& cluster, - const std::string&, auto warming_cb) -> bool { - EXPECT_EQ(cluster_name, cluster.name()); - EXPECT_EQ(warming_cbs_.cend(), warming_cbs_.find(cluster.name())); - warming_cbs_[cluster.name()] = warming_cb; - warming_cb(cluster.name(), ClusterManager::ClusterWarmingState::Starting); - if (immediately_warm_up) { - warming_cbs_.erase(cluster.name()); - warming_cb(cluster.name(), ClusterManager::ClusterWarmingState::Finished); - } - return true; - })); - } - - void expectWarmingClusterCount(int times = 1) { - EXPECT_CALL(*this, warmingClusterCount()).Times(times).WillRepeatedly(Invoke([this]() { - return warming_cbs_.size(); - })); - } - - void finishClusterWarming() { - for (const auto& cluster : clusters_to_warm_up_) { - EXPECT_NE(warming_cbs_.cend(), warming_cbs_.find(cluster)); - auto callback = warming_cbs_[cluster]; - warming_cbs_.erase(cluster); - callback(cluster, ClusterManager::ClusterWarmingState::Finished); - } - clusters_to_warm_up_.clear(); - } - - void clustersToWarmUp(const std::vector&& clusters) { - clusters_to_warm_up_ = clusters; - } - - private: - std::map warming_cbs_; - std::vector clusters_to_warm_up_; - }; - - NiceMock cm_; + NiceMock cm_; Upstream::ClusterManager::ClusterInfoMap cluster_map_; Upstream::MockClusterMockPrioritySet mock_cluster_; Stats::IsolatedStoreImpl store_; @@ -164,7 +107,7 @@ version_info: '0' auto response1 = TestUtility::parseYaml(response1_yaml); EXPECT_CALL(cm_, clusters()).WillOnce(Return(ClusterManager::ClusterInfoMap{})); - cm_.expectAdd("cluster1", "0"); + expectAdd("cluster1", "0"); EXPECT_CALL(initialized_, ready()); EXPECT_EQ("", cds_->versionInfo()); @@ -227,12 +170,12 @@ TEST_F(CdsApiImplTest, ConfigUpdateWith2ValidClusters) { envoy::api::v2::Cluster cluster_1; cluster_1.set_name("cluster_1"); clusters.Add()->PackFrom(cluster_1); - cm_.expectAdd("cluster_1"); + expectAdd("cluster_1"); envoy::api::v2::Cluster cluster_2; cluster_2.set_name("cluster_2"); clusters.Add()->PackFrom(cluster_2); - cm_.expectAdd("cluster_2"); + expectAdd("cluster_2"); cds_callbacks_->onConfigUpdate(clusters, ""); } @@ -249,7 +192,7 @@ TEST_F(CdsApiImplTest, DeltaConfigUpdate) { { envoy::api::v2::Cluster cluster; cluster.set_name("cluster_1"); - cm_.expectAdd("cluster_1", "v1"); + expectAdd("cluster_1", "v1"); auto* resource = resources.Add(); resource->mutable_resource()->PackFrom(cluster); resource->set_name("cluster_1"); @@ -258,7 +201,7 @@ TEST_F(CdsApiImplTest, DeltaConfigUpdate) { { envoy::api::v2::Cluster cluster; cluster.set_name("cluster_2"); - cm_.expectAdd("cluster_2", "v1"); + expectAdd("cluster_2", "v1"); auto* resource = resources.Add(); resource->mutable_resource()->PackFrom(cluster); resource->set_name("cluster_2"); @@ -272,7 +215,7 @@ TEST_F(CdsApiImplTest, DeltaConfigUpdate) { { envoy::api::v2::Cluster cluster; cluster.set_name("cluster_3"); - cm_.expectAdd("cluster_3", "v2"); + expectAdd("cluster_3", "v2"); auto* resource = resources.Add(); resource->mutable_resource()->PackFrom(cluster); resource->set_name("cluster_3"); @@ -299,17 +242,17 @@ TEST_F(CdsApiImplTest, ConfigUpdateAddsSecondClusterEvenIfFirstThrows) { envoy::api::v2::Cluster cluster_1; cluster_1.set_name("cluster_1"); clusters.Add()->PackFrom(cluster_1); - cm_.expectAddToThrow("cluster_1", "An exception"); + expectAddToThrow("cluster_1", "An exception"); envoy::api::v2::Cluster cluster_2; cluster_2.set_name("cluster_2"); clusters.Add()->PackFrom(cluster_2); - cm_.expectAdd("cluster_2"); + expectAdd("cluster_2"); envoy::api::v2::Cluster cluster_3; cluster_3.set_name("cluster_3"); clusters.Add()->PackFrom(cluster_3); - cm_.expectAddToThrow("cluster_3", "Another exception"); + expectAddToThrow("cluster_3", "Another exception"); EXPECT_THROW_WITH_MESSAGE( cds_callbacks_->onConfigUpdate(clusters, ""), EnvoyException, @@ -340,8 +283,8 @@ version_info: '0' auto response1 = TestUtility::parseYaml(response1_yaml); EXPECT_CALL(cm_, clusters()).WillOnce(Return(ClusterManager::ClusterInfoMap{})); - cm_.expectAdd("cluster1", "0"); - cm_.expectAdd("cluster2", "0"); + expectAdd("cluster1", "0"); + expectAdd("cluster2", "0"); EXPECT_CALL(initialized_, ready()); EXPECT_EQ("", cds_->versionInfo()); cds_callbacks_->onConfigUpdate(response1.resources(), response1.version_info()); @@ -366,145 +309,14 @@ version_info: '1' auto response2 = TestUtility::parseYaml(response2_yaml); EXPECT_CALL(cm_, clusters()).WillOnce(Return(makeClusterMap({"cluster1", "cluster2"}))); - cm_.expectAdd("cluster1", "1"); - cm_.expectAdd("cluster3", "1"); + expectAdd("cluster1", "1"); + expectAdd("cluster3", "1"); EXPECT_CALL(cm_, removeCluster("cluster2")); cds_callbacks_->onConfigUpdate(response2.resources(), response2.version_info()); EXPECT_EQ("1", cds_->versionInfo()); } -TEST_F(CdsApiImplTest, CdsPauseOnWarming) { - EXPECT_CALL(cm_, clusters()).WillRepeatedly(Return(ClusterManager::ClusterInfoMap{})); - InSequence s; - - setup(); - - const std::string response1_yaml = R"EOF( -version_info: '0' -resources: -- "@type": type.googleapis.com/envoy.api.v2.Cluster - name: cluster1 - type: EDS - eds_cluster_config: - eds_config: - path: eds path -- "@type": type.googleapis.com/envoy.api.v2.Cluster - name: cluster2 - type: EDS - eds_cluster_config: - eds_config: - path: eds path -)EOF"; - auto response1 = TestUtility::parseYaml(response1_yaml); - - // Two clusters updated, both warmed up. - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)); - cm_.expectAddWithWarming("cluster1", "0"); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)); - cm_.expectAddWithWarming("cluster2", "0"); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(initialized_, ready()); - cm_.expectWarmingClusterCount(2); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)); - cm_.clustersToWarmUp({"cluster1", "cluster2"}); - cds_callbacks_->onConfigUpdate(response1.resources(), response1.version_info()); - - // Two clusters updated, only one warmed up. - const std::string response2_yaml = R"EOF( -version_info: '1' -resources: -- "@type": type.googleapis.com/envoy.api.v2.Cluster - name: cluster1 - type: EDS - eds_cluster_config: - eds_config: - path: eds path -- "@type": type.googleapis.com/envoy.api.v2.Cluster - name: cluster3 - type: EDS - eds_cluster_config: - eds_config: - path: eds path -)EOF"; - auto response2 = TestUtility::parseYaml(response2_yaml); - - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)); - cm_.expectAddWithWarming("cluster1", "1"); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)); - cm_.expectAddWithWarming("cluster3", "1"); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(initialized_, ready()); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)); - resetCdsInitializedCb(); - cm_.clustersToWarmUp({"cluster1"}); - cds_callbacks_->onConfigUpdate(response2.resources(), response2.version_info()); - - // One cluster updated and warmed up. Also finish warming up of the previously added cluster3. - const std::string response3_yaml = R"EOF( -version_info: '2' -resources: -- "@type": type.googleapis.com/envoy.api.v2.Cluster - name: cluster4 - type: EDS - eds_cluster_config: - eds_config: - path: eds path -)EOF"; - auto response3 = TestUtility::parseYaml(response3_yaml); - - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)); - cm_.expectAddWithWarming("cluster4", "2"); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(initialized_, ready()); - cm_.expectWarmingClusterCount(2); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)); - resetCdsInitializedCb(); - cm_.clustersToWarmUp({"cluster4", "cluster3"}); - cds_callbacks_->onConfigUpdate(response3.resources(), response3.version_info()); - - const std::string response4_yaml = R"EOF( -version_info: '3' -resources: -- "@type": type.googleapis.com/envoy.api.v2.Cluster - name: cluster5 - type: EDS - eds_cluster_config: - eds_config: - path: eds path -- "@type": type.googleapis.com/envoy.api.v2.Cluster - name: cluster6 - type: EDS - eds_cluster_config: - eds_config: - path: eds path -)EOF"; - auto response4 = TestUtility::parseYaml(response4_yaml); - - // Two clusters updated, first one warmed up before processing of the second one starts. - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)); - cm_.expectAddWithWarming("cluster5", "3", true); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)); - cm_.expectAddWithWarming("cluster6", "3"); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)); - EXPECT_CALL(initialized_, ready()); - cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)); - resetCdsInitializedCb(); - cm_.clustersToWarmUp({"cluster6"}); - cds_callbacks_->onConfigUpdate(response4.resources(), response4.version_info()); -} - // Validate behavior when the config is delivered but it fails PGV validation. TEST_F(CdsApiImplTest, FailureInvalidConfig) { InSequence s; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 002a15e3a2d4..47f469f73001 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -216,8 +216,6 @@ std::string clustersJson(const std::vector& clusters) { return fmt::sprintf("\"clusters\": [%s]", StringUtil::join(clusters, ",")); } -const ClusterManager::ClusterWarmingCallback dummyWarmingCb = [](auto, auto) {}; - class ClusterManagerImplTest : public testing::Test { public: ClusterManagerImplTest() @@ -1019,21 +1017,18 @@ TEST_F(ClusterManagerImplTest, InitializeOrder) { EXPECT_CALL(factory_, clusterFromProto_(_, _, _, _)) .WillOnce(Return(std::make_pair(cluster3, nullptr))); ON_CALL(*cluster3, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); - cluster_manager_->addOrUpdateCluster(defaultStaticCluster("cluster3"), "version1", - dummyWarmingCb); + cluster_manager_->addOrUpdateCluster(defaultStaticCluster("cluster3"), "version1"); EXPECT_CALL(factory_, clusterFromProto_(_, _, _, _)) .WillOnce(Return(std::make_pair(cluster4, nullptr))); ON_CALL(*cluster4, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Primary)); EXPECT_CALL(*cluster4, initialize(_)); - cluster_manager_->addOrUpdateCluster(defaultStaticCluster("cluster4"), "version2", - dummyWarmingCb); + cluster_manager_->addOrUpdateCluster(defaultStaticCluster("cluster4"), "version2"); EXPECT_CALL(factory_, clusterFromProto_(_, _, _, _)) .WillOnce(Return(std::make_pair(cluster5, nullptr))); ON_CALL(*cluster5, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); - cluster_manager_->addOrUpdateCluster(defaultStaticCluster("cluster5"), "version3", - dummyWarmingCb); + cluster_manager_->addOrUpdateCluster(defaultStaticCluster("cluster5"), "version3"); cds->initialized_callback_(); EXPECT_CALL(*cds, versionInfo()).WillOnce(Return("version3")); @@ -1166,7 +1161,7 @@ TEST_F(ClusterManagerImplTest, DynamicRemoveWithLocalCluster) { .WillOnce(Return(std::make_pair(cluster1, nullptr))); ON_CALL(*cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Primary)); EXPECT_CALL(*cluster1, initialize(_)); - cluster_manager_->addOrUpdateCluster(defaultStaticCluster("cluster1"), "", dummyWarmingCb); + cluster_manager_->addOrUpdateCluster(defaultStaticCluster("cluster1"), ""); // Add another update callback on foo so we make sure callbacks keep working. ReadyWatcher membership_updated; @@ -1205,8 +1200,8 @@ TEST_F(ClusterManagerImplTest, RemoveWarmingCluster) { .WillOnce(Return(std::make_pair(cluster1, nullptr))); EXPECT_CALL(*cluster1, initializePhase()).Times(0); EXPECT_CALL(*cluster1, initialize(_)); - EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "version1", - dummyWarmingCb)); + EXPECT_TRUE( + cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "version1")); checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 0 /*active*/, 1 /*warming*/); EXPECT_EQ(nullptr, cluster_manager_->get("fake_cluster")); checkConfigDump(R"EOF( @@ -1245,8 +1240,8 @@ TEST_F(ClusterManagerImplTest, ShutdownWithWarming) { .WillOnce(Return(std::make_pair(cluster1, nullptr))); EXPECT_CALL(*cluster1, initializePhase()).Times(0); EXPECT_CALL(*cluster1, initialize(_)); - EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "version1", - dummyWarmingCb)); + EXPECT_TRUE( + cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "version1")); checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 0 /*active*/, 1 /*warming*/); cluster_manager_->shutdown(); checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 0 /*active*/, 0 /*warming*/); @@ -1267,36 +1262,23 @@ TEST_F(ClusterManagerImplTest, DynamicAddRemove) { cluster_manager_->addThreadLocalClusterUpdateCallbacks(*callbacks); std::shared_ptr cluster1(new NiceMock()); - int warming_cb_calls = 0; - ClusterManager::ClusterWarmingState last_warming_state = - ClusterManager::ClusterWarmingState::Starting; EXPECT_CALL(factory_, clusterFromProto_(_, _, _, _)) .WillOnce(Return(std::make_pair(cluster1, nullptr))); EXPECT_CALL(*cluster1, initializePhase()).Times(0); EXPECT_CALL(*cluster1, initialize(_)); EXPECT_CALL(*callbacks, onClusterAddOrUpdate(_)).Times(1); - EXPECT_TRUE(cluster_manager_->addOrUpdateCluster( - defaultStaticCluster("fake_cluster"), "", - [&last_warming_state, &warming_cb_calls](auto, auto state) { - warming_cb_calls++; - last_warming_state = state; - })); + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "")); checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 0 /*active*/, 1 /*warming*/); EXPECT_EQ(1, cluster_manager_->warmingClusterCount()); EXPECT_EQ(nullptr, cluster_manager_->get("fake_cluster")); - EXPECT_EQ(1, warming_cb_calls); - EXPECT_EQ(ClusterManager::ClusterWarmingState::Starting, last_warming_state); cluster1->initialize_callback_(); EXPECT_EQ(cluster1->info_, cluster_manager_->get("fake_cluster")->info()); checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 1 /*active*/, 0 /*warming*/); EXPECT_EQ(0, cluster_manager_->warmingClusterCount()); - EXPECT_EQ(2, warming_cb_calls); - EXPECT_EQ(ClusterManager::ClusterWarmingState::Finished, last_warming_state); // Now try to update again but with the same hash. - EXPECT_FALSE(cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "", - dummyWarmingCb)); + EXPECT_FALSE(cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "")); // Now do it again with a different hash. auto update_cluster = defaultStaticCluster("fake_cluster"); @@ -1314,7 +1296,7 @@ TEST_F(ClusterManagerImplTest, DynamicAddRemove) { initialize_callback(); })); EXPECT_CALL(*callbacks, onClusterAddOrUpdate(_)).Times(1); - EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(update_cluster, "", dummyWarmingCb)); + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(update_cluster, "")); EXPECT_EQ(cluster2->info_, cluster_manager_->get("fake_cluster")->info()); EXPECT_EQ(1UL, cluster_manager_->clusters().size()); @@ -1385,8 +1367,7 @@ TEST_F(ClusterManagerImplTest, addOrUpdateClusterStaticExists) { EXPECT_CALL(initialized, ready()); cluster1->initialize_callback_(); - EXPECT_FALSE(cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "", - dummyWarmingCb)); + EXPECT_FALSE(cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "")); // Attempt to remove a static cluster. EXPECT_FALSE(cluster_manager_->removeCluster("fake_cluster")); @@ -2547,8 +2528,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) { common_lb_config: update_merge_window: 3s )EOF"; - EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(parseClusterFromV2Yaml(yaml), "version1", - dummyWarmingCb)); + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(parseClusterFromV2Yaml(yaml), "version1")); Cluster& cluster = cluster_manager_->activeClusters().find("new_cluster")->second; HostVectorSharedPtr hosts( @@ -2615,8 +2595,8 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) { EXPECT_EQ(0, factory_.stats_ .gauge("cluster_manager.warming_clusters", Stats::Gauge::ImportMode::NeverImport) .value()); - EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(parseClusterFromV2Yaml(yaml_updated), "version2", - dummyWarmingCb)); + EXPECT_TRUE( + cluster_manager_->addOrUpdateCluster(parseClusterFromV2Yaml(yaml_updated), "version2")); EXPECT_EQ(2, factory_.stats_ .gauge("cluster_manager.active_clusters", Stats::Gauge::ImportMode::NeverImport) .value()); diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 129947cf1bdf..8866d05da640 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -572,11 +572,15 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); makeSingleRequest(); + EXPECT_FALSE( + test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); // Send the first warming cluster. sendDiscoveryResponse( Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_1")}, {buildCluster("warming_cluster_1")}, {"cluster_0"}, "2"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + EXPECT_TRUE( + test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", {"warming_cluster_1"}, {"warming_cluster_1"}, {"cluster_0"})); @@ -591,6 +595,8 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { {"warming_cluster_2", "warming_cluster_1"}, {"warming_cluster_2"}, {})); + EXPECT_TRUE( + test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); // Finish warming the clusters. sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, @@ -602,6 +608,8 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { // Validate that clusters are warmed. test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + EXPECT_FALSE( + test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); // CDS is resumed and EDS response was acknowledged. EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index f7cd61191bce..8396fff4fc31 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -83,6 +83,7 @@ class MockGrpcMux : public GrpcMux { GrpcMuxCallbacks& callbacks) override; MOCK_METHOD1(pause, void(const std::string& type_url)); MOCK_METHOD1(resume, void(const std::string& type_url)); + MOCK_CONST_METHOD1(paused, bool(const std::string& type_url)); }; class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 00ae7bd017b1..e9b86baf2be6 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -299,9 +299,8 @@ class MockClusterManager : public ClusterManager { ClusterManagerFactory& clusterManagerFactory() override { return cluster_manager_factory_; } // Upstream::ClusterManager - MOCK_METHOD3(addOrUpdateCluster, - bool(const envoy::api::v2::Cluster& cluster, const std::string& version_info, - ClusterWarmingCallback cluster_warming_cb)); + MOCK_METHOD2(addOrUpdateCluster, + bool(const envoy::api::v2::Cluster& cluster, const std::string& version_info)); MOCK_METHOD1(setInitializedCb, void(std::function)); MOCK_METHOD0(clusters, ClusterInfoMap()); MOCK_METHOD1(get, ThreadLocalCluster*(absl::string_view cluster)); From d884048a442997caaed3031b54cc5ec3cfeb9476 Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 13 Jun 2019 18:51:27 -0400 Subject: [PATCH 006/542] fuzz: replace invalid characters *everywhere* (in virtual hosts) (#7263) Signed-off-by: Asra Ali --- ...minimized-route_fuzz_test-5731276071370752 | 27 +++++++++++++ test/common/router/route_fuzz_test.cc | 38 ++++++++++++++----- 2 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5731276071370752 diff --git a/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5731276071370752 b/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5731276071370752 new file mode 100644 index 000000000000..8a611620c540 --- /dev/null +++ b/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5731276071370752 @@ -0,0 +1,27 @@ +config { + virtual_hosts { + name: "e" + domains: "e" + cors { + filter_enabled { + default_value { + } + } + shadow_enabled { + default_value { + } + } + } + response_headers_to_add { + header { + key: "A\000\000\000\000\000\000\000" + } + } + } + vhds { + config_source { + ads { + } + } + } +} diff --git a/test/common/router/route_fuzz_test.cc b/test/common/router/route_fuzz_test.cc index 206bcd876575..0d9b6f7d0486 100644 --- a/test/common/router/route_fuzz_test.cc +++ b/test/common/router/route_fuzz_test.cc @@ -11,20 +11,38 @@ namespace Envoy { namespace Router { namespace { -// Return a new RouteConfiguration with invalid characters replaces in all header fields. -envoy::api::v2::RouteConfiguration -replaceInvalidHeaders(envoy::api::v2::RouteConfiguration route_config) { - envoy::api::v2::RouteConfiguration clean_config = route_config; +// A templated method to replace invalid characters in a protocol buffer that contains +// (request/response)_headers_to_(add/remove). +template T replaceInvalidHeaders(const T& config) { + T clean_config = config; clean_config.mutable_request_headers_to_add()->CopyFrom( - Fuzz::replaceInvalidHeaders(route_config.request_headers_to_add())); + Fuzz::replaceInvalidHeaders(config.request_headers_to_add())); clean_config.mutable_response_headers_to_add()->CopyFrom( - Fuzz::replaceInvalidHeaders(route_config.response_headers_to_add())); - auto internal_only_headers = clean_config.mutable_internal_only_headers(); - std::for_each(internal_only_headers->begin(), internal_only_headers->end(), - [](std::string& n) { n = Fuzz::replaceInvalidCharacters(n); }); + Fuzz::replaceInvalidHeaders(config.response_headers_to_add())); auto request_headers_to_remove = clean_config.mutable_request_headers_to_remove(); std::for_each(request_headers_to_remove->begin(), request_headers_to_remove->end(), [](std::string& n) { n = Fuzz::replaceInvalidCharacters(n); }); + auto response_headers_to_remove = clean_config.mutable_response_headers_to_remove(); + std::for_each(response_headers_to_remove->begin(), response_headers_to_remove->end(), + [](std::string& n) { n = Fuzz::replaceInvalidCharacters(n); }); + return clean_config; +} + +// Removes invalid headers from the RouteConfiguration as well as in each of the virtual hosts. +envoy::api::v2::RouteConfiguration +cleanRouteConfig(envoy::api::v2::RouteConfiguration route_config) { + envoy::api::v2::RouteConfiguration clean_config = + replaceInvalidHeaders(route_config); + auto internal_only_headers = clean_config.mutable_internal_only_headers(); + std::for_each(internal_only_headers->begin(), internal_only_headers->end(), + [](std::string& n) { n = Fuzz::replaceInvalidCharacters(n); }); + // Remove invalid characters in each virtual host. + auto virtual_hosts = clean_config.mutable_virtual_hosts(); + std::for_each(virtual_hosts->begin(), virtual_hosts->end(), + [](envoy::api::v2::route::VirtualHost& virtual_host) { + virtual_host = + replaceInvalidHeaders(virtual_host); + }); return clean_config; } @@ -34,7 +52,7 @@ DEFINE_PROTO_FUZZER(const test::common::router::RouteTestCase& input) { NiceMock stream_info; NiceMock factory_context; MessageUtil::validate(input.config()); - ConfigImpl config(replaceInvalidHeaders(input.config()), factory_context, true); + ConfigImpl config(cleanRouteConfig(input.config()), factory_context, true); Http::TestHeaderMapImpl headers = Fuzz::fromHeaders(input.headers()); // It's a precondition of routing that {:authority, :path, x-forwarded-proto} headers exists, // HCM enforces this. From 65cbe1430aec5b2a9639a297989ec23087fc729b Mon Sep 17 00:00:00 2001 From: cmluciano Date: Thu, 13 Jun 2019 18:59:14 -0400 Subject: [PATCH 007/542] build: add ppc llvm steps for ubuntu build container (#6970) Signed-off-by: Christopher M. Luciano --- ci/build_container/build_container_ubuntu.sh | 43 ++++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/ci/build_container/build_container_ubuntu.sh b/ci/build_container/build_container_ubuntu.sh index 3ffe5f1996ed..08697220d797 100755 --- a/ci/build_container/build_container_ubuntu.sh +++ b/ci/build_container/build_container_ubuntu.sh @@ -2,6 +2,8 @@ set -e +ARCH="$(uname -m)" + # Setup basic requirements and install them. apt-get update export DEBIAN_FRONTEND=noninteractive @@ -9,10 +11,24 @@ apt-get install -y wget software-properties-common make cmake git python python- unzip bc libtool ninja-build automake zip time golang gdb strace wireshark tshark tcpdump lcov \ apt-transport-https # clang 8. -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -apt-add-repository "deb https://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" -apt-get update -apt-get install -y clang-8 clang-format-8 clang-tidy-8 lld-8 libc++-8-dev libc++abi-8-dev +case $ARCH in + 'ppc64le' ) + LLVM_VERSION=8.0.0 + LLVM_RELEASE="clang+llvm-${LLVM_VERSION}-powerpc64le-unknown-unknown" + wget "https://releases.llvm.org/${LLVM_VERSION}/${LLVM_RELEASE}.tar.xz" + tar Jxf "${LLVM_RELEASE}.tar.xz" + mv "./${LLVM_RELEASE}" /opt/llvm + rm "./${LLVM_RELEASE}.tar.xz" + echo "/opt/llvm/lib" > /etc/ld.so.conf.d/llvm.conf + ldconfig + ;; + 'x86_64' ) + wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" + apt-get update + apt-get install -y clang-8 clang-format-8 clang-tidy-8 lld-8 libc++-8-dev libc++abi-8-dev + ;; +esac # gcc-7 add-apt-repository -y ppa:ubuntu-toolchain-r/test apt update @@ -25,10 +41,21 @@ update-alternatives --config g++ update-alternatives --config gcov # Bazel and related dependencies. apt-get install -y openjdk-8-jdk curl -echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list -curl https://bazel.build/bazel-release.pub.gpg | apt-key add - -apt-get update -apt-get install -y bazel +case $ARCH in + 'ppc64le' ) + BAZEL_LATEST="$(curl https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_16.04/latest/ 2>&1 \ + | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep '^bazel' | head -n 1)" + curl -fSL https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_16.04/latest/${BAZEL_LATEST} \ + -o /usr/local/bin/bazel + chmod +x /usr/local/bin/bazel + ;; + 'x86_64' ) + echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list + curl https://bazel.build/bazel-release.pub.gpg | apt-key add - + apt-get update + apt-get install -y bazel + ;; +esac apt-get install -y aspell rm -rf /var/lib/apt/lists/* From 3b4b5005985df400f2380aed7c7c53d55d96cc31 Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Thu, 13 Jun 2019 16:26:54 -0700 Subject: [PATCH 008/542] filter: check if connection is open before and after filter read/write (#7256) Signed-off-by: Yan Xue --- source/common/network/filter_manager_impl.cc | 20 +- .../network/filter_manager_impl_test.cc | 198 ++++++++++++++++++ 2 files changed, 214 insertions(+), 4 deletions(-) diff --git a/source/common/network/filter_manager_impl.cc b/source/common/network/filter_manager_impl.cc index 3a78ca849969..9701881a9f2f 100644 --- a/source/common/network/filter_manager_impl.cc +++ b/source/common/network/filter_manager_impl.cc @@ -38,6 +38,12 @@ bool FilterManagerImpl::initializeReadFilters() { void FilterManagerImpl::onContinueReading(ActiveReadFilter* filter, ReadBufferSource& buffer_source) { + // Filter could return status == FilterStatus::StopIteration immediately, close the connection and + // use callback to call this function. + if (connection_.state() != Connection::State::Open) { + return; + } + std::list::iterator entry; if (!filter) { entry = upstream_filters_.begin(); @@ -49,7 +55,7 @@ void FilterManagerImpl::onContinueReading(ActiveReadFilter* filter, if (!(*entry)->initialized_) { (*entry)->initialized_ = true; FilterStatus status = (*entry)->filter_->onNewConnection(); - if (status == FilterStatus::StopIteration) { + if (status == FilterStatus::StopIteration || connection_.state() != Connection::State::Open) { return; } } @@ -57,7 +63,7 @@ void FilterManagerImpl::onContinueReading(ActiveReadFilter* filter, StreamBuffer read_buffer = buffer_source.getReadBuffer(); if (read_buffer.buffer.length() > 0 || read_buffer.end_stream) { FilterStatus status = (*entry)->filter_->onData(read_buffer.buffer, read_buffer.end_stream); - if (status == FilterStatus::StopIteration) { + if (status == FilterStatus::StopIteration || connection_.state() != Connection::State::Open) { return; } } @@ -73,6 +79,12 @@ FilterStatus FilterManagerImpl::onWrite() { return onWrite(nullptr, connection_) FilterStatus FilterManagerImpl::onWrite(ActiveWriteFilter* filter, WriteBufferSource& buffer_source) { + // Filter could return status == FilterStatus::StopIteration immediately, close the connection and + // use callback to call this function. + if (connection_.state() != Connection::State::Open) { + return FilterStatus::StopIteration; + } + std::list::iterator entry; if (!filter) { entry = downstream_filters_.begin(); @@ -83,8 +95,8 @@ FilterStatus FilterManagerImpl::onWrite(ActiveWriteFilter* filter, for (; entry != downstream_filters_.end(); entry++) { StreamBuffer write_buffer = buffer_source.getWriteBuffer(); FilterStatus status = (*entry)->filter_->onWrite(write_buffer.buffer, write_buffer.end_stream); - if (status == FilterStatus::StopIteration) { - return status; + if (status == FilterStatus::StopIteration || connection_.state() != Connection::State::Open) { + return FilterStatus::StopIteration; } } diff --git a/test/common/network/filter_manager_impl_test.cc b/test/common/network/filter_manager_impl_test.cc index 0e8d2cd438f6..cccbac597e73 100644 --- a/test/common/network/filter_manager_impl_test.cc +++ b/test/common/network/filter_manager_impl_test.cc @@ -109,6 +109,204 @@ TEST_F(NetworkFilterManagerTest, All) { manager.onWrite(); } +TEST_F(NetworkFilterManagerTest, ConnectionClosedBeforeRunningFilter) { + InSequence s; + + Upstream::HostDescription* host_description(new NiceMock()); + MockReadFilter* read_filter(new MockReadFilter()); + MockFilter* filter(new LocalMockFilter()); + + FilterManagerImpl manager(connection_); + manager.addReadFilter(ReadFilterSharedPtr{read_filter}); + manager.addFilter(FilterSharedPtr{filter}); + + read_filter->callbacks_->upstreamHost(Upstream::HostDescriptionConstSharedPtr{host_description}); + EXPECT_EQ(read_filter->callbacks_->upstreamHost(), filter->callbacks_->upstreamHost()); + + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Closing)); + EXPECT_CALL(*read_filter, onNewConnection()).Times(0); + EXPECT_CALL(*read_filter, onData(_, _)).Times(0); + EXPECT_CALL(*filter, onNewConnection()).Times(0); + EXPECT_CALL(*filter, onData(_, _)).Times(0); + manager.onRead(); + + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Closed)); + EXPECT_CALL(*filter, onWrite(_, _)).Times(0); + manager.onWrite(); +} + +TEST_F(NetworkFilterManagerTest, FilterReturnStopAndNoCallback) { + InSequence s; + + Upstream::HostDescription* host_description(new NiceMock()); + MockReadFilter* read_filter(new MockReadFilter()); + MockWriteFilter* write_filter(new MockWriteFilter()); + MockFilter* filter(new LocalMockFilter()); + + FilterManagerImpl manager(connection_); + manager.addReadFilter(ReadFilterSharedPtr{read_filter}); + manager.addWriteFilter(WriteFilterSharedPtr{write_filter}); + manager.addFilter(FilterSharedPtr{filter}); + + read_filter->callbacks_->upstreamHost(Upstream::HostDescriptionConstSharedPtr{host_description}); + EXPECT_EQ(read_filter->callbacks_->upstreamHost(), filter->callbacks_->upstreamHost()); + + read_buffer_.add("hello"); + EXPECT_CALL(*read_filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*read_filter, onData(BufferStringEqual("hello"), _)) + .WillOnce(Return(FilterStatus::StopIteration)); + EXPECT_CALL(*filter, onNewConnection()).Times(0); + EXPECT_CALL(*filter, onData(_, _)).Times(0); + manager.onRead(); + + EXPECT_CALL(*filter, onWrite(_, _)).WillOnce(Return(FilterStatus::StopIteration)); + EXPECT_CALL(*write_filter, onWrite(_, _)).Times(0); + manager.onWrite(); +} + +TEST_F(NetworkFilterManagerTest, ReadFilterCloseConnectionAndReturnContinue) { + InSequence s; + + Upstream::HostDescription* host_description(new NiceMock()); + MockReadFilter* read_filter(new MockReadFilter()); + MockFilter* filter(new LocalMockFilter()); + + FilterManagerImpl manager(connection_); + manager.addReadFilter(ReadFilterSharedPtr{read_filter}); + manager.addFilter(FilterSharedPtr{filter}); + + read_filter->callbacks_->upstreamHost(Upstream::HostDescriptionConstSharedPtr{host_description}); + EXPECT_EQ(read_filter->callbacks_->upstreamHost(), filter->callbacks_->upstreamHost()); + + EXPECT_CALL(*read_filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_EQ(manager.initializeReadFilters(), true); + + read_buffer_.add("hello"); + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Open)); + EXPECT_CALL(*read_filter, onData(BufferStringEqual("hello"), _)) + .WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Closing)); + EXPECT_CALL(*filter, onData(_, _)).Times(0); + manager.onRead(); + + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Closed)); + EXPECT_CALL(*filter, onWrite(_, _)).Times(0); + manager.onWrite(); +} + +TEST_F(NetworkFilterManagerTest, WriteFilterCloseConnectionAndReturnContinue) { + InSequence s; + + Upstream::HostDescription* host_description(new NiceMock()); + MockReadFilter* read_filter(new MockReadFilter()); + MockWriteFilter* write_filter(new MockWriteFilter()); + MockFilter* filter(new LocalMockFilter()); + + FilterManagerImpl manager(connection_); + manager.addReadFilter(ReadFilterSharedPtr{read_filter}); + manager.addWriteFilter(WriteFilterSharedPtr{write_filter}); + manager.addFilter(FilterSharedPtr{filter}); + + read_filter->callbacks_->upstreamHost(Upstream::HostDescriptionConstSharedPtr{host_description}); + EXPECT_EQ(read_filter->callbacks_->upstreamHost(), filter->callbacks_->upstreamHost()); + + EXPECT_CALL(*read_filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_EQ(manager.initializeReadFilters(), true); + + read_buffer_.add("hello"); + EXPECT_CALL(*read_filter, onData(BufferStringEqual("hello"), _)) + .WillOnce(Return(FilterStatus::StopIteration)); + manager.onRead(); + + read_buffer_.add("world"); + EXPECT_CALL(*filter, onData(BufferStringEqual("helloworld"), _)) + .WillOnce(Return(FilterStatus::Continue)); + read_filter->callbacks_->continueReading(); + + write_buffer_.add("foo"); + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Open)); + EXPECT_CALL(*filter, onWrite(BufferStringEqual("foo"), _)) + .WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Closing)); + EXPECT_CALL(*write_filter, onWrite(_, _)).Times(0); + manager.onWrite(); +} + +TEST_F(NetworkFilterManagerTest, ReadCloseConnectionReturnStopAndCallback) { + InSequence s; + + Upstream::HostDescription* host_description(new NiceMock()); + MockReadFilter* read_filter(new MockReadFilter()); + MockWriteFilter* write_filter(new MockWriteFilter()); + MockFilter* filter(new LocalMockFilter()); + + FilterManagerImpl manager(connection_); + manager.addReadFilter(ReadFilterSharedPtr{read_filter}); + manager.addWriteFilter(WriteFilterSharedPtr{write_filter}); + manager.addFilter(FilterSharedPtr{filter}); + + read_filter->callbacks_->upstreamHost(Upstream::HostDescriptionConstSharedPtr{host_description}); + EXPECT_EQ(read_filter->callbacks_->upstreamHost(), filter->callbacks_->upstreamHost()); + + EXPECT_CALL(*read_filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_EQ(manager.initializeReadFilters(), true); + + read_buffer_.add("hello"); + EXPECT_CALL(*read_filter, onData(BufferStringEqual("hello"), _)) + .WillOnce(Return(FilterStatus::StopIteration)); + manager.onRead(); + + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Closing)); + EXPECT_CALL(*filter, onData(_, _)).Times(0); + read_filter->callbacks_->continueReading(); + + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Closed)); + EXPECT_CALL(*filter, onWrite(_, _)).Times(0); + manager.onWrite(); +} + +TEST_F(NetworkFilterManagerTest, WriteCloseConnectionReturnStopAndCallback) { + InSequence s; + + Upstream::HostDescription* host_description(new NiceMock()); + MockReadFilter* read_filter(new MockReadFilter()); + MockWriteFilter* write_filter(new MockWriteFilter()); + MockFilter* filter(new LocalMockFilter()); + + FilterManagerImpl manager(connection_); + manager.addReadFilter(ReadFilterSharedPtr{read_filter}); + manager.addWriteFilter(WriteFilterSharedPtr{write_filter}); + manager.addFilter(FilterSharedPtr{filter}); + + read_filter->callbacks_->upstreamHost(Upstream::HostDescriptionConstSharedPtr{host_description}); + EXPECT_EQ(read_filter->callbacks_->upstreamHost(), filter->callbacks_->upstreamHost()); + + EXPECT_CALL(*read_filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*filter, onNewConnection()).WillOnce(Return(FilterStatus::Continue)); + EXPECT_EQ(manager.initializeReadFilters(), true); + + read_buffer_.add("hello"); + EXPECT_CALL(*read_filter, onData(BufferStringEqual("hello"), _)) + .WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*filter, onData(BufferStringEqual("hello"), _)) + .WillOnce(Return(FilterStatus::Continue)); + manager.onRead(); + + write_buffer_.add("foo"); + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Open)); + EXPECT_CALL(*filter, onWrite(BufferStringEqual("foo"), _)) + .WillOnce(Return(FilterStatus::StopIteration)); + manager.onWrite(); + + EXPECT_CALL(connection_, state()).WillOnce(Return(Connection::State::Closed)); + EXPECT_CALL(*filter, onWrite(_, _)).Times(0); + EXPECT_CALL(*write_filter, onWrite(_, _)).Times(0); + manager.onWrite(); +} + // Test that end_stream is delivered in the correct order with the data, even // if FilterStatus::StopIteration occurs. TEST_F(NetworkFilterManagerTest, EndStream) { From ea63c3a9110558ef16ec032bd395a192c5eca316 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Thu, 13 Jun 2019 20:03:59 -0400 Subject: [PATCH 009/542] stats: Resolve loss of importMode for gauges that are initialized first, then merged. (#7255) Signed-off-by: Joshua Marantz --- include/envoy/stats/stats.h | 5 +- .../common/stats/stat_data_allocator_impl.h | 58 ++++++++++++------- source/common/stats/stat_merger.cc | 46 +++++++++------ test/common/stats/stat_merger_test.cc | 20 ++++++- test/integration/hotrestart_test.sh | 8 +++ 5 files changed, 95 insertions(+), 42 deletions(-) diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index 7dca08d43da4..ac302a08bb76 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -97,11 +97,8 @@ class Metric { */ struct Flags { static const uint8_t Used = 0x01; - // TODO(fredlas) these logic flags should be removed if we move to indicating combine logic in - // the stat declaration macros themselves. (Now that stats no longer use shared memory, it's - // safe to mess with what these flag bits mean whenever we want). static const uint8_t LogicAccumulate = 0x02; - static const uint8_t ImportModeUninitialized = 0x04; // Stat was discovered during import. + static const uint8_t NeverImport = 0x04; }; virtual SymbolTable& symbolTable() PURE; virtual const SymbolTable& constSymbolTable() const PURE; diff --git a/source/common/stats/stat_data_allocator_impl.h b/source/common/stats/stat_data_allocator_impl.h index 6c7b63e203fd..578d23e9c0b6 100644 --- a/source/common/stats/stat_data_allocator_impl.h +++ b/source/common/stats/stat_data_allocator_impl.h @@ -119,10 +119,19 @@ template class GaugeImpl : public Gauge, public MetricImpl { absl::string_view tag_extracted_name, const std::vector& tags, ImportMode import_mode) : MetricImpl(tag_extracted_name, tags, alloc.symbolTable()), data_(data), alloc_(alloc) { - if (import_mode == ImportMode::Uninitialized) { - data_.flags_ |= Flags::ImportModeUninitialized; - } else if (import_mode == ImportMode::Accumulate) { + switch (import_mode) { + case ImportMode::Accumulate: data_.flags_ |= Flags::LogicAccumulate; + break; + case ImportMode::NeverImport: + data_.flags_ |= Flags::NeverImport; + break; + case ImportMode::Uninitialized: + // Note that we don't clear any flag bits for import_mode==Uninitialized, + // as we may have an established import_mode when this stat was created in + // an alternate scope. See + // https://github.com/envoyproxy/envoy/issues/7227. + break; } } ~GaugeImpl() override { @@ -155,29 +164,38 @@ template class GaugeImpl : public Gauge, public MetricImpl { bool used() const override { return data_.flags_ & Flags::Used; } ImportMode importMode() const override { - if (data_.flags_ & Flags::ImportModeUninitialized) { - return ImportMode::Uninitialized; + if (data_.flags_ & Flags::NeverImport) { + return ImportMode::NeverImport; } else if (data_.flags_ & Flags::LogicAccumulate) { return ImportMode::Accumulate; } - return ImportMode::NeverImport; + return ImportMode::Uninitialized; } void mergeImportMode(ImportMode import_mode) override { - if (import_mode != ImportMode::Uninitialized && - (data_.flags_ & Flags::ImportModeUninitialized)) { - data_.flags_ &= ~Flags::ImportModeUninitialized; - if (import_mode == ImportMode::Accumulate) { - data_.flags_ |= Flags::LogicAccumulate; - } else { - // A previous revision of Envoy must have transferred a gauge that it - // thought was Accumulate. But the new version thinks its NeverImport, - // so we clear the accumulated value. - data_.value_ = 0; - data_.flags_ &= ~Flags::Used; - } - } else { - ASSERT(importMode() == import_mode); + ImportMode current = importMode(); + if (current == import_mode) { + return; + } + + switch (import_mode) { + case ImportMode::Uninitialized: + // mergeImportNode(ImportMode::Uninitialized) is called when merging an + // existing stat with importMode() == Accumulate or NeverImport. + break; + case ImportMode::Accumulate: + ASSERT(current == ImportMode::Uninitialized); + data_.flags_ |= Flags::LogicAccumulate; + break; + case ImportMode::NeverImport: + ASSERT(current == ImportMode::Uninitialized); + // A previous revision of Envoy may have transferred a gauge that it + // thought was Accumulate. But the new version thinks it's NeverImport, so + // we clear the accumulated value. + data_.value_ = 0; + data_.flags_ &= ~Flags::Used; + data_.flags_ |= Flags::NeverImport; + break; } } diff --git a/source/common/stats/stat_merger.cc b/source/common/stats/stat_merger.cc index b978ee5a6e35..0aab1fe0b48e 100644 --- a/source/common/stats/stat_merger.cc +++ b/source/common/stats/stat_merger.cc @@ -1,7 +1,5 @@ #include "common/stats/stat_merger.h" -#include - namespace Envoy { namespace Stats { @@ -15,15 +13,36 @@ void StatMerger::mergeCounters(const Protobuf::Map& count void StatMerger::mergeGauges(const Protobuf::Map& gauges) { for (const auto& gauge : gauges) { + // Merging gauges via RPC from the parent has 4 cases; case 2 and 4b are the + // most common. + // + // 1. Parent process thinks gauge is NeverImport: no data sent, and we + // do not run this loop for such a gauge. Only if the parent process + // thinks the gauge is Accumulate do we consider cases 2-4. + // 2. Child thinks gauge is Accumulate : data is combined in + // gauge_ref.add() below. + // 3. Child thinks gauge is NeverImport: we skip this loop entry via + // 'continue'. This only happens with a code-change where the child + // contains a code-change relative to parent, switching a Gauge from + // Accumulate to NeverImport. + // 4. Child has not yet initialized gauge yet -- this merge is the + // first time the child learns of the gauge. It's possible the child + // will think the gauge is NeverImport due to a code change. But for + // now we will leave the gauge in the child process as + // import_mode==Uninitialized, and accumulate the parent value in + // gauge_ref.add(). Gauges in this mode will not be included in + // stats-sinks or the admin /stats calls, until the child initializes + // the gauge, in which case: + // 4a. Child later initializes gauges as NeverImport: the parent value is + // cleared during the mergeImportMode call. + // 4b. Child later initializes gauges as Accumulate: the parent value is + // retained. + StatNameManagedStorage storage(gauge.first, temp_scope_->symbolTable()); StatName stat_name = storage.statName(); absl::optional> gauge_opt = temp_scope_->findGauge(stat_name); - // If the stat named in the protobuf map is already initialized, and has a - // mode of NeverImport, then we simply skip over the map entry. This is a - // case where a new revision of Envoy has been built where a previously - // Accumulated gauge has been switched to NeverImport mode. Gauge::ImportMode import_mode = Gauge::ImportMode::Uninitialized; if (gauge_opt) { import_mode = gauge_opt->get().importMode(); @@ -32,23 +51,16 @@ void StatMerger::mergeGauges(const Protobuf::Map& gauges) } } - // We establish here a tentative import-mode of Uninitialized. Gauges in - // this mode will not be reported in stats sinks or in the admin /stats - // endpoint. However, we'll retain the transferred value, and if the running - // system initialized the stat as Accumulate, we'll have the accumulated - // value ready to go. If the system re-initializes it as NeverImport, we'll - // clear the value during the mergeImportMode call. auto& gauge_ref = temp_scope_->gaugeFromStatName(stat_name, import_mode); uint64_t& parent_value_ref = parent_gauge_values_[gauge_ref.statName()]; uint64_t old_parent_value = parent_value_ref; uint64_t new_parent_value = gauge.second; parent_value_ref = new_parent_value; - if (new_parent_value > old_parent_value) { - gauge_ref.add(new_parent_value - old_parent_value); - } else { - gauge_ref.sub(old_parent_value - new_parent_value); - } + // Note that new_parent_value may be less than old_parent_value, in which + // case 2s complement does its magic (-1 == 0xffffffffffffffff) and adding + // that to the gauge's current value works the same as subtraction. + gauge_ref.add(new_parent_value - old_parent_value); } } diff --git a/test/common/stats/stat_merger_test.cc b/test/common/stats/stat_merger_test.cc index 504a2a2946f6..bd437374803e 100644 --- a/test/common/stats/stat_merger_test.cc +++ b/test/common/stats/stat_merger_test.cc @@ -207,7 +207,6 @@ TEST_F(StatMergerThreadLocalTest, newStatFromParent) { { StatMerger stat_merger(store_); - Protobuf::Map counter_values; Protobuf::Map counter_deltas; Protobuf::Map gauges; counter_deltas["newcounter0"] = 0; @@ -231,6 +230,25 @@ TEST_F(StatMergerThreadLocalTest, newStatFromParent) { EXPECT_FALSE(TestUtility::findGauge(store_, "newgauge2")); } +// Verify that if we create a stat in the child process which then gets merged +// from the parent, that we retain the import-mode, accumulating the updated +// value. https://github.com/envoyproxy/envoy/issues/7227 +TEST_F(StatMergerThreadLocalTest, retainImportModeAfterMerge) { + Gauge& gauge = store_.gauge("mygauge", Gauge::ImportMode::Accumulate); + gauge.set(42); + EXPECT_EQ(Gauge::ImportMode::Accumulate, gauge.importMode()); + EXPECT_EQ(42, gauge.value()); + { + StatMerger stat_merger(store_); + Protobuf::Map counter_deltas; + Protobuf::Map gauges; + gauges["mygauge"] = 789; + stat_merger.mergeStats(counter_deltas, gauges); + } + EXPECT_EQ(Gauge::ImportMode::Accumulate, gauge.importMode()); + EXPECT_EQ(789 + 42, gauge.value()); +} + } // namespace } // namespace Stats } // namespace Envoy diff --git a/test/integration/hotrestart_test.sh b/test/integration/hotrestart_test.sh index 07fda57455a3..f5c30aecef6f 100755 --- a/test/integration/hotrestart_test.sh +++ b/test/integration/hotrestart_test.sh @@ -116,6 +116,10 @@ do --max-obj-name-len 500 2>&1) check [ "${ADMIN_HOT_RESTART_VERSION}" = "${CLI_HOT_RESTART_VERSION}" ] + # Verify we can see server.live in the admin port. + SERVER_LIVE_0=$(curl -sg http://${ADMIN_ADDRESS_0}/stats | grep server.live) + check [ "$SERVER_LIVE_0" = "server.live: 1" ]; + enableHeapCheck start_test Starting epoch 1 @@ -129,6 +133,10 @@ do # Wait for stat flushing sleep 7 + ADMIN_ADDRESS_1=$(cat "${ADMIN_ADDRESS_PATH_1}") + SERVER_LIVE_1=$(curl -sg http://${ADMIN_ADDRESS_1}/stats | grep server.live) + check [ "$SERVER_LIVE_1" = "server.live: 2" ]; + start_test Checking that listener addresses have not changed HOT_RESTART_JSON_1="${TEST_TMPDIR}"/hot_restart.1."${TEST_INDEX}".yaml "${TEST_RUNDIR}"/tools/socket_passing "-o" "${UPDATED_HOT_RESTART_JSON}" "-a" "${ADMIN_ADDRESS_PATH_1}" \ From 9c80afd6d45a42deee07f84f42dec39b8ce0cac7 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Thu, 13 Jun 2019 20:31:58 -0700 Subject: [PATCH 010/542] format: fix the ability to find buildifier in non-standard path. (#7270) Signed-off-by: Piotr Sikora --- tools/check_format_test_helper.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/check_format_test_helper.py b/tools/check_format_test_helper.py index 45b36d149c95..646512239866 100755 --- a/tools/check_format_test_helper.py +++ b/tools/check_format_test_helper.py @@ -14,8 +14,6 @@ import subprocess import sys -os.putenv("BUILDIFIER_BIN", "/usr/local/bin/buildifier") - tools = os.path.dirname(os.path.realpath(__file__)) tmp = os.path.join(os.getenv('TEST_TMPDIR', "/tmp"), "check_format_test") src = os.path.join(tools, 'testdata', 'check_format') From 1a60b343665cf2ffb966f37bbe48fed21805df57 Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 14 Jun 2019 06:10:00 +0200 Subject: [PATCH 011/542] upstream: Fallback policy per selector support (#7087) This code change allows to redefine fallback policy per specific subset selector. Because of how existing LbSubsetMap trie data structure is organised (mapping subset key to values), is not possible to do lookups for fallback policy only based on subset keys (had to introduce additional trie that maps subset keys to keys and has fallback policy on leaf level). Additional LbSubsetSelectorFallbackPolicy enum required to correctly identify the case when fallback policy is not set for given selector (otherwise it would always default to NO_FALLBACK, breaking backwards compatibility, if field is not set we should use top level fallback policy instead). Risk Level: Medium Testing: Done Docs Changes: Updated related docs Release Notes: added Fixes #5130 Signed-off-by: Kateryna Nezdolii --- api/envoy/api/v2/cds.proto | 18 + .../arch_overview/load_balancing/subsets.rst | 11 +- docs/root/intro/version_history.rst | 1 + include/envoy/upstream/load_balancer_type.h | 10 +- source/common/upstream/load_balancer_impl.h | 11 +- source/common/upstream/subset_lb.cc | 126 +++++- source/common/upstream/subset_lb.h | 29 +- .../upstream/load_balancer_impl_test.cc | 21 +- test/common/upstream/subset_lb_test.cc | 385 +++++++++++++++--- test/mocks/upstream/cluster_info.cc | 2 +- test/mocks/upstream/cluster_info.h | 4 +- 11 files changed, 544 insertions(+), 74 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 633218817564..4f79fda5ab53 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -367,6 +367,24 @@ message Cluster { message LbSubsetSelector { // List of keys to match with the weighted cluster metadata. repeated string keys = 1; + // The behavior used when no endpoint subset matches the selected route's + // metadata. + LbSubsetSelectorFallbackPolicy fallback_policy = 2 + [(validate.rules).enum.defined_only = true]; + + // Allows to override top level fallback policy per selector. + enum LbSubsetSelectorFallbackPolicy { + // If NOT_DEFINED top level config fallback policy is used instead. + NOT_DEFINED = 0; + // If NO_FALLBACK is selected, a result equivalent to no healthy hosts is reported. + NO_FALLBACK = 1; + // If ANY_ENDPOINT is selected, any cluster endpoint may be returned + // (subject to policy, health checks, etc). + ANY_ENDPOINT = 2; + // If DEFAULT_SUBSET is selected, load balancing is performed over the + // endpoints matching the values from the default_subset field. + DEFAULT_SUBSET = 3; + } } // For each entry, LbEndpoint.Metadata's diff --git a/docs/root/intro/arch_overview/load_balancing/subsets.rst b/docs/root/intro/arch_overview/load_balancing/subsets.rst index 637d8d3bc23a..710a72f43b40 100644 --- a/docs/root/intro/arch_overview/load_balancing/subsets.rst +++ b/docs/root/intro/arch_overview/load_balancing/subsets.rst @@ -18,7 +18,8 @@ specifies no metadata or no subset matching the metadata exists, the subset load its fallback policy. The default policy is ``NO_FALLBACK``, in which case the request fails as if the cluster had no hosts. Conversely, the ``ANY_ENDPOINT`` fallback policy load balances across all hosts in the cluster, without regard to host metadata. Finally, the ``DEFAULT_SUBSET`` causes -fallback to load balance among hosts that match a specific set of metadata. +fallback to load balance among hosts that match a specific set of metadata. It is possible to +override fallback policy for specific subset selector. Subsets must be predefined to allow the subset load balancer to efficiently select the correct subset of hosts. Each definition is a set of keys, which translates to zero or more @@ -79,20 +80,22 @@ The cluster may enable subset load balancing like this: - stage - keys: - stage + fallback_policy: NO_FALLBACK The following table describes some routes and the result of their application to the cluster. Typically the match criteria would be used with routes matching specific aspects of the request, such as the path or header information. -====================== ============= ========================================== +====================== ============= ====================================================================== Match Criteria Balances Over Reason -====================== ============= ========================================== +====================== ============= ====================================================================== stage: canary host3 Subset of hosts selected v: 1.2-pre, stage: dev host4 Subset of hosts selected v: 1.0 host1, host2 Fallback: No subset selector for "v" alone other: x host1, host2 Fallback: No subset selector for "other" (none) host1, host2 Fallback: No subset requested -====================== ============= ========================================== +stage: test empty cluster As fallback policy is overriden per selector with "NO_FALLBACK" value +====================== ============= ====================================================================== Metadata match criteria may also be specified on a route's weighted clusters. Metadata match criteria from the selected weighted cluster are merged with and override the criteria from the diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index cb231b444dd3..9c700155aebc 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -76,6 +76,7 @@ Version history that allows ignoring new hosts for the purpose of load balancing calculations until they have been health checked for the first time. * upstream: added runtime error checking to prevent setting dns type to STRICT_DNS or LOGICAL_DNS when custom resolver name is specified. +* upstream: added possibility to override fallback_policy per specific selector in :ref:`subset load balancer `. * upstream: the :ref:`logical DNS cluster ` now displays the current resolved IP address in admin output instead of 0.0.0.0. diff --git a/include/envoy/upstream/load_balancer_type.h b/include/envoy/upstream/load_balancer_type.h index 5c508aa4edc3..91848df3e34a 100644 --- a/include/envoy/upstream/load_balancer_type.h +++ b/include/envoy/upstream/load_balancer_type.h @@ -25,6 +25,14 @@ enum class LoadBalancerType { ClusterProvided }; +struct SubsetSelector { + std::set selector_keys_; + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::LbSubsetSelectorFallbackPolicy + fallback_policy_; +}; + +using SubsetSelectorPtr = std::shared_ptr; + /** * Load Balancer subset configuration. */ @@ -54,7 +62,7 @@ class LoadBalancerSubsetInfo { * @return const std:vector>& a vector of * sorted keys used to define load balancer subsets. */ - virtual const std::vector>& subsetKeys() const PURE; + virtual const std::vector& subsetSelectors() const PURE; /* * @return bool whether routing to subsets should take locality weights into account. diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 7b26302c35cd..2f5d0de1360b 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -488,8 +488,9 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { panic_mode_any_(subset_config.panic_mode_any()) { for (const auto& subset : subset_config.subset_selectors()) { if (!subset.keys().empty()) { - subset_keys_.emplace_back( - std::set(subset.keys().begin(), subset.keys().end())); + subset_selectors_.emplace_back(std::make_shared( + SubsetSelector{std::set(subset.keys().begin(), subset.keys().end()), + subset.fallback_policy()})); } } } @@ -500,7 +501,9 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { return fallback_policy_; } const ProtobufWkt::Struct& defaultSubset() const override { return default_subset_; } - const std::vector>& subsetKeys() const override { return subset_keys_; } + const std::vector& subsetSelectors() const override { + return subset_selectors_; + } bool localityWeightAware() const override { return locality_weight_aware_; } bool scaleLocalityWeight() const override { return scale_locality_weight_; } bool panicModeAny() const override { return panic_mode_any_; } @@ -509,7 +512,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { const bool enabled_; const envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy fallback_policy_; const ProtobufWkt::Struct default_subset_; - std::vector> subset_keys_; + std::vector subset_selectors_; const bool locality_weight_aware_; const bool scale_locality_weight_; const bool panic_mode_any_; diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index a8ccfec9245c..197c8d1ef9a2 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -29,7 +29,7 @@ SubsetLoadBalancer::SubsetLoadBalancer( scope_(scope), runtime_(runtime), random_(random), fallback_policy_(subsets.fallbackPolicy()), default_subset_metadata_(subsets.defaultSubset().fields().begin(), subsets.defaultSubset().fields().end()), - subset_keys_(subsets.subsetKeys()), original_priority_set_(priority_set), + subset_selectors_(subsets.subsetSelectors()), original_priority_set_(priority_set), original_local_priority_set_(local_priority_set), locality_weight_aware_(subsets.localityWeightAware()), scale_locality_weight_(subsets.scaleLocalityWeight()) { @@ -66,6 +66,8 @@ SubsetLoadBalancer::SubsetLoadBalancer( // Create filtered default subset (if necessary) and other subsets based on current hosts. refreshSubsets(); + initSubsetSelectorMap(); + // Configure future updates. original_priority_set_callback_handle_ = priority_set.addPriorityUpdateCb( [this](uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed) { @@ -110,6 +112,60 @@ void SubsetLoadBalancer::refreshSubsets(uint32_t priority) { update(priority, host_sets[priority]->hosts(), {}); } +void SubsetLoadBalancer::initSubsetSelectorMap() { + selectors_ = std::make_shared(); + SubsetSelectorMapPtr selectors; + for (const auto& subset_selector : subset_selectors_) { + const auto& selector_keys = subset_selector->selector_keys_; + const auto& selector_fallback_policy = subset_selector->fallback_policy_; + if (selector_fallback_policy == + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED) { + continue; + } + uint32_t pos = 0; + selectors = selectors_; + for (const auto& key : selector_keys) { + const auto& selector_it = selectors->subset_keys_.find(key); + pos++; + if (selector_it == selectors->subset_keys_.end()) { + selectors->subset_keys_.emplace(std::make_pair(key, std::make_shared())); + const auto& child_selector = selectors->subset_keys_.find(key); + // if this is last key for given selector, check if it has fallback specified + if (pos == selector_keys.size()) { + child_selector->second->fallback_policy_ = selector_fallback_policy; + initSelectorFallbackSubset(selector_fallback_policy); + } + selectors = child_selector->second; + } else { + selectors = selector_it->second; + } + } + selectors = selectors_; + } +} + +void SubsetLoadBalancer::initSelectorFallbackSubset( + const envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::LbSubsetSelectorFallbackPolicy& + fallback_policy) { + if (fallback_policy == envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT && + selector_fallback_subset_any_ == nullptr) { + ENVOY_LOG(debug, "subset lb: creating any-endpoint fallback load balancer for selector"); + HostPredicate predicate = [](const Host&) -> bool { return true; }; + selector_fallback_subset_any_ = std::make_shared(); + selector_fallback_subset_any_->priority_subset_.reset( + new PrioritySubsetImpl(*this, predicate, locality_weight_aware_, scale_locality_weight_)); + } else if (fallback_policy == + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET && + selector_fallback_subset_default_ == nullptr) { + ENVOY_LOG(debug, "subset lb: creating default subset fallback load balancer for selector"); + HostPredicate predicate = std::bind(&SubsetLoadBalancer::hostMatches, this, + default_subset_metadata_, std::placeholders::_1); + selector_fallback_subset_default_ = std::make_shared(); + selector_fallback_subset_default_->priority_subset_.reset( + new PrioritySubsetImpl(*this, predicate, locality_weight_aware_, scale_locality_weight_)); + } +} + HostConstSharedPtr SubsetLoadBalancer::chooseHost(LoadBalancerContext* context) { if (context) { bool host_chosen; @@ -118,6 +174,14 @@ HostConstSharedPtr SubsetLoadBalancer::chooseHost(LoadBalancerContext* context) // Subset lookup succeeded, return this result even if it's nullptr. return host; } + // otherwise check if there is fallback policy configured for given route metadata + absl::optional< + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::LbSubsetSelectorFallbackPolicy> + selector_fallback_policy = tryFindSelectorFallbackPolicy(context); + if (selector_fallback_policy) { + // return result according to configured fallback policy + return chooseHostForSelectorFallbackPolicy(selector_fallback_policy.value(), context); + } } if (fallback_subset_ == nullptr) { @@ -141,6 +205,52 @@ HostConstSharedPtr SubsetLoadBalancer::chooseHost(LoadBalancerContext* context) return nullptr; } +absl::optional< + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::LbSubsetSelectorFallbackPolicy> +SubsetLoadBalancer::tryFindSelectorFallbackPolicy(LoadBalancerContext* context) { + const Router::MetadataMatchCriteria* match_criteria = context->metadataMatchCriteria(); + if (!match_criteria) { + return absl::nullopt; + } + const auto match_criteria_vec = match_criteria->metadataMatchCriteria(); + SubsetSelectorMapPtr selectors = selectors_; + if (selectors == nullptr) { + return absl::nullopt; + } + for (uint32_t i = 0; i < match_criteria_vec.size(); i++) { + const Router::MetadataMatchCriterion& match_criterion = *match_criteria_vec[i]; + const auto& subset_it = selectors->subset_keys_.find(match_criterion.name()); + if (subset_it == selectors->subset_keys_.end()) { + // No subsets with this key (at this level in the hierarchy). + break; + } + + if (i + 1 == match_criteria_vec.size()) { + // We've reached the end of the criteria, and they all matched. + return subset_it->second->fallback_policy_; + } + selectors = subset_it->second; + } + + return absl::nullopt; +} + +HostConstSharedPtr SubsetLoadBalancer::chooseHostForSelectorFallbackPolicy( + const envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::LbSubsetSelectorFallbackPolicy& + fallback_policy, + LoadBalancerContext* context) { + if (fallback_policy == envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT && + selector_fallback_subset_any_ != nullptr) { + return selector_fallback_subset_any_->priority_subset_->lb_->chooseHost(context); + } else if (fallback_policy == + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET && + selector_fallback_subset_default_ != nullptr) { + return selector_fallback_subset_default_->priority_subset_->lb_->chooseHost(context); + } else { + return nullptr; + } +} + // Find a host from the subsets. Sets host_chosen to false and returns nullptr if the context has // no metadata match criteria, if there is no matching subset, or if the matching subset contains // no hosts (ignoring health). Otherwise, host_chosen is true and the returns HostConstSharedPtr is @@ -206,6 +316,16 @@ SubsetLoadBalancer::LbSubsetEntryPtr SubsetLoadBalancer::findSubset( void SubsetLoadBalancer::updateFallbackSubset(uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed) { + + if (selector_fallback_subset_any_ != nullptr) { + selector_fallback_subset_any_->priority_subset_->update(priority, hosts_added, hosts_removed); + } + + if (selector_fallback_subset_default_ != nullptr) { + selector_fallback_subset_default_->priority_subset_->update(priority, hosts_added, + hosts_removed); + } + if (fallback_subset_ == nullptr) { ENVOY_LOG(debug, "subset lb: fallback load balancer disabled"); return; @@ -235,9 +355,9 @@ void SubsetLoadBalancer::processSubsets( for (const auto& step : steps) { const auto& hosts = step.first; const bool adding_hosts = step.second; - for (const auto& host : hosts) { - for (const auto& keys : subset_keys_) { + for (const auto& subset_selector : subset_selectors_) { + const auto& keys = subset_selector->selector_keys_; // For each host, for each subset key, attempt to extract the metadata corresponding to the // key from the host. SubsetMetadata kvs = extractSubsetMetadata(keys, *host); diff --git a/source/common/upstream/subset_lb.h b/source/common/upstream/subset_lb.h index 08499782ca26..fa45a4fa0dc3 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/common/upstream/subset_lb.h @@ -37,6 +37,14 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable; + void initSubsetSelectorMap(); + void initSelectorFallbackSubset(const envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy&); + HostConstSharedPtr chooseHostForSelectorFallbackPolicy( + const envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy& fallback_policy, + LoadBalancerContext* context); + // Represents a subset of an original HostSet. class HostSubsetImpl : public HostSetImpl { public: @@ -111,10 +119,19 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable>; class LbSubsetEntry; + struct SubsetSelectorMap; + using LbSubsetEntryPtr = std::shared_ptr; + using SubsetSelectorMapPtr = std::shared_ptr; using ValueSubsetMap = std::unordered_map; using LbSubsetMap = std::unordered_map; + struct SubsetSelectorMap { + std::unordered_map subset_keys_; + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::LbSubsetSelectorFallbackPolicy + fallback_policy_; + }; + // Entry in the subset hierarchy. class LbSubsetEntry { public: @@ -145,6 +162,10 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable + tryFindSelectorFallbackPolicy(LoadBalancerContext* context); + bool hostMatches(const SubsetMetadata& kvs, const Host& host); LbSubsetEntryPtr @@ -168,7 +189,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable> subset_keys_; + const std::vector subset_selectors_; const PrioritySet& original_priority_set_; const PrioritySet* original_local_priority_set_; @@ -177,8 +198,14 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggablemutable_fields()->insert({"key", subset_value}); - auto subset_selector = subset_config.mutable_subset_selectors()->Add(); - subset_selector->add_keys("selector_key"); + auto subset_selector1 = subset_config.mutable_subset_selectors()->Add(); + subset_selector1->add_keys("selector_key1"); + auto subset_selector2 = subset_config.mutable_subset_selectors()->Add(); + subset_selector2->add_keys("selector_key2"); + subset_selector2->set_fallback_policy( + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT); auto subset_info = LoadBalancerSubsetInfoImpl(subset_config); @@ -1419,8 +1423,15 @@ TEST(LoadBalancerSubsetInfoImplTest, SubsetConfig) { EXPECT_EQ(subset_info.defaultSubset().fields_size(), 1); EXPECT_EQ(subset_info.defaultSubset().fields().at("key").string_value(), std::string("the value")); - EXPECT_EQ(subset_info.subsetKeys().size(), 1); - EXPECT_EQ(subset_info.subsetKeys()[0], std::set({"selector_key"})); + EXPECT_EQ(subset_info.subsetSelectors().size(), 2); + EXPECT_EQ(subset_info.subsetSelectors()[0]->selector_keys_, + std::set({"selector_key1"})); + EXPECT_EQ(subset_info.subsetSelectors()[0]->fallback_policy_, + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED); + EXPECT_EQ(subset_info.subsetSelectors()[1]->selector_keys_, + std::set({"selector_key2"})); + EXPECT_EQ(subset_info.subsetSelectors()[1]->fallback_policy_, + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT); } } // namespace diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index b9119ea16cf2..44b64701f4f6 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -570,8 +570,11 @@ TEST_F(SubsetLoadBalancerTest, BalancesSubset) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, @@ -595,8 +598,11 @@ TEST_P(SubsetLoadBalancerTest, BalancesSubsetAfterUpdate) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, @@ -633,8 +639,12 @@ TEST_P(SubsetLoadBalancerTest, UpdateFailover) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + TestLoadBalancerContext context_10({{"version", "1.0"}}); // Start with an empty lb. Choosing a host should result in failure. @@ -660,15 +670,21 @@ TEST_P(SubsetLoadBalancerTest, OnlyMetadataChanged) { TestLoadBalancerContext context_12({{"version", "1.2"}}); TestLoadBalancerContext context_13({{"version", "1.3"}}); TestLoadBalancerContext context_default({{"default", "true"}}); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED}), + std::make_shared(SubsetSelector{ + {"default"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"default", "true"}}); EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - std::vector> subset_keys = {{"version"}, {"default"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); - // Add hosts initial hosts. init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, {"tcp://127.0.0.1:8001", {{"version", "1.0"}, {"default", "true"}}}}); @@ -748,8 +764,12 @@ TEST_P(SubsetLoadBalancerTest, MetadataChangedHostsAddedRemoved) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); - std::vector> subset_keys = {{"version"}, {"default"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED}), + std::make_shared(SubsetSelector{ + {"default"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); // Add hosts initial hosts. init({{"tcp://127.0.0.1:8000", {{"version", "1.2"}}}, @@ -827,8 +847,10 @@ TEST_P(SubsetLoadBalancerTest, UpdateRemovingLastSubsetHost) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, @@ -860,8 +882,14 @@ TEST_P(SubsetLoadBalancerTest, UpdateRemovingUnknownHost) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"stage", "version"}, {"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared( + SubsetSelector{{"stage", "version"}, + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED}), + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"stage", "prod"}, {"version", "1.0"}}}, @@ -882,8 +910,13 @@ TEST_F(SubsetLoadBalancerTest, UpdateModifyingOnlyHostHealth) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}, {"hardware"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED}), + std::make_shared(SubsetSelector{ + {"hardware"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, @@ -917,8 +950,13 @@ TEST_F(SubsetLoadBalancerTest, BalancesDisjointSubsets) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}, {"hardware"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED}), + std::make_shared(SubsetSelector{ + {"hardware"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"hardware", "std"}}}, @@ -940,11 +978,14 @@ TEST_F(SubsetLoadBalancerTest, BalancesOverlappingSubsets) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = { - {"stage", "version"}, - {"version"}, - }; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared( + SubsetSelector{{"stage", "version"}, + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED}), + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "prod"}}}, @@ -977,11 +1018,14 @@ TEST_F(SubsetLoadBalancerTest, BalancesNestedSubsets) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = { - {"stage", "version"}, - {"stage"}, - }; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared( + SubsetSelector{{"stage", "version"}, + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED}), + std::make_shared(SubsetSelector{ + {"stage"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "prod"}}}, @@ -1012,8 +1056,11 @@ TEST_F(SubsetLoadBalancerTest, IgnoresUnselectedMetadata) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}, {"stage", "ignored"}}}, @@ -1035,8 +1082,11 @@ TEST_F(SubsetLoadBalancerTest, IgnoresHostsWithoutMetadata) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); HostVector hosts; hosts.emplace_back(makeTestHost(info_, "tcp://127.0.0.1:80")); @@ -1081,8 +1131,11 @@ TEST_F(SubsetLoadBalancerTest, ZoneAwareFallback) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - std::vector> subset_keys = {{"x"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"x"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); common_config_.mutable_healthy_panic_threshold()->set_value(40); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 40)) @@ -1125,8 +1178,11 @@ TEST_P(SubsetLoadBalancerTest, ZoneAwareFallbackAfterUpdate) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - std::vector> subset_keys = {{"x"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"x"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillRepeatedly(Return(50)); @@ -1184,8 +1240,11 @@ TEST_F(SubsetLoadBalancerTest, ZoneAwareFallbackDefaultSubset) { const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillRepeatedly(Return(50)); @@ -1238,8 +1297,11 @@ TEST_P(SubsetLoadBalancerTest, ZoneAwareFallbackDefaultSubsetAfterUpdate) { const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillRepeatedly(Return(50)); @@ -1302,8 +1364,10 @@ TEST_F(SubsetLoadBalancerTest, ZoneAwareBalancesSubsets) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillRepeatedly(Return(50)); @@ -1355,8 +1419,10 @@ TEST_P(SubsetLoadBalancerTest, ZoneAwareBalancesSubsetsAfterUpdate) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillRepeatedly(Return(50)); @@ -1511,8 +1577,11 @@ TEST_F(SubsetLoadBalancerTest, EnabledLocalityWeightAwareness) { } TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeights) { - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); @@ -1552,8 +1621,11 @@ TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeights) { } TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeightsRounding) { - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); @@ -1590,8 +1662,10 @@ TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeightsRounding) { // Regression for bug where missing locality weights crashed scaling and locality aware subset LBs. TEST_F(SubsetLoadBalancerTest, ScaleLocalityWeightsWithNoLocalityWeights) { - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); EXPECT_CALL(subset_info_, localityWeightAware()).WillRepeatedly(Return(true)); EXPECT_CALL(subset_info_, scaleLocalityWeight()).WillRepeatedly(Return(true)); @@ -1612,8 +1686,10 @@ TEST_P(SubsetLoadBalancerTest, GaugesUpdatedOnDestroy) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - std::vector> subset_keys = {{"version"}}; - EXPECT_CALL(subset_info_, subsetKeys()).WillRepeatedly(ReturnRef(subset_keys)); + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, @@ -1628,6 +1704,209 @@ TEST_P(SubsetLoadBalancerTest, GaugesUpdatedOnDestroy) { EXPECT_EQ(1U, stats_.lb_subsets_removed_.value()); } +TEST_P(SubsetLoadBalancerTest, SubsetSelectorNoFallbackPerSelector) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.1"}}}, + {"tcp://127.0.0.1:83", {{"version", "1.1"}}}, + }); + + TestLoadBalancerContext context_10({{"version", "1.0"}}); + TestLoadBalancerContext context_11({{"version", "1.1"}}); + TestLoadBalancerContext context_12({{"version", "1.2"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_11)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_10)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_11)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_12)); + EXPECT_EQ(0U, stats_.lb_subsets_fallback_.value()); + EXPECT_EQ(4U, stats_.lb_subsets_selected_.value()); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorFallbackOverridesTopLevelOne) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init(); + + TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); + TestLoadBalancerContext context_unknown_value({{"version", "unknown"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_unknown_key)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorNoFallbackMatchesTopLevelOne) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init(); + + TestLoadBalancerContext context_unknown_key({{"unknown", "unknown"}}); + TestLoadBalancerContext context_unknown_value({{"version", "unknown"}}); + + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_key)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_unknown_value)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorDefaultAnyFallbackPerSelector) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET}), + std::make_shared(SubsetSelector{ + {"app"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT}), + std::make_shared(SubsetSelector{ + {"foo"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"bar", "default"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + // Add hosts initial hosts. + init({{"tcp://127.0.0.1:81", {{"version", "0.0"}}}, + {"tcp://127.0.0.1:82", {{"version", "1.0"}}}, + {"tcp://127.0.0.1:83", {{"app", "envoy"}}}, + {"tcp://127.0.0.1:84", {{"foo", "abc"}, {"bar", "default"}}}}); + + TestLoadBalancerContext context_ver_10({{"version", "1.0"}}); + TestLoadBalancerContext context_ver_nx({{"version", "x"}}); + TestLoadBalancerContext context_app({{"app", "envoy"}}); + TestLoadBalancerContext context_app_nx({{"app", "ngnix"}}); + TestLoadBalancerContext context_foo({{"foo", "abc"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_app_nx)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_app_nx)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_app)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_ver_nx)); + EXPECT_EQ(host_set_.hosts_[3], lb_->chooseHost(&context_foo)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorDefaultAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::DEFAULT_SUBSET)); + + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"version", "default"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:80", {{"version", "new"}}}, + {"tcp://127.0.0.1:81", {{"version", "default"}}}, + }); + + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(nullptr)); + + HostSharedPtr added_host1 = makeHost("tcp://127.0.0.1:8000", {{"version", "new"}}); + HostSharedPtr added_host2 = makeHost("tcp://127.0.0.1:8001", {{"version", "default"}}); + + TestLoadBalancerContext context_ver_nx({{"version", "x"}}); + + modifyHosts({added_host1, added_host2}, {host_set_.hosts_.back()}); + + EXPECT_EQ(added_host2, lb_->chooseHost(&context_ver_nx)); +} + +TEST_P(SubsetLoadBalancerTest, SubsetSelectorAnyAfterUpdate) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({ + {"tcp://127.0.0.1:81", {{"version", "1"}}}, + {"tcp://127.0.0.1:82", {{"version", "2"}}}, + }); + + TestLoadBalancerContext context_ver_nx({{"version", "x"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_ver_nx)); + + HostSharedPtr added_host1 = makeHost("tcp://127.0.0.1:83", {{"version", "3"}}); + + modifyHosts({added_host1}, {host_set_.hosts_.back()}); + + EXPECT_EQ(added_host1, lb_->chooseHost(&context_ver_nx)); +} + +TEST_P(SubsetLoadBalancerTest, FallbackForCompoundSelector) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + const ProtobufWkt::Struct default_subset = makeDefaultSubset({{"foo", "bar"}}); + EXPECT_CALL(subset_info_, defaultSubset()).WillRepeatedly(ReturnRef(default_subset)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED}), + std::make_shared( + SubsetSelector{{"version", "hardware", "stage"}, + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NO_FALLBACK}), + std::make_shared(SubsetSelector{ + {"version", "hardware"}, + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::DEFAULT_SUBSET})}; + + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + // Add hosts initial hosts. + init({{"tcp://127.0.0.1:80", {{"version", "1.0"}, {"hardware", "c32"}}}, + {"tcp://127.0.0.1:81", {{"version", "1.0"}, {"hardware", "c32"}, {"foo", "bar"}}}, + {"tcp://127.0.0.1:82", {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "dev"}}}}); + + TestLoadBalancerContext context_match_host0({{"version", "1.0"}, {"hardware", "c32"}}); + TestLoadBalancerContext context_ver_nx({{"version", "x"}, {"hardware", "c32"}}); + TestLoadBalancerContext context_stage_nx( + {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "x"}}); + TestLoadBalancerContext context_hardware_nx( + {{"version", "2.0"}, {"hardware", "zzz"}, {"stage", "dev"}}); + TestLoadBalancerContext context_match_host2( + {{"version", "2.0"}, {"hardware", "c32"}, {"stage", "dev"}}); + TestLoadBalancerContext context_ver_20({{"version", "2.0"}}); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(&context_match_host0)); + EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_ver_nx)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_hardware_nx)); + EXPECT_EQ(nullptr, lb_->chooseHost(&context_stage_nx)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_match_host2)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_match_host2)); + EXPECT_EQ(host_set_.hosts_[2], lb_->chooseHost(&context_ver_20)); +} + INSTANTIATE_TEST_SUITE_P(UpdateOrderings, SubsetLoadBalancerTest, testing::ValuesIn({REMOVES_FIRST, SIMULTANEOUS})); diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index 08894d788321..ee8cf04f2889 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -22,7 +22,7 @@ MockLoadBalancerSubsetInfo::MockLoadBalancerSubsetInfo() { ON_CALL(*this, fallbackPolicy()) .WillByDefault(Return(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT)); ON_CALL(*this, defaultSubset()).WillByDefault(ReturnRef(ProtobufWkt::Struct::default_instance())); - ON_CALL(*this, subsetKeys()).WillByDefault(ReturnRef(subset_keys_)); + ON_CALL(*this, subsetSelectors()).WillByDefault(ReturnRef(subset_selectors_)); } MockLoadBalancerSubsetInfo::~MockLoadBalancerSubsetInfo() {} diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index f1ae8657a6f6..c55a31a42585 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -33,12 +33,12 @@ class MockLoadBalancerSubsetInfo : public LoadBalancerSubsetInfo { MOCK_CONST_METHOD0(fallbackPolicy, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy()); MOCK_CONST_METHOD0(defaultSubset, const ProtobufWkt::Struct&()); - MOCK_CONST_METHOD0(subsetKeys, const std::vector>&()); + MOCK_CONST_METHOD0(subsetSelectors, const std::vector&()); MOCK_CONST_METHOD0(localityWeightAware, bool()); MOCK_CONST_METHOD0(scaleLocalityWeight, bool()); MOCK_CONST_METHOD0(panicModeAny, bool()); - std::vector> subset_keys_; + std::vector subset_selectors_; }; // While this mock class doesn't have any direct use in public Envoy tests, it's From baa9d2d3d26129dfe3a98dc73c92687e9cc019f2 Mon Sep 17 00:00:00 2001 From: ivitjuk <875244+ivitjuk@users.noreply.github.com> Date: Fri, 14 Jun 2019 00:49:57 -0700 Subject: [PATCH 012/542] Added AWS region provider (#7172) Description: Added AWS region provider This will be used by the AWS SigV4 signing Task 3 as described in: https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html Risk Level: Low Testing: Added new unit tests Docs Changes: N/A Release Notes: N/A Signed-off-by: Ivan Vitjuk --- source/common/common/logger.h | 2 + .../extensions/filters/http/common/aws/BUILD | 16 +++++++ .../filters/http/common/aws/region_provider.h | 33 +++++++++++++++ .../http/common/aws/region_provider_impl.cc | 30 +++++++++++++ .../http/common/aws/region_provider_impl.h | 38 +++++++++++++++++ test/extensions/filters/http/common/aws/BUILD | 9 ++++ .../common/aws/region_provider_impl_test.cc | 42 +++++++++++++++++++ test/test_common/environment.cc | 14 ++++++- test/test_common/environment.h | 5 +++ 9 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 source/extensions/filters/http/common/aws/region_provider.h create mode 100644 source/extensions/filters/http/common/aws/region_provider_impl.cc create mode 100644 source/extensions/filters/http/common/aws/region_provider_impl.h create mode 100644 test/extensions/filters/http/common/aws/region_provider_impl_test.cc diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 735aafa18f98..67580a841a01 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -20,8 +20,10 @@ namespace Envoy { namespace Logger { // clang-format off +// TODO: find out a way for extensions to register new logger IDs #define ALL_LOGGER_IDS(FUNCTION) \ FUNCTION(admin) \ + FUNCTION(aws) \ FUNCTION(assert) \ FUNCTION(backtrace) \ FUNCTION(client) \ diff --git a/source/extensions/filters/http/common/aws/BUILD b/source/extensions/filters/http/common/aws/BUILD index e21ff2645279..0a0c543fb426 100644 --- a/source/extensions/filters/http/common/aws/BUILD +++ b/source/extensions/filters/http/common/aws/BUILD @@ -49,3 +49,19 @@ envoy_cc_library( "//source/common/http:headers_lib", ], ) + +envoy_cc_library( + name = "region_provider_interface", + hdrs = ["region_provider.h"], + external_deps = ["abseil_optional"], +) + +envoy_cc_library( + name = "region_provider_impl_lib", + srcs = ["region_provider_impl.cc"], + hdrs = ["region_provider_impl.h"], + deps = [ + ":region_provider_interface", + "//source/common/common:logger_lib", + ], +) diff --git a/source/extensions/filters/http/common/aws/region_provider.h b/source/extensions/filters/http/common/aws/region_provider.h new file mode 100644 index 000000000000..58e2ad64da74 --- /dev/null +++ b/source/extensions/filters/http/common/aws/region_provider.h @@ -0,0 +1,33 @@ +#pragma once + +#include "envoy/common/pure.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Common { +namespace Aws { + +/** + * Interface for classes capable of discovering the AWS region from the execution environment. + */ +class RegionProvider { +public: + virtual ~RegionProvider() = default; + + /** + * Discover and return the AWS region. + * @return AWS region, or nullopt if unable to discover the region. + */ + virtual absl::optional getRegion() PURE; +}; + +typedef std::shared_ptr RegionProviderSharedPtr; + +} // namespace Aws +} // namespace Common +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/common/aws/region_provider_impl.cc b/source/extensions/filters/http/common/aws/region_provider_impl.cc new file mode 100644 index 000000000000..f535eaef3ec3 --- /dev/null +++ b/source/extensions/filters/http/common/aws/region_provider_impl.cc @@ -0,0 +1,30 @@ +#include "extensions/filters/http/common/aws/region_provider_impl.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Common { +namespace Aws { + +static const char AWS_REGION[] = "AWS_REGION"; + +StaticRegionProvider::StaticRegionProvider(const std::string& region) : region_(region) {} + +absl::optional StaticRegionProvider::getRegion() { + return absl::optional(region_); +} + +absl::optional EnvironmentRegionProvider::getRegion() { + const auto region = std::getenv(AWS_REGION); + if (region == nullptr) { + return absl::nullopt; + } + ENVOY_LOG(debug, "Found environment region {}={}", AWS_REGION, region); + return absl::optional(region); +} + +} // namespace Aws +} // namespace Common +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/common/aws/region_provider_impl.h b/source/extensions/filters/http/common/aws/region_provider_impl.h new file mode 100644 index 000000000000..114d1e294fb8 --- /dev/null +++ b/source/extensions/filters/http/common/aws/region_provider_impl.h @@ -0,0 +1,38 @@ +#pragma once + +#include "common/common/logger.h" + +#include "extensions/filters/http/common/aws/region_provider.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Common { +namespace Aws { + +/** + * Retrieve AWS region name from the environment + */ +class EnvironmentRegionProvider : public RegionProvider, public Logger::Loggable { +public: + absl::optional getRegion() override; +}; + +/** + * Return statically configured AWS region name + */ +class StaticRegionProvider : public RegionProvider { +public: + StaticRegionProvider(const std::string& region); + + absl::optional getRegion() override; + +private: + const std::string region_; +}; + +} // namespace Aws +} // namespace Common +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/common/aws/BUILD b/test/extensions/filters/http/common/aws/BUILD index 1c5c0541e4eb..67bc0a150c6e 100644 --- a/test/extensions/filters/http/common/aws/BUILD +++ b/test/extensions/filters/http/common/aws/BUILD @@ -40,3 +40,12 @@ envoy_cc_test( "//test/test_common:utility_lib", ], ) + +envoy_cc_test( + name = "region_provider_impl_test", + srcs = ["region_provider_impl_test.cc"], + deps = [ + "//source/extensions/filters/http/common/aws:region_provider_impl_lib", + "//test/test_common:environment_lib", + ], +) diff --git a/test/extensions/filters/http/common/aws/region_provider_impl_test.cc b/test/extensions/filters/http/common/aws/region_provider_impl_test.cc new file mode 100644 index 000000000000..42ce4d907bef --- /dev/null +++ b/test/extensions/filters/http/common/aws/region_provider_impl_test.cc @@ -0,0 +1,42 @@ +#include "extensions/filters/http/common/aws/region_provider_impl.h" + +#include "test/test_common/environment.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Common { +namespace Aws { + +class EnvironmentRegionProviderTest : public testing::Test { +public: + virtual ~EnvironmentRegionProviderTest() { TestEnvironment::unsetEnvVar("AWS_REGION"); } + + EnvironmentRegionProvider provider_; +}; + +class StaticRegionProviderTest : public testing::Test { +public: + StaticRegionProviderTest() : provider_("test-region") {} + + StaticRegionProvider provider_; +}; + +TEST_F(EnvironmentRegionProviderTest, SomeRegion) { + TestEnvironment::setEnvVar("AWS_REGION", "test-region", 1); + EXPECT_EQ("test-region", provider_.getRegion().value()); +} + +TEST_F(EnvironmentRegionProviderTest, NoRegion) { EXPECT_FALSE(provider_.getRegion().has_value()); } + +TEST_F(StaticRegionProviderTest, SomeRegion) { + EXPECT_EQ("test-region", provider_.getRegion().value()); +} + +} // namespace Aws +} // namespace Common +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index da848c98eb2b..ea6c11d6f0ea 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -315,7 +315,7 @@ void TestEnvironment::setEnvVar(const std::string& name, const std::string& valu if (!overwrite) { size_t requiredSize; const int rc = ::getenv_s(&requiredSize, nullptr, 0, name.c_str()); - ASSERT_EQ(rc, 0); + ASSERT_EQ(0, rc); if (requiredSize != 0) { return; } @@ -324,7 +324,17 @@ void TestEnvironment::setEnvVar(const std::string& name, const std::string& valu ASSERT_EQ(0, rc); #else const int rc = ::setenv(name.c_str(), value.c_str(), overwrite); - ASSERT_EQ(rc, 0); + ASSERT_EQ(0, rc); +#endif +} + +void TestEnvironment::unsetEnvVar(const std::string& name) { +#ifdef WIN32 + const int rc = ::_putenv_s(name.c_str(), ""); + ASSERT_EQ(0, rc); +#else + const int rc = ::unsetenv(name.c_str()); + ASSERT_EQ(0, rc); #endif } diff --git a/test/test_common/environment.h b/test/test_common/environment.h index f77d54226dcb..77ca0c20311b 100644 --- a/test/test_common/environment.h +++ b/test/test_common/environment.h @@ -203,6 +203,11 @@ class TestEnvironment { * Set environment variable. Same args as setenv(2). */ static void setEnvVar(const std::string& name, const std::string& value, int overwrite); + + /** + * Removes environment variable. Same args as unsetenv(3). + */ + static void unsetEnvVar(const std::string& name); }; } // namespace Envoy From 6ac9a4e3c0dda2dc5c958a7a7ca4eab8b0b9513d Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 14 Jun 2019 17:18:36 +0200 Subject: [PATCH 013/542] subsets fix comment (#7277) Signed-off-by: Kateryna Nezdolii --- source/common/upstream/subset_lb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/upstream/subset_lb.h b/source/common/upstream/subset_lb.h index fa45a4fa0dc3..43337ccc5763 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/common/upstream/subset_lb.h @@ -203,7 +203,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable Date: Fri, 14 Jun 2019 17:37:39 +0200 Subject: [PATCH 014/542] router: auto_host_rewrite_header support (#7220) Signed-off-by: Bartosz Borkowski --- api/envoy/api/v2/route/route.proto | 12 ++++- docs/root/intro/version_history.rst | 1 + source/common/router/config_impl.cc | 12 +++++ source/common/router/config_impl.h | 1 + test/common/router/config_impl_test.cc | 70 ++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 1 deletion(-) diff --git a/api/envoy/api/v2/route/route.proto b/api/envoy/api/v2/route/route.proto index 7d6b382383e6..8390e4348526 100644 --- a/api/envoy/api/v2/route/route.proto +++ b/api/envoy/api/v2/route/route.proto @@ -460,7 +460,7 @@ message CorsPolicy { core.RuntimeFractionalPercent shadow_enabled = 10; } -// [#comment:next free field: 27] +// [#comment:next free field: 30] message RouteAction { oneof cluster_specifier { option (validate.required) = true; @@ -548,6 +548,16 @@ message RouteAction { // type *strict_dns* or *logical_dns*. Setting this to true with other cluster // types has no effect. google.protobuf.BoolValue auto_host_rewrite = 7; + + // Indicates that during forwarding, the host header will be swapped with the content of given + // downstream or :ref:`custom ` header. + // If header value is empty, host header is left intact. + // + // .. attention:: + // + // Pay attention to the potential security implications of using this option. Provided header + // must come from trusted source. + string auto_host_rewrite_header = 29; } // Specifies the upstream timeout for the route. If not specified, the default is 15s. This diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 9c700155aebc..ca2608b014f6 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -55,6 +55,7 @@ Version history * router: per try timeouts will no longer start before the downstream request has been received in full by the router. This ensures that the per try timeout does not account for slow downstreams and that will not start before the global timeout. +* router: added :ref:`RouteAction's auto_host_rewrite_header ` to allow upstream host header substitution with some other header's value * router: added support for UPSTREAM_REMOTE_ADDRESS :ref:`header formatter `. * runtime: added support for :ref:`flexible layering configuration diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 56670d9d93a2..a215dbc0038c 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -366,6 +366,10 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, prefix_rewrite_(route.route().prefix_rewrite()), host_rewrite_(route.route().host_rewrite()), vhost_(vhost), auto_host_rewrite_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(route.route(), auto_host_rewrite, false)), + auto_host_rewrite_header_(!route.route().auto_host_rewrite_header().empty() + ? absl::optional(Http::LowerCaseString( + route.route().auto_host_rewrite_header())) + : absl::nullopt), cluster_name_(route.route().cluster()), cluster_header_name_(route.route().cluster_header()), cluster_not_found_response_code_(ConfigUtility::parseClusterNotFoundResponseCode( route.route().cluster_not_found_response_code())), @@ -513,6 +517,14 @@ void RouteEntryImplBase::finalizeRequestHeaders(Http::HeaderMap& headers, vhost_.globalRouteConfig().requestHeaderParser().evaluateHeaders(headers, stream_info); if (!host_rewrite_.empty()) { headers.Host()->value(host_rewrite_); + } else if (auto_host_rewrite_header_) { + Http::HeaderEntry* header = headers.get(*auto_host_rewrite_header_); + if (header != nullptr) { + absl::string_view header_value = header->value().getStringView(); + if (!header_value.empty()) { + headers.Host()->value(header_value); + } + } } // Handle path rewrite diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 24fbf3d73644..8ec35a02330c 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -646,6 +646,7 @@ class RouteEntryImplBase : public RouteEntry, const VirtualHostImpl& vhost_; // See note in RouteEntryImplBase::clusterEntry() on why raw ref // to virtual host is currently safe. const bool auto_host_rewrite_; + const absl::optional auto_host_rewrite_header_; const std::string cluster_name_; const Http::LowerCaseString cluster_header_name_; const Http::Code cluster_not_found_response_code_; diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index eb9adf152501..0df7229963a2 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -257,6 +257,20 @@ TEST_F(RouteMatcherTest, TestRoutes) { route: prefix_rewrite: "/oranGES" cluster: instant-server + - match: + path: "/rewrite-host-with-header-value" + request_headers_to_add: + - header: + key: x-rewrite-host + value: rewrote + route: + cluster: ats + auto_host_rewrite_header: x-rewrite-host + - match: + path: "/do-not-rewrite-host-with-header-value" + route: + cluster: ats + auto_host_rewrite_header: x-rewrite-host - match: prefix: "/" route: @@ -418,6 +432,24 @@ TEST_F(RouteMatcherTest, TestRoutes) { EXPECT_EQ("new_host", headers.get_(Http::Headers::get().Host)); } + // Rewrites host using supplied header. + { + Http::TestHeaderMapImpl headers = + genHeaders("api.lyft.com", "/rewrite-host-with-header-value", "GET"); + const RouteEntry* route = config.route(headers, 0)->routeEntry(); + route->finalizeRequestHeaders(headers, stream_info, true); + EXPECT_EQ("rewrote", headers.get_(Http::Headers::get().Host)); + } + + // Does not rewrite host because of missing header. + { + Http::TestHeaderMapImpl headers = + genHeaders("api.lyft.com", "/do-not-rewrite-host-with-header-value", "GET"); + const RouteEntry* route = config.route(headers, 0)->routeEntry(); + route->finalizeRequestHeaders(headers, stream_info, true); + EXPECT_EQ("api.lyft.com", headers.get_(Http::Headers::get().Host)); + } + // Case sensitive rewrite matching test. { Http::TestHeaderMapImpl headers = @@ -1067,6 +1099,44 @@ TEST_F(RouteMatcherTest, NoHostRewriteAndAutoRewrite) { EnvoyException); } +TEST_F(RouteMatcherTest, NoHostRewriteAndAutoRewriteHeader) { + const std::string yaml = R"EOF( +virtual_hosts: +- name: local_service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: local_service + host_rewrite: foo + auto_host_rewrite_header: "dummy-header" + )EOF"; + + EXPECT_THROW(TestConfigImpl(parseRouteConfigurationFromV2Yaml(yaml), factory_context_, true), + EnvoyException); +} + +TEST_F(RouteMatcherTest, NoAutoRewriteAndAutoRewriteHeader) { + const std::string yaml = R"EOF( +virtual_hosts: +- name: local_service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: local_service + auto_host_rewrite: true + auto_host_rewrite_header: "dummy-header" + )EOF"; + + EXPECT_THROW(TestConfigImpl(parseRouteConfigurationFromV2Yaml(yaml), factory_context_, true), + EnvoyException); +} + TEST_F(RouteMatcherTest, HeaderMatchedRouting) { const std::string yaml = R"EOF( virtual_hosts: From f1597d08e2964c45af11b779a38bb44d70475feb Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 14 Jun 2019 10:27:36 -0700 Subject: [PATCH 015/542] docs: redo arch overview (#7273) The current arch overview is a little out of control. This attempts to organize it better. Signed-off-by: Matt Klein --- .../intro/arch_overview/advanced/advanced.rst | 7 +++ .../data_sharing_between_filters.rst | 10 ++--- .../intro/arch_overview/arch_overview.rst | 45 ++++--------------- docs/root/intro/arch_overview/http/http.rst | 10 +++++ .../{ => http}/http_connection_management.rst | 0 .../arch_overview/{ => http}/http_filters.rst | 0 .../arch_overview/{ => http}/http_routing.rst | 0 .../arch_overview/{ => http}/websocket.rst | 0 docs/root/intro/arch_overview/intro/intro.rst | 8 ++++ .../arch_overview/{ => intro}/terminology.rst | 0 .../{ => intro}/threading_model.rst | 0 .../{ => listeners}/listener_filters.rst | 0 .../{ => listeners}/listeners.rst | 0 .../arch_overview/listeners/listeners_toc.rst | 9 ++++ .../{ => listeners}/network_filters.rst | 0 .../{ => observability}/access_logging.rst | 0 .../observability/observability.rst | 9 ++++ .../{ => observability}/statistics.rst | 0 .../{ => observability}/tracing.rst | 0 .../{ => operations}/draining.rst | 0 .../dynamic_configuration.rst | 0 .../{ => operations}/hot_restart.rst | 0 .../arch_overview/{ => operations}/init.rst | 0 .../arch_overview/operations/operations.rst | 12 +++++ .../{ => operations}/overload_manager.rst | 0 .../{ => operations}/runtime.rst | 0 .../{ => other_features}/ext_authz_filter.rst | 2 +- .../global_rate_limiting.rst | 0 .../{ => other_features}/ip_transparency.rst | 0 .../other_features/other_features.rst | 11 +++++ .../{ => other_features}/scripting.rst | 0 .../{ => other_features}/ssl.rst | 0 .../{ => other_protocols}/dynamo.rst | 0 .../{ => other_protocols}/grpc.rst | 0 .../{ => other_protocols}/mongo.rst | 0 .../other_protocols/other_protocols.rst | 11 +++++ .../{ => other_protocols}/redis.rst | 0 .../{ => other_protocols}/tcp_proxy.rst | 0 .../{ => upstream}/circuit_breaking.rst | 0 .../{ => upstream}/cluster_manager.rst | 0 .../{ => upstream}/connection_pooling.rst | 0 .../{ => upstream}/health_checking.rst | 0 .../load_balancing/degraded.rst | 0 .../load_balancing/load_balancers.rst | 0 .../load_balancing/load_balancing.rst | 0 .../load_balancing/locality_weight.rst | 0 .../load_balancing/original_dst.rst | 0 .../load_balancing/overprovisioning.rst | 0 .../load_balancing/overview.rst | 0 .../load_balancing/panic_threshold.rst | 0 .../load_balancing/priority.rst | 0 .../{ => upstream}/load_balancing/subsets.rst | 0 .../load_balancing/zone_aware.rst | 0 .../arch_overview/{ => upstream}/outlier.rst | 0 .../{ => upstream}/service_discovery.rst | 0 .../intro/arch_overview/upstream/upstream.rst | 13 ++++++ 56 files changed, 105 insertions(+), 42 deletions(-) create mode 100644 docs/root/intro/arch_overview/advanced/advanced.rst rename docs/root/intro/arch_overview/{ => advanced}/data_sharing_between_filters.rst (98%) create mode 100644 docs/root/intro/arch_overview/http/http.rst rename docs/root/intro/arch_overview/{ => http}/http_connection_management.rst (100%) rename docs/root/intro/arch_overview/{ => http}/http_filters.rst (100%) rename docs/root/intro/arch_overview/{ => http}/http_routing.rst (100%) rename docs/root/intro/arch_overview/{ => http}/websocket.rst (100%) create mode 100644 docs/root/intro/arch_overview/intro/intro.rst rename docs/root/intro/arch_overview/{ => intro}/terminology.rst (100%) rename docs/root/intro/arch_overview/{ => intro}/threading_model.rst (100%) rename docs/root/intro/arch_overview/{ => listeners}/listener_filters.rst (100%) rename docs/root/intro/arch_overview/{ => listeners}/listeners.rst (100%) create mode 100644 docs/root/intro/arch_overview/listeners/listeners_toc.rst rename docs/root/intro/arch_overview/{ => listeners}/network_filters.rst (100%) rename docs/root/intro/arch_overview/{ => observability}/access_logging.rst (100%) create mode 100644 docs/root/intro/arch_overview/observability/observability.rst rename docs/root/intro/arch_overview/{ => observability}/statistics.rst (100%) rename docs/root/intro/arch_overview/{ => observability}/tracing.rst (100%) rename docs/root/intro/arch_overview/{ => operations}/draining.rst (100%) rename docs/root/intro/arch_overview/{ => operations}/dynamic_configuration.rst (100%) rename docs/root/intro/arch_overview/{ => operations}/hot_restart.rst (100%) rename docs/root/intro/arch_overview/{ => operations}/init.rst (100%) create mode 100644 docs/root/intro/arch_overview/operations/operations.rst rename docs/root/intro/arch_overview/{ => operations}/overload_manager.rst (100%) rename docs/root/intro/arch_overview/{ => operations}/runtime.rst (100%) rename docs/root/intro/arch_overview/{ => other_features}/ext_authz_filter.rst (98%) rename docs/root/intro/arch_overview/{ => other_features}/global_rate_limiting.rst (100%) rename docs/root/intro/arch_overview/{ => other_features}/ip_transparency.rst (100%) create mode 100644 docs/root/intro/arch_overview/other_features/other_features.rst rename docs/root/intro/arch_overview/{ => other_features}/scripting.rst (100%) rename docs/root/intro/arch_overview/{ => other_features}/ssl.rst (100%) rename docs/root/intro/arch_overview/{ => other_protocols}/dynamo.rst (100%) rename docs/root/intro/arch_overview/{ => other_protocols}/grpc.rst (100%) rename docs/root/intro/arch_overview/{ => other_protocols}/mongo.rst (100%) create mode 100644 docs/root/intro/arch_overview/other_protocols/other_protocols.rst rename docs/root/intro/arch_overview/{ => other_protocols}/redis.rst (100%) rename docs/root/intro/arch_overview/{ => other_protocols}/tcp_proxy.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/circuit_breaking.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/cluster_manager.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/connection_pooling.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/health_checking.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/degraded.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/load_balancers.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/load_balancing.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/locality_weight.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/original_dst.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/overprovisioning.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/overview.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/panic_threshold.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/priority.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/subsets.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/load_balancing/zone_aware.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/outlier.rst (100%) rename docs/root/intro/arch_overview/{ => upstream}/service_discovery.rst (100%) create mode 100644 docs/root/intro/arch_overview/upstream/upstream.rst diff --git a/docs/root/intro/arch_overview/advanced/advanced.rst b/docs/root/intro/arch_overview/advanced/advanced.rst new file mode 100644 index 000000000000..c26e0fee562c --- /dev/null +++ b/docs/root/intro/arch_overview/advanced/advanced.rst @@ -0,0 +1,7 @@ +Advanced +======== + +.. toctree:: + :maxdepth: 2 + + data_sharing_between_filters diff --git a/docs/root/intro/arch_overview/data_sharing_between_filters.rst b/docs/root/intro/arch_overview/advanced/data_sharing_between_filters.rst similarity index 98% rename from docs/root/intro/arch_overview/data_sharing_between_filters.rst rename to docs/root/intro/arch_overview/advanced/data_sharing_between_filters.rst index 05423e21c6e6..a614068b6fab 100644 --- a/docs/root/intro/arch_overview/data_sharing_between_filters.rst +++ b/docs/root/intro/arch_overview/advanced/data_sharing_between_filters.rst @@ -1,12 +1,14 @@ .. _arch_overview_data_sharing_between_filters: +Sharing data between filters +============================ + Envoy provides the following mechanisms for the transfer of configuration, metadata and per-request/connection state to, from and between filters, as well as to other core subsystems (e.g., access logging). - Static State -============ +^^^^^^^^^^^^ Static state is any immutable state specified at configuration load time (e.g., through xDS). There are three categories of static state: @@ -51,7 +53,6 @@ into an instance of `ServicePolicy` class (inherited from metadata is not a new source of metadata. It is obtained from metadata that is specified as part of the configuration. - HTTP Per-Route Filter Configuration ----------------------------------- @@ -71,9 +72,8 @@ convert it into a typed class object that’s stored with the route itself. HTTP filters can then query the route-specific filter config during request processing. - Dynamic State -============= +^^^^^^^^^^^^^ Dynamic state is generated per network connection or per HTTP stream. Dynamic state can be mutable if desired by the filter generating diff --git a/docs/root/intro/arch_overview/arch_overview.rst b/docs/root/intro/arch_overview/arch_overview.rst index 0a118df54b1a..23fb2e43fd5d 100644 --- a/docs/root/intro/arch_overview/arch_overview.rst +++ b/docs/root/intro/arch_overview/arch_overview.rst @@ -4,39 +4,12 @@ Architecture overview .. toctree:: :maxdepth: 2 - terminology - threading_model - listeners - listener_filters - network_filters - http_connection_management - http_filters - data_sharing_between_filters - http_routing - grpc - websocket - cluster_manager - service_discovery - health_checking - connection_pooling - load_balancing/load_balancing - outlier - circuit_breaking - global_rate_limiting - ssl - statistics - runtime - tracing - tcp_proxy - access_logging - mongo - dynamo - redis - hot_restart - dynamic_configuration - init - draining - scripting - ext_authz_filter - overload_manager - ip_transparency + intro/intro + listeners/listeners_toc + http/http + upstream/upstream + observability/observability + operations/operations + other_features/other_features + other_protocols/other_protocols + advanced/advanced diff --git a/docs/root/intro/arch_overview/http/http.rst b/docs/root/intro/arch_overview/http/http.rst new file mode 100644 index 000000000000..d32e20e4aa07 --- /dev/null +++ b/docs/root/intro/arch_overview/http/http.rst @@ -0,0 +1,10 @@ +HTTP +==== + +.. toctree:: + :maxdepth: 2 + + http_connection_management + http_filters + http_routing + websocket diff --git a/docs/root/intro/arch_overview/http_connection_management.rst b/docs/root/intro/arch_overview/http/http_connection_management.rst similarity index 100% rename from docs/root/intro/arch_overview/http_connection_management.rst rename to docs/root/intro/arch_overview/http/http_connection_management.rst diff --git a/docs/root/intro/arch_overview/http_filters.rst b/docs/root/intro/arch_overview/http/http_filters.rst similarity index 100% rename from docs/root/intro/arch_overview/http_filters.rst rename to docs/root/intro/arch_overview/http/http_filters.rst diff --git a/docs/root/intro/arch_overview/http_routing.rst b/docs/root/intro/arch_overview/http/http_routing.rst similarity index 100% rename from docs/root/intro/arch_overview/http_routing.rst rename to docs/root/intro/arch_overview/http/http_routing.rst diff --git a/docs/root/intro/arch_overview/websocket.rst b/docs/root/intro/arch_overview/http/websocket.rst similarity index 100% rename from docs/root/intro/arch_overview/websocket.rst rename to docs/root/intro/arch_overview/http/websocket.rst diff --git a/docs/root/intro/arch_overview/intro/intro.rst b/docs/root/intro/arch_overview/intro/intro.rst new file mode 100644 index 000000000000..19102425b140 --- /dev/null +++ b/docs/root/intro/arch_overview/intro/intro.rst @@ -0,0 +1,8 @@ +Introduction +============ + +.. toctree:: + :maxdepth: 2 + + terminology + threading_model diff --git a/docs/root/intro/arch_overview/terminology.rst b/docs/root/intro/arch_overview/intro/terminology.rst similarity index 100% rename from docs/root/intro/arch_overview/terminology.rst rename to docs/root/intro/arch_overview/intro/terminology.rst diff --git a/docs/root/intro/arch_overview/threading_model.rst b/docs/root/intro/arch_overview/intro/threading_model.rst similarity index 100% rename from docs/root/intro/arch_overview/threading_model.rst rename to docs/root/intro/arch_overview/intro/threading_model.rst diff --git a/docs/root/intro/arch_overview/listener_filters.rst b/docs/root/intro/arch_overview/listeners/listener_filters.rst similarity index 100% rename from docs/root/intro/arch_overview/listener_filters.rst rename to docs/root/intro/arch_overview/listeners/listener_filters.rst diff --git a/docs/root/intro/arch_overview/listeners.rst b/docs/root/intro/arch_overview/listeners/listeners.rst similarity index 100% rename from docs/root/intro/arch_overview/listeners.rst rename to docs/root/intro/arch_overview/listeners/listeners.rst diff --git a/docs/root/intro/arch_overview/listeners/listeners_toc.rst b/docs/root/intro/arch_overview/listeners/listeners_toc.rst new file mode 100644 index 000000000000..bc4f4b62ac10 --- /dev/null +++ b/docs/root/intro/arch_overview/listeners/listeners_toc.rst @@ -0,0 +1,9 @@ +Listeners +========= + +.. toctree:: + :maxdepth: 2 + + listeners + listener_filters + network_filters diff --git a/docs/root/intro/arch_overview/network_filters.rst b/docs/root/intro/arch_overview/listeners/network_filters.rst similarity index 100% rename from docs/root/intro/arch_overview/network_filters.rst rename to docs/root/intro/arch_overview/listeners/network_filters.rst diff --git a/docs/root/intro/arch_overview/access_logging.rst b/docs/root/intro/arch_overview/observability/access_logging.rst similarity index 100% rename from docs/root/intro/arch_overview/access_logging.rst rename to docs/root/intro/arch_overview/observability/access_logging.rst diff --git a/docs/root/intro/arch_overview/observability/observability.rst b/docs/root/intro/arch_overview/observability/observability.rst new file mode 100644 index 000000000000..105eb6c43d8b --- /dev/null +++ b/docs/root/intro/arch_overview/observability/observability.rst @@ -0,0 +1,9 @@ +Observability +============= + +.. toctree:: + :maxdepth: 2 + + statistics + access_logging + tracing diff --git a/docs/root/intro/arch_overview/statistics.rst b/docs/root/intro/arch_overview/observability/statistics.rst similarity index 100% rename from docs/root/intro/arch_overview/statistics.rst rename to docs/root/intro/arch_overview/observability/statistics.rst diff --git a/docs/root/intro/arch_overview/tracing.rst b/docs/root/intro/arch_overview/observability/tracing.rst similarity index 100% rename from docs/root/intro/arch_overview/tracing.rst rename to docs/root/intro/arch_overview/observability/tracing.rst diff --git a/docs/root/intro/arch_overview/draining.rst b/docs/root/intro/arch_overview/operations/draining.rst similarity index 100% rename from docs/root/intro/arch_overview/draining.rst rename to docs/root/intro/arch_overview/operations/draining.rst diff --git a/docs/root/intro/arch_overview/dynamic_configuration.rst b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst similarity index 100% rename from docs/root/intro/arch_overview/dynamic_configuration.rst rename to docs/root/intro/arch_overview/operations/dynamic_configuration.rst diff --git a/docs/root/intro/arch_overview/hot_restart.rst b/docs/root/intro/arch_overview/operations/hot_restart.rst similarity index 100% rename from docs/root/intro/arch_overview/hot_restart.rst rename to docs/root/intro/arch_overview/operations/hot_restart.rst diff --git a/docs/root/intro/arch_overview/init.rst b/docs/root/intro/arch_overview/operations/init.rst similarity index 100% rename from docs/root/intro/arch_overview/init.rst rename to docs/root/intro/arch_overview/operations/init.rst diff --git a/docs/root/intro/arch_overview/operations/operations.rst b/docs/root/intro/arch_overview/operations/operations.rst new file mode 100644 index 000000000000..c4b6441f8570 --- /dev/null +++ b/docs/root/intro/arch_overview/operations/operations.rst @@ -0,0 +1,12 @@ +Operations & configuration +========================== + +.. toctree:: + :maxdepth: 2 + + dynamic_configuration + init + draining + runtime + hot_restart + overload_manager diff --git a/docs/root/intro/arch_overview/overload_manager.rst b/docs/root/intro/arch_overview/operations/overload_manager.rst similarity index 100% rename from docs/root/intro/arch_overview/overload_manager.rst rename to docs/root/intro/arch_overview/operations/overload_manager.rst diff --git a/docs/root/intro/arch_overview/runtime.rst b/docs/root/intro/arch_overview/operations/runtime.rst similarity index 100% rename from docs/root/intro/arch_overview/runtime.rst rename to docs/root/intro/arch_overview/operations/runtime.rst diff --git a/docs/root/intro/arch_overview/ext_authz_filter.rst b/docs/root/intro/arch_overview/other_features/ext_authz_filter.rst similarity index 98% rename from docs/root/intro/arch_overview/ext_authz_filter.rst rename to docs/root/intro/arch_overview/other_features/ext_authz_filter.rst index 875b80e41b09..aaa8b2a3610c 100644 --- a/docs/root/intro/arch_overview/ext_authz_filter.rst +++ b/docs/root/intro/arch_overview/other_features/ext_authz_filter.rst @@ -38,4 +38,4 @@ The content of the request that are passed to an authorization service is specif :glob: :maxdepth: 2 - ../../api-v2/service/auth/v2/* + ../../../api-v2/service/auth/v2/* diff --git a/docs/root/intro/arch_overview/global_rate_limiting.rst b/docs/root/intro/arch_overview/other_features/global_rate_limiting.rst similarity index 100% rename from docs/root/intro/arch_overview/global_rate_limiting.rst rename to docs/root/intro/arch_overview/other_features/global_rate_limiting.rst diff --git a/docs/root/intro/arch_overview/ip_transparency.rst b/docs/root/intro/arch_overview/other_features/ip_transparency.rst similarity index 100% rename from docs/root/intro/arch_overview/ip_transparency.rst rename to docs/root/intro/arch_overview/other_features/ip_transparency.rst diff --git a/docs/root/intro/arch_overview/other_features/other_features.rst b/docs/root/intro/arch_overview/other_features/other_features.rst new file mode 100644 index 000000000000..7c5f004502e4 --- /dev/null +++ b/docs/root/intro/arch_overview/other_features/other_features.rst @@ -0,0 +1,11 @@ +Other features +============== + +.. toctree:: + :maxdepth: 2 + + global_rate_limiting + ssl + scripting + ext_authz_filter + ip_transparency diff --git a/docs/root/intro/arch_overview/scripting.rst b/docs/root/intro/arch_overview/other_features/scripting.rst similarity index 100% rename from docs/root/intro/arch_overview/scripting.rst rename to docs/root/intro/arch_overview/other_features/scripting.rst diff --git a/docs/root/intro/arch_overview/ssl.rst b/docs/root/intro/arch_overview/other_features/ssl.rst similarity index 100% rename from docs/root/intro/arch_overview/ssl.rst rename to docs/root/intro/arch_overview/other_features/ssl.rst diff --git a/docs/root/intro/arch_overview/dynamo.rst b/docs/root/intro/arch_overview/other_protocols/dynamo.rst similarity index 100% rename from docs/root/intro/arch_overview/dynamo.rst rename to docs/root/intro/arch_overview/other_protocols/dynamo.rst diff --git a/docs/root/intro/arch_overview/grpc.rst b/docs/root/intro/arch_overview/other_protocols/grpc.rst similarity index 100% rename from docs/root/intro/arch_overview/grpc.rst rename to docs/root/intro/arch_overview/other_protocols/grpc.rst diff --git a/docs/root/intro/arch_overview/mongo.rst b/docs/root/intro/arch_overview/other_protocols/mongo.rst similarity index 100% rename from docs/root/intro/arch_overview/mongo.rst rename to docs/root/intro/arch_overview/other_protocols/mongo.rst diff --git a/docs/root/intro/arch_overview/other_protocols/other_protocols.rst b/docs/root/intro/arch_overview/other_protocols/other_protocols.rst new file mode 100644 index 000000000000..abdd2018e852 --- /dev/null +++ b/docs/root/intro/arch_overview/other_protocols/other_protocols.rst @@ -0,0 +1,11 @@ +Other protocols +=============== + +.. toctree:: + :maxdepth: 2 + + grpc + tcp_proxy + mongo + dynamo + redis diff --git a/docs/root/intro/arch_overview/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst similarity index 100% rename from docs/root/intro/arch_overview/redis.rst rename to docs/root/intro/arch_overview/other_protocols/redis.rst diff --git a/docs/root/intro/arch_overview/tcp_proxy.rst b/docs/root/intro/arch_overview/other_protocols/tcp_proxy.rst similarity index 100% rename from docs/root/intro/arch_overview/tcp_proxy.rst rename to docs/root/intro/arch_overview/other_protocols/tcp_proxy.rst diff --git a/docs/root/intro/arch_overview/circuit_breaking.rst b/docs/root/intro/arch_overview/upstream/circuit_breaking.rst similarity index 100% rename from docs/root/intro/arch_overview/circuit_breaking.rst rename to docs/root/intro/arch_overview/upstream/circuit_breaking.rst diff --git a/docs/root/intro/arch_overview/cluster_manager.rst b/docs/root/intro/arch_overview/upstream/cluster_manager.rst similarity index 100% rename from docs/root/intro/arch_overview/cluster_manager.rst rename to docs/root/intro/arch_overview/upstream/cluster_manager.rst diff --git a/docs/root/intro/arch_overview/connection_pooling.rst b/docs/root/intro/arch_overview/upstream/connection_pooling.rst similarity index 100% rename from docs/root/intro/arch_overview/connection_pooling.rst rename to docs/root/intro/arch_overview/upstream/connection_pooling.rst diff --git a/docs/root/intro/arch_overview/health_checking.rst b/docs/root/intro/arch_overview/upstream/health_checking.rst similarity index 100% rename from docs/root/intro/arch_overview/health_checking.rst rename to docs/root/intro/arch_overview/upstream/health_checking.rst diff --git a/docs/root/intro/arch_overview/load_balancing/degraded.rst b/docs/root/intro/arch_overview/upstream/load_balancing/degraded.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/degraded.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/degraded.rst diff --git a/docs/root/intro/arch_overview/load_balancing/load_balancers.rst b/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/load_balancers.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst diff --git a/docs/root/intro/arch_overview/load_balancing/load_balancing.rst b/docs/root/intro/arch_overview/upstream/load_balancing/load_balancing.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/load_balancing.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/load_balancing.rst diff --git a/docs/root/intro/arch_overview/load_balancing/locality_weight.rst b/docs/root/intro/arch_overview/upstream/load_balancing/locality_weight.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/locality_weight.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/locality_weight.rst diff --git a/docs/root/intro/arch_overview/load_balancing/original_dst.rst b/docs/root/intro/arch_overview/upstream/load_balancing/original_dst.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/original_dst.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/original_dst.rst diff --git a/docs/root/intro/arch_overview/load_balancing/overprovisioning.rst b/docs/root/intro/arch_overview/upstream/load_balancing/overprovisioning.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/overprovisioning.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/overprovisioning.rst diff --git a/docs/root/intro/arch_overview/load_balancing/overview.rst b/docs/root/intro/arch_overview/upstream/load_balancing/overview.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/overview.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/overview.rst diff --git a/docs/root/intro/arch_overview/load_balancing/panic_threshold.rst b/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/panic_threshold.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst diff --git a/docs/root/intro/arch_overview/load_balancing/priority.rst b/docs/root/intro/arch_overview/upstream/load_balancing/priority.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/priority.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/priority.rst diff --git a/docs/root/intro/arch_overview/load_balancing/subsets.rst b/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/subsets.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst diff --git a/docs/root/intro/arch_overview/load_balancing/zone_aware.rst b/docs/root/intro/arch_overview/upstream/load_balancing/zone_aware.rst similarity index 100% rename from docs/root/intro/arch_overview/load_balancing/zone_aware.rst rename to docs/root/intro/arch_overview/upstream/load_balancing/zone_aware.rst diff --git a/docs/root/intro/arch_overview/outlier.rst b/docs/root/intro/arch_overview/upstream/outlier.rst similarity index 100% rename from docs/root/intro/arch_overview/outlier.rst rename to docs/root/intro/arch_overview/upstream/outlier.rst diff --git a/docs/root/intro/arch_overview/service_discovery.rst b/docs/root/intro/arch_overview/upstream/service_discovery.rst similarity index 100% rename from docs/root/intro/arch_overview/service_discovery.rst rename to docs/root/intro/arch_overview/upstream/service_discovery.rst diff --git a/docs/root/intro/arch_overview/upstream/upstream.rst b/docs/root/intro/arch_overview/upstream/upstream.rst new file mode 100644 index 000000000000..4c8a12db3137 --- /dev/null +++ b/docs/root/intro/arch_overview/upstream/upstream.rst @@ -0,0 +1,13 @@ +Upstream clusters +================= + +.. toctree:: + :maxdepth: 2 + + cluster_manager + service_discovery + health_checking + connection_pooling + load_balancing/load_balancing + outlier + circuit_breaking From 2f569b9a8d3f0d7a43ffa69e3e5ba947cd3a9f8b Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 14 Jun 2019 13:38:40 -0700 Subject: [PATCH 016/542] docs: further arch overview fixes from feedback (#7286) Signed-off-by: Matt Klein --- docs/root/intro/arch_overview/arch_overview.rst | 1 + docs/root/intro/arch_overview/listeners/listeners_toc.rst | 1 + .../{other_protocols => listeners}/tcp_proxy.rst | 0 .../intro/arch_overview/other_features/other_features.rst | 2 -- .../arch_overview/other_protocols/other_protocols.rst | 1 - .../{other_features => security}/ext_authz_filter.rst | 0 docs/root/intro/arch_overview/security/security.rst | 8 ++++++++ .../arch_overview/{other_features => security}/ssl.rst | 0 8 files changed, 10 insertions(+), 3 deletions(-) rename docs/root/intro/arch_overview/{other_protocols => listeners}/tcp_proxy.rst (100%) rename docs/root/intro/arch_overview/{other_features => security}/ext_authz_filter.rst (100%) create mode 100644 docs/root/intro/arch_overview/security/security.rst rename docs/root/intro/arch_overview/{other_features => security}/ssl.rst (100%) diff --git a/docs/root/intro/arch_overview/arch_overview.rst b/docs/root/intro/arch_overview/arch_overview.rst index 23fb2e43fd5d..59433b5950eb 100644 --- a/docs/root/intro/arch_overview/arch_overview.rst +++ b/docs/root/intro/arch_overview/arch_overview.rst @@ -9,6 +9,7 @@ Architecture overview http/http upstream/upstream observability/observability + security/security operations/operations other_features/other_features other_protocols/other_protocols diff --git a/docs/root/intro/arch_overview/listeners/listeners_toc.rst b/docs/root/intro/arch_overview/listeners/listeners_toc.rst index bc4f4b62ac10..5b488ad2488f 100644 --- a/docs/root/intro/arch_overview/listeners/listeners_toc.rst +++ b/docs/root/intro/arch_overview/listeners/listeners_toc.rst @@ -7,3 +7,4 @@ Listeners listeners listener_filters network_filters + tcp_proxy diff --git a/docs/root/intro/arch_overview/other_protocols/tcp_proxy.rst b/docs/root/intro/arch_overview/listeners/tcp_proxy.rst similarity index 100% rename from docs/root/intro/arch_overview/other_protocols/tcp_proxy.rst rename to docs/root/intro/arch_overview/listeners/tcp_proxy.rst diff --git a/docs/root/intro/arch_overview/other_features/other_features.rst b/docs/root/intro/arch_overview/other_features/other_features.rst index 7c5f004502e4..6b8dc5f03736 100644 --- a/docs/root/intro/arch_overview/other_features/other_features.rst +++ b/docs/root/intro/arch_overview/other_features/other_features.rst @@ -5,7 +5,5 @@ Other features :maxdepth: 2 global_rate_limiting - ssl scripting - ext_authz_filter ip_transparency diff --git a/docs/root/intro/arch_overview/other_protocols/other_protocols.rst b/docs/root/intro/arch_overview/other_protocols/other_protocols.rst index abdd2018e852..1081a96993af 100644 --- a/docs/root/intro/arch_overview/other_protocols/other_protocols.rst +++ b/docs/root/intro/arch_overview/other_protocols/other_protocols.rst @@ -5,7 +5,6 @@ Other protocols :maxdepth: 2 grpc - tcp_proxy mongo dynamo redis diff --git a/docs/root/intro/arch_overview/other_features/ext_authz_filter.rst b/docs/root/intro/arch_overview/security/ext_authz_filter.rst similarity index 100% rename from docs/root/intro/arch_overview/other_features/ext_authz_filter.rst rename to docs/root/intro/arch_overview/security/ext_authz_filter.rst diff --git a/docs/root/intro/arch_overview/security/security.rst b/docs/root/intro/arch_overview/security/security.rst new file mode 100644 index 000000000000..00488a82559a --- /dev/null +++ b/docs/root/intro/arch_overview/security/security.rst @@ -0,0 +1,8 @@ +Security +======== + +.. toctree:: + :maxdepth: 2 + + ssl + ext_authz_filter diff --git a/docs/root/intro/arch_overview/other_features/ssl.rst b/docs/root/intro/arch_overview/security/ssl.rst similarity index 100% rename from docs/root/intro/arch_overview/other_features/ssl.rst rename to docs/root/intro/arch_overview/security/ssl.rst From 54e22406c38987d5b0401f18b614501bc184f41f Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 14 Jun 2019 15:12:21 -0700 Subject: [PATCH 017/542] stats: further hot restart gauge merging fixes (#7282) Follow up to https://github.com/envoyproxy/envoy/issues/7227 Signed-off-by: Matt Klein --- source/common/stats/stat_merger.cc | 34 +++++++++++++++++---------- test/common/stats/stat_merger_test.cc | 19 +++++++++++++++ test/integration/hotrestart_test.sh | 2 +- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/source/common/stats/stat_merger.cc b/source/common/stats/stat_merger.cc index 0aab1fe0b48e..db2e7d49a319 100644 --- a/source/common/stats/stat_merger.cc +++ b/source/common/stats/stat_merger.cc @@ -13,19 +13,14 @@ void StatMerger::mergeCounters(const Protobuf::Map& count void StatMerger::mergeGauges(const Protobuf::Map& gauges) { for (const auto& gauge : gauges) { - // Merging gauges via RPC from the parent has 4 cases; case 2 and 4b are the + // Merging gauges via RPC from the parent has 3 cases; case 1 and 3b are the // most common. // - // 1. Parent process thinks gauge is NeverImport: no data sent, and we - // do not run this loop for such a gauge. Only if the parent process - // thinks the gauge is Accumulate do we consider cases 2-4. - // 2. Child thinks gauge is Accumulate : data is combined in + // 1. Child thinks gauge is Accumulate : data is combined in // gauge_ref.add() below. - // 3. Child thinks gauge is NeverImport: we skip this loop entry via - // 'continue'. This only happens with a code-change where the child - // contains a code-change relative to parent, switching a Gauge from - // Accumulate to NeverImport. - // 4. Child has not yet initialized gauge yet -- this merge is the + // 2. Child thinks gauge is NeverImport: we skip this loop entry via + // 'continue'. + // 3. Child has not yet initialized gauge yet -- this merge is the // first time the child learns of the gauge. It's possible the child // will think the gauge is NeverImport due to a code change. But for // now we will leave the gauge in the child process as @@ -33,9 +28,9 @@ void StatMerger::mergeGauges(const Protobuf::Map& gauges) // gauge_ref.add(). Gauges in this mode will not be included in // stats-sinks or the admin /stats calls, until the child initializes // the gauge, in which case: - // 4a. Child later initializes gauges as NeverImport: the parent value is + // 3a. Child later initializes gauges as NeverImport: the parent value is // cleared during the mergeImportMode call. - // 4b. Child later initializes gauges as Accumulate: the parent value is + // 3b. Child later initializes gauges as Accumulate: the parent value is // retained. StatNameManagedStorage storage(gauge.first, temp_scope_->symbolTable()); @@ -52,6 +47,21 @@ void StatMerger::mergeGauges(const Protobuf::Map& gauges) } auto& gauge_ref = temp_scope_->gaugeFromStatName(stat_name, import_mode); + if (gauge_ref.importMode() == Gauge::ImportMode::NeverImport) { + // On the first iteration through the loop, the gauge will not be loaded into the scope + // cache even though it might exist in another scope. Thus, we need to check again for + // the import status to see if we should skip this gauge. + // + // TODO(mattklein123): There is a race condition here. It's technically possible that + // between the time we created this stat, the stat might be created by the child as a + // never import stat, making the below math invalid. A follow up solution is to take the + // store lock starting from gaugeFromStatName() to the end of this function, but this will + // require adding some type of mergeGauge() function to the scope and dealing with recursive + // lock acquisition, etc. so we will leave this as a follow up. This race should be incredibly + // rare. + continue; + } + uint64_t& parent_value_ref = parent_gauge_values_[gauge_ref.statName()]; uint64_t old_parent_value = parent_value_ref; uint64_t new_parent_value = gauge.second; diff --git a/test/common/stats/stat_merger_test.cc b/test/common/stats/stat_merger_test.cc index bd437374803e..e91cb3dda992 100644 --- a/test/common/stats/stat_merger_test.cc +++ b/test/common/stats/stat_merger_test.cc @@ -249,6 +249,25 @@ TEST_F(StatMergerThreadLocalTest, retainImportModeAfterMerge) { EXPECT_EQ(789 + 42, gauge.value()); } +// Verify that if we create a never import stat in the child process which then gets merged +// from the parent, that we retain the import-mode, and don't accumulate the updated +// value. https://github.com/envoyproxy/envoy/issues/7227 +TEST_F(StatMergerThreadLocalTest, retainNeverImportModeAfterMerge) { + Gauge& gauge = store_.gauge("mygauge", Gauge::ImportMode::NeverImport); + gauge.set(42); + EXPECT_EQ(Gauge::ImportMode::NeverImport, gauge.importMode()); + EXPECT_EQ(42, gauge.value()); + { + StatMerger stat_merger(store_); + Protobuf::Map counter_deltas; + Protobuf::Map gauges; + gauges["mygauge"] = 789; + stat_merger.mergeStats(counter_deltas, gauges); + } + EXPECT_EQ(Gauge::ImportMode::NeverImport, gauge.importMode()); + EXPECT_EQ(42, gauge.value()); +} + } // namespace } // namespace Stats } // namespace Envoy diff --git a/test/integration/hotrestart_test.sh b/test/integration/hotrestart_test.sh index f5c30aecef6f..25def55875e7 100755 --- a/test/integration/hotrestart_test.sh +++ b/test/integration/hotrestart_test.sh @@ -135,7 +135,7 @@ do ADMIN_ADDRESS_1=$(cat "${ADMIN_ADDRESS_PATH_1}") SERVER_LIVE_1=$(curl -sg http://${ADMIN_ADDRESS_1}/stats | grep server.live) - check [ "$SERVER_LIVE_1" = "server.live: 2" ]; + check [ "$SERVER_LIVE_1" = "server.live: 1" ]; start_test Checking that listener addresses have not changed HOT_RESTART_JSON_1="${TEST_TMPDIR}"/hot_restart.1."${TEST_INDEX}".yaml From c1b98bbb93996a3b6ead6df34e65da384e6f3d7b Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 17 Jun 2019 09:15:46 -0700 Subject: [PATCH 018/542] docs: fix requirements for csrf sandbox (#7296) Signed-off-by: Derek Schaller --- docs/root/start/sandboxes/csrf.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/start/sandboxes/csrf.rst b/docs/root/start/sandboxes/csrf.rst index 419ca98fcc69..0574e436e8aa 100644 --- a/docs/root/start/sandboxes/csrf.rst +++ b/docs/root/start/sandboxes/csrf.rst @@ -34,7 +34,7 @@ The following documentation runs through the setup of both services. **Step 1: Install Docker** -Ensure that you have a recent versions of ``docker``, ``docker-compose``, and ``docker-machine``. +Ensure that you have a recent versions of ``docker`` and ``docker-compose``. A simple way to achieve this is via the `Docker Toolbox `_. From 5dd7cc99af303cb92741718836ad4e73639d630f Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Mon, 17 Jun 2019 21:55:59 +0530 Subject: [PATCH 019/542] docs: link http filter and network filters in listener proto (#7274) Signed-off-by: Rama Chavali --- api/envoy/api/v2/listener/listener.proto | 21 +++----------- .../v2/http_connection_manager.proto | 28 ++----------------- 2 files changed, 6 insertions(+), 43 deletions(-) diff --git a/api/envoy/api/v2/listener/listener.proto b/api/envoy/api/v2/listener/listener.proto index 9c931cd51ae6..b9397ab7afc7 100644 --- a/api/envoy/api/v2/listener/listener.proto +++ b/api/envoy/api/v2/listener/listener.proto @@ -25,17 +25,8 @@ option (gogoproto.equal_all) = true; // Listener :ref:`configuration overview ` message Filter { - // The name of the filter to instantiate. The name must match a supported - // filter. The built-in filters are: - // - // [#comment:TODO(mattklein123): Auto generate the following list] - // * :ref:`envoy.client_ssl_auth` - // * :ref:`envoy.echo ` - // * :ref:`envoy.http_connection_manager ` - // * :ref:`envoy.mongo_proxy ` - // * :ref:`envoy.ratelimit ` - // * :ref:`envoy.redis_proxy ` - // * :ref:`envoy.tcp_proxy ` + // The name of the filter to instantiate. The name must match a + // :ref:`supported filter `. string name = 1 [(validate.rules).string.min_bytes = 1]; // Filter specific configuration which depends on the filter being @@ -199,12 +190,8 @@ message FilterChain { } message ListenerFilter { - // The name of the filter to instantiate. The name must match a supported - // filter. The built-in filters are: - // - // [#comment:TODO(mattklein123): Auto generate the following list] - // * :ref:`envoy.listener.original_dst ` - // * :ref:`envoy.listener.tls_inspector ` + // The name of the filter to instantiate. The name must match a + // :ref:`supported filter `. string name = 1 [(validate.rules).string.min_bytes = 1]; // Filter specific configuration which depends on the filter being instantiated. diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index f5a1394b954d..4cee964e1d2c 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -552,32 +552,8 @@ message ScopedRds { } message HttpFilter { - // The name of the filter to instantiate. The name must match a supported - // filter. The built-in filters are: - // - // [#comment:TODO(mattklein123): Auto generate the following list] - // * :ref:`envoy.buffer ` - // * :ref:`envoy.cors ` - // * :ref:`envoy.ext_authz ` - // * :ref:`envoy.fault ` - // * :ref:`envoy.filters.http.csrf ` - // * :ref:`envoy.filters.http.header_to_metadata ` - // * :ref:`envoy.filters.http.grpc_http1_reverse_bridge \ - // ` - // * :ref:`envoy.filters.http.jwt_authn ` - // * :ref:`envoy.filters.http.rbac ` - // * :ref:`envoy.filters.http.tap ` - // * :ref:`envoy.gzip ` - // * :ref:`envoy.http_dynamo_filter ` - // * :ref:`envoy.grpc_http1_bridge ` - // * :ref:`envoy.grpc_json_transcoder ` - // * :ref:`envoy.grpc_web ` - // * :ref:`envoy.health_check ` - // * :ref:`envoy.ip_tagging ` - // * :ref:`envoy.lua ` - // * :ref:`envoy.rate_limit ` - // * :ref:`envoy.router ` - // * :ref:`envoy.squash ` + // The name of the filter to instantiate. The name must match a + // :ref:`supported filter `. string name = 1 [(validate.rules).string.min_bytes = 1]; // Filter specific configuration which depends on the filter being instantiated. See the supported From bfd9c7180e895684abde21c4af80d8815f06a658 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Mon, 17 Jun 2019 15:12:12 -0400 Subject: [PATCH 020/542] add aspell-en to fedora instructions (#7302) I had to install it in fedora 30 for the git hooks to work. Signed-off-by: Yuval Kohavi --- bazel/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/README.md b/bazel/README.md index faf3da2f0b92..0b7cbeea22bd 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -40,7 +40,7 @@ for how to update or override dependencies. On Fedora (maybe also other red hat distros), run the following: ``` - dnf install cmake libtool libstdc++ ninja-build lld patch + dnf install cmake libtool libstdc++ ninja-build lld patch aspell-en ``` On macOS, you'll need to install several dependencies. This can be accomplished via [Homebrew](https://brew.sh/): From 54e65f89a106b692ebe266eb5b0f852bde2a627e Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 17 Jun 2019 19:00:07 -0400 Subject: [PATCH 021/542] tools: check_format verification that we add owners for new extensions (#7265) Signed-off-by: Alyssa Wilk --- CODEOWNERS | 4 +- tools/check_format.py | 133 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 132 insertions(+), 5 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 5df7c9c8ea9d..767bd14d6bb0 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -10,6 +10,8 @@ /*/extensions/filters/http/original_src @snowp @klarose # original_src listener filter extension /*/extensions/filters/listener/original_src @snowp @klarose +# original_src common extension +extensions/filters/common/original_src @snowp @klarose # dubbo_proxy extension /*/extensions/filters/network/dubbo_proxy @zyfjeff @lizan # thrift_proxy extension @@ -33,4 +35,4 @@ # zookeeper_proxy extension /*/extensions/filters/network/zookeeper_proxy @rgs1 @snowp # redis cluster extension -/*/extensions/clusters/redis @msukalski @henryyyang @mattklein123 \ No newline at end of file +/*/extensions/clusters/redis @msukalski @henryyyang @mattklein123 diff --git a/tools/check_format.py b/tools/check_format.py index ec748006c4a0..94446a829471 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -78,6 +78,81 @@ LIBCXX_REPLACEMENTS = { "absl::make_unique<": "std::make_unique<", } + +UNOWNED_EXTENSIONS = { + "extensions/filters/http/ratelimit", + "extensions/filters/http/buffer", + "extensions/filters/http/grpc_http1_bridge", + "extensions/filters/http/grpc_http1_reverse_bridge", + "extensions/filters/http/rbac", + "extensions/filters/http/gzip", + "extensions/filters/http/ip_tagging", + "extensions/filters/http/tap", + "extensions/filters/http/fault", + "extensions/filters/http/grpc_json_transcoder", + "extensions/filters/http/health_check", + "extensions/filters/http/router", + "extensions/filters/http/cors", + "extensions/filters/http/ext_authz", + "extensions/filters/http/dynamo", + "extensions/filters/http/lua", + "extensions/filters/http/grpc_web", + "extensions/filters/http/common", + "extensions/filters/http/common/aws", + "extensions/filters/http/squash", + "extensions/filters/common", + "extensions/filters/common/ratelimit", + "extensions/filters/common/rbac", + "extensions/filters/common/fault", + "extensions/filters/common/ext_authz", + "extensions/filters/common/lua", + "extensions/filters/common/original_src", + "extensions/filters/listener/original_dst", + "extensions/filters/listener/proxy_protocol", + "extensions/filters/listener/tls_inspector", + "extensions/grpc_credentials/example", + "extensions/grpc_credentials/file_based_metadata", + "extensions/stat_sinks/dog_statsd", + "extensions/stat_sinks/hystrix", + "extensions/stat_sinks/metrics_service", + "extensions/stat_sinks/statsd", + "extensions/stat_sinks/common", + "extensions/stat_sinks/common/statsd", + "extensions/health_checkers/redis", + "extensions/access_loggers/http_grpc", + "extensions/access_loggers/file", + "extensions/common/tap", + "extensions/transport_sockets/raw_buffer", + "extensions/transport_sockets/tap", + "extensions/transport_sockets/tls", + "extensions/tracers/zipkin", + "extensions/tracers/dynamic_ot", + "extensions/tracers/opencensus", + "extensions/tracers/lightstep", + "extensions/tracers/common", + "extensions/tracers/common/ot", + "extensions/resource_monitors/injected_resource", + "extensions/resource_monitors/fixed_heap", + "extensions/resource_monitors/common", + "extensions/retry/priority", + "extensions/retry/priority/previous_priorities", + "extensions/retry/host", + "extensions/retry/host/previous_hosts", + "extensions/filters/network/ratelimit", + "extensions/filters/network/client_ssl_auth", + "extensions/filters/network/http_connection_manager", + "extensions/filters/network/rbac", + "extensions/filters/network/tcp_proxy", + "extensions/filters/network/echo", + "extensions/filters/network/ext_authz", + "extensions/filters/network/redis_proxy", + "extensions/filters/network/kafka", + "extensions/filters/network/kafka/protocol_code_generator", + "extensions/filters/network/kafka/serialization_code_generator", + "extensions/filters/network/mongo_proxy", + "extensions/filters/network/common", + "extensions/filters/network/common/redis", +} # yapf: enable @@ -622,11 +697,30 @@ def checkFormatReturnTraceOnError(file_path): return traceback.format_exc().split("\n") +def checkOwners(dir_name, owned_directories, error_messages): + """Checks to make sure a given directory is present either in CODEOWNERS or OWNED_EXTENSIONS + + Args: + dir_name: the directory being checked. + owned_directories: directories currently listed in CODEOWNERS. + error_messages: where to put an error message for new unowned directories. + """ + found = False + for owned in owned_directories: + if owned.startswith(dir_name) or dir_name.startswith(owned): + found = True + if not found and dir_name not in UNOWNED_EXTENSIONS: + error_messages.append("New directory %s appears to not have owners in CODEOWNERS" % dir_name) + + def checkFormatVisitor(arg, dir_name, names): """Run checkFormat in parallel for the given files. Args: - arg: a tuple (pool, result_list) for starting tasks asynchronously. + arg: a tuple (pool, result_list, owned_directories, error_messages) + pool and result_list are for starting tasks asynchronously. + owned_directories tracks directories listed in the CODEOWNERS file. + error_messages is a list of string format errors. dir_name: the parent directory of the given files. names: a list of file names. """ @@ -635,7 +729,18 @@ def checkFormatVisitor(arg, dir_name, names): # python lists are passed as references, this is used to collect the list of # async results (futures) from running checkFormat and passing them back to # the caller. - pool, result_list = arg + pool, result_list, owned_directories, error_messags = arg + + # Sanity check CODEOWNERS. This doesn't need to be done in a multi-threaded + # manner as it is a small and limited list. + source_prefix = './source/' + full_prefix = './source/extensions/' + # Check to see if this directory is a subdir under /source/extensions + # Also ignore top level directories under /source/extensions since we don't + # need owners for source/extensions/access_loggers etc, just the subdirectories. + if dir_name.startswith(full_prefix) and '/' in dir_name[len(full_prefix):]: + checkOwners(dir_name[len(source_prefix):], owned_directories, error_messages) + for file_name in names: result = pool.apply_async(checkFormatReturnTraceOnError, args=(dir_name + "/" + file_name,)) result_list.append(result) @@ -705,6 +810,24 @@ def checkErrorMessages(error_messages): if args.add_excluded_prefixes: EXCLUDED_PREFIXES += tuple(args.add_excluded_prefixes) + # Returns the list of directories with owners listed in CODEOWNERS + def ownedDirectories(): + owned = [] + try: + with open('./CODEOWNERS') as f: + for line in f: + # If this line is of the form "extensions/... @owner1 @owner2" capture the directory + # name and store it in the list of directories with documented owners. + m = re.search(r'..*(extensions[^@]* )@.*@.*', line) + if m is not None: + owned.append(m.group(1).strip()) + return owned + except IOError: + return [] # for the check format tests. + + # Calculate the list of owned directories once per run. + owned_directories = ownedDirectories() + # Check whether all needed external tools are available. ct_error_messages = checkTools() if checkErrorMessages(ct_error_messages): @@ -715,15 +838,17 @@ def checkErrorMessages(error_messages): else: pool = multiprocessing.Pool(processes=args.num_workers) results = [] + error_messages = [] # For each file in target_path, start a new task in the pool and collect the # results (results is passed by reference, and is used as an output). - os.path.walk(target_path, checkFormatVisitor, (pool, results)) + os.path.walk(target_path, checkFormatVisitor, + (pool, results, owned_directories, error_messages)) # Close the pool to new tasks, wait for all of the running tasks to finish, # then collect the error messages. pool.close() pool.join() - error_messages = sum((r.get() for r in results), []) + error_messages += sum((r.get() for r in results), []) if checkErrorMessages(error_messages): print("ERROR: check format failed. run 'tools/check_format.py fix'") From 6a6e43a94201e9059de61fd6e94fda984615b54c Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 17 Jun 2019 17:54:25 -0700 Subject: [PATCH 022/542] ci: make protodoc.bzl ready for Bazel 0.27 (#7304) * ci: make protodoc.bzl ready for Bazel 0.27 Risk Level: Low Testing: local Signed-off-by: Lizan Zhou --- docs/build.sh | 2 +- tools/protodoc/protodoc.bzl | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/build.sh b/docs/build.sh index 7b1725c4daa5..284222935c36 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -48,7 +48,7 @@ pip install -r "${SCRIPT_DIR}"/requirements.txt bazel build ${BAZEL_BUILD_OPTIONS} @envoy_api//docs:protos --aspects \ tools/protodoc/protodoc.bzl%proto_doc_aspect --output_groups=rst --action_env=CPROFILE_ENABLED \ - --action_env=ENVOY_BLOB_SHA --spawn_strategy=standalone + --action_env=ENVOY_BLOB_SHA --spawn_strategy=standalone --host_force_python=PY2 # These are the protos we want to put in docs, this list will grow. # TODO(htuch): Factor this out of this script. diff --git a/tools/protodoc/protodoc.bzl b/tools/protodoc/protodoc.bzl index bc9540a88888..e619f8b457c1 100644 --- a/tools/protodoc/protodoc.bzl +++ b/tools/protodoc/protodoc.bzl @@ -44,7 +44,7 @@ def _proto_doc_aspect_impl(target, ctx): # these don't include source_code_info, which we need for comment # extractions. See https://github.com/bazelbuild/bazel/issues/3971. import_paths = [] - for f in target[ProtoInfo].transitive_sources: + for f in target[ProtoInfo].transitive_sources.to_list(): if f.root.path: import_path = f.root.path + "/" + f.owner.workspace_root else: @@ -64,10 +64,11 @@ def _proto_doc_aspect_impl(target, ctx): args += ["-I" + import_path for import_path in import_paths] args += ["--plugin=protoc-gen-protodoc=" + ctx.executable._protodoc.path, "--protodoc_out=" + output_path] args += [_proto_path(src) for src in target[ProtoInfo].direct_sources] - ctx.action( + ctx.actions.run( executable = ctx.executable._protoc, arguments = args, - inputs = [ctx.executable._protodoc] + target[ProtoInfo].transitive_sources.to_list(), + inputs = target[ProtoInfo].transitive_sources, + tools = [ctx.executable._protodoc], outputs = outputs, mnemonic = "ProtoDoc", use_default_shell_env = True, From b53cb3734d640d9c3c378a5af36a60c54e2f9809 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 18 Jun 2019 05:42:06 -0700 Subject: [PATCH 023/542] Fix macOS CI with Bazel 0.27 (#7305) * Fix macOS CI with Bazel 0.27 Signed-off-by: Lizan Zhou --- test/extensions/filters/network/thrift_proxy/driver/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/extensions/filters/network/thrift_proxy/driver/BUILD b/test/extensions/filters/network/thrift_proxy/driver/BUILD index beed670c9e20..b0461509c7a1 100644 --- a/test/extensions/filters/network/thrift_proxy/driver/BUILD +++ b/test/extensions/filters/network/thrift_proxy/driver/BUILD @@ -16,6 +16,7 @@ filegroup( py_binary( name = "client", srcs = ["client.py"], + python_version = "PY2", deps = [ "//test/extensions/filters/network/thrift_proxy/driver/fbthrift:fbthrift_lib", "//test/extensions/filters/network/thrift_proxy/driver/finagle:finagle_lib", @@ -27,6 +28,7 @@ py_binary( py_binary( name = "server", srcs = ["server.py"], + python_version = "PY2", deps = [ "//test/extensions/filters/network/thrift_proxy/driver/fbthrift:fbthrift_lib", "//test/extensions/filters/network/thrift_proxy/driver/finagle:finagle_lib", From d2068d3e3cd1b5165d8d1044e4ec7b5483643c6c Mon Sep 17 00:00:00 2001 From: ahedberg Date: Tue, 18 Jun 2019 09:08:33 -0400 Subject: [PATCH 024/542] test: add process_object_ to BaseIntegrationTest (#7183) Description: Adds a process_object_ member to BaseIntegrationTest. If an integration test subclassing from BaseIntegrationTest sets this member, the envoy server will be started with a ProcessContext referencing that object. Risk Level: low (test-only, defaults to nullopt) Testing: new integration test with ProcessContext filter Docs Changes: none Release Notes: n/a Fixes #6969 Signed-off-by: Ashley Hedberg --- test/integration/BUILD | 1 + test/integration/filters/BUILD | 17 ++++++ .../filters/process_context_filter.cc | 56 +++++++++++++++++++ .../filters/process_context_filter.h | 17 ++++++ test/integration/integration.cc | 2 +- test/integration/integration.h | 5 ++ test/integration/integration_test.cc | 41 ++++++++++++++ test/integration/server.cc | 35 ++++++++---- test/integration/server.h | 41 ++++++++------ 9 files changed, 185 insertions(+), 30 deletions(-) create mode 100644 test/integration/filters/process_context_filter.cc create mode 100644 test/integration/filters/process_context_filter.h diff --git a/test/integration/BUILD b/test/integration/BUILD index ce5cb1e55458..35aa1046dec1 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -447,6 +447,7 @@ envoy_cc_test( "//source/extensions/filters/http/grpc_http1_bridge:config", "//source/extensions/filters/http/health_check:config", "//test/integration/filters:clear_route_cache_filter_lib", + "//test/integration/filters:process_context_lib", "//test/mocks/http:http_mocks", "//test/test_common:utility_lib", ], diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD index e2a1fecc2c5f..7e2647880eb7 100644 --- a/test/integration/filters/BUILD +++ b/test/integration/filters/BUILD @@ -111,6 +111,23 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "process_context_lib", + srcs = [ + "process_context_filter.cc", + ], + hdrs = [ + "process_context_filter.h", + ], + deps = [ + "//include/envoy/http:filter_interface", + "//include/envoy/registry", + "//include/envoy/server:process_context_interface", + "//source/extensions/filters/http/common:empty_http_filter_config_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + ], +) + envoy_cc_test_library( name = "stop_iteration_and_continue", srcs = [ diff --git a/test/integration/filters/process_context_filter.cc b/test/integration/filters/process_context_filter.cc new file mode 100644 index 000000000000..6765042b2c2a --- /dev/null +++ b/test/integration/filters/process_context_filter.cc @@ -0,0 +1,56 @@ +#include "test/integration/filters/process_context_filter.h" + +#include +#include + +#include "envoy/http/filter.h" +#include "envoy/http/header_map.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "extensions/filters/http/common/empty_http_filter_config.h" +#include "extensions/filters/http/common/pass_through_filter.h" + +namespace Envoy { + +// A test filter that rejects all requests if the ProcessObject held by the +// ProcessContext is unhealthy, and responds OK to all requests otherwise. +class ProcessContextFilter : public Http::PassThroughFilter { +public: + ProcessContextFilter(ProcessContext& process_context) + : process_object_(dynamic_cast(process_context.get())) {} + Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap&, bool) override { + if (!process_object_.isHealthy()) { + decoder_callbacks_->sendLocalReply(Envoy::Http::Code::InternalServerError, + "ProcessObjectForFilter is unhealthy", nullptr, + absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + decoder_callbacks_->sendLocalReply(Envoy::Http::Code::OK, "ProcessObjectForFilter is healthy", + nullptr, absl::nullopt, ""); + return Http::FilterHeadersStatus::StopIteration; + } + +private: + ProcessObjectForFilter& process_object_; +}; + +class ProcessContextFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpFilterConfig { +public: + ProcessContextFilterConfig() : EmptyHttpFilterConfig("process-context-filter") {} + + Http::FilterFactoryCb + createFilter(const std::string&, + Server::Configuration::FactoryContext& factory_context) override { + return [&factory_context](Http::FilterChainFactoryCallbacks& callbacks) { + callbacks.addStreamFilter( + std::make_shared(factory_context.processContext())); + }; + } +}; + +static Registry::RegisterFactory + register_; + +} // namespace Envoy diff --git a/test/integration/filters/process_context_filter.h b/test/integration/filters/process_context_filter.h new file mode 100644 index 000000000000..7e1fde35ff64 --- /dev/null +++ b/test/integration/filters/process_context_filter.h @@ -0,0 +1,17 @@ +#pragma once + +#include "envoy/server/process_context.h" + +namespace Envoy { + +class ProcessObjectForFilter : public ProcessObject { +public: + explicit ProcessObjectForFilter(bool is_healthy) : is_healthy_(is_healthy) {} + ~ProcessObjectForFilter() override {} + + bool isHealthy() { return is_healthy_; } + +private: + bool is_healthy_; +}; +} // namespace Envoy diff --git a/test/integration/integration.cc b/test/integration/integration.cc index 8cbbf958e6ca..daa2a3911fc6 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -409,7 +409,7 @@ void BaseIntegrationTest::createGeneratedApiTestServer(const std::string& bootst const std::vector& port_names) { test_server_ = IntegrationTestServer::create(bootstrap_path, version_, on_server_init_function_, deterministic_, timeSystem(), *api_, - defer_listener_finalization_); + defer_listener_finalization_, process_object_); if (config_helper_.bootstrap().static_resources().listeners_size() > 0 && !defer_listener_finalization_) { diff --git a/test/integration/integration.h b/test/integration/integration.h index 72f00790a9c6..873dde090a2c 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -5,6 +5,8 @@ #include #include +#include "envoy/server/process_context.h" + #include "common/http/codec_client.h" #include "test/common/grpc/grpc_client_integration.h" @@ -20,6 +22,7 @@ #include "test/test_common/simulated_time_system.h" #include "test/test_common/test_time.h" +#include "absl/types/optional.h" #include "spdlog/spdlog.h" namespace Envoy { @@ -322,6 +325,8 @@ class BaseIntegrationTest : Logger::Loggable { InstanceConstSharedPtrFn upstream_address_fn_; // The config for envoy start-up. ConfigHelper config_helper_; + // The ProcessObject to use when constructing the envoy server. + absl::optional> process_object_{absl::nullopt}; // Steps that should be done in parallel with the envoy server starting. E.g., xDS // pre-init, control plane synchronization needed for server start. diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index b3e9829bf8b1..7ad5e339703b 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -9,6 +9,7 @@ #include "common/protobuf/utility.h" #include "test/integration/autonomous_upstream.h" +#include "test/integration/filters/process_context_filter.h" #include "test/integration/utility.h" #include "test/mocks/http/mocks.h" #include "test/test_common/network_utility.h" @@ -792,6 +793,46 @@ TEST_P(IntegrationTest, NoConnectionPoolsFree) { EXPECT_EQ(test_server_->counter("cluster.cluster_0.upstream_cx_pool_overflow")->value(), 1); } +TEST_P(IntegrationTest, ProcessObjectHealthy) { + config_helper_.addFilter("{ name: process-context-filter, config: {} }"); + + ProcessObjectForFilter healthy_object(true); + process_object_ = healthy_object; + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = + codec_client_->makeHeaderOnlyRequest(Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/healthcheck"}, + {":authority", "host"}, + {"connection", "close"}}); + response->waitForEndStream(); + codec_client_->waitForDisconnect(); + + EXPECT_TRUE(response->complete()); + EXPECT_THAT(response->headers(), HttpStatusIs("200")); +} + +TEST_P(IntegrationTest, ProcessObjectUnealthy) { + config_helper_.addFilter("{ name: process-context-filter, config: {} }"); + + ProcessObjectForFilter unhealthy_object(false); + process_object_ = unhealthy_object; + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto response = + codec_client_->makeHeaderOnlyRequest(Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/healthcheck"}, + {":authority", "host"}, + {"connection", "close"}}); + response->waitForEndStream(); + codec_client_->waitForDisconnect(); + + EXPECT_TRUE(response->complete()); + EXPECT_THAT(response->headers(), HttpStatusIs("500")); +} + INSTANTIATE_TEST_SUITE_P(IpVersions, UpstreamEndpointIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); diff --git a/test/integration/server.cc b/test/integration/server.cc index 9f314549906b..ddd12b31191d 100644 --- a/test/integration/server.cc +++ b/test/integration/server.cc @@ -47,10 +47,12 @@ OptionsImpl createTestOptionsImpl(const std::string& config_path, const std::str IntegrationTestServerPtr IntegrationTestServer::create( const std::string& config_path, const Network::Address::IpVersion version, std::function on_server_init_function, bool deterministic, - Event::TestTimeSystem& time_system, Api::Api& api, bool defer_listener_finalization) { + Event::TestTimeSystem& time_system, Api::Api& api, bool defer_listener_finalization, + absl::optional> process_object) { IntegrationTestServerPtr server{ std::make_unique(time_system, api, config_path)}; - server->start(version, on_server_init_function, deterministic, defer_listener_finalization); + server->start(version, on_server_init_function, deterministic, defer_listener_finalization, + process_object); return server; } @@ -64,13 +66,16 @@ void IntegrationTestServer::waitUntilListenersReady() { ENVOY_LOG(info, "listener wait complete"); } -void IntegrationTestServer::start(const Network::Address::IpVersion version, - std::function on_server_init_function, bool deterministic, - bool defer_listener_finalization) { +void IntegrationTestServer::start( + const Network::Address::IpVersion version, std::function on_server_init_function, + bool deterministic, bool defer_listener_finalization, + absl::optional> process_object) { ENVOY_LOG(info, "starting integration test server"); ASSERT(!thread_); - thread_ = api_.threadFactory().createThread( - [version, deterministic, this]() -> void { threadRoutine(version, deterministic); }); + thread_ = + api_.threadFactory().createThread([version, deterministic, process_object, this]() -> void { + threadRoutine(version, deterministic, process_object); + }); // If any steps need to be done prior to workers starting, do them now. E.g., xDS pre-init. // Note that there is no synchronization guaranteeing this happens either @@ -139,8 +144,9 @@ void IntegrationTestServer::serverReady() { server_set_.setReady(); } -void IntegrationTestServer::threadRoutine(const Network::Address::IpVersion version, - bool deterministic) { +void IntegrationTestServer::threadRoutine( + const Network::Address::IpVersion version, bool deterministic, + absl::optional> process_object) { OptionsImpl options(Server::createTestOptionsImpl(config_path_, "", version)); Thread::MutexBasicLockable lock; @@ -151,7 +157,7 @@ void IntegrationTestServer::threadRoutine(const Network::Address::IpVersion vers random_generator = std::make_unique(); } createAndRunEnvoyServer(options, time_system_, Network::Utility::getLocalAddress(version), *this, - lock, *this, std::move(random_generator)); + lock, *this, std::move(random_generator), process_object); } void IntegrationTestServer::onRuntimeCreated() { @@ -168,17 +174,22 @@ void IntegrationTestServerImpl::createAndRunEnvoyServer( OptionsImpl& options, Event::TimeSystem& time_system, Network::Address::InstanceConstSharedPtr local_address, ListenerHooks& hooks, Thread::BasicLockable& access_log_lock, Server::ComponentFactory& component_factory, - Runtime::RandomGeneratorPtr&& random_generator) { + Runtime::RandomGeneratorPtr&& random_generator, + absl::optional> process_object) { { Stats::FakeSymbolTableImpl symbol_table; Server::HotRestartNopImpl restarter; ThreadLocal::InstanceImpl tls; Stats::HeapStatDataAllocator stats_allocator(symbol_table); Stats::ThreadLocalStoreImpl stat_store(stats_allocator); + std::unique_ptr process_context; + if (process_object.has_value()) { + process_context = std::make_unique(process_object->get()); + } Server::InstanceImpl server(options, time_system, local_address, hooks, restarter, stat_store, access_log_lock, component_factory, std::move(random_generator), tls, Thread::threadFactoryForTest(), - Filesystem::fileSystemForTest(), nullptr); + Filesystem::fileSystemForTest(), std::move(process_context)); // This is technically thread unsafe (assigning to a shared_ptr accessed // across threads), but because we synchronize below through serverReady(), the only // consumer on the main test thread in ~IntegrationTestServerImpl will not race. diff --git a/test/integration/server.h b/test/integration/server.h index 801b4ac75865..ab21e43b3538 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -7,6 +7,7 @@ #include #include "envoy/server/options.h" +#include "envoy/server/process_context.h" #include "envoy/stats/stats.h" #include "common/common/assert.h" @@ -24,6 +25,7 @@ #include "test/test_common/utility.h" #include "absl/synchronization/notification.h" +#include "absl/types/optional.h" namespace Envoy { namespace Server { @@ -228,11 +230,12 @@ class IntegrationTestServer : public Logger::Loggable, public IntegrationTestServerStats, public Server::ComponentFactory { public: - static IntegrationTestServerPtr create(const std::string& config_path, - const Network::Address::IpVersion version, - std::function on_server_init_function, - bool deterministic, Event::TestTimeSystem& time_system, - Api::Api& api, bool defer_listener_finalization = false); + static IntegrationTestServerPtr + create(const std::string& config_path, const Network::Address::IpVersion version, + std::function on_server_init_function, bool deterministic, + Event::TestTimeSystem& time_system, Api::Api& api, + bool defer_listener_finalization = false, + absl::optional> process_object = absl::nullopt); // Note that the derived class is responsible for tearing down the server in its // destructor. ~IntegrationTestServer(); @@ -250,7 +253,8 @@ class IntegrationTestServer : public Logger::Loggable, void start(const Network::Address::IpVersion version, std::function on_server_init_function, bool deterministic, - bool defer_listener_finalization); + bool defer_listener_finalization, + absl::optional> process_object); void waitForCounterEq(const std::string& name, uint64_t value) override { while (counter(name) == nullptr || counter(name)->value() != value) { @@ -320,11 +324,12 @@ class IntegrationTestServer : public Logger::Loggable, // functions server(), stat_store(), and admin_address() may be called, but before the server // has been started. // The subclass is also responsible for tearing down this server in its destructor. - virtual void createAndRunEnvoyServer(OptionsImpl& options, Event::TimeSystem& time_system, - Network::Address::InstanceConstSharedPtr local_address, - ListenerHooks& hooks, Thread::BasicLockable& access_log_lock, - Server::ComponentFactory& component_factory, - Runtime::RandomGeneratorPtr&& random_generator) PURE; + virtual void createAndRunEnvoyServer( + OptionsImpl& options, Event::TimeSystem& time_system, + Network::Address::InstanceConstSharedPtr local_address, ListenerHooks& hooks, + Thread::BasicLockable& access_log_lock, Server::ComponentFactory& component_factory, + Runtime::RandomGeneratorPtr&& random_generator, + absl::optional> process_object) PURE; // Will be called by subclass on server thread when the server is ready to be accessed. The // server may not have been run yet, but all server access methods (server(), stat_store(), @@ -335,7 +340,8 @@ class IntegrationTestServer : public Logger::Loggable, /** * Runs the real server on a thread. */ - void threadRoutine(const Network::Address::IpVersion version, bool deterministic); + void threadRoutine(const Network::Address::IpVersion version, bool deterministic, + absl::optional> process_object); Event::TestTimeSystem& time_system_; Api::Api& api_; @@ -371,11 +377,12 @@ class IntegrationTestServerImpl : public IntegrationTestServer { Network::Address::InstanceConstSharedPtr admin_address() override { return admin_address_; } private: - void createAndRunEnvoyServer(OptionsImpl& options, Event::TimeSystem& time_system, - Network::Address::InstanceConstSharedPtr local_address, - ListenerHooks& hooks, Thread::BasicLockable& access_log_lock, - Server::ComponentFactory& component_factory, - Runtime::RandomGeneratorPtr&& random_generator) override; + void createAndRunEnvoyServer( + OptionsImpl& options, Event::TimeSystem& time_system, + Network::Address::InstanceConstSharedPtr local_address, ListenerHooks& hooks, + Thread::BasicLockable& access_log_lock, Server::ComponentFactory& component_factory, + Runtime::RandomGeneratorPtr&& random_generator, + absl::optional> process_object) override; // Owned by this class. An owning pointer is not used because the actual allocation is done // on a stack in a non-main thread. From 4fa3e9fae1fd170576e73ce4bc3a2efe4ff6aaed Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 18 Jun 2019 11:55:50 -0400 Subject: [PATCH 025/542] test: Add macros for doing memory-usage tests of varying precision, based on platform (#7243) * add macros and helper functions for doing approximate and exact memory tests. Signed-off-by: Joshua Marantz --- ci/do_ci.sh | 17 +++- test/common/stats/BUILD | 6 +- test/common/stats/stat_test_utility.cc | 58 +++++++----- test/common/stats/stat_test_utility.h | 91 +++++++++++++++++-- test/common/stats/symbol_table_impl_test.cc | 33 ++----- test/common/stats/thread_local_store_test.cc | 51 ++++------- .../quiche/platform/quic_platform_test.cc | 14 ++- test/integration/stats_integration_test.cc | 33 +++---- tools/spelling_dictionary.txt | 1 + 9 files changed, 183 insertions(+), 121 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 9c621749cc59..430f8d33c579 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -55,10 +55,19 @@ function cp_binary_for_image_build() { strip "${ENVOY_DELIVERY_DIR}"/envoy -o "${ENVOY_SRCDIR}"/build_"$1"_stripped/envoy } +# When testing memory consumption, we want to test against exact byte-counts +# where possible. As these differ between platforms and compile options, we +# define the 'release' builds as canonical and test them only in CI, so the +# toolchain is kept consistent. This ifdef is checked in +# test/common/stats/stat_test_utility.cc when computing +# Stats::TestUtil::MemoryTest::mode(). +MEMORY_TEST_EXACT_ARGS="--cxxopt=-DMEMORY_TEST_EXACT=1" + function bazel_binary_build() { BINARY_TYPE="$1" if [[ "${BINARY_TYPE}" == "release" ]]; then COMPILE_TYPE="opt" + CONFIG_ARGS="$MEMORY_TEST_EXACT_ARGS" elif [[ "${BINARY_TYPE}" == "debug" ]]; then COMPILE_TYPE="dbg" elif [[ "${BINARY_TYPE}" == "sizeopt" ]]; then @@ -85,21 +94,23 @@ if [[ "$1" == "bazel.release" ]]; then setup_clang_toolchain echo "bazel release build with tests..." bazel_binary_build release + RELEASE_OPTIONS="-c opt ${MEMORY_TEST_EXACT_ARGS}" + BAZEL_TEST_RELEASE_OPTIONS="test ${BAZEL_TEST_OPTIONS} ${RELEASE_OPTIONS}" if [[ $# -gt 1 ]]; then shift echo "Testing $* ..." # Run only specified tests. Argument can be a single test # (e.g. '//test/common/common:assert_test') or a test group (e.g. '//test/common/...') - bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c opt $* + bazel_with_collection $BAZEL_TEST_RELEASE_OPTIONS $* else echo "Testing..." # We have various test binaries in the test directory such as tools, benchmarks, etc. We # run a build pass to make sure they compile. - bazel build ${BAZEL_BUILD_OPTIONS} -c opt //include/... //source/... //test/... + bazel build ${BAZEL_BUILD_OPTIONS} ${RELEASE_OPTIONS} //include/... //source/... //test/... # Now run all of the tests which should already be compiled. - bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c opt //test/... + bazel_with_collection $BAZEL_TEST_RELEASE_OPTIONS //test/... fi exit 0 elif [[ "$1" == "bazel.release.server_only" ]]; then diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 83b6e6fcd3c7..49f3a6168aaf 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -58,13 +58,17 @@ envoy_cc_test_library( external_deps = [ "abseil_strings", ], - deps = ["//source/common/memory:stats_lib"], + deps = [ + "//source/common/common:assert_lib", + "//source/common/memory:stats_lib", + ], ) envoy_cc_test( name = "stats_matcher_impl_test", srcs = ["stats_matcher_impl_test.cc"], deps = [ + "//source/common/memory:stats_lib", "//source/common/stats:stats_matcher_lib", "//test/test_common:utility_lib", ], diff --git a/test/common/stats/stat_test_utility.cc b/test/common/stats/stat_test_utility.cc index dad7b452f364..7a9241d2c5b4 100644 --- a/test/common/stats/stat_test_utility.cc +++ b/test/common/stats/stat_test_utility.cc @@ -1,35 +1,12 @@ #include "test/common/stats/stat_test_utility.h" +#include "common/common/assert.h" #include "common/memory/stats.h" namespace Envoy { namespace Stats { namespace TestUtil { -bool hasDeterministicMallocStats() { -#if defined(TCMALLOC) && !defined(ENVOY_MEMORY_DEBUG_ENABLED) - // We can only test absolute memory usage if the malloc library is a known - // quantity. This decision is centralized here. As the preferred malloc - // library for Envoy is TCMALLOC that's what we test for here. If we switch - // to a different malloc library than we'd have to re-evaluate all the - // thresholds in the tests referencing hasDeterministicMallocStats(). - // - // Note that different versions of STL and other compiler/architecture - // differences may also impact memory usage, so unfortunately memory - // comparisons need to have some slack. There have recently emerged - // some memory-allocation differences between development and Envoy CI - // and Bazel CI (which compiles Envoy as a test of Bazel). - - // Do one quick sanity check to see if we can measure memory usage at all. - const size_t start_mem = Memory::Stats::totalCurrentlyAllocated(); - std::unique_ptr data(new char[10000]); - const size_t end_mem = Memory::Stats::totalCurrentlyAllocated(); - return end_mem - start_mem >= 10000; // actually 10240 -#else - return false; -#endif -} - void forEachSampleStat(int num_clusters, std::function fn) { // These are stats that are repeated for each cluster as of Oct 2018, with a // very basic configuration with no traffic. @@ -123,6 +100,39 @@ void forEachSampleStat(int num_clusters, std::function } } +MemoryTest::Mode MemoryTest::mode() { +#if !defined(TCMALLOC) || defined(ENVOY_MEMORY_DEBUG_ENABLED) + // We can only test absolute memory usage if the malloc library is a known + // quantity. This decision is centralized here. As the preferred malloc + // library for Envoy is TCMALLOC that's what we test for here. If we switch + // to a different malloc library than we'd have to re-evaluate all the + // thresholds in the tests referencing MemoryTest. + return Mode::Disabled; +#else + // Even when using TCMALLOC is defined, it appears that + // Memory::Stats::totalCurrentlyAllocated() does not work as expected + // on some platforms, so try to force-allocate some heap memory + // and determine whether we can measure it. + const size_t start_mem = Memory::Stats::totalCurrentlyAllocated(); + volatile std::string long_string("more than 22 chars to exceed libc++ short-string optimization"); + const size_t end_mem = Memory::Stats::totalCurrentlyAllocated(); + bool can_measure_memory = end_mem > start_mem; + +#if defined(MEMORY_TEST_EXACT) // Set in "ci/do_ci.sh" for 'release' tests. + RELEASE_ASSERT(can_measure_memory, "Compilation is set up for canonical memory measurements, " + "but memory measurement looks broken"); + return Mode::Canonical; +#else + // Different versions of STL and other compiler/architecture differences may + // also impact memory usage, so when not compiling with MEMORY_TEST_EXACT, + // memory comparisons must be given some slack. There have recently emerged + // some memory-allocation differences between development and Envoy CI and + // Bazel CI (which compiles Envoy as a test of Bazel). + return can_measure_memory ? Mode::Approximate : Mode::Disabled; +#endif +#endif +} + } // namespace TestUtil } // namespace Stats } // namespace Envoy diff --git a/test/common/stats/stat_test_utility.h b/test/common/stats/stat_test_utility.h index 321ce0855468..afc97bb9c6cc 100644 --- a/test/common/stats/stat_test_utility.h +++ b/test/common/stats/stat_test_utility.h @@ -1,5 +1,8 @@ #pragma once +#include "common/common/logger.h" +#include "common/memory/stats.h" + #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" @@ -7,13 +10,6 @@ namespace Envoy { namespace Stats { namespace TestUtil { -/** - * Determines whether the library has deterministic malloc-stats results. - * - * @return bool true if the Memory::Stats::totalCurrentlyAllocated() has stable results. - */ -bool hasDeterministicMallocStats(); - /** * Calls fn for a sampling of plausible stat names given a number of clusters. * This is intended for memory and performance benchmarking, where the syntax of @@ -27,6 +23,87 @@ bool hasDeterministicMallocStats(); */ void forEachSampleStat(int num_clusters, std::function fn); +// Tracks memory consumption over a span of time. Test classes instantiate a +// MemoryTest object to start measuring heap memory, and call consumedBytes() to +// determine how many bytes have been consumed since the class was instantiated. +// +// That value should then be passed to EXPECT_MEMORY_EQ and EXPECT_MEMORY_LE, +// defined below, as the interpretation of this value can differ based on +// platform and compilation mode. +class MemoryTest { +public: + // There are 3 cases: + // 1. Memory usage API is available, and is built using with a canonical + // toolchain, enabling exact comparisons against an expected number of + // bytes consumed. The canonical environment is Envoy CI release builds. + // 2. Memory usage API is available, but the current build may subtly differ + // in memory consumption from #1. We'd still like to track memory usage + // but it needs to be approximate. + // 3. Memory usage API is not available. In this case, the code is executed + // but no testing occurs. + enum class Mode { + Disabled, // No memory usage data available on platform. + Canonical, // Memory usage is available, and current platform is canonical. + Approximate, // Memory usage is available, but variances form canonical expected. + }; + + MemoryTest() : memory_at_construction_(Memory::Stats::totalCurrentlyAllocated()) {} + + /** + * @return the memory execution testability mode for the current compiler, architecture, + * and compile flags. + */ + static Mode mode(); + + size_t consumedBytes() const { + // Note that this subtraction of two unsigned numbers will yield a very + // large number if memory has actually shrunk since construction. In that + // case, the EXPECT_MEMORY_EQ and EXPECT_MEMORY_LE macros will both report + // failures, as desired, though the failure log may look confusing. + // + // Note also that tools like ubsan may report this as an unsigned integer + // underflow, if run with -fsanitize=unsigned-integer-overflow, though + // strictly speaking this is legal and well-defined for unsigned integers. + return Memory::Stats::totalCurrentlyAllocated() - memory_at_construction_; + } + +private: + const size_t memory_at_construction_; +}; + +// Compares the memory consumed against an exact expected value, but only on +// canonical platforms, or when the expected value is zero. Canonical platforms +// currently include only for 'release' tests in ci. On other platforms an info +// log is emitted, indicating that the test is being skipped. +#define EXPECT_MEMORY_EQ(consumed_bytes, expected_value) \ + do { \ + if (expected_value == 0 || \ + Stats::TestUtil::MemoryTest::mode() == Stats::TestUtil::MemoryTest::Mode::Canonical) { \ + EXPECT_EQ(consumed_bytes, expected_value); \ + } else { \ + ENVOY_LOG_MISC(info, \ + "Skipping exact memory test of actual={} versus expected={} " \ + "bytes as platform is non-canonical", \ + consumed_bytes, expected_value); \ + } \ + } while (false) + +// Compares the memory consumed against an expected upper bound, but only +// on platforms where memory consumption can be measured via API. This is +// currently enabled only for builds with TCMALLOC. On other platforms, an info +// log is emitted, indicating that the test is being skipped. +#define EXPECT_MEMORY_LE(consumed_bytes, upper_bound) \ + do { \ + if (Stats::TestUtil::MemoryTest::mode() != Stats::TestUtil::MemoryTest::Mode::Disabled) { \ + EXPECT_LE(consumed_bytes, upper_bound); \ + EXPECT_GT(consumed_bytes, 0); \ + } else { \ + ENVOY_LOG_MISC( \ + info, "Skipping upper-bound memory test against {} bytes as platform lacks tcmalloc", \ + upper_bound); \ + } \ + } while (false) + } // namespace TestUtil } // namespace Stats } // namespace Envoy diff --git a/test/common/stats/symbol_table_impl_test.cc b/test/common/stats/symbol_table_impl_test.cc index 7196797a82ba..374ab5b80195 100644 --- a/test/common/stats/symbol_table_impl_test.cc +++ b/test/common/stats/symbol_table_impl_test.cc @@ -542,19 +542,11 @@ TEST_P(StatNameTest, SharedStatNameStorageSetSwap) { // 2M. Note that only SymbolTableImpl is tested for memory consumption, // and not FakeSymbolTableImpl. TEST(SymbolTableTest, Memory) { - if (!TestUtil::hasDeterministicMallocStats()) { - return; - } - // Tests a stat-name allocation strategy. auto test_memory_usage = [](std::function fn) -> size_t { - const size_t start_mem = Memory::Stats::totalCurrentlyAllocated(); + TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat(1000, fn); - const size_t end_mem = Memory::Stats::totalCurrentlyAllocated(); - if (end_mem != 0) { // See warning below for asan, tsan, and mac. - EXPECT_GT(end_mem, start_mem); - } - return end_mem - start_mem; + return memory_test.consumedBytes(); }; size_t string_mem_used, symbol_table_mem_used; @@ -575,20 +567,13 @@ TEST(SymbolTableTest, Memory) { } } - // This test only works if Memory::Stats::totalCurrentlyAllocated() works, which - // appears not to be the case in some tests, including asan, tsan, and mac. - if (Memory::Stats::totalCurrentlyAllocated() == 0) { - ENVOY_LOG_MISC(info, - "SymbolTableTest.Memory comparison skipped due to malloc-stats returning 0."); - } else { - // Make sure we don't regress. Data as of 2019/05/29: - // - // string_mem_used: 6710912 (libc++), 7759488 (libstdc++). - // symbol_table_mem_used: 1726056 (3.9x) -- does not seem to depend on STL sizes. - EXPECT_LE(string_mem_used, 7759488); - EXPECT_LT(symbol_table_mem_used, string_mem_used / 3); - EXPECT_EQ(symbol_table_mem_used, 1726056); - } + // Make sure we don't regress. Data as of 2019/05/29: + // + // string_mem_used: 6710912 (libc++), 7759488 (libstdc++). + // symbol_table_mem_used: 1726056 (3.9x) -- does not seem to depend on STL sizes. + EXPECT_MEMORY_LE(string_mem_used, 7759488); + EXPECT_MEMORY_LE(symbol_table_mem_used, string_mem_used / 3); + EXPECT_MEMORY_EQ(symbol_table_mem_used, 1726056); } } // namespace Stats diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 120e17e81e66..389bc4b80758 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -847,60 +847,43 @@ TEST_F(StatsThreadLocalStoreTest, NonHotRestartNoTruncation) { // Tests how much memory is consumed allocating 100k stats. TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTls) { - if (!TestUtil::hasDeterministicMallocStats()) { - return; - } - MockSink sink; Stats::FakeSymbolTableImpl symbol_table; HeapStatDataAllocator alloc(symbol_table); - auto store = std::make_unique(alloc); - store->addSink(sink); + ThreadLocalStoreImpl store(alloc); + store.addSink(sink); // Use a tag producer that will produce tags. envoy::config::metrics::v2::StatsConfig stats_config; - store->setTagProducer(std::make_unique(stats_config)); + store.setTagProducer(std::make_unique(stats_config)); - const size_t start_mem = Memory::Stats::totalCurrentlyAllocated(); - if (start_mem == 0) { - // Skip this test for platforms where we can't measure memory. - return; - } + TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat( - 1000, [&store](absl::string_view name) { store->counter(std::string(name)); }); - const size_t end_mem = Memory::Stats::totalCurrentlyAllocated(); - EXPECT_LT(start_mem, end_mem); + 1000, [&store](absl::string_view name) { store.counter(std::string(name)); }); const size_t million = 1000 * 1000; - EXPECT_LT(end_mem - start_mem, 20 * million); // actual value: 19601552 as of March 14, 2019 + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 19602128); // June 13, 2019 + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 20 * million); } TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithTls) { - if (!TestUtil::hasDeterministicMallocStats()) { - return; - } Stats::FakeSymbolTableImpl symbol_table; HeapStatDataAllocator alloc(symbol_table); - auto store = std::make_unique(alloc); + NiceMock main_thread_dispatcher; + NiceMock tls; + ThreadLocalStoreImpl store(alloc); // Use a tag producer that will produce tags. envoy::config::metrics::v2::StatsConfig stats_config; - store->setTagProducer(std::make_unique(stats_config)); + store.setTagProducer(std::make_unique(stats_config)); - NiceMock main_thread_dispatcher; - NiceMock tls; - store->initializeThreading(main_thread_dispatcher, tls); - const size_t start_mem = Memory::Stats::totalCurrentlyAllocated(); - if (start_mem == 0) { - // Skip this test for platforms where we can't measure memory. - return; - } + store.initializeThreading(main_thread_dispatcher, tls); + TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat( - 1000, [&store](absl::string_view name) { store->counter(std::string(name)); }); - const size_t end_mem = Memory::Stats::totalCurrentlyAllocated(); - EXPECT_LT(start_mem, end_mem); + 1000, [&store](absl::string_view name) { store.counter(std::string(name)); }); const size_t million = 1000 * 1000; - EXPECT_LT(end_mem - start_mem, 23 * million); // actual value: 22880912 as of March 14, 2019 - store->shutdownThreading(); + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 22879216); + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 23 * million); + store.shutdownThreading(); tls.shutdownThread(); } diff --git a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc index 618ff0cd1cf4..b77b6805c394 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc @@ -684,19 +684,17 @@ TEST_F(QuicPlatformTest, FailToPickUnsedPort) { } TEST_F(QuicPlatformTest, TestEnvoyQuicBufferAllocator) { - bool deterministic_stats = Envoy::Stats::TestUtil::hasDeterministicMallocStats(); - const size_t start_mem = Envoy::Memory::Stats::totalCurrentlyAllocated(); QuicStreamBufferAllocator allocator; - char* p = allocator.New(1024); - if (deterministic_stats) { - EXPECT_LT(start_mem, Envoy::Memory::Stats::totalCurrentlyAllocated()); + Envoy::Stats::TestUtil::MemoryTest memory_test; + if (memory_test.mode() == Envoy::Stats::TestUtil::MemoryTest::Mode::Disabled) { + return; } + char* p = allocator.New(1024); EXPECT_NE(nullptr, p); + EXPECT_GT(memory_test.consumedBytes(), 0); memset(p, 'a', 1024); allocator.Delete(p); - if (deterministic_stats) { - EXPECT_EQ(start_mem, Envoy::Memory::Stats::totalCurrentlyAllocated()); - } + EXPECT_EQ(memory_test.consumedBytes(), 0); } TEST_F(QuicPlatformTest, TestSystemEventLoop) { diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index d5076ffb743f..5195c2b0b950 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -145,13 +145,19 @@ class ClusterMemoryTestHelper : public BaseIntegrationTest { ClusterMemoryTestHelper() : BaseIntegrationTest(testing::TestWithParam::GetParam()) {} + static size_t computeMemory(int num_clusters) { + ClusterMemoryTestHelper helper; + return helper.clusterMemoryHelper(num_clusters, true); + } + +private: /** - * * @param num_clusters number of clusters appended to bootstrap_config * @param allow_stats if false, enable set_reject_all in stats_config * @return size_t the total memory allocated */ - size_t ClusterMemoryHelper(int num_clusters, bool allow_stats) { + size_t clusterMemoryHelper(int num_clusters, bool allow_stats) { + Stats::TestUtil::MemoryTest memory_test; config_helper_.addConfigModifier([&](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { if (!allow_stats) { bootstrap.mutable_stats_config()->mutable_stats_matcher()->set_reject_all(true); @@ -163,15 +169,7 @@ class ClusterMemoryTestHelper : public BaseIntegrationTest { }); initialize(); - return Memory::Stats::totalCurrentlyAllocated(); - } - - static size_t computeMemory(int num_clusters) { - const size_t start_mem = Memory::Stats::totalCurrentlyAllocated(); - ClusterMemoryTestHelper helper; - size_t memory = helper.ClusterMemoryHelper(num_clusters, true); - EXPECT_LT(start_mem, memory); - return memory; + return memory_test.consumedBytes(); } }; class ClusterMemoryTestRunner : public testing::TestWithParam {}; @@ -181,11 +179,6 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ClusterMemoryTestRunner, TestUtility::ipTestParamsToString); TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { - // Skip test if we cannot measure memory with TCMALLOC - if (!Stats::TestUtil::hasDeterministicMallocStats()) { - return; - } - const size_t start_mem = Memory::Stats::totalCurrentlyAllocated(); // A unique instance of ClusterMemoryTest allows for multiple runs of Envoy with // differing configuration. This is necessary for measuring the memory consumption // between the different instances within the same test. @@ -193,9 +186,6 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { const size_t m1001 = ClusterMemoryTestHelper::computeMemory(1001); const size_t m_per_cluster = (m1001 - m1) / 1000; - EXPECT_LT(start_mem, m1); - EXPECT_LT(start_mem, m1001); - // Note: if you are increasing this golden value because you are adding a // stat, please confirm that this will be generally useful to most Envoy // users. Otherwise you are adding to the per-cluster memory overhead, which @@ -205,6 +195,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // History of golden values: // // Date PR Bytes Per Cluster Notes + // exact upper-bound // ---------- ----- ----------------- ----- // 2019/03/20 6329 59015 Initial version // 2019/04/12 6477 59576 Implementing Endpoint lease... @@ -215,8 +206,10 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // 2019/05/31 6866 50157 libstdc++ upgrade in CI // 2019/06/03 7199 49393 absl update // 2019/06/06 7208 49650 make memory targets approximate + // 2019/06/17 7243 49412 49700 macros for exact/upper-bound memory checks - EXPECT_LE(m_per_cluster, 49650); + EXPECT_MEMORY_EQ(m_per_cluster, 49412); + EXPECT_MEMORY_LE(m_per_cluster, 49700); } } // namespace diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index ced4cb24a3d7..b18242af6237 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -355,6 +355,7 @@ casted chrono chroot chunked +ci circllhist cmd codec From 154d7a0fc31d279a07f10f33ccdabb136ea25413 Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Tue, 18 Jun 2019 19:15:30 -0400 Subject: [PATCH 026/542] TapDS API (#7287) Signed-off-by: Yuval Kohavi --- api/envoy/config/common/tap/v2alpha/BUILD | 1 + .../config/common/tap/v2alpha/common.proto | 16 ++++++ api/envoy/data/tap/v2alpha/BUILD | 2 + api/envoy/service/tap/v2alpha/BUILD | 25 ++++++++++ api/envoy/service/tap/v2alpha/common.proto | 27 ++++++++++ api/envoy/service/tap/v2alpha/tap.proto | 50 +++++++++++++++++++ api/envoy/service/tap/v2alpha/tapds.proto | 43 ++++++++++++++++ 7 files changed, 164 insertions(+) create mode 100644 api/envoy/service/tap/v2alpha/tap.proto create mode 100644 api/envoy/service/tap/v2alpha/tapds.proto diff --git a/api/envoy/config/common/tap/v2alpha/BUILD b/api/envoy/config/common/tap/v2alpha/BUILD index 4b780575154e..863ba519d128 100644 --- a/api/envoy/config/common/tap/v2alpha/BUILD +++ b/api/envoy/config/common/tap/v2alpha/BUILD @@ -7,6 +7,7 @@ api_proto_library_internal( srcs = ["common.proto"], visibility = ["//visibility:public"], deps = [ + "//envoy/api/v2/core:config_source", "//envoy/service/tap/v2alpha:common", ], ) diff --git a/api/envoy/config/common/tap/v2alpha/common.proto b/api/envoy/config/common/tap/v2alpha/common.proto index a52016d81a6f..b8180f3cdd98 100644 --- a/api/envoy/config/common/tap/v2alpha/common.proto +++ b/api/envoy/config/common/tap/v2alpha/common.proto @@ -1,8 +1,10 @@ syntax = "proto3"; import "envoy/service/tap/v2alpha/common.proto"; +import "envoy/api/v2/core/config_source.proto"; import "validate/validate.proto"; +import "gogoproto/gogo.proto"; package envoy.config.common.tap.v2alpha; @@ -14,6 +16,17 @@ option java_package = "io.envoyproxy.envoy.config.common.tap.v2alpha"; // Common configuration for all tap extensions. message CommonExtensionConfig { + + // [#not-implemented-hide:] + message TapDSConfig { + // Configuration for the source of TapDS updates for this Cluster. + envoy.api.v2.core.ConfigSource config_source = 1 + [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + + // Tap config to request from XDS server. + string name = 2 [(validate.rules).string.min_bytes = 1]; + } + oneof config_type { option (validate.required) = true; @@ -23,6 +36,9 @@ message CommonExtensionConfig { // If specified, the tap filter will be configured via a static configuration that cannot be // changed. service.tap.v2alpha.TapConfig static_config = 2; + + // [#not-implemented-hide:] Configuration to use for TapDS updates for the filter. + TapDSConfig tapds_config = 3; } } diff --git a/api/envoy/data/tap/v2alpha/BUILD b/api/envoy/data/tap/v2alpha/BUILD index 90cd317006d5..1b373eee86df 100644 --- a/api/envoy/data/tap/v2alpha/BUILD +++ b/api/envoy/data/tap/v2alpha/BUILD @@ -28,8 +28,10 @@ api_proto_library_internal( api_proto_library_internal( name = "wrapper", srcs = ["wrapper.proto"], + visibility = ["//visibility:public"], deps = [ ":http", ":transport", + "//envoy/api/v2/core:address", ], ) diff --git a/api/envoy/service/tap/v2alpha/BUILD b/api/envoy/service/tap/v2alpha/BUILD index e2e67d5d7d78..6651b9c5e10d 100644 --- a/api/envoy/service/tap/v2alpha/BUILD +++ b/api/envoy/service/tap/v2alpha/BUILD @@ -7,6 +7,31 @@ api_proto_library_internal( srcs = ["common.proto"], visibility = ["//visibility:public"], deps = [ + "//envoy/api/v2/core:base", + "//envoy/api/v2/core:grpc_service", "//envoy/api/v2/route", + "//envoy/type/matcher:string", + ], +) + +api_proto_library_internal( + name = "tap", + srcs = ["tap.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v2:discovery", + "//envoy/api/v2/core:base", + "//envoy/data/tap/v2alpha:wrapper", + ], +) + +api_proto_library_internal( + name = "tapds", + srcs = ["tapds.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v2:discovery", + "//envoy/api/v2/core:base", + "//envoy/service/tap/v2alpha:common", ], ) diff --git a/api/envoy/service/tap/v2alpha/common.proto b/api/envoy/service/tap/v2alpha/common.proto index c078e2a93a6a..3c7bf498f42a 100644 --- a/api/envoy/service/tap/v2alpha/common.proto +++ b/api/envoy/service/tap/v2alpha/common.proto @@ -1,6 +1,9 @@ syntax = "proto3"; import "envoy/api/v2/route/route.proto"; +import "envoy/api/v2/core/base.proto"; +import "envoy/api/v2/core/grpc_service.proto"; +import "envoy/type/matcher/string.proto"; import "google/protobuf/wrappers.proto"; @@ -24,6 +27,16 @@ message TapConfig { // a tap will occur and the data will be written to the configured output. OutputConfig output_config = 2 [(validate.rules).message.required = true]; + // [#not-implemented-hide:] Specify if Tap matching is enabled. The % of requests\connections for + // which the tap matching is enabled. When not enabled, the request\connection will not be + // recorded. + // + // .. note:: + // + // This field defaults to 100/:ref:`HUNDRED + // `. + envoy.api.v2.core.RuntimeFractionalPercent tap_enabled = 3; + // [#comment:TODO(mattklein123): Rate limiting] } @@ -158,6 +171,10 @@ message OutputSink { // Tap output will be written to a file per tap sink. FilePerTapSink file_per_tap = 3; + + // [#not-implemented-hide:] + // GrpcService to stream data to. The format argument must be PROTO_BINARY. + StreamingGrpcSink streaming_grpc = 4; } } @@ -172,3 +189,13 @@ message FilePerTapSink { // connection ID, HTTP stream ID, etc.). string path_prefix = 1 [(validate.rules).string.min_bytes = 1]; } + +// [#not-implemented-hide:] Streaming gRPC sink configuration sends the taps to an external gRPC +// server. +message StreamingGrpcSink { + // Opaque identifier, that will be sent back to the streaming grpc server. + string tap_id = 1; + + // The gRPC server that hosts the Tap Sink Service. + envoy.api.v2.core.GrpcService grpc_service = 2 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/service/tap/v2alpha/tap.proto b/api/envoy/service/tap/v2alpha/tap.proto new file mode 100644 index 000000000000..c4cc919c275c --- /dev/null +++ b/api/envoy/service/tap/v2alpha/tap.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +import "envoy/api/v2/core/base.proto"; +import "envoy/data/tap/v2alpha/wrapper.proto"; + +package envoy.service.tap.v2alpha; + +import "validate/validate.proto"; + +option java_outer_classname = "CommonProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.tap.v2alpha"; + +// [#protodoc-title: Tap Sink Service] + +// [#not-implemented-hide:] Stream message for the Tap API. Envoy will open a stream to the server +// and stream taps without ever expecting a response. +message StreamTapsRequest { + message Identifier { + // The node sending taps over the stream. + envoy.api.v2.core.Node node = 1 [(validate.rules).message.required = true]; + // The opaque identifier that was set in the :ref:`output config + // `. + string tap_id = 2; + } + + // Identifier data effectively is a structured metadata. As a performance optimization this will + // only be sent in the first message on the stream. + Identifier identifier = 1; + // The trace id. this can be used to merge together a streaming trace. Note that the trace_id + // is not guaranteed to be spatially or temporally unique. + uint64 trace_id = 2; + // The trace data. + envoy.data.tap.v2alpha.TraceWrapper trace = 3; +} + +// [#not-implemented-hide:] +message StreamTapsResponse { +} + +// [#not-implemented-hide:] A tap service to receive incoming taps. Envoy will call +// StreamTaps to deliver captured taps to the server +service TapSinkService { + + // Envoy will connect and send StreamTapsRequest messages forever. It does not expect any + // response to be sent as nothing would be done in the case of failure. The server should + // disconnect if it expects Envoy to reconnect. + rpc StreamTaps(stream StreamTapsRequest) returns (StreamTapsResponse) { + } +} \ No newline at end of file diff --git a/api/envoy/service/tap/v2alpha/tapds.proto b/api/envoy/service/tap/v2alpha/tapds.proto new file mode 100644 index 000000000000..a7aea7de9e7c --- /dev/null +++ b/api/envoy/service/tap/v2alpha/tapds.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; + +import "envoy/api/v2/discovery.proto"; +import "envoy/service/tap/v2alpha/common.proto"; +import "validate/validate.proto"; + +package envoy.service.tap.v2alpha; + +import "google/api/annotations.proto"; + +option java_outer_classname = "CommonProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.tap.v2alpha"; + +// [#protodoc-title: Tap discovery service] + +// [#not-implemented-hide:] Tap discovery service. +service TapDiscoveryService { + rpc StreamTapConfigs(stream envoy.api.v2.DiscoveryRequest) + returns (stream envoy.api.v2.DiscoveryResponse) { + } + + rpc DeltaTapConfigs(stream envoy.api.v2.DeltaDiscoveryRequest) + returns (stream envoy.api.v2.DeltaDiscoveryResponse) { + } + + rpc FetchTapConfigs(envoy.api.v2.DiscoveryRequest) returns (envoy.api.v2.DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:tap_configs" + body: "*" + }; + } +} + +// [#not-implemented-hide:] A tap resource is essentially a tap configuration with a name +// The filter TapDS config references this name. +message TapResource { + // The name of the tap configuration. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Tap config to apply + TapConfig config = 2; +} \ No newline at end of file From a4953ec7dc2c1658f38f756e325c40ea9457be6d Mon Sep 17 00:00:00 2001 From: Derek Date: Tue, 18 Jun 2019 16:15:58 -0700 Subject: [PATCH 027/542] csrf filter: add support for additional source origins (#7297) Signed-off-by: Derek Schaller --- api/envoy/config/filter/http/csrf/v2/BUILD | 5 ++- .../config/filter/http/csrf/v2/csrf.proto | 10 ++++- .../http_filters/csrf_filter.rst | 26 ++++++++++- docs/root/intro/version_history.rst | 1 + examples/csrf/index.html | 1 + examples/csrf/samesite/front-envoy.yaml | 12 +++++ source/extensions/filters/http/csrf/BUILD | 1 + .../filters/http/csrf/csrf_filter.cc | 18 +++++++- .../filters/http/csrf/csrf_filter.h | 12 ++++- .../filters/http/csrf/csrf_filter_test.cc | 44 +++++++++++++++++++ 10 files changed, 124 insertions(+), 6 deletions(-) diff --git a/api/envoy/config/filter/http/csrf/v2/BUILD b/api/envoy/config/filter/http/csrf/v2/BUILD index b236868c2cc0..0d58b1ef6d43 100644 --- a/api/envoy/config/filter/http/csrf/v2/BUILD +++ b/api/envoy/config/filter/http/csrf/v2/BUILD @@ -5,5 +5,8 @@ licenses(["notice"]) # Apache 2 api_proto_library_internal( name = "csrf", srcs = ["csrf.proto"], - deps = ["//envoy/api/v2/core:base"], + deps = [ + "//envoy/api/v2/core:base", + "//envoy/type/matcher:string", + ], ) diff --git a/api/envoy/config/filter/http/csrf/v2/csrf.proto b/api/envoy/config/filter/http/csrf/v2/csrf.proto index eed59de5edd1..525ed118a71b 100644 --- a/api/envoy/config/filter/http/csrf/v2/csrf.proto +++ b/api/envoy/config/filter/http/csrf/v2/csrf.proto @@ -8,6 +8,7 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.csrf.v2"; option go_package = "v2"; import "envoy/api/v2/core/base.proto"; +import "envoy/type/matcher/string.proto"; import "validate/validate.proto"; import "gogoproto/gogo.proto"; @@ -17,7 +18,7 @@ import "gogoproto/gogo.proto"; // CSRF filter config. message CsrfPolicy { - // Specify if CSRF is enabled. + // Specifies if CSRF is enabled. // // More information on how this can be controlled via runtime can be found // :ref:`here `. @@ -40,4 +41,11 @@ message CsrfPolicy { // This field defaults to 100/:ref:`HUNDRED // `. envoy.api.v2.core.RuntimeFractionalPercent shadow_enabled = 2; + + // Specifies additional source origins that will be allowed in addition to + // the destination origin. + // + // More information on how this can be configured via runtime can be found + // :ref:`here `. + repeated envoy.type.matcher.StringMatcher additional_origins = 3; } diff --git a/docs/root/configuration/http_filters/csrf_filter.rst b/docs/root/configuration/http_filters/csrf_filter.rst index ebb4ac2fc8f8..66470be85881 100644 --- a/docs/root/configuration/http_filters/csrf_filter.rst +++ b/docs/root/configuration/http_filters/csrf_filter.rst @@ -26,7 +26,8 @@ a request originated from the same host. When the filter is evaluating a request, it ensures both pieces of information are present and compares their values. If the source origin is missing or the origins do not match -the request is rejected. +the request is rejected. The exception to this being if the source origin has been +added to the policy as valid. .. note:: Due to differing functionality between browsers this filter will determine @@ -44,6 +45,29 @@ For more information on CSRF please refer to the pages below. This filter should be configured with the name *envoy.csrf*. +.. _csrf-configuration: + +Configuration +------------- + +The CSRF filter supports the ability to extend the source origins it will consider +valid. The reason it is able to do this while still mitigating cross-site request +forgery attempts is because the target origin has already been reached by the time +front-envoy is applying the filter. This means that while endpoints may support +cross-origin requests they are still protected from malicious third-parties who +have not been whitelisted. + +It's important to note that requests should generally originate from the same +origin as the target but there are use cases where that may not be possible. +For example, if you are hosting a static site on a third-party vendor but need +to make requests for tracking purposes. + +.. warning:: + + Additional origins can be either an exact string, regex pattern, prefix string, + or suffix string. It's advised to be cautious when adding regex, prefix, or suffix + origins since an ambiguous origin can pose a security vulnerability. + .. _csrf-runtime: Runtime diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ca2608b014f6..28b229278305 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -13,6 +13,7 @@ Version history * api: track and report requests issued since last load report. * build: releases are built with Clang and linked with LLD. * control-plane: management servers can respond with HTTP 304 to indicate that config is up to date for Envoy proxies polling a :ref:`REST API Config Type ` +* csrf: added support for whitelisting additional source origins. * dubbo_proxy: support the :ref:`Dubbo proxy filter `. * eds: added support to specify max time for which endpoints can be used :ref:`gRPC filter `. * event: added :ref:`loop duration and poll delay statistics `. diff --git a/examples/csrf/index.html b/examples/csrf/index.html index 386c52feae80..e39a5cb61659 100644 --- a/examples/csrf/index.html +++ b/examples/csrf/index.html @@ -56,6 +56,7 @@
CSRF Enforcement
Shadow Mode
Enabled
Ignored
+ Additional Origin

diff --git a/examples/csrf/samesite/front-envoy.yaml b/examples/csrf/samesite/front-envoy.yaml index 8f7cfbb38392..ed82deeb2236 100644 --- a/examples/csrf/samesite/front-envoy.yaml +++ b/examples/csrf/samesite/front-envoy.yaml @@ -66,6 +66,18 @@ static_resources: default_value: numerator: 100 denominator: HUNDRED + - match: + prefix: "/csrf/additional_origin" + route: + cluster: generic_service + per_filter_config: + envoy.csrf: + filter_enabled: + default_value: + numerator: 100 + denominator: HUNDRED + additional_origins: + - regex: .* - match: prefix: "/" route: diff --git a/source/extensions/filters/http/csrf/BUILD b/source/extensions/filters/http/csrf/BUILD index b9b6fd26b007..f33294cea8e7 100644 --- a/source/extensions/filters/http/csrf/BUILD +++ b/source/extensions/filters/http/csrf/BUILD @@ -18,6 +18,7 @@ envoy_cc_library( deps = [ "//include/envoy/http:filter_interface", "//source/common/buffer:buffer_lib", + "//source/common/common:matchers_lib", "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", diff --git a/source/extensions/filters/http/csrf/csrf_filter.cc b/source/extensions/filters/http/csrf/csrf_filter.cc index f8065b6d58ed..85edec966ce3 100644 --- a/source/extensions/filters/http/csrf/csrf_filter.cc +++ b/source/extensions/filters/http/csrf/csrf_filter.cc @@ -91,8 +91,7 @@ Http::FilterHeadersStatus CsrfFilter::decodeHeaders(Http::HeaderMap& headers, bo config_->stats().missing_source_origin_.inc(); } - const absl::string_view target_origin = targetOriginValue(headers); - if (source_origin != target_origin) { + if (!isValid(source_origin, headers)) { is_valid = false; config_->stats().request_invalid_.inc(); } @@ -122,6 +121,21 @@ void CsrfFilter::determinePolicy() { } } +bool CsrfFilter::isValid(const absl::string_view source_origin, Http::HeaderMap& headers) { + const absl::string_view target_origin = targetOriginValue(headers); + if (source_origin == target_origin) { + return true; + } + + for (const auto& additional_origin : policy_->additional_origins()) { + if (additional_origin.match(source_origin)) { + return true; + } + } + + return false; +} + } // namespace Csrf } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/csrf/csrf_filter.h b/source/extensions/filters/http/csrf/csrf_filter.h index 4ee3af9d1fa7..da62a7800873 100644 --- a/source/extensions/filters/http/csrf/csrf_filter.h +++ b/source/extensions/filters/http/csrf/csrf_filter.h @@ -7,6 +7,7 @@ #include "envoy/stats/stats_macros.h" #include "common/buffer/buffer_impl.h" +#include "common/common/matchers.h" namespace Envoy { namespace Extensions { @@ -36,7 +37,11 @@ struct CsrfStats { class CsrfPolicy : public Router::RouteSpecificFilterConfig { public: CsrfPolicy(const envoy::config::filter::http::csrf::v2::CsrfPolicy& policy, - Runtime::Loader& runtime) : policy_(policy), runtime_(runtime) {} + Runtime::Loader& runtime) : policy_(policy), runtime_(runtime) { + for (const auto& additional_origin : policy.additional_origins()) { + additional_origins_.emplace_back(Matchers::StringMatcher(additional_origin)); + } + } bool enabled() const { const envoy::api::v2::core::RuntimeFractionalPercent& filter_enabled = policy_.filter_enabled(); @@ -53,9 +58,13 @@ class CsrfPolicy : public Router::RouteSpecificFilterConfig { shadow_enabled.default_value()); } + const std::vector& additional_origins() const { return additional_origins_; }; + private: const envoy::config::filter::http::csrf::v2::CsrfPolicy policy_; + std::vector additional_origins_; Runtime::Loader& runtime_; + }; /** @@ -97,6 +106,7 @@ class CsrfFilter : public Http::StreamDecoderFilter { private: void determinePolicy(); + bool isValid(const absl::string_view source_origin, Http::HeaderMap& headers); Http::StreamDecoderFilterCallbacks* callbacks_{}; CsrfFilterConfigSharedPtr config_; diff --git a/test/extensions/filters/http/csrf/csrf_filter_test.cc b/test/extensions/filters/http/csrf/csrf_filter_test.cc index bed3a7ae064a..55def661cb19 100644 --- a/test/extensions/filters/http/csrf/csrf_filter_test.cc +++ b/test/extensions/filters/http/csrf/csrf_filter_test.cc @@ -39,6 +39,13 @@ class CsrfFilterTest : public testing::Test { shadow_enabled->mutable_default_value()->set_denominator( envoy::type::FractionalPercent::HUNDRED); shadow_enabled->set_runtime_key("csrf.shadow_enabled"); + + const auto& add_exact_origin = policy.mutable_additional_origins()->Add(); + add_exact_origin->set_exact("additionalhost"); + + const auto& add_regex_origin = policy.mutable_additional_origins()->Add(); + add_regex_origin->set_regex(R"(www\-[0-9]\.allow\.com)"); + return std::make_shared(policy, "test", stats_, runtime_); } @@ -331,6 +338,43 @@ TEST_F(CsrfFilterTest, NoVHostCsrfEntry) { EXPECT_EQ(1U, config_->stats().request_invalid_.value()); EXPECT_EQ(0U, config_->stats().request_valid_.value()); } + +TEST_F(CsrfFilterTest, RequestFromAdditionalExactOrigin) { + Http::TestHeaderMapImpl request_headers{{":method", "PUT"}, {"origin", "additionalhost"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(request_headers_)); + + EXPECT_EQ(0U, config_->stats().missing_source_origin_.value()); + EXPECT_EQ(0U, config_->stats().request_invalid_.value()); + EXPECT_EQ(1U, config_->stats().request_valid_.value()); +} + +TEST_F(CsrfFilterTest, RequestFromAdditionalRegexOrigin) { + Http::TestHeaderMapImpl request_headers{{":method", "PUT"}, {"origin", "www-1.allow.com"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(request_headers_)); + + EXPECT_EQ(0U, config_->stats().missing_source_origin_.value()); + EXPECT_EQ(0U, config_->stats().request_invalid_.value()); + EXPECT_EQ(1U, config_->stats().request_valid_.value()); +} + +TEST_F(CsrfFilterTest, RequestFromInvalidAdditionalRegexOrigin) { + Http::TestHeaderMapImpl request_headers{{":method", "PUT"}, {"origin", "www.allow.com"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_.decodeHeaders(request_headers, false)); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(request_headers_)); + + EXPECT_EQ(0U, config_->stats().missing_source_origin_.value()); + EXPECT_EQ(1U, config_->stats().request_invalid_.value()); + EXPECT_EQ(0U, config_->stats().request_valid_.value()); +} } // namespace Csrf } // namespace HttpFilters } // namespace Extensions From 3290085f92706ba23205e26d6354a87d421cfd33 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 18 Jun 2019 19:16:26 -0400 Subject: [PATCH 028/542] stats: Simplify stats structures, part 1 (#7295) Signed-off-by: Joshua Marantz --- source/common/stats/BUILD | 35 ++- source/common/stats/heap_stat_data.cc | 152 ++++++++++- source/common/stats/heap_stat_data.h | 54 ++-- source/common/stats/isolated_store_impl.h | 1 + source/common/stats/null_counter.h | 33 +++ source/common/stats/null_gauge.h | 36 +++ .../common/stats/stat_data_allocator_impl.h | 235 ------------------ source/common/stats/thread_local_store.cc | 4 +- source/common/stats/thread_local_store.h | 6 +- test/common/config/grpc_mux_impl_test.cc | 6 +- test/common/stats/isolated_store_impl_test.cc | 2 + .../network/dubbo_proxy/conn_manager_test.cc | 2 +- 12 files changed, 275 insertions(+), 291 deletions(-) create mode 100644 source/common/stats/null_counter.h create mode 100644 source/common/stats/null_gauge.h delete mode 100644 source/common/stats/stat_data_allocator_impl.h diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index 2f3478652bca..16c0174eaab0 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -14,7 +14,6 @@ envoy_cc_library( hdrs = ["heap_stat_data.h"], deps = [ ":metric_impl_lib", - ":stat_data_allocator_lib", ":stat_merger_lib", "//source/common/common:assert_lib", "//source/common/common:hash_lib", @@ -46,6 +45,8 @@ envoy_cc_library( deps = [ ":fake_symbol_table_lib", ":histogram_lib", + ":null_counter_lib", + ":null_gauge_lib", ":scope_prefixer_lib", ":stats_lib", ":store_impl_lib", @@ -66,32 +67,42 @@ envoy_cc_library( ) envoy_cc_library( - name = "store_impl_lib", - hdrs = ["store_impl.h"], + name = "null_counter_lib", + hdrs = ["null_counter.h"], deps = [ + ":metric_impl_lib", ":symbol_table_lib", "//include/envoy/stats:stats_interface", ], ) envoy_cc_library( - name = "scope_prefixer_lib", - srcs = ["scope_prefixer.cc"], - hdrs = ["scope_prefixer.h"], + name = "null_gauge_lib", + hdrs = ["null_gauge.h"], deps = [ + ":metric_impl_lib", ":symbol_table_lib", - ":utility_lib", "//include/envoy/stats:stats_interface", ], ) envoy_cc_library( - name = "stat_data_allocator_lib", - hdrs = ["stat_data_allocator_impl.h"], + name = "store_impl_lib", + hdrs = ["store_impl.h"], deps = [ - ":metric_impl_lib", + ":symbol_table_lib", + "//include/envoy/stats:stats_interface", + ], +) + +envoy_cc_library( + name = "scope_prefixer_lib", + srcs = ["scope_prefixer.cc"], + hdrs = ["scope_prefixer.h"], + deps = [ + ":symbol_table_lib", + ":utility_lib", "//include/envoy/stats:stats_interface", - "//source/common/common:assert_lib", ], ) @@ -192,6 +203,8 @@ envoy_cc_library( hdrs = ["thread_local_store.h"], deps = [ ":heap_stat_data_lib", + ":null_counter_lib", + ":null_gauge_lib", ":scope_prefixer_lib", ":stats_lib", ":stats_matcher_lib", diff --git a/source/common/stats/heap_stat_data.cc b/source/common/stats/heap_stat_data.cc index 12df79dcff87..8a26ceb2dd07 100644 --- a/source/common/stats/heap_stat_data.cc +++ b/source/common/stats/heap_stat_data.cc @@ -1,9 +1,21 @@ #include "common/stats/heap_stat_data.h" +#include + +#include "envoy/stats/stats.h" +#include "envoy/stats/symbol_table.h" + +#include "common/common/hash.h" #include "common/common/lock_guard.h" #include "common/common/logger.h" #include "common/common/thread.h" +#include "common/common/thread_annotations.h" #include "common/common/utility.h" +#include "common/stats/metric_impl.h" +#include "common/stats/stat_merger.h" +#include "common/stats/symbol_table_impl.h" + +#include "absl/container/flat_hash_set.h" namespace Envoy { namespace Stats { @@ -61,7 +73,145 @@ void HeapStatDataAllocator::debugPrint() { } #endif -template class StatDataAllocatorImpl; +class CounterImpl : public Counter, public MetricImpl { +public: + CounterImpl(HeapStatData& data, HeapStatDataAllocator& alloc, + absl::string_view tag_extracted_name, const std::vector& tags) + : MetricImpl(tag_extracted_name, tags, alloc.symbolTable()), data_(data), alloc_(alloc) {} + + ~CounterImpl() override { + // MetricImpl must be explicitly cleared() before destruction, otherwise it + // will not be able to access the SymbolTable& to free the symbols. An RAII + // alternative would be to store the SymbolTable reference in the + // MetricImpl, costing 8 bytes per stat. + MetricImpl::clear(); + alloc_.free(data_); + } + + // Stats::Counter + void add(uint64_t amount) override { + data_.value_ += amount; + data_.pending_increment_ += amount; + data_.flags_ |= Flags::Used; + } + void inc() override { add(1); } + uint64_t latch() override { return data_.pending_increment_.exchange(0); } + void reset() override { data_.value_ = 0; } + bool used() const override { return data_.flags_ & Flags::Used; } + uint64_t value() const override { return data_.value_; } + + SymbolTable& symbolTable() override { return alloc_.symbolTable(); } + StatName statName() const override { return data_.statName(); } + +private: + HeapStatData& data_; + HeapStatDataAllocator& alloc_; +}; + +class GaugeImpl : public Gauge, public MetricImpl { +public: + GaugeImpl(HeapStatData& data, HeapStatDataAllocator& alloc, absl::string_view tag_extracted_name, + const std::vector& tags, ImportMode import_mode) + : MetricImpl(tag_extracted_name, tags, alloc.symbolTable()), data_(data), alloc_(alloc) { + switch (import_mode) { + case ImportMode::Accumulate: + data_.flags_ |= Flags::LogicAccumulate; + break; + case ImportMode::NeverImport: + data_.flags_ |= Flags::NeverImport; + break; + case ImportMode::Uninitialized: + // Note that we don't clear any flag bits for import_mode==Uninitialized, + // as we may have an established import_mode when this stat was created in + // an alternate scope. See + // https://github.com/envoyproxy/envoy/issues/7227. + break; + } + } + + ~GaugeImpl() override { + // MetricImpl must be explicitly cleared() before destruction, otherwise it + // will not be able to access the SymbolTable& to free the symbols. An RAII + // alternative would be to store the SymbolTable reference in the + // MetricImpl, costing 8 bytes per stat. + MetricImpl::clear(); + alloc_.free(data_); + } + + // Stats::Gauge + void add(uint64_t amount) override { + data_.value_ += amount; + data_.flags_ |= Flags::Used; + } + void dec() override { sub(1); } + void inc() override { add(1); } + void set(uint64_t value) override { + data_.value_ = value; + data_.flags_ |= Flags::Used; + } + void sub(uint64_t amount) override { + ASSERT(data_.value_ >= amount); + ASSERT(used() || amount == 0); + data_.value_ -= amount; + } + uint64_t value() const override { return data_.value_; } + bool used() const override { return data_.flags_ & Flags::Used; } + + ImportMode importMode() const override { + if (data_.flags_ & Flags::NeverImport) { + return ImportMode::NeverImport; + } else if (data_.flags_ & Flags::LogicAccumulate) { + return ImportMode::Accumulate; + } + return ImportMode::Uninitialized; + } + + void mergeImportMode(ImportMode import_mode) override { + ImportMode current = importMode(); + if (current == import_mode) { + return; + } + + switch (import_mode) { + case ImportMode::Uninitialized: + // mergeImportNode(ImportMode::Uninitialized) is called when merging an + // existing stat with importMode() == Accumulate or NeverImport. + break; + case ImportMode::Accumulate: + ASSERT(current == ImportMode::Uninitialized); + data_.flags_ |= Flags::LogicAccumulate; + break; + case ImportMode::NeverImport: + ASSERT(current == ImportMode::Uninitialized); + // A previous revision of Envoy may have transferred a gauge that it + // thought was Accumulate. But the new version thinks it's NeverImport, so + // we clear the accumulated value. + data_.value_ = 0; + data_.flags_ &= ~Flags::Used; + data_.flags_ |= Flags::NeverImport; + break; + } + } + + SymbolTable& symbolTable() override { return alloc_.symbolTable(); } + StatName statName() const override { return data_.statName(); } + +private: + HeapStatData& data_; + HeapStatDataAllocator& alloc_; +}; + +CounterSharedPtr HeapStatDataAllocator::makeCounter(StatName name, + absl::string_view tag_extracted_name, + const std::vector& tags) { + return std::make_shared(alloc(name), *this, tag_extracted_name, tags); +} + +GaugeSharedPtr HeapStatDataAllocator::makeGauge(StatName name, absl::string_view tag_extracted_name, + const std::vector& tags, + Gauge::ImportMode import_mode) { + return std::make_shared(alloc(name), *this, tag_extracted_name, tags, import_mode); +} } // namespace Stats } // namespace Envoy diff --git a/source/common/stats/heap_stat_data.h b/source/common/stats/heap_stat_data.h index b640ac6bdafd..9536d0e00849 100644 --- a/source/common/stats/heap_stat_data.h +++ b/source/common/stats/heap_stat_data.h @@ -1,28 +1,25 @@ +// TODO(jmarantz): rename this file and class to heap_allocator.h. + #pragma once -#include -#include -#include +#include +#include "envoy/stats/stat_data_allocator.h" #include "envoy/stats/stats.h" #include "envoy/stats/symbol_table.h" -#include "common/common/hash.h" -#include "common/common/thread.h" -#include "common/common/thread_annotations.h" #include "common/stats/metric_impl.h" -#include "common/stats/stat_data_allocator_impl.h" -#include "common/stats/stat_merger.h" -#include "common/stats/symbol_table_impl.h" #include "absl/container/flat_hash_set.h" +#include "absl/strings/string_view.h" namespace Envoy { namespace Stats { /** - * This structure is an alternate backing store for both CounterImpl and GaugeImpl. It is designed - * so that it can be allocated efficiently from the heap on demand. + * Holds backing store for both CounterImpl and GaugeImpl. This provides a level + * of indirection needed to enable stats created with the same name from + * different scopes to share the same value. */ struct HeapStatData : public InlineStorage { private: @@ -44,40 +41,21 @@ struct HeapStatData : public InlineStorage { SymbolTable::Storage symbol_storage_; // This is a 'using' nickname for uint8_t[]. }; -template class HeapStat : public Stat { -public: - HeapStat(HeapStatData& data, StatDataAllocatorImpl& alloc, - absl::string_view tag_extracted_name, const std::vector& tags) - : Stat(data, alloc, tag_extracted_name, tags) {} - - HeapStat(HeapStatData& data, StatDataAllocatorImpl& alloc, - absl::string_view tag_extracted_name, const std::vector& tags, - Gauge::ImportMode import_mode) - : Stat(data, alloc, tag_extracted_name, tags, import_mode) {} - - StatName statName() const override { return this->data_.statName(); } -}; - -class HeapStatDataAllocator : public StatDataAllocatorImpl { +class HeapStatDataAllocator : public StatDataAllocator { public: - HeapStatDataAllocator(SymbolTable& symbol_table) : StatDataAllocatorImpl(symbol_table) {} + HeapStatDataAllocator(SymbolTable& symbol_table) : symbol_table_(symbol_table) {} ~HeapStatDataAllocator() override; HeapStatData& alloc(StatName name); - void free(HeapStatData& data) override; + void free(HeapStatData& data); // StatDataAllocator CounterSharedPtr makeCounter(StatName name, absl::string_view tag_extracted_name, - const std::vector& tags) override { - return std::make_shared>>(alloc(name), *this, - tag_extracted_name, tags); - } - + const std::vector& tags) override; GaugeSharedPtr makeGauge(StatName name, absl::string_view tag_extracted_name, - const std::vector& tags, Gauge::ImportMode import_mode) override { - return std::make_shared>>( - alloc(name), *this, tag_extracted_name, tags, import_mode); - } + const std::vector& tags, Gauge::ImportMode import_mode) override; + SymbolTable& symbolTable() override { return symbol_table_; } + const SymbolTable& constSymbolTable() const override { return symbol_table_; } #ifndef ENVOY_CONFIG_COVERAGE void debugPrint(); @@ -97,6 +75,8 @@ class HeapStatDataAllocator : public StatDataAllocatorImpl { using StatSet = absl::flat_hash_set; StatSet stats_ GUARDED_BY(mutex_); + SymbolTable& symbol_table_; + // A mutex is needed here to protect both the stats_ object from both // alloc() and free() operations. Although alloc() operations are called under existing locking, // free() operations are made from the destructors of the individual stat objects, which are not diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index b71cab6a30bd..867946e4382c 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -10,6 +10,7 @@ #include "common/common/utility.h" #include "common/stats/heap_stat_data.h" +#include "common/stats/null_gauge.h" #include "common/stats/store_impl.h" #include "common/stats/symbol_table_impl.h" #include "common/stats/utility.h" diff --git a/source/common/stats/null_counter.h b/source/common/stats/null_counter.h new file mode 100644 index 000000000000..f0ca00163cc3 --- /dev/null +++ b/source/common/stats/null_counter.h @@ -0,0 +1,33 @@ +#pragma once + +#include "envoy/stats/stats.h" + +#include "common/stats/metric_impl.h" + +namespace Envoy { +namespace Stats { + +/** + * Null counter implementation. + * No-ops on all calls and requires no underlying metric or data. + */ +class NullCounterImpl : public Counter, NullMetricImpl { +public: + explicit NullCounterImpl(SymbolTable& symbol_table) : NullMetricImpl(symbol_table) {} + ~NullCounterImpl() override { + // MetricImpl must be explicitly cleared() before destruction, otherwise it + // will not be able to access the SymbolTable& to free the symbols. An RAII + // alternative would be to store the SymbolTable reference in the + // MetricImpl, costing 8 bytes per stat. + MetricImpl::clear(); + } + + void add(uint64_t) override {} + void inc() override {} + uint64_t latch() override { return 0; } + void reset() override {} + uint64_t value() const override { return 0; } +}; + +} // namespace Stats +} // namespace Envoy diff --git a/source/common/stats/null_gauge.h b/source/common/stats/null_gauge.h new file mode 100644 index 000000000000..568f0d70510e --- /dev/null +++ b/source/common/stats/null_gauge.h @@ -0,0 +1,36 @@ +#pragma once + +#include "envoy/stats/stats.h" + +#include "common/stats/metric_impl.h" + +namespace Envoy { +namespace Stats { + +/** + * Null gauge implementation. + * No-ops on all calls and requires no underlying metric or data. + */ +class NullGaugeImpl : public Gauge, NullMetricImpl { +public: + explicit NullGaugeImpl(SymbolTable& symbol_table) : NullMetricImpl(symbol_table) {} + ~NullGaugeImpl() override { + // MetricImpl must be explicitly cleared() before destruction, otherwise it + // will not be able to access the SymbolTable& to free the symbols. An RAII + // alternative would be to store the SymbolTable reference in the + // MetricImpl, costing 8 bytes per stat. + MetricImpl::clear(); + } + + void add(uint64_t) override {} + void inc() override {} + void dec() override {} + void set(uint64_t) override {} + void sub(uint64_t) override {} + uint64_t value() const override { return 0; } + ImportMode importMode() const override { return ImportMode::NeverImport; } + void mergeImportMode(ImportMode /* import_mode */) override {} +}; + +} // namespace Stats +} // namespace Envoy diff --git a/source/common/stats/stat_data_allocator_impl.h b/source/common/stats/stat_data_allocator_impl.h deleted file mode 100644 index 578d23e9c0b6..000000000000 --- a/source/common/stats/stat_data_allocator_impl.h +++ /dev/null @@ -1,235 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/stats/stat_data_allocator.h" -#include "envoy/stats/stats.h" -#include "envoy/stats/symbol_table.h" - -#include "common/common/assert.h" -#include "common/stats/metric_impl.h" - -#include "absl/strings/string_view.h" - -namespace Envoy { -namespace Stats { - -// Partially implements a StatDataAllocator, leaving alloc & free for subclasses. -// We templatize on StatData rather than defining a virtual base StatData class -// for performance reasons; stat increment is on the hot path. -// -// The two production derivations cover using a fixed block of shared-memory for -// hot restart stat continuity, and heap allocation for more efficient RAM usage -// for when hot-restart is not required. -// -// TODO(fredlas) the above paragraph is obsolete; it's now only heap. So, this -// interface can hopefully be collapsed down a bit. -template class StatDataAllocatorImpl : public StatDataAllocator { -public: - explicit StatDataAllocatorImpl(SymbolTable& symbol_table) : symbol_table_(symbol_table) {} - - /** - * Free a raw stat data block. The allocator should handle reference counting and only truly - * free the block if it is no longer needed. - * @param data the data returned by alloc(). - */ - virtual void free(StatData& data) PURE; - - SymbolTable& symbolTable() override { return symbol_table_; } - const SymbolTable& constSymbolTable() const override { return symbol_table_; } - -private: - // SymbolTable encodes stat names as back into strings. This does not - // get guarded by a mutex, since it has its own internal mutex to guarantee - // thread safety. - SymbolTable& symbol_table_; -}; - -/** - * Counter implementation that wraps a StatData. StatData must have data members: - * std::atomic value_; - * std::atomic pending_increment_; - * std::atomic flags_; - * std::atomic ref_count_; - */ -template class CounterImpl : public Counter, public MetricImpl { -public: - CounterImpl(StatData& data, StatDataAllocatorImpl& alloc, - absl::string_view tag_extracted_name, const std::vector& tags) - : MetricImpl(tag_extracted_name, tags, alloc.symbolTable()), data_(data), alloc_(alloc) {} - ~CounterImpl() override { - alloc_.free(data_); - - // MetricImpl must be explicitly cleared() before destruction, otherwise it - // will not be able to access the SymbolTable& to free the symbols. An RAII - // alternative would be to store the SymbolTable reference in the - // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); - } - - // Stats::Counter - void add(uint64_t amount) override { - data_.value_ += amount; - data_.pending_increment_ += amount; - data_.flags_ |= Flags::Used; - } - - void inc() override { add(1); } - uint64_t latch() override { return data_.pending_increment_.exchange(0); } - void reset() override { data_.value_ = 0; } - bool used() const override { return data_.flags_ & Flags::Used; } - uint64_t value() const override { return data_.value_; } - - SymbolTable& symbolTable() override { return alloc_.symbolTable(); } - -protected: - StatData& data_; - StatDataAllocatorImpl& alloc_; -}; - -/** - * Null counter implementation. - * No-ops on all calls and requires no underlying metric or data. - */ -class NullCounterImpl : public Counter, NullMetricImpl { -public: - explicit NullCounterImpl(SymbolTable& symbol_table) : NullMetricImpl(symbol_table) {} - ~NullCounterImpl() override { - // MetricImpl must be explicitly cleared() before destruction, otherwise it - // will not be able to access the SymbolTable& to free the symbols. An RAII - // alternative would be to store the SymbolTable reference in the - // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); - } - - void add(uint64_t) override {} - void inc() override {} - uint64_t latch() override { return 0; } - void reset() override {} - uint64_t value() const override { return 0; } -}; - -/** - * Gauge implementation that wraps a StatData. - */ -template class GaugeImpl : public Gauge, public MetricImpl { -public: - GaugeImpl(StatData& data, StatDataAllocatorImpl& alloc, - absl::string_view tag_extracted_name, const std::vector& tags, - ImportMode import_mode) - : MetricImpl(tag_extracted_name, tags, alloc.symbolTable()), data_(data), alloc_(alloc) { - switch (import_mode) { - case ImportMode::Accumulate: - data_.flags_ |= Flags::LogicAccumulate; - break; - case ImportMode::NeverImport: - data_.flags_ |= Flags::NeverImport; - break; - case ImportMode::Uninitialized: - // Note that we don't clear any flag bits for import_mode==Uninitialized, - // as we may have an established import_mode when this stat was created in - // an alternate scope. See - // https://github.com/envoyproxy/envoy/issues/7227. - break; - } - } - ~GaugeImpl() override { - alloc_.free(data_); - - // MetricImpl must be explicitly cleared() before destruction, otherwise it - // will not be able to access the SymbolTable& to free the symbols. An RAII - // alternative would be to store the SymbolTable reference in the - // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); - } - - // Stats::Gauge - void add(uint64_t amount) override { - data_.value_ += amount; - data_.flags_ |= Flags::Used; - } - void dec() override { sub(1); } - void inc() override { add(1); } - void set(uint64_t value) override { - data_.value_ = value; - data_.flags_ |= Flags::Used; - } - void sub(uint64_t amount) override { - ASSERT(data_.value_ >= amount); - ASSERT(used() || amount == 0); - data_.value_ -= amount; - } - uint64_t value() const override { return data_.value_; } - bool used() const override { return data_.flags_ & Flags::Used; } - - ImportMode importMode() const override { - if (data_.flags_ & Flags::NeverImport) { - return ImportMode::NeverImport; - } else if (data_.flags_ & Flags::LogicAccumulate) { - return ImportMode::Accumulate; - } - return ImportMode::Uninitialized; - } - - void mergeImportMode(ImportMode import_mode) override { - ImportMode current = importMode(); - if (current == import_mode) { - return; - } - - switch (import_mode) { - case ImportMode::Uninitialized: - // mergeImportNode(ImportMode::Uninitialized) is called when merging an - // existing stat with importMode() == Accumulate or NeverImport. - break; - case ImportMode::Accumulate: - ASSERT(current == ImportMode::Uninitialized); - data_.flags_ |= Flags::LogicAccumulate; - break; - case ImportMode::NeverImport: - ASSERT(current == ImportMode::Uninitialized); - // A previous revision of Envoy may have transferred a gauge that it - // thought was Accumulate. But the new version thinks it's NeverImport, so - // we clear the accumulated value. - data_.value_ = 0; - data_.flags_ &= ~Flags::Used; - data_.flags_ |= Flags::NeverImport; - break; - } - } - - SymbolTable& symbolTable() override { return alloc_.symbolTable(); } - -protected: - StatData& data_; - StatDataAllocatorImpl& alloc_; -}; - -/** - * Null gauge implementation. - * No-ops on all calls and requires no underlying metric or data. - */ -class NullGaugeImpl : public Gauge, NullMetricImpl { -public: - explicit NullGaugeImpl(SymbolTable& symbol_table) : NullMetricImpl(symbol_table) {} - ~NullGaugeImpl() override { - // MetricImpl must be explicitly cleared() before destruction, otherwise it - // will not be able to access the SymbolTable& to free the symbols. An RAII - // alternative would be to store the SymbolTable reference in the - // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); - } - - void add(uint64_t) override {} - void inc() override {} - void dec() override {} - void set(uint64_t) override {} - void sub(uint64_t) override {} - uint64_t value() const override { return 0; } - ImportMode importMode() const override { return ImportMode::NeverImport; } - void mergeImportMode(ImportMode /* import_mode */) override {} -}; - -} // namespace Stats -} // namespace Envoy diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 69b73a4cf58a..334f018666d9 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -569,7 +569,7 @@ ThreadLocalHistogramImpl::ThreadLocalHistogramImpl(StatName name, absl::string_view tag_extracted_name, const std::vector& tags, SymbolTable& symbol_table) - : MetricImpl(tag_extracted_name, tags, symbol_table), current_active_(0), flags_(0), + : MetricImpl(tag_extracted_name, tags, symbol_table), current_active_(0), used_(false), created_thread_id_(std::this_thread::get_id()), name_(name, symbol_table), symbol_table_(symbol_table) { histograms_[0] = hist_alloc(); @@ -586,7 +586,7 @@ ThreadLocalHistogramImpl::~ThreadLocalHistogramImpl() { void ThreadLocalHistogramImpl::recordValue(uint64_t value) { ASSERT(std::this_thread::get_id() == created_thread_id_); hist_insert_intscale(histograms_[current_active_], value, 0, 1); - flags_ |= Flags::Used; + used_ = true; } void ThreadLocalHistogramImpl::merge(histogram_t* target) { diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 7867a3bcd87c..97b19109602b 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -11,6 +11,8 @@ #include "common/common/hash.h" #include "common/stats/heap_stat_data.h" #include "common/stats/histogram_impl.h" +#include "common/stats/null_counter.h" +#include "common/stats/null_gauge.h" #include "common/stats/symbol_table_impl.h" #include "common/stats/utility.h" @@ -46,7 +48,7 @@ class ThreadLocalHistogramImpl : public Histogram, public MetricImpl { // Stats::Histogram void recordValue(uint64_t value) override; - bool used() const override { return flags_ & Flags::Used; } + bool used() const override { return used_; } // Stats::Metric StatName statName() const override { return name_.statName(); } @@ -56,7 +58,7 @@ class ThreadLocalHistogramImpl : public Histogram, public MetricImpl { uint64_t otherHistogramIndex() const { return 1 - current_active_; } uint64_t current_active_; histogram_t* histograms_[2]; - std::atomic flags_; + std::atomic used_; std::thread::id created_thread_id_; StatNameStorage name_; SymbolTable& symbol_table_; diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 0276aa7bd28b..7a515a37f7f1 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -539,14 +539,16 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(500))).Times(AtLeast(1)); onReceiveMessage(160); EXPECT_EQ(12, stats_.counter("control_plane.rate_limit_enforced").value()); - EXPECT_EQ(12, stats_.counter("control_plane.pending_requests").value()); + Stats::Gauge& pending_requests = + stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::Accumulate); + EXPECT_EQ(12, pending_requests.value()); // Validate that drain requests call when there are multiple requests in queue. time_system_.setMonotonicTime(std::chrono::seconds(10)); drain_timer_cb(); // Check that the pending_requests stat is updated with the queue drain. - EXPECT_EQ(0, stats_.counter("control_plane.pending_requests").value()); + EXPECT_EQ(0, pending_requests.value()); } // Verifies that a message with no resources is accepted. diff --git a/test/common/stats/isolated_store_impl_test.cc b/test/common/stats/isolated_store_impl_test.cc index 73e83407f6c0..a91f01f93c51 100644 --- a/test/common/stats/isolated_store_impl_test.cc +++ b/test/common/stats/isolated_store_impl_test.cc @@ -3,6 +3,8 @@ #include "envoy/stats/stats_macros.h" #include "common/stats/isolated_store_impl.h" +#include "common/stats/null_counter.h" +#include "common/stats/null_gauge.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index e497406eb804..833bf34c1ad2 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -933,7 +933,7 @@ TEST_F(ConnectionManagerTest, EmptyRequestData) { EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(0); EXPECT_EQ(filter_->onData(buffer_, true), Network::FilterStatus::StopIteration); - EXPECT_EQ(0U, store_.counter("test.request_active").value()); + EXPECT_EQ(0U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); } TEST_F(ConnectionManagerTest, StopHandleRequest) { From 07da3346bee84a7762052d9ce9f343a009ae994f Mon Sep 17 00:00:00 2001 From: htuch Date: Wed, 19 Jun 2019 00:16:56 +0100 Subject: [PATCH 029/542] rtds: Runtime Discovery Service implementation. (#7251) Fixes #6708. Risk level: Low Testing: Unit and integration tests added. Signed-off-by: Harvey Tuch --- api/docs/BUILD | 1 + api/envoy/config/bootstrap/v2/bootstrap.proto | 28 +- api/envoy/service/discovery/v2/BUILD | 8 +- .../discovery/v2/{tds.proto => rtds.proto} | 22 +- api/test/build/BUILD | 2 +- api/xds_protocol.rst | 1 + docs/build.sh | 1 + docs/root/api-v2/service/service.rst | 1 + docs/root/configuration/runtime.rst | 13 +- docs/root/intro/version_history.rst | 1 + include/envoy/runtime/runtime.h | 13 + source/common/config/BUILD | 3 +- source/common/config/protobuf_link_hacks.h | 4 +- source/common/config/resources.h | 1 + source/common/config/runtime_utility.cc | 12 +- source/common/config/type_to_endpoint.cc | 52 ++- source/common/runtime/BUILD | 6 + source/common/runtime/runtime_impl.cc | 116 ++++- source/common/runtime/runtime_impl.h | 69 ++- source/server/BUILD | 2 +- source/server/server.cc | 11 +- test/common/config/runtime_utility_test.cc | 44 +- test/common/grpc/grpc_client_integration.h | 22 +- test/common/protobuf/BUILD | 3 + test/common/protobuf/utility_test.cc | 9 +- test/common/runtime/BUILD | 4 + test/common/runtime/runtime_impl_test.cc | 396 ++++++++++++++++-- test/integration/BUILD | 10 + test/integration/rtds_integration_test.cc | 164 ++++++++ test/mocks/runtime/BUILD | 1 + test/mocks/runtime/mocks.h | 2 + test/server/BUILD | 2 + test/server/configuration_impl_test.cc | 24 +- ...nvalid_layered_runtime_duplicate_name.yaml | 8 + .../invalid_layered_runtime_missing_name.yaml | 4 + test/server/runtime_bootstrap.yaml | 9 +- test/server/server_test.cc | 13 + test/test_common/BUILD | 1 + test/test_common/utility.cc | 4 + tools/spelling_dictionary.txt | 2 +- 40 files changed, 926 insertions(+), 163 deletions(-) rename api/envoy/service/discovery/v2/{tds.proto => rtds.proto} (59%) create mode 100644 test/integration/rtds_integration_test.cc create mode 100644 test/server/invalid_layered_runtime_duplicate_name.yaml create mode 100644 test/server/invalid_layered_runtime_missing_name.yaml diff --git a/api/docs/BUILD b/api/docs/BUILD index d2dbcd26adfa..1cd82c78fc05 100644 --- a/api/docs/BUILD +++ b/api/docs/BUILD @@ -89,6 +89,7 @@ proto_library( "//envoy/service/auth/v2:attribute_context", "//envoy/service/auth/v2:external_auth", "//envoy/service/discovery/v2:ads", + "//envoy/service/discovery/v2:rtds", "//envoy/service/load_stats/v2:lrs", "//envoy/service/metrics/v2:metrics_service", "//envoy/service/ratelimit/v2:rls", diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index caa9dce53d67..d7d412da4fdd 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -237,13 +237,6 @@ message Runtime { } message RuntimeLayer { - // :ref:`Static runtime ` layer. - message StaticLayer { - // This follows the :ref:`runtime protobuf JSON representation encoding - // `. - google.protobuf.Struct value = 1; - } - // :ref:`Disk runtime ` layer. message DiskLayer { // The implementation assumes that the file system tree is accessed via a @@ -264,27 +257,28 @@ message RuntimeLayer { message AdminLayer { } - // [#not-implemented-hide:] - message TdsLayer { - // Resource to subscribe to at *tds_config* for the TDS layer. + // :ref:`Runtime Discovery Service (RTDS) ` layer. + message RtdsLayer { + // Resource to subscribe to at *rtds_config* for the RTDS layer. string name = 1; - // TDS configuration source. - envoy.api.v2.core.ConfigSource tds_config = 2; + // RTDS configuration source. + envoy.api.v2.core.ConfigSource rtds_config = 2; } // Descriptive name for the runtime layer. This is only used for the runtime // :http:get:`/runtime` output. - string name = 1; + string name = 1 [(validate.rules).string.min_bytes = 1]; oneof layer_specifier { - // Unlike static xDS resources, this static layer is overridable by later - // layers in the runtime virtual filesystem. + // :ref:`Static runtime ` layer. + // This follows the :ref:`runtime protobuf JSON representation encoding + // `. Unlike static xDS resources, this static + // layer is overridable by later layers in the runtime virtual filesystem. google.protobuf.Struct static_layer = 2; DiskLayer disk_layer = 3; AdminLayer admin_layer = 4; - // [#not-implemented-hide:] - TdsLayer tds_layer = 5; + RtdsLayer rtds_layer = 5; } } diff --git a/api/envoy/service/discovery/v2/BUILD b/api/envoy/service/discovery/v2/BUILD index fdd939758c4a..a9c2efd02fb9 100644 --- a/api/envoy/service/discovery/v2/BUILD +++ b/api/envoy/service/discovery/v2/BUILD @@ -58,8 +58,8 @@ api_go_grpc_library( ) api_proto_library_internal( - name = "tds", - srcs = ["tds.proto"], + name = "rtds", + srcs = ["rtds.proto"], has_services = 1, deps = [ "//envoy/api/v2:discovery", @@ -67,8 +67,8 @@ api_proto_library_internal( ) api_go_grpc_library( - name = "tds", - proto = ":tds", + name = "rtds", + proto = ":rtds", deps = [ "//envoy/api/v2:discovery_go_proto", ], diff --git a/api/envoy/service/discovery/v2/tds.proto b/api/envoy/service/discovery/v2/rtds.proto similarity index 59% rename from api/envoy/service/discovery/v2/tds.proto rename to api/envoy/service/discovery/v2/rtds.proto index eb317633f917..582a14366ff1 100644 --- a/api/envoy/service/discovery/v2/tds.proto +++ b/api/envoy/service/discovery/v2/rtds.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.service.discovery.v2; -option java_outer_classname = "TdsProto"; +option java_outer_classname = "RtdsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.discovery.v2"; option java_generic_services = true; @@ -12,18 +12,26 @@ import "envoy/api/v2/discovery.proto"; import "google/api/annotations.proto"; import "google/protobuf/struct.proto"; +import "validate/validate.proto"; + +// [#protodoc-title: runTime Discovery Service (RTDS)] +// RTDS :ref:`configuration overview ` + // [#not-implemented-hide:] Not configuration. Workaround c++ protobuf issue with importing // services: https://github.com/google/protobuf/issues/4221 -message TdsDummy { +message RtdsDummy { } // Discovery service for Runtime resources. -// [#not-implemented-hide:] service RuntimeDiscoveryService { rpc StreamRuntime(stream envoy.api.v2.DiscoveryRequest) returns (stream envoy.api.v2.DiscoveryResponse) { } + rpc DeltaRuntime(stream envoy.api.v2.DeltaDiscoveryRequest) + returns (stream envoy.api.v2.DeltaDiscoveryResponse) { + } + rpc FetchRuntime(envoy.api.v2.DiscoveryRequest) returns (envoy.api.v2.DiscoveryResponse) { option (google.api.http) = { post: "/v2/discovery:runtime" @@ -32,8 +40,10 @@ service RuntimeDiscoveryService { } } -// TDS resource type. This describes a layer in the runtime virtual filesystem. -// [#not-implemented-hide:] +// RTDS resource type. This describes a layer in the runtime virtual filesystem. message Runtime { - google.protobuf.Struct layer = 1; + // Runtime resource name. This makes the Runtime a self-describing xDS + // resource. + string name = 1 [(validate.rules).string.min_bytes = 1]; + google.protobuf.Struct layer = 2; } diff --git a/api/test/build/BUILD b/api/test/build/BUILD index 9eb525246458..ab1e8640a97d 100644 --- a/api/test/build/BUILD +++ b/api/test/build/BUILD @@ -13,7 +13,7 @@ api_cc_test( "//envoy/service/accesslog/v2:als", "//envoy/service/discovery/v2:ads", "//envoy/service/discovery/v2:hds", - "//envoy/service/discovery/v2:tds", + "//envoy/service/discovery/v2:rtds", "//envoy/service/metrics/v2:metrics_service", "//envoy/service/ratelimit/v2:rls", ], diff --git a/api/xds_protocol.rst b/api/xds_protocol.rst index 3906d64f1e7a..515c86343836 100644 --- a/api/xds_protocol.rst +++ b/api/xds_protocol.rst @@ -60,6 +60,7 @@ correspondence between an xDS API and a resource type. That is: - CDS: :ref:`envoy.api.v2.Cluster ` - EDS: :ref:`envoy.api.v2.ClusterLoadAssignment ` - SDS: :ref:`envoy.api.v2.Auth.Secret ` +- RTDS: :ref:`envoy.service.discovery.v2.Runtime ` The concept of `type URLs `_ appears below, and takes the form `type.googleapis.com/`, e.g. diff --git a/docs/build.sh b/docs/build.sh index 284222935c36..566fd061e4d6 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -142,6 +142,7 @@ PROTO_RST=" /envoy/service/accesslog/v2/als/envoy/service/accesslog/v2/als.proto.rst /envoy/service/auth/v2/external_auth/envoy/service/auth/v2/attribute_context.proto.rst /envoy/service/auth/v2/external_auth/envoy/service/auth/v2/external_auth.proto.rst + /envoy/service/discovery/v2/rtds/envoy/service/discovery/v2/rtds.proto.rst /envoy/service/ratelimit/v2/rls/envoy/service/ratelimit/v2/rls.proto.rst /envoy/service/tap/v2alpha/common/envoy/service/tap/v2alpha/common.proto.rst /envoy/type/http_status/envoy/type/http_status.proto.rst diff --git a/docs/root/api-v2/service/service.rst b/docs/root/api-v2/service/service.rst index 4cf1c5e9f284..5b2c37635c02 100644 --- a/docs/root/api-v2/service/service.rst +++ b/docs/root/api-v2/service/service.rst @@ -6,5 +6,6 @@ Services :maxdepth: 2 accesslog/v2/* + discovery/v2/* ratelimit/v2/* tap/v2alpha/* diff --git a/docs/root/configuration/runtime.rst b/docs/root/configuration/runtime.rst index 3f617eb2285c..c0e7109ba166 100644 --- a/docs/root/configuration/runtime.rst +++ b/docs/root/configuration/runtime.rst @@ -5,7 +5,7 @@ Runtime The :ref:`runtime configuration ` specifies a virtual file system tree that contains re-loadable configuration elements. This virtual file system can be realized via a series -of local file system, static bootstrap configuration and admin console derived overlays. +of local file system, static bootstrap configuration, RTDS and admin console derived overlays. * :ref:`v2 API reference ` @@ -134,6 +134,17 @@ old tree to the new runtime tree, using the equivalent of the following command: It's beyond the scope of this document how the file system data is deployed, garbage collected, etc. +.. _config_runtime_rtds: + +runTime Discovery Service (RTDS) +++++++++++++++++++++++++++++++++ + +One or more runtime layers may be specified and delivered by specifying a :ref:`rtds_layer +`. This points the runtime layer at a +regular :ref:`xDS ` endpoint, subscribing to a single xDS resource for the given +layer. The resource type for these layers is a :ref:`Runtime message +`. + .. _config_runtime_admin: Admin console diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 28b229278305..ed32dd6ca384 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -63,6 +63,7 @@ Version history `. * runtime: added support for statically :ref:`specifying the runtime in the bootstrap configuration `. +* runtime: :ref:`runTime Discovery Service (RTDS) ` support added to layered runtime configuration. * sandbox: added :ref:`CSRF sandbox `. * server: ``--define manual_stamp=manual_stamp`` was added to allow server stamping outside of binary rules. more info in the `bazel docs `_. diff --git a/include/envoy/runtime/runtime.h b/include/envoy/runtime/runtime.h index 5f1686570bf1..b3e3ebeb468a 100644 --- a/include/envoy/runtime/runtime.h +++ b/include/envoy/runtime/runtime.h @@ -16,6 +16,11 @@ #include "absl/types/optional.h" namespace Envoy { + +namespace Upstream { +class ClusterManager; +} + namespace Runtime { /** @@ -206,6 +211,14 @@ class Loader { public: virtual ~Loader() = default; + /** + * Post-construction initialization. Runtime will be generally available after + * the constructor is finished, with the exception of dynamic RTDS layers, + * which require ClusterManager. + * @param cm cluster manager reference. + */ + virtual void initialize(Upstream::ClusterManager& cm) PURE; + /** * @return Snapshot& the current snapshot. This reference is safe to use for the duration of * the calling routine, but may be overwritten on a future event loop cycle so should be diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 6dee36decdbb..c7fb5b067410 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -262,8 +262,8 @@ envoy_cc_library( hdrs = ["protobuf_link_hacks.h"], deps = [ "@envoy_api//envoy/service/discovery/v2:ads_cc", + "@envoy_api//envoy/service/discovery/v2:rtds_cc", "@envoy_api//envoy/service/discovery/v2:sds_cc", - "@envoy_api//envoy/service/discovery/v2:tds_cc", "@envoy_api//envoy/service/ratelimit/v2:rls_cc", ], ) @@ -353,6 +353,7 @@ envoy_cc_library( "@envoy_api//envoy/api/v2:lds_cc", "@envoy_api//envoy/api/v2:rds_cc", "@envoy_api//envoy/api/v2:srds_cc", + "@envoy_api//envoy/service/discovery/v2:rtds_cc", ], ) diff --git a/source/common/config/protobuf_link_hacks.h b/source/common/config/protobuf_link_hacks.h index fa0afdc4dff7..90682dcac177 100644 --- a/source/common/config/protobuf_link_hacks.h +++ b/source/common/config/protobuf_link_hacks.h @@ -1,8 +1,8 @@ #pragma once #include "envoy/service/discovery/v2/ads.pb.h" +#include "envoy/service/discovery/v2/rtds.pb.h" #include "envoy/service/discovery/v2/sds.pb.h" -#include "envoy/service/discovery/v2/tds.pb.h" #include "envoy/service/ratelimit/v2/rls.pb.h" namespace Envoy { @@ -12,5 +12,5 @@ namespace Envoy { const envoy::service::discovery::v2::AdsDummy _ads_dummy; const envoy::service::ratelimit::v2::RateLimitRequest _rls_dummy; const envoy::service::discovery::v2::SdsDummy _sds_dummy; -const envoy::service::discovery::v2::TdsDummy _tds_dummy; +const envoy::service::discovery::v2::RtdsDummy _tds_dummy; } // namespace Envoy diff --git a/source/common/config/resources.h b/source/common/config/resources.h index d4baae1622f1..7af6a74e4f85 100644 --- a/source/common/config/resources.h +++ b/source/common/config/resources.h @@ -20,6 +20,7 @@ class TypeUrlValues { const std::string VirtualHost{"type.googleapis.com/envoy.api.v2.route.VirtualHost"}; const std::string ScopedRouteConfiguration{ "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration"}; + const std::string Runtime{"type.googleapis.com/envoy.service.discovery.v2.Runtime"}; }; using TypeUrl = ConstSingleton; diff --git a/source/common/config/runtime_utility.cc b/source/common/config/runtime_utility.cc index e29e58f4b911..ffd4d3a9657c 100644 --- a/source/common/config/runtime_utility.cc +++ b/source/common/config/runtime_utility.cc @@ -5,7 +5,11 @@ namespace Config { void translateRuntime(const envoy::config::bootstrap::v2::Runtime& runtime_config, envoy::config::bootstrap::v2::LayeredRuntime& layered_runtime_config) { - layered_runtime_config.add_layers()->mutable_static_layer()->MergeFrom(runtime_config.base()); + { + auto* layer = layered_runtime_config.add_layers(); + layer->set_name("base"); + layer->mutable_static_layer()->MergeFrom(runtime_config.base()); + } if (!runtime_config.symlink_root().empty()) { { auto* layer = layered_runtime_config.add_layers(); @@ -21,7 +25,11 @@ void translateRuntime(const envoy::config::bootstrap::v2::Runtime& runtime_confi layer->mutable_disk_layer()->set_append_service_cluster(true); } } - layered_runtime_config.add_layers()->mutable_admin_layer(); + { + auto* layer = layered_runtime_config.add_layers(); + layer->set_name("admin"); + layer->mutable_admin_layer(); + } } } // namespace Config diff --git a/source/common/config/type_to_endpoint.cc b/source/common/config/type_to_endpoint.cc index 71905d570b0b..800d04271435 100644 --- a/source/common/config/type_to_endpoint.cc +++ b/source/common/config/type_to_endpoint.cc @@ -5,6 +5,7 @@ #include "envoy/api/v2/lds.pb.h" #include "envoy/api/v2/rds.pb.h" #include "envoy/api/v2/srds.pb.h" +#include "envoy/service/discovery/v2/rtds.pb.h" #include "common/grpc/common.h" @@ -13,24 +14,29 @@ namespace Config { const char UnknownMethod[] = "could_not_lookup_method_due_to_unknown_type_url"; -#define TYPE_URL_IS(x) \ +#define API_TYPE_URL_IS(x) \ (type_url == Grpc::Common::typeUrl(envoy::api::v2::x().GetDescriptor()->full_name())) +#define DISCOVERY_TYPE_URL_IS(x) \ + (type_url == \ + Grpc::Common::typeUrl(envoy::service::discovery::v2::x().GetDescriptor()->full_name())) const Protobuf::MethodDescriptor& deltaGrpcMethod(absl::string_view type_url) { std::string method_name = UnknownMethod; - if (TYPE_URL_IS(RouteConfiguration)) { + if (API_TYPE_URL_IS(RouteConfiguration)) { method_name = "envoy.api.v2.RouteDiscoveryService.DeltaRoutes"; - } else if (TYPE_URL_IS(ScopedRouteConfiguration)) { + } else if (API_TYPE_URL_IS(ScopedRouteConfiguration)) { method_name = "envoy.api.v2.ScopedRoutesDiscoveryService.DeltaScopedRoutes"; - } else if (TYPE_URL_IS(route::VirtualHost)) { + } else if (API_TYPE_URL_IS(route::VirtualHost)) { method_name = "envoy.api.v2.VirtualHostDiscoveryService.DeltaVirtualHosts"; - } else if (TYPE_URL_IS(auth::Secret)) { + } else if (API_TYPE_URL_IS(auth::Secret)) { method_name = "envoy.service.discovery.v2.SecretDiscoveryService.DeltaSecrets"; - } else if (TYPE_URL_IS(Cluster)) { + } else if (API_TYPE_URL_IS(Cluster)) { method_name = "envoy.api.v2.ClusterDiscoveryService.DeltaClusters"; - } else if (TYPE_URL_IS(ClusterLoadAssignment)) { + } else if (API_TYPE_URL_IS(ClusterLoadAssignment)) { method_name = "envoy.api.v2.EndpointDiscoveryService.DeltaEndpoints"; - } else if (TYPE_URL_IS(Listener)) { + } else if (API_TYPE_URL_IS(Listener)) { method_name = "envoy.api.v2.ListenerDiscoveryService.DeltaListeners"; + } else if (DISCOVERY_TYPE_URL_IS(Runtime)) { + method_name = "envoy.service.discovery.v2.RuntimeDiscoveryService.DeltaRuntime"; } ASSERT(method_name != UnknownMethod); return *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(method_name); @@ -38,18 +44,20 @@ const Protobuf::MethodDescriptor& deltaGrpcMethod(absl::string_view type_url) { const Protobuf::MethodDescriptor& sotwGrpcMethod(absl::string_view type_url) { std::string method_name = UnknownMethod; - if (TYPE_URL_IS(RouteConfiguration)) { + if (API_TYPE_URL_IS(RouteConfiguration)) { method_name = "envoy.api.v2.RouteDiscoveryService.StreamRoutes"; - } else if (TYPE_URL_IS(ScopedRouteConfiguration)) { + } else if (API_TYPE_URL_IS(ScopedRouteConfiguration)) { method_name = "envoy.api.v2.ScopedRoutesDiscoveryService.StreamScopedRoutes"; - } else if (TYPE_URL_IS(auth::Secret)) { + } else if (API_TYPE_URL_IS(auth::Secret)) { method_name = "envoy.service.discovery.v2.SecretDiscoveryService.StreamSecrets"; - } else if (TYPE_URL_IS(Cluster)) { + } else if (API_TYPE_URL_IS(Cluster)) { method_name = "envoy.api.v2.ClusterDiscoveryService.StreamClusters"; - } else if (TYPE_URL_IS(ClusterLoadAssignment)) { + } else if (API_TYPE_URL_IS(ClusterLoadAssignment)) { method_name = "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"; - } else if (TYPE_URL_IS(Listener)) { + } else if (API_TYPE_URL_IS(Listener)) { method_name = "envoy.api.v2.ListenerDiscoveryService.StreamListeners"; + } else if (DISCOVERY_TYPE_URL_IS(Runtime)) { + method_name = "envoy.service.discovery.v2.RuntimeDiscoveryService.StreamRuntime"; } ASSERT(method_name != UnknownMethod); return *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(method_name); @@ -57,23 +65,25 @@ const Protobuf::MethodDescriptor& sotwGrpcMethod(absl::string_view type_url) { const Protobuf::MethodDescriptor& restMethod(absl::string_view type_url) { std::string method_name = UnknownMethod; - if (TYPE_URL_IS(RouteConfiguration)) { + if (API_TYPE_URL_IS(RouteConfiguration)) { method_name = "envoy.api.v2.RouteDiscoveryService.FetchRoutes"; - } else if (TYPE_URL_IS(ScopedRouteConfiguration)) { + } else if (API_TYPE_URL_IS(ScopedRouteConfiguration)) { method_name = "envoy.api.v2.ScopedRoutesDiscoveryService.FetchScopedRoutes"; - } else if (TYPE_URL_IS(auth::Secret)) { + } else if (API_TYPE_URL_IS(auth::Secret)) { method_name = "envoy.service.discovery.v2.SecretDiscoveryService.FetchSecrets"; - } else if (TYPE_URL_IS(Cluster)) { + } else if (API_TYPE_URL_IS(Cluster)) { method_name = "envoy.api.v2.ClusterDiscoveryService.FetchClusters"; - } else if (TYPE_URL_IS(ClusterLoadAssignment)) { + } else if (API_TYPE_URL_IS(ClusterLoadAssignment)) { method_name = "envoy.api.v2.EndpointDiscoveryService.FetchEndpoints"; - } else if (TYPE_URL_IS(Listener)) { + } else if (API_TYPE_URL_IS(Listener)) { method_name = "envoy.api.v2.ListenerDiscoveryService.FetchListeners"; + } else if (DISCOVERY_TYPE_URL_IS(Runtime)) { + method_name = "envoy.service.discovery.v2.RuntimeDiscoveryService.FetchRuntime"; } ASSERT(method_name != UnknownMethod); return *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(method_name); } -#undef TYPE_URL_IS +#undef API_TYPE_URL_IS } // namespace Config } // namespace Envoy diff --git a/source/common/runtime/BUILD b/source/common/runtime/BUILD index 4f848a10918a..2cbf39b5d36a 100644 --- a/source/common/runtime/BUILD +++ b/source/common/runtime/BUILD @@ -20,19 +20,25 @@ envoy_cc_library( ], external_deps = ["ssl"], deps = [ + "//include/envoy/config:subscription_interface", "//include/envoy/event:dispatcher_interface", + "//include/envoy/init:manager_interface", "//include/envoy/runtime:runtime_interface", "//include/envoy/stats:stats_interface", "//include/envoy/stats:stats_macros", "//include/envoy/thread_local:thread_local_interface", + "//include/envoy/upstream:cluster_manager_interface", "//source/common/common:empty_string", "//source/common/common:minimal_logger_lib", "//source/common/common:thread_lib", "//source/common/common:utility_lib", "//source/common/filesystem:directory_lib", + "//source/common/grpc:common_lib", + "//source/common/init:target_lib", "//source/common/protobuf:message_validator_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/config/bootstrap/v2:bootstrap_cc", + "@envoy_api//envoy/service/discovery/v2:rtds_cc", ], ) diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index fd449f63fc90..03f654dc1b18 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -13,6 +13,7 @@ #include "common/common/fmt.h" #include "common/common/utility.h" #include "common/filesystem/directory.h" +#include "common/grpc/common.h" #include "common/protobuf/message_validator_impl.h" #include "common/protobuf/utility.h" #include "common/runtime/runtime_features.h" @@ -350,7 +351,7 @@ void AdminLayer::mergeValues(const std::unordered_map& stats_.admin_overrides_active_.set(values_.empty() ? 0 : 1); } -DiskLayer::DiskLayer(const std::string& name, const std::string& path, Api::Api& api) +DiskLayer::DiskLayer(absl::string_view name, const std::string& path, Api::Api& api) : OverrideLayerImpl{name} { walkDirectory(path, "", 1, api); } @@ -409,7 +410,8 @@ void DiskLayer::walkDirectory(const std::string& path, const std::string& prefix } } -ProtoLayer::ProtoLayer(const ProtobufWkt::Struct& proto) : OverrideLayerImpl{"base"} { +ProtoLayer::ProtoLayer(absl::string_view name, const ProtobufWkt::Struct& proto) + : OverrideLayerImpl{name} { for (const auto& f : proto.fields()) { walkProtoValue(f.second, f.first); } @@ -448,30 +450,106 @@ void ProtoLayer::walkProtoValue(const ProtobufWkt::Value& v, const std::string& LoaderImpl::LoaderImpl(Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls, const envoy::config::bootstrap::v2::LayeredRuntime& config, - absl::string_view service_cluster, Stats::Store& store, - RandomGenerator& generator, Api::Api& api) - : generator_(generator), stats_(generateStats(store)), admin_layer_(stats_), - tls_(tls.allocateSlot()), config_(config), service_cluster_(service_cluster), api_(api) { + const LocalInfo::LocalInfo& local_info, Init::Manager& init_manager, + Stats::Store& store, RandomGenerator& generator, + ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api) + : generator_(generator), stats_(generateStats(store)), tls_(tls.allocateSlot()), + config_(config), service_cluster_(local_info.clusterName()), api_(api) { + std::unordered_set layer_names; for (const auto& layer : config_.layers()) { - if (layer.has_admin_layer()) { - if (config_has_admin_layer_) { + auto ret = layer_names.insert(layer.name()); + if (!ret.second) { + throw EnvoyException(absl::StrCat("Duplicate layer name: ", layer.name())); + } + switch (layer.layer_specifier_case()) { + case envoy::config::bootstrap::v2::RuntimeLayer::kAdminLayer: + if (admin_layer_ != nullptr) { throw EnvoyException( "Too many admin layers specified in LayeredRuntime, at most one may be specified"); - } else { - config_has_admin_layer_ = true; } - } else if (layer.has_disk_layer()) { + admin_layer_ = std::make_unique(layer.name(), stats_); + break; + case envoy::config::bootstrap::v2::RuntimeLayer::kDiskLayer: if (watcher_ == nullptr) { watcher_ = dispatcher.createFilesystemWatcher(); } watcher_->addWatch(layer.disk_layer().symlink_root(), Filesystem::Watcher::Events::MovedTo, [this](uint32_t) -> void { loadNewSnapshot(); }); + break; + case envoy::config::bootstrap::v2::RuntimeLayer::kRtdsLayer: + subscriptions_.emplace_back( + std::make_unique(*this, layer.rtds_layer(), store, validation_visitor)); + init_manager.add(subscriptions_.back()->init_target_); + break; + default: + ENVOY_LOG(warn, "Skipping unsupported runtime layer: {}", layer.DebugString()); + break; } } loadNewSnapshot(); } +void LoaderImpl::initialize(Upstream::ClusterManager& cm) { cm_ = &cm; } + +RtdsSubscription::RtdsSubscription( + LoaderImpl& parent, const envoy::config::bootstrap::v2::RuntimeLayer::RtdsLayer& rtds_layer, + Stats::Store& store, ProtobufMessage::ValidationVisitor& validation_visitor) + : parent_(parent), config_source_(rtds_layer.rtds_config()), store_(store), + resource_name_(rtds_layer.name()), + init_target_("RTDS " + resource_name_, [this]() { start(); }), + validation_visitor_(validation_visitor) {} + +void RtdsSubscription::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string&) { + validateUpdateSize(resources.size()); + auto runtime = MessageUtil::anyConvert( + resources[0], validation_visitor_); + MessageUtil::validate(runtime); + if (runtime.name() != resource_name_) { + throw EnvoyException( + fmt::format("Unexpected RTDS runtime (expecting {}): {}", resource_name_, runtime.name())); + } + ENVOY_LOG(debug, "Reloading RTDS snapshot for onConfigUpdate"); + proto_.CopyFrom(runtime.layer()); + parent_.loadNewSnapshot(); + init_target_.ready(); +} + +void RtdsSubscription::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const Protobuf::RepeatedPtrField&, const std::string&) { + validateUpdateSize(resources.size()); + Protobuf::RepeatedPtrField unwrapped_resource; + *unwrapped_resource.Add() = resources[0].resource(); + onConfigUpdate(unwrapped_resource, resources[0].version()); +} + +void RtdsSubscription::onConfigUpdateFailed(const EnvoyException*) { + // We need to allow server startup to continue, even if we have a bad + // config. + init_target_.ready(); +} + +void RtdsSubscription::start() { + // We have to delay the subscription creation until init-time, since the + // cluster manager resources are not available in the constructor when + // instantiated in the server instance. + subscription_ = parent_.cm_->subscriptionFactory().subscriptionFromConfigSource( + config_source_, + Grpc::Common::typeUrl(envoy::service::discovery::v2::Runtime().GetDescriptor()->full_name()), + store_, *this); + subscription_->start({resource_name_}); +} + +void RtdsSubscription::validateUpdateSize(uint32_t num_resources) { + if (num_resources != 1) { + init_target_.ready(); + throw EnvoyException(fmt::format("Unexpected RTDS resource length: {}", num_resources)); + // (would be a return false here) + } +} + void LoaderImpl::loadNewSnapshot() { ThreadLocal::ThreadLocalObjectSharedPtr ptr = createNewSnapshot(); tls_->set([ptr = std::move(ptr)](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { @@ -482,10 +560,10 @@ void LoaderImpl::loadNewSnapshot() { Snapshot& LoaderImpl::snapshot() { return tls_->getTyped(); } void LoaderImpl::mergeValues(const std::unordered_map& values) { - if (!config_has_admin_layer_) { + if (admin_layer_ == nullptr) { throw EnvoyException("No admin layer specified"); } - admin_layer_.mergeValues(values); + admin_layer_->mergeValues(values); loadNewSnapshot(); } @@ -500,10 +578,11 @@ std::unique_ptr LoaderImpl::createNewSnapshot() { std::vector layers; uint32_t disk_layers = 0; uint32_t error_layers = 0; + uint32_t rtds_layer = 0; for (const auto& layer : config_.layers()) { switch (layer.layer_specifier_case()) { case envoy::config::bootstrap::v2::RuntimeLayer::kStaticLayer: - layers.emplace_back(std::make_unique(layer.static_layer())); + layers.emplace_back(std::make_unique(layer.name(), layer.static_layer())); break; case envoy::config::bootstrap::v2::RuntimeLayer::kDiskLayer: { std::string path = layer.disk_layer().symlink_root(); @@ -516,7 +595,7 @@ std::unique_ptr LoaderImpl::createNewSnapshot() { ++disk_layers; } catch (EnvoyException& e) { // TODO(htuch): Consider latching here, rather than ignoring the - // layer. This would be consistent with filesystem TDS. + // layer. This would be consistent with filesystem RTDS. ++error_layers; ENVOY_LOG(debug, "error loading runtime values for layer {} from disk: {}", layer.DebugString(), e.what()); @@ -525,8 +604,13 @@ std::unique_ptr LoaderImpl::createNewSnapshot() { break; } case envoy::config::bootstrap::v2::RuntimeLayer::kAdminLayer: - layers.push_back(std::make_unique(admin_layer_)); + layers.push_back(std::make_unique(*admin_layer_)); break; + case envoy::config::bootstrap::v2::RuntimeLayer::kRtdsLayer: { + auto* subscription = subscriptions_[rtds_layer++].get(); + layers.emplace_back(std::make_unique(layer.name(), subscription->proto_)); + break; + } default: NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index 3054feb89828..855dc1ea370e 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -8,16 +8,21 @@ #include "envoy/api/api.h" #include "envoy/common/exception.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.h" +#include "envoy/config/subscription.h" +#include "envoy/init/manager.h" #include "envoy/runtime/runtime.h" +#include "envoy/service/discovery/v2/rtds.pb.validate.h" #include "envoy/stats/stats_macros.h" #include "envoy/stats/store.h" #include "envoy/thread_local/thread_local.h" #include "envoy/type/percent.pb.validate.h" +#include "envoy/upstream/cluster_manager.h" #include "common/common/assert.h" #include "common/common/empty_string.h" #include "common/common/logger.h" #include "common/common/thread.h" +#include "common/init/target_impl.h" #include "common/singleton/threadsafe_singleton.h" #include "spdlog/spdlog.h" @@ -121,7 +126,7 @@ class SnapshotImpl : public Snapshot, */ class OverrideLayerImpl : public Snapshot::OverrideLayer { public: - explicit OverrideLayerImpl(const std::string& name) : name_{name} {} + explicit OverrideLayerImpl(absl::string_view name) : name_{name} {} const Snapshot::EntryMap& values() const override { return values_; } const std::string& name() const override { return name_; } @@ -137,11 +142,12 @@ class OverrideLayerImpl : public Snapshot::OverrideLayer { */ class AdminLayer : public OverrideLayerImpl { public: - explicit AdminLayer(RuntimeStats& stats) : OverrideLayerImpl{"admin"}, stats_{stats} {} + explicit AdminLayer(absl::string_view name, RuntimeStats& stats) + : OverrideLayerImpl{name}, stats_{stats} {} /** * Copy-constructible so that it can snapshotted. */ - AdminLayer(const AdminLayer& admin_layer) : AdminLayer{admin_layer.stats_} { + AdminLayer(const AdminLayer& admin_layer) : AdminLayer{admin_layer.name_, admin_layer.stats_} { values_ = admin_layer.values(); } @@ -155,12 +161,14 @@ class AdminLayer : public OverrideLayerImpl { RuntimeStats& stats_; }; +using AdminLayerPtr = std::unique_ptr; + /** * Extension of OverrideLayerImpl that loads values from the file system upon construction. */ class DiskLayer : public OverrideLayerImpl, Logger::Loggable { public: - DiskLayer(const std::string& name, const std::string& path, Api::Api& api); + DiskLayer(absl::string_view name, const std::string& path, Api::Api& api); private: void walkDirectory(const std::string& path, const std::string& prefix, uint32_t depth, @@ -177,12 +185,48 @@ class DiskLayer : public OverrideLayerImpl, Logger::Loggable { public: - ProtoLayer(const ProtobufWkt::Struct& proto); + ProtoLayer(absl::string_view name, const ProtobufWkt::Struct& proto); private: void walkProtoValue(const ProtobufWkt::Value& v, const std::string& prefix); }; +class LoaderImpl; + +struct RtdsSubscription : Config::SubscriptionCallbacks, Logger::Loggable { + RtdsSubscription(LoaderImpl& parent, + const envoy::config::bootstrap::v2::RuntimeLayer::RtdsLayer& rtds_layer, + Stats::Store& store, ProtobufMessage::ValidationVisitor& validation_visitor); + + // Config::SubscriptionCallbacks + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string&) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string&) override; + + void onConfigUpdateFailed(const EnvoyException* e) override; + std::string resourceName(const ProtobufWkt::Any& resource) override { + return MessageUtil::anyConvert(resource, + validation_visitor_) + .name(); + } + + void start(); + void validateUpdateSize(uint32_t num_resources); + + LoaderImpl& parent_; + const envoy::api::v2::core::ConfigSource config_source_; + Stats::Store& store_; + Config::SubscriptionPtr subscription_; + std::string resource_name_; + Init::TargetImpl init_target_; + ProtobufWkt::Struct proto_; + ProtobufMessage::ValidationVisitor& validation_visitor_; +}; + +using RtdsSubscriptionPtr = std::unique_ptr; + /** * Implementation of Loader that provides Snapshots of values added via mergeValues(). * A single snapshot is shared among all threads and referenced by shared_ptr such that @@ -193,29 +237,34 @@ class LoaderImpl : public Loader, Logger::Loggable { public: LoaderImpl(Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls, const envoy::config::bootstrap::v2::LayeredRuntime& config, - absl::string_view service_cluster, Stats::Store& store, RandomGenerator& generator, - Api::Api& api); + const LocalInfo::LocalInfo& local_info, Init::Manager& init_manager, + Stats::Store& store, RandomGenerator& generator, + ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api); // Runtime::Loader + void initialize(Upstream::ClusterManager& cm) override; Snapshot& snapshot() override; void mergeValues(const std::unordered_map& values) override; +private: + friend RtdsSubscription; + // Create a new Snapshot virtual std::unique_ptr createNewSnapshot(); // Load a new Snapshot into TLS void loadNewSnapshot(); RuntimeStats generateStats(Stats::Store& store); - bool config_has_admin_layer_{}; RandomGenerator& generator_; RuntimeStats stats_; - AdminLayer admin_layer_; - const ProtobufWkt::Struct base_; + AdminLayerPtr admin_layer_; ThreadLocal::SlotPtr tls_; const envoy::config::bootstrap::v2::LayeredRuntime config_; const std::string service_cluster_; Filesystem::WatcherPtr watcher_; Api::Api& api_; + std::vector subscriptions_; + Upstream::ClusterManager* cm_{}; }; } // namespace Runtime diff --git a/source/server/BUILD b/source/server/BUILD index 3f442837f2e9..168e7aa62795 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -311,7 +311,7 @@ envoy_cc_library( "@envoy_api//envoy/api/v2:srds_cc", "@envoy_api//envoy/service/discovery/v2:ads_cc", "@envoy_api//envoy/service/discovery/v2:hds_cc", - "@envoy_api//envoy/service/discovery/v2:tds_cc", + "@envoy_api//envoy/service/discovery/v2:rtds_cc", "@envoy_api//envoy/service/ratelimit/v2:rls_cc", ], ) diff --git a/source/server/server.cc b/source/server/server.cc index 94fd9ac5c560..3f5205347198 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -361,6 +361,10 @@ void InstanceImpl::initialize(const Options& options, listener_manager_->createLdsApi(bootstrap_.dynamic_resources().lds_config()); } + // We have to defer RTDS initialization until after the cluster manager is + // instantiated (which in turn relies on runtime...). + Runtime::LoaderSingleton::get().initialize(clusterManager()); + if (bootstrap_.has_hds_config()) { const auto& hds_config = bootstrap_.hds_config(); async_client_manager_ = std::make_unique( @@ -402,9 +406,10 @@ void InstanceImpl::startWorkers() { Runtime::LoaderPtr InstanceUtil::createRuntime(Instance& server, Server::Configuration::Initial& config) { ENVOY_LOG(info, "runtime: {}", MessageUtil::getYamlStringFromMessage(config.runtime())); - return std::make_unique(server.dispatcher(), server.threadLocal(), - config.runtime(), server.localInfo().clusterName(), - server.stats(), server.random(), server.api()); + return std::make_unique( + server.dispatcher(), server.threadLocal(), config.runtime(), server.localInfo(), + server.initManager(), server.stats(), server.random(), server.messageValidationVisitor(), + server.api()); } void InstanceImpl::loadServerFlags(const absl::optional& flags_path) { diff --git a/test/common/config/runtime_utility_test.cc b/test/common/config/runtime_utility_test.cc index 016d0c7c992f..6b5d35b434d6 100644 --- a/test/common/config/runtime_utility_test.cc +++ b/test/common/config/runtime_utility_test.cc @@ -13,8 +13,16 @@ TEST(RuntimeUtility, TranslateEmpty) { envoy::config::bootstrap::v2::LayeredRuntime layered_runtime_config; translateRuntime({}, layered_runtime_config); envoy::config::bootstrap::v2::LayeredRuntime expected_runtime_config; - expected_runtime_config.add_layers()->mutable_static_layer(); - expected_runtime_config.add_layers()->mutable_admin_layer(); + { + auto* layer = expected_runtime_config.add_layers(); + layer->set_name("base"); + layer->mutable_static_layer(); + } + { + auto* layer = expected_runtime_config.add_layers(); + layer->set_name("admin"); + layer->mutable_admin_layer(); + } EXPECT_THAT(layered_runtime_config, ProtoEq(expected_runtime_config)); } @@ -25,11 +33,21 @@ TEST(RuntimeUtility, TranslateSubdirOnly) { envoy::config::bootstrap::v2::LayeredRuntime layered_runtime_config; translateRuntime(runtime_config, layered_runtime_config); envoy::config::bootstrap::v2::LayeredRuntime expected_runtime_config; - expected_runtime_config.add_layers()->mutable_static_layer(); - auto* layer = expected_runtime_config.add_layers(); - layer->set_name("root"); - layer->mutable_disk_layer()->set_symlink_root("foo/bar"); - expected_runtime_config.add_layers()->mutable_admin_layer(); + { + auto* layer = expected_runtime_config.add_layers(); + layer->set_name("base"); + layer->mutable_static_layer(); + } + { + auto* layer = expected_runtime_config.add_layers(); + layer->set_name("root"); + layer->mutable_disk_layer()->set_symlink_root("foo/bar"); + } + { + auto* layer = expected_runtime_config.add_layers(); + layer->set_name("admin"); + layer->mutable_admin_layer(); + } EXPECT_THAT(layered_runtime_config, ProtoEq(expected_runtime_config)); } @@ -41,7 +59,11 @@ TEST(RuntimeUtility, TranslateSubdirOverride) { envoy::config::bootstrap::v2::LayeredRuntime layered_runtime_config; translateRuntime(runtime_config, layered_runtime_config); envoy::config::bootstrap::v2::LayeredRuntime expected_runtime_config; - expected_runtime_config.add_layers()->mutable_static_layer(); + { + auto* layer = expected_runtime_config.add_layers(); + layer->set_name("base"); + layer->mutable_static_layer(); + } { auto* layer = expected_runtime_config.add_layers(); layer->set_name("root"); @@ -53,7 +75,11 @@ TEST(RuntimeUtility, TranslateSubdirOverride) { layer->mutable_disk_layer()->set_symlink_root("foo/baz"); layer->mutable_disk_layer()->set_append_service_cluster(true); } - expected_runtime_config.add_layers()->mutable_admin_layer(); + { + auto* layer = expected_runtime_config.add_layers(); + layer->set_name("admin"); + layer->mutable_admin_layer(); + } EXPECT_THAT(layered_runtime_config, ProtoEq(expected_runtime_config)); } diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index 7585a14fd1e9..ff6d4d3a7b53 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -55,21 +55,17 @@ class GrpcClientIntegrationParamTest }; class DeltaSotwIntegrationParamTest - : public testing::TestWithParam< - std::tuple> { + : public testing::TestWithParam> { public: ~DeltaSotwIntegrationParamTest() override = default; - static std::string - protocolTestParamsToString(const ::testing::TestParamInfo< - std::tuple>& p) { + static std::string protocolTestParamsToString( + const ::testing::TestParamInfo>& p) { return fmt::format("{}_{}_{}", std::get<0>(p.param) == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6", - std::get<1>(p.param) == ClientType::GoogleGrpc ? "GoogleGrpc" : "EnvoyGrpc", - std::get<2>(p.param) == SotwOrDelta::Delta ? "Delta" : "StateOfTheWorld"); + std::get<1>(p.param) == SotwOrDelta::Delta ? "Delta" : "StateOfTheWorld"); } Network::Address::IpVersion ipVersion() const { return std::get<0>(GetParam()); } - ClientType clientType() const { return std::get<1>(GetParam()); } - SotwOrDelta sotwOrDelta() const { return std::get<2>(GetParam()); } + SotwOrDelta sotwOrDelta() const { return std::get<1>(GetParam()); } }; // Skip tests based on gRPC client type. @@ -88,19 +84,15 @@ class DeltaSotwIntegrationParamTest #define GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc)) -#define DELTA_INTEGRATION_PARAMS \ - testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ - testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc), \ - testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) #else #define GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc)) +#endif // ENVOY_GOOGLE_GRPC + #define DELTA_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ - testing::Values(Grpc::ClientType::EnvoyGrpc), \ testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) -#endif // ENVOY_GOOGLE_GRPC } // namespace Grpc } // namespace Envoy diff --git a/test/common/protobuf/BUILD b/test/common/protobuf/BUILD index 7280fe87106f..94fbcced56c7 100644 --- a/test/common/protobuf/BUILD +++ b/test/common/protobuf/BUILD @@ -15,6 +15,9 @@ envoy_cc_test( deps = [ "//source/common/protobuf:utility_lib", "//source/common/stats:isolated_store_lib", + "//test/mocks/init:init_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/server:server_mocks", "//test/proto:deprecated_proto_cc", "//test/test_common:environment_lib", diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index bf3bb13c1ec8..99a7dcb607fd 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -8,6 +8,9 @@ #include "common/runtime/runtime_impl.h" #include "common/stats/isolated_store_impl.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/protobuf/mocks.h" #include "test/mocks/server/mocks.h" #include "test/proto/deprecated.pb.h" #include "test/test_common/environment.h" @@ -455,7 +458,8 @@ class DeprecatedFieldsTest : public testing::Test { envoy::config::bootstrap::v2::LayeredRuntime config; config.add_layers()->mutable_admin_layer(); loader_ = std::make_unique(Runtime::LoaderPtr{ - new Runtime::LoaderImpl(dispatcher_, tls_, config, "", store_, generator_, *api_)}); + new Runtime::LoaderImpl(dispatcher_, tls_, config, local_info_, init_manager_, store_, + generator_, validation_visitor_, *api_)}); } Event::MockDispatcher dispatcher_; @@ -466,6 +470,9 @@ class DeprecatedFieldsTest : public testing::Test { Runtime::MockRandomGenerator rand_; std::unique_ptr loader_; Stats::Counter& runtime_deprecated_feature_use_; + NiceMock local_info_; + Init::MockManager init_manager_; + NiceMock validation_visitor_; }; TEST_F(DeprecatedFieldsTest, NoCrashIfRuntimeMissing) { diff --git a/test/common/runtime/BUILD b/test/common/runtime/BUILD index b3a4b44b3f42..a4eab5e8d007 100644 --- a/test/common/runtime/BUILD +++ b/test/common/runtime/BUILD @@ -37,8 +37,12 @@ envoy_cc_test( "//source/common/stats:stats_lib", "//test/mocks/event:event_mocks", "//test/mocks/filesystem:filesystem_mocks", + "//test/mocks/init:init_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/thread_local:thread_local_mocks", + "//test/mocks/upstream:upstream_mocks", "//test/test_common:environment_lib", ], ) diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 86fb91912681..c192268e3c0d 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -7,8 +7,12 @@ #include "test/mocks/event/mocks.h" #include "test/mocks/filesystem/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/protobuf/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/thread_local/mocks.h" +#include "test/mocks/upstream/mocks.h" #include "test/test_common/environment.h" #include "gmock/gmock.h" @@ -20,6 +24,7 @@ using testing::InvokeWithoutArgs; using testing::NiceMock; using testing::Return; using testing::ReturnNew; +using testing::ReturnRef; namespace Envoy { namespace Runtime { @@ -68,16 +73,7 @@ TEST(UUID, SanityCheckOfUniqueness) { class LoaderImplTest : public testing::Test { protected: - LoaderImplTest() : api_(Api::createApiForTest(store_)) {} - - static void SetUpTestSuite() { - TestEnvironment::exec( - {TestEnvironment::runfilesPath("test/common/runtime/filesystem_setup.sh")}); - } - - static void TearDownTestSuite() { - TestEnvironment::removePath(TestEnvironment::temporaryPath("test/common/runtime/test_data")); - } + LoaderImplTest() : api_(Api::createApiForTest(store_)) { local_info_.node_.set_cluster(""); } virtual void setup() { EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([this] { @@ -91,6 +87,30 @@ class LoaderImplTest : public testing::Test { })); } + Event::MockDispatcher dispatcher_; + NiceMock tls_; + Stats::IsolatedStoreImpl store_; + MockRandomGenerator generator_; + std::unique_ptr loader_; + Api::ApiPtr api_; + Upstream::MockClusterManager cm_; + NiceMock local_info_; + Init::MockManager init_manager_; + std::vector on_changed_cbs_; + NiceMock validation_visitor_; +}; + +class DiskLoaderImplTest : public LoaderImplTest { +public: + static void SetUpTestSuite() { + TestEnvironment::exec( + {TestEnvironment::runfilesPath("test/common/runtime/filesystem_setup.sh")}); + } + + static void TearDownTestSuite() { + TestEnvironment::removePath(TestEnvironment::temporaryPath("test/common/runtime/test_data")); + } + void run(const std::string& primary_dir, const std::string& override_dir) { envoy::config::bootstrap::v2::Runtime runtime; runtime.mutable_base()->MergeFrom(base_); @@ -100,8 +120,9 @@ class LoaderImplTest : public testing::Test { envoy::config::bootstrap::v2::LayeredRuntime layered_runtime; Config::translateRuntime(runtime, layered_runtime); - loader_ = std::make_unique(dispatcher_, tls_, layered_runtime, "", store_, - generator_, *api_); + loader_ = + std::make_unique(dispatcher_, tls_, layered_runtime, local_info_, init_manager_, + store_, generator_, validation_visitor_, *api_); } void write(const std::string& path, const std::string& value) { @@ -113,18 +134,10 @@ class LoaderImplTest : public testing::Test { on_changed_cbs_[layer](Filesystem::Watcher::Events::MovedTo); } - Event::MockDispatcher dispatcher_; - NiceMock tls_; - - std::vector on_changed_cbs_; - Stats::IsolatedStoreImpl store_; - MockRandomGenerator generator_; - std::unique_ptr loader_; - Api::ApiPtr api_; ProtobufWkt::Struct base_; }; -TEST_F(LoaderImplTest, All) { +TEST_F(DiskLoaderImplTest, All) { setup(); run("test/common/runtime/test_data/current", "envoy_override"); @@ -138,10 +151,16 @@ TEST_F(LoaderImplTest, All) { EXPECT_EQ(2UL, loader_->snapshot().getInteger("file3", 1)); EXPECT_EQ(123UL, loader_->snapshot().getInteger("file4", 1)); - // Boolean getting. bool value; SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); + // Validate that the layer name is set properly for static layers. + EXPECT_EQ("base", snapshot->getLayers()[0]->name()); + EXPECT_EQ("root", snapshot->getLayers()[1]->name()); + EXPECT_EQ("override", snapshot->getLayers()[2]->name()); + EXPECT_EQ("admin", snapshot->getLayers()[3]->name()); + + // Boolean getting. EXPECT_EQ(true, snapshot->getBoolean("file11", value)); EXPECT_EQ(true, value); EXPECT_EQ(true, snapshot->getBoolean("file12", value)); @@ -222,7 +241,7 @@ TEST_F(LoaderImplTest, All) { EXPECT_EQ(4, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); } -TEST_F(LoaderImplTest, GetLayers) { +TEST_F(DiskLoaderImplTest, GetLayers) { base_ = TestUtility::parseYaml(R"EOF( foo: whatevs )EOF"); @@ -246,7 +265,7 @@ TEST_F(LoaderImplTest, GetLayers) { EXPECT_EQ(2, store_.counter("runtime.load_success").value()); } -TEST_F(LoaderImplTest, BadDirectory) { +TEST_F(DiskLoaderImplTest, BadDirectory) { setup(); run("/baddir", "/baddir"); EXPECT_EQ(0, store_.counter("runtime.load_error").value()); @@ -257,7 +276,7 @@ TEST_F(LoaderImplTest, BadDirectory) { } // Validate that an error in a layer will results in appropriate stats tracking. -TEST_F(LoaderImplTest, DiskLayerFailure) { +TEST_F(DiskLoaderImplTest, DiskLayerFailure) { setup(); // Symlink loopy configuration will result in an error. run("test/common/runtime/test_data", "loop"); @@ -268,7 +287,7 @@ TEST_F(LoaderImplTest, DiskLayerFailure) { EXPECT_EQ(1, store_.counter("runtime.override_dir_not_exists").value()); } -TEST_F(LoaderImplTest, OverrideFolderDoesNotExist) { +TEST_F(DiskLoaderImplTest, OverrideFolderDoesNotExist) { setup(); run("test/common/runtime/test_data/current", "envoy_override_does_not_exist"); @@ -280,7 +299,7 @@ TEST_F(LoaderImplTest, OverrideFolderDoesNotExist) { EXPECT_EQ(1, store_.counter("runtime.override_dir_not_exists").value()); } -TEST_F(LoaderImplTest, PercentHandling) { +TEST_F(DiskLoaderImplTest, PercentHandling) { setup(); run("test/common/runtime/test_data/current", "envoy_override"); @@ -347,7 +366,7 @@ void testNewOverrides(Loader& loader, Stats::Store& store) { EXPECT_EQ(0, admin_overrides_active.value()); } -TEST_F(LoaderImplTest, MergeValues) { +TEST_F(DiskLoaderImplTest, MergeValues) { setup(); run("test/common/runtime/test_data/current", "envoy_override"); testNewOverrides(*loader_, store_); @@ -391,7 +410,7 @@ TEST_F(LoaderImplTest, MergeValues) { } // Validate that admin overrides disk, disk overrides bootstrap. -TEST_F(LoaderImplTest, LayersOverride) { +TEST_F(DiskLoaderImplTest, LayersOverride) { base_ = TestUtility::parseYaml(R"EOF( some: thing other: thang @@ -422,31 +441,50 @@ TEST_F(LoaderImplTest, LayersOverride) { } // Validate that multiple admin layers leads to a configuration load failure. -TEST_F(LoaderImplTest, MultipleAdminLayersFail) { +TEST_F(DiskLoaderImplTest, MultipleAdminLayersFail) { setup(); envoy::config::bootstrap::v2::LayeredRuntime layered_runtime; - layered_runtime.add_layers()->mutable_admin_layer(); - layered_runtime.add_layers()->mutable_admin_layer(); + { + auto* layer = layered_runtime.add_layers(); + layer->set_name("admin_0"); + layer->mutable_admin_layer(); + } + { + auto* layer = layered_runtime.add_layers(); + layer->set_name("admin_1"); + layer->mutable_admin_layer(); + } EXPECT_THROW_WITH_MESSAGE( - std::make_unique(dispatcher_, tls_, layered_runtime, "", store_, generator_, - *api_), + std::make_unique(dispatcher_, tls_, layered_runtime, local_info_, init_manager_, + store_, generator_, validation_visitor_, *api_), EnvoyException, "Too many admin layers specified in LayeredRuntime, at most one may be specified"); } -class DisklessLoaderImplTest : public LoaderImplTest { +class StaticLoaderImplTest : public LoaderImplTest { protected: void setup() override { LoaderImplTest::setup(); envoy::config::bootstrap::v2::LayeredRuntime layered_runtime; - layered_runtime.add_layers()->mutable_static_layer()->MergeFrom(base_); - layered_runtime.add_layers()->mutable_admin_layer(); - loader_ = std::make_unique(dispatcher_, tls_, layered_runtime, "", store_, - generator_, *api_); + { + auto* layer = layered_runtime.add_layers(); + layer->set_name("base"); + layer->mutable_static_layer()->MergeFrom(base_); + } + { + auto* layer = layered_runtime.add_layers(); + layer->set_name("admin"); + layer->mutable_admin_layer(); + } + loader_ = + std::make_unique(dispatcher_, tls_, layered_runtime, local_info_, init_manager_, + store_, generator_, validation_visitor_, *api_); } + + ProtobufWkt::Struct base_; }; -TEST_F(DisklessLoaderImplTest, All) { +TEST_F(StaticLoaderImplTest, All) { setup(); EXPECT_EQ("", loader_->snapshot().get("foo")); EXPECT_EQ(1UL, loader_->snapshot().getInteger("foo", 1)); @@ -456,7 +494,7 @@ TEST_F(DisklessLoaderImplTest, All) { } // Validate proto parsing sanity. -TEST_F(DisklessLoaderImplTest, ProtoParsing) { +TEST_F(StaticLoaderImplTest, ProtoParsing) { base_ = TestUtility::parseYaml(R"EOF( file1: hello override file2: world @@ -611,6 +649,282 @@ TEST(NoRuntime, FeatureEnabled) { EXPECT_EQ(true, runtimeFeatureEnabled("envoy.reloadable_features.test_feature_true")); } +// Test RTDS layer(s). +class RtdsLoaderImplTest : public LoaderImplTest { +public: + void setup() override { + LoaderImplTest::setup(); + + envoy::config::bootstrap::v2::LayeredRuntime config; + *config.add_layers()->mutable_static_layer() = + TestUtility::parseYaml(R"EOF( + foo: whatevs + bar: yar + )EOF"); + for (const auto& layer_resource_name : layers_) { + auto* layer = config.add_layers(); + layer->set_name(layer_resource_name); + auto* rtds_layer = layer->mutable_rtds_layer(); + rtds_layer->set_name(layer_resource_name); + rtds_layer->mutable_rtds_config(); + } + EXPECT_CALL(cm_, subscriptionFactory()).Times(layers_.size()); + EXPECT_CALL(init_manager_, add(_)).WillRepeatedly(Invoke([this](const Init::Target& target) { + init_target_handles_.emplace_back(target.createHandle("test")); + })); + ON_CALL(cm_.subscription_factory_, subscriptionFromConfigSource(_, _, _, _)) + .WillByDefault(testing::Invoke( + [this](const envoy::api::v2::core::ConfigSource&, absl::string_view, Stats::Scope&, + Config::SubscriptionCallbacks& callbacks) -> Config::SubscriptionPtr { + auto ret = std::make_unique>(); + rtds_subscriptions_.push_back(ret.get()); + rtds_callbacks_.push_back(&callbacks); + return ret; + })); + loader_ = std::make_unique(dispatcher_, tls_, config, local_info_, init_manager_, + store_, generator_, validation_visitor_, *api_); + loader_->initialize(cm_); + for (auto* sub : rtds_subscriptions_) { + EXPECT_CALL(*sub, start(_)); + } + for (auto& handle : init_target_handles_) { + handle->initialize(init_watcher_); + } + + // Validate that the layer name is set properly for dynamic layers. + EXPECT_EQ(layers_[0], loader_->snapshot().getLayers()[1]->name()); + + EXPECT_EQ("whatevs", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("", loader_->snapshot().get("baz")); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(1, store_.counter("runtime.load_success").value()); + EXPECT_EQ(2, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(1 + layers_.size(), + store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); + } + + void addLayer(absl::string_view name) { layers_.emplace_back(name); } + + void doOnConfigUpdateVerifyNoThrow(const envoy::service::discovery::v2::Runtime& runtime, + uint32_t callback_index = 0) { + Protobuf::RepeatedPtrField resources; + resources.Add()->PackFrom(runtime); + VERBOSE_EXPECT_NO_THROW(rtds_callbacks_[callback_index]->onConfigUpdate(resources, "")); + } + + void doDeltaOnConfigUpdateVerifyNoThrow(const envoy::service::discovery::v2::Runtime& runtime) { + Protobuf::RepeatedPtrField resources; + auto* resource = resources.Add(); + resource->mutable_resource()->PackFrom(runtime); + resource->set_version(""); + VERBOSE_EXPECT_NO_THROW(rtds_callbacks_[0]->onConfigUpdate(resources, {}, "")); + } + + std::vector layers_{"some_resource"}; + std::vector rtds_callbacks_; + std::vector rtds_subscriptions_; + Init::ExpectableWatcherImpl init_watcher_; + std::vector init_target_handles_; +}; + +// Empty resource lists are rejected. +TEST_F(RtdsLoaderImplTest, UnexpectedSizeEmpty) { + setup(); + + Protobuf::RepeatedPtrField runtimes; + + EXPECT_CALL(init_watcher_, ready()); + EXPECT_THROW_WITH_MESSAGE(rtds_callbacks_[0]->onConfigUpdate(runtimes, ""), EnvoyException, + "Unexpected RTDS resource length: 0"); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(1, store_.counter("runtime.load_success").value()); + EXPECT_EQ(2, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); +} + +// > 1 length lists are rejected. +TEST_F(RtdsLoaderImplTest, UnexpectedSizeTooMany) { + setup(); + + Protobuf::RepeatedPtrField runtimes; + runtimes.Add(); + runtimes.Add(); + + EXPECT_CALL(init_watcher_, ready()); + EXPECT_THROW_WITH_MESSAGE(rtds_callbacks_[0]->onConfigUpdate(runtimes, ""), EnvoyException, + "Unexpected RTDS resource length: 2"); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(1, store_.counter("runtime.load_success").value()); + EXPECT_EQ(2, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); +} + +// Validate behavior when the config fails delivery at the subscription level. +TEST_F(RtdsLoaderImplTest, FailureSubscription) { + setup(); + + EXPECT_CALL(init_watcher_, ready()); + rtds_callbacks_[0]->onConfigUpdateFailed({}); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(1, store_.counter("runtime.load_success").value()); + EXPECT_EQ(2, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); +} + +// Unexpected runtime resource name. +TEST_F(RtdsLoaderImplTest, WrongResourceName) { + setup(); + + auto runtime = TestUtility::parseYaml(R"EOF( + name: other_resource + layer: + foo: bar + baz: meh + )EOF"); + Protobuf::RepeatedPtrField resources; + resources.Add()->PackFrom(runtime); + EXPECT_THROW_WITH_MESSAGE(rtds_callbacks_[0]->onConfigUpdate(resources, ""), EnvoyException, + "Unexpected RTDS runtime (expecting some_resource): other_resource"); + + EXPECT_EQ("whatevs", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("", loader_->snapshot().get("baz")); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(1, store_.counter("runtime.load_success").value()); + EXPECT_EQ(2, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); +} + +// Successful update. +TEST_F(RtdsLoaderImplTest, OnConfigUpdateSuccess) { + setup(); + + auto runtime = TestUtility::parseYaml(R"EOF( + name: some_resource + layer: + foo: bar + baz: meh + )EOF"); + EXPECT_CALL(init_watcher_, ready()); + doOnConfigUpdateVerifyNoThrow(runtime); + + EXPECT_EQ("bar", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("meh", loader_->snapshot().get("baz")); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(2, store_.counter("runtime.load_success").value()); + EXPECT_EQ(3, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); + + runtime = TestUtility::parseYaml(R"EOF( + name: some_resource + layer: + baz: saz + )EOF"); + doOnConfigUpdateVerifyNoThrow(runtime); + + EXPECT_EQ("whatevs", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("saz", loader_->snapshot().get("baz")); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(3, store_.counter("runtime.load_success").value()); + EXPECT_EQ(3, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); +} + +// Delta style successful update. +TEST_F(RtdsLoaderImplTest, DeltaOnConfigUpdateSuccess) { + setup(); + + auto runtime = TestUtility::parseYaml(R"EOF( + name: some_resource + layer: + foo: bar + baz: meh + )EOF"); + EXPECT_CALL(init_watcher_, ready()); + doDeltaOnConfigUpdateVerifyNoThrow(runtime); + + EXPECT_EQ("bar", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("meh", loader_->snapshot().get("baz")); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(2, store_.counter("runtime.load_success").value()); + EXPECT_EQ(3, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); + + runtime = TestUtility::parseYaml(R"EOF( + name: some_resource + layer: + baz: saz + )EOF"); + doDeltaOnConfigUpdateVerifyNoThrow(runtime); + + EXPECT_EQ("whatevs", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("saz", loader_->snapshot().get("baz")); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(3, store_.counter("runtime.load_success").value()); + EXPECT_EQ(3, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); +} + +// Updates with multiple RTDS layers. +TEST_F(RtdsLoaderImplTest, MultipleRtdsLayers) { + addLayer("another_resource"); + setup(); + + EXPECT_EQ("whatevs", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("", loader_->snapshot().get("baz")); + + auto runtime = TestUtility::parseYaml(R"EOF( + name: some_resource + layer: + foo: bar + baz: meh + )EOF"); + EXPECT_CALL(init_watcher_, ready()).Times(2); + doOnConfigUpdateVerifyNoThrow(runtime, 0); + + EXPECT_EQ("bar", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("meh", loader_->snapshot().get("baz")); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(2, store_.counter("runtime.load_success").value()); + EXPECT_EQ(3, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(3, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); + + runtime = TestUtility::parseYaml(R"EOF( + name: another_resource + layer: + baz: saz + )EOF"); + doOnConfigUpdateVerifyNoThrow(runtime, 1); + + // Unlike in OnConfigUpdateSuccess, foo latches onto bar as the some_resource + // layer still applies. + EXPECT_EQ("bar", loader_->snapshot().get("foo")); + EXPECT_EQ("yar", loader_->snapshot().get("bar")); + EXPECT_EQ("saz", loader_->snapshot().get("baz")); + + EXPECT_EQ(0, store_.counter("runtime.load_error").value()); + EXPECT_EQ(3, store_.counter("runtime.load_success").value()); + EXPECT_EQ(3, store_.gauge("runtime.num_keys", Stats::Gauge::ImportMode::NeverImport).value()); + EXPECT_EQ(3, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); +} + } // namespace } // namespace Runtime } // namespace Envoy diff --git a/test/integration/BUILD b/test/integration/BUILD index 35aa1046dec1..cc8752ebabc0 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -598,6 +598,16 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "rtds_integration_test", + srcs = ["rtds_integration_test.cc"], + deps = [ + ":http_integration_lib", + "//test/common/grpc:grpc_client_integration_lib", + "@envoy_api//envoy/service/discovery/v2:rtds_cc", + ], +) + envoy_cc_test_library( name = "server_stats_interface", hdrs = ["server_stats.h"], diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc new file mode 100644 index 000000000000..63122162e184 --- /dev/null +++ b/test/integration/rtds_integration_test.cc @@ -0,0 +1,164 @@ +#include "test/common/grpc/grpc_client_integration.h" +#include "test/integration/http_integration.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace { + +std::string tdsBootstrapConfig(absl::string_view api_type) { + return fmt::format(R"EOF( +static_resources: + clusters: + - name: dummy_cluster + hosts: + socket_address: + address: 127.0.0.1 + port_value: 0 + - name: rtds_cluster + http2_protocol_options: {{}} + hosts: + socket_address: + address: 127.0.0.1 + port_value: 0 +layered_runtime: + layers: + - name: some_static_layer + static_layer: + foo: whatevs + bar: yar + - name: some_rtds_layer + rtds_layer: + name: some_rtds_layer + rtds_config: + api_config_source: + api_type: {} + grpc_services: + envoy_grpc: + cluster_name: rtds_cluster + - name: some_admin_layer + admin_layer: {{}} +admin: + access_log_path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 0 +)EOF", + api_type); +} + +class RtdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { +public: + RtdsIntegrationTest() + : HttpIntegrationTest( + Http::CodecClient::Type::HTTP2, ipVersion(), + tdsBootstrapConfig(sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { + use_lds_ = false; + create_xds_upstream_ = true; + sotw_or_delta_ = sotwOrDelta(); + } + + void TearDown() override { + cleanUpXdsConnection(); + test_server_.reset(); + fake_upstreams_.clear(); + } + + void initialize() override { + // The tests infra expects the xDS server to be the second fake upstream, so + // we need a dummy data plane cluster. + setUpstreamCount(1); + setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + HttpIntegrationTest::initialize(); + // Initial RTDS connection. + createXdsConnection(); + AssertionResult result = xds_connection_->waitForNewStream(*dispatcher_, xds_stream_); + RELEASE_ASSERT(result, result.message()); + xds_stream_->startGrpcStream(); + // Register admin port. + registerTestServerPorts({}); + initial_load_success_ = test_server_->counter("runtime.load_success")->value(); + initial_keys_ = test_server_->gauge("runtime.num_keys")->value(); + } + + void acceptXdsConnection() { + AssertionResult result = // xds_connection_ is filled with the new FakeHttpConnection. + fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, xds_connection_); + RELEASE_ASSERT(result, result.message()); + result = xds_connection_->waitForNewStream(*dispatcher_, xds_stream_); + RELEASE_ASSERT(result, result.message()); + xds_stream_->startGrpcStream(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + } + + std::string getRuntimeKey(const std::string& key) { + auto response = IntegrationUtil::makeSingleRequest( + lookupPort("admin"), "GET", "/runtime?format=json", "", downstreamProtocol(), version_); + EXPECT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + Json::ObjectSharedPtr loader = TestEnvironment::jsonLoadFromString(response->body()); + auto entries = loader->getObject("entries"); + if (entries->hasObject(key)) { + return entries->getObject(key)->getString("final_value"); + } + return ""; + } + + uint32_t initial_load_success_{}; + uint32_t initial_keys_{}; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, RtdsIntegrationTest, DELTA_INTEGRATION_PARAMS); + +TEST_P(RtdsIntegrationTest, RtdsReload) { + initialize(); + + EXPECT_EQ("whatevs", getRuntimeKey("foo")); + EXPECT_EQ("yar", getRuntimeKey("bar")); + EXPECT_EQ("", getRuntimeKey("baz")); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "", {"some_rtds_layer"}, + {"some_rtds_layer"}, {})); + auto some_rtds_layer = TestUtility::parseYaml(R"EOF( + name: some_rtds_layer + layer: + foo: bar + baz: meh + )EOF"); + sendDiscoveryResponse( + Config::TypeUrl::get().Runtime, {some_rtds_layer}, {some_rtds_layer}, {}, "1"); + test_server_->waitForCounterGe("runtime.load_success", initial_load_success_ + 1); + + EXPECT_EQ("bar", getRuntimeKey("foo")); + EXPECT_EQ("yar", getRuntimeKey("bar")); + EXPECT_EQ("meh", getRuntimeKey("baz")); + + EXPECT_EQ(0, test_server_->counter("runtime.load_error")->value()); + EXPECT_EQ(initial_load_success_ + 1, test_server_->counter("runtime.load_success")->value()); + EXPECT_EQ(initial_keys_ + 1, test_server_->gauge("runtime.num_keys")->value()); + EXPECT_EQ(3, test_server_->gauge("runtime.num_layers")->value()); + + EXPECT_TRUE( + compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "1", {"some_rtds_layer"}, {}, {})); + some_rtds_layer = TestUtility::parseYaml(R"EOF( + name: some_rtds_layer + layer: + baz: saz + )EOF"); + sendDiscoveryResponse( + Config::TypeUrl::get().Runtime, {some_rtds_layer}, {some_rtds_layer}, {}, "2"); + test_server_->waitForCounterGe("runtime.load_success", initial_load_success_ + 2); + + EXPECT_EQ("whatevs", getRuntimeKey("foo")); + EXPECT_EQ("yar", getRuntimeKey("bar")); + EXPECT_EQ("saz", getRuntimeKey("baz")); + + EXPECT_EQ(0, test_server_->counter("runtime.load_error")->value()); + EXPECT_EQ(initial_load_success_ + 2, test_server_->counter("runtime.load_success")->value()); + EXPECT_EQ(initial_keys_ + 1, test_server_->gauge("runtime.num_keys")->value()); + EXPECT_EQ(3, test_server_->gauge("runtime.num_layers")->value()); +} + +} // namespace +} // namespace Envoy diff --git a/test/mocks/runtime/BUILD b/test/mocks/runtime/BUILD index e373cb891685..7aed549bc387 100644 --- a/test/mocks/runtime/BUILD +++ b/test/mocks/runtime/BUILD @@ -15,6 +15,7 @@ envoy_cc_mock( external_deps = ["abseil_optional"], deps = [ "//include/envoy/runtime:runtime_interface", + "//include/envoy/upstream:cluster_manager_interface", "//test/mocks:common_lib", ], ) diff --git a/test/mocks/runtime/mocks.h b/test/mocks/runtime/mocks.h index 76bac6b3c392..e67e34361958 100644 --- a/test/mocks/runtime/mocks.h +++ b/test/mocks/runtime/mocks.h @@ -5,6 +5,7 @@ #include #include "envoy/runtime/runtime.h" +#include "envoy/upstream/cluster_manager.h" #include "gmock/gmock.h" @@ -61,6 +62,7 @@ class MockLoader : public Loader { MockLoader(); ~MockLoader(); + MOCK_METHOD1(initialize, void(Upstream::ClusterManager& cm)); MOCK_METHOD0(snapshot, Snapshot&()); MOCK_METHOD1(mergeValues, void(const std::unordered_map&)); diff --git a/test/server/BUILD b/test/server/BUILD index 4cfa5be816ee..f37d8ff8bc7e 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -217,6 +217,8 @@ envoy_cc_test( ":cluster_health_check_bootstrap.yaml", ":empty_bootstrap.yaml", ":invalid_bootstrap.yaml", + ":invalid_layered_runtime_duplicate_name.yaml", + ":invalid_layered_runtime_missing_name.yaml", ":invalid_runtime_bootstrap.yaml", ":node_bootstrap.yaml", ":node_bootstrap_no_admin_port.yaml", diff --git a/test/server/configuration_impl_test.cc b/test/server/configuration_impl_test.cc index 702363379c59..13fc76644559 100644 --- a/test/server/configuration_impl_test.cc +++ b/test/server/configuration_impl_test.cc @@ -375,12 +375,16 @@ TEST(InitialImplTest, LayeredRuntime) { const std::string yaml = R"EOF( layered_runtime: layers: - - static_layer: + - name: base + static_layer: health_check: min_interval: 5 - - disk_layer: { symlink_root: /srv/runtime/current/envoy } - - disk_layer: { symlink_root: /srv/runtime/current/envoy_override, append_service_cluster: true } - - admin_layer: {} + - name: root + disk_layer: { symlink_root: /srv/runtime/current/envoy } + - name: override + disk_layer: { symlink_root: /srv/runtime/current/envoy_override, append_service_cluster: true } + - name: admin + admin_layer: {} )EOF"; const auto bootstrap = TestUtility::parseYaml(yaml); InitialImpl config(bootstrap); @@ -412,8 +416,10 @@ TEST(InitialImplTest, EmptyDeprecatedRuntime) { const std::string expected_yaml = R"EOF( layers: - - static_layer: {} - - admin_layer: {} + - name: base + static_layer: {} + - name: admin + admin_layer: {} )EOF"; const auto expected_runtime = TestUtility::parseYaml(expected_yaml); @@ -437,14 +443,16 @@ TEST(InitialImplTest, DeprecatedRuntimeTranslation) { const std::string expected_yaml = R"EOF( layers: - - static_layer: + - name: base + static_layer: health_check: min_interval: 5 - name: root disk_layer: { symlink_root: /srv/runtime/current/envoy } - name: override disk_layer: { symlink_root: /srv/runtime/current/envoy_override, append_service_cluster: true } - - admin_layer: {} + - name: admin + admin_layer: {} )EOF"; const auto expected_runtime = TestUtility::parseYaml(expected_yaml); diff --git a/test/server/invalid_layered_runtime_duplicate_name.yaml b/test/server/invalid_layered_runtime_duplicate_name.yaml new file mode 100644 index 000000000000..9115d8a0c382 --- /dev/null +++ b/test/server/invalid_layered_runtime_duplicate_name.yaml @@ -0,0 +1,8 @@ +layered_runtime: + layers: + - name: some_static_layer + static_layer: + foo: bar + - name: some_static_layer + static_layer: + foo: baz diff --git a/test/server/invalid_layered_runtime_missing_name.yaml b/test/server/invalid_layered_runtime_missing_name.yaml new file mode 100644 index 000000000000..ec91562e9f28 --- /dev/null +++ b/test/server/invalid_layered_runtime_missing_name.yaml @@ -0,0 +1,4 @@ +layered_runtime: + layers: + - static_layer: + foo: bar diff --git a/test/server/runtime_bootstrap.yaml b/test/server/runtime_bootstrap.yaml index a69a176aa51e..e92c3fd5a903 100644 --- a/test/server/runtime_bootstrap.yaml +++ b/test/server/runtime_bootstrap.yaml @@ -1,6 +1,9 @@ layered_runtime: layers: - - static_layer: + - name: some_static_layer + static_layer: foo: bar - - disk_layer: { symlink_root: {{ test_rundir }}/test/server/test_data/runtime/primary } - - disk_layer: { symlink_root: {{ test_rundir }}/test/server/test_data/runtime/override, append_service_cluster: true } + - name: base_disk_layer + disk_layer: { symlink_root: {{ test_rundir }}/test/server/test_data/runtime/primary } + - name: overlay_disk_layer + disk_layer: { symlink_root: {{ test_rundir }}/test/server/test_data/runtime/override, append_service_cluster: true } diff --git a/test/server/server_test.cc b/test/server/server_test.cc index d18bf0a56c62..b91ab2bac861 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -362,6 +362,19 @@ TEST_P(ServerInstanceImplTest, InvalidBootstrapRuntime) { EnvoyException, "Invalid runtime entry value for foo"); } +// Validate invalid layered runtime missing a name is rejected. +TEST_P(ServerInstanceImplTest, InvalidLayeredBootstrapMissingName) { + EXPECT_THROW_WITH_REGEX(initialize("test/server/invalid_layered_runtime_missing_name.yaml"), + EnvoyException, + "RuntimeLayerValidationError.Name: \\[\"value length must be at least"); +} + +// Validate invalid layered runtime with duplicate names is rejected. +TEST_P(ServerInstanceImplTest, InvalidLayeredBootstrapDuplicateName) { + EXPECT_THROW_WITH_REGEX(initialize("test/server/invalid_layered_runtime_duplicate_name.yaml"), + EnvoyException, "Duplicate layer name: some_static_laye"); +} + // Regression test for segfault when server initialization fails prior to // ClusterManager initialization. TEST_P(ServerInstanceImplTest, BootstrapClusterManagerInitializationFail) { diff --git a/test/test_common/BUILD b/test/test_common/BUILD index a25726bfc7ab..c1d4c4f692bf 100644 --- a/test/test_common/BUILD +++ b/test/test_common/BUILD @@ -113,6 +113,7 @@ envoy_cc_test_library( "//source/common/stats:stats_lib", "//test/mocks/stats:stats_mocks", "@envoy_api//envoy/config/bootstrap/v2:bootstrap_cc", + "@envoy_api//envoy/service/discovery/v2:rtds_cc", ], ) diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 0211691228c5..9ea731e47ac3 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -26,6 +26,7 @@ #include "envoy/api/v2/route/route.pb.h" #include "envoy/buffer/buffer.h" #include "envoy/http/codec.h" +#include "envoy/service/discovery/v2/rtds.pb.h" #include "common/api/api_impl.h" #include "common/common/empty_string.h" @@ -194,6 +195,9 @@ std::string TestUtility::xdsResourceName(const ProtobufWkt::Any& resource) { if (resource.type_url() == Config::TypeUrl::get().VirtualHost) { return TestUtility::anyConvert(resource).name(); } + if (resource.type_url() == Config::TypeUrl::get().Runtime) { + return TestUtility::anyConvert(resource).name(); + } throw EnvoyException( fmt::format("xdsResourceName does not know about type URL {}", resource.type_url())); } diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index b18242af6237..e1a53572eccf 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -213,6 +213,7 @@ RNG RPC RSA RST +RTDS RTTI RW RX @@ -248,7 +249,6 @@ TBD TCLAP TCMalloc TCP -TDS TE TFO TID From 139cca5f6f150e4adeed235ac7ab6f9ae5f844ba Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Tue, 18 Jun 2019 20:19:01 -0700 Subject: [PATCH 030/542] clang-tidy fixes (#7318) Signed-off-by: Derek Argueta --- .clang-tidy | 2 +- include/envoy/event/timer.h | 2 +- include/envoy/http/header_map.h | 9 ++-- include/envoy/registry/registry.h | 2 +- include/envoy/stats/timespan.h | 2 +- include/envoy/upstream/types.h | 3 +- .../common/access_log/access_log_formatter.cc | 10 ++-- source/common/api/os_sys_calls_impl.cc | 3 +- .../api/os_sys_calls_impl_hot_restart.cc | 2 +- source/common/api/os_sys_calls_impl_linux.cc | 3 +- source/common/buffer/buffer_impl.h | 2 +- .../buffer/zero_copy_input_stream_impl.h | 8 +-- source/common/common/assert.cc | 2 +- source/common/common/assert.h | 2 +- source/common/common/empty_string.h | 2 +- source/common/common/linked_object.h | 6 +-- source/common/common/logger.h | 4 +- source/common/common/matchers.h | 2 +- source/common/common/non_copyable.h | 2 +- source/common/common/perf_annotation.cc | 4 +- source/common/common/scalar_to_byte_vector.h | 3 +- source/common/common/stack_array.h | 2 +- source/common/common/utility.h | 2 +- source/common/config/metadata.cc | 2 +- source/common/config/well_known_names.cc | 36 ++++++------- source/common/event/file_event_impl.cc | 2 +- source/common/event/libevent.cc | 2 +- source/common/filesystem/file_shared_impl.h | 2 +- .../common/filesystem/inotify/watcher_impl.cc | 2 +- .../common/filesystem/inotify/watcher_impl.h | 2 +- .../posix/directory_iterator_impl.cc | 2 +- .../common/filesystem/posix/filesystem_impl.h | 2 +- source/common/http/async_client_impl.h | 4 +- source/common/http/codec_client.cc | 2 +- source/common/http/conn_manager_impl.h | 2 +- source/common/http/conn_pool_base.h | 2 +- source/common/http/http1/codec_impl.cc | 2 +- source/common/http/http1/conn_pool.h | 6 +-- source/common/http/http2/codec_impl.cc | 2 +- source/common/http/http2/codec_impl.h | 4 +- source/common/http/rest_api_fetcher.h | 2 +- source/common/network/address_impl.cc | 2 +- source/common/network/cidr_range.h | 2 +- source/common/network/connection_impl.h | 2 +- .../common/network/io_socket_handle_impl.cc | 4 +- source/common/network/lc_trie.h | 2 +- source/common/network/udp_listener_impl.h | 2 +- source/common/network/utility.cc | 6 +-- source/common/router/config_impl.cc | 2 +- source/common/stats/histogram_impl.h | 2 +- .../upstream/health_checker_base_impl.h | 2 +- source/common/upstream/health_checker_impl.h | 6 +-- source/common/upstream/logical_dns_cluster.h | 2 +- .../upstream/priority_conn_pool_map_impl.h | 2 +- source/common/upstream/subset_lb.h | 4 +- source/exe/signal_action.cc | 3 +- .../clusters/redis/redis_cluster.cc | 2 +- .../extensions/clusters/redis/redis_cluster.h | 4 +- .../common/tap/extension_config_base.h | 2 +- .../header_to_metadata_filter.cc | 2 +- .../filters/http/squash/squash_filter.h | 2 +- .../filters/network/mongo_proxy/codec_impl.h | 4 +- .../network/redis_proxy/conn_pool_impl.cc | 4 +- .../thrift_proxy/binary_protocol_impl.h | 4 +- .../network/thrift_proxy/router/router_impl.h | 4 +- .../file_based_metadata/config.h | 2 +- .../quiche/envoy_quic_alarm_factory.h | 2 +- .../common/ot/opentracing_driver_impl.cc | 2 +- source/server/backtrace.h | 2 +- source/server/config_validation/connection.h | 2 +- source/server/connection_handler_impl.h | 6 +-- source/server/guarddog_impl.h | 2 +- source/server/hot_restart_impl.cc | 2 +- source/server/hot_restart_nop_impl.h | 4 +- source/server/server.h | 2 +- test/common/grpc/google_grpc_creds_test.cc | 2 +- test/common/http/conn_manager_impl_test.cc | 18 +++---- test/common/router/router_test.cc | 2 +- test/common/stats/thread_local_store_test.cc | 2 +- .../filters/common/ext_authz/mocks.cc | 8 +-- .../filters/common/ratelimit/mocks.cc | 4 +- .../http/dynamo/dynamo_request_parser_test.cc | 6 +-- .../grpc_json_transcoder_integration_test.cc | 2 +- .../filters/http/lua/wrappers_test.cc | 4 +- .../squash/squash_filter_integration_test.cc | 4 +- .../filters/network/common/redis/mocks.cc | 16 +++--- .../filters/network/dubbo_proxy/mocks.cc | 22 ++++---- .../network/mysql_proxy/mysql_codec_test.cc | 2 +- .../network/mysql_proxy/mysql_filter_test.cc | 2 +- .../thrift_proxy/auto_protocol_impl_test.cc | 8 +-- .../stats_sinks/statsd/config_test.cc | 4 +- test/integration/header_integration_test.cc | 8 +-- test/integration/integration.cc | 2 +- test/integration/integration_admin_test.cc | 2 +- test/integration/protocol_integration_test.cc | 2 +- test/integration/stats_integration_test.cc | 2 +- test/integration/uds_integration_test.cc | 2 +- test/mocks/filesystem/mocks.cc | 8 +-- test/mocks/filesystem/mocks.h | 6 +-- test/mocks/http/mocks.cc | 50 +++++++++---------- test/mocks/http/stream.h | 2 +- test/mocks/http/stream_decoder.h | 2 +- test/mocks/stats/mocks.cc | 24 ++++----- test/server/listener_manager_impl_test.cc | 2 +- test/test_common/environment.cc | 14 +++--- test/test_common/test_time.cc | 2 +- test/test_common/thread_factory_for_test.cc | 4 +- test/test_common/utility.cc | 7 ++- test/test_common/utility.h | 5 +- 109 files changed, 248 insertions(+), 256 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index a62ee3c94414..870690b2d183 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,7 +1,7 @@ Checks: 'clang-diagnostic-*,clang-analyzer-*,abseil-*,bugprone-*,modernize-*,performance-*,readability-redundant-*,readability-braces-around-statements,readability-container-size-empty' #TODO(lizan): grow this list, fix possible warnings and make more checks as error -WarningsAsErrors: 'bugprone-assert-side-effect,modernize-make-shared,modernize-make-unique,readability-redundant-smartptr-get,readability-braces-around-statements,readability-redundant-string-cstr,bugprone-use-after-move,readability-container-size-empty' +WarningsAsErrors: 'bugprone-assert-side-effect,modernize-make-shared,modernize-make-unique,readability-redundant-smartptr-get,readability-braces-around-statements,readability-redundant-string-cstr,bugprone-use-after-move,readability-container-size-empty,performance-faster-string-find' CheckOptions: - key: bugprone-assert-side-effect.AssertMacros diff --git a/include/envoy/event/timer.h b/include/envoy/event/timer.h index 8166a1308ad8..05fc533ac235 100644 --- a/include/envoy/event/timer.h +++ b/include/envoy/event/timer.h @@ -58,7 +58,7 @@ using SchedulerPtr = std::unique_ptr; */ class TimeSystem : public TimeSource { public: - virtual ~TimeSystem() = default; + ~TimeSystem() override = default; using Duration = MonotonicTime::duration; diff --git a/include/envoy/http/header_map.h b/include/envoy/http/header_map.h index ee202a2fccc2..a9c0ea95058f 100644 --- a/include/envoy/http/header_map.h +++ b/include/envoy/http/header_map.h @@ -1,9 +1,8 @@ #pragma once -#include - #include #include +#include #include #include #include @@ -133,9 +132,7 @@ class HeaderString { * * @return an absl::string_view. */ - absl::string_view getStringView() const { - return absl::string_view(buffer_.ref_, string_length_); - } + absl::string_view getStringView() const { return {buffer_.ref_, string_length_}; } /** * Return the string to a default state. Reference strings are not touched. Both inline/dynamic @@ -478,7 +475,7 @@ class HeaderMap { * @param context supplies the context passed to iterate(). * @return Iterate::Continue to continue iteration. */ - typedef Iterate (*ConstIterateCb)(const HeaderEntry& header, void* context); + using ConstIterateCb = Iterate (*)(const HeaderEntry&, void*); /** * Iterate over a constant header map. diff --git a/include/envoy/registry/registry.h b/include/envoy/registry/registry.h index 201434e47787..37cd4f2f9f21 100644 --- a/include/envoy/registry/registry.h +++ b/include/envoy/registry/registry.h @@ -52,7 +52,7 @@ template class FactoryRegistry { * Gets the current map of factory implementations. This is an ordered map for sorting reasons. */ static std::map& factories() { - static std::map* factories = new std::map; + static auto* factories = new std::map; return *factories; } diff --git a/include/envoy/stats/timespan.h b/include/envoy/stats/timespan.h index 8fad46bfb2ce..c0c7c1b18d24 100644 --- a/include/envoy/stats/timespan.h +++ b/include/envoy/stats/timespan.h @@ -37,7 +37,7 @@ template class TimespanWithUnit : public CompletableTimespan { /** * Complete the timespan and send the time to the histogram. */ - void complete() { histogram_.recordValue(getRawDuration().count()); } + void complete() override { histogram_.recordValue(getRawDuration().count()); } /** * Get duration in the time unit since the creation of the span. diff --git a/include/envoy/upstream/types.h b/include/envoy/upstream/types.h index 322392aadff3..d9304766345a 100644 --- a/include/envoy/upstream/types.h +++ b/include/envoy/upstream/types.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include #include #include "common/common/phantom.h" diff --git a/source/common/access_log/access_log_formatter.cc b/source/common/access_log/access_log_formatter.cc index bf31ad1c16d9..47752f08efcd 100644 --- a/source/common/access_log/access_log_formatter.cc +++ b/source/common/access_log/access_log_formatter.cc @@ -238,7 +238,7 @@ std::vector AccessLogFormatParser::parse(const std::string pos += 1; int command_end_position = pos + token.length(); - if (token.find("REQ(") == 0) { + if (absl::StartsWith(token, "REQ(")) { std::string main_header, alternative_header; absl::optional max_length; @@ -246,7 +246,7 @@ std::vector AccessLogFormatParser::parse(const std::string formatters.emplace_back(FormatterProviderPtr{ new RequestHeaderFormatter(main_header, alternative_header, max_length)}); - } else if (token.find("RESP(") == 0) { + } else if (absl::StartsWith(token, "RESP(")) { std::string main_header, alternative_header; absl::optional max_length; @@ -254,7 +254,7 @@ std::vector AccessLogFormatParser::parse(const std::string formatters.emplace_back(FormatterProviderPtr{ new ResponseHeaderFormatter(main_header, alternative_header, max_length)}); - } else if (token.find("TRAILER(") == 0) { + } else if (absl::StartsWith(token, "TRAILER(")) { std::string main_header, alternative_header; absl::optional max_length; @@ -262,7 +262,7 @@ std::vector AccessLogFormatParser::parse(const std::string formatters.emplace_back(FormatterProviderPtr{ new ResponseTrailerFormatter(main_header, alternative_header, max_length)}); - } else if (token.find(DYNAMIC_META_TOKEN) == 0) { + } else if (absl::StartsWith(token, DYNAMIC_META_TOKEN)) { std::string filter_namespace; absl::optional max_length; std::vector path; @@ -271,7 +271,7 @@ std::vector AccessLogFormatParser::parse(const std::string parseCommand(token, start, ":", filter_namespace, path, max_length); formatters.emplace_back( FormatterProviderPtr{new DynamicMetadataFormatter(filter_namespace, path, max_length)}); - } else if (token.find("START_TIME") == 0) { + } else if (absl::StartsWith(token, "START_TIME")) { const size_t parameters_length = pos + StartTimeParamStart + 1; const size_t parameters_end = command_end_position - parameters_length; diff --git a/source/common/api/os_sys_calls_impl.cc b/source/common/api/os_sys_calls_impl.cc index e0fc4f6cbcae..24577fbccb94 100644 --- a/source/common/api/os_sys_calls_impl.cc +++ b/source/common/api/os_sys_calls_impl.cc @@ -1,10 +1,11 @@ #include "common/api/os_sys_calls_impl.h" -#include #include #include #include +#include + namespace Envoy { namespace Api { diff --git a/source/common/api/os_sys_calls_impl_hot_restart.cc b/source/common/api/os_sys_calls_impl_hot_restart.cc index ab0496ccecad..1df6bc57f918 100644 --- a/source/common/api/os_sys_calls_impl_hot_restart.cc +++ b/source/common/api/os_sys_calls_impl_hot_restart.cc @@ -1,6 +1,6 @@ #include "common/api/os_sys_calls_impl_hot_restart.h" -#include +#include namespace Envoy { namespace Api { diff --git a/source/common/api/os_sys_calls_impl_linux.cc b/source/common/api/os_sys_calls_impl_linux.cc index fcf2fafdc7d0..9b2d213c0816 100644 --- a/source/common/api/os_sys_calls_impl_linux.cc +++ b/source/common/api/os_sys_calls_impl_linux.cc @@ -4,9 +4,10 @@ #include "common/api/os_sys_calls_impl_linux.h" -#include #include +#include + namespace Envoy { namespace Api { diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 9251bfc8fd50..99ca54af12ff 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -507,7 +507,7 @@ class OwnedImpl : public LibEventInstance { // LibEventInstance Event::Libevent::BufferPtr& buffer() override { return buffer_; } - virtual void postProcess() override; + void postProcess() override; /** * Create a new slice at the end of the buffer, and copy the supplied content into it. diff --git a/source/common/buffer/zero_copy_input_stream_impl.h b/source/common/buffer/zero_copy_input_stream_impl.h index abffdc0560a2..96bdea0be9ea 100644 --- a/source/common/buffer/zero_copy_input_stream_impl.h +++ b/source/common/buffer/zero_copy_input_stream_impl.h @@ -34,10 +34,10 @@ class ZeroCopyInputStreamImpl : public virtual Protobuf::io::ZeroCopyInputStream // Note Next() will return true with no data until more data is available if the stream is not // finished. It is the caller's responsibility to finish the stream or wrap with // LimitingInputStream before passing to protobuf code to avoid a spin loop. - virtual bool Next(const void** data, int* size) override; - virtual void BackUp(int count) override; - virtual bool Skip(int count) override; // Not implemented - virtual ProtobufTypes::Int64 ByteCount() const override { return byte_count_; } + bool Next(const void** data, int* size) override; + void BackUp(int count) override; + bool Skip(int count) override; // Not implemented + ProtobufTypes::Int64 ByteCount() const override { return byte_count_; } protected: Buffer::InstancePtr buffer_; diff --git a/source/common/common/assert.cc b/source/common/common/assert.cc index d3e483c9410c..cb01008c121e 100644 --- a/source/common/common/assert.cc +++ b/source/common/common/assert.cc @@ -30,7 +30,7 @@ class ActionRegistrationImpl : public ActionRegistration { std::function ActionRegistrationImpl::debug_assertion_failure_record_action_; -ActionRegistrationPtr setDebugAssertionFailureRecordAction(std::function action) { +ActionRegistrationPtr setDebugAssertionFailureRecordAction(const std::function& action) { return std::make_unique(action); } diff --git a/source/common/common/assert.h b/source/common/common/assert.h index 3bd13367ac5e..5cd0e462f2bc 100644 --- a/source/common/common/assert.h +++ b/source/common/common/assert.h @@ -30,7 +30,7 @@ using ActionRegistrationPtr = std::unique_ptr; * @param action The action to take when an assertion fails. * @return A registration object. The registration is removed when the object is destructed. */ -ActionRegistrationPtr setDebugAssertionFailureRecordAction(std::function action); +ActionRegistrationPtr setDebugAssertionFailureRecordAction(const std::function& action); /** * Invokes the action set by setDebugAssertionFailureRecordAction, or does nothing if diff --git a/source/common/common/empty_string.h b/source/common/common/empty_string.h index b9355eae9ab3..9d8b9e988900 100644 --- a/source/common/common/empty_string.h +++ b/source/common/common/empty_string.h @@ -3,5 +3,5 @@ #include namespace Envoy { -static const std::string EMPTY_STRING = ""; +static const std::string EMPTY_STRING; } // namespace Envoy diff --git a/source/common/common/linked_object.h b/source/common/common/linked_object.h index ee0ab8275989..2efbc14cf6b7 100644 --- a/source/common/common/linked_object.h +++ b/source/common/common/linked_object.h @@ -76,11 +76,11 @@ template class LinkedObject { } protected: - LinkedObject() : inserted_(false) {} + LinkedObject() = default; private: typename ListType::iterator entry_; - bool inserted_; // iterators do not have any "invalid" value so we need this boolean for sanity - // checking. + bool inserted_{false}; // iterators do not have any "invalid" value so we need this boolean for + // sanity checking. }; } // namespace Envoy diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 67580a841a01..00d083bfaacc 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -74,7 +74,7 @@ class Logger { * but the method to log at err level is called LOGGER.error not LOGGER.err. All other level are * fine spdlog::info corresponds to LOGGER.info method. */ - typedef enum { + using levels = enum { trace = spdlog::level::trace, debug = spdlog::level::debug, info = spdlog::level::info, @@ -82,7 +82,7 @@ class Logger { error = spdlog::level::err, critical = spdlog::level::critical, off = spdlog::level::off - } levels; + }; spdlog::string_view_t levelString() const { return spdlog::level::level_string_views[logger_->level()]; diff --git a/source/common/common/matchers.h b/source/common/common/matchers.h index ca021f56b461..8305a3ee4b3f 100644 --- a/source/common/common/matchers.h +++ b/source/common/common/matchers.h @@ -107,7 +107,7 @@ class ListMatcher : public ValueMatcher { public: ListMatcher(const envoy::type::matcher::ListMatcher& matcher); - bool match(const ProtobufWkt::Value& value) const; + bool match(const ProtobufWkt::Value& value) const override; private: const envoy::type::matcher::ListMatcher matcher_; diff --git a/source/common/common/non_copyable.h b/source/common/common/non_copyable.h index 7c394c41de18..c248a37f48e4 100644 --- a/source/common/common/non_copyable.h +++ b/source/common/common/non_copyable.h @@ -6,7 +6,7 @@ namespace Envoy { */ class NonCopyable { protected: - NonCopyable() {} + NonCopyable() = default; private: NonCopyable(const NonCopyable&); diff --git a/source/common/common/perf_annotation.cc b/source/common/common/perf_annotation.cc index 65e496cb2415..b31eb76a5548 100644 --- a/source/common/common/perf_annotation.cc +++ b/source/common/common/perf_annotation.cc @@ -30,7 +30,7 @@ void PerfOperation::record(absl::string_view category, absl::string_view descrip // The ctor is explicitly declared private to encourage clients to use getOrCreate(), at // least for now. Given that it's declared it must be instantiated. It's not inlined // because the constructor is non-trivial due to the contained unordered_map. -PerfAnnotationContext::PerfAnnotationContext() {} +PerfAnnotationContext::PerfAnnotationContext() = default; void PerfAnnotationContext::record(std::chrono::nanoseconds duration, absl::string_view category, absl::string_view description) { @@ -146,7 +146,7 @@ void PerfAnnotationContext::clear() { } PerfAnnotationContext* PerfAnnotationContext::getOrCreate() { - static PerfAnnotationContext* context = new PerfAnnotationContext(); + static auto* context = new PerfAnnotationContext(); return context; } diff --git a/source/common/common/scalar_to_byte_vector.h b/source/common/common/scalar_to_byte_vector.h index 9db11f90e56f..4ca2654bcedc 100644 --- a/source/common/common/scalar_to_byte_vector.h +++ b/source/common/common/scalar_to_byte_vector.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include #include namespace Envoy { diff --git a/source/common/common/stack_array.h b/source/common/common/stack_array.h index 21e2c7aa97b5..f96e42063729 100644 --- a/source/common/common/stack_array.h +++ b/source/common/common/stack_array.h @@ -7,7 +7,7 @@ #include #endif -#include +#include #include "common/common/assert.h" diff --git a/source/common/common/utility.h b/source/common/common/utility.h index 3697c9fb0542..6ff000af46dc 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -719,7 +719,7 @@ class InlineString : public InlineStorage { /** * @return a string_view into the InlineString. */ - absl::string_view toStringView() const { return absl::string_view(data_, size_); } + absl::string_view toStringView() const { return {data_, size_}; } /** * @return the number of bytes in the string diff --git a/source/common/config/metadata.cc b/source/common/config/metadata.cc index 22a2d9f05be7..9b06dca26dd8 100644 --- a/source/common/config/metadata.cc +++ b/source/common/config/metadata.cc @@ -13,7 +13,7 @@ const ProtobufWkt::Value& Metadata::metadataValue(const envoy::api::v2::core::Me const ProtobufWkt::Struct* data_struct = &(filter_it->second); const ProtobufWkt::Value* val = nullptr; // go through path to select sub entries - for (const auto p : path) { + for (const auto& p : path) { if (nullptr == data_struct) { // sub entry not found return ProtobufWkt::Value::default_instance(); } diff --git a/source/common/config/well_known_names.cc b/source/common/config/well_known_names.cc index 12b6072195ed..a9b1662d20f6 100644 --- a/source/common/config/well_known_names.cc +++ b/source/common/config/well_known_names.cc @@ -44,62 +44,62 @@ TagNameValues::TagNameValues() { // mongo.[.]collection.[.]callsite.(.)query. addRegex(MONGO_CALLSITE, - "^mongo(?=\\.).*?\\.collection(?=\\.).*?\\.callsite\\.((.*?)\\.).*?query.\\w+?$", + R"(^mongo(?=\.).*?\.collection(?=\.).*?\.callsite\.((.*?)\.).*?query.\w+?$)", ".collection."); // http.[.]dynamodb.table.(.) or // http.[.]dynamodb.error.(.)* - addRegex(DYNAMO_TABLE, "^http(?=\\.).*?\\.dynamodb.(?:table|error)\\.((.*?)\\.)", ".dynamodb."); + addRegex(DYNAMO_TABLE, R"(^http(?=\.).*?\.dynamodb.(?:table|error)\.((.*?)\.))", ".dynamodb."); // mongo.[.]collection.(.)query. - addRegex(MONGO_COLLECTION, "^mongo(?=\\.).*?\\.collection\\.((.*?)\\.).*?query.\\w+?$", + addRegex(MONGO_COLLECTION, R"(^mongo(?=\.).*?\.collection\.((.*?)\.).*?query.\w+?$)", ".collection."); // mongo.[.]cmd.(.) - addRegex(MONGO_CMD, "^mongo(?=\\.).*?\\.cmd\\.((.*?)\\.)\\w+?$", ".cmd."); + addRegex(MONGO_CMD, R"(^mongo(?=\.).*?\.cmd\.((.*?)\.)\w+?$)", ".cmd."); // cluster.[.]grpc.[.](.) - addRegex(GRPC_BRIDGE_METHOD, "^cluster(?=\\.).*?\\.grpc(?=\\.).*\\.((.*?)\\.)\\w+?$", ".grpc."); + addRegex(GRPC_BRIDGE_METHOD, R"(^cluster(?=\.).*?\.grpc(?=\.).*\.((.*?)\.)\w+?$)", ".grpc."); // http.[.]user_agent.(.) - addRegex(HTTP_USER_AGENT, "^http(?=\\.).*?\\.user_agent\\.((.*?)\\.)\\w+?$", ".user_agent."); + addRegex(HTTP_USER_AGENT, R"(^http(?=\.).*?\.user_agent\.((.*?)\.)\w+?$)", ".user_agent."); // vhost.[.]vcluster.(.) - addRegex(VIRTUAL_CLUSTER, "^vhost(?=\\.).*?\\.vcluster\\.((.*?)\\.)\\w+?$", ".vcluster."); + addRegex(VIRTUAL_CLUSTER, R"(^vhost(?=\.).*?\.vcluster\.((.*?)\.)\w+?$)", ".vcluster."); // http.[.]fault.(.) - addRegex(FAULT_DOWNSTREAM_CLUSTER, "^http(?=\\.).*?\\.fault\\.((.*?)\\.)\\w+?$", ".fault."); + addRegex(FAULT_DOWNSTREAM_CLUSTER, R"(^http(?=\.).*?\.fault\.((.*?)\.)\w+?$)", ".fault."); // listener.[
.]ssl.cipher.() - addRegex(SSL_CIPHER, "^listener(?=\\.).*?\\.ssl\\.cipher(\\.(.*?))$"); + addRegex(SSL_CIPHER, R"(^listener(?=\.).*?\.ssl\.cipher(\.(.*?))$)"); // cluster.[.]ssl.ciphers.() - addRegex(SSL_CIPHER_SUITE, "^cluster(?=\\.).*?\\.ssl\\.ciphers(\\.(.*?))$", ".ssl.ciphers."); + addRegex(SSL_CIPHER_SUITE, R"(^cluster(?=\.).*?\.ssl\.ciphers(\.(.*?))$)", ".ssl.ciphers."); // cluster.[.]grpc.(.)* - addRegex(GRPC_BRIDGE_SERVICE, "^cluster(?=\\.).*?\\.grpc\\.((.*?)\\.)", ".grpc."); + addRegex(GRPC_BRIDGE_SERVICE, R"(^cluster(?=\.).*?\.grpc\.((.*?)\.))", ".grpc."); // tcp.(.) - addRegex(TCP_PREFIX, "^tcp\\.((.*?)\\.)\\w+?$"); + addRegex(TCP_PREFIX, R"(^tcp\.((.*?)\.)\w+?$)"); // auth.clientssl.(.) - addRegex(CLIENTSSL_PREFIX, "^auth\\.clientssl\\.((.*?)\\.)\\w+?$"); + addRegex(CLIENTSSL_PREFIX, R"(^auth\.clientssl\.((.*?)\.)\w+?$)"); // ratelimit.(.) - addRegex(RATELIMIT_PREFIX, "^ratelimit\\.((.*?)\\.)\\w+?$"); + addRegex(RATELIMIT_PREFIX, R"(^ratelimit\.((.*?)\.)\w+?$)"); // cluster.(.)* addRegex(CLUSTER_NAME, "^cluster\\.((.*?)\\.)"); // listener.[
.]http.(.)* - addRegex(HTTP_CONN_MANAGER_PREFIX, "^listener(?=\\.).*?\\.http\\.((.*?)\\.)", ".http."); + addRegex(HTTP_CONN_MANAGER_PREFIX, R"(^listener(?=\.).*?\.http\.((.*?)\.))", ".http."); // http.(.)* addRegex(HTTP_CONN_MANAGER_PREFIX, "^http\\.((.*?)\\.)"); // listener.(
.)* addRegex(LISTENER_ADDRESS, - "^listener\\.(((?:[_.[:digit:]]*|[_\\[\\]aAbBcCdDeEfF[:digit:]]*))\\.)"); + R"(^listener\.(((?:[_.[:digit:]]*|[_\[\]aAbBcCdDeEfF[:digit:]]*))\.))"); // vhost.(.)* addRegex(VIRTUAL_HOST, "^vhost\\.((.*?)\\.)"); @@ -108,10 +108,10 @@ TagNameValues::TagNameValues() { addRegex(MONGO_PREFIX, "^mongo\\.((.*?)\\.)"); // http.[.]rds.(.) - addRegex(RDS_ROUTE_CONFIG, "^http(?=\\.).*?\\.rds\\.((.*?)\\.)\\w+?$", ".rds."); + addRegex(RDS_ROUTE_CONFIG, R"(^http(?=\.).*?\.rds\.((.*?)\.)\w+?$)", ".rds."); // listener_manager.(worker_.)* - addRegex(WORKER_ID, "^listener_manager\\.((worker_\\d+)\\.)", "listener_manager.worker_"); + addRegex(WORKER_ID, R"(^listener_manager\.((worker_\d+)\.))", "listener_manager.worker_"); } void TagNameValues::addRegex(const std::string& name, const std::string& regex, diff --git a/source/common/event/file_event_impl.cc b/source/common/event/file_event_impl.cc index feee927132ee..a1331e3359ea 100644 --- a/source/common/event/file_event_impl.cc +++ b/source/common/event/file_event_impl.cc @@ -47,7 +47,7 @@ void FileEventImpl::assignEvents(uint32_t events) { (events & FileReadyType::Write ? EV_WRITE : 0) | (events & FileReadyType::Closed ? EV_CLOSED : 0), [](evutil_socket_t, short what, void* arg) -> void { - FileEventImpl* event = static_cast(arg); + auto* event = static_cast(arg); uint32_t events = 0; if (what & EV_READ) { events |= FileReadyType::Read; diff --git a/source/common/event/libevent.cc b/source/common/event/libevent.cc index bf894858898b..c7ec364ddf24 100644 --- a/source/common/event/libevent.cc +++ b/source/common/event/libevent.cc @@ -1,6 +1,6 @@ #include "common/event/libevent.h" -#include +#include #include "common/common/assert.h" diff --git a/source/common/filesystem/file_shared_impl.h b/source/common/filesystem/file_shared_impl.h index 42d4ddb0f939..5e05b607cfa8 100644 --- a/source/common/filesystem/file_shared_impl.h +++ b/source/common/filesystem/file_shared_impl.h @@ -13,7 +13,7 @@ class IoFileError : public Api::IoError { public: explicit IoFileError(int sys_errno) : errno_(sys_errno) {} - ~IoFileError() override {} + ~IoFileError() override = default; Api::IoError::IoErrorCode getErrorCode() const override; std::string getErrorDetails() const override; diff --git a/source/common/filesystem/inotify/watcher_impl.cc b/source/common/filesystem/inotify/watcher_impl.cc index a8956d348d9a..766411f1b728 100644 --- a/source/common/filesystem/inotify/watcher_impl.cc +++ b/source/common/filesystem/inotify/watcher_impl.cc @@ -63,7 +63,7 @@ void WatcherImpl::onInotifyEvent() { const size_t event_count = rc; size_t index = 0; while (index < event_count) { - inotify_event* file_event = reinterpret_cast(&buffer[index]); + auto* file_event = reinterpret_cast(&buffer[index]); ASSERT(callback_map_.count(file_event->wd) == 1); std::string file; diff --git a/source/common/filesystem/inotify/watcher_impl.h b/source/common/filesystem/inotify/watcher_impl.h index 2885c55b578e..d83b5a428fda 100644 --- a/source/common/filesystem/inotify/watcher_impl.h +++ b/source/common/filesystem/inotify/watcher_impl.h @@ -21,7 +21,7 @@ namespace Filesystem { class WatcherImpl : public Watcher, Logger::Loggable { public: WatcherImpl(Event::Dispatcher& dispatcher); - ~WatcherImpl(); + ~WatcherImpl() override; // Filesystem::Watcher void addWatch(const std::string& path, uint32_t events, OnChangedCb cb) override; diff --git a/source/common/filesystem/posix/directory_iterator_impl.cc b/source/common/filesystem/posix/directory_iterator_impl.cc index 1b8480d56854..f4d7b97419e4 100644 --- a/source/common/filesystem/posix/directory_iterator_impl.cc +++ b/source/common/filesystem/posix/directory_iterator_impl.cc @@ -7,7 +7,7 @@ namespace Envoy { namespace Filesystem { DirectoryIteratorImpl::DirectoryIteratorImpl(const std::string& directory_path) - : DirectoryIterator(), directory_path_(directory_path), dir_(nullptr), + : directory_path_(directory_path), dir_(nullptr), os_sys_calls_(Api::OsSysCallsSingleton::get()) { openDirectory(); nextEntry(); diff --git a/source/common/filesystem/posix/filesystem_impl.h b/source/common/filesystem/posix/filesystem_impl.h index 8c6279e66499..b0c6c6388321 100644 --- a/source/common/filesystem/posix/filesystem_impl.h +++ b/source/common/filesystem/posix/filesystem_impl.h @@ -13,7 +13,7 @@ namespace Filesystem { class FileImplPosix : public FileSharedImpl { public: FileImplPosix(const std::string& path) : FileSharedImpl(path) {} - ~FileImplPosix(); + ~FileImplPosix() override; protected: // Filesystem::FileSharedImpl diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 8beae3707065..9830b98627cc 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -164,7 +164,7 @@ class AsyncStreamImpl : public AsyncClient::Stream, struct NullVirtualHost : public Router::VirtualHost { // Router::VirtualHost - Stats::StatName statName() const override { return Stats::StatName(); } + Stats::StatName statName() const override { return {}; } const Router::RateLimitPolicy& rateLimitPolicy() const override { return rate_limit_policy_; } const Router::CorsPolicy* corsPolicy() const override { return nullptr; } const Router::Config& routeConfig() const override { return route_configuration_; } @@ -368,7 +368,7 @@ class AsyncRequestImpl final : public AsyncClient::Request, const AsyncClient::RequestOptions& options); // AsyncClient::Request - virtual void cancel() override; + void cancel() override; private: void initialize(); diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index 134f8cad313e..6c5488ef7883 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -36,7 +36,7 @@ CodecClient::CodecClient(Type type, Network::ClientConnectionPtr&& connection, connection_->noDelay(true); } -CodecClient::~CodecClient() {} +CodecClient::~CodecClient() = default; void CodecClient::close() { connection_->close(Network::ConnectionCloseType::NoFlush); } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index c6e54e889522..66f819d3abc3 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -53,7 +53,7 @@ class ConnectionManagerImpl : Logger::Loggable, Runtime::Loader& runtime, const LocalInfo::LocalInfo& local_info, Upstream::ClusterManager& cluster_manager, Server::OverloadManager* overload_manager, TimeSource& time_system); - ~ConnectionManagerImpl(); + ~ConnectionManagerImpl() override; static ConnectionManagerStats generateStats(const std::string& prefix, Stats::Scope& scope); static ConnectionManagerTracingStats generateTracingStats(const std::string& prefix, diff --git a/source/common/http/conn_pool_base.h b/source/common/http/conn_pool_base.h index 7317acde39ec..ab4ea2de8255 100644 --- a/source/common/http/conn_pool_base.h +++ b/source/common/http/conn_pool_base.h @@ -19,7 +19,7 @@ class ConnPoolImplBase : protected Logger::Loggable { struct PendingRequest : LinkedObject, public ConnectionPool::Cancellable { PendingRequest(ConnPoolImplBase& parent, StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks); - ~PendingRequest(); + ~PendingRequest() override; // ConnectionPool::Cancellable void cancel() override { parent_.onPendingRequestCancel(*this); } diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index ded79fb00a93..cdc166e9ac85 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -315,7 +315,7 @@ http_parser_settings ConnectionImpl::settings_{ }; const ToLowerTable& ConnectionImpl::toLowerTable() { - static ToLowerTable* table = new ToLowerTable(); + static auto* table = new ToLowerTable(); return *table; } diff --git a/source/common/http/http1/conn_pool.h b/source/common/http/http1/conn_pool.h index 40228d27515d..91f854d514ac 100644 --- a/source/common/http/http1/conn_pool.h +++ b/source/common/http/http1/conn_pool.h @@ -34,7 +34,7 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase { Upstream::ResourcePriority priority, const Network::ConnectionSocket::OptionsSharedPtr& options); - ~ConnPoolImpl(); + ~ConnPoolImpl() override; // ConnectionPool::Instance Http::Protocol protocol() const override { return Http::Protocol::Http11; } @@ -55,7 +55,7 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase { public StreamDecoderWrapper, public StreamCallbacks { StreamWrapper(StreamDecoder& response_decoder, ActiveClient& parent); - ~StreamWrapper(); + ~StreamWrapper() override; // StreamEncoderWrapper void onEncodeComplete() override; @@ -84,7 +84,7 @@ class ConnPoolImpl : public ConnectionPool::Instance, public ConnPoolImplBase { public Network::ConnectionCallbacks, public Event::DeferredDeletable { ActiveClient(ConnPoolImpl& parent); - ~ActiveClient(); + ~ActiveClient() override; void onConnectTimeout(); diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 5f288926387e..475aa1d314e2 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -738,7 +738,7 @@ void ConnectionImpl::sendSettings(const Http2Settings& http2_settings, bool disa ASSERT(rc == 0); } else { // nghttp2_submit_settings need to be called at least once - int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, 0, 0); + int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, nullptr, 0); ASSERT(rc == 0); } diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 9bf50ad9b9e0..0b8c8bd0c4be 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -168,8 +168,8 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable(&rhs); + const auto* rhs_casted = dynamic_cast(&rhs); return (rhs_casted && (ip_.ipv6_.address() == rhs_casted->ip_.ipv6_.address()) && (ip_.port() == rhs_casted->ip_.port())); } diff --git a/source/common/network/cidr_range.h b/source/common/network/cidr_range.h index 309bae37c147..44e72585973f 100644 --- a/source/common/network/cidr_range.h +++ b/source/common/network/cidr_range.h @@ -129,7 +129,7 @@ class IpList { IpList(const std::vector& subnets); IpList(const Json::Object& config, const std::string& member_name); IpList(const Protobuf::RepeatedPtrField& cidrs); - IpList(){}; + IpList() = default; bool contains(const Instance& address) const; bool empty() const { return ip_list_.empty(); } diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index b0ba9671de93..f695b1acb416 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -53,7 +53,7 @@ class ConnectionImpl : public FilterManagerConnection, ConnectionImpl(Event::Dispatcher& dispatcher, ConnectionSocketPtr&& socket, TransportSocketPtr&& transport_socket, bool connected); - ~ConnectionImpl(); + ~ConnectionImpl() override; // Network::FilterManager void addWriteFilter(WriteFilterSharedPtr filter) override; diff --git a/source/common/network/io_socket_handle_impl.cc b/source/common/network/io_socket_handle_impl.cc index 56ae4469ba4d..cc9cbcdbc0b0 100644 --- a/source/common/network/io_socket_handle_impl.cc +++ b/source/common/network/io_socket_handle_impl.cc @@ -72,7 +72,7 @@ Api::IoCallUint64Result IoSocketHandleImpl::writev(const Buffer::RawSlice* slice Api::IoCallUint64Result IoSocketHandleImpl::sendto(const Buffer::RawSlice& slice, int flags, const Address::Instance& address) { - const Address::InstanceBase* address_base = dynamic_cast(&address); + const auto* address_base = dynamic_cast(&address); sockaddr* sock_addr = const_cast(address_base->sockAddr()); auto& os_syscalls = Api::OsSysCallsSingleton::get(); @@ -84,7 +84,7 @@ Api::IoCallUint64Result IoSocketHandleImpl::sendto(const Buffer::RawSlice& slice Api::IoCallUint64Result IoSocketHandleImpl::sendmsg(const Buffer::RawSlice* slices, uint64_t num_slice, int flags, const Address::Instance& address) { - const Address::InstanceBase* address_base = dynamic_cast(&address); + const auto* address_base = dynamic_cast(&address); sockaddr* sock_addr = const_cast(address_base->sockAddr()); STACK_ARRAY(iov, iovec, num_slice); diff --git a/source/common/network/lc_trie.h b/source/common/network/lc_trie.h index 059cb038d412..cf6dfdb0d65a 100644 --- a/source/common/network/lc_trie.h +++ b/source/common/network/lc_trie.h @@ -238,7 +238,7 @@ template class LcTrie { */ template struct IpPrefix { - IpPrefix() {} + IpPrefix() = default; IpPrefix(const IpType& ip, uint32_t length, const T& data) : ip_(ip), length_(length) { data_.insert(data); diff --git a/source/common/network/udp_listener_impl.h b/source/common/network/udp_listener_impl.h index 1c9c8a90c321..2a871959b3e0 100644 --- a/source/common/network/udp_listener_impl.h +++ b/source/common/network/udp_listener_impl.h @@ -20,7 +20,7 @@ class UdpListenerImpl : public BaseListenerImpl, public: UdpListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, UdpListenerCallbacks& cb); - ~UdpListenerImpl(); + ~UdpListenerImpl() override; // Network::Listener Interface void disable() override; diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 00bea45974cf..0290ff8fd169 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -65,7 +65,7 @@ namespace { std::string hostFromUrl(const std::string& url, const std::string& scheme, const std::string& scheme_name) { - if (url.find(scheme) != 0) { + if (!absl::StartsWith(url, scheme)) { throw EnvoyException(fmt::format("expected {} scheme, got: {}", scheme_name, url)); } @@ -80,7 +80,7 @@ std::string hostFromUrl(const std::string& url, const std::string& scheme, uint32_t portFromUrl(const std::string& url, const std::string& scheme, const std::string& scheme_name) { - if (url.find(scheme) != 0) { + if (!absl::StartsWith(url, scheme)) { throw EnvoyException(fmt::format("expected {} scheme, got: {}", scheme_name, url)); } @@ -163,7 +163,7 @@ Address::InstanceConstSharedPtr Utility::parseInternetAddressAndPort(const std:: return std::make_shared(sa6, v6only); } // Treat it as an IPv4 address followed by a port. - auto pos = ip_address.rfind(":"); + auto pos = ip_address.rfind(':'); if (pos == std::string::npos) { throwWithMalformedIp(ip_address); } diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index a215dbc0038c..25cfea0397a7 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -664,7 +664,7 @@ RouteEntryImplBase::parseOpaqueConfig(const envoy::api::v2::route::Route& route) if (filter_metadata == route.metadata().filter_metadata().end()) { return ret; } - for (auto it : filter_metadata->second.fields()) { + for (const auto& it : filter_metadata->second.fields()) { if (it.second.kind_case() == ProtobufWkt::Value::kStringValue) { ret.emplace(it.first, it.second.string_value()); } diff --git a/source/common/stats/histogram_impl.h b/source/common/stats/histogram_impl.h index 9977e357cb8b..5af61def2fcd 100644 --- a/source/common/stats/histogram_impl.h +++ b/source/common/stats/histogram_impl.h @@ -92,7 +92,7 @@ class HistogramImpl : public Histogram, public MetricImpl { class NullHistogramImpl : public Histogram, NullMetricImpl { public: explicit NullHistogramImpl(SymbolTable& symbol_table) : NullMetricImpl(symbol_table) {} - ~NullHistogramImpl() { + ~NullHistogramImpl() override { // MetricImpl must be explicitly cleared() before destruction, otherwise it // will not be able to access the SymbolTable& to free the symbols. An RAII // alternative would be to store the SymbolTable reference in the diff --git a/source/common/upstream/health_checker_base_impl.h b/source/common/upstream/health_checker_base_impl.h index 5920462adf8a..39aa9a8687aa 100644 --- a/source/common/upstream/health_checker_base_impl.h +++ b/source/common/upstream/health_checker_base_impl.h @@ -85,7 +85,7 @@ class HealthCheckerImplBase : public HealthChecker, HealthCheckerImplBase(const Cluster& cluster, const envoy::api::v2::core::HealthCheck& config, Event::Dispatcher& dispatcher, Runtime::Loader& runtime, Runtime::RandomGenerator& random, HealthCheckEventLoggerPtr&& event_logger); - ~HealthCheckerImplBase(); + ~HealthCheckerImplBase() override; virtual ActiveHealthCheckSessionPtr makeSession(HostSharedPtr host) PURE; virtual envoy::data::core::v2alpha::HealthCheckerType healthCheckerType() const PURE; diff --git a/source/common/upstream/health_checker_impl.h b/source/common/upstream/health_checker_impl.h index f47ddac7467c..6ca106d947b9 100644 --- a/source/common/upstream/health_checker_impl.h +++ b/source/common/upstream/health_checker_impl.h @@ -68,7 +68,7 @@ class HttpHealthCheckerImpl : public HealthCheckerImplBase { public Http::StreamDecoder, public Http::StreamCallbacks { HttpActiveHealthCheckSession(HttpHealthCheckerImpl& parent, const HostSharedPtr& host); - ~HttpActiveHealthCheckSession(); + ~HttpActiveHealthCheckSession() override; void onResponseComplete(); enum class HealthCheckResult { Succeeded, Degraded, Failed }; @@ -241,7 +241,7 @@ class TcpHealthCheckerImpl : public HealthCheckerImplBase { struct TcpActiveHealthCheckSession : public ActiveHealthCheckSession { TcpActiveHealthCheckSession(TcpHealthCheckerImpl& parent, const HostSharedPtr& host) : ActiveHealthCheckSession(parent, host), parent_(parent) {} - ~TcpActiveHealthCheckSession(); + ~TcpActiveHealthCheckSession() override; void onData(Buffer::Instance& data); void onEvent(Network::ConnectionEvent event); @@ -287,7 +287,7 @@ class GrpcHealthCheckerImpl : public HealthCheckerImplBase { public Http::StreamDecoder, public Http::StreamCallbacks { GrpcActiveHealthCheckSession(GrpcHealthCheckerImpl& parent, const HostSharedPtr& host); - ~GrpcActiveHealthCheckSession(); + ~GrpcActiveHealthCheckSession() override; void onRpcComplete(Grpc::Status::GrpcStatus grpc_status, const std::string& grpc_message, bool end_stream); diff --git a/source/common/upstream/logical_dns_cluster.h b/source/common/upstream/logical_dns_cluster.h index 56a2a0f82e19..07829f286533 100644 --- a/source/common/upstream/logical_dns_cluster.h +++ b/source/common/upstream/logical_dns_cluster.h @@ -37,7 +37,7 @@ class LogicalDnsCluster : public ClusterImplBase { Server::Configuration::TransportSocketFactoryContext& factory_context, Stats::ScopePtr&& stats_scope, bool added_via_api); - ~LogicalDnsCluster(); + ~LogicalDnsCluster() override; // Upstream::Cluster InitializePhase initializePhase() const override { return InitializePhase::Primary; } diff --git a/source/common/upstream/priority_conn_pool_map_impl.h b/source/common/upstream/priority_conn_pool_map_impl.h index cfe1c021393b..7d6f537ff1a3 100644 --- a/source/common/upstream/priority_conn_pool_map_impl.h +++ b/source/common/upstream/priority_conn_pool_map_impl.h @@ -10,7 +10,7 @@ template PriorityConnPoolMap::PriorityConnPoolMap(Envoy::Event::Dispatcher& dispatcher, const HostConstSharedPtr& host) { for (size_t pool_map_index = 0; pool_map_index < NumResourcePriorities; ++pool_map_index) { - ResourcePriority priority = static_cast(pool_map_index); + auto priority = static_cast(pool_map_index); conn_pool_maps_[pool_map_index].reset(new ConnPoolMapType(dispatcher, host, priority)); } } diff --git a/source/common/upstream/subset_lb.h b/source/common/upstream/subset_lb.h index 43337ccc5763..02b6108d56a4 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/common/upstream/subset_lb.h @@ -29,7 +29,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable& lb_ring_hash_config, const absl::optional& least_request_config, const envoy::api::v2::Cluster::CommonLbConfig& common_config); - ~SubsetLoadBalancer(); + ~SubsetLoadBalancer() override; // Upstream::LoadBalancer HostConstSharedPtr chooseHost(LoadBalancerContext* context) override; @@ -135,7 +135,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggableempty(); } diff --git a/source/exe/signal_action.cc b/source/exe/signal_action.cc index 5ff87da9ccbd..00cfaf5d2c40 100644 --- a/source/exe/signal_action.cc +++ b/source/exe/signal_action.cc @@ -1,8 +1,9 @@ #include "exe/signal_action.h" -#include #include +#include + #include "common/common/assert.h" namespace Envoy { diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index c8066478ab71..70cf9e2d9051 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -160,7 +160,7 @@ ProcessCluster(const NetworkFilters::Common::Redis::RespValue& value) { } std::string address = array[0].asString(); - bool ipv6 = (address.find(":") != std::string::npos); + bool ipv6 = (address.find(':') != std::string::npos); if (ipv6) { return std::make_shared(address, array[1].asInteger()); } diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index d21b69a7e176..fad26cafcaa6 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -100,7 +100,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { struct ClusterSlotsRequest : public Extensions::NetworkFilters::Common::Redis::RespValue { public: - ClusterSlotsRequest() : Extensions::NetworkFilters::Common::Redis::RespValue() { + ClusterSlotsRequest() { type(Extensions::NetworkFilters::Common::Redis::RespType::Array); std::vector values(2); values[0].type(NetworkFilters::Common::Redis::RespType::BulkString); @@ -193,7 +193,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { RedisDiscoverySession(RedisCluster& parent, NetworkFilters::Common::Redis::Client::ClientFactory& client_factory); - ~RedisDiscoverySession(); + ~RedisDiscoverySession() override; void registerDiscoveryAddress( const std::list& address_list, diff --git a/source/extensions/common/tap/extension_config_base.h b/source/extensions/common/tap/extension_config_base.h index 3e42a098c501..7e72447cf348 100644 --- a/source/extensions/common/tap/extension_config_base.h +++ b/source/extensions/common/tap/extension_config_base.h @@ -27,7 +27,7 @@ class ExtensionConfigBase : public ExtensionConfig, Logger::LoggabledoRequest()) { diff --git a/source/extensions/filters/http/squash/squash_filter.h b/source/extensions/filters/http/squash/squash_filter.h index 4dc2ea532712..0e35c3dd5245 100644 --- a/source/extensions/filters/http/squash/squash_filter.h +++ b/source/extensions/filters/http/squash/squash_filter.h @@ -71,7 +71,7 @@ class SquashFilter : public Http::StreamDecoderFilter, protected Logger::Loggable { public: SquashFilter(SquashFilterConfigSharedPtr config, Upstream::ClusterManager& cm); - ~SquashFilter(); + ~SquashFilter() override; // Http::StreamFilterBase void onDestroy() override; diff --git a/source/extensions/filters/network/mongo_proxy/codec_impl.h b/source/extensions/filters/network/mongo_proxy/codec_impl.h index 1736220ce835..4a404d14089e 100644 --- a/source/extensions/filters/network/mongo_proxy/codec_impl.h +++ b/source/extensions/filters/network/mongo_proxy/codec_impl.h @@ -136,9 +136,9 @@ class QueryMessageImpl : public MessageImpl, void numberToSkip(int32_t skip) override { number_to_skip_ = skip; } int32_t numberToReturn() const override { return number_to_return_; } void numberToReturn(int32_t to_return) override { number_to_return_ = to_return; } - virtual const Bson::Document* query() const override { return query_.get(); } + const Bson::Document* query() const override { return query_.get(); } void query(Bson::DocumentSharedPtr&& query) override { query_ = std::move(query); } - virtual const Bson::Document* returnFieldsSelector() const override { + const Bson::Document* returnFieldsSelector() const override { return return_fields_selector_.get(); } void returnFieldsSelector(Bson::DocumentSharedPtr&& fields) override { diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index e3a9c860931e..c9da9ef18806 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -188,13 +188,13 @@ InstanceImpl::ThreadLocalPool::makeRequestToHost(const std::string& host_address return nullptr; } - auto colon_pos = host_address.rfind(":"); + auto colon_pos = host_address.rfind(':'); if ((colon_pos == std::string::npos) || (colon_pos == (host_address.size() - 1))) { return nullptr; } const std::string ip_address = host_address.substr(0, colon_pos); - const bool ipv6 = (ip_address.find(":") != std::string::npos); + const bool ipv6 = (ip_address.find(':') != std::string::npos); std::string host_address_map_key; Network::Address::InstanceConstSharedPtr address_ptr; diff --git a/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h b/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h index 75e519f3c8ad..cfd687ba1878 100644 --- a/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h +++ b/source/extensions/filters/network/thrift_proxy/binary_protocol_impl.h @@ -18,7 +18,7 @@ namespace ThriftProxy { */ class BinaryProtocolImpl : public Protocol { public: - BinaryProtocolImpl() {} + BinaryProtocolImpl() = default; // Protocol const std::string& name() const override { return ProtocolNames::get().BINARY; } @@ -89,7 +89,7 @@ class BinaryProtocolImpl : public Protocol { */ class LaxBinaryProtocolImpl : public BinaryProtocolImpl { public: - LaxBinaryProtocolImpl() {} + LaxBinaryProtocolImpl() = default; const std::string& name() const override { return ProtocolNames::get().LAX_BINARY; } diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.h b/source/extensions/filters/network/thrift_proxy/router/router_impl.h index 85468dd1702a..de8bc9e0d470 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.h +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.h @@ -142,7 +142,7 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, public: Router(Upstream::ClusterManager& cluster_manager) : cluster_manager_(cluster_manager) {} - ~Router() {} + ~Router() override = default; // ThriftFilters::DecoderFilter void onDestroy() override; @@ -174,7 +174,7 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, UpstreamRequest(Router& parent, Tcp::ConnectionPool::Instance& pool, MessageMetadataSharedPtr& metadata, TransportType transport_type, ProtocolType protocol_type); - ~UpstreamRequest(); + ~UpstreamRequest() override; FilterStatus start(); void resetStream(); diff --git a/source/extensions/grpc_credentials/file_based_metadata/config.h b/source/extensions/grpc_credentials/file_based_metadata/config.h index 2a0ba6dcc38a..961688d1db31 100644 --- a/source/extensions/grpc_credentials/file_based_metadata/config.h +++ b/source/extensions/grpc_credentials/file_based_metadata/config.h @@ -37,7 +37,7 @@ class FileBasedMetadataGrpcCredentialsFactory : public Grpc::GoogleGrpcCredentia class FileBasedMetadataAuthenticator : public grpc::MetadataCredentialsPlugin { public: FileBasedMetadataAuthenticator( - const envoy::config::grpc_credential::v2alpha::FileBasedMetadataConfig config, Api::Api& api) + const envoy::config::grpc_credential::v2alpha::FileBasedMetadataConfig& config, Api::Api& api) : config_(config), api_(api) {} grpc::Status GetMetadata(grpc::string_ref, grpc::string_ref, const grpc::AuthContext&, diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h index 24f91efc4bc9..f70b953fab71 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h @@ -22,7 +22,7 @@ class EnvoyQuicAlarmFactory : public quic::QuicAlarmFactory, NonCopyable { EnvoyQuicAlarmFactory(Event::Scheduler& scheduler, quic::QuicClock& clock) : scheduler_(scheduler), clock_(clock) {} - ~EnvoyQuicAlarmFactory() override {} + ~EnvoyQuicAlarmFactory() override = default; // QuicAlarmFactory quic::QuicAlarm* CreateAlarm(quic::QuicAlarm::Delegate* delegate) override; diff --git a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc index c1e40522ec39..819638d4f155 100644 --- a/source/extensions/tracers/common/ot/opentracing_driver_impl.cc +++ b/source/extensions/tracers/common/ot/opentracing_driver_impl.cc @@ -70,7 +70,7 @@ class OpenTracingHTTPHeadersReader : public opentracing::HTTPHeadersReader { static Http::HeaderMap::Iterate headerMapCallback(const Http::HeaderEntry& header, void* context) { - OpenTracingCb* callback = static_cast(context); + auto* callback = static_cast(context); opentracing::string_view key{header.key().getStringView().data(), header.key().getStringView().length()}; opentracing::string_view value{header.value().getStringView().data(), diff --git a/source/server/backtrace.h b/source/server/backtrace.h index 1963a5ec0666..c452f88cba37 100644 --- a/source/server/backtrace.h +++ b/source/server/backtrace.h @@ -35,7 +35,7 @@ namespace Envoy { */ class BackwardsTrace : Logger::Loggable { public: - BackwardsTrace() {} + BackwardsTrace() = default; /** * Capture a stack trace. diff --git a/source/server/config_validation/connection.h b/source/server/config_validation/connection.h index 085067c16f62..7c6d5b6db062 100644 --- a/source/server/config_validation/connection.h +++ b/source/server/config_validation/connection.h @@ -28,7 +28,7 @@ class ConfigValidateConnection : public Network::ClientConnectionImpl { // connect may be called in config verification mode. // It is redefined as no-op. Calling parent's method triggers connection to upstream host. - virtual void connect() override {} + void connect() override {} }; } // namespace Network diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index 3c5de7f99852..35f0e67b242f 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -130,7 +130,7 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { ActiveTcpListener(ConnectionHandlerImpl& parent, Network::ListenerPtr&& listener, Network::ListenerConfig& config); - ~ActiveTcpListener(); + ~ActiveTcpListener() override; // Network::ListenerCallbacks void onAccept(Network::ConnectionSocketPtr&& socket, @@ -160,7 +160,7 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { public Network::ConnectionCallbacks { ActiveConnection(ActiveTcpListener& listener, Network::ConnectionPtr&& new_connection, TimeSource& time_system); - ~ActiveConnection(); + ~ActiveConnection() override; // Network::ConnectionCallbacks void onEvent(Network::ConnectionEvent event) override { @@ -192,7 +192,7 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { iter_(accept_filters_.end()) { listener_.stats_.downstream_pre_cx_active_.inc(); } - ~ActiveSocket() { + ~ActiveSocket() override { accept_filters_.clear(); listener_.stats_.downstream_pre_cx_active_.dec(); } diff --git a/source/server/guarddog_impl.h b/source/server/guarddog_impl.h index 7f07ba898ce7..c1ba688e9ba0 100644 --- a/source/server/guarddog_impl.h +++ b/source/server/guarddog_impl.h @@ -67,7 +67,7 @@ class GuardDogImpl : public GuardDog { GuardDogImpl(Stats::Scope& stats_scope, const Server::Configuration::Main& config, Api::Api& api, std::unique_ptr&& test_interlock); GuardDogImpl(Stats::Scope& stats_scope, const Server::Configuration::Main& config, Api::Api& api); - ~GuardDogImpl(); + ~GuardDogImpl() override; /** * Exposed for testing purposes only (but harmless to call): diff --git a/source/server/hot_restart_impl.cc b/source/server/hot_restart_impl.cc index 0a511bd741e1..2a39c9c425d2 100644 --- a/source/server/hot_restart_impl.cc +++ b/source/server/hot_restart_impl.cc @@ -1,10 +1,10 @@ #include "server/hot_restart_impl.h" -#include #include #include #include +#include #include #include #include diff --git a/source/server/hot_restart_nop_impl.h b/source/server/hot_restart_nop_impl.h index 010f93340ed5..dc4e0662270d 100644 --- a/source/server/hot_restart_nop_impl.h +++ b/source/server/hot_restart_nop_impl.h @@ -21,9 +21,7 @@ class HotRestartNopImpl : public Server::HotRestart { void initialize(Event::Dispatcher&, Server::Instance&) override {} void sendParentAdminShutdownRequest(time_t&) override {} void sendParentTerminateRequest() override {} - ServerStatsFromParent mergeParentStatsIfAny(Stats::StoreRoot&) override { - return ServerStatsFromParent(); - } + ServerStatsFromParent mergeParentStatsIfAny(Stats::StoreRoot&) override { return {}; } void shutdown() override {} std::string version() override { return "disabled"; } Thread::BasicLockable& logLock() override { return log_lock_; } diff --git a/source/server/server.h b/source/server/server.h index 871c71ca0355..fe77f35cd874 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -181,7 +181,7 @@ class InstanceImpl : Logger::Loggable, Runtime::RandomGenerator& random() override { return *random_generator_; } Runtime::Loader& runtime() override; void shutdown() override; - bool isShutdown() override final { return shutdown_; } + bool isShutdown() final { return shutdown_; } void shutdownAdmin() override; Singleton::Manager& singletonManager() override { return *singleton_manager_; } bool healthCheckFailed() override; diff --git a/test/common/grpc/google_grpc_creds_test.cc b/test/common/grpc/google_grpc_creds_test.cc index 819b758e1614..a08fcf162567 100644 --- a/test/common/grpc/google_grpc_creds_test.cc +++ b/test/common/grpc/google_grpc_creds_test.cc @@ -1,4 +1,4 @@ -#include +#include #include "common/grpc/google_grpc_creds_impl.h" diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 983adc19f9aa..4a407492b78f 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -545,7 +545,7 @@ TEST_F(HttpConnectionManagerImplTest, InvalidPathWithDualFilter) { })); // This test also verifies that decoder/encoder filters have onDestroy() called only once. - MockStreamFilter* filter = new MockStreamFilter(); + auto* filter = new MockStreamFilter(); EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillOnce(Invoke([&](FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(StreamFilterSharedPtr{filter}); @@ -584,7 +584,7 @@ TEST_F(HttpConnectionManagerImplTest, PathFailedtoSanitize) { })); // This test also verifies that decoder/encoder filters have onDestroy() called only once. - MockStreamFilter* filter = new MockStreamFilter(); + auto* filter = new MockStreamFilter(); EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillOnce(Invoke([&](FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(StreamFilterSharedPtr{filter}); @@ -614,7 +614,7 @@ TEST_F(HttpConnectionManagerImplTest, FilterShouldUseSantizedPath) { const std::string original_path = "/x/%2E%2e/z"; const std::string normalized_path = "/z"; - MockStreamFilter* filter = new MockStreamFilter(); + auto* filter = new MockStreamFilter(); EXPECT_CALL(filter_factory_, createFilterChain(_)) .WillOnce(Invoke([&](FilterChainFactoryCallbacks& callbacks) -> void { @@ -680,7 +680,7 @@ TEST_F(HttpConnectionManagerImplTest, RouteShouldUseSantizedPath) { TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlow) { setup(false, ""); - NiceMock* span = new NiceMock(); + auto* span = new NiceMock(); EXPECT_CALL(tracer_, startSpan_(_, _, _, _)) .WillOnce( Invoke([&](const Tracing::Config& config, const HeaderMap&, const StreamInfo::StreamInfo&, @@ -748,7 +748,7 @@ TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlow) { TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlowIngressDecorator) { setup(false, ""); - NiceMock* span = new NiceMock(); + auto* span = new NiceMock(); EXPECT_CALL(tracer_, startSpan_(_, _, _, _)) .WillOnce( Invoke([&](const Tracing::Config& config, const HeaderMap&, const StreamInfo::StreamInfo&, @@ -811,7 +811,7 @@ TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlowIngressDecorat TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlowIngressDecoratorOverrideOp) { setup(false, ""); - NiceMock* span = new NiceMock(); + auto* span = new NiceMock(); EXPECT_CALL(tracer_, startSpan_(_, _, _, _)) .WillOnce( Invoke([&](const Tracing::Config& config, const HeaderMap&, const StreamInfo::StreamInfo&, @@ -889,7 +889,7 @@ TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlowEgressDecorato percent1, false}); - NiceMock* span = new NiceMock(); + auto* span = new NiceMock(); EXPECT_CALL(tracer_, startSpan_(_, _, _, _)) .WillOnce( Invoke([&](const Tracing::Config& config, const HeaderMap&, const StreamInfo::StreamInfo&, @@ -967,7 +967,7 @@ TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlowEgressDecorato percent1, false}); - NiceMock* span = new NiceMock(); + auto* span = new NiceMock(); EXPECT_CALL(tracer_, startSpan_(_, _, _, _)) .WillOnce( Invoke([&](const Tracing::Config& config, const HeaderMap&, const StreamInfo::StreamInfo&, @@ -2049,7 +2049,7 @@ TEST_F(HttpConnectionManagerImplTest, FooUpgradeDrainClose) { setup(false, "envoy-custom-server", false); // Store the basic request encoder during filter chain setup. - MockStreamFilter* filter = new MockStreamFilter(); + auto* filter = new MockStreamFilter(); EXPECT_CALL(drain_close_, drainClose()).WillOnce(Return(true)); EXPECT_CALL(*filter, decodeHeaders(_, false)) diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 22458eb26b52..256bb7926fc3 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -3991,7 +3991,7 @@ TEST_F(WatermarkTest, RetryRequestNotComplete) { EXPECT_CALL(callbacks_.stream_info_, setResponseFlag(StreamInfo::ResponseFlag::UpstreamRemoteReset)); EXPECT_CALL(callbacks_.stream_info_, onUpstreamHostSelected(_)) - .WillRepeatedly(Invoke([&](const Upstream::HostDescriptionConstSharedPtr host) -> void { + .WillRepeatedly(Invoke([&](const Upstream::HostDescriptionConstSharedPtr& host) -> void { EXPECT_EQ(host_address_, host->address()); })); diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 389bc4b80758..cc7bcf242490 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -715,7 +715,7 @@ class RememberStatsMatcherTest : public testing::TestWithParam { void testAcceptsAll(const LookupStatFn lookup_stat) { InSequence s; - MockStatsMatcher* matcher = new MockStatsMatcher; + auto* matcher = new MockStatsMatcher; matcher->accepts_all_ = true; StatsMatcherPtr matcher_ptr(matcher); store_.setStatsMatcher(std::move(matcher_ptr)); diff --git a/test/extensions/filters/common/ext_authz/mocks.cc b/test/extensions/filters/common/ext_authz/mocks.cc index 99bbd23ab0ea..1423f6ce9100 100644 --- a/test/extensions/filters/common/ext_authz/mocks.cc +++ b/test/extensions/filters/common/ext_authz/mocks.cc @@ -6,11 +6,11 @@ namespace Filters { namespace Common { namespace ExtAuthz { -MockClient::MockClient() {} -MockClient::~MockClient() {} +MockClient::MockClient() = default; +MockClient::~MockClient() = default; -MockRequestCallbacks::MockRequestCallbacks() {} -MockRequestCallbacks::~MockRequestCallbacks() {} +MockRequestCallbacks::MockRequestCallbacks() = default; +MockRequestCallbacks::~MockRequestCallbacks() = default; } // namespace ExtAuthz } // namespace Common diff --git a/test/extensions/filters/common/ratelimit/mocks.cc b/test/extensions/filters/common/ratelimit/mocks.cc index 88abd53bcc2e..0cd8194ed64a 100644 --- a/test/extensions/filters/common/ratelimit/mocks.cc +++ b/test/extensions/filters/common/ratelimit/mocks.cc @@ -6,8 +6,8 @@ namespace Filters { namespace Common { namespace RateLimit { -MockClient::MockClient() {} -MockClient::~MockClient() {} +MockClient::MockClient() = default; +MockClient::~MockClient() = default; } // namespace RateLimit } // namespace Common diff --git a/test/extensions/filters/http/dynamo/dynamo_request_parser_test.cc b/test/extensions/filters/http/dynamo/dynamo_request_parser_test.cc index 45fa567f287d..8ebaeee2145a 100644 --- a/test/extensions/filters/http/dynamo/dynamo_request_parser_test.cc +++ b/test/extensions/filters/http/dynamo/dynamo_request_parser_test.cc @@ -69,7 +69,7 @@ TEST(DynamoRequestParser, parseTableNameSingleOperation) { } { - Json::ObjectSharedPtr json_data = Json::Factory::loadFromString("{\"TableName\":\"Pets\"}"); + Json::ObjectSharedPtr json_data = Json::Factory::loadFromString(R"({"TableName":"Pets"})"); EXPECT_EQ("Pets", RequestParser::parseTable("GetItem", *json_data).table_name); } } @@ -197,7 +197,7 @@ TEST(DynamoRequestParser, parseBatchUnProcessedKeys) { { std::vector unprocessed_tables = RequestParser::parseBatchUnProcessedKeys( - *Json::Factory::loadFromString("{\"UnprocessedKeys\":{\"table_1\" :{}}}")); + *Json::Factory::loadFromString(R"({"UnprocessedKeys":{"table_1" :{}}})")); EXPECT_EQ("table_1", unprocessed_tables[0]); EXPECT_EQ(1u, unprocessed_tables.size()); } @@ -236,7 +236,7 @@ TEST(DynamoRequestParser, parsePartitionIds) { } { std::vector partitions = RequestParser::parsePartitions( - *Json::Factory::loadFromString("{\"ConsumedCapacity\":{ \"Partitions\":{}}}")); + *Json::Factory::loadFromString(R"({"ConsumedCapacity":{ "Partitions":{}}})")); EXPECT_EQ(0u, partitions.size()); } { diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index e83933a33806..d4672646d22f 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -135,7 +135,7 @@ class GrpcJsonTranscoderIntegrationTest response_headers.iterate( [](const Http::HeaderEntry& entry, void* context) -> Http::HeaderMap::Iterate { - IntegrationStreamDecoder* response = static_cast(context); + auto* response = static_cast(context); Http::LowerCaseString lower_key{std::string(entry.key().getStringView())}; EXPECT_EQ(entry.value().getStringView(), response->headers().get(lower_key)->value().getStringView()); diff --git a/test/extensions/filters/http/lua/wrappers_test.cc b/test/extensions/filters/http/lua/wrappers_test.cc index 78eb7021e600..a1e151593132 100644 --- a/test/extensions/filters/http/lua/wrappers_test.cc +++ b/test/extensions/filters/http/lua/wrappers_test.cc @@ -19,7 +19,7 @@ namespace { class LuaHeaderMapWrapperTest : public Filters::Common::Lua::LuaWrappersTestBase { public: - virtual void setup(const std::string& script) { + void setup(const std::string& script) override { Filters::Common::Lua::LuaWrappersTestBase::setup(script); state_->registerType(); } @@ -220,7 +220,7 @@ TEST_F(LuaHeaderMapWrapperTest, IteratorAcrossYield) { class LuaStreamInfoWrapperTest : public Filters::Common::Lua::LuaWrappersTestBase { public: - virtual void setup(const std::string& script) { + void setup(const std::string& script) override { Filters::Common::Lua::LuaWrappersTestBase::setup(script); state_->registerType(); state_->registerType(); diff --git a/test/extensions/filters/http/squash/squash_filter_integration_test.cc b/test/extensions/filters/http/squash/squash_filter_integration_test.cc index 5e1634a606f9..2abddb9344e9 100644 --- a/test/extensions/filters/http/squash/squash_filter_integration_test.cc +++ b/test/extensions/filters/http/squash/squash_filter_integration_test.cc @@ -1,5 +1,3 @@ -#include - #include #include "common/protobuf/protobuf.h" @@ -21,7 +19,7 @@ class SquashFilterIntegrationTest : public testing::TestWithParamclose(); RELEASE_ASSERT(result, result.message()); diff --git a/test/extensions/filters/network/common/redis/mocks.cc b/test/extensions/filters/network/common/redis/mocks.cc index 3a2c2110f415..29d0a35725ea 100644 --- a/test/extensions/filters/network/common/redis/mocks.cc +++ b/test/extensions/filters/network/common/redis/mocks.cc @@ -28,10 +28,10 @@ MockEncoder::MockEncoder() { })); } -MockEncoder::~MockEncoder() {} +MockEncoder::~MockEncoder() = default; -MockDecoder::MockDecoder() {} -MockDecoder::~MockDecoder() {} +MockDecoder::MockDecoder() = default; +MockDecoder::~MockDecoder() = default; namespace Client { @@ -45,13 +45,13 @@ MockClient::MockClient() { })); } -MockClient::~MockClient() {} +MockClient::~MockClient() = default; -MockPoolRequest::MockPoolRequest() {} -MockPoolRequest::~MockPoolRequest() {} +MockPoolRequest::MockPoolRequest() = default; +MockPoolRequest::~MockPoolRequest() = default; -MockPoolCallbacks::MockPoolCallbacks() {} -MockPoolCallbacks::~MockPoolCallbacks() {} +MockPoolCallbacks::MockPoolCallbacks() = default; +MockPoolCallbacks::~MockPoolCallbacks() = default; } // namespace Client diff --git a/test/extensions/filters/network/dubbo_proxy/mocks.cc b/test/extensions/filters/network/dubbo_proxy/mocks.cc index d6ecb1e10546..14a55f348d81 100644 --- a/test/extensions/filters/network/dubbo_proxy/mocks.cc +++ b/test/extensions/filters/network/dubbo_proxy/mocks.cc @@ -34,18 +34,18 @@ MockProtocol::MockProtocol() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, type()).WillByDefault(Return(type_)); } -MockProtocol::~MockProtocol() {} +MockProtocol::~MockProtocol() = default; MockDeserializer::MockDeserializer() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, type()).WillByDefault(Return(type_)); } -MockDeserializer::~MockDeserializer() {} +MockDeserializer::~MockDeserializer() = default; namespace DubboFilters { -MockFilterChainFactoryCallbacks::MockFilterChainFactoryCallbacks() {} -MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() {} +MockFilterChainFactoryCallbacks::MockFilterChainFactoryCallbacks() = default; +MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() = default; MockDecoderFilter::MockDecoderFilter() { ON_CALL(*this, transportBegin()).WillByDefault(Return(Network::FilterStatus::Continue)); @@ -63,7 +63,7 @@ MockDecoderFilter::MockDecoderFilter() { return Network::FilterStatus::Continue; })); } -MockDecoderFilter::~MockDecoderFilter() {} +MockDecoderFilter::~MockDecoderFilter() = default; MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { route_.reset(new NiceMock()); @@ -73,16 +73,16 @@ MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { ON_CALL(*this, route()).WillByDefault(Return(route_)); ON_CALL(*this, streamInfo()).WillByDefault(ReturnRef(stream_info_)); } -MockDecoderFilterCallbacks::~MockDecoderFilterCallbacks() {} +MockDecoderFilterCallbacks::~MockDecoderFilterCallbacks() = default; -MockDirectResponse::MockDirectResponse() {} -MockDirectResponse::~MockDirectResponse() {} +MockDirectResponse::MockDirectResponse() = default; +MockDirectResponse::~MockDirectResponse() = default; MockFilterConfigFactory::MockFilterConfigFactory() : MockFactoryBase("envoy.filters.dubbo.mock_filter"), mock_filter_(std::make_shared>()) {} -MockFilterConfigFactory::~MockFilterConfigFactory() {} +MockFilterConfigFactory::~MockFilterConfigFactory() = default; FilterFactoryCb MockFilterConfigFactory::createFilterFactoryFromProtoTyped(const ProtobufWkt::Struct& proto_config, @@ -103,10 +103,10 @@ namespace Router { MockRouteEntry::MockRouteEntry() { ON_CALL(*this, clusterName()).WillByDefault(ReturnRef(cluster_name_)); } -MockRouteEntry::~MockRouteEntry() {} +MockRouteEntry::~MockRouteEntry() = default; MockRoute::MockRoute() { ON_CALL(*this, routeEntry()).WillByDefault(Return(&route_entry_)); } -MockRoute::~MockRoute() {} +MockRoute::~MockRoute() = default; } // namespace Router diff --git a/test/extensions/filters/network/mysql_proxy/mysql_codec_test.cc b/test/extensions/filters/network/mysql_proxy/mysql_codec_test.cc index fb79a345d124..2a186b3c35b9 100644 --- a/test/extensions/filters/network/mysql_proxy/mysql_codec_test.cc +++ b/test/extensions/filters/network/mysql_proxy/mysql_codec_test.cc @@ -838,7 +838,7 @@ TEST_F(MySQLCodecTest, MySQLLoginOldAuthSwitch) { TEST_F(MySQLCodecTest, MySQLLoginOkIncompleteRespCode) { ClientLoginResponse mysql_loginok_encode{}; mysql_loginok_encode.setRespCode(MYSQL_UT_RESP_OK); - std::string data = ""; + std::string data; Buffer::InstancePtr decode_data(new Buffer::OwnedImpl(data)); ClientLoginResponse mysql_loginok_decode{}; diff --git a/test/extensions/filters/network/mysql_proxy/mysql_filter_test.cc b/test/extensions/filters/network/mysql_proxy/mysql_filter_test.cc index 5d10600a940e..43ee4bf7802c 100644 --- a/test/extensions/filters/network/mysql_proxy/mysql_filter_test.cc +++ b/test/extensions/filters/network/mysql_proxy/mysql_filter_test.cc @@ -786,7 +786,7 @@ TEST_F(MySQLFilterTest, MySqlHandshake320WrongServerRespCode) { EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*server_resp_ok_data, false)); EXPECT_EQ(MySQLSession::State::MYSQL_NOT_HANDLED, filter_->getSession().getState()); - std::string msg_data = ""; + std::string msg_data; std::string mysql_msg = BufferHelper::encodeHdr(msg_data, 3); Buffer::InstancePtr client_query_data(new Buffer::OwnedImpl(mysql_msg)); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(*client_query_data, false)); diff --git a/test/extensions/filters/network/thrift_proxy/auto_protocol_impl_test.cc b/test/extensions/filters/network/thrift_proxy/auto_protocol_impl_test.cc index ffb756103f0d..63f33d6f103d 100644 --- a/test/extensions/filters/network/thrift_proxy/auto_protocol_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/auto_protocol_impl_test.cc @@ -56,7 +56,7 @@ class AutoProtocolTest : public testing::Test { TEST(ProtocolNames, FromType) { for (int i = 0; i <= static_cast(ProtocolType::LastProtocolType); i++) { - ProtocolType type = static_cast(i); + auto type = static_cast(i); EXPECT_NE("", ProtocolNames::get().fromType(type)); } } @@ -158,7 +158,7 @@ TEST_F(AutoProtocolTest, ReadMessageBegin) { } TEST_F(AutoProtocolTest, ReadDelegation) { - NiceMock* proto = new NiceMock(); + auto* proto = new NiceMock(); AutoProtocolImpl auto_proto; auto_proto.setProtocol(ProtocolPtr{proto}); @@ -281,7 +281,7 @@ TEST_F(AutoProtocolTest, ReadDelegation) { } TEST_F(AutoProtocolTest, WriteDelegation) { - NiceMock* proto = new NiceMock(); + auto* proto = new NiceMock(); AutoProtocolImpl auto_proto; auto_proto.setProtocol(ProtocolPtr{proto}); @@ -369,7 +369,7 @@ TEST_F(AutoProtocolTest, WriteDelegation) { // Test that protocol-upgrade methods are delegated to the detected protocol. TEST_F(AutoProtocolTest, ProtocolUpgradeDelegation) { - NiceMock* proto = new NiceMock(); + auto* proto = new NiceMock(); AutoProtocolImpl auto_proto; auto_proto.setProtocol(ProtocolPtr{proto}); diff --git a/test/extensions/stats_sinks/statsd/config_test.cc b/test/extensions/stats_sinks/statsd/config_test.cc index bcb6b41404ac..b71d29c94a6f 100644 --- a/test/extensions/stats_sinks/statsd/config_test.cc +++ b/test/extensions/stats_sinks/statsd/config_test.cc @@ -55,7 +55,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, StatsConfigParameterizedTest, TEST_P(StatsConfigParameterizedTest, UdpSinkDefaultPrefix) { const std::string name = StatsSinkNames::get().Statsd; - auto defaultPrefix = Common::Statsd::getDefaultPrefix(); + const auto& defaultPrefix = Common::Statsd::getDefaultPrefix(); envoy::config::metrics::v2::StatsdSink sink_config; envoy::api::v2::core::Address& address = *sink_config.mutable_address(); @@ -120,7 +120,7 @@ TEST(StatsConfigTest, TcpSinkDefaultPrefix) { const std::string name = StatsSinkNames::get().Statsd; envoy::config::metrics::v2::StatsdSink sink_config; - auto defaultPrefix = Common::Statsd::getDefaultPrefix(); + const auto& defaultPrefix = Common::Statsd::getDefaultPrefix(); sink_config.set_tcp_cluster_name("fake_cluster"); Server::Configuration::StatsSinkFactory* factory = diff --git a/test/integration/header_integration_test.cc b/test/integration/header_integration_test.cc index 9520fae8202d..2fa40efd43ce 100644 --- a/test/integration/header_integration_test.cc +++ b/test/integration/header_integration_test.cc @@ -285,24 +285,24 @@ class HeaderIntegrationTest if (use_eds_) { addHeader(route_config->mutable_response_headers_to_add(), "x-routeconfig-dynamic", - "%UPSTREAM_METADATA([\"test.namespace\", \"key\"])%", append); + R"(%UPSTREAM_METADATA(["test.namespace", "key"])%)", append); // Iterate over VirtualHosts, nested Routes and WeightedClusters, adding a dynamic // response header. for (auto& vhost : *route_config->mutable_virtual_hosts()) { addHeader(vhost.mutable_response_headers_to_add(), "x-vhost-dynamic", - "vhost:%UPSTREAM_METADATA([\"test.namespace\", \"key\"])%", append); + R"(vhost:%UPSTREAM_METADATA(["test.namespace", "key"])%)", append); for (auto& route : *vhost.mutable_routes()) { addHeader(route.mutable_response_headers_to_add(), "x-route-dynamic", - "route:%UPSTREAM_METADATA([\"test.namespace\", \"key\"])%", append); + R"(route:%UPSTREAM_METADATA(["test.namespace", "key"])%)", append); if (route.has_route()) { auto* route_action = route.mutable_route(); if (route_action->has_weighted_clusters()) { for (auto& c : *route_action->mutable_weighted_clusters()->mutable_clusters()) { addHeader(c.mutable_response_headers_to_add(), "x-weighted-cluster-dynamic", - "weighted:%UPSTREAM_METADATA([\"test.namespace\", \"key\"])%", + R"(weighted:%UPSTREAM_METADATA(["test.namespace", "key"])%)", append); } } diff --git a/test/integration/integration.cc b/test/integration/integration.cc index daa2a3911fc6..28e5eb47827c 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -126,7 +126,7 @@ void IntegrationStreamDecoder::decodeTrailers(Http::HeaderMapPtr&& trailers) { void IntegrationStreamDecoder::decodeMetadata(Http::MetadataMapPtr&& metadata_map) { // Combines newly received metadata with the existing metadata. - for (const auto metadata : *metadata_map) { + for (const auto& metadata : *metadata_map) { duplicated_metadata_key_count_[metadata.first]++; metadata_map_->insert(metadata); } diff --git a/test/integration/integration_admin_test.cc b/test/integration/integration_admin_test.cc index 2a84e580e570..137e972e0055 100644 --- a/test/integration/integration_admin_test.cc +++ b/test/integration/integration_admin_test.cc @@ -395,7 +395,7 @@ TEST_P(IntegrationAdminTest, Admin) { "type.googleapis.com/envoy.admin.v2alpha.ScopedRoutesConfigDump", "type.googleapis.com/envoy.admin.v2alpha.RoutesConfigDump"}; - for (Json::ObjectSharedPtr obj_ptr : json->getObjectArray("configs")) { + for (const Json::ObjectSharedPtr& obj_ptr : json->getObjectArray("configs")) { EXPECT_TRUE(expected_types[index].compare(obj_ptr->getString("@type")) == 0); index++; } diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 3c302f3c8e33..1e7c6bcb0b72 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -520,7 +520,7 @@ TEST_P(ProtocolIntegrationTest, HittingEncoderFilterLimit) { auto encoder_decoder = codec_client_->startRequest(default_request_headers_); auto downstream_request = &encoder_decoder.first; auto response = std::move(encoder_decoder.second); - Buffer::OwnedImpl data("{\"TableName\":\"locations\"}"); + Buffer::OwnedImpl data(R"({"TableName":"locations"})"); codec_client_->sendData(*downstream_request, data, true); waitForNextUpstreamRequest(); diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 5195c2b0b950..5bdc4b751de8 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -111,7 +111,7 @@ TEST_P(StatsIntegrationTest, WithTagSpecifierWithRegex) { bootstrap.mutable_stats_config()->mutable_use_all_default_tags()->set_value(false); auto tag_specifier = bootstrap.mutable_stats_config()->mutable_stats_tags()->Add(); tag_specifier->set_tag_name("my.http_conn_manager_prefix"); - tag_specifier->set_regex("^(?:|listener(?=\\.).*?\\.)http\\.((.*?)\\.)"); + tag_specifier->set_regex(R"(^(?:|listener(?=\.).*?\.)http\.((.*?)\.))"); }); initialize(); diff --git a/test/integration/uds_integration_test.cc b/test/integration/uds_integration_test.cc index caa7c93b9c95..e12d6d2a6c04 100644 --- a/test/integration/uds_integration_test.cc +++ b/test/integration/uds_integration_test.cc @@ -56,7 +56,7 @@ void UdsListenerIntegrationTest::initialize() { admin_addr->mutable_pipe()->set_path(getAdminSocketName()); auto* listeners = bootstrap.mutable_static_resources()->mutable_listeners(); - RELEASE_ASSERT(listeners->size() > 0, ""); + RELEASE_ASSERT(!listeners->empty(), ""); auto filter_chains = listeners->Get(0).filter_chains(); listeners->Clear(); auto* listener = listeners->Add(); diff --git a/test/mocks/filesystem/mocks.cc b/test/mocks/filesystem/mocks.cc index 5e585b5c8816..a9eca9c4815c 100644 --- a/test/mocks/filesystem/mocks.cc +++ b/test/mocks/filesystem/mocks.cc @@ -40,11 +40,11 @@ Api::IoCallBoolResult MockFile::close() { return result; } -MockInstance::MockInstance() {} -MockInstance::~MockInstance() {} +MockInstance::MockInstance() = default; +MockInstance::~MockInstance() = default; -MockWatcher::MockWatcher() {} -MockWatcher::~MockWatcher() {} +MockWatcher::MockWatcher() = default; +MockWatcher::~MockWatcher() = default; } // namespace Filesystem } // namespace Envoy diff --git a/test/mocks/filesystem/mocks.h b/test/mocks/filesystem/mocks.h index e1cd62f79b8e..b62af89b68a4 100644 --- a/test/mocks/filesystem/mocks.h +++ b/test/mocks/filesystem/mocks.h @@ -16,7 +16,7 @@ namespace Filesystem { class MockFile : public File { public: MockFile(); - ~MockFile(); + ~MockFile() override; // Filesystem::File Api::IoCallBoolResult open() override; @@ -43,7 +43,7 @@ class MockFile : public File { class MockInstance : public Instance { public: MockInstance(); - ~MockInstance(); + ~MockInstance() override; // Filesystem::Instance MOCK_METHOD1(createFile, FilePtr(const std::string&)); @@ -57,7 +57,7 @@ class MockInstance : public Instance { class MockWatcher : public Watcher { public: MockWatcher(); - ~MockWatcher(); + ~MockWatcher() override; MOCK_METHOD3(addWatch, void(const std::string&, uint32_t, OnChangedCb)); }; diff --git a/test/mocks/http/mocks.cc b/test/mocks/http/mocks.cc index 2e071d5a11c0..d1dab5be2225 100644 --- a/test/mocks/http/mocks.cc +++ b/test/mocks/http/mocks.cc @@ -19,26 +19,26 @@ using testing::SaveArg; namespace Envoy { namespace Http { -MockConnectionCallbacks::MockConnectionCallbacks() {} -MockConnectionCallbacks::~MockConnectionCallbacks() {} +MockConnectionCallbacks::MockConnectionCallbacks() = default; +MockConnectionCallbacks::~MockConnectionCallbacks() = default; -MockServerConnectionCallbacks::MockServerConnectionCallbacks() {} -MockServerConnectionCallbacks::~MockServerConnectionCallbacks() {} +MockServerConnectionCallbacks::MockServerConnectionCallbacks() = default; +MockServerConnectionCallbacks::~MockServerConnectionCallbacks() = default; -MockStreamCallbacks::MockStreamCallbacks() {} -MockStreamCallbacks::~MockStreamCallbacks() {} +MockStreamCallbacks::MockStreamCallbacks() = default; +MockStreamCallbacks::~MockStreamCallbacks() = default; MockServerConnection::MockServerConnection() { ON_CALL(*this, protocol()).WillByDefault(Return(protocol_)); } -MockServerConnection::~MockServerConnection() {} +MockServerConnection::~MockServerConnection() = default; -MockClientConnection::MockClientConnection() {} -MockClientConnection::~MockClientConnection() {} +MockClientConnection::MockClientConnection() = default; +MockClientConnection::~MockClientConnection() = default; -MockFilterChainFactory::MockFilterChainFactory() {} -MockFilterChainFactory::~MockFilterChainFactory() {} +MockFilterChainFactory::MockFilterChainFactory() = default; +MockFilterChainFactory::~MockFilterChainFactory() = default; template static void initializeMockStreamFilterCallbacks(T& callbacks) { callbacks.cluster_info_.reset(new NiceMock()); @@ -67,7 +67,7 @@ MockStreamDecoderFilterCallbacks::MockStreamDecoderFilterCallbacks() { ON_CALL(*this, tracingConfig()).WillByDefault(ReturnRef(tracing_config_)); } -MockStreamDecoderFilterCallbacks::~MockStreamDecoderFilterCallbacks() {} +MockStreamDecoderFilterCallbacks::~MockStreamDecoderFilterCallbacks() = default; MockStreamEncoderFilterCallbacks::MockStreamEncoderFilterCallbacks() { initializeMockStreamFilterCallbacks(*this); @@ -76,7 +76,7 @@ MockStreamEncoderFilterCallbacks::MockStreamEncoderFilterCallbacks() { ON_CALL(*this, tracingConfig()).WillByDefault(ReturnRef(tracing_config_)); } -MockStreamEncoderFilterCallbacks::~MockStreamEncoderFilterCallbacks() {} +MockStreamEncoderFilterCallbacks::~MockStreamEncoderFilterCallbacks() = default; MockStreamDecoderFilter::MockStreamDecoderFilter() { ON_CALL(*this, setDecoderFilterCallbacks(_)) @@ -84,7 +84,7 @@ MockStreamDecoderFilter::MockStreamDecoderFilter() { [this](StreamDecoderFilterCallbacks& callbacks) -> void { callbacks_ = &callbacks; })); } -MockStreamDecoderFilter::~MockStreamDecoderFilter() {} +MockStreamDecoderFilter::~MockStreamDecoderFilter() = default; MockStreamEncoderFilter::MockStreamEncoderFilter() { ON_CALL(*this, setEncoderFilterCallbacks(_)) @@ -92,7 +92,7 @@ MockStreamEncoderFilter::MockStreamEncoderFilter() { [this](StreamEncoderFilterCallbacks& callbacks) -> void { callbacks_ = &callbacks; })); } -MockStreamEncoderFilter::~MockStreamEncoderFilter() {} +MockStreamEncoderFilter::~MockStreamEncoderFilter() = default; MockStreamFilter::MockStreamFilter() { ON_CALL(*this, setDecoderFilterCallbacks(_)) @@ -105,27 +105,27 @@ MockStreamFilter::MockStreamFilter() { })); } -MockStreamFilter::~MockStreamFilter() {} +MockStreamFilter::~MockStreamFilter() = default; MockAsyncClient::MockAsyncClient() { ON_CALL(*this, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); } -MockAsyncClient::~MockAsyncClient() {} +MockAsyncClient::~MockAsyncClient() = default; -MockAsyncClientCallbacks::MockAsyncClientCallbacks() {} -MockAsyncClientCallbacks::~MockAsyncClientCallbacks() {} +MockAsyncClientCallbacks::MockAsyncClientCallbacks() = default; +MockAsyncClientCallbacks::~MockAsyncClientCallbacks() = default; -MockAsyncClientStreamCallbacks::MockAsyncClientStreamCallbacks() {} -MockAsyncClientStreamCallbacks::~MockAsyncClientStreamCallbacks() {} +MockAsyncClientStreamCallbacks::MockAsyncClientStreamCallbacks() = default; +MockAsyncClientStreamCallbacks::~MockAsyncClientStreamCallbacks() = default; MockAsyncClientRequest::MockAsyncClientRequest(MockAsyncClient* client) : client_(client) {} MockAsyncClientRequest::~MockAsyncClientRequest() { client_->onRequestDestroy(); } -MockAsyncClientStream::MockAsyncClientStream() {} -MockAsyncClientStream::~MockAsyncClientStream() {} +MockAsyncClientStream::MockAsyncClientStream() = default; +MockAsyncClientStream::~MockAsyncClientStream() = default; -MockFilterChainFactoryCallbacks::MockFilterChainFactoryCallbacks() {} -MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() {} +MockFilterChainFactoryCallbacks::MockFilterChainFactoryCallbacks() = default; +MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() = default; } // namespace Http diff --git a/test/mocks/http/stream.h b/test/mocks/http/stream.h index e99855404f82..54b81c4fd912 100644 --- a/test/mocks/http/stream.h +++ b/test/mocks/http/stream.h @@ -10,7 +10,7 @@ namespace Http { class MockStream : public Stream { public: MockStream(); - ~MockStream(); + ~MockStream() override; // Http::Stream MOCK_METHOD1(addCallbacks, void(StreamCallbacks& callbacks)); diff --git a/test/mocks/http/stream_decoder.h b/test/mocks/http/stream_decoder.h index 632e765fd3ce..1a5cd60b874f 100644 --- a/test/mocks/http/stream_decoder.h +++ b/test/mocks/http/stream_decoder.h @@ -9,7 +9,7 @@ namespace Http { class MockStreamDecoder : public StreamDecoder { public: MockStreamDecoder(); - ~MockStreamDecoder(); + ~MockStreamDecoder() override; void decode100ContinueHeaders(HeaderMapPtr&& headers) override { decode100ContinueHeaders_(headers); diff --git a/test/mocks/stats/mocks.cc b/test/mocks/stats/mocks.cc index aa7c6303e7cf..33ea887eb1ab 100644 --- a/test/mocks/stats/mocks.cc +++ b/test/mocks/stats/mocks.cc @@ -18,7 +18,7 @@ namespace Envoy { namespace Stats { MockMetric::MockMetric() : name_(*this), tag_pool_(*symbol_table_) {} -MockMetric::~MockMetric() {} +MockMetric::~MockMetric() = default; MockMetric::MetricName::~MetricName() { if (stat_name_storage_ != nullptr) { @@ -73,14 +73,14 @@ MockCounter::MockCounter() { ON_CALL(*this, value()).WillByDefault(ReturnPointee(&value_)); ON_CALL(*this, latch()).WillByDefault(ReturnPointee(&latch_)); } -MockCounter::~MockCounter() {} +MockCounter::~MockCounter() = default; MockGauge::MockGauge() : used_(false), value_(0), import_mode_(ImportMode::Accumulate) { ON_CALL(*this, used()).WillByDefault(ReturnPointee(&used_)); ON_CALL(*this, value()).WillByDefault(ReturnPointee(&value_)); ON_CALL(*this, importMode()).WillByDefault(ReturnPointee(&import_mode_)); } -MockGauge::~MockGauge() {} +MockGauge::~MockGauge() = default; MockHistogram::MockHistogram() { ON_CALL(*this, recordValue(_)).WillByDefault(Invoke([this](uint64_t value) { @@ -89,7 +89,7 @@ MockHistogram::MockHistogram() { } })); } -MockHistogram::~MockHistogram() {} +MockHistogram::~MockHistogram() = default; MockParentHistogram::MockParentHistogram() { ON_CALL(*this, recordValue(_)).WillByDefault(Invoke([this](uint64_t value) { @@ -101,7 +101,7 @@ MockParentHistogram::MockParentHistogram() { ON_CALL(*this, cumulativeStatistics()).WillByDefault(ReturnRef(*histogram_stats_)); ON_CALL(*this, used()).WillByDefault(ReturnPointee(&used_)); } -MockParentHistogram::~MockParentHistogram() {} +MockParentHistogram::~MockParentHistogram() = default; MockMetricSnapshot::MockMetricSnapshot() { ON_CALL(*this, counters()).WillByDefault(ReturnRef(counters_)); @@ -109,10 +109,10 @@ MockMetricSnapshot::MockMetricSnapshot() { ON_CALL(*this, histograms()).WillByDefault(ReturnRef(histograms_)); } -MockMetricSnapshot::~MockMetricSnapshot() {} +MockMetricSnapshot::~MockMetricSnapshot() = default; -MockSink::MockSink() {} -MockSink::~MockSink() {} +MockSink::MockSink() = default; +MockSink::~MockSink() = default; MockStore::MockStore() : StoreImpl(*fake_symbol_table_) { ON_CALL(*this, counter(_)).WillByDefault(ReturnRef(counter_)); @@ -124,14 +124,14 @@ MockStore::MockStore() : StoreImpl(*fake_symbol_table_) { return *histogram; })); } -MockStore::~MockStore() {} +MockStore::~MockStore() = default; MockIsolatedStatsStore::MockIsolatedStatsStore() : IsolatedStoreImpl(Test::Global::get()) {} -MockIsolatedStatsStore::~MockIsolatedStatsStore() {} +MockIsolatedStatsStore::~MockIsolatedStatsStore() = default; -MockStatsMatcher::MockStatsMatcher() {} -MockStatsMatcher::~MockStatsMatcher() {} +MockStatsMatcher::MockStatsMatcher() = default; +MockStatsMatcher::~MockStatsMatcher() = default; } // namespace Stats } // namespace Envoy diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index e9c4c5fdc47f..a35196a81fd1 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -730,7 +730,7 @@ TEST_F(ListenerManagerImplTest, AddOrUpdateListener) { InSequence s; - MockLdsApi* lds_api = new MockLdsApi(); + auto* lds_api = new MockLdsApi(); EXPECT_CALL(listener_factory_, createLdsApi_(_)).WillOnce(Return(lds_api)); envoy::api::v2::core::ConfigSource lds_config; manager_->createLdsApi(lds_config); diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index ea6c11d6f0ea..68915dba5f1d 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -188,26 +188,26 @@ std::string TestEnvironment::substitute(const std::string& str, {"test_rundir", TestEnvironment::runfilesDirectory()}, }; std::string out_json_string = str; - for (auto it : path_map) { + for (const auto& it : path_map) { const std::regex port_regex("\\{\\{ " + it.first + " \\}\\}"); out_json_string = std::regex_replace(out_json_string, port_regex, it.second); } // Substitute IP loopback addresses. - const std::regex loopback_address_regex("\\{\\{ ip_loopback_address \\}\\}"); + const std::regex loopback_address_regex(R"(\{\{ ip_loopback_address \}\})"); out_json_string = std::regex_replace(out_json_string, loopback_address_regex, Network::Test::getLoopbackAddressString(version)); - const std::regex ntop_loopback_address_regex("\\{\\{ ntop_ip_loopback_address \\}\\}"); + const std::regex ntop_loopback_address_regex(R"(\{\{ ntop_ip_loopback_address \}\})"); out_json_string = std::regex_replace(out_json_string, ntop_loopback_address_regex, Network::Test::getLoopbackAddressString(version)); // Substitute IP any addresses. - const std::regex any_address_regex("\\{\\{ ip_any_address \\}\\}"); + const std::regex any_address_regex(R"(\{\{ ip_any_address \}\})"); out_json_string = std::regex_replace(out_json_string, any_address_regex, Network::Test::getAnyAddressString(version)); // Substitute dns lookup family. - const std::regex lookup_family_regex("\\{\\{ dns_lookup_family \\}\\}"); + const std::regex lookup_family_regex(R"(\{\{ dns_lookup_family \}\})"); switch (version) { case Network::Address::IpVersion::v4: out_json_string = std::regex_replace(out_json_string, lookup_family_regex, "v4_only"); @@ -251,13 +251,13 @@ std::string TestEnvironment::temporaryFileSubstitute(const std::string& path, std::string out_json_string = readFileToStringForTest(json_path); // Substitute params. - for (auto it : param_map) { + for (const auto& it : param_map) { const std::regex param_regex("\\{\\{ " + it.first + " \\}\\}"); out_json_string = std::regex_replace(out_json_string, param_regex, it.second); } // Substitute ports. - for (auto it : port_map) { + for (const auto& it : port_map) { const std::regex port_regex("\\{\\{ " + it.first + " \\}\\}"); out_json_string = std::regex_replace(out_json_string, port_regex, std::to_string(it.second)); } diff --git a/test/test_common/test_time.cc b/test/test_common/test_time.cc index 09a5391cf128..c89f2b9e199c 100644 --- a/test/test_common/test_time.cc +++ b/test/test_common/test_time.cc @@ -6,7 +6,7 @@ namespace Envoy { -DangerousDeprecatedTestTime::DangerousDeprecatedTestTime() {} +DangerousDeprecatedTestTime::DangerousDeprecatedTestTime() = default; namespace Event { diff --git a/test/test_common/thread_factory_for_test.cc b/test/test_common/thread_factory_for_test.cc index 0bdd93e0cdde..38b966bae066 100644 --- a/test/test_common/thread_factory_for_test.cc +++ b/test/test_common/thread_factory_for_test.cc @@ -7,9 +7,9 @@ namespace Thread { // TODO(sesmith177) Tests should get the ThreadFactory from the same location as the main code ThreadFactory& threadFactoryForTest() { #ifdef WIN32 - static ThreadFactoryImplWin32* thread_factory = new ThreadFactoryImplWin32(); + static auto* thread_factory = new ThreadFactoryImplWin32(); #else - static ThreadFactoryImplPosix* thread_factory = new ThreadFactoryImplPosix(); + static auto* thread_factory = new ThreadFactoryImplPosix(); #endif return *thread_factory; } diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 9ea731e47ac3..983932374d32 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -287,7 +287,7 @@ std::string TestUtility::convertTime(const std::string& input, const std::string } // static -bool TestUtility::gaugesZeroed(const std::vector gauges) { +bool TestUtility::gaugesZeroed(const std::vector& gauges) { // Returns true if all gauges are 0 except the circuit_breaker remaining resource // gauges which default to the resource max. std::regex omitted(".*circuit_breakers\\..*\\.remaining.*"); @@ -357,11 +357,10 @@ const uint32_t Http2Settings::DEFAULT_INITIAL_STREAM_WINDOW_SIZE; const uint32_t Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE; const uint32_t Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE; -TestHeaderMapImpl::TestHeaderMapImpl() : HeaderMapImpl() {} +TestHeaderMapImpl::TestHeaderMapImpl() = default; TestHeaderMapImpl::TestHeaderMapImpl( - const std::initializer_list>& values) - : HeaderMapImpl() { + const std::initializer_list>& values) { for (auto& value : values) { addCopy(value.first, value.second); } diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 2c80f49b431f..f60d6452cb64 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include #include #include #include @@ -407,7 +406,7 @@ class TestUtility { * @param vector of gauges to check. * @return bool indicating that passed gauges not matching the omitted regex have a value of 0. */ - static bool gaugesZeroed(const std::vector gauges); + static bool gaugesZeroed(const std::vector& gauges); // Strict variants of Protobuf::MessageUtil static void loadFromJson(const std::string& json, Protobuf::Message& message) { From 49277cebb22ad584fff375de2c31abfd381aa30d Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Tue, 18 Jun 2019 21:40:24 -0700 Subject: [PATCH 031/542] bazel: avoid invalidating analysis cache between build and tests. (#7294) This is achieved by pulling BAZEL_TEST_OPTIONS into BAZEL_BUILD_OPTIONS, i.e. using the same build options for "build" and "test" commands. Signed-off-by: Piotr Sikora --- .bazelrc | 2 +- ci/build_setup.ps1 | 3 +- ci/build_setup.sh | 7 +++-- ci/do_ci.ps1 | 4 +-- ci/do_ci.sh | 51 +++++++++++++++----------------- ci/mac_ci_steps.sh | 8 ++--- test/run_envoy_bazel_coverage.sh | 4 +-- 7 files changed, 38 insertions(+), 41 deletions(-) diff --git a/.bazelrc b/.bazelrc index 8a549c6e8dc0..d0cd3e3f007a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -65,4 +65,4 @@ build:libc++ --define force_libcpp=enabled build:sizeopt -c opt --copt -Os # Test options -test --test_env=HEAPCHECK=normal --test_env=PPROF_PATH +build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH diff --git a/ci/build_setup.ps1 b/ci/build_setup.ps1 index d832c1796b29..49bb4a0990da 100755 --- a/ci/build_setup.ps1 +++ b/ci/build_setup.ps1 @@ -18,5 +18,4 @@ echo "ENVOY_BAZEL_ROOT: $env:ENVOY_BAZEL_ROOT" echo "ENVOY_SRCDIR: $env:ENVOY_SRCDIR" $env:BAZEL_BASE_OPTIONS="--noworkspace_rc --output_base=$env:ENVOY_BAZEL_ROOT --bazelrc=$env:ENVOY_SRCDIR\windows\tools\bazel.rc" -$env:BAZEL_BUILD_OPTIONS="--strategy=Genrule=standalone --spawn_strategy=standalone --verbose_failures --jobs=$env:NUM_CPUS --show_task_finish $env:BAZEL_BUILD_EXTRA_OPTIONS" -$env:BAZEL_TEST_OPTIONS="$env:BAZEL_BUILD_OPTIONS --cache_test_results=no --test_output=all $env:BAZEL_EXTRA_TEST_OPTIONS" +$env:BAZEL_BUILD_OPTIONS="--strategy=Genrule=standalone --spawn_strategy=standalone --verbose_failures --jobs=$env:NUM_CPUS --show_task_finish --cache_test_results=no --test_output=all $env:BAZEL_BUILD_EXTRA_OPTIONS $env:BAZEL_EXTRA_TEST_OPTIONS" diff --git a/ci/build_setup.sh b/ci/build_setup.sh index beef261cc69e..a8dc27e51a6a 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -71,9 +71,10 @@ fi export BAZEL_QUERY_OPTIONS="${BAZEL_OPTIONS}" export BAZEL_BUILD_OPTIONS="--strategy=Genrule=standalone --spawn_strategy=standalone \ --verbose_failures ${BAZEL_OPTIONS} --action_env=HOME --action_env=PYTHONUSERBASE \ - --jobs=${NUM_CPUS} --show_task_finish --experimental_generate_json_trace_profile ${BAZEL_BUILD_EXTRA_OPTIONS}" -export BAZEL_TEST_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_env=HOME --test_env=PYTHONUSERBASE \ - --cache_test_results=no --test_output=all ${BAZEL_EXTRA_TEST_OPTIONS}" + --jobs=${NUM_CPUS} --show_task_finish --experimental_generate_json_trace_profile \ + --test_env=HOME --test_env=PYTHONUSERBASE --cache_test_results=no --test_output=all \ + ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}" + [[ "${BAZEL_EXPUNGE}" == "1" ]] && "${BAZEL}" clean --expunge if [ "$1" != "-nofetch" ]; then diff --git a/ci/do_ci.ps1 b/ci/do_ci.ps1 index 6d85985d89c8..216b3eb712a5 100755 --- a/ci/do_ci.ps1 +++ b/ci/do_ci.ps1 @@ -15,10 +15,10 @@ function bazel_binary_build($type) { function bazel_test($type, $test) { if ($test -ne "") { - bazel $env:BAZEL_BASE_OPTIONS.Split(" ") test $env:BAZEL_TEST_OPTIONS.Split(" ") -c $type $test + bazel $env:BAZEL_BASE_OPTIONS.Split(" ") test $env:BAZEL_BUILD_OPTIONS.Split(" ") -c $type $test } else { echo "running windows tests" - bazel $env:BAZEL_BASE_OPTIONS.Split(" ") test $env:BAZEL_TEST_OPTIONS.Split(" ") -c $type "//test/..." + bazel $env:BAZEL_BASE_OPTIONS.Split(" ") test $env:BAZEL_BUILD_OPTIONS.Split(" ") -c $type "//test/..." } exit $LASTEXITCODE } diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 430f8d33c579..6aa1cd457138 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -55,19 +55,10 @@ function cp_binary_for_image_build() { strip "${ENVOY_DELIVERY_DIR}"/envoy -o "${ENVOY_SRCDIR}"/build_"$1"_stripped/envoy } -# When testing memory consumption, we want to test against exact byte-counts -# where possible. As these differ between platforms and compile options, we -# define the 'release' builds as canonical and test them only in CI, so the -# toolchain is kept consistent. This ifdef is checked in -# test/common/stats/stat_test_utility.cc when computing -# Stats::TestUtil::MemoryTest::mode(). -MEMORY_TEST_EXACT_ARGS="--cxxopt=-DMEMORY_TEST_EXACT=1" - function bazel_binary_build() { BINARY_TYPE="$1" if [[ "${BINARY_TYPE}" == "release" ]]; then COMPILE_TYPE="opt" - CONFIG_ARGS="$MEMORY_TEST_EXACT_ARGS" elif [[ "${BINARY_TYPE}" == "debug" ]]; then COMPILE_TYPE="dbg" elif [[ "${BINARY_TYPE}" == "sizeopt" ]]; then @@ -91,26 +82,32 @@ function bazel_binary_build() { } if [[ "$1" == "bazel.release" ]]; then + # When testing memory consumption, we want to test against exact byte-counts + # where possible. As these differ between platforms and compile options, we + # define the 'release' builds as canonical and test them only in CI, so the + # toolchain is kept consistent. This ifdef is checked in + # test/common/stats/stat_test_utility.cc when computing + # Stats::TestUtil::MemoryTest::mode(). + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --cxxopt=-DMEMORY_TEST_EXACT=1" + setup_clang_toolchain echo "bazel release build with tests..." bazel_binary_build release - RELEASE_OPTIONS="-c opt ${MEMORY_TEST_EXACT_ARGS}" - BAZEL_TEST_RELEASE_OPTIONS="test ${BAZEL_TEST_OPTIONS} ${RELEASE_OPTIONS}" if [[ $# -gt 1 ]]; then shift echo "Testing $* ..." # Run only specified tests. Argument can be a single test # (e.g. '//test/common/common:assert_test') or a test group (e.g. '//test/common/...') - bazel_with_collection $BAZEL_TEST_RELEASE_OPTIONS $* + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c opt $* else echo "Testing..." # We have various test binaries in the test directory such as tools, benchmarks, etc. We # run a build pass to make sure they compile. - bazel build ${BAZEL_BUILD_OPTIONS} ${RELEASE_OPTIONS} //include/... //source/... //test/... + bazel build ${BAZEL_BUILD_OPTIONS} -c opt //include/... //source/... //test/... # Now run all of the tests which should already be compiled. - bazel_with_collection $BAZEL_TEST_RELEASE_OPTIONS //test/... + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c opt //test/... fi exit 0 elif [[ "$1" == "bazel.release.server_only" ]]; then @@ -128,14 +125,14 @@ elif [[ "$1" == "bazel.sizeopt" ]]; then echo "bazel size optimized build with tests..." bazel_binary_build sizeopt echo "Testing..." - bazel test ${BAZEL_TEST_OPTIONS} //test/... --config=sizeopt + bazel test ${BAZEL_BUILD_OPTIONS} //test/... --config=sizeopt exit 0 elif [[ "$1" == "bazel.debug" ]]; then setup_clang_toolchain echo "bazel debug build with tests..." bazel_binary_build debug echo "Testing..." - bazel test ${BAZEL_TEST_OPTIONS} -c dbg //test/... + bazel test ${BAZEL_BUILD_OPTIONS} -c dbg //test/... exit 0 elif [[ "$1" == "bazel.debug.server_only" ]]; then setup_clang_toolchain @@ -146,10 +143,10 @@ elif [[ "$1" == "bazel.asan" ]]; then setup_clang_toolchain echo "bazel ASAN/UBSAN debug build with tests" echo "Building and testing envoy tests..." - bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan //test/... + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-asan //test/... echo "Building and testing envoy-filter-example tests..." pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" - bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan \ + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-asan \ //:echo2_integration_test //:envoy_binary_test popd # Also validate that integration test traffic tapping (useful when debugging etc.) @@ -159,7 +156,7 @@ elif [[ "$1" == "bazel.asan" ]]; then TAP_TMP=/tmp/tap/ rm -rf "${TAP_TMP}" mkdir -p "${TAP_TMP}" - bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-asan \ + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-asan \ //test/extensions/transport_sockets/tls/integration:ssl_integration_test \ --test_env=TAP_PATH="${TAP_TMP}/tap" # Verify that some pb_text files have been created. We can't check for pcap, @@ -171,10 +168,10 @@ elif [[ "$1" == "bazel.tsan" ]]; then setup_clang_toolchain echo "bazel TSAN debug build with tests" echo "Building and testing envoy tests..." - bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-tsan //test/... + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-tsan //test/... echo "Building and testing envoy-filter-example tests..." cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" - bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c dbg --config=clang-tsan \ + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-tsan \ //:echo2_integration_test //:envoy_binary_test exit 0 elif [[ "$1" == "bazel.dev" ]]; then @@ -185,7 +182,7 @@ elif [[ "$1" == "bazel.dev" ]]; then bazel_binary_build fastbuild echo "Building and testing..." - bazel test ${BAZEL_TEST_OPTIONS} -c fastbuild //test/... + bazel test ${BAZEL_BUILD_OPTIONS} -c fastbuild //test/... exit 0 elif [[ "$1" == "bazel.compile_time_options" ]]; then # Right now, none of the available compile-time options conflict with each other. If this @@ -208,11 +205,11 @@ elif [[ "$1" == "bazel.compile_time_options" ]]; then echo "Building..." bazel build ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg //source/exe:envoy-static echo "Building and testing..." - bazel test ${BAZEL_TEST_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg //test/... + bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg //test/... # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only # these tests under "-c opt" to save time in CI. - bazel test ${BAZEL_TEST_OPTIONS} ${COMPILE_TIME_OPTIONS} -c opt //test/common/common:assert_test //test/server:server_test + bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c opt //test/common/common:assert_test //test/server:server_test exit 0 elif [[ "$1" == "bazel.ipv6_tests" ]]; then # This is around until Circle supports IPv6. We try to run a limited set of IPv6 tests as fast @@ -230,7 +227,7 @@ elif [[ "$1" == "bazel.ipv6_tests" ]]; then setup_clang_toolchain echo "Testing..." - bazel_with_collection test ${BAZEL_TEST_OPTIONS} --test_env=ENVOY_IP_TEST_VERSIONS=v6only -c fastbuild \ + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} --test_env=ENVOY_IP_TEST_VERSIONS=v6only -c fastbuild \ //test/integration/... //test/common/network/... exit 0 elif [[ "$1" == "bazel.api" ]]; then @@ -238,7 +235,7 @@ elif [[ "$1" == "bazel.api" ]]; then echo "Building API..." bazel build ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api//envoy/... echo "Testing API..." - bazel_with_collection test ${BAZEL_TEST_OPTIONS} -c fastbuild @envoy_api//test/... @envoy_api//tools/... \ + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api//test/... @envoy_api//tools/... \ @envoy_api//tools:tap2pcap_test exit 0 elif [[ "$1" == "bazel.coverage" ]]; then @@ -257,7 +254,7 @@ elif [[ "$1" == "bazel.coverage" ]]; then # https://github.com/envoyproxy/envoy/pull/5611. # TODO(akonradi): use --local_cpu_resources flag once Bazel has a release # after 0.21. - [ -z "$CIRCLECI" ] || export BAZEL_TEST_OPTIONS="${BAZEL_TEST_OPTIONS} --local_resources=12288,4,1" + [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_resources=12288,4,1" test/run_envoy_bazel_coverage.sh collect_build_profile coverage diff --git a/ci/mac_ci_steps.sh b/ci/mac_ci_steps.sh index 50d7bba3fb40..79f47984f241 100755 --- a/ci/mac_ci_steps.sh +++ b/ci/mac_ci_steps.sh @@ -13,15 +13,15 @@ df -h . "$(dirname "$0")"/setup_cache.sh -BAZEL_BUILD_OPTIONS="--curses=no --show_task_finish --verbose_failures ${BAZEL_BUILD_EXTRA_OPTIONS} \ - --action_env=PATH=/usr/local/bin:/opt/local/bin:/usr/bin:/bin" # TODO(zuercher): remove --flaky_test_attempts when https://github.com/envoyproxy/envoy/issues/2428 # is resolved. -BAZEL_TEST_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_output=all --flaky_test_attempts=integration@2" +BAZEL_BUILD_OPTIONS="--curses=no --show_task_finish --verbose_failures \ + --action_env=PATH=/usr/local/bin:/opt/local/bin:/usr/bin:/bin --test_output=all \ + --flaky_test_attempts=integration@2 ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}" # Build envoy and run tests as separate steps so that failure output # is somewhat more deterministic (rather than interleaving the build # and test steps). bazel build ${BAZEL_BUILD_OPTIONS} //source/... //test/... -bazel test ${BAZEL_TEST_OPTIONS} //test/... +bazel test ${BAZEL_BUILD_OPTIONS} //test/... diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 78cc3c9dabb9..9306bb0dca33 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -42,7 +42,7 @@ done echo "Cleanup completed. ${NUM_PREVIOUS_GCOV_FILES} files deleted." # Force dbg for path consistency later, don't include debug code in coverage. -BAZEL_TEST_OPTIONS="${BAZEL_TEST_OPTIONS} -c dbg --copt=-DNDEBUG" +BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} -c dbg --copt=-DNDEBUG" # Run all tests under "bazel test", no sandbox. We're going to generate the # .gcda inplace in the bazel-out/ directory. This is in contrast to the "bazel @@ -50,7 +50,7 @@ BAZEL_TEST_OPTIONS="${BAZEL_TEST_OPTIONS} -c dbg --copt=-DNDEBUG" # https://github.com/bazelbuild/bazel/issues/1118). This works today as we have # a single coverage test binary and do not require the "bazel coverage" support # for collecting multiple traces and glueing them together. -"${BAZEL_COVERAGE}" test "${COVERAGE_TARGET}" ${BAZEL_TEST_OPTIONS} \ +"${BAZEL_COVERAGE}" test "${COVERAGE_TARGET}" ${BAZEL_BUILD_OPTIONS} \ --cache_test_results=no --cxxopt="--coverage" --cxxopt="-DENVOY_CONFIG_COVERAGE=1" \ --linkopt="--coverage" --define ENVOY_CONFIG_COVERAGE=1 --test_output=streamed \ --strategy=Genrule=standalone --spawn_strategy=standalone --test_timeout=4000 \ From 1dc5f64ef7738462ac0a4992b2e54a14cf6a8b95 Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Wed, 19 Jun 2019 12:45:44 +0800 Subject: [PATCH 032/542] Add OpenLab CI configuration (#6981) This patch adds the OpenLab CI configuration to enable the support for Envoy arm64 build in OpenLab. After this, each pull request in envoy will trigger the envoy-arm64-build job which verified the arm build on OpenLab ARM cluster. Related: #1861 Signed-off-by: Yikun Jiang --- .zuul.yaml | 14 ++++++++++++++ .zuul/playbooks/envoy-build/run.yaml | 28 ++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 .zuul.yaml create mode 100644 .zuul/playbooks/envoy-build/run.yaml diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 000000000000..e1e01e446e6b --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,14 @@ +- project: + name: envoyproxy/envoy + check: + jobs: + - envoy-build-arm64 + +- job: + name: envoy-build-arm64 + parent: init-test + description: | + Envoy build in openlab cluster. + run: .zuul/playbooks/envoy-build/run.yaml + nodeset: ubuntu-xenial-arm64 + voting: false diff --git a/.zuul/playbooks/envoy-build/run.yaml b/.zuul/playbooks/envoy-build/run.yaml new file mode 100644 index 000000000000..1ca8f832e08a --- /dev/null +++ b/.zuul/playbooks/envoy-build/run.yaml @@ -0,0 +1,28 @@ +- hosts: all + become: yes + roles: + - role: config-gcc + gcc_version: 7 + tasks: + - name: Build envoy + shell: + cmd: | + apt update + apt-get update + apt-get install -y \ + libtool \ + cmake \ + automake \ + autoconf \ + make \ + ninja-build \ + curl \ + unzip \ + virtualenv + + bazel build //source/exe:envoy-static | tee $LOGS_PATH//bazel.txt + + cp -r ./bazel-bin $RESULTS_PATH + chdir: '{{ zuul.project.src_dir }}' + executable: /bin/bash + environment: '{{ global_env }}' From b166f11f3710e9084d95436365ba61e8947ece4d Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 19 Jun 2019 14:36:45 -0400 Subject: [PATCH 033/542] test: deflaking */LoadStatsIntegrationTest.LocalityWeighted/* (#7225) For posterity #6875 got us the information it was a "waiting for stats" bug, which is then debuggable by lowering load stats interval to 1ms. This exposed two issues with merge. When request 1 arrived in a separate interval from request 2, the stats protos could arrive in the opposite order (fixed by ignoring order) and that if there was a stats-update with a request in progress it never resolved because we were always adding fields (oops!) I'm tempted to leave the interval low now that this is thoroughly debugged, but there's expectations on the time being within a range which wouldn't work if we lower it so meh? Risk Level: n/a (test only) Testing: yes. very much so. Docs Changes: n/a Release Notes: n/a Fixes #6874 Signed-off-by: Alyssa Wilk --- source/common/protobuf/utility.h | 16 +++---- source/common/router/scoped_rds.cc | 3 +- .../load_stats_integration_test.cc | 34 +++++++++------ test/test_common/utility.h | 43 ++++++++++++++++--- 4 files changed, 67 insertions(+), 29 deletions(-) diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index fa60e8ede2f4..5adba55f0f66 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -142,22 +142,22 @@ class RepeatedPtrUtil { } /** - * Converts a proto repeated field into a generic vector of const Protobuf::Message unique_ptr's. + * Converts a proto repeated field into a container of const Protobuf::Message unique_ptr's. * * @param repeated_field the proto repeated field to convert. - * @return ProtobufType::ConstMessagePtrVector the vector of const Message pointers. + * @return ReturnType the container of const Message pointers. */ - template - static ProtobufTypes::ConstMessagePtrVector - convertToConstMessagePtrVector(const Protobuf::RepeatedPtrField& repeated_field) { - ProtobufTypes::ConstMessagePtrVector ret_vector; - std::transform(repeated_field.begin(), repeated_field.end(), std::back_inserter(ret_vector), + template + static ReturnType + convertToConstMessagePtrContainer(const Protobuf::RepeatedPtrField& repeated_field) { + ReturnType ret_container; + std::transform(repeated_field.begin(), repeated_field.end(), std::back_inserter(ret_container), [](const ProtoType& proto_message) -> std::unique_ptr { Protobuf::Message* clone = proto_message.New(); clone->MergeFrom(proto_message); return std::unique_ptr(clone); }); - return ret_vector; + return ret_container; } }; diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9694c38f78f9..0ea1c4ed5a2d 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -35,7 +35,8 @@ create(const envoy::config::filter::network::http_connection_manager::v2::HttpCo ScopedRouteConfigurationsList& scoped_route_list = config.scoped_routes().scoped_route_configurations_list(); return scoped_routes_config_provider_manager.createStaticConfigProvider( - RepeatedPtrUtil::convertToConstMessagePtrVector( + RepeatedPtrUtil::convertToConstMessagePtrContainer( scoped_route_list.scoped_route_configurations()), factory_context, ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), diff --git a/test/integration/load_stats_integration_test.cc b/test/integration/load_stats_integration_test.cc index c846a5649580..de9a3e3434cb 100644 --- a/test/integration/load_stats_integration_test.cc +++ b/test/integration/load_stats_integration_test.cc @@ -191,15 +191,13 @@ class LoadStatsIntegrationTest : public testing::TestWithParamset_total_successful_requests( upstream_locality_stats->total_successful_requests() + local_upstream_locality_stats.total_successful_requests()); - upstream_locality_stats->set_total_requests_in_progress( - upstream_locality_stats->total_requests_in_progress() + - local_upstream_locality_stats.total_requests_in_progress()); upstream_locality_stats->set_total_error_requests( upstream_locality_stats->total_error_requests() + local_upstream_locality_stats.total_error_requests()); upstream_locality_stats->set_total_issued_requests( upstream_locality_stats->total_issued_requests() + local_upstream_locality_stats.total_issued_requests()); + // Unlike most stats, current requests in progress replaces old requests in progress. break; } } @@ -208,6 +206,25 @@ class LoadStatsIntegrationTest : public testing::TestWithParamCopyFrom(local_upstream_locality_stats); } } + + // Unfortunately because we don't issue an update when total_requests_in_progress goes from + // non-zero to zero, we have to go through and zero it out for any locality stats we didn't see. + for (int i = 0; i < cluster_stats->upstream_locality_stats_size(); ++i) { + auto upstream_locality_stats = cluster_stats->mutable_upstream_locality_stats(i); + bool found = false; + for (int j = 0; j < local_cluster_stats.upstream_locality_stats_size(); ++j) { + auto& local_upstream_locality_stats = local_cluster_stats.upstream_locality_stats(j); + if (TestUtility::protoEqual(upstream_locality_stats->locality(), + local_upstream_locality_stats.locality()) && + upstream_locality_stats->priority() == local_upstream_locality_stats.priority()) { + found = true; + break; + } + } + if (!found) { + upstream_locality_stats->set_total_requests_in_progress(0); + } + } } void waitForLoadStatsRequest( @@ -257,7 +274,7 @@ class LoadStatsIntegrationTest : public testing::TestWithParamheaders().ContentType()->value().getStringView()); } while (!TestUtility::assertRepeatedPtrFieldEqual(expected_cluster_stats, - loadstats_request.cluster_stats())); + loadstats_request.cluster_stats(), true)); } void waitForUpstreamResponse(uint32_t endpoint_index, uint32_t response_code = 200) { @@ -460,23 +477,16 @@ TEST_P(LoadStatsIntegrationTest, LocalityWeighted) { locality_weighted_lb_ = true; initialize(); - // Debug logs for #6874 - std::cerr << "Waiting for load stats stream." << std::endl; waitForLoadStatsStream(); - std::cerr << "Waiting for load stats request." << std::endl; waitForLoadStatsRequest({}); - std::cerr << "Done waiting." << std::endl; loadstats_stream_->startGrpcStream(); - std::cerr << "Starting response." << std::endl; requestLoadStatsResponse({"cluster_0"}); - std::cerr << "Updating assignments." << std::endl; // Simple 33%/67% split between dragon/winter localities. // Even though there are more endpoints in the dragon locality, the winter locality gets the // expected weighting in the WRR locality schedule. updateClusterLoadAssignment({{0}, 2}, {{1, 2}, 1}, {}, {}); - std::cerr << "Sending traffic." << std::endl; sendAndReceiveUpstream(0); sendAndReceiveUpstream(1); @@ -486,10 +496,8 @@ TEST_P(LoadStatsIntegrationTest, LocalityWeighted) { sendAndReceiveUpstream(0); // Verify we get the expect request distribution. - std::cerr << "Waiting for load stats request 2." << std::endl; waitForLoadStatsRequest( {localityStats("winter", 4, 0, 0, 4), localityStats("dragon", 2, 0, 0, 2)}); - std::cerr << "Done waiting." << std::endl; EXPECT_EQ(1, test_server_->counter("load_reporter.requests")->value()); // On slow machines, more than one load stats response may be pushed while we are simulating load. diff --git a/test/test_common/utility.h b/test/test_common/utility.h index f60d6452cb64..29d19a96d6b6 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -264,30 +264,59 @@ class TestUtility { * * @param lhs RepeatedPtrField on LHS. * @param rhs RepeatedPtrField on RHS. + * @param ignore_ordering if ordering should be ignored. Note if true this turns + * comparison into an N^2 operation. * @return bool indicating whether the RepeatedPtrField are equal. TestUtility::protoEqual() is * used for individual element testing. */ - template + template static bool repeatedPtrFieldEqual(const Protobuf::RepeatedPtrField& lhs, - const Protobuf::RepeatedPtrField& rhs) { + const Protobuf::RepeatedPtrField& rhs, + bool ignore_ordering = false) { if (lhs.size() != rhs.size()) { return false; } - for (int i = 0; i < lhs.size(); ++i) { - if (!TestUtility::protoEqual(lhs[i], rhs[i], /*ignore_repeated_field_ordering=*/false)) { + if (!ignore_ordering) { + for (int i = 0; i < lhs.size(); ++i) { + if (!TestUtility::protoEqual(lhs[i], rhs[i], /*ignore_ordering=*/false)) { + return false; + } + } + + return true; + } + typedef std::list> ProtoList; + // Iterate through using protoEqual as ignore_ordering is true, and fields + // in the sub-protos may also be out of order. + ProtoList lhs_list = + RepeatedPtrUtil::convertToConstMessagePtrContainer(lhs); + ProtoList rhs_list = + RepeatedPtrUtil::convertToConstMessagePtrContainer(rhs); + while (!lhs_list.empty()) { + bool found = false; + for (auto it = rhs_list.begin(); it != rhs_list.end(); ++it) { + if (TestUtility::protoEqual(*lhs_list.front(), **it, + /*ignore_ordering=*/true)) { + lhs_list.pop_front(); + rhs_list.erase(it); + found = true; + break; + } + } + if (!found) { return false; } } - return true; } template static AssertionResult assertRepeatedPtrFieldEqual(const Protobuf::RepeatedPtrField& lhs, - const Protobuf::RepeatedPtrField& rhs) { - if (!repeatedPtrFieldEqual(lhs, rhs)) { + const Protobuf::RepeatedPtrField& rhs, + bool ignore_ordering = false) { + if (!repeatedPtrFieldEqual(lhs, rhs, ignore_ordering)) { return AssertionFailure() << RepeatedPtrUtil::debugString(lhs) << " does not match " << RepeatedPtrUtil::debugString(rhs); } From 280c7224647ee0758033c659c1fcbc7c9b40dd2e Mon Sep 17 00:00:00 2001 From: danzh Date: Wed, 19 Jun 2019 14:45:45 -0400 Subject: [PATCH 034/542] quiche: more platform implementations and enhancement (#7283) Signed-off-by: Dan Zhang --- bazel/external/quiche.BUILD | 51 +++++++++---- bazel/repository_locations.bzl | 6 +- source/common/common/stl_helpers.h | 15 ++++ source/extensions/quic_listeners/quiche/BUILD | 4 +- .../quic_listeners/quiche/platform/BUILD | 7 +- .../quiche/platform/flags_list.h | 75 ++++++++++++++++++- .../quiche/platform/quic_client_stats_impl.h | 4 + .../platform/quic_error_code_wrappers_impl.h | 11 +++ .../quiche/platform/quic_ip_address_impl.h | 56 -------------- .../quiche/platform/quic_macros_impl.h | 12 +++ .../quiche/platform/quic_mem_slice_impl.h | 2 + .../platform/quic_mem_slice_span_impl.h | 1 + .../quiche/platform/quic_optional_impl.h | 15 ++++ .../platform/quic_socket_address_impl.h | 8 +- test/common/common/BUILD | 8 ++ test/common/common/stl_helpers_test.cc | 16 ++++ .../quic_listeners/quiche/platform/BUILD | 4 +- .../quiche/platform/quic_platform_test.cc | 32 ++++++++ 18 files changed, 243 insertions(+), 84 deletions(-) create mode 100644 source/extensions/quic_listeners/quiche/platform/quic_error_code_wrappers_impl.h delete mode 100644 source/extensions/quic_listeners/quiche/platform/quic_ip_address_impl.h create mode 100644 source/extensions/quic_listeners/quiche/platform/quic_macros_impl.h create mode 100644 source/extensions/quic_listeners/quiche/platform/quic_optional_impl.h create mode 100644 test/common/common/stl_helpers_test.cc diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index c094ad2e5125..0fec16d9d74d 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -314,11 +314,34 @@ envoy_cc_test_library( ) envoy_cc_library( - name = "quic_platform_base", - srcs = [ - "quiche/quic/platform/api/quic_ip_address.cc", - "quiche/quic/platform/api/quic_socket_address.cc", + name = "quic_platform_ip_address", + srcs = ["quiche/quic/platform/api/quic_ip_address.cc"], + hdrs = ["quiche/quic/platform/api/quic_ip_address.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_platform_base", + ":quic_platform_export", + ":quic_platform_ip_address_family", ], +) + +envoy_cc_library( + name = "quic_platform_socket_address", + srcs = ["quiche/quic/platform/api/quic_socket_address.cc"], + hdrs = ["quiche/quic/platform/api/quic_socket_address.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_platform_export", + ":quic_platform_ip_address", + ], +) + +envoy_cc_library( + name = "quic_platform_base", hdrs = [ "quiche/quic/platform/api/quic_aligned.h", "quiche/quic/platform/api/quic_arraysize.h", @@ -326,21 +349,22 @@ envoy_cc_library( "quiche/quic/platform/api/quic_client_stats.h", "quiche/quic/platform/api/quic_containers.h", "quiche/quic/platform/api/quic_endian.h", + "quiche/quic/platform/api/quic_error_code_wrappers.h", "quiche/quic/platform/api/quic_estimate_memory_usage.h", "quiche/quic/platform/api/quic_exported_stats.h", "quiche/quic/platform/api/quic_fallthrough.h", "quiche/quic/platform/api/quic_flag_utils.h", "quiche/quic/platform/api/quic_flags.h", "quiche/quic/platform/api/quic_iovec.h", - "quiche/quic/platform/api/quic_ip_address.h", "quiche/quic/platform/api/quic_logging.h", + "quiche/quic/platform/api/quic_macros.h", "quiche/quic/platform/api/quic_map_util.h", "quiche/quic/platform/api/quic_mem_slice.h", + "quiche/quic/platform/api/quic_optional.h", "quiche/quic/platform/api/quic_prefetch.h", "quiche/quic/platform/api/quic_ptr_util.h", "quiche/quic/platform/api/quic_reference_counted.h", "quiche/quic/platform/api/quic_server_stats.h", - "quiche/quic/platform/api/quic_socket_address.h", "quiche/quic/platform/api/quic_stack_trace.h", "quiche/quic/platform/api/quic_str_cat.h", "quiche/quic/platform/api/quic_stream_buffer_allocator.h", @@ -371,7 +395,7 @@ envoy_cc_library( ) envoy_cc_library( - name = "quic_core_alarm_lib", + name = "quic_core_alarm_interface", srcs = ["quiche/quic/core/quic_alarm.cc"], hdrs = ["quiche/quic/core/quic_alarm.h"], repository = "@envoy", @@ -383,12 +407,12 @@ envoy_cc_library( ) envoy_cc_library( - name = "quic_core_alarm_factory_lib", + name = "quic_core_alarm_factory_interface", hdrs = ["quiche/quic/core/quic_alarm_factory.h"], repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":quic_core_alarm_lib", + ":quic_core_alarm_interface", ":quic_core_one_block_arena_lib", ], ) @@ -589,7 +613,7 @@ envoy_cc_library( ":quic_core_interval_lib", ":quic_core_types_lib", ":quic_platform_base", - ":quic_platform_mem_slice_span_lib", + ":quic_platform_mem_slice_span", ], ) @@ -658,6 +682,7 @@ envoy_cc_library( ":quic_core_types_lib", ":quic_core_versions_lib", ":quic_platform_base", + ":quic_platform_socket_address", ], ) @@ -676,7 +701,7 @@ envoy_cc_library( ":quic_core_types_lib", ":quic_core_utils_lib", ":quic_platform_base", - ":quic_platform_mem_slice_span_lib", + ":quic_platform_mem_slice_span", ], ) @@ -788,7 +813,7 @@ envoy_cc_test( ) envoy_cc_library( - name = "quic_platform_mem_slice_span_lib", + name = "quic_platform_mem_slice_span", hdrs = [ "quiche/quic/platform/api/quic_mem_slice_span.h", ], @@ -841,7 +866,7 @@ envoy_cc_test( deps = [ ":quic_core_buffer_allocator_lib", ":quic_platform", - ":quic_platform_mem_slice_span_lib", + ":quic_platform_mem_slice_span", ":quic_platform_mem_slice_storage_lib", ":quic_platform_test", ":quic_platform_test_mem_slice_vector_lib", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 3469b49c5d5f..92f7b5f3b594 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -242,8 +242,8 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/curl/curl/releases/download/curl-7_65_1/curl-7.65.1.tar.gz"], ), com_googlesource_quiche = dict( - # Static snapshot of https://quiche.googlesource.com/quiche/+archive/7bf7c3c358eb954e463bde14ea27444f4bd8ea05.tar.gz - sha256 = "36fe180d532a9ccb18cd32328af5231636c7408104523f9ed5eebbad75f1e039", - urls = ["https://storage.googleapis.com/quiche-envoy-integration/7bf7c3c358eb954e463bde14ea27444f4bd8ea05.tar.gz"], + # Static snapshot of https://quiche.googlesource.com/quiche/+archive/88e3e05c341147f4052e17a2769ac2722739c498.tar.gz + sha256 = "74ac2169adecfdc086a4942be2758b9ec3bf1782f75d1b421b695b8f4a19cc82", + urls = ["https://storage.googleapis.com/quiche-envoy-integration/88e3e05c341147f4052e17a2769ac2722739c498.tar.gz"], ), ) diff --git a/source/common/common/stl_helpers.h b/source/common/common/stl_helpers.h index c535525191fc..74a9e4930179 100644 --- a/source/common/common/stl_helpers.h +++ b/source/common/common/stl_helpers.h @@ -2,6 +2,10 @@ #include #include +#include +#include + +#include "absl/strings/str_join.h" namespace Envoy { /** @@ -13,3 +17,14 @@ template bool containsReference(const Container& c, c }) != c.end(); } } // namespace Envoy + +// NOLINT(namespace-envoy) +// Overload functions in std library. +namespace std { +// Overload std::operator<< to output a vector. +template std::ostream& operator<<(std::ostream& out, const std::vector& v) { + out << "vector { " << absl::StrJoin(v, ", ", absl::StreamFormatter()) << " }"; + return out; +} + +} // namespace std diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 09c338fec0a5..1d9baa577b87 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -15,7 +15,7 @@ envoy_cc_library( external_deps = ["quiche_quic_platform"], deps = [ "//include/envoy/event:timer_interface", - "@com_googlesource_quiche//:quic_core_alarm_lib", + "@com_googlesource_quiche//:quic_core_alarm_interface", ], ) @@ -26,7 +26,7 @@ envoy_cc_library( external_deps = ["quiche_quic_platform"], deps = [ ":envoy_quic_alarm_lib", - "@com_googlesource_quiche//:quic_core_alarm_factory_lib", + "@com_googlesource_quiche//:quic_core_alarm_factory_interface", "@com_googlesource_quiche//:quic_core_arena_scoped_ptr_lib", "@com_googlesource_quiche//:quic_core_one_block_arena_lib", ], diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD index 49108d806959..f1660355d196 100644 --- a/source/extensions/quic_listeners/quiche/platform/BUILD +++ b/source/extensions/quic_listeners/quiche/platform/BUILD @@ -119,14 +119,16 @@ envoy_cc_library( "quic_client_stats_impl.h", "quic_containers_impl.h", "quic_endian_impl.h", + "quic_error_code_wrappers_impl.h", "quic_estimate_memory_usage_impl.h", "quic_fallthrough_impl.h", "quic_flag_utils_impl.h", "quic_flags_impl.h", "quic_iovec_impl.h", - "quic_ip_address_impl.h", + "quic_macros_impl.h", "quic_map_util_impl.h", "quic_mem_slice_impl.h", + "quic_optional_impl.h", "quic_prefetch_impl.h", "quic_ptr_util_impl.h", "quic_reference_counted_impl.h", @@ -146,6 +148,7 @@ envoy_cc_library( "abseil_memory", "abseil_node_hash_map", "abseil_node_hash_set", + "abseil_optional", "googletest", ], visibility = ["//visibility:public"], @@ -220,7 +223,7 @@ envoy_cc_library( deps = [ "@com_googlesource_quiche//:quic_core_buffer_allocator_lib", "@com_googlesource_quiche//:quic_core_utils_lib", - "@com_googlesource_quiche//:quic_platform_mem_slice_span_lib", + "@com_googlesource_quiche//:quic_platform_mem_slice_span", ], ) diff --git a/source/extensions/quic_listeners/quiche/platform/flags_list.h b/source/extensions/quic_listeners/quiche/platform/flags_list.h index 2c99774b557c..e334ff8f21e1 100644 --- a/source/extensions/quic_listeners/quiche/platform/flags_list.h +++ b/source/extensions/quic_listeners/quiche/platform/flags_list.h @@ -292,8 +292,53 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_validate_packet_number_post_decrypti "If true, a QUIC endpoint will valid a received packet number after " "successfully decrypting the packet.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_v44_disable_trial_decryption, false, - "Disables trial decryption in QUIC v44 and above.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_send_version_negotiation_fixed_bit, false, + "When true, version negotiation packets sent by the server will set the fixed bit.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_eliminate_static_stream_map_3, false, + "If true, static streams in a QuicSession will be stored inside dynamic stream map. " + "static_stream_map will no longer be used.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_simplify_stop_waiting, false, + "Do not send STOP_WAITING if no_stop_waiting_frame_ is true.") + +QUICHE_FLAG(bool, quic_reloadable_flag_send_quic_fallback_server_config_on_leto_error, false, + "If true and using Leto for QUIC shared-key calculations, GFE will react to a failure " + "to contact Leto by sending a REJ containing a fallback ServerConfig, allowing the " + "client to continue the handshake.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_bbr_cwnd_in_bandwidth_resumption, true, + " If true, adjust congestion window when doing bandwidth resumption in BBR.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_lumpy_pacing_at_low_bw, false, + "If true, disable lumpy pacing for low bandwidth flows.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_cwnd_and_pacing_gains, false, + "If true, uses conservative cwnd gain and pacing gain.") + +QUICHE_FLAG( + bool, quic_reloadable_flag_quic_do_not_accept_stop_waiting, false, + "In v44 and above, where STOP_WAITING is never sent, close the connection if it's received.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_loss_removes_from_inflight, false, + "When true, remove packets from inflight where they're declared lost, rather than in " + "MarkForRetransmission. Also no longer marks handshake packets as no longer inflight " + "when they're retransmitted.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_bursts, false, + "If true, set burst token to 2 in cwnd bootstrapping experiment.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_deprecate_queued_control_frames, false, + "If true, deprecate queued_control_frames_ from QuicPacketGenerator.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_check_connected_before_flush, false, + "If true, check whether connection is connected before flush.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_ignore_tlpr_if_sending_ping, false, + "If true, ignore TLPR for retransmission delay when sending pings from ping alarm.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_terminate_gquic_connection_as_ietf, false, + "If true, terminate Google QUIC connections similarly as IETF QUIC.") QUICHE_FLAG(bool, quic_restart_flag_quic_allow_loas_multipacket_chlo, false, "If true, inspects QUIC CHLOs for kLOAS and early creates sessions " @@ -345,6 +390,29 @@ QUICHE_FLAG(bool, quic_restart_flag_quic_use_pigeon_socket_to_backend, false, "If true, create a shared pigeon socket for all quic to backend " "connections and switch to use it after successful handshake.") +QUICHE_FLAG(bool, quic_restart_flag_quic_do_not_override_connection_id, false, + " When true, QuicFramer will not override connection IDs in headers and will instead " + "respect the source/destination direction as expected by IETF QUIC.") + +QUICHE_FLAG(bool, quic_restart_flag_quic_server_drop_version_negotiation, false, + "When true, QUIC server will drop IETF QUIC Version Negotiation packets.") + +QUICHE_FLAG( + bool, quic_restart_flag_quic_allow_variable_length_connection_id_for_negotiation, false, + "When true, allow variable length QUIC connection IDs for unsupported versions. This allows " + "performing version negotiation when the client-chosen server connection ID length is not 8") + +QUICHE_FLAG(bool, quic_restart_flag_quic_no_framer_object_in_dispatcher, false, + "If true, make QuicDispatcher no longer have an instance of QuicFramer.") + +QUICHE_FLAG( + bool, quic_restart_flag_dont_fetch_quic_private_keys_from_leto, false, + "If true, GFE will not request private keys when fetching QUIC ServerConfigs from Leto.") + +QUICHE_FLAG(bool, quic_restart_flag_quic_use_allocated_connection_ids, false, + "When true, QuicConnectionId will allocate long connection IDs on the heap instead of " + "inline in the object.") + QUICHE_FLAG(bool, quic_allow_chlo_buffering, true, "If true, allows packets to be buffered in anticipation of a " "future CHLO, and allow CHLO packets to be buffered until next " @@ -399,6 +467,9 @@ QUICHE_FLAG(double, quic_pace_time_into_future_srtt_fraction, 0.125f, // One-eighth smoothed RTT "Smoothed RTT fraction that a connection can pace packets into the future.") +QUICHE_FLAG(bool, quic_export_server_num_packets_per_write_histogram, false, + "If true, export number of packets written per write operation histogram.") + QUICHE_FLAG(bool, http2_reloadable_flag_http2_testonly_default_false, false, "A testonly reloadable flag that will always default to false.") diff --git a/source/extensions/quic_listeners/quiche/platform/quic_client_stats_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_client_stats_impl.h index 7108de135e2b..af9850d0bd22 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_client_stats_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_client_stats_impl.h @@ -14,15 +14,19 @@ #define QUIC_CLIENT_HISTOGRAM_ENUM_IMPL(name, sample, enum_size, docstring) \ do { \ + (void)(sample); \ } while (0) #define QUIC_CLIENT_HISTOGRAM_BOOL_IMPL(name, sample, docstring) \ + (void)(sample); \ do { \ } while (0) #define QUIC_CLIENT_HISTOGRAM_TIMES_IMPL(name, sample, min, max, num_buckets, docstring) \ do { \ + (void)(sample); \ } while (0) #define QUIC_CLIENT_HISTOGRAM_COUNTS_IMPL(name, sample, min, max, num_buckets, docstring) \ do { \ + (void)(sample); \ } while (0) namespace quic { diff --git a/source/extensions/quic_listeners/quiche/platform/quic_error_code_wrappers_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_error_code_wrappers_impl.h new file mode 100644 index 000000000000..8d6901c04700 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/platform/quic_error_code_wrappers_impl.h @@ -0,0 +1,11 @@ +#pragma once + +// NOLINT(namespace-envoy) + +// This file is part of the QUICHE platform implementation, and is not to be +// consumed or referenced directly by other Envoy code. It serves purely as a +// porting layer for QUICHE. + +#include + +#define QUIC_EMSGSIZE_IMPL EMSGSIZE diff --git a/source/extensions/quic_listeners/quiche/platform/quic_ip_address_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_ip_address_impl.h deleted file mode 100644 index 6d848b6aa5aa..000000000000 --- a/source/extensions/quic_listeners/quiche/platform/quic_ip_address_impl.h +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -// NOLINT(namespace-envoy) - -// This file is part of the QUICHE platform implementation, and is not to be -// consumed or referenced directly by other Envoy code. It serves purely as a -// porting layer for QUICHE. - -#include - -#include - -#include "quiche/quic/platform/api/quic_ip_address_family.h" - -namespace quic { - -// Implements the interface required by -// https://quiche.googlesource.com/quiche/+/refs/heads/master/quic/platform/api/quic_ip_address.h -// This is a dummy implementation which just allows its dependency to build. -// TODO(vasilvv) Remove this impl once QuicSocketAddress and QuicIpAddress are -// removed from platform API. - -class QuicIpAddressImpl { -public: - enum : size_t { kIPv4AddressSize = sizeof(in_addr), kIPv6AddressSize = sizeof(in6_addr) }; - static QuicIpAddressImpl Loopback4() { return QuicIpAddressImpl(); } - static QuicIpAddressImpl Loopback6() { return QuicIpAddressImpl(); } - static QuicIpAddressImpl Any4() { return QuicIpAddressImpl(); } - static QuicIpAddressImpl Any6() { return QuicIpAddressImpl(); } - - QuicIpAddressImpl() = default; - QuicIpAddressImpl(const in_addr&) {} - QuicIpAddressImpl(const in6_addr&) {} - QuicIpAddressImpl(const QuicIpAddressImpl& other) = default; - QuicIpAddressImpl& operator=(const QuicIpAddressImpl& other) = default; - QuicIpAddressImpl& operator=(QuicIpAddressImpl&& other) = default; - friend bool operator==(QuicIpAddressImpl, QuicIpAddressImpl) { return false; } - friend bool operator!=(QuicIpAddressImpl, QuicIpAddressImpl) { return true; } - - bool IsInitialized() const { return false; } - IpAddressFamily address_family() const { return IpAddressFamily::IP_V4; } - int AddressFamilyToInt() const { return 4; } - std::string ToPackedString() const { return "Unimplemented"; } - std::string ToString() const { return "Unimplemented"; } - QuicIpAddressImpl Normalized() const { return QuicIpAddressImpl(); } - QuicIpAddressImpl DualStacked() const { return QuicIpAddressImpl(); } - bool FromPackedString(const char*, size_t) { return false; } - bool FromString(const std::string&) { return false; } - bool IsIPv4() const { return true; } - bool IsIPv6() const { return false; } - in_addr GetIPv4() const { return in_addr(); } - in6_addr GetIPv6() const { return in6_addr(); } - bool InSameSubnet(const QuicIpAddressImpl&, int) { return false; } -}; - -} // namespace quic diff --git a/source/extensions/quic_listeners/quiche/platform/quic_macros_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_macros_impl.h new file mode 100644 index 000000000000..eb8ce413fb8a --- /dev/null +++ b/source/extensions/quic_listeners/quiche/platform/quic_macros_impl.h @@ -0,0 +1,12 @@ +#pragma once + +// NOLINT(namespace-envoy) + +// This file is part of the QUICHE platform implementation, and is not to be +// consumed or referenced directly by other Envoy code. It serves purely as a +// porting layer for QUICHE. + +#include "absl/base/attributes.h" + +#define QUIC_MUST_USE_RESULT_IMPL ABSL_MUST_USE_RESULT +#define QUIC_UNUSED_IMPL ABSL_ATTRIBUTE_UNUSED diff --git a/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_impl.h index d0f9c434fd6e..e1dc857ae581 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_impl.h @@ -56,6 +56,8 @@ class QuicMemSliceImpl { size_t length() const { return single_slice_buffer_.length(); } bool empty() const { return length() == 0; } + Envoy::Buffer::OwnedImpl& single_slice_buffer() { return single_slice_buffer_; } + private: // Prerequisite: buffer has at least one slice. size_t firstSliceLength(Envoy::Buffer::Instance& buffer); diff --git a/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_span_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_span_impl.h index f23487862e8f..675de20ad4fd 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_span_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_span_impl.h @@ -26,6 +26,7 @@ class QuicMemSliceSpanImpl { * @param buffer has to outlive the life time of this class. */ explicit QuicMemSliceSpanImpl(Envoy::Buffer::Instance& buffer) : buffer_(&buffer) {} + explicit QuicMemSliceSpanImpl(QuicMemSliceImpl* slice) : buffer_(&slice->single_slice_buffer()) {} QuicMemSliceSpanImpl(const QuicMemSliceSpanImpl& other) = default; QuicMemSliceSpanImpl& operator=(const QuicMemSliceSpanImpl& other) = default; diff --git a/source/extensions/quic_listeners/quiche/platform/quic_optional_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_optional_impl.h new file mode 100644 index 000000000000..0c03fc3a0aa6 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/platform/quic_optional_impl.h @@ -0,0 +1,15 @@ +#pragma once + +// NOLINT(namespace-envoy) + +// This file is part of the QUICHE platform implementation, and is not to be +// consumed or referenced directly by other Envoy code. It serves purely as a +// porting layer for QUICHE. + +#include "absl/types/optional.h" + +namespace quic { + +template using QuicOptionalImpl = absl::optional; + +} // namespace quic diff --git a/source/extensions/quic_listeners/quiche/platform/quic_socket_address_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_socket_address_impl.h index 5b33538719e2..77a5a38edef4 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_socket_address_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_socket_address_impl.h @@ -10,7 +10,7 @@ #include -#include "extensions/quic_listeners/quiche/platform/quic_ip_address_impl.h" +#include "quiche/quic/platform/api/quic_ip_address.h" namespace quic { @@ -23,9 +23,9 @@ namespace quic { class QuicSocketAddressImpl { public: QuicSocketAddressImpl() = default; - QuicSocketAddressImpl(QuicIpAddressImpl, uint16_t) {} + QuicSocketAddressImpl(QuicIpAddress, uint16_t) {} explicit QuicSocketAddressImpl(const struct sockaddr_storage&) {} - explicit QuicSocketAddressImpl(const struct sockaddr&) {} + QuicSocketAddressImpl(const struct sockaddr*, socklen_t&) {} QuicSocketAddressImpl(const QuicSocketAddressImpl&) = default; QuicSocketAddressImpl& operator=(const QuicSocketAddressImpl&) = default; QuicSocketAddressImpl& operator=(QuicSocketAddressImpl&&) = default; @@ -37,7 +37,7 @@ class QuicSocketAddressImpl { int FromSocket(int) { return -1; } QuicSocketAddressImpl Normalized() const { return QuicSocketAddressImpl(); } - QuicIpAddressImpl host() const { return QuicIpAddressImpl(); } + QuicIpAddress host() const { return QuicIpAddress(); } uint16_t port() const { return 0; } sockaddr_storage generic_address() const { return sockaddr_storage{}; } diff --git a/test/common/common/BUILD b/test/common/common/BUILD index 2aa4a66dc28f..9d2d93940b54 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -203,3 +203,11 @@ envoy_cc_test( "//source/common/common:thread_lib", ], ) + +envoy_cc_test( + name = "stl_helpers_test", + srcs = ["stl_helpers_test.cc"], + deps = [ + "//source/common/common:stl_helpers", + ], +) diff --git a/test/common/common/stl_helpers_test.cc b/test/common/common/stl_helpers_test.cc new file mode 100644 index 000000000000..2baa0ed559e9 --- /dev/null +++ b/test/common/common/stl_helpers_test.cc @@ -0,0 +1,16 @@ +#include + +#include "common/common/stl_helpers.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +TEST(StlHelpersTest, TestOutputToStreamOperator) { + std::stringstream os; + std::vector v{1, 2, 3, 4, 5}; + os << v; + EXPECT_EQ("vector { 1, 2, 3, 4, 5 }", os.str()); +} + +} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/platform/BUILD b/test/extensions/quic_listeners/quiche/platform/BUILD index 8028d199a63d..346c76ce24b7 100644 --- a/test/extensions/quic_listeners/quiche/platform/BUILD +++ b/test/extensions/quic_listeners/quiche/platform/BUILD @@ -50,7 +50,7 @@ envoy_cc_test( "@com_googlesource_quiche//:quic_core_error_codes_lib", "@com_googlesource_quiche//:quic_core_types_lib", "@com_googlesource_quiche//:quic_platform_expect_bug", - "@com_googlesource_quiche//:quic_platform_mem_slice_span_lib", + "@com_googlesource_quiche//:quic_platform_mem_slice_span", "@com_googlesource_quiche//:quic_platform_mem_slice_storage_lib", "@com_googlesource_quiche//:quic_platform_mock_log", "@com_googlesource_quiche//:quic_platform_port_utils", @@ -155,7 +155,7 @@ envoy_cc_test_library( hdrs = ["quic_test_mem_slice_vector_impl.h"], deps = [ "//include/envoy/buffer:buffer_interface", - "@com_googlesource_quiche//:quic_platform_mem_slice_span_lib", + "@com_googlesource_quiche//:quic_platform_mem_slice_span", ], ) diff --git a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc index b77b6805c394..8e25b2dd1337 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc @@ -42,12 +42,14 @@ #include "quiche/quic/platform/api/quic_flags.h" #include "quiche/quic/platform/api/quic_hostname_utils.h" #include "quiche/quic/platform/api/quic_logging.h" +#include "quiche/quic/platform/api/quic_macros.h" #include "quiche/quic/platform/api/quic_map_util.h" #include "quiche/quic/platform/api/quic_mem_slice.h" #include "quiche/quic/platform/api/quic_mem_slice_span.h" #include "quiche/quic/platform/api/quic_mem_slice_storage.h" #include "quiche/quic/platform/api/quic_mock_log.h" #include "quiche/quic/platform/api/quic_mutex.h" +#include "quiche/quic/platform/api/quic_optional.h" #include "quiche/quic/platform/api/quic_pcc_sender.h" #include "quiche/quic/platform/api/quic_port_utils.h" #include "quiche/quic/platform/api/quic_ptr_util.h" @@ -120,6 +122,9 @@ TEST_F(QuicPlatformTest, QuicClientStats) { 100, "doc"); QUIC_CLIENT_HISTOGRAM_COUNTS("my.count.histogram", 123, 0, 1000, 100, "doc"); QuicClientSparseHistogram("my.sparse.histogram", 345); + // Make sure compiler doesn't report unused-parameter error. + bool should_be_used; + QUIC_CLIENT_HISTOGRAM_BOOL("my.bool.histogram", should_be_used, "doc"); } TEST_F(QuicPlatformTest, QuicExpectBug) { @@ -704,6 +709,21 @@ TEST_F(QuicPlatformTest, TestSystemEventLoop) { QuicSystemEventLoop("dummy"); } +QUIC_MUST_USE_RESULT bool dummyTestFunction() { return false; } + +TEST_F(QuicPlatformTest, TestQuicMacros) { + // Just make sure it compiles. + EXPECT_FALSE(dummyTestFunction()); + int a QUIC_UNUSED; +} + +TEST_F(QuicPlatformTest, TestQuicOptional) { + QuicOptional maybe_a; + EXPECT_FALSE(maybe_a.has_value()); + maybe_a = 1; + EXPECT_EQ(1, *maybe_a); +} + class QuicMemSliceTest : public Envoy::Buffer::BufferImplementationParamTest { public: ~QuicMemSliceTest() override {} @@ -752,6 +772,18 @@ TEST_P(QuicMemSliceTest, ConstructMemSliceFromBuffer) { EXPECT_TRUE(fragment_releaser_called); } +TEST_P(QuicMemSliceTest, ConstructQuicMemSliceSpan) { + Envoy::Buffer::OwnedImpl buffer; + Envoy::Buffer::BufferImplementationParamTest::verifyImplementation(buffer); + std::string str(1024, 'a'); + buffer.add(str); + quic::QuicMemSlice slice{quic::QuicMemSliceImpl(buffer, str.length())}; + + QuicMemSliceSpan span(&slice); + EXPECT_EQ(1024u, span.total_length()); + EXPECT_EQ(str, span.GetData(0)); +} + TEST_P(QuicMemSliceTest, QuicMemSliceStorage) { std::string str(512, 'a'); struct iovec iov = {const_cast(str.data()), str.length()}; From 2d810b4add11d6d05e87113736f440d3d23e69fa Mon Sep 17 00:00:00 2001 From: Yangmin Zhu Date: Wed, 19 Jun 2019 20:02:44 -0700 Subject: [PATCH 035/542] rbac: add rbac to architecture overview (#7289) Signed-off-by: Yangmin Zhu --- .../arch_overview/security/rbac_filter.rst | 35 +++++++++++++++++++ .../intro/arch_overview/security/security.rst | 1 + 2 files changed, 36 insertions(+) create mode 100644 docs/root/intro/arch_overview/security/rbac_filter.rst diff --git a/docs/root/intro/arch_overview/security/rbac_filter.rst b/docs/root/intro/arch_overview/security/rbac_filter.rst new file mode 100644 index 000000000000..a0fba7097e78 --- /dev/null +++ b/docs/root/intro/arch_overview/security/rbac_filter.rst @@ -0,0 +1,35 @@ +.. _arch_overview_rbac: + +Role Based Access Control +========================= + +* :ref:`Network filter configuration `. +* :ref:`HTTP filter configuration `. + +The Role Based Access Control (RBAC) filter checks if the incoming request is authorized or not. +Unlike external authorization, the check of RBAC filter happens in the Envoy process and is +based on a list of policies from the filter config. + +The RBAC filter can be either configured as a :ref:`network filter `, +or as a :ref:`HTTP filter ` or both. If the request is deemed unauthorized +by the network filter then the connection will be closed. If the request is deemed unauthorized by +the HTTP filter the request will be denied with 403 (Forbidden) response. + +Policy +------ + +The RBAC filter checks the request based on a list of +:ref:`policies `. A policy consists of a list of +:ref:`permissions ` and +:ref:`principals `. The permission specifies the actions of +the request, for example, the method and path of a HTTP request. The principal specifies the +downstream client identities of the request, for example, the URI SAN of the downstream client +certificate. A policy is matched if its permissions and principals are matched at the same time. + +Shadow Policy +------------- + +The filter can be configured with a +:ref:`shadow policy ` that doesn't +have any effect (i.e. not deny the request) but only emit stats and log the result. This is useful +for testing a rule before applying in production. diff --git a/docs/root/intro/arch_overview/security/security.rst b/docs/root/intro/arch_overview/security/security.rst index 00488a82559a..9a2be26f81f2 100644 --- a/docs/root/intro/arch_overview/security/security.rst +++ b/docs/root/intro/arch_overview/security/security.rst @@ -6,3 +6,4 @@ Security ssl ext_authz_filter + rbac_filter From 9dd4ded601016607b55893e7f14c278ad0bc7c0f Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Thu, 20 Jun 2019 09:03:15 -0700 Subject: [PATCH 036/542] fault: destroy rate limiter during reset (#7327) Probably related to https://github.com/envoyproxy/envoy/issues/7179 Signed-off-by: Matt Klein --- .../filters/http/fault/fault_filter.cc | 8 +++++++- .../filters/http/fault/fault_filter.h | 7 +++++++ .../filters/http/fault/fault_filter_test.cc | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 2d5ad0c9d62a..4c7a73c3f43d 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -72,7 +72,10 @@ FaultFilterConfig::FaultFilterConfig(const envoy::config::filter::http::fault::v FaultFilter::FaultFilter(FaultFilterConfigSharedPtr config) : config_(config) {} -FaultFilter::~FaultFilter() { ASSERT(!delay_timer_); } +FaultFilter::~FaultFilter() { + ASSERT(delay_timer_ == nullptr); + ASSERT(response_limiter_ == nullptr || response_limiter_->destroyed()); +} // Delays and aborts are independent events. One can inject a delay // followed by an abort or inject just a delay or abort. In this callback, @@ -323,6 +326,9 @@ void FaultFilter::maybeIncActiveFaults() { void FaultFilter::onDestroy() { resetTimerState(); + if (response_limiter_ != nullptr) { + response_limiter_->destroy(); + } if (fault_active_) { config_->stats().active_faults_.dec(); } diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index 8185133e73ab..b350ce61ce64 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -136,6 +136,13 @@ class StreamRateLimiter : Logger::Loggable { */ Http::FilterTrailersStatus onTrailers(); + /** + * Like the owning filter, we must handle inline destruction, so we have a destroy() method which + * kills any callbacks. + */ + void destroy() { token_timer_.reset(); } + bool destroyed() { return token_timer_ == nullptr; } + private: void onTokenTimer(); diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index 536be5232b71..a22a9819a7f7 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -1015,6 +1015,23 @@ TEST_F(FaultFilterRateLimitTest, ResponseRateLimitDisabled) { EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(request_headers_)); } +// Make sure we destroy the rate limiter if we are reset. +TEST_F(FaultFilterRateLimitTest, DestroyWithResponseRateLimitEnabled) { + setupRateLimitTest(true); + + ON_CALL(encoder_filter_callbacks_, encoderBufferLimit()).WillByDefault(Return(1100)); + // The timer is consumed but not used by this test. + new NiceMock(&decoder_filter_callbacks_.dispatcher_); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, true)); + + EXPECT_EQ(1UL, config_->stats().response_rl_injected_.value()); + EXPECT_EQ(1UL, config_->stats().active_faults_.value()); + + filter_->onDestroy(); + + EXPECT_EQ(0UL, config_->stats().active_faults_.value()); +} + TEST_F(FaultFilterRateLimitTest, ResponseRateLimitEnabled) { setupRateLimitTest(true); From 49b5a915324e44b9445e3fc5434e0deffaf3f94f Mon Sep 17 00:00:00 2001 From: htuch Date: Thu, 20 Jun 2019 17:16:02 +0100 Subject: [PATCH 037/542] runtime: distinct symlink root and subdir in layered runtime. (#7335) Without a distinction between the root and subdir, we can't watch for symlink swaps that cover multiple layers. Risk level: Low Testing: additional test expects added. Signed-off-by: Harvey Tuch --- api/envoy/config/bootstrap/v2/bootstrap.proto | 5 +++++ docs/root/configuration/runtime.rst | 4 ++-- source/common/config/runtime_utility.cc | 8 ++++---- source/common/runtime/runtime_impl.cc | 3 ++- test/common/config/runtime_utility_test.cc | 9 ++++++--- test/common/runtime/runtime_impl_test.cc | 9 ++++++--- test/server/configuration_impl_test.cc | 8 ++++---- 7 files changed, 29 insertions(+), 17 deletions(-) diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index d7d412da4fdd..f8083ca6bf8a 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -248,6 +248,11 @@ message RuntimeLayer { // treated. string symlink_root = 1; + // Specifies the subdirectory to load within the root directory. This is + // useful if multiple systems share the same delivery mechanism. Envoy + // configuration elements can be contained in a dedicated subdirectory. + string subdirectory = 3; + // :ref:`Append ` the // service cluster to the path under symlink root. bool append_service_cluster = 2; diff --git a/docs/root/configuration/runtime.rst b/docs/root/configuration/runtime.rst index c0e7109ba166..b240c472d930 100644 --- a/docs/root/configuration/runtime.rst +++ b/docs/root/configuration/runtime.rst @@ -30,8 +30,8 @@ be: - static_layer: health_check: min_interval: 5 - - disk_layer: { symlink_root: /srv/runtime/current/envoy } - - disk_layer: { symlink_root: /srv/runtime/current/envoy_override, append_service_cluster: true } + - disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy } + - disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy_override, append_service_cluster: true } - admin_layer: {} In the deprecated :ref:`runtime ` bootstrap diff --git a/source/common/config/runtime_utility.cc b/source/common/config/runtime_utility.cc index ffd4d3a9657c..ef1f8e45ca15 100644 --- a/source/common/config/runtime_utility.cc +++ b/source/common/config/runtime_utility.cc @@ -14,14 +14,14 @@ void translateRuntime(const envoy::config::bootstrap::v2::Runtime& runtime_confi { auto* layer = layered_runtime_config.add_layers(); layer->set_name("root"); - layer->mutable_disk_layer()->set_symlink_root(runtime_config.symlink_root() + "/" + - runtime_config.subdirectory()); + layer->mutable_disk_layer()->set_symlink_root(runtime_config.symlink_root()); + layer->mutable_disk_layer()->set_subdirectory(runtime_config.subdirectory()); } if (!runtime_config.override_subdirectory().empty()) { auto* layer = layered_runtime_config.add_layers(); layer->set_name("override"); - layer->mutable_disk_layer()->set_symlink_root(runtime_config.symlink_root() + "/" + - runtime_config.override_subdirectory()); + layer->mutable_disk_layer()->set_symlink_root(runtime_config.symlink_root()); + layer->mutable_disk_layer()->set_subdirectory(runtime_config.override_subdirectory()); layer->mutable_disk_layer()->set_append_service_cluster(true); } } diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 03f654dc1b18..a1369e6ab141 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -585,7 +585,8 @@ std::unique_ptr LoaderImpl::createNewSnapshot() { layers.emplace_back(std::make_unique(layer.name(), layer.static_layer())); break; case envoy::config::bootstrap::v2::RuntimeLayer::kDiskLayer: { - std::string path = layer.disk_layer().symlink_root(); + std::string path = + layer.disk_layer().symlink_root() + "/" + layer.disk_layer().subdirectory(); if (layer.disk_layer().append_service_cluster()) { path += "/" + service_cluster_; } diff --git a/test/common/config/runtime_utility_test.cc b/test/common/config/runtime_utility_test.cc index 6b5d35b434d6..0451f1a6b9a2 100644 --- a/test/common/config/runtime_utility_test.cc +++ b/test/common/config/runtime_utility_test.cc @@ -41,7 +41,8 @@ TEST(RuntimeUtility, TranslateSubdirOnly) { { auto* layer = expected_runtime_config.add_layers(); layer->set_name("root"); - layer->mutable_disk_layer()->set_symlink_root("foo/bar"); + layer->mutable_disk_layer()->set_symlink_root("foo"); + layer->mutable_disk_layer()->set_subdirectory("bar"); } { auto* layer = expected_runtime_config.add_layers(); @@ -67,12 +68,14 @@ TEST(RuntimeUtility, TranslateSubdirOverride) { { auto* layer = expected_runtime_config.add_layers(); layer->set_name("root"); - layer->mutable_disk_layer()->set_symlink_root("foo/bar"); + layer->mutable_disk_layer()->set_symlink_root("foo"); + layer->mutable_disk_layer()->set_subdirectory("bar"); } { auto* layer = expected_runtime_config.add_layers(); layer->set_name("override"); - layer->mutable_disk_layer()->set_symlink_root("foo/baz"); + layer->mutable_disk_layer()->set_symlink_root("foo"); + layer->mutable_disk_layer()->set_subdirectory("baz"); layer->mutable_disk_layer()->set_append_service_cluster(true); } { diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index c192268e3c0d..3c18b387e289 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -79,8 +79,9 @@ class LoaderImplTest : public testing::Test { EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillRepeatedly(InvokeWithoutArgs([this] { Filesystem::MockWatcher* mock_watcher = new NiceMock(); EXPECT_CALL(*mock_watcher, addWatch(_, Filesystem::Watcher::Events::MovedTo, _)) - .WillRepeatedly( - Invoke([this](const std::string&, uint32_t, Filesystem::Watcher::OnChangedCb cb) { + .WillRepeatedly(Invoke( + [this](const std::string& path, uint32_t, Filesystem::Watcher::OnChangedCb cb) { + EXPECT_EQ(path, expected_watch_root_); on_changed_cbs_.emplace_back(cb); })); return mock_watcher; @@ -98,6 +99,7 @@ class LoaderImplTest : public testing::Test { Init::MockManager init_manager_; std::vector on_changed_cbs_; NiceMock validation_visitor_; + std::string expected_watch_root_; }; class DiskLoaderImplTest : public LoaderImplTest { @@ -114,7 +116,8 @@ class DiskLoaderImplTest : public LoaderImplTest { void run(const std::string& primary_dir, const std::string& override_dir) { envoy::config::bootstrap::v2::Runtime runtime; runtime.mutable_base()->MergeFrom(base_); - runtime.set_symlink_root(TestEnvironment::temporaryPath(primary_dir)); + expected_watch_root_ = TestEnvironment::temporaryPath(primary_dir); + runtime.set_symlink_root(expected_watch_root_); runtime.set_subdirectory("envoy"); runtime.set_override_subdirectory(override_dir); diff --git a/test/server/configuration_impl_test.cc b/test/server/configuration_impl_test.cc index 13fc76644559..732c47126c51 100644 --- a/test/server/configuration_impl_test.cc +++ b/test/server/configuration_impl_test.cc @@ -380,9 +380,9 @@ TEST(InitialImplTest, LayeredRuntime) { health_check: min_interval: 5 - name: root - disk_layer: { symlink_root: /srv/runtime/current/envoy } + disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy } - name: override - disk_layer: { symlink_root: /srv/runtime/current/envoy_override, append_service_cluster: true } + disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy_override, append_service_cluster: true } - name: admin admin_layer: {} )EOF"; @@ -448,9 +448,9 @@ TEST(InitialImplTest, DeprecatedRuntimeTranslation) { health_check: min_interval: 5 - name: root - disk_layer: { symlink_root: /srv/runtime/current/envoy } + disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy } - name: override - disk_layer: { symlink_root: /srv/runtime/current/envoy_override, append_service_cluster: true } + disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy_override, append_service_cluster: true } - name: admin admin_layer: {} )EOF"; From e0859623248fe445473d88eb85af717354e006d6 Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 20 Jun 2019 14:41:31 -0400 Subject: [PATCH 038/542] fuzz: fix oss-fuzz build failure (#7339) Signed-off-by: Asra Ali --- test/test_common/environment.cc | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index 68915dba5f1d..70050aaab309 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -4,7 +4,10 @@ #include #ifdef __has_include -#if __has_include() +#if __has_include() +#include +// TODO(asraa): Remove this when Envoy requires Clang >= 9. +#elif __has_include() #include #endif #endif @@ -38,7 +41,11 @@ std::string makeTempDir(char* name_template) { char* dirname = ::_mktemp(name_template); RELEASE_ASSERT(dirname != nullptr, fmt::format("failed to create tempdir from template: {} {}", name_template, strerror(errno))); +#ifdef __cpp_lib_filesystem + std::filesystem::create_directories(dirname); +#elifdef __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories(dirname); +#endif #else char* dirname = ::mkdtemp(name_template); RELEASE_ASSERT(dirname != nullptr, fmt::format("failed to create tempdir from template: {} {}", @@ -77,39 +84,48 @@ char** argv_; } // namespace void TestEnvironment::createPath(const std::string& path) { -#ifdef __cpp_lib_experimental_filesystem +#ifdef __cpp_lib_filesystem // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. + std::filesystem::create_directories(std::filesystem::path(path)); +#elifdef __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories(std::experimental::filesystem::path(path)); #else - // No support on this system for std::experimental::filesystem. + // No support on this system for std::filesystem or std::experimental::filesystem. RELEASE_ASSERT(::system(("mkdir -p " + path).c_str()) == 0, ""); #endif } void TestEnvironment::createParentPath(const std::string& path) { -#ifdef __cpp_lib_experimental_filesystem +#ifdef __cpp_lib_filesystem // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. + std::filesystem::create_directories(std::filesystem::path(path).parent_path()); +#elifdef __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories( std::experimental::filesystem::path(path).parent_path()); #else - // No support on this system for std::experimental::filesystem. + // No support on this system for std::filesystem or std::experimental::filesystem. RELEASE_ASSERT(::system(("mkdir -p $(dirname " + path + ")").c_str()) == 0, ""); #endif } void TestEnvironment::removePath(const std::string& path) { RELEASE_ASSERT(absl::StartsWith(path, TestEnvironment::temporaryDirectory()), ""); -#ifdef __cpp_lib_experimental_filesystem - // We don't want to rely on rm etc. if we can avoid it, since it might not +#ifdef __cpp_lib_filesystem + // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. + if (!std::filesystem::exists(path)) { + return; + } + std::filesystem::remove_all(std::filesystem::path(path)); +#elifdef __cpp_lib_experimental_filesystem if (!std::experimental::filesystem::exists(path)) { return; } std::experimental::filesystem::remove_all(std::experimental::filesystem::path(path)); #else - // No support on this system for std::experimental::filesystem. + // No support on this system for std::filesystem or std::experimental::filesystem. RELEASE_ASSERT(::system(("rm -rf " + path).c_str()) == 0, ""); #endif } From ce78705977f499688558fa1fa7d2722bc90580e6 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Thu, 20 Jun 2019 11:42:40 -0700 Subject: [PATCH 039/542] ci: add the ability to run only a subset of tests. (#7269) While there, cleanup and add missing targets to the README.md file. Signed-off-by: Piotr Sikora --- bazel/README.md | 9 +-- ci/README.md | 17 ++++-- ci/do_ci.sh | 98 ++++++++++++++++---------------- ci/do_circle_ci.sh | 2 +- ci/mac_ci_steps.sh | 10 +++- test/coverage/gen_build.sh | 14 ++++- test/run_envoy_bazel_coverage.sh | 18 ++++-- 7 files changed, 102 insertions(+), 66 deletions(-) diff --git a/bazel/README.md b/bazel/README.md index 0b7cbeea22bd..0eba43900fa5 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -454,12 +454,13 @@ then log back in and it should start working. The latest coverage report for master is available [here](https://s3.amazonaws.com/lyft-envoy/coverage/report-master/coverage.html). -It's also possible to specialize the coverage build to a single test target. This is useful -when doing things like exploring the coverage of a fuzzer over its corpus. This can be done with -the `COVERAGE_TARGET` and `VALIDATE_COVERAGE` environment variables, e.g.: +It's also possible to specialize the coverage build to a specified test or test dir. This is useful +when doing things like exploring the coverage of a fuzzer over its corpus. This can be done by +passing coverage targets as the command-line arguments and using the `VALIDATE_COVERAGE` environment +variable, e.g.: ``` -COVERAGE_TARGET=//test/common/common:base64_fuzz_test VALIDATE_COVERAGE=false test/run_envoy_bazel_coverage.sh +VALIDATE_COVERAGE=false test/run_envoy_bazel_coverage.sh //test/common/common:base64_fuzz_test ``` # Cleaning the build and test artifacts diff --git a/ci/README.md b/ci/README.md index 1676ab83ace6..0316f30e776d 100644 --- a/ci/README.md +++ b/ci/README.md @@ -80,16 +80,25 @@ The `./ci/run_envoy_docker.sh './ci/do_ci.sh '` targets are: * `bazel.api` — build and run API tests under `-c fastbuild` with clang. * `bazel.asan` — build and run tests under `-c dbg --config=clang-asan` with clang. +* `bazel.asan ` — build and run a specified test or test dir under `-c dbg --config=clang-asan` with clang. * `bazel.debug` — build Envoy static binary and run tests under `-c dbg`. +* `bazel.debug ` — build Envoy static binary and run a specified test or test dir under `-c dbg`. * `bazel.debug.server_only` — build Envoy static binary under `-c dbg`. * `bazel.dev` — build Envoy static binary and run tests under `-c fastbuild` with clang. -* `bazel.release` — build Envoy static binary and run tests under `-c opt` with gcc. -* `bazel.release ` — build Envoy static binary and run a specified test or test dir under `-c opt` with gcc. -* `bazel.release.server_only` — build Envoy static binary under `-c opt` with gcc. +* `bazel.dev ` — build Envoy static binary and run a specified test or test dir under `-c fastbuild` with clang. +* `bazel.release` — build Envoy static binary and run tests under `-c opt` with clang. +* `bazel.release ` — build Envoy static binary and run a specified test or test dir under `-c opt` with clang. +* `bazel.release.server_only` — build Envoy static binary under `-c opt` with clang. +* `bazel.sizeopt` — build Envoy static binary and run tests under `-c opt --config=sizeopt` with clang. +* `bazel.sizeopt ` — build Envoy static binary and run a specified test or test dir under `-c opt --config=sizeopt` with clang. +* `bazel.sizeopt.server_only` — build Envoy static binary under `-c opt --config=sizeopt` with clang. * `bazel.coverage` — build and run tests under `-c dbg` with gcc, generating coverage information in `$ENVOY_DOCKER_BUILD_DIR/envoy/generated/coverage/coverage.html`. +* `bazel.coverage ` — build and run a specified test or test dir under `-c dbg` with gcc, generating coverage information in `$ENVOY_DOCKER_BUILD_DIR/envoy/generated/coverage/coverage.html`. * `bazel.coverity` — build Envoy static binary and run Coverity Scan static analysis. * `bazel.tsan` — build and run tests under `-c dbg --config=clang-tsan` with clang. -* `bazel.compile_time_options` — build Envoy and test with various compile-time options toggled to their non-default state, to ensure they still build. +* `bazel.tsan ` — build and run a specified test or test dir under `-c dbg --config=clang-tsan` with clang. +* `bazel.compile_time_options` — build Envoy and run tests with various compile-time options toggled to their non-default state, to ensure they still build. +* `bazel.compile_time_options ` — build Envoy and run a specified test or test dir with various compile-time options toggled to their non-default state, to ensure they still build. * `bazel.clang_tidy` — build and run clang-tidy over all source files. * `check_format`— run `clang-format` and `buildifier` on entire source tree. * `fix_format`— run and enforce `clang-format` and `buildifier` on entire source tree. diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 6aa1cd457138..d55381c86de0 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -81,7 +81,16 @@ function bazel_binary_build() { cp_binary_for_image_build "${BINARY_TYPE}" } -if [[ "$1" == "bazel.release" ]]; then +CI_TARGET=$1 + +if [[ $# -gt 1 ]]; then + shift + TEST_TARGETS=$* +else + TEST_TARGETS=//test/... +fi + +if [[ "$CI_TARGET" == "bazel.release" ]]; then # When testing memory consumption, we want to test against exact byte-counts # where possible. As these differ between platforms and compile options, we # define the 'release' builds as canonical and test them only in CI, so the @@ -94,56 +103,49 @@ if [[ "$1" == "bazel.release" ]]; then echo "bazel release build with tests..." bazel_binary_build release - if [[ $# -gt 1 ]]; then - shift - echo "Testing $* ..." - # Run only specified tests. Argument can be a single test - # (e.g. '//test/common/common:assert_test') or a test group (e.g. '//test/common/...') - bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c opt $* - else - echo "Testing..." + echo "Testing ${TEST_TARGETS}" + if [[ "$TEST_TARGETS" == "//test/..." ]]; then # We have various test binaries in the test directory such as tools, benchmarks, etc. We # run a build pass to make sure they compile. - bazel build ${BAZEL_BUILD_OPTIONS} -c opt //include/... //source/... //test/... - # Now run all of the tests which should already be compiled. - bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c opt //test/... fi + # Now run all of the tests which should already be compiled. + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c opt ${TEST_TARGETS} exit 0 -elif [[ "$1" == "bazel.release.server_only" ]]; then +elif [[ "$CI_TARGET" == "bazel.release.server_only" ]]; then setup_clang_toolchain echo "bazel release build..." bazel_binary_build release exit 0 -elif [[ "$1" == "bazel.sizeopt.server_only" ]]; then +elif [[ "$CI_TARGET" == "bazel.sizeopt.server_only" ]]; then setup_clang_toolchain echo "bazel size optimized build..." bazel_binary_build sizeopt exit 0 -elif [[ "$1" == "bazel.sizeopt" ]]; then +elif [[ "$CI_TARGET" == "bazel.sizeopt" ]]; then setup_clang_toolchain echo "bazel size optimized build with tests..." bazel_binary_build sizeopt - echo "Testing..." - bazel test ${BAZEL_BUILD_OPTIONS} //test/... --config=sizeopt + echo "Testing ${TEST_TARGETS}" + bazel test ${BAZEL_BUILD_OPTIONS} --config=sizeopt ${TEST_TARGETS} exit 0 -elif [[ "$1" == "bazel.debug" ]]; then +elif [[ "$CI_TARGET" == "bazel.debug" ]]; then setup_clang_toolchain echo "bazel debug build with tests..." bazel_binary_build debug - echo "Testing..." - bazel test ${BAZEL_BUILD_OPTIONS} -c dbg //test/... + echo "Testing ${TEST_TARGETS}" + bazel test ${BAZEL_BUILD_OPTIONS} -c dbg ${TEST_TARGETS} exit 0 -elif [[ "$1" == "bazel.debug.server_only" ]]; then +elif [[ "$CI_TARGET" == "bazel.debug.server_only" ]]; then setup_clang_toolchain echo "bazel debug build..." bazel_binary_build debug exit 0 -elif [[ "$1" == "bazel.asan" ]]; then +elif [[ "$CI_TARGET" == "bazel.asan" ]]; then setup_clang_toolchain echo "bazel ASAN/UBSAN debug build with tests" - echo "Building and testing envoy tests..." - bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-asan //test/... + echo "Building and testing envoy tests ${TEST_TARGETS}" + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-asan ${TEST_TARGETS} echo "Building and testing envoy-filter-example tests..." pushd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-asan \ @@ -164,27 +166,27 @@ elif [[ "$1" == "bazel.asan" ]]; then # for privileged Docker executors. ls -l "${TAP_TMP}"/tap_*.pb_text > /dev/null exit 0 -elif [[ "$1" == "bazel.tsan" ]]; then +elif [[ "$CI_TARGET" == "bazel.tsan" ]]; then setup_clang_toolchain echo "bazel TSAN debug build with tests" - echo "Building and testing envoy tests..." - bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-tsan //test/... + echo "Building and testing envoy tests ${TEST_TARGETS}" + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-tsan ${TEST_TARGETS} echo "Building and testing envoy-filter-example tests..." cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-tsan \ //:echo2_integration_test //:envoy_binary_test exit 0 -elif [[ "$1" == "bazel.dev" ]]; then +elif [[ "$CI_TARGET" == "bazel.dev" ]]; then setup_clang_toolchain # This doesn't go into CI but is available for developer convenience. echo "bazel fastbuild build with tests..." echo "Building..." bazel_binary_build fastbuild - echo "Building and testing..." - bazel test ${BAZEL_BUILD_OPTIONS} -c fastbuild //test/... + echo "Building and testing ${TEST_TARGETS}" + bazel test ${BAZEL_BUILD_OPTIONS} -c fastbuild ${TEST_TARGETS} exit 0 -elif [[ "$1" == "bazel.compile_time_options" ]]; then +elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then # Right now, none of the available compile-time options conflict with each other. If this # changes, this build type may need to be broken up. # TODO(mpwarres): remove quiche=enabled once QUICHE is built by default. @@ -204,14 +206,14 @@ elif [[ "$1" == "bazel.compile_time_options" ]]; then # Building all the dependencies from scratch to link them against libc++. echo "Building..." bazel build ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg //source/exe:envoy-static - echo "Building and testing..." - bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg //test/... + echo "Building and testing ${TEST_TARGETS}" + bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg ${TEST_TARGETS} # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only # these tests under "-c opt" to save time in CI. bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c opt //test/common/common:assert_test //test/server:server_test exit 0 -elif [[ "$1" == "bazel.ipv6_tests" ]]; then +elif [[ "$CI_TARGET" == "bazel.ipv6_tests" ]]; then # This is around until Circle supports IPv6. We try to run a limited set of IPv6 tests as fast # as possible for basic sanity testing. @@ -230,7 +232,7 @@ elif [[ "$1" == "bazel.ipv6_tests" ]]; then bazel_with_collection test ${BAZEL_BUILD_OPTIONS} --test_env=ENVOY_IP_TEST_VERSIONS=v6only -c fastbuild \ //test/integration/... //test/common/network/... exit 0 -elif [[ "$1" == "bazel.api" ]]; then +elif [[ "$CI_TARGET" == "bazel.api" ]]; then setup_clang_toolchain echo "Building API..." bazel build ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api//envoy/... @@ -238,9 +240,9 @@ elif [[ "$1" == "bazel.api" ]]; then bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c fastbuild @envoy_api//test/... @envoy_api//tools/... \ @envoy_api//tools:tap2pcap_test exit 0 -elif [[ "$1" == "bazel.coverage" ]]; then +elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then setup_gcc_toolchain - echo "bazel coverage build with tests..." + echo "bazel coverage build with tests ${TEST_TARGETS}" # gcovr is a pain to run with `bazel run`, so package it up into a # relocatable and hermetic-ish .par file. @@ -256,14 +258,14 @@ elif [[ "$1" == "bazel.coverage" ]]; then # after 0.21. [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_resources=12288,4,1" - test/run_envoy_bazel_coverage.sh + test/run_envoy_bazel_coverage.sh ${TEST_TARGETS} collect_build_profile coverage exit 0 -elif [[ "$1" == "bazel.clang_tidy" ]]; then +elif [[ "$CI_TARGET" == "bazel.clang_tidy" ]]; then setup_clang_toolchain ci/run_clang_tidy.sh exit 0 -elif [[ "$1" == "bazel.coverity" ]]; then +elif [[ "$CI_TARGET" == "bazel.coverity" ]]; then # Coverity Scan version 2017.07 fails to analyze the entirely of the Envoy # build when compiled with Clang 5. Revisit when Coverity Scan explicitly # supports Clang 5. Until this issue is resolved, run Coverity Scan with @@ -280,39 +282,39 @@ elif [[ "$1" == "bazel.coverity" ]]; then "${ENVOY_BUILD_DIR}"/envoy-coverity-output.tgz \ "${ENVOY_DELIVERY_DIR}"/envoy-coverity-output.tgz exit 0 -elif [[ "$1" == "fix_format" ]]; then +elif [[ "$CI_TARGET" == "fix_format" ]]; then echo "fix_format..." ./tools/check_format.py fix ./tools/format_python_tools.sh fix exit 0 -elif [[ "$1" == "check_format" ]]; then +elif [[ "$CI_TARGET" == "check_format" ]]; then echo "check_format_test..." ./tools/check_format_test_helper.py --log=WARN echo "check_format..." ./tools/check_format.py check ./tools/format_python_tools.sh check exit 0 -elif [[ "$1" == "check_repositories" ]]; then +elif [[ "$CI_TARGET" == "check_repositories" ]]; then echo "check_repositories..." ./tools/check_repositories.sh exit 0 -elif [[ "$1" == "check_spelling" ]]; then +elif [[ "$CI_TARGET" == "check_spelling" ]]; then echo "check_spelling..." ./tools/check_spelling.sh check exit 0 -elif [[ "$1" == "fix_spelling" ]];then +elif [[ "$CI_TARGET" == "fix_spelling" ]];then echo "fix_spell..." ./tools/check_spelling.sh fix exit 0 -elif [[ "$1" == "check_spelling_pedantic" ]]; then +elif [[ "$CI_TARGET" == "check_spelling_pedantic" ]]; then echo "check_spelling_pedantic..." ./tools/check_spelling_pedantic.py check exit 0 -elif [[ "$1" == "fix_spelling_pedantic" ]]; then +elif [[ "$CI_TARGET" == "fix_spelling_pedantic" ]]; then echo "fix_spelling_pedantic..." ./tools/check_spelling_pedantic.py fix exit 0 -elif [[ "$1" == "docs" ]]; then +elif [[ "$CI_TARGET" == "docs" ]]; then echo "generating docs..." docs/build.sh exit 0 diff --git a/ci/do_circle_ci.sh b/ci/do_circle_ci.sh index 5dcdca04231b..c51ee029c3ee 100755 --- a/ci/do_circle_ci.sh +++ b/ci/do_circle_ci.sh @@ -29,4 +29,4 @@ trap finish EXIT echo "disk space at beginning of build:" df -h -ci/do_ci.sh "$1" +ci/do_ci.sh $* diff --git a/ci/mac_ci_steps.sh b/ci/mac_ci_steps.sh index 79f47984f241..351282757be7 100755 --- a/ci/mac_ci_steps.sh +++ b/ci/mac_ci_steps.sh @@ -23,5 +23,11 @@ BAZEL_BUILD_OPTIONS="--curses=no --show_task_finish --verbose_failures \ # is somewhat more deterministic (rather than interleaving the build # and test steps). -bazel build ${BAZEL_BUILD_OPTIONS} //source/... //test/... -bazel test ${BAZEL_BUILD_OPTIONS} //test/... +if [[ $# -gt 0 ]]; then + TEST_TARGETS=$* +else + TEST_TARGETS=//test/... +fi + +bazel build ${BAZEL_BUILD_OPTIONS} //source/... ${TEST_TARGETS} +bazel test ${BAZEL_BUILD_OPTIONS} ${TEST_TARGETS} diff --git a/test/coverage/gen_build.sh b/test/coverage/gen_build.sh index 0dcca9cad10a..5532e78a5e95 100755 --- a/test/coverage/gen_build.sh +++ b/test/coverage/gen_build.sh @@ -25,10 +25,20 @@ set -e rm -f "${BUILD_PATH}" -TARGETS=$("${BAZEL_BIN}" query ${BAZEL_QUERY_OPTIONS} "attr('tags', 'coverage_test_lib', ${REPOSITORY}//test/...)" | grep "^//") +if [[ $# -gt 0 ]]; then + COVERAGE_TARGETS=$* +else + COVERAGE_TARGETS=//test/... +fi + +for target in ${COVERAGE_TARGETS}; do + TARGETS="$TARGETS $("${BAZEL_BIN}" query ${BAZEL_QUERY_OPTIONS} "attr('tags', 'coverage_test_lib', ${REPOSITORY}${target})" | grep "^//")" +done # Run the QUICHE platform api tests for coverage. -TARGETS="$TARGETS $("${BAZEL_BIN}" query ${BAZEL_QUERY_OPTIONS} "attr('tags', 'coverage_test_lib', '@com_googlesource_quiche//:all')" | grep "^@com_googlesource_quiche")" +if [[ "${COVERAGE_TARGETS}" == "//test/..." ]]; then + TARGETS="$TARGETS $("${BAZEL_BIN}" query ${BAZEL_QUERY_OPTIONS} "attr('tags', 'coverage_test_lib', '@com_googlesource_quiche//:all')" | grep "^@com_googlesource_quiche")" +fi if [ -n "${EXTRA_QUERY_PATHS}" ]; then TARGETS="$TARGETS $("${BAZEL_BIN}" query ${BAZEL_QUERY_OPTIONS} "attr('tags', 'coverage_test_lib', ${EXTRA_QUERY_PATHS})" | grep "^//")" diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 9306bb0dca33..57a7e2e28120 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -22,15 +22,23 @@ echo " VALIDATE_COVERAGE=${VALIDATE_COVERAGE}" # This is the target that will be run to generate coverage data. It can be overridden by consumer # projects that want to run coverage on a different/combined target. -[[ -z "${COVERAGE_TARGET}" ]] && COVERAGE_TARGET="//test/coverage:coverage_tests" +# Command-line arguments take precedence over ${COVERAGE_TARGET}. +if [[ $# -gt 0 ]]; then + COVERAGE_TARGETS=$* +elif [[ -n "${COVERAGE_TARGET}" ]]; then + COVERAGE_TARGETS=${COVERAGE_TARGET} +else + COVERAGE_TARGETS=//test/... +fi + # This is where we are going to copy the .gcno files into. -GCNO_ROOT=bazel-out/k8-dbg/bin/"${COVERAGE_TARGET/:/\/}".runfiles/"${WORKSPACE}" +GCNO_ROOT=bazel-out/k8-dbg/bin/test/coverage/coverage_tests.runfiles/"${WORKSPACE}" echo " GCNO_ROOT=${GCNO_ROOT}" rm -rf ${GCNO_ROOT} -# Make sure ${COVERAGE_TARGET} is up-to-date. +# Make sure //test/coverage:coverage_tests is up-to-date. SCRIPT_DIR="$(realpath "$(dirname "$0")")" -(BAZEL_BIN="${BAZEL_COVERAGE}" "${SCRIPT_DIR}"/coverage/gen_build.sh) +(BAZEL_BIN="${BAZEL_COVERAGE}" "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS}) echo "Cleaning .gcda/.gcov from previous coverage runs..." NUM_PREVIOUS_GCOV_FILES=0 @@ -50,7 +58,7 @@ BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} -c dbg --copt=-DNDEBUG" # https://github.com/bazelbuild/bazel/issues/1118). This works today as we have # a single coverage test binary and do not require the "bazel coverage" support # for collecting multiple traces and glueing them together. -"${BAZEL_COVERAGE}" test "${COVERAGE_TARGET}" ${BAZEL_BUILD_OPTIONS} \ +"${BAZEL_COVERAGE}" test //test/coverage:coverage_tests ${BAZEL_BUILD_OPTIONS} \ --cache_test_results=no --cxxopt="--coverage" --cxxopt="-DENVOY_CONFIG_COVERAGE=1" \ --linkopt="--coverage" --define ENVOY_CONFIG_COVERAGE=1 --test_output=streamed \ --strategy=Genrule=standalone --spawn_strategy=standalone --test_timeout=4000 \ From cb7fca2685ea533e5b7ec4489933d1642d1e4bdc Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 20 Jun 2019 12:31:23 -0700 Subject: [PATCH 040/542] bazel: Force python 2 for host tools (#7337) https://github.com/bazelbuild/bazel/issues/8626 Signed-off-by: Keith Smiley --- .bazelrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.bazelrc b/.bazelrc index d0cd3e3f007a..786af870ed82 100644 --- a/.bazelrc +++ b/.bazelrc @@ -7,6 +7,7 @@ startup --host_jvm_args=-Xmx512m build --workspace_status_command=bazel/get_workspace_status build --experimental_remap_main_repo +build --host_force_python=PY2 # Basic ASAN/UBSAN that works for gcc build:asan --define ENVOY_CONFIG_ASAN=1 From 2e03ef24060d479fb825d2d0257a1961f502f08d Mon Sep 17 00:00:00 2001 From: Sriduth Jayhari Date: Fri, 21 Jun 2019 03:28:59 +0530 Subject: [PATCH 041/542] extenstions: retry predicate - omit_canary_hosts (#7230) In case a user wants to deploy a new change, this can help in cases where any failure in the canary downstream is retried in a stable downstream. Signed-off-by: Sriduth Jayhari --- CODEOWNERS | 2 + .../http/http_connection_management.rst | 4 ++ docs/root/intro/version_history.rst | 1 + source/extensions/extensions_build_config.bzl | 3 +- .../retry/host/omit_canary_hosts/BUILD | 29 +++++++++++++ .../retry/host/omit_canary_hosts/config.cc | 16 +++++++ .../retry/host/omit_canary_hosts/config.h | 29 +++++++++++++ .../omit_canary_hosts/omit_canary_hosts.h | 15 +++++++ .../retry/host/previous_hosts/config.h | 2 +- .../extensions/retry/host/well_known_names.h | 1 + .../retry/host/omit_canary_hosts/BUILD | 22 ++++++++++ .../host/omit_canary_hosts/config_test.cc | 43 +++++++++++++++++++ 12 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 source/extensions/retry/host/omit_canary_hosts/BUILD create mode 100644 source/extensions/retry/host/omit_canary_hosts/config.cc create mode 100644 source/extensions/retry/host/omit_canary_hosts/config.h create mode 100644 source/extensions/retry/host/omit_canary_hosts/omit_canary_hosts.h create mode 100644 test/extensions/retry/host/omit_canary_hosts/BUILD create mode 100644 test/extensions/retry/host/omit_canary_hosts/config_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index 767bd14d6bb0..e3f69eac6bb4 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -36,3 +36,5 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/network/zookeeper_proxy @rgs1 @snowp # redis cluster extension /*/extensions/clusters/redis @msukalski @henryyyang @mattklein123 +# omit_canary_hosts retry predicate +/*/extensions/retry/host/omit_canary_hosts @sriduth @snowp diff --git a/docs/root/intro/arch_overview/http/http_connection_management.rst b/docs/root/intro/arch_overview/http/http_connection_management.rst index 68bdeacb33e2..6a5df8b7c066 100644 --- a/docs/root/intro/arch_overview/http/http_connection_management.rst +++ b/docs/root/intro/arch_overview/http/http_connection_management.rst @@ -60,6 +60,10 @@ can be used to modify this behavior, and they fall into two categories: * *envoy.retry_host_predicates.previous_hosts*: This will keep track of previously attempted hosts, and rejects hosts that have already been attempted. + * *envoy.retry_host_predicates.omit_canary_hosts*: This will reject any host that is a marked as canary host. + Hosts are marked by setting ``canary: true`` for the ```envoy.lb`` filter in the endpoint's filter metadata. + See :ref:`LbEndpoint ` for more details. + * :ref:`Priority Predicates`: These predicates can be used to adjust the priority load used when selecting a priority for a retry attempt. Only one such predicate may be specified. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ed32dd6ca384..e5848c7e4834 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -143,6 +143,7 @@ Version history * redis: added :ref:`success and error stats ` for commands. * redis: migrate hash function for host selection to `MurmurHash2 `_ from std::hash. MurmurHash2 is compatible with std::hash in GNU libstdc++ 3.4.20 or above. This is typically the case when compiled on Linux and not macOS. * redis: added :ref:`latency_in_micros ` to specify the redis commands stats time unit in microseconds. +* retry: added a retry predicate that :ref:`rejects canary hosts. ` * router: added ability to configure a :ref:`retry policy ` at the virtual host level. * router: added reset reason to response body when upstream reset happens. After this change, the response body will be of the form `upstream connect error or disconnect/reset before headers. reset reason:` diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 93fd4108aa8b..a132b9aa59ce 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -131,7 +131,8 @@ EXTENSIONS = { # Retry host predicates "envoy.retry_host_predicates.previous_hosts": "//source/extensions/retry/host/previous_hosts:config", - + "envoy.retry_host_predicates.omit_canary_hosts": "//source/extensions/retry/host/omit_canary_hosts:config", + # Retry priorities "envoy.retry_priorities.previous_priorities": "//source/extensions/retry/priority/previous_priorities:config", } diff --git a/source/extensions/retry/host/omit_canary_hosts/BUILD b/source/extensions/retry/host/omit_canary_hosts/BUILD new file mode 100644 index 000000000000..5ee8c65978a0 --- /dev/null +++ b/source/extensions/retry/host/omit_canary_hosts/BUILD @@ -0,0 +1,29 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "omit_canary_hosts_predicate_lib", + hdrs = ["omit_canary_hosts.h"], + deps = [ + "//include/envoy/upstream:retry_interface", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":omit_canary_hosts_predicate_lib", + "//include/envoy/registry", + "//include/envoy/upstream:retry_interface", + "//source/extensions/retry/host:well_known_names", + ], +) diff --git a/source/extensions/retry/host/omit_canary_hosts/config.cc b/source/extensions/retry/host/omit_canary_hosts/config.cc new file mode 100644 index 000000000000..495790ecc731 --- /dev/null +++ b/source/extensions/retry/host/omit_canary_hosts/config.cc @@ -0,0 +1,16 @@ +#include "extensions/retry/host/omit_canary_hosts/config.h" + +#include "envoy/registry/registry.h" +#include "envoy/upstream/retry.h" + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Host { + +REGISTER_FACTORY(OmitCanaryHostsRetryPredicateFactory, Upstream::RetryHostPredicateFactory); + +} +} // namespace Retry +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/retry/host/omit_canary_hosts/config.h b/source/extensions/retry/host/omit_canary_hosts/config.h new file mode 100644 index 000000000000..53e3476499d8 --- /dev/null +++ b/source/extensions/retry/host/omit_canary_hosts/config.h @@ -0,0 +1,29 @@ +#include "envoy/upstream/retry.h" + +#include "extensions/retry/host/omit_canary_hosts/omit_canary_hosts.h" +#include "extensions/retry/host/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Host { + +class OmitCanaryHostsRetryPredicateFactory : public Upstream::RetryHostPredicateFactory { + +public: + Upstream::RetryHostPredicateSharedPtr createHostPredicate(const Protobuf::Message&, + uint32_t) override { + return std::make_shared(); + } + + std::string name() override { return RetryHostPredicateValues::get().OmitCanaryHostsPredicate; } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } +}; + +} // namespace Host +} // namespace Retry +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/retry/host/omit_canary_hosts/omit_canary_hosts.h b/source/extensions/retry/host/omit_canary_hosts/omit_canary_hosts.h new file mode 100644 index 000000000000..e174100d8bbd --- /dev/null +++ b/source/extensions/retry/host/omit_canary_hosts/omit_canary_hosts.h @@ -0,0 +1,15 @@ +#pragma once + +#include "envoy/upstream/retry.h" +#include "envoy/upstream/upstream.h" + +namespace Envoy { +class OmitCanaryHostsRetryPredicate : public Upstream::RetryHostPredicate { +public: + bool shouldSelectAnotherHost(const Upstream::Host& candidate_host) override { + return candidate_host.canary(); + } + + void onHostAttempted(Upstream::HostDescriptionConstSharedPtr) override {} +}; +} // namespace Envoy diff --git a/source/extensions/retry/host/previous_hosts/config.h b/source/extensions/retry/host/previous_hosts/config.h index 5cb50a2983c6..ddc762f661eb 100644 --- a/source/extensions/retry/host/previous_hosts/config.h +++ b/source/extensions/retry/host/previous_hosts/config.h @@ -20,7 +20,7 @@ class PreviousHostsRetryPredicateFactory : public Upstream::RetryHostPredicateFa std::string name() override { return RetryHostPredicateValues::get().PreviousHostsPredicate; } ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Empty()}; + return std::make_unique(); } }; diff --git a/source/extensions/retry/host/well_known_names.h b/source/extensions/retry/host/well_known_names.h index a79002befab3..a2e50df9e185 100644 --- a/source/extensions/retry/host/well_known_names.h +++ b/source/extensions/retry/host/well_known_names.h @@ -16,6 +16,7 @@ class RetryHostPredicatesNameValues { public: // Previous host predicate. Rejects hosts that have already been tried. const std::string PreviousHostsPredicate = "envoy.retry_host_predicates.previous_hosts"; + const std::string OmitCanaryHostsPredicate = "envoy.retry_host_predicates.omit_canary_hosts"; }; using RetryHostPredicateValues = ConstSingleton; diff --git a/test/extensions/retry/host/omit_canary_hosts/BUILD b/test/extensions/retry/host/omit_canary_hosts/BUILD new file mode 100644 index 000000000000..605fc111a64c --- /dev/null +++ b/test/extensions/retry/host/omit_canary_hosts/BUILD @@ -0,0 +1,22 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_name = "envoy.retry_host_predicates.omit_canary_hosts", + deps = [ + "//source/extensions/retry/host/omit_canary_hosts:config", + "//test/mocks/upstream:upstream_mocks", + ], +) diff --git a/test/extensions/retry/host/omit_canary_hosts/config_test.cc b/test/extensions/retry/host/omit_canary_hosts/config_test.cc new file mode 100644 index 000000000000..4794aefa32a1 --- /dev/null +++ b/test/extensions/retry/host/omit_canary_hosts/config_test.cc @@ -0,0 +1,43 @@ +#include "envoy/registry/registry.h" +#include "envoy/upstream/retry.h" + +#include "extensions/retry/host/omit_canary_hosts/config.h" +#include "extensions/retry/host/well_known_names.h" + +#include "test/mocks/upstream/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace testing; + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Host { +namespace { + +TEST(OmitCanaryHostsRetryPredicateTest, PredicateTest) { + auto factory = Registry::FactoryRegistry::getFactory( + RetryHostPredicateValues::get().OmitCanaryHostsPredicate); + + ASSERT_NE(nullptr, factory); + + ProtobufWkt::Struct config; + auto predicate = factory->createHostPredicate(config, 3); + + auto host1 = std::make_shared>(); + auto host2 = std::make_shared>(); + + ON_CALL(*host1, canary()).WillByDefault(Return(false)); + ON_CALL(*host2, canary()).WillByDefault(Return(true)); + + ASSERT_FALSE(predicate->shouldSelectAnotherHost(*host1)); + ASSERT_TRUE(predicate->shouldSelectAnotherHost(*host2)); +} + +} // namespace +} // namespace Host +} // namespace Retry +} // namespace Extensions +} // namespace Envoy From 79e53f21c680b4a02695b1761e7d91868c3481d4 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Thu, 20 Jun 2019 16:47:40 -0700 Subject: [PATCH 042/542] http: add dynamic forward proxy filter and cluster (#7307) This allows using Envoy as a generic HTTP proxy without any prior configuration of DNS targets. See the included documentation for more information. Part of https://github.com/envoyproxy/envoy/issues/1606 Signed-off-by: Matt Klein --- CODEOWNERS | 4 + api/docs/BUILD | 3 + .../dynamic_forward_proxy/v2alpha/BUILD | 11 + .../v2alpha/cluster.proto | 24 ++ .../dynamic_forward_proxy/v2alpha/BUILD | 12 + .../v2alpha/dns_cache.proto | 58 +++ .../http/dynamic_forward_proxy/v2alpha/BUILD | 11 + .../v2alpha/dynamic_forward_proxy.proto | 24 ++ docs/build.sh | 3 + docs/root/api-v2/config/cluster/cluster.rst | 1 + docs/root/api-v2/config/common/common.rst | 1 + .../dynamic_forward_proxy_filter.rst | 94 +++++ .../http_filters/http_filters.rst | 1 + docs/root/intro/arch_overview/http/http.rst | 1 + .../intro/arch_overview/http/http_proxy.rst | 54 +++ docs/root/intro/version_history.rst | 1 + source/common/common/logger.h | 1 + source/common/upstream/upstream_impl.cc | 7 +- source/common/upstream/upstream_impl.h | 4 +- .../clusters/dynamic_forward_proxy/BUILD | 23 ++ .../clusters/dynamic_forward_proxy/cluster.cc | 163 ++++++++ .../clusters/dynamic_forward_proxy/cluster.h | 134 +++++++ source/extensions/clusters/well_known_names.h | 4 + .../common/dynamic_forward_proxy/BUILD | 44 +++ .../common/dynamic_forward_proxy/dns_cache.h | 165 ++++++++ .../dynamic_forward_proxy/dns_cache_impl.cc | 237 +++++++++++ .../dynamic_forward_proxy/dns_cache_impl.h | 115 ++++++ .../dns_cache_manager_impl.cc | 46 +++ .../dns_cache_manager_impl.h | 55 +++ source/extensions/extensions_build_config.bzl | 2 + .../filters/http/dynamic_forward_proxy/BUILD | 35 ++ .../http/dynamic_forward_proxy/config.cc | 32 ++ .../http/dynamic_forward_proxy/config.h | 32 ++ .../dynamic_forward_proxy/proxy_filter.cc | 65 +++ .../http/dynamic_forward_proxy/proxy_filter.h | 53 +++ .../filters/http/well_known_names.h | 2 + test/common/upstream/BUILD | 1 + .../clusters/dynamic_forward_proxy/BUILD | 32 ++ .../dynamic_forward_proxy/cluster_test.cc | 224 +++++++++++ .../common/dynamic_forward_proxy/BUILD | 32 ++ .../dns_cache_impl_test.cc | 369 ++++++++++++++++++ .../common/dynamic_forward_proxy/mocks.cc | 37 ++ .../common/dynamic_forward_proxy/mocks.h | 84 ++++ .../filters/http/dynamic_forward_proxy/BUILD | 35 ++ .../proxy_filter_integration_test.cc | 87 +++++ .../proxy_filter_test.cc | 108 +++++ test/mocks/upstream/cluster_info.cc | 6 +- 47 files changed, 2534 insertions(+), 3 deletions(-) create mode 100644 api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD create mode 100644 api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto create mode 100644 api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD create mode 100644 api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto create mode 100644 api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD create mode 100644 api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto create mode 100644 docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst create mode 100644 docs/root/intro/arch_overview/http/http_proxy.rst create mode 100644 source/extensions/clusters/dynamic_forward_proxy/BUILD create mode 100644 source/extensions/clusters/dynamic_forward_proxy/cluster.cc create mode 100644 source/extensions/clusters/dynamic_forward_proxy/cluster.h create mode 100644 source/extensions/common/dynamic_forward_proxy/BUILD create mode 100644 source/extensions/common/dynamic_forward_proxy/dns_cache.h create mode 100644 source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc create mode 100644 source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h create mode 100644 source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc create mode 100644 source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h create mode 100644 source/extensions/filters/http/dynamic_forward_proxy/BUILD create mode 100644 source/extensions/filters/http/dynamic_forward_proxy/config.cc create mode 100644 source/extensions/filters/http/dynamic_forward_proxy/config.h create mode 100644 source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc create mode 100644 source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h create mode 100644 test/extensions/clusters/dynamic_forward_proxy/BUILD create mode 100644 test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc create mode 100644 test/extensions/common/dynamic_forward_proxy/BUILD create mode 100644 test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc create mode 100644 test/extensions/common/dynamic_forward_proxy/mocks.cc create mode 100644 test/extensions/common/dynamic_forward_proxy/mocks.h create mode 100644 test/extensions/filters/http/dynamic_forward_proxy/BUILD create mode 100644 test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc create mode 100644 test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index e3f69eac6bb4..18869803bd5e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -36,5 +36,9 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/network/zookeeper_proxy @rgs1 @snowp # redis cluster extension /*/extensions/clusters/redis @msukalski @henryyyang @mattklein123 +# dynamic forward proxy +/*/extensions/clusters/dynamic_forward_proxy @mattklein123 @alyssawilk +/*/extensions/common/dynamic_forward_proxy @mattklein123 @alyssawilk +/*/extensions/filters/http/dynamic_forward_proxy @mattklein123 @alyssawilk # omit_canary_hosts retry predicate /*/extensions/retry/host/omit_canary_hosts @sriduth @snowp diff --git a/api/docs/BUILD b/api/docs/BUILD index 1cd82c78fc05..b1ec7f0a3388 100644 --- a/api/docs/BUILD +++ b/api/docs/BUILD @@ -34,12 +34,15 @@ proto_library( "//envoy/config/accesslog/v2:als", "//envoy/config/accesslog/v2:file", "//envoy/config/bootstrap/v2:bootstrap", + "//envoy/config/cluster/dynamic_forward_proxy/v2alpha:cluster", "//envoy/config/cluster/redis:redis_cluster", + "//envoy/config/common/dynamic_forward_proxy/v2alpha:dns_cache", "//envoy/config/common/tap/v2alpha:common", "//envoy/config/filter/accesslog/v2:accesslog", "//envoy/config/filter/dubbo/router/v2alpha1:router", "//envoy/config/filter/http/buffer/v2:buffer", "//envoy/config/filter/http/csrf/v2:csrf", + "//envoy/config/filter/http/dynamic_forward_proxy/v2alpha:dynamic_forward_proxy", "//envoy/config/filter/http/ext_authz/v2:ext_authz", "//envoy/config/filter/http/fault/v2:fault", "//envoy/config/filter/http/gzip/v2:gzip", diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD new file mode 100644 index 000000000000..b09f5c858ba9 --- /dev/null +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "cluster", + srcs = ["cluster.proto"], + deps = [ + "//envoy/config/common/dynamic_forward_proxy/v2alpha:dns_cache", + ], +) diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto new file mode 100644 index 000000000000..d9ae85903264 --- /dev/null +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.config.cluster.dynamic_forward_proxy.v2alpha; + +option java_outer_classname = "DynamicForwardProxyClusterProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.cluster.dynamic_forward_proxy.v2alpha"; +option go_package = "v2alpha"; + +import "envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Dynamic forward proxy cluster configuration] + +// Configuration for the dynamic forward proxy cluster. See the :ref:`architecture overview +// ` for more information. +message ClusterConfig { + // The DNS cache configuration that the cluster will attach to. Note this configuration must + // match that of associated :ref:`dynamic forward proxy HTTP filter configuration + // `. + common.dynamic_forward_proxy.v2alpha.DnsCacheConfig dns_cache_config = 1 + [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD new file mode 100644 index 000000000000..53095826454d --- /dev/null +++ b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD @@ -0,0 +1,12 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "dns_cache", + srcs = ["dns_cache.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v2:cds", + ], +) diff --git a/api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto new file mode 100644 index 000000000000..33803393fcea --- /dev/null +++ b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto @@ -0,0 +1,58 @@ +syntax = "proto3"; + +package envoy.config.common.dynamic_forward_proxy.v2alpha; + +option java_outer_classname = "DnsCacheProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.common.dynamic_forward_proxy.v2alpha"; + +import "envoy/api/v2/cds.proto"; + +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Dynamic forward proxy common configuration] + +// Configuration for the dynamic forward proxy DNS cache. See the :ref:`architecture overview +// ` for more information. +message DnsCacheConfig { + // The name of the cache. Multiple named caches allow independent dynamic forward proxy + // configurations to operate within a single Envoy process using different configurations. All + // configurations with the same name *must* otherwise have the same settings when referenced + // from different configuration components. Configuration will fail to load if this is not + // the case. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // The DNS lookup family to use during resolution. + // + // [#comment:TODO(mattklein123): Figure out how to support IPv4/IPv6 "happy eyeballs" mode. The + // way this might work is a new lookup family which returns both IPv4 and IPv6 addresses, and + // then configures a host to have a primary and fall back address. With this, we could very + // likely build a "happy eyeballs" connection pool which would race the primary / fall back + // address and return the one that wins. This same method could potentially also be used for + // QUIC to TCP fall back.] + api.v2.Cluster.DnsLookupFamily dns_lookup_family = 2 [(validate.rules).enum.defined_only = true]; + + // The DNS refresh rate for currently cached DNS hosts. If not specified defaults to 60s. + // + // .. note: + // + // The returned DNS TTL is not currently used to alter the refresh rate. This feature will be + // added in a future change. + google.protobuf.Duration dns_refresh_rate = 3 [(validate.rules).duration.gt = {}]; + + // The TTL for hosts that are unused. Hosts that have not been used in the configured time + // interval will be purged. If not specified defaults to 5m. + // + // .. note: + // + // The TTL is only checked at the time of DNS refresh, as specified by *dns_refresh_rate*. This + // means that if the configured TTL is shorter than the refresh rate the host may not be removed + // immediately. + // + // .. note: + // + // The TTL has no relation to DNS TTL and is only used to control Envoy's resource usage. + google.protobuf.Duration host_ttl = 4 [(validate.rules).duration.gt = {}]; +} diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD new file mode 100644 index 000000000000..4fd1d84399fa --- /dev/null +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "dynamic_forward_proxy", + srcs = ["dynamic_forward_proxy.proto"], + deps = [ + "//envoy/config/common/dynamic_forward_proxy/v2alpha:dns_cache", + ], +) diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto new file mode 100644 index 000000000000..631363a6d95e --- /dev/null +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.config.filter.http.dynamic_forward_proxy.v2alpha; + +option java_outer_classname = "DynamicForwardProxyProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.dynamic_forward_proxy.v2alpha"; +option go_package = "v2alpha"; + +import "envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Dynamic forward proxy] + +// Configuration for the dynamic forward proxy HTTP filter. See the :ref:`architecture overview +// ` for more information. +message FilterConfig { + // The DNS cache configuration that the filter will attach to. Note this configuration must + // match that of associated :ref:`dynamic forward proxy cluster configuration + // `. + common.dynamic_forward_proxy.v2alpha.DnsCacheConfig dns_cache_config = 1 + [(validate.rules).message.required = true]; +} diff --git a/docs/build.sh b/docs/build.sh index 566fd061e4d6..8a4c7850908f 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -85,7 +85,9 @@ PROTO_RST=" /envoy/config/accesslog/v2/als/envoy/config/accesslog/v2/als.proto.rst /envoy/config/accesslog/v2/file/envoy/config/accesslog/v2/file.proto.rst /envoy/config/bootstrap/v2/bootstrap/envoy/config/bootstrap/v2/bootstrap.proto.rst + /envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto.rst /envoy/config/cluster/redis/redis_cluster/envoy/config/cluster/redis/redis_cluster.proto.rst + /envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto.rst /envoy/config/common/tap/v2alpha/common/envoy/config/common/tap/v2alpha/common.proto.rst /envoy/config/ratelimit/v2/rls/envoy/config/ratelimit/v2/rls.proto.rst /envoy/config/metrics/v2/metrics_service/envoy/config/metrics/v2/metrics_service.proto.rst @@ -95,6 +97,7 @@ PROTO_RST=" /envoy/config/filter/fault/v2/fault/envoy/config/filter/fault/v2/fault.proto.rst /envoy/config/filter/http/buffer/v2/buffer/envoy/config/filter/http/buffer/v2/buffer.proto.rst /envoy/config/filter/http/csrf/v2/csrf/envoy/config/filter/http/csrf/v2/csrf.proto.rst + /envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto.rst /envoy/config/filter/http/ext_authz/v2/ext_authz/envoy/config/filter/http/ext_authz/v2/ext_authz.proto.rst /envoy/config/filter/http/fault/v2/fault/envoy/config/filter/http/fault/v2/fault.proto.rst /envoy/config/filter/http/gzip/v2/gzip/envoy/config/filter/http/gzip/v2/gzip.proto.rst diff --git a/docs/root/api-v2/config/cluster/cluster.rst b/docs/root/api-v2/config/cluster/cluster.rst index 7bb5343e81dd..0e8dabab3a8a 100644 --- a/docs/root/api-v2/config/cluster/cluster.rst +++ b/docs/root/api-v2/config/cluster/cluster.rst @@ -5,4 +5,5 @@ Cluster :glob: :maxdepth: 1 + dynamic_forward_proxy/v2alpha/* redis/* diff --git a/docs/root/api-v2/config/common/common.rst b/docs/root/api-v2/config/common/common.rst index d3c5a2848b56..c8a0a212cc5b 100644 --- a/docs/root/api-v2/config/common/common.rst +++ b/docs/root/api-v2/config/common/common.rst @@ -5,4 +5,5 @@ Common :glob: :maxdepth: 2 + dynamic_forward_proxy/v2alpha/* tap/v2alpha/* diff --git a/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst b/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst new file mode 100644 index 000000000000..407dafb5dc26 --- /dev/null +++ b/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst @@ -0,0 +1,94 @@ +.. _config_http_filters_dynamic_forward_proxy: + +Dynamic forward proxy +===================== + +.. attention:: + + HTTP dynamic forward proxy support should be considered alpha and not production ready. Stats + as well as circuit breakers are missing and will be added soon. + +* HTTP dynamic forward proxy :ref:`architecture overview ` +* :ref:`v2 API reference ` +* This filter should be configured with the name *envoy.filters.http.dynamic_forward_proxy*. + +The following is a complete configuration that configures both the +:ref:`dynamic forward proxy HTTP filter +` +as well as the :ref:`dynamic forward proxy cluster +`. Both filter and cluster +must be configured together and point to the same DNS cache parameters for Envoy to operate as an +HTTP dynamic forward proxy. + +.. note:: + + The HTTP connection manager :ref:`allow_absolute_url + ` parameter has been set to true + to allow Envoy to proxy absolute HTTP URLs. + +.. attention:: + + While configuring a :ref:`tls_context ` on the cluster with + *trusted_ca* certificates instructs Envoy to use TLS when connecting to upstream hosts and verify + the certificate chain, currently it is not possible to configure per-host TLS configuration + parameters including SNI, subject alt name verification, etc. This will be added in a future + change. **This means that the following configuration will not fully validate TLS certificates**. + Use with care until full support for per-host validation is implemented. + +.. code-block:: yaml + + admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: + protocol: TCP + address: 127.0.0.1 + port_value: 9901 + static_resources: + listeners: + - name: listener_0 + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 10000 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + stat_prefix: ingress_http + http_protocol_options: + allow_absolute_url: true + route_config: + name: local_route + virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: dynamic_forward_proxy_cluster + http_filters: + - name: envoy.filters.http.dynamic_forward_proxy + config: + dns_cache_config: + name: dynamic_forward_proxy_cache_config + dns_lookup_family: V4_ONLY + - name: envoy.router + clusters: + - name: dynamic_forward_proxy_cluster + connect_timeout: 1s + lb_policy: CLUSTER_PROVIDED + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.config.cluster.dynamic_forward_proxy.v2alpha.ClusterConfig + dns_cache_config: + name: dynamic_forward_proxy_cache_config + dns_lookup_family: V4_ONLY + tls_context: + common_tls_context: + validation_context: + trusted_ca: {filename: /etc/ssl/certs/ca-certificates.crt} diff --git a/docs/root/configuration/http_filters/http_filters.rst b/docs/root/configuration/http_filters/http_filters.rst index 5fcbedd731ed..1ec3af560711 100644 --- a/docs/root/configuration/http_filters/http_filters.rst +++ b/docs/root/configuration/http_filters/http_filters.rst @@ -9,6 +9,7 @@ HTTP filters buffer_filter cors_filter csrf_filter + dynamic_forward_proxy_filter dynamodb_filter ext_authz_filter fault_filter diff --git a/docs/root/intro/arch_overview/http/http.rst b/docs/root/intro/arch_overview/http/http.rst index d32e20e4aa07..f5729560e0f6 100644 --- a/docs/root/intro/arch_overview/http/http.rst +++ b/docs/root/intro/arch_overview/http/http.rst @@ -8,3 +8,4 @@ HTTP http_filters http_routing websocket + http_proxy diff --git a/docs/root/intro/arch_overview/http/http_proxy.rst b/docs/root/intro/arch_overview/http/http_proxy.rst new file mode 100644 index 000000000000..f094e0c3fa01 --- /dev/null +++ b/docs/root/intro/arch_overview/http/http_proxy.rst @@ -0,0 +1,54 @@ +.. _arch_overview_http_dynamic_forward_proxy: + +HTTP dynamic forward proxy +========================== + +.. attention:: + + HTTP dynamic forward proxy support should be considered alpha and not production ready. Stats + as well as circuit breakers are missing and will be added soon. + +Through the combination of both an :ref:`HTTP filter ` and +:ref:`custom cluster `, +Envoy supports HTTP dynamic forward proxy. This means that Envoy can perform the role of an HTTP +proxy without prior knowledge of all configured DNS addresses, while still retaining the vast +majority of Envoy's benefits including asynchronous DNS resolution. The implementation works as +follows: + +* The dynamic forward proxy HTTP filter is used to pause requests if the target DNS host is not + already in cache. +* Envoy will begin asynchronously resolving the DNS address, unblocking any requests waiting on + the response when the resolution completes. +* Any future requests will not be blocked as the DNS address is already in cache. The resolution + process works similarly to the :ref:`logical DNS + ` service discovery type with a single target + address being remembered at any given time. +* All known hosts are stored in the dynamic forward proxy cluster such that they can be displayed + in :ref:`admin output `. +* A special load balancer will select the right host to use based on the HTTP host/authority header + during forwarding. +* Hosts that have not been used for a period of time are subject to a TTL that will purge them. + +The above implementation details mean that at steady state Envoy can forward a large volume of +HTTP proxy traffic while all DNS resolution happens asynchronously in the background. Additionally, +all other Envoy filters and extensions can be used in conjunction with dynamic forward proxy support +including authentication, RBAC, rate limiting, etc. + +For further configuration information see the :ref:`HTTP filter configuration documentation +`. + +Memory usage details +-------------------- + +.. attention:: + + HTTP dynamic forward proxy support currently can use unbounded memory and is not ready for + production use. Circuit breakers and other limits will be added soon. + +Memory usage detail's for Envoy's dynamic forward proxy support are as follows: + +* Each resolved host/port pair uses a fixed amount of memory global to the server and shared + amongst all workers. +* Address changes are performed inline using read/write locks and require no host reallocations. +* Hosts removed via TTL are purged once all active connections stop referring to them and all used + memory is regained. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index e5848c7e4834..b666d151d989 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -31,6 +31,7 @@ Version history * http: mitigated a race condition with the :ref:`delayed_close_timeout` where it could trigger while actively flushing a pending write buffer for a downstream connection. * http: added support for :ref:`preserve_external_request_id` that represents whether the x-request-id should not be reset on edge entry inside mesh * http: changed `sendLocalReply` to send percent-encoded `GrpcMessage`. +* http: added :ref:`dynamic forward proxy ` support. * jwt_authn: make filter's parsing of JWT more flexible, allowing syntax like ``jwt=eyJhbGciOiJS...ZFnFIw,extra=7,realm=123`` * listener: added :ref:`source IP ` and :ref:`source port ` filter diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 00d083bfaacc..b01f210fa5bb 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -32,6 +32,7 @@ namespace Logger { FUNCTION(dubbo) \ FUNCTION(file) \ FUNCTION(filter) \ + FUNCTION(forward_proxy) \ FUNCTION(grpc) \ FUNCTION(hc) \ FUNCTION(health_checker) \ diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 138e180055b1..a94ccd5d2094 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1265,7 +1265,12 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const HostVector& new_hosts, } Network::DnsLookupFamily getDnsLookupFamilyFromCluster(const envoy::api::v2::Cluster& cluster) { - switch (cluster.dns_lookup_family()) { + return getDnsLookupFamilyFromEnum(cluster.dns_lookup_family()); +} + +Network::DnsLookupFamily +getDnsLookupFamilyFromEnum(envoy::api::v2::Cluster::DnsLookupFamily family) { + switch (family) { case envoy::api::v2::Cluster::V6_ONLY: return Network::DnsLookupFamily::V6Only; case envoy::api::v2::Cluster::V4_ONLY: diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index fe87258f5910..5d07e9a1673f 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -820,9 +820,11 @@ class BaseDynamicClusterImpl : public ClusterImplBase { }; /** - * Utility function to get Dns from cluster. + * Utility function to get Dns from cluster/enum. */ Network::DnsLookupFamily getDnsLookupFamilyFromCluster(const envoy::api::v2::Cluster& cluster); +Network::DnsLookupFamily +getDnsLookupFamilyFromEnum(envoy::api::v2::Cluster::DnsLookupFamily family); /** * Utility function to report upstream cx destroy metrics diff --git a/source/extensions/clusters/dynamic_forward_proxy/BUILD b/source/extensions/clusters/dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..861a2da0a10b --- /dev/null +++ b/source/extensions/clusters/dynamic_forward_proxy/BUILD @@ -0,0 +1,23 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "cluster", + srcs = ["cluster.cc"], + hdrs = ["cluster.h"], + deps = [ + "//source/common/upstream:cluster_factory_lib", + "//source/common/upstream:logical_host_lib", + "//source/extensions/clusters:well_known_names", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_interface", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_manager_impl", + "@envoy_api//envoy/config/cluster/dynamic_forward_proxy/v2alpha:cluster_cc", + ], +) diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc new file mode 100644 index 000000000000..f56e529bf14a --- /dev/null +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -0,0 +1,163 @@ +#include "extensions/clusters/dynamic_forward_proxy/cluster.h" + +#include "extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" + +namespace Envoy { +namespace Extensions { +namespace Clusters { +namespace DynamicForwardProxy { + +// TODO(mattklein123): Make sure that the cluster's hosts display their host name in admin output. +// TODO(mattklein123): Allow customizing TLS on a per-host basis. For example, setting SNI and +// doing certificate validation. + +Cluster::Cluster( + const envoy::api::v2::Cluster& cluster, + const envoy::config::cluster::dynamic_forward_proxy::v2alpha::ClusterConfig& config, + Runtime::Loader& runtime, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + const LocalInfo::LocalInfo& local_info, + Server::Configuration::TransportSocketFactoryContext& factory_context, + Stats::ScopePtr&& stats_scope, bool added_via_api) + : Upstream::BaseDynamicClusterImpl(cluster, runtime, factory_context, std::move(stats_scope), + added_via_api), + dns_cache_manager_(cache_manager_factory.get()), + dns_cache_(dns_cache_manager_->getCache(config.dns_cache_config())), + update_callbacks_handle_(dns_cache_->addUpdateCallbacks(*this)), local_info_(local_info), + host_map_(std::make_shared()) { + // TODO(mattklein123): Technically, we should support attaching to an already warmed DNS cache. + // This will require adding a hosts() or similar API to the cache and + // reading it during initialization. + + // Block certain TLS context parameters that don't make sense on a cluster-wide scale. We will + // support these parameters dynamically in the future. This is not an exhaustive list of + // parameters that don't make sense but should be the most obvious ones that a user might set + // in error. + if (!cluster.tls_context().sni().empty() || !cluster.tls_context() + .common_tls_context() + .validation_context() + .verify_subject_alt_name() + .empty()) { + throw EnvoyException( + "dynamic_forward_proxy cluster cannot configure 'sni' or 'verify_subject_alt_name'"); + } +} + +void Cluster::onDnsHostAddOrUpdate( + const std::string& host, + const Extensions::Common::DynamicForwardProxy::DnsHostInfoSharedPtr& host_info) { + // We should never get a host with no address from the cache. + ASSERT(host_info->address() != nullptr); + + HostInfoMapSharedPtr current_map = getCurrentHostMap(); + const auto host_map_it = current_map->find(host); + if (host_map_it != current_map->end()) { + // If we only have an address change, we can do that swap inline without any other updates. The + // appropriate R/W locking is in place to allow this. The details of this locking are: + // - Hosts are not thread local, they are global. + // - We take a read lock when reading the address and a write lock when changing it. + // - Address updates are very rare. + // - Address reads are only done when a connection is being made and a "real" host description + // is created or the host is queries via the admin endpoint. Both of these operations are + // relatively rare and the read lock is held for a short period of time. + // + // TODO(mattklein123): Right now the dynamic forward proxy / DNS cache works similar to how + // logical DNS works, meaning that we only store a single address per + // resolution. It would not be difficult to also expose strict DNS + // semantics, meaning the cache would expose multiple addresses and the + // cluster would create multiple logical hosts based on those addresses. + // We will leave this is a follow up depending on need. + ASSERT(host_info == host_map_it->second.shared_host_info_); + ASSERT(host_map_it->second.shared_host_info_->address() != + host_map_it->second.logical_host_->address()); + ENVOY_LOG(debug, "updating dfproxy cluster host address '{}'", host); + host_map_it->second.logical_host_->setNewAddress(host_info->address(), dummy_lb_endpoint_); + return; + } + + ENVOY_LOG(debug, "adding new dfproxy cluster host '{}'", host); + const auto new_host_map = std::make_shared(*current_map); + const auto emplaced = new_host_map->try_emplace( + host, host_info, + std::make_shared(info(), host, host_info->address(), + dummy_locality_lb_endpoint_, dummy_lb_endpoint_)); + Upstream::HostVector hosts_added; + hosts_added.emplace_back(emplaced.first->second.logical_host_); + + // Swap in the new map. This will be picked up when the per-worker LBs are recreated via + // the host set update. + swapAndUpdateMap(new_host_map, hosts_added, {}); +} + +void Cluster::swapAndUpdateMap(const HostInfoMapSharedPtr& new_hosts_map, + const Upstream::HostVector& hosts_added, + const Upstream::HostVector& hosts_removed) { + { + absl::WriterMutexLock lock(&host_map_lock_); + host_map_ = new_hosts_map; + } + + Upstream::PriorityStateManager priority_state_manager(*this, local_info_, nullptr); + priority_state_manager.initializePriorityFor(dummy_locality_lb_endpoint_); + for (const auto& host : (*new_hosts_map)) { + priority_state_manager.registerHostForPriority(host.second.logical_host_, + dummy_locality_lb_endpoint_); + } + priority_state_manager.updateClusterPrioritySet( + 0, std::move(priority_state_manager.priorityState()[0].first), hosts_added, hosts_removed, + absl::nullopt, absl::nullopt); +} + +void Cluster::onDnsHostRemove(const std::string& host) { + HostInfoMapSharedPtr current_map = getCurrentHostMap(); + const auto host_map_it = current_map->find(host); + ASSERT(host_map_it != current_map->end()); + const auto new_host_map = std::make_shared(*current_map); + Upstream::HostVector hosts_removed; + hosts_removed.emplace_back(host_map_it->second.logical_host_); + new_host_map->erase(host); + ENVOY_LOG(debug, "removing dfproxy cluster host '{}'", host); + + // Swap in the new map. This will be picked up when the per-worker LBs are recreated via + // the host set update. + swapAndUpdateMap(new_host_map, {}, hosts_removed); +} + +Upstream::HostConstSharedPtr +Cluster::LoadBalancer::chooseHost(Upstream::LoadBalancerContext* context) { + if (!context || !context->downstreamHeaders()) { + return nullptr; + } + + const auto host_it = + host_map_->find(context->downstreamHeaders()->Host()->value().getStringView()); + if (host_it == host_map_->end()) { + return nullptr; + } else { + host_it->second.shared_host_info_->touch(); + return host_it->second.logical_host_; + } +} + +std::pair +ClusterFactory::createClusterWithConfig( + const envoy::api::v2::Cluster& cluster, + const envoy::config::cluster::dynamic_forward_proxy::v2alpha::ClusterConfig& proto_config, + Upstream::ClusterFactoryContext& context, + Server::Configuration::TransportSocketFactoryContext& socket_factory_context, + Stats::ScopePtr&& stats_scope) { + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( + context.singletonManager(), context.dispatcher(), context.tls()); + auto new_cluster = std::make_shared( + cluster, proto_config, context.runtime(), cache_manager_factory, context.localInfo(), + socket_factory_context, std::move(stats_scope), context.addedViaApi()); + auto lb = std::make_unique(*new_cluster); + return std::make_pair(new_cluster, std::move(lb)); +} + +REGISTER_FACTORY(ClusterFactory, Upstream::ClusterFactory); + +} // namespace DynamicForwardProxy +} // namespace Clusters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.h b/source/extensions/clusters/dynamic_forward_proxy/cluster.h new file mode 100644 index 000000000000..7635e91cb49b --- /dev/null +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.h @@ -0,0 +1,134 @@ +#pragma once + +#include "envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.pb.h" +#include "envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.pb.validate.h" + +#include "common/upstream/cluster_factory_impl.h" +#include "common/upstream/logical_host.h" + +#include "extensions/clusters/well_known_names.h" +#include "extensions/common/dynamic_forward_proxy/dns_cache.h" + +namespace Envoy { +namespace Extensions { +namespace Clusters { +namespace DynamicForwardProxy { + +class Cluster : public Upstream::BaseDynamicClusterImpl, + public Extensions::Common::DynamicForwardProxy::DnsCache::UpdateCallbacks { +public: + Cluster(const envoy::api::v2::Cluster& cluster, + const envoy::config::cluster::dynamic_forward_proxy::v2alpha::ClusterConfig& config, + Runtime::Loader& runtime, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + const LocalInfo::LocalInfo& local_info, + Server::Configuration::TransportSocketFactoryContext& factory_context, + Stats::ScopePtr&& stats_scope, bool added_via_api); + + // Upstream::Cluster + Upstream::Cluster::InitializePhase initializePhase() const override { + return Upstream::Cluster::InitializePhase::Primary; + } + + // Upstream::ClusterImplBase + void startPreInit() override { + // Nothing to do during initialization. + onPreInitComplete(); + } + + // Extensions::Common::DynamicForwardProxy::DnsCache::UpdateCallbacks + void onDnsHostAddOrUpdate( + const std::string& host, + const Extensions::Common::DynamicForwardProxy::DnsHostInfoSharedPtr& host_info) override; + void onDnsHostRemove(const std::string& host) override; + +private: + struct HostInfo { + HostInfo(const Extensions::Common::DynamicForwardProxy::DnsHostInfoSharedPtr& shared_host_info, + const Upstream::LogicalHostSharedPtr& logical_host) + : shared_host_info_(shared_host_info), logical_host_(logical_host) {} + + const Extensions::Common::DynamicForwardProxy::DnsHostInfoSharedPtr shared_host_info_; + const Upstream::LogicalHostSharedPtr logical_host_; + }; + + using HostInfoMap = absl::flat_hash_map; + using HostInfoMapSharedPtr = std::shared_ptr; + + struct LoadBalancer : public Upstream::LoadBalancer { + LoadBalancer(const HostInfoMapSharedPtr& host_map) : host_map_(host_map) {} + + // Upstream::LoadBalancer + Upstream::HostConstSharedPtr chooseHost(Upstream::LoadBalancerContext* context) override; + + const HostInfoMapSharedPtr host_map_; + }; + + struct LoadBalancerFactory : public Upstream::LoadBalancerFactory { + LoadBalancerFactory(Cluster& cluster) : cluster_(cluster) {} + + // Upstream::LoadBalancerFactory + Upstream::LoadBalancerPtr create() override { + return std::make_unique(cluster_.getCurrentHostMap()); + } + + Cluster& cluster_; + }; + + struct ThreadAwareLoadBalancer : public Upstream::ThreadAwareLoadBalancer { + ThreadAwareLoadBalancer(Cluster& cluster) : cluster_(cluster) {} + + // Upstream::ThreadAwareLoadBalancer + Upstream::LoadBalancerFactorySharedPtr factory() override { + return std::make_shared(cluster_); + } + void initialize() override {} + + Cluster& cluster_; + }; + + HostInfoMapSharedPtr getCurrentHostMap() { + absl::ReaderMutexLock lock(&host_map_lock_); + return host_map_; + } + + void swapAndUpdateMap(const HostInfoMapSharedPtr& new_hosts_map, + const Upstream::HostVector& hosts_added, + const Upstream::HostVector& hosts_removed); + + const Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr dns_cache_manager_; + const Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache_; + const Extensions::Common::DynamicForwardProxy::DnsCache::AddUpdateCallbacksHandlePtr + update_callbacks_handle_; + const envoy::api::v2::endpoint::LocalityLbEndpoints dummy_locality_lb_endpoint_; + const envoy::api::v2::endpoint::LbEndpoint dummy_lb_endpoint_; + const LocalInfo::LocalInfo& local_info_; + + absl::Mutex host_map_lock_; + HostInfoMapSharedPtr host_map_ ABSL_GUARDED_BY(host_map_lock_); + + friend class ClusterFactory; + friend class ClusterTest; +}; + +class ClusterFactory : public Upstream::ConfigurableClusterFactoryBase< + envoy::config::cluster::dynamic_forward_proxy::v2alpha::ClusterConfig> { +public: + ClusterFactory() + : ConfigurableClusterFactoryBase( + Extensions::Clusters::ClusterTypes::get().DynamicForwardProxy) {} + +private: + std::pair + createClusterWithConfig( + const envoy::api::v2::Cluster& cluster, + const envoy::config::cluster::dynamic_forward_proxy::v2alpha::ClusterConfig& proto_config, + Upstream::ClusterFactoryContext& context, + Server::Configuration::TransportSocketFactoryContext& socket_factory_context, + Stats::ScopePtr&& stats_scope) override; +}; + +} // namespace DynamicForwardProxy +} // namespace Clusters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/clusters/well_known_names.h b/source/extensions/clusters/well_known_names.h index b074bdd644bc..48654e4f2c71 100644 --- a/source/extensions/clusters/well_known_names.h +++ b/source/extensions/clusters/well_known_names.h @@ -31,6 +31,10 @@ class ClusterTypeValues { // Redis cluster (cluster that reads host information using the redis cluster protocol). const std::string Redis = "envoy.clusters.redis"; + + // Dynamic forward proxy cluster. This cluster is designed to work directly with the + // dynamic forward proxy HTTP filter. + const std::string DynamicForwardProxy = "envoy.clusters.dynamic_forward_proxy"; }; using ClusterTypes = ConstSingleton; diff --git a/source/extensions/common/dynamic_forward_proxy/BUILD b/source/extensions/common/dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..0b3ddf4492b3 --- /dev/null +++ b/source/extensions/common/dynamic_forward_proxy/BUILD @@ -0,0 +1,44 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "dns_cache_interface", + hdrs = ["dns_cache.h"], + deps = [ + "//include/envoy/event:dispatcher_interface", + "//include/envoy/singleton:manager_interface", + "//include/envoy/thread_local:thread_local_interface", + "@envoy_api//envoy/config/common/dynamic_forward_proxy/v2alpha:dns_cache_cc", + ], +) + +envoy_cc_library( + name = "dns_cache_manager_impl", + srcs = ["dns_cache_manager_impl.cc"], + hdrs = ["dns_cache_manager_impl.h"], + deps = [ + ":dns_cache_impl", + "//source/common/protobuf", + ], +) + +envoy_cc_library( + name = "dns_cache_impl", + srcs = ["dns_cache_impl.cc"], + hdrs = ["dns_cache_impl.h"], + deps = [ + ":dns_cache_interface", + "//include/envoy/network:dns_interface", + "//include/envoy/thread_local:thread_local_interface", + "//source/common/common:cleanup_lib", + "//source/common/network:utility_lib", + "//source/common/upstream:upstream_lib", + ], +) diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache.h b/source/extensions/common/dynamic_forward_proxy/dns_cache.h new file mode 100644 index 000000000000..eb804396ef0f --- /dev/null +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache.h @@ -0,0 +1,165 @@ +#pragma once + +#include "envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.pb.h" +#include "envoy/event/dispatcher.h" +#include "envoy/singleton/manager.h" +#include "envoy/thread_local/thread_local.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +/** + * A cached DNS host. + */ +class DnsHostInfo { +public: + virtual ~DnsHostInfo() = default; + + /** + * Returns the host's currently resolved address. This address may change periodically due to + * async re-resolution. + */ + virtual Network::Address::InstanceConstSharedPtr address() PURE; + + /** + * Indicates that the host has been used and should not be purged depending on any configured + * TTL policy + */ + virtual void touch() PURE; +}; + +using DnsHostInfoSharedPtr = std::shared_ptr; + +/** + * A cache of DNS hosts. Hosts will re-resolve their addresses or be automatically purged + * depending on configured policy. + */ +class DnsCache { +public: + /** + * Callbacks used in the loadDnsCache() method. + */ + class LoadDnsCacheCallbacks { + public: + virtual ~LoadDnsCacheCallbacks() = default; + + /** + * Called when the DNS cache load is complete (or failed). + */ + virtual void onLoadDnsCacheComplete() PURE; + }; + + /** + * Handle returned from loadDnsCache(). Destruction of the handle will cancel any future callback. + */ + class LoadDnsCacheHandle { + public: + virtual ~LoadDnsCacheHandle() = default; + }; + + using LoadDnsCacheHandlePtr = std::unique_ptr; + + /** + * Update callbacks that can be registered in the addUpdateCallbacks() method. + */ + class UpdateCallbacks { + public: + virtual ~UpdateCallbacks() = default; + + /** + * Called when a host has been added or has had its address updated. + * @param host supplies the added/updated host. + * @param host_info supplies the associated host info. + */ + virtual void onDnsHostAddOrUpdate(const std::string& host, + const DnsHostInfoSharedPtr& host_info) PURE; + + /** + * Called when a host has been removed. + * @param host supplies the removed host. + */ + virtual void onDnsHostRemove(const std::string& host) PURE; + }; + + /** + * Handle returned from addUpdateCallbacks(). Destruction of the handle will remove the + * registered callbacks. + */ + class AddUpdateCallbacksHandle { + public: + virtual ~AddUpdateCallbacksHandle() = default; + }; + + using AddUpdateCallbacksHandlePtr = std::unique_ptr; + + virtual ~DnsCache() = default; + + /** + * Initiate a DNS cache load. + * @param host supplies the host to load. Hosts are cached inclusive of port, even though the + * port will be stripped during resolution. This means that 'a.b.c' and 'a.b.c:9001' + * will both resolve 'a.b.c' but will generate different host entries with different + * target ports. + * @param default_port supplies the port to use if the host does not have a port embedded in it. + * @param callbacks supplies the cache load callbacks to invoke if async processing is needed. + * @return a cache load handle, in which case the callbacks will be invoked at a later time, + * or nullptr if the cache is already loaded. In this case, callbacks will never be + * called. + */ + virtual LoadDnsCacheHandlePtr loadDnsCache(absl::string_view host, uint16_t default_port, + LoadDnsCacheCallbacks& callbacks) PURE; + + /** + * Add update callbacks to the cache. + * @param callbacks supplies the callbacks to add. + * @return a handle that on destruction will de-register the callbacks. + */ + virtual AddUpdateCallbacksHandlePtr addUpdateCallbacks(UpdateCallbacks& callbacks) PURE; +}; + +using DnsCacheSharedPtr = std::shared_ptr; + +/** + * A manager for multiple DNS caches. + */ +class DnsCacheManager { +public: + virtual ~DnsCacheManager() = default; + + /** + * Get a DNS cache. + * @param config supplies the cache parameters. If a cache exists with the same parameters it + * will be returned, otherwise a new one will be created. + */ + virtual DnsCacheSharedPtr getCache( + const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config) PURE; +}; + +using DnsCacheManagerSharedPtr = std::shared_ptr; + +/** + * Get the singleton cache manager for the entire server. + */ +DnsCacheManagerSharedPtr getCacheManager(Singleton::Manager& manager, + Event::Dispatcher& main_thread_dispatcher, + ThreadLocal::SlotAllocator& tls); + +/** + * Factory for getting a DNS cache manager. + */ +class DnsCacheManagerFactory { +public: + virtual ~DnsCacheManagerFactory() = default; + + /** + * Get a DNS cache manager. + */ + virtual DnsCacheManagerSharedPtr get() PURE; +}; + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc new file mode 100644 index 000000000000..6eab79ba0e3f --- /dev/null +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -0,0 +1,237 @@ +#include "extensions/common/dynamic_forward_proxy/dns_cache_impl.h" + +#include "common/network/utility.h" + +// TODO(mattklein123): Move DNS family helpers to a smaller include. +#include "common/upstream/upstream_impl.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +// TODO(mattklein123): circuit breakers / maximums on the number of hosts that the cache can +// contain. +// TODO(mattklein123): stats + +DnsCacheImpl::DnsCacheImpl( + Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls, + const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config) + : main_thread_dispatcher_(main_thread_dispatcher), + dns_lookup_family_(Upstream::getDnsLookupFamilyFromEnum(config.dns_lookup_family())), + resolver_(main_thread_dispatcher.createDnsResolver({})), tls_slot_(tls.allocateSlot()), + refresh_interval_(PROTOBUF_GET_MS_OR_DEFAULT(config, dns_refresh_rate, 60000)), + host_ttl_(PROTOBUF_GET_MS_OR_DEFAULT(config, host_ttl, 300000)) { + tls_slot_->set([](Event::Dispatcher&) { return std::make_shared(); }); + updateTlsHostsMap(); +} + +DnsCacheImpl::~DnsCacheImpl() { + for (const auto& primary_host : primary_hosts_) { + if (primary_host.second->active_query_ != nullptr) { + primary_host.second->active_query_->cancel(); + } + } + + for (auto update_callbacks : update_callbacks_) { + update_callbacks->cancel(); + } +} + +DnsCacheImpl::LoadDnsCacheHandlePtr DnsCacheImpl::loadDnsCache(absl::string_view host, + uint16_t default_port, + LoadDnsCacheCallbacks& callbacks) { + ENVOY_LOG(debug, "thread local lookup for host '{}'", host); + auto& tls_host_info = tls_slot_->getTyped(); + auto tls_host = tls_host_info.host_map_->find(host); + if (tls_host != tls_host_info.host_map_->end()) { + ENVOY_LOG(debug, "thread local hit for host '{}'", host); + return nullptr; + } else { + ENVOY_LOG(debug, "thread local miss for host '{}', posting to main thread", host); + main_thread_dispatcher_.post( + [this, host = std::string(host), default_port]() { startCacheLoad(host, default_port); }); + return std::make_unique(tls_host_info.pending_resolutions_, host, + callbacks); + } +} + +DnsCacheImpl::AddUpdateCallbacksHandlePtr +DnsCacheImpl::addUpdateCallbacks(UpdateCallbacks& callbacks) { + return std::make_unique(update_callbacks_, callbacks); +} + +void DnsCacheImpl::startCacheLoad(const std::string& host, uint16_t default_port) { + // It's possible for multiple requests to race trying to start a resolution. If a host is + // already in the map it's either in the process of being resolved or the resolution is already + // heading out to the worker threads. Either way the pending resolution will be completed. + const auto primary_host_it = primary_hosts_.find(host); + if (primary_host_it != primary_hosts_.end()) { + ENVOY_LOG(debug, "main thread resolve for host '{}' skipped. Entry present", host); + return; + } + + // TODO(mattklein123): Figure out if we want to support addresses of the form :. This + // seems unlikely to be useful in TLS scenarios, but it is technically + // supported. We might want to block this form for now. + const auto colon_pos = host.find(':'); + absl::string_view host_to_resolve = host; + if (colon_pos != absl::string_view::npos) { + const absl::string_view string_view_host = host; + host_to_resolve = string_view_host.substr(0, colon_pos); + const auto port_str = string_view_host.substr(colon_pos + 1); + uint64_t port64; + if (port_str.empty() || !absl::SimpleAtoi(port_str, &port64) || port64 > 65535) { + // Just attempt to resolve whatever we were given. This will very likely fail. + // TODO(mattklein123): Should we actually fail here or do something different? + host_to_resolve = host; + } else { + default_port = port64; + } + } + + // TODO(mattklein123): Right now, the same host with different ports will become two + // independent primary hosts with independent DNS resolutions. I'm not sure how much this will + // matter, but we could consider collapsing these down and sharing the underlying DNS resolution. + auto& primary_host = + *primary_hosts_ + // try_emplace() is used here for direct argument forwarding. + .try_emplace(host, + std::make_unique(*this, host_to_resolve, default_port, + [this, host]() { onReResolve(host); })) + .first->second; + startResolve(host, primary_host); +} + +void DnsCacheImpl::onReResolve(const std::string& host) { + const auto primary_host_it = primary_hosts_.find(host); + ASSERT(primary_host_it != primary_hosts_.end()); + + const std::chrono::steady_clock::duration now_duration = + main_thread_dispatcher_.timeSource().monotonicTime().time_since_epoch(); + ENVOY_LOG(debug, "host='{}' TTL check: now={} last_used={}", primary_host_it->first, + now_duration.count(), + primary_host_it->second->host_info_->last_used_time_.load().count()); + if (now_duration - primary_host_it->second->host_info_->last_used_time_.load() > host_ttl_) { + ENVOY_LOG(debug, "host='{}' TTL expired, removing", host); + runRemoveCallbacks(host); + primary_hosts_.erase(primary_host_it); + updateTlsHostsMap(); + } else { + startResolve(host, *primary_host_it->second); + } +} + +void DnsCacheImpl::startResolve(const std::string& host, PrimaryHostInfo& host_info) { + ENVOY_LOG(debug, "starting main thread resolve for host='{}' dns='{}' port='{}'", host, + host_info.host_to_resolve_, host_info.port_); + ASSERT(host_info.active_query_ == nullptr); + host_info.active_query_ = resolver_->resolve( + host_info.host_to_resolve_, dns_lookup_family_, + [this, host](const std::list&& address_list) { + finishResolve(host, address_list); + }); +} + +void DnsCacheImpl::finishResolve( + const std::string& host, + const std::list& address_list) { + ENVOY_LOG(debug, "main thread resolve complete for host '{}'. {} results", host, + address_list.size()); + const auto primary_host_it = primary_hosts_.find(host); + ASSERT(primary_host_it != primary_hosts_.end()); + + auto& primary_host_info = *primary_host_it->second; + primary_host_info.active_query_ = nullptr; + const bool first_resolve = primary_host_info.host_info_ == nullptr; + if (primary_host_info.host_info_ == nullptr) { + primary_host_info.host_info_ = + std::make_shared(main_thread_dispatcher_.timeSource()); + } + + const auto new_address = + !address_list.empty() + ? Network::Utility::getAddressWithPort(*address_list.front(), primary_host_info.port_) + : nullptr; + + // Only the change the address if: + // 1) The new address is valid && + // 2a) The host doesn't yet have an address || + // 2b) The host has a changed address. + // + // This means that once a host gets an address it will stick even in the case of a subsequent + // resolution failure. + bool address_changed = false; + if (new_address != nullptr && (primary_host_info.host_info_->address_ == nullptr || + *primary_host_info.host_info_->address_ != *new_address)) { + ENVOY_LOG(debug, "host '{}' address has changed", host); + primary_host_info.host_info_->address_ = new_address; + runAddUpdateCallbacks(host, primary_host_info.host_info_); + address_changed = true; + } + + if (first_resolve || address_changed) { + updateTlsHostsMap(); + } + + // Kick off the refresh timer. + // TODO(mattklein123): Consider jitter here. It may not be necessary since the initial host + // is populated dynamically. + primary_host_info.refresh_timer_->enableTimer(refresh_interval_); +} + +void DnsCacheImpl::runAddUpdateCallbacks(const std::string& host, + const DnsHostInfoSharedPtr& host_info) { + for (auto callbacks : update_callbacks_) { + callbacks->callbacks_.onDnsHostAddOrUpdate(host, host_info); + } +} + +void DnsCacheImpl::runRemoveCallbacks(const std::string& host) { + for (auto callbacks : update_callbacks_) { + callbacks->callbacks_.onDnsHostRemove(host); + } +} + +void DnsCacheImpl::updateTlsHostsMap() { + TlsHostMapSharedPtr new_host_map = std::make_shared(); + for (const auto& primary_host : primary_hosts_) { + // Do not include hosts without host info. This only happens before we get the first + // resolution. + if (primary_host.second->host_info_ != nullptr) { + new_host_map->emplace(primary_host.first, primary_host.second->host_info_); + } + } + + tls_slot_->runOnAllThreads([this, new_host_map]() { + tls_slot_->getTyped().updateHostMap(new_host_map); + }); +} + +DnsCacheImpl::ThreadLocalHostInfo::~ThreadLocalHostInfo() { + // Make sure we cancel any handles that still exist. + for (auto pending_resolution : pending_resolutions_) { + pending_resolution->cancel(); + } +} + +void DnsCacheImpl::ThreadLocalHostInfo::updateHostMap(const TlsHostMapSharedPtr& new_host_map) { + host_map_ = new_host_map; + for (auto pending_resolution_it = pending_resolutions_.begin(); + pending_resolution_it != pending_resolutions_.end();) { + auto& pending_resolution = **pending_resolution_it; + if (host_map_->count(pending_resolution.host_) != 0) { + auto& callbacks = pending_resolution.callbacks_; + pending_resolution.cancel(); + pending_resolution_it = pending_resolutions_.erase(pending_resolution_it); + callbacks.onLoadDnsCacheComplete(); + } else { + ++pending_resolution_it; + } + } +} + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h new file mode 100644 index 000000000000..741eceb04966 --- /dev/null +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -0,0 +1,115 @@ +#pragma once + +#include "envoy/network/dns.h" +#include "envoy/thread_local/thread_local.h" + +#include "common/common/cleanup.h" + +#include "extensions/common/dynamic_forward_proxy/dns_cache.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +class DnsCacheImpl : public DnsCache, Logger::Loggable { +public: + DnsCacheImpl(Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls, + const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config); + ~DnsCacheImpl(); + + // DnsCache + LoadDnsCacheHandlePtr loadDnsCache(absl::string_view host, uint16_t default_port, + LoadDnsCacheCallbacks& callbacks) override; + AddUpdateCallbacksHandlePtr addUpdateCallbacks(UpdateCallbacks& callbacks) override; + +private: + using TlsHostMap = absl::flat_hash_map; + using TlsHostMapSharedPtr = std::shared_ptr; + + struct LoadDnsCacheHandleImpl : public LoadDnsCacheHandle, + RaiiListElement { + LoadDnsCacheHandleImpl(std::list& parent, absl::string_view host, + LoadDnsCacheCallbacks& callbacks) + : RaiiListElement(parent, this), host_(host), + callbacks_(callbacks) {} + + const std::string host_; + LoadDnsCacheCallbacks& callbacks_; + }; + + // Per-thread DNS cache info including the currently known hosts as well as any pending callbacks. + struct ThreadLocalHostInfo : public ThreadLocal::ThreadLocalObject { + ~ThreadLocalHostInfo(); + void updateHostMap(const TlsHostMapSharedPtr& new_host_map); + + TlsHostMapSharedPtr host_map_; + std::list pending_resolutions_; + }; + + struct DnsHostInfoImpl : public DnsHostInfo { + DnsHostInfoImpl(TimeSource& time_source) : time_source_(time_source) { touch(); } + + // DnsHostInfo + Network::Address::InstanceConstSharedPtr address() override { return address_; } + void touch() override { last_used_time_ = time_source_.monotonicTime().time_since_epoch(); } + + TimeSource& time_source_; + Network::Address::InstanceConstSharedPtr address_; + // Using std::chrono::steady_clock::duration is required for compilation within an atomic vs. + // using MonotonicTime. + std::atomic last_used_time_; + }; + + using DnsHostInfoImplSharedPtr = std::shared_ptr; + + // Primary host information that accounts for TTL, re-resolution, etc. + struct PrimaryHostInfo { + PrimaryHostInfo(DnsCacheImpl& parent, absl::string_view host_to_resolve, uint16_t port, + const Event::TimerCb& timer_cb) + : host_to_resolve_(host_to_resolve), port_(port), + refresh_timer_(parent.main_thread_dispatcher_.createTimer(timer_cb)) {} + + const std::string host_to_resolve_; + const uint16_t port_; + const Event::TimerPtr refresh_timer_; + DnsHostInfoImplSharedPtr host_info_; + Network::ActiveDnsQuery* active_query_{}; + }; + + using PrimaryHostInfoPtr = std::unique_ptr; + + struct AddUpdateCallbacksHandleImpl : public AddUpdateCallbacksHandle, + RaiiListElement { + AddUpdateCallbacksHandleImpl(std::list& parent, + UpdateCallbacks& callbacks) + : RaiiListElement(parent, this), callbacks_(callbacks) {} + + UpdateCallbacks& callbacks_; + }; + + void startCacheLoad(const std::string& host, uint16_t default_port); + void startResolve(const std::string& host, PrimaryHostInfo& host_info); + void finishResolve(const std::string& host, + const std::list& address_list); + void runAddUpdateCallbacks(const std::string& host, const DnsHostInfoSharedPtr& host_info); + void runRemoveCallbacks(const std::string& host); + void updateTlsHostsMap(); + void onReResolve(const std::string& host); + + Event::Dispatcher& main_thread_dispatcher_; + const Network::DnsLookupFamily dns_lookup_family_; + const Network::DnsResolverSharedPtr resolver_; + const ThreadLocal::SlotPtr tls_slot_; + std::list update_callbacks_; + absl::flat_hash_map primary_hosts_; + const std::chrono::milliseconds refresh_interval_; + const std::chrono::milliseconds host_ttl_; +}; + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc new file mode 100644 index 000000000000..0219746d0116 --- /dev/null +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc @@ -0,0 +1,46 @@ +#include "extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" + +#include "common/protobuf/protobuf.h" + +#include "extensions/common/dynamic_forward_proxy/dns_cache_impl.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +SINGLETON_MANAGER_REGISTRATION(dns_cache_manager); + +DnsCacheSharedPtr DnsCacheManagerImpl::getCache( + const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config) { + const auto& existing_cache = caches_.find(config.name()); + if (existing_cache != caches_.end()) { + if (!Protobuf::util::MessageDifferencer::Equivalent(config, existing_cache->second.config_)) { + throw EnvoyException( + fmt::format("config specified DNS cache '{}' with different settings", config.name())); + } + + return existing_cache->second.cache_; + } + + DnsCacheSharedPtr new_cache = + std::make_shared(main_thread_dispatcher_, tls_, config); + caches_.emplace(config.name(), ActiveCache{config, new_cache}); + return new_cache; +} + +DnsCacheManagerSharedPtr getCacheManager(Singleton::Manager& singleton_manager, + Event::Dispatcher& main_thread_dispatcher, + ThreadLocal::SlotAllocator& tls) { + return singleton_manager.getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(dns_cache_manager), [&main_thread_dispatcher, &tls] { + return std::make_shared(main_thread_dispatcher, tls); + }); +} + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h new file mode 100644 index 000000000000..7217028932d4 --- /dev/null +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h @@ -0,0 +1,55 @@ +#pragma once + +#include "extensions/common/dynamic_forward_proxy/dns_cache.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +class DnsCacheManagerImpl : public DnsCacheManager, public Singleton::Instance { +public: + DnsCacheManagerImpl(Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls) + : main_thread_dispatcher_(main_thread_dispatcher), tls_(tls) {} + + // DnsCacheManager + DnsCacheSharedPtr + getCache(const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config); + +private: + struct ActiveCache { + ActiveCache(const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config, + DnsCacheSharedPtr cache) + : config_(config), cache_(cache) {} + + const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig config_; + DnsCacheSharedPtr cache_; + }; + + Event::Dispatcher& main_thread_dispatcher_; + ThreadLocal::SlotAllocator& tls_; + absl::flat_hash_map caches_; +}; + +class DnsCacheManagerFactoryImpl : public DnsCacheManagerFactory { +public: + DnsCacheManagerFactoryImpl(Singleton::Manager& singleton_manager, Event::Dispatcher& dispatcher, + ThreadLocal::SlotAllocator& tls) + : singleton_manager_(singleton_manager), dispatcher_(dispatcher), tls_(tls) {} + + DnsCacheManagerSharedPtr get() override { + return getCacheManager(singleton_manager_, dispatcher_, tls_); + } + +private: + Singleton::Manager& singleton_manager_; + Event::Dispatcher& dispatcher_; + ThreadLocal::SlotAllocator& tls_; +}; + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index a132b9aa59ce..94d75a42835b 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -10,6 +10,7 @@ EXTENSIONS = { # # Clusters # + "envoy.clusters.dynamic_forward_proxy": "//source/extensions/clusters/dynamic_forward_proxy:cluster", "envoy.clusters.redis": "//source/extensions/clusters/redis:redis_cluster", # @@ -31,6 +32,7 @@ EXTENSIONS = { "envoy.filters.http.buffer": "//source/extensions/filters/http/buffer:config", "envoy.filters.http.cors": "//source/extensions/filters/http/cors:config", "envoy.filters.http.csrf": "//source/extensions/filters/http/csrf:config", + "envoy.filters.http.dynamic_forward_proxy": "//source/extensions/filters/http/dynamic_forward_proxy:config", "envoy.filters.http.dynamo": "//source/extensions/filters/http/dynamo:config", "envoy.filters.http.ext_authz": "//source/extensions/filters/http/ext_authz:config", "envoy.filters.http.fault": "//source/extensions/filters/http/fault:config", diff --git a/source/extensions/filters/http/dynamic_forward_proxy/BUILD b/source/extensions/filters/http/dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..a11ce93fbdec --- /dev/null +++ b/source/extensions/filters/http/dynamic_forward_proxy/BUILD @@ -0,0 +1,35 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "proxy_filter_lib", + srcs = ["proxy_filter.cc"], + hdrs = ["proxy_filter.h"], + deps = [ + "//include/envoy/http:filter_interface", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_interface", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "@envoy_api//envoy/config/filter/http/dynamic_forward_proxy/v2alpha:dynamic_forward_proxy_cc", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//include/envoy/registry", + "//include/envoy/server:filter_config_interface", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_manager_impl", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/common:factory_base_lib", + "//source/extensions/filters/http/dynamic_forward_proxy:proxy_filter_lib", + ], +) diff --git a/source/extensions/filters/http/dynamic_forward_proxy/config.cc b/source/extensions/filters/http/dynamic_forward_proxy/config.cc new file mode 100644 index 000000000000..dc4b93374186 --- /dev/null +++ b/source/extensions/filters/http/dynamic_forward_proxy/config.cc @@ -0,0 +1,32 @@ +#include "extensions/filters/http/dynamic_forward_proxy/config.h" + +#include "extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" +#include "extensions/filters/http/dynamic_forward_proxy/proxy_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace DynamicForwardProxy { + +Http::FilterFactoryCb DynamicForwardProxyFilterFactory::createFilterFactoryFromProtoTyped( + const envoy::config::filter::http::dynamic_forward_proxy::v2alpha::FilterConfig& proto_config, + const std::string&, Server::Configuration::FactoryContext& context) { + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( + context.singletonManager(), context.dispatcher(), context.threadLocal()); + ProxyFilterConfigSharedPtr filter_config(std::make_shared( + proto_config, cache_manager_factory, context.clusterManager())); + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamDecoderFilter(std::make_shared(filter_config)); + }; +} + +/** + * Static registration for the dynamic forward proxy filter. @see RegisterFactory. + */ +REGISTER_FACTORY(DynamicForwardProxyFilterFactory, + Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace DynamicForwardProxy +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/dynamic_forward_proxy/config.h b/source/extensions/filters/http/dynamic_forward_proxy/config.h new file mode 100644 index 000000000000..9f8596c280ea --- /dev/null +++ b/source/extensions/filters/http/dynamic_forward_proxy/config.h @@ -0,0 +1,32 @@ +#pragma once + +#include "envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.pb.h" +#include "envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.pb.validate.h" + +#include "extensions/filters/http/common/factory_base.h" +#include "extensions/filters/http/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace DynamicForwardProxy { + +/** + * Config registration for the dynamic forward proxy filter. + */ +class DynamicForwardProxyFilterFactory + : public Common::FactoryBase< + envoy::config::filter::http::dynamic_forward_proxy::v2alpha::FilterConfig> { +public: + DynamicForwardProxyFilterFactory() : FactoryBase(HttpFilterNames::get().DynamicForwardProxy) {} + +private: + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::config::filter::http::dynamic_forward_proxy::v2alpha::FilterConfig& proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +} // namespace DynamicForwardProxy +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc new file mode 100644 index 000000000000..f0df762ac708 --- /dev/null +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc @@ -0,0 +1,65 @@ +#include "extensions/filters/http/dynamic_forward_proxy/proxy_filter.h" + +#include "extensions/common/dynamic_forward_proxy/dns_cache.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace DynamicForwardProxy { + +ProxyFilterConfig::ProxyFilterConfig( + const envoy::config::filter::http::dynamic_forward_proxy::v2alpha::FilterConfig& proto_config, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + Upstream::ClusterManager& cluster_manager) + : dns_cache_manager_(cache_manager_factory.get()), + dns_cache_(dns_cache_manager_->getCache(proto_config.dns_cache_config())), + cluster_manager_(cluster_manager) {} + +void ProxyFilter::onDestroy() { + // Make sure we destroy any active cache load handle in case we are getting reset and deferred + // deleted. + cache_load_handle_.reset(); +} + +Http::FilterHeadersStatus ProxyFilter::decodeHeaders(Http::HeaderMap& headers, bool) { + Router::RouteConstSharedPtr route = decoder_callbacks_->route(); + if (!route || !route->routeEntry()) { + return Http::FilterHeadersStatus::Continue; + } + + Upstream::ThreadLocalCluster* cluster = + config_->clusterManager().get(route->routeEntry()->clusterName()); + if (!cluster) { + return Http::FilterHeadersStatus::Continue; + } + + uint16_t default_port = 80; + if (cluster->info()->transportSocketFactory().implementsSecureTransport()) { + default_port = 443; + } + + // See the comments in dns_cache.h for how loadDnsCache() handles hosts with embedded ports. + // TODO(mattklein123): Because the filter and cluster have independent configuration, it is + // not obvious to the user if something is misconfigured. We should see if + // we can do better here, perhaps by checking the cache to see if anything + // else is attached to it or something else? + cache_load_handle_ = + config_->cache().loadDnsCache(headers.Host()->value().getStringView(), default_port, *this); + if (cache_load_handle_ == nullptr) { + ENVOY_STREAM_LOG(debug, "DNS cache already loaded, continuing", *decoder_callbacks_); + return Http::FilterHeadersStatus::Continue; + } + + ENVOY_STREAM_LOG(debug, "waiting to load DNS cache", *decoder_callbacks_); + return Http::FilterHeadersStatus::StopAllIterationAndWatermark; +} + +void ProxyFilter::onLoadDnsCacheComplete() { + ENVOY_STREAM_LOG(debug, "load DNS cache complete, continuing", *decoder_callbacks_); + decoder_callbacks_->continueDecoding(); +} + +} // namespace DynamicForwardProxy +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h new file mode 100644 index 000000000000..83f431eb1d93 --- /dev/null +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h @@ -0,0 +1,53 @@ +#pragma once + +#include "envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.pb.h" +#include "envoy/upstream/cluster_manager.h" + +#include "extensions/common/dynamic_forward_proxy/dns_cache.h" +#include "extensions/filters/http/common/pass_through_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace DynamicForwardProxy { + +class ProxyFilterConfig { +public: + ProxyFilterConfig( + const envoy::config::filter::http::dynamic_forward_proxy::v2alpha::FilterConfig& proto_config, + Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, + Upstream::ClusterManager& cluster_manager); + + Extensions::Common::DynamicForwardProxy::DnsCache& cache() { return *dns_cache_; } + Upstream::ClusterManager& clusterManager() { return cluster_manager_; } + +private: + const Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr dns_cache_manager_; + const Extensions::Common::DynamicForwardProxy::DnsCacheSharedPtr dns_cache_; + Upstream::ClusterManager& cluster_manager_; +}; + +using ProxyFilterConfigSharedPtr = std::shared_ptr; + +class ProxyFilter : public Http::PassThroughDecoderFilter, + public Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheCallbacks, + Logger::Loggable { +public: + ProxyFilter(const ProxyFilterConfigSharedPtr& config) : config_(config) {} + + // Http::PassThroughDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap& headers, bool end_stream) override; + void onDestroy() override; + + // Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheCallbacks + void onLoadDnsCacheComplete() override; + +private: + const ProxyFilterConfigSharedPtr config_; + Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheHandlePtr cache_load_handle_; +}; + +} // namespace DynamicForwardProxy +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/well_known_names.h b/source/extensions/filters/http/well_known_names.h index d62f3886d5bc..3ce6643fa8fc 100644 --- a/source/extensions/filters/http/well_known_names.h +++ b/source/extensions/filters/http/well_known_names.h @@ -56,6 +56,8 @@ class HttpFilterNameValues { const std::string Tap = "envoy.filters.http.tap"; // Original Src Filter const std::string OriginalSrc = "envoy.filters.http.original_src"; + // Dynamic forward proxy filter + const std::string DynamicForwardProxy = "envoy.filters.http.dynamic_forward_proxy"; // Converts names from v1 to v2 const Config::V1Converter v1_converter_; diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index cfc06fe84930..8ced77fcc2c4 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -401,6 +401,7 @@ envoy_cc_test_library( "//source/common/stats:stats_lib", "//source/common/upstream:upstream_includes", "//source/common/upstream:upstream_lib", + "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/clusters/dynamic_forward_proxy/BUILD b/test/extensions/clusters/dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..974cac40c1ae --- /dev/null +++ b/test/extensions/clusters/dynamic_forward_proxy/BUILD @@ -0,0 +1,32 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_mock", + "envoy_cc_test", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "cluster_test", + srcs = ["cluster_test.cc"], + data = ["//test/extensions/transport_sockets/tls/test_data:certs"], + extension_name = "envoy.filters.http.dynamic_forward_proxy", + deps = [ + "//source/extensions/clusters/dynamic_forward_proxy:cluster", + "//source/extensions/transport_sockets/raw_buffer:config", + "//source/extensions/transport_sockets/tls:config", + "//test/common/upstream:utility_lib", + "//test/extensions/common/dynamic_forward_proxy:mocks", + "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/server:server_mocks", + "//test/mocks/ssl:ssl_mocks", + "//test/test_common:environment_lib", + ], +) diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc new file mode 100644 index 000000000000..568637c431a2 --- /dev/null +++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc @@ -0,0 +1,224 @@ +#include "common/singleton/manager_impl.h" + +#include "extensions/clusters/dynamic_forward_proxy/cluster.h" + +#include "test/common/upstream/utility.h" +#include "test/extensions/common/dynamic_forward_proxy/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/server/mocks.h" +#include "test/mocks/ssl/mocks.h" +#include "test/test_common/environment.h" + +using testing::AtLeast; +using testing::DoAll; +using testing::InSequence; +using testing::Return; +using testing::SizeIs; + +namespace Envoy { +namespace Extensions { +namespace Clusters { +namespace DynamicForwardProxy { + +class ClusterTest : public testing::Test, + public Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory { +public: + void initialize(const std::string& yaml_config, bool uses_tls) { + envoy::api::v2::Cluster cluster_config = Upstream::parseClusterFromV2Yaml(yaml_config); + envoy::config::cluster::dynamic_forward_proxy::v2alpha::ClusterConfig config; + Config::Utility::translateOpaqueConfig(cluster_config.cluster_type().typed_config(), + ProtobufWkt::Struct::default_instance(), + ProtobufMessage::getStrictValidationVisitor(), config); + Stats::ScopePtr scope = stats_store_.createScope("cluster.name."); + Server::Configuration::TransportSocketFactoryContextImpl factory_context( + admin_, ssl_context_manager_, *scope, cm_, local_info_, dispatcher_, random_, stats_store_, + singleton_manager_, tls_, validation_visitor_, *api_); + if (uses_tls) { + EXPECT_CALL(ssl_context_manager_, createSslClientContext(_, _)); + } + EXPECT_CALL(*dns_cache_manager_, getCache(_)); + // Below we return a nullptr handle which has no effect on the code under test but isn't + // actually correct. It's possible this will have to change in the future. + EXPECT_CALL(*dns_cache_manager_->dns_cache_, addUpdateCallbacks_(_)) + .WillOnce(DoAll(SaveArgAddress(&update_callbacks_), Return(nullptr))); + cluster_ = std::make_shared(cluster_config, config, runtime_, *this, local_info_, + factory_context, std::move(scope), false); + thread_aware_lb_ = std::make_unique(*cluster_); + lb_factory_ = thread_aware_lb_->factory(); + refreshLb(); + + ON_CALL(lb_context_, downstreamHeaders()).WillByDefault(Return(&downstream_headers_)); + + cluster_->prioritySet().addMemberUpdateCb( + [this](const Upstream::HostVector& hosts_added, + const Upstream::HostVector& hosts_removed) -> void { + onMemberUpdateCb(hosts_added, hosts_removed); + }); + } + + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr get() override { + return dns_cache_manager_; + } + + void makeTestHost(const std::string& host, const std::string& address) { + EXPECT_TRUE(host_map_.find(host) == host_map_.end()); + host_map_[host] = std::make_shared(); + host_map_[host]->address_ = Network::Utility::parseInternetAddress(address); + + // Allow touch() to still be strict. + EXPECT_CALL(*host_map_[host], address()).Times(AtLeast(0)); + } + + void updateTestHostAddress(const std::string& host, const std::string& address) { + EXPECT_FALSE(host_map_.find(host) == host_map_.end()); + host_map_[host]->address_ = Network::Utility::parseInternetAddress(address); + } + + void refreshLb() { lb_ = lb_factory_->create(); } + + Upstream::MockLoadBalancerContext* setHostAndReturnContext(const std::string& host) { + downstream_headers_.remove(":authority"); + downstream_headers_.addCopy(":authority", host); + return &lb_context_; + } + + MOCK_METHOD2(onMemberUpdateCb, void(const Upstream::HostVector& hosts_added, + const Upstream::HostVector& hosts_removed)); + + Stats::IsolatedStoreImpl stats_store_; + Ssl::MockContextManager ssl_context_manager_; + NiceMock cm_; + NiceMock random_; + NiceMock tls_; + NiceMock runtime_; + NiceMock dispatcher_; + NiceMock local_info_; + NiceMock admin_; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + NiceMock validation_visitor_; + Api::ApiPtr api_{Api::createApiForTest(stats_store_)}; + std::shared_ptr dns_cache_manager_{ + new Extensions::Common::DynamicForwardProxy::MockDnsCacheManager()}; + std::shared_ptr cluster_; + Upstream::ThreadAwareLoadBalancerPtr thread_aware_lb_; + Upstream::LoadBalancerFactorySharedPtr lb_factory_; + Upstream::LoadBalancerPtr lb_; + NiceMock lb_context_; + Http::TestHeaderMapImpl downstream_headers_; + Extensions::Common::DynamicForwardProxy::DnsCache::UpdateCallbacks* update_callbacks_{}; + absl::flat_hash_map> + host_map_; + + const std::string default_yaml_config_ = R"EOF( +name: name +connect_timeout: 0.25s +cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.config.cluster.dynamic_forward_proxy.v2alpha.ClusterConfig + dns_cache_config: + name: foo + dns_lookup_family: AUTO +)EOF"; +}; + +// Basic flow of the cluster including adding hosts and removing them. +TEST_F(ClusterTest, BasicFlow) { + initialize(default_yaml_config_, false); + makeTestHost("host1", "1.2.3.4"); + InSequence s; + + // Verify no host LB cases. + EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext("foo"))); + + // LB will not resolve host1 until it has been updated. + EXPECT_CALL(*this, onMemberUpdateCb(SizeIs(1), SizeIs(0))); + update_callbacks_->onDnsHostAddOrUpdate("host1", host_map_["host1"]); + EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext("host1"))); + EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); + EXPECT_EQ("1.2.3.4:0", + cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->address()->asString()); + refreshLb(); + EXPECT_CALL(*host_map_["host1"], touch()); + EXPECT_EQ("1.2.3.4:0", lb_->chooseHost(setHostAndReturnContext("host1"))->address()->asString()); + + // After changing the address, LB will immediately resolve the new address with a refresh. + updateTestHostAddress("host1", "2.3.4.5"); + update_callbacks_->onDnsHostAddOrUpdate("host1", host_map_["host1"]); + EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); + EXPECT_EQ("2.3.4.5:0", + cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->address()->asString()); + EXPECT_CALL(*host_map_["host1"], touch()); + EXPECT_EQ("2.3.4.5:0", lb_->chooseHost(setHostAndReturnContext("host1"))->address()->asString()); + + // Remove the host, LB will still resolve until it is refreshed. + EXPECT_CALL(*this, onMemberUpdateCb(SizeIs(0), SizeIs(1))); + update_callbacks_->onDnsHostRemove("host1"); + EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); + EXPECT_CALL(*host_map_["host1"], touch()); + EXPECT_EQ("2.3.4.5:0", lb_->chooseHost(setHostAndReturnContext("host1"))->address()->asString()); + refreshLb(); + EXPECT_EQ(nullptr, lb_->chooseHost(setHostAndReturnContext("host1"))); +} + +// Various invalid LB context permutations in case the cluster is used outside of HTTP. +TEST_F(ClusterTest, InvalidLbContext) { + initialize(default_yaml_config_, false); + ON_CALL(lb_context_, downstreamHeaders()).WillByDefault(Return(nullptr)); + EXPECT_EQ(nullptr, lb_->chooseHost(&lb_context_)); + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); +} + +// Verify that using 'sni' causes a failure. +TEST_F(ClusterTest, InvalidSNI) { + const std::string yaml_config = TestEnvironment::substitute(R"EOF( +name: name +connect_timeout: 0.25s +cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.config.cluster.dynamic_forward_proxy.v2alpha.ClusterConfig + dns_cache_config: + name: foo +tls_context: + sni: api.lyft.com + common_tls_context: + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" +)EOF"); + + EXPECT_THROW_WITH_MESSAGE( + initialize(yaml_config, true), EnvoyException, + "dynamic_forward_proxy cluster cannot configure 'sni' or 'verify_subject_alt_name'"); +} + +// Verify that using 'verify_subject_alt_name' causes a failure. +TEST_F(ClusterTest, InvalidVerifySubjectAltName) { + const std::string yaml_config = TestEnvironment::substitute(R"EOF( +name: name +connect_timeout: 0.25s +cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.config.cluster.dynamic_forward_proxy.v2alpha.ClusterConfig + dns_cache_config: + name: foo +tls_context: + common_tls_context: + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + verify_subject_alt_name: [api.lyft.com] +)EOF"); + + EXPECT_THROW_WITH_MESSAGE( + initialize(yaml_config, true), EnvoyException, + "dynamic_forward_proxy cluster cannot configure 'sni' or 'verify_subject_alt_name'"); +} + +} // namespace DynamicForwardProxy +} // namespace Clusters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/common/dynamic_forward_proxy/BUILD b/test/extensions/common/dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..c9bb27447c2a --- /dev/null +++ b/test/extensions/common/dynamic_forward_proxy/BUILD @@ -0,0 +1,32 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_mock", + "envoy_cc_test", + "envoy_package", +) + +envoy_package() + +envoy_cc_test( + name = "dns_cache_impl_test", + srcs = ["dns_cache_impl_test.cc"], + deps = [ + ":mocks", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_impl", + "//source/extensions/common/dynamic_forward_proxy:dns_cache_manager_impl", + "//test/mocks/network:network_mocks", + "//test/mocks/thread_local:thread_local_mocks", + "//test/test_common:simulated_time_system_lib", + ], +) + +envoy_cc_mock( + name = "mocks", + srcs = ["mocks.cc"], + hdrs = ["mocks.h"], + deps = [ + "//source/extensions/common/dynamic_forward_proxy:dns_cache_interface", + ], +) diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc new file mode 100644 index 000000000000..a92d668f06f8 --- /dev/null +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -0,0 +1,369 @@ +#include "extensions/common/dynamic_forward_proxy/dns_cache_impl.h" +#include "extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" + +#include "test/extensions/common/dynamic_forward_proxy/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/thread_local/mocks.h" +#include "test/test_common/simulated_time_system.h" + +using testing::InSequence; +using testing::Return; +using testing::SaveArg; + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { +namespace { + +std::list +makeAddressList(const std::list address_list) { + std::list ret; + for (const auto& address : address_list) { + ret.emplace_back(Network::Utility::parseInternetAddress(address)); + } + return ret; +} + +class DnsCacheImplTest : public testing::Test, public Event::TestUsingSimulatedTime { +public: + void initialize() { + config_.set_dns_lookup_family(envoy::api::v2::Cluster::V4_ONLY); + + EXPECT_CALL(dispatcher_, createDnsResolver(_)).WillOnce(Return(resolver_)); + dns_cache_ = std::make_unique(dispatcher_, tls_, config_); + update_callbacks_handle_ = dns_cache_->addUpdateCallbacks(update_callbacks_); + } + + envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig config_; + NiceMock dispatcher_; + std::shared_ptr resolver_{std::make_shared()}; + NiceMock tls_; + std::unique_ptr dns_cache_; + MockUpdateCallbacks update_callbacks_; + DnsCache::AddUpdateCallbacksHandlePtr update_callbacks_handle_; +}; + +MATCHER_P(SharedAddressEquals, expected, "") { + const bool equal = expected == arg->address()->asString(); + if (!equal) { + *result_listener << fmt::format("'{}' != '{}'", expected, arg->address()->asString()); + } + return equal; +} + +// Basic successful resolution and then re-resolution. +TEST_F(DnsCacheImplTest, ResolveSuccess) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_NE(handle, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(makeAddressList({"10.0.0.1"})); + + // Re-resolve timer. + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + resolve_timer->invokeCallback(); + + // Address does not change. + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(makeAddressList({"10.0.0.1"})); + + // Address does change. + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.2:80"))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(makeAddressList({"10.0.0.2"})); +} + +// TTL purge test. +TEST_F(DnsCacheImplTest, TTL) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_NE(handle, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(makeAddressList({"10.0.0.1"})); + + // Re-resolve with ~60s passed. TTL should still be OK at default of 5 minutes. + simTime().sleep(std::chrono::milliseconds(60001)); + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + resolve_timer->invokeCallback(); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(makeAddressList({"10.0.0.1"})); + + // Re-resolve with ~5m passed. This is not realistic as we would have re-resolved many times + // during this period but it's good enough for the test. + simTime().sleep(std::chrono::milliseconds(300000)); + EXPECT_CALL(update_callbacks_, onDnsHostRemove("foo.com")); + resolve_timer->invokeCallback(); + + // Make sure we don't get a cache hit the next time the host is requested. + resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_NE(handle, nullptr); +} + +// TTL purge test with different refresh/TTL parameters. +TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { + *config_.mutable_dns_refresh_rate() = Protobuf::util::TimeUtil::SecondsToDuration(30); + *config_.mutable_host_ttl() = Protobuf::util::TimeUtil::SecondsToDuration(60); + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_NE(handle, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000))); + resolve_cb(makeAddressList({"10.0.0.1"})); + + // Re-resolve with ~30s passed. TTL should still be OK at 60s. + simTime().sleep(std::chrono::milliseconds(30001)); + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + resolve_timer->invokeCallback(); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000))); + resolve_cb(makeAddressList({"10.0.0.1"})); + + // Re-resolve with ~30s passed. TTL should expire. + simTime().sleep(std::chrono::milliseconds(30001)); + EXPECT_CALL(update_callbacks_, onDnsHostRemove("foo.com")); + resolve_timer->invokeCallback(); +} + +// Resolve that completes inline without any callback. +TEST_F(DnsCacheImplTest, InlineResolve) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Event::PostCb post_cb; + EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb)); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("localhost", 80, callbacks); + EXPECT_NE(handle, nullptr); + + Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("localhost", _, _)) + .WillOnce(Invoke([](const std::string&, Network::DnsLookupFamily, + Network::DnsResolver::ResolveCb callback) { + callback(makeAddressList({"127.0.0.1"})); + return nullptr; + })); + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("localhost", SharedAddressEquals("127.0.0.1:80"))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + post_cb(); +} + +// Resolve failure that returns no addresses. +TEST_F(DnsCacheImplTest, ResolveFailure) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_NE(handle, nullptr); + + EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + resolve_cb(makeAddressList({})); + + handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_EQ(handle, nullptr); +} + +// Cancel a cache load before the resolve completes. +TEST_F(DnsCacheImplTest, CancelResolve) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_NE(handle, nullptr); + + handle.reset(); + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + resolve_cb(makeAddressList({"10.0.0.1"})); +} + +// Two cache loads that are trying to resolve the same host. Make sure we only do a single resolve +// and fire both callbacks on completion. +TEST_F(DnsCacheImplTest, MultipleResolveSameHost) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks1; + Network::DnsResolver::ResolveCb resolve_cb; + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle1 = dns_cache_->loadDnsCache("foo.com", 80, callbacks1); + EXPECT_NE(handle1, nullptr); + + MockLoadDnsCacheCallbacks callbacks2; + DnsCache::LoadDnsCacheHandlePtr handle2 = dns_cache_->loadDnsCache("foo.com", 80, callbacks2); + EXPECT_NE(handle2, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + EXPECT_CALL(callbacks2, onLoadDnsCacheComplete()); + EXPECT_CALL(callbacks1, onLoadDnsCacheComplete()); + resolve_cb(makeAddressList({"10.0.0.1"})); +} + +// Two cache loads that are resolving different hosts. +TEST_F(DnsCacheImplTest, MultipleResolveDifferentHost) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks1; + Network::DnsResolver::ResolveCb resolve_cb1; + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb1), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle1 = dns_cache_->loadDnsCache("foo.com", 80, callbacks1); + EXPECT_NE(handle1, nullptr); + + MockLoadDnsCacheCallbacks callbacks2; + Network::DnsResolver::ResolveCb resolve_cb2; + EXPECT_CALL(*resolver_, resolve("bar.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb2), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle2 = dns_cache_->loadDnsCache("bar.com", 443, callbacks2); + EXPECT_NE(handle2, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("bar.com", SharedAddressEquals("10.0.0.1:443"))); + EXPECT_CALL(callbacks2, onLoadDnsCacheComplete()); + resolve_cb2(makeAddressList({"10.0.0.1"})); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.2:80"))); + EXPECT_CALL(callbacks1, onLoadDnsCacheComplete()); + resolve_cb1(makeAddressList({"10.0.0.2"})); +} + +// A successful resolve followed by a cache hit. +TEST_F(DnsCacheImplTest, CacheHit) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_NE(handle, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + resolve_cb(makeAddressList({"10.0.0.1"})); + + EXPECT_EQ(nullptr, dns_cache_->loadDnsCache("foo.com", 80, callbacks)); +} + +// Make sure we destroy active queries if the cache goes away. +TEST_F(DnsCacheImplTest, CancelActiveQueriesOnDestroy) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); + EXPECT_NE(handle, nullptr); + + EXPECT_CALL(resolver_->active_query_, cancel()); + dns_cache_.reset(); +} + +// Invalid port +TEST_F(DnsCacheImplTest, InvalidPort) { + initialize(); + InSequence s; + + MockLoadDnsCacheCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + EXPECT_CALL(*resolver_, resolve("foo.com:abc", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com:abc", 80, callbacks); + EXPECT_NE(handle, nullptr); + + EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + resolve_cb(makeAddressList({})); +} + +// DNS cache manager config tests. +TEST(DnsCacheManagerImplTest, LoadViaConfig) { + NiceMock dispatcher; + NiceMock tls; + DnsCacheManagerImpl cache_manager(dispatcher, tls); + + envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig config1; + config1.set_name("foo"); + + auto cache1 = cache_manager.getCache(config1); + EXPECT_NE(cache1, nullptr); + + envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig config2; + config2.set_name("foo"); + EXPECT_EQ(cache1, cache_manager.getCache(config2)); + + envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig config3; + config3.set_name("bar"); + auto cache2 = cache_manager.getCache(config3); + EXPECT_NE(cache2, nullptr); + EXPECT_NE(cache1, cache2); + + envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig config4; + config4.set_name("foo"); + config4.set_dns_lookup_family(envoy::api::v2::Cluster::V6_ONLY); + EXPECT_THROW_WITH_MESSAGE(cache_manager.getCache(config4), EnvoyException, + "config specified DNS cache 'foo' with different settings"); +} + +} // namespace +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.cc b/test/extensions/common/dynamic_forward_proxy/mocks.cc new file mode 100644 index 000000000000..2ec514837de3 --- /dev/null +++ b/test/extensions/common/dynamic_forward_proxy/mocks.cc @@ -0,0 +1,37 @@ +#include "test/extensions/common/dynamic_forward_proxy/mocks.h" + +using testing::_; +using testing::Return; +using testing::ReturnPointee; + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +MockDnsCache::MockDnsCache() = default; +MockDnsCache::~MockDnsCache() = default; + +MockLoadDnsCacheHandle::MockLoadDnsCacheHandle() = default; +MockLoadDnsCacheHandle::~MockLoadDnsCacheHandle() { onDestroy(); } + +MockDnsCacheManager::MockDnsCacheManager() { + ON_CALL(*this, getCache(_)).WillByDefault(Return(dns_cache_)); +} +MockDnsCacheManager::~MockDnsCacheManager() = default; + +MockDnsHostInfo::MockDnsHostInfo() { + ON_CALL(*this, address()).WillByDefault(ReturnPointee(&address_)); +} +MockDnsHostInfo::~MockDnsHostInfo() = default; + +MockUpdateCallbacks::MockUpdateCallbacks() = default; +MockUpdateCallbacks::~MockUpdateCallbacks() = default; + +MockLoadDnsCacheCallbacks::MockLoadDnsCacheCallbacks() = default; +MockLoadDnsCacheCallbacks::~MockLoadDnsCacheCallbacks() = default; + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.h b/test/extensions/common/dynamic_forward_proxy/mocks.h new file mode 100644 index 000000000000..5fd19e34f481 --- /dev/null +++ b/test/extensions/common/dynamic_forward_proxy/mocks.h @@ -0,0 +1,84 @@ +#pragma once + +#include "extensions/common/dynamic_forward_proxy/dns_cache.h" + +#include "gmock/gmock.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace DynamicForwardProxy { + +class MockDnsCache : public DnsCache { +public: + MockDnsCache(); + ~MockDnsCache(); + + LoadDnsCacheHandlePtr loadDnsCache(absl::string_view host, uint16_t default_port, + LoadDnsCacheCallbacks& callbacks) override { + return LoadDnsCacheHandlePtr{loadDnsCache_(host, default_port, callbacks)}; + } + MOCK_METHOD3(loadDnsCache_, + DnsCache::LoadDnsCacheHandle*(absl::string_view host, uint16_t default_port, + LoadDnsCacheCallbacks& callbacks)); + AddUpdateCallbacksHandlePtr addUpdateCallbacks(UpdateCallbacks& callbacks) override { + return AddUpdateCallbacksHandlePtr{addUpdateCallbacks_(callbacks)}; + } + MOCK_METHOD1(addUpdateCallbacks_, + DnsCache::AddUpdateCallbacksHandle*(UpdateCallbacks& callbacks)); +}; + +class MockLoadDnsCacheHandle : public DnsCache::LoadDnsCacheHandle { +public: + MockLoadDnsCacheHandle(); + ~MockLoadDnsCacheHandle(); + + MOCK_METHOD0(onDestroy, void()); +}; + +class MockDnsCacheManager : public DnsCacheManager { +public: + MockDnsCacheManager(); + ~MockDnsCacheManager(); + + MOCK_METHOD1( + getCache, + DnsCacheSharedPtr( + const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config)); + + std::shared_ptr dns_cache_{new MockDnsCache()}; +}; + +class MockDnsHostInfo : public DnsHostInfo { +public: + MockDnsHostInfo(); + ~MockDnsHostInfo(); + + MOCK_METHOD0(address, Network::Address::InstanceConstSharedPtr()); + MOCK_METHOD0(touch, void()); + + Network::Address::InstanceConstSharedPtr address_; +}; + +class MockUpdateCallbacks : public DnsCache::UpdateCallbacks { +public: + MockUpdateCallbacks(); + ~MockUpdateCallbacks(); + + MOCK_METHOD2(onDnsHostAddOrUpdate, + void(const std::string& host, const DnsHostInfoSharedPtr& address)); + MOCK_METHOD1(onDnsHostRemove, void(const std::string& host)); +}; + +class MockLoadDnsCacheCallbacks : public DnsCache::LoadDnsCacheCallbacks { +public: + MockLoadDnsCacheCallbacks(); + ~MockLoadDnsCacheCallbacks(); + + MOCK_METHOD0(onLoadDnsCacheComplete, void()); +}; + +} // namespace DynamicForwardProxy +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/dynamic_forward_proxy/BUILD b/test/extensions/filters/http/dynamic_forward_proxy/BUILD new file mode 100644 index 000000000000..f3b9f0a3edff --- /dev/null +++ b/test/extensions/filters/http/dynamic_forward_proxy/BUILD @@ -0,0 +1,35 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "proxy_filter_test", + srcs = ["proxy_filter_test.cc"], + extension_name = "envoy.filters.http.dynamic_forward_proxy", + deps = [ + "//source/extensions/filters/http/dynamic_forward_proxy:config", + "//test/extensions/common/dynamic_forward_proxy:mocks", + "//test/mocks/http:http_mocks", + "//test/mocks/upstream:upstream_mocks", + ], +) + +envoy_extension_cc_test( + name = "proxy_filter_integration_test", + srcs = ["proxy_filter_integration_test.cc"], + extension_name = "envoy.filters.http.dynamic_forward_proxy", + deps = [ + "//source/extensions/clusters/dynamic_forward_proxy:cluster", + "//source/extensions/filters/http/dynamic_forward_proxy:config", + "//test/integration:http_integration_lib", + ], +) diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc new file mode 100644 index 000000000000..5ab1a65505e9 --- /dev/null +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -0,0 +1,87 @@ +#include "test/integration/http_integration.h" + +namespace Envoy { +namespace { + +class ProxyFilterIntegrationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + ProxyFilterIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} + + static std::string ipVersionToDnsFamily(Network::Address::IpVersion version) { + switch (version) { + case Network::Address::IpVersion::v4: + return "V4_ONLY"; + case Network::Address::IpVersion::v6: + return "V6_ONLY"; + } + + // This seems to be needed on the coverage build for some reason. + NOT_REACHED_GCOVR_EXCL_LINE; + } + + void SetUp() override { + setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + + const std::string filter = fmt::format(R"EOF( +name: envoy.filters.http.dynamic_forward_proxy +config: + dns_cache_config: + name: foo + dns_lookup_family: {} +)EOF", + ipVersionToDnsFamily(GetParam())); + config_helper_.addFilter(filter); + + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters(0); + cluster_0->clear_hosts(); + cluster_0->set_lb_policy(envoy::api::v2::Cluster::CLUSTER_PROVIDED); + + const std::string cluster_type_config = fmt::format(R"EOF( +name: envoy.clusters.dynamic_forward_proxy +typed_config: + "@type": type.googleapis.com/envoy.config.cluster.dynamic_forward_proxy.v2alpha.ClusterConfig + dns_cache_config: + name: foo + dns_lookup_family: {} +)EOF", + ipVersionToDnsFamily(GetParam())); + + TestUtility::loadFromYaml(cluster_type_config, *cluster_0->mutable_cluster_type()); + }); + + HttpIntegrationTest::initialize(); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, ProxyFilterIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// A basic test where we pause a request to lookup localhost, and then do another request which +// should hit the TLS cache. +TEST_P(ProxyFilterIntegrationTest, RequestWithBody) { + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = + sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + + // Now send another request. This should hit the DNS cache. + // TODO(mattklein123): Verify this with stats once stats are added. + response = sendRequestAndWaitForResponse(request_headers, 512, default_response_headers_, 512); + checkSimpleRequestSuccess(512, 512, response.get()); +} + +// TODO(mattklein123): Add a test for host expiration. We can do this both with simulated time +// and by checking stats. + +} // namespace +} // namespace Envoy diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc new file mode 100644 index 000000000000..9fae4430601f --- /dev/null +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc @@ -0,0 +1,108 @@ +#include "extensions/filters/http/dynamic_forward_proxy/proxy_filter.h" + +#include "test/extensions/common/dynamic_forward_proxy/mocks.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/upstream/mocks.h" + +using testing::AtLeast; +using testing::Eq; +using testing::InSequence; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace DynamicForwardProxy { +namespace { + +class ProxyFilterTest : public testing::Test, + public Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory { +public: + ProxyFilterTest() { + cm_.thread_local_cluster_.cluster_.info_->transport_socket_factory_.reset( + transport_socket_factory_); + + envoy::config::filter::http::dynamic_forward_proxy::v2alpha::FilterConfig proto_config; + EXPECT_CALL(*dns_cache_manager_, getCache(_)); + filter_config_ = std::make_shared(proto_config, *this, cm_); + filter_ = std::make_unique(filter_config_); + filter_->setDecoderFilterCallbacks(callbacks_); + + // Allow for an otherwise strict mock. + EXPECT_CALL(callbacks_, connection()).Times(AtLeast(0)); + EXPECT_CALL(callbacks_, streamId()).Times(AtLeast(0)); + } + + Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr get() override { + return dns_cache_manager_; + } + + std::shared_ptr dns_cache_manager_{ + new Extensions::Common::DynamicForwardProxy::MockDnsCacheManager()}; + Network::MockTransportSocketFactory* transport_socket_factory_{ + new Network::MockTransportSocketFactory}; + Upstream::MockClusterManager cm_; + ProxyFilterConfigSharedPtr filter_config_; + std::unique_ptr filter_; + Http::MockStreamDecoderFilterCallbacks callbacks_; + Http::TestHeaderMapImpl request_headers_{{":authority", "foo"}}; +}; + +// Default port 80 if upstream TLS not configured. +TEST_F(ProxyFilterTest, HttpDefaultPort) { + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(cm_, get(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCache_(Eq("foo"), 80, _)) + .WillOnce(Return(handle)); + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_CALL(*handle, onDestroy()); + filter_->onDestroy(); +} + +// Default port 443 if upstream TLS is configured. +TEST_F(ProxyFilterTest, HttpsDefaultPort) { + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(cm_, get(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCache_(Eq("foo"), 443, _)) + .WillOnce(Return(handle)); + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_CALL(*handle, onDestroy()); + filter_->onDestroy(); +} + +// No route handling. +TEST_F(ProxyFilterTest, NoRoute) { + InSequence s; + + EXPECT_CALL(callbacks_, route()).WillOnce(Return(nullptr)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); +} + +// No cluster handling. +TEST_F(ProxyFilterTest, NoCluster) { + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(cm_, get(_)).WillOnce(Return(nullptr)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, false)); +} + +} // namespace +} // namespace DynamicForwardProxy +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index ee8cf04f2889..d34c8fa20fe1 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -52,7 +52,11 @@ MockClusterInfo::MockClusterInfo() .WillByDefault(ReturnPointee(&max_requests_per_connection_)); ON_CALL(*this, stats()).WillByDefault(ReturnRef(stats_)); ON_CALL(*this, statsScope()).WillByDefault(ReturnRef(stats_store_)); - ON_CALL(*this, transportSocketFactory()).WillByDefault(ReturnRef(*transport_socket_factory_)); + // TODO(mattklein123): The following is a hack because it's not possible to directly embed + // a mock transport socket factory due to circular dependencies. Fix this up in a follow up. + ON_CALL(*this, transportSocketFactory()) + .WillByDefault(Invoke( + [this]() -> Network::TransportSocketFactory& { return *transport_socket_factory_; })); ON_CALL(*this, loadReportStats()).WillByDefault(ReturnRef(load_report_stats_)); ON_CALL(*this, sourceAddress()).WillByDefault(ReturnRef(source_address_)); ON_CALL(*this, resourceManager(_)) From 3c42cf460a2710bbe4dc43e72afc456f72411f04 Mon Sep 17 00:00:00 2001 From: danzh Date: Thu, 20 Jun 2019 19:53:38 -0400 Subject: [PATCH 043/542] quiche: build quiche libraries (#7340) Signed-off-by: Dan Zhang --- bazel/envoy_library.bzl | 4 +- bazel/external/quiche.BUILD | 1263 +++++++++++++++-- bazel/external/quiche.genrule_cmd | 2 + bazel/repository_locations.bzl | 6 +- source/extensions/quic_listeners/quiche/BUILD | 4 +- .../quic_listeners/quiche/platform/BUILD | 13 +- .../quiche/platform/flags_list.h | 46 +- .../quiche/platform/quic_bbr2_sender_impl.h | 15 + .../quiche/platform/quic_logging_impl.h | 1 + .../platform/quic_reference_counted_impl.h | 2 - .../platform/quic_socket_address_impl.h | 46 - 11 files changed, 1163 insertions(+), 239 deletions(-) create mode 100644 source/extensions/quic_listeners/quiche/platform/quic_bbr2_sender_impl.h delete mode 100644 source/extensions/quic_listeners/quiche/platform/quic_socket_address_impl.h diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index 09d13c0a5c33..33e9e93fc049 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -38,7 +38,8 @@ def envoy_cc_library( linkstamp = None, tags = [], deps = [], - strip_include_prefix = None): + strip_include_prefix = None, + textual_hdrs = None): if tcmalloc_dep: deps += _tcmalloc_external_deps(repository) @@ -49,6 +50,7 @@ def envoy_cc_library( copts = envoy_copts(repository) + copts, visibility = visibility, tags = tags, + textual_hdrs = textual_hdrs, deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + [ repository + "//include/envoy/common:base_includes", repository + "//source/common/common:fmt_lib", diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 0fec16d9d74d..75c2c90572c0 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -31,6 +31,7 @@ load( "envoy_cc_library", "envoy_cc_test", "envoy_cc_test_library", + "envoy_proto_library", ) src_files = glob([ @@ -172,7 +173,7 @@ envoy_cc_library( ) envoy_cc_library( - name = "spdy_core_headers_handler_interface", + name = "spdy_core_headers_handler_interface_lib", hdrs = ["quiche/spdy/core/spdy_headers_handler_interface.h"], copts = quiche_copt, repository = "@envoy", @@ -180,6 +181,18 @@ envoy_cc_library( deps = [":spdy_platform"], ) +envoy_cc_library( + name = "spdy_core_priority_write_scheduler_lib", + srcs = ["quiche/spdy/core/priority_write_scheduler.h"], + repository = "@envoy", + deps = [ + ":http2_platform", + ":spdy_core_protocol_lib", + ":spdy_core_write_scheduler_lib", + ":spdy_platform", + ], +) + envoy_cc_library( name = "spdy_core_protocol_lib", hdrs = [ @@ -196,6 +209,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "spdy_core_write_scheduler_lib", + hdrs = ["quiche/spdy/core/write_scheduler.h"], + repository = "@envoy", + deps = [ + ":spdy_core_protocol_lib", + ":spdy_platform", + ], +) + envoy_cc_test_library( name = "spdy_core_test_utils_lib", srcs = ["quiche/spdy/core/spdy_test_utils.cc"], @@ -204,7 +227,7 @@ envoy_cc_test_library( repository = "@envoy", deps = [ ":spdy_core_header_block_lib", - ":spdy_core_headers_handler_interface", + ":spdy_core_headers_handler_interface_lib", ":spdy_core_protocol_lib", ":spdy_platform", ], @@ -235,6 +258,58 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quic_platform_base", + hdrs = [ + "quiche/quic/platform/api/quic_aligned.h", + "quiche/quic/platform/api/quic_arraysize.h", + "quiche/quic/platform/api/quic_bug_tracker.h", + "quiche/quic/platform/api/quic_client_stats.h", + "quiche/quic/platform/api/quic_containers.h", + "quiche/quic/platform/api/quic_endian.h", + "quiche/quic/platform/api/quic_error_code_wrappers.h", + "quiche/quic/platform/api/quic_estimate_memory_usage.h", + "quiche/quic/platform/api/quic_exported_stats.h", + "quiche/quic/platform/api/quic_fallthrough.h", + "quiche/quic/platform/api/quic_flag_utils.h", + "quiche/quic/platform/api/quic_flags.h", + "quiche/quic/platform/api/quic_iovec.h", + "quiche/quic/platform/api/quic_logging.h", + "quiche/quic/platform/api/quic_macros.h", + "quiche/quic/platform/api/quic_map_util.h", + "quiche/quic/platform/api/quic_mem_slice.h", + "quiche/quic/platform/api/quic_optional.h", + "quiche/quic/platform/api/quic_prefetch.h", + "quiche/quic/platform/api/quic_ptr_util.h", + "quiche/quic/platform/api/quic_reference_counted.h", + "quiche/quic/platform/api/quic_server_stats.h", + "quiche/quic/platform/api/quic_stack_trace.h", + "quiche/quic/platform/api/quic_str_cat.h", + "quiche/quic/platform/api/quic_stream_buffer_allocator.h", + "quiche/quic/platform/api/quic_string_piece.h", + "quiche/quic/platform/api/quic_string_utils.h", + "quiche/quic/platform/api/quic_uint128.h", + "quiche/quic/platform/api/quic_text_utils.h", + # TODO: uncomment the following files as implementations are added. + # "quiche/quic/platform/api/quic_fuzzed_data_provider.h", + # "quiche/quic/platform/api/quic_test_loopback.h", + ], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_platform_export", + ":quiche_common_lib", + "@envoy//source/extensions/quic_listeners/quiche/platform:quic_platform_base_impl_lib", + ], +) + +envoy_cc_library( + name = "quic_platform_bbr2_sender", + hdrs = ["quiche/quic/platform/api/quic_bbr2_sender.h"], + repository = "@envoy", + deps = ["@envoy//source/extensions/quic_listeners/quiche/platform:quic_platform_bbr2_sender_impl_lib"], +) + envoy_cc_test_library( name = "quic_platform_epoll_lib", hdrs = ["quiche/quic/platform/api/quic_epoll.h"], @@ -264,6 +339,20 @@ envoy_cc_library( visibility = ["//visibility:public"], ) +envoy_cc_library( + name = "quic_platform_ip_address", + srcs = ["quiche/quic/platform/api/quic_ip_address.cc"], + hdrs = ["quiche/quic/platform/api/quic_ip_address.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_platform_base", + ":quic_platform_export", + ":quic_platform_ip_address_family", + ], +) + envoy_cc_test_library( name = "quic_platform_mock_log", hdrs = ["quiche/quic/platform/api/quic_mock_log.h"], @@ -285,6 +374,19 @@ envoy_cc_test_library( deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_sleep_impl_lib"], ) +envoy_cc_library( + name = "quic_platform_socket_address", + srcs = ["quiche/quic/platform/api/quic_socket_address.cc"], + hdrs = ["quiche/quic/platform/api/quic_socket_address.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_platform_export", + ":quic_platform_ip_address", + ], +) + envoy_cc_test_library( name = "quic_platform_test", hdrs = ["quiche/quic/platform/api/quic_test.h"], @@ -313,108 +415,103 @@ envoy_cc_test_library( deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_thread_impl_lib"], ) +#TODO(danzh) Figure out why using envoy_proto_library() fails. +proto_library( + name = "quic_core_proto_cached_network_parameters_proto", + srcs = ["quiche/quic/core/proto/cached_network_parameters.proto"], +) + +cc_proto_library( + name = "quic_core_proto_cached_network_parameters_proto_cc", + deps = [":quic_core_proto_cached_network_parameters_proto"], +) + +proto_library( + name = "quic_core_proto_source_address_token_proto", + srcs = ["quiche/quic/core/proto/source_address_token.proto"], + deps = [":quic_core_proto_cached_network_parameters_proto"], +) + +cc_proto_library( + name = "quic_core_proto_source_address_token_proto_cc", + deps = [":quic_core_proto_source_address_token_proto"], +) + +proto_library( + name = "quic_core_proto_crypto_server_config_proto", + srcs = ["quiche/quic/core/proto/crypto_server_config.proto"], +) + +cc_proto_library( + name = "quic_core_proto_crypto_server_config_proto_cc", + deps = [":quic_core_proto_crypto_server_config_proto"], +) + envoy_cc_library( - name = "quic_platform_ip_address", - srcs = ["quiche/quic/platform/api/quic_ip_address.cc"], - hdrs = ["quiche/quic/platform/api/quic_ip_address.h"], + name = "quic_core_ack_listener_interface_lib", + srcs = ["quiche/quic/core/quic_ack_listener_interface.cc"], + hdrs = ["quiche/quic/core/quic_ack_listener_interface.h"], copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], deps = [ + ":quic_core_time_lib", + ":quic_core_types_lib", ":quic_platform_base", - ":quic_platform_export", - ":quic_platform_ip_address_family", ], ) envoy_cc_library( - name = "quic_platform_socket_address", - srcs = ["quiche/quic/platform/api/quic_socket_address.cc"], - hdrs = ["quiche/quic/platform/api/quic_socket_address.h"], - copts = quiche_copt, + name = "quic_core_alarm_interface_lib", + srcs = ["quiche/quic/core/quic_alarm.cc"], + hdrs = ["quiche/quic/core/quic_alarm.h"], repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":quic_platform_export", - ":quic_platform_ip_address", + ":quic_core_arena_scoped_ptr_lib", + ":quic_core_time_lib", ], ) envoy_cc_library( - name = "quic_platform_base", - hdrs = [ - "quiche/quic/platform/api/quic_aligned.h", - "quiche/quic/platform/api/quic_arraysize.h", - "quiche/quic/platform/api/quic_bug_tracker.h", - "quiche/quic/platform/api/quic_client_stats.h", - "quiche/quic/platform/api/quic_containers.h", - "quiche/quic/platform/api/quic_endian.h", - "quiche/quic/platform/api/quic_error_code_wrappers.h", - "quiche/quic/platform/api/quic_estimate_memory_usage.h", - "quiche/quic/platform/api/quic_exported_stats.h", - "quiche/quic/platform/api/quic_fallthrough.h", - "quiche/quic/platform/api/quic_flag_utils.h", - "quiche/quic/platform/api/quic_flags.h", - "quiche/quic/platform/api/quic_iovec.h", - "quiche/quic/platform/api/quic_logging.h", - "quiche/quic/platform/api/quic_macros.h", - "quiche/quic/platform/api/quic_map_util.h", - "quiche/quic/platform/api/quic_mem_slice.h", - "quiche/quic/platform/api/quic_optional.h", - "quiche/quic/platform/api/quic_prefetch.h", - "quiche/quic/platform/api/quic_ptr_util.h", - "quiche/quic/platform/api/quic_reference_counted.h", - "quiche/quic/platform/api/quic_server_stats.h", - "quiche/quic/platform/api/quic_stack_trace.h", - "quiche/quic/platform/api/quic_str_cat.h", - "quiche/quic/platform/api/quic_stream_buffer_allocator.h", - "quiche/quic/platform/api/quic_string_piece.h", - "quiche/quic/platform/api/quic_string_utils.h", - "quiche/quic/platform/api/quic_uint128.h", - "quiche/quic/platform/api/quic_text_utils.h", - # TODO: uncomment the following files as implementations are added. - # "quiche/quic/platform/api/quic_fuzzed_data_provider.h", - # "quiche/quic/platform/api/quic_test_loopback.h", - ], + name = "quic_core_alarm_factory_interface_lib", + hdrs = ["quiche/quic/core/quic_alarm_factory.h"], repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":quic_platform_export", - ":quiche_common_lib", - "@envoy//source/extensions/quic_listeners/quiche/platform:quic_platform_base_impl_lib", - "@envoy//source/extensions/quic_listeners/quiche/platform:quic_platform_logging_impl_lib", + ":quic_core_alarm_interface_lib", + ":quic_core_one_block_arena_lib", ], ) envoy_cc_library( - name = "quic_core_arena_scoped_ptr_lib", - hdrs = ["quiche/quic/core/quic_arena_scoped_ptr.h"], + name = "quic_core_bandwidth_lib", + srcs = ["quiche/quic/core/quic_bandwidth.cc"], + hdrs = ["quiche/quic/core/quic_bandwidth.h"], + copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], - deps = [":quic_platform_base"], + deps = [ + ":quic_core_constants_lib", + ":quic_core_time_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], ) envoy_cc_library( - name = "quic_core_alarm_interface", - srcs = ["quiche/quic/core/quic_alarm.cc"], - hdrs = ["quiche/quic/core/quic_alarm.h"], + name = "quic_core_blocked_writer_interface_lib", + hdrs = ["quiche/quic/core/quic_blocked_writer_interface.h"], repository = "@envoy", - visibility = ["//visibility:public"], - deps = [ - ":quic_core_arena_scoped_ptr_lib", - ":quic_core_time_lib", - ], + deps = [":quic_platform_export"], ) envoy_cc_library( - name = "quic_core_alarm_factory_interface", - hdrs = ["quiche/quic/core/quic_alarm_factory.h"], + name = "quic_core_arena_scoped_ptr_lib", + hdrs = ["quiche/quic/core/quic_arena_scoped_ptr.h"], repository = "@envoy", visibility = ["//visibility:public"], - deps = [ - ":quic_core_alarm_interface", - ":quic_core_one_block_arena_lib", - ], + deps = [":quic_platform_base"], ) envoy_cc_library( @@ -433,122 +530,530 @@ envoy_cc_library( ) envoy_cc_library( - name = "quic_core_error_codes_lib", - srcs = ["quiche/quic/core/quic_error_codes.cc"], - hdrs = ["quiche/quic/core/quic_error_codes.h"], + name = "quic_core_config_lib", + srcs = ["quiche/quic/core/quic_config.cc"], + hdrs = ["quiche/quic/core/quic_config.h"], copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], - deps = [":quic_platform_export"], + deps = [ + ":quic_core_constants_lib", + ":quic_core_crypto_crypto_handshake_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_packets_lib", + ":quic_core_socket_address_coder_lib", + ":quic_core_time_lib", + ":quic_core_utils_lib", + ":quic_platform_base", + ], ) envoy_cc_library( - name = "quic_core_one_block_arena_lib", - srcs = ["quiche/quic/core/quic_one_block_arena.h"], + name = "quic_core_congestion_control_bandwidth_sampler_lib", + srcs = ["quiche/quic/core/congestion_control/bandwidth_sampler.cc"], + hdrs = ["quiche/quic/core/congestion_control/bandwidth_sampler.h"], + copts = quiche_copt, repository = "@envoy", - visibility = ["//visibility:public"], deps = [ - ":quic_core_arena_scoped_ptr_lib", + ":quic_core_bandwidth_lib", + ":quic_core_packet_number_indexed_queue_lib", + ":quic_core_packets_lib", + ":quic_core_time_lib", ":quic_core_types_lib", ":quic_platform_base", ], ) envoy_cc_library( - name = "quic_core_time_lib", - srcs = ["quiche/quic/core/quic_time.cc"], - hdrs = ["quiche/quic/core/quic_time.h"], - repository = "@envoy", - visibility = ["//visibility:public"], - deps = [":quic_platform_base"], -) - -envoy_cc_library( - name = "quic_core_types_lib", - srcs = [ - "quiche/quic/core/quic_connection_id.cc", - "quiche/quic/core/quic_packet_number.cc", - "quiche/quic/core/quic_types.cc", - ], - hdrs = [ - "quiche/quic/core/quic_connection_id.h", - "quiche/quic/core/quic_packet_number.h", - "quiche/quic/core/quic_types.h", - ], + name = "quic_core_congestion_control_bbr_lib", + srcs = ["quiche/quic/core/congestion_control/bbr_sender.cc"], + hdrs = ["quiche/quic/core/congestion_control/bbr_sender.h"], copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":quic_core_error_codes_lib", + ":quic_core_bandwidth_lib", + ":quic_core_congestion_control_bandwidth_sampler_lib", + ":quic_core_congestion_control_congestion_control_interface_lib", + ":quic_core_congestion_control_rtt_stats_lib", + ":quic_core_congestion_control_windowed_filter_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_crypto_random_lib", + ":quic_core_packets_lib", ":quic_core_time_lib", + ":quic_core_unacked_packet_map_lib", ":quic_platform_base", ], ) envoy_cc_library( - name = "quic_core_crypto_random_lib", - srcs = ["quiche/quic/core/crypto/quic_random.cc"], - hdrs = ["quiche/quic/core/crypto/quic_random.h"], + name = "quic_core_congestion_control_general_loss_algorithm_lib", + srcs = ["quiche/quic/core/congestion_control/general_loss_algorithm.cc"], + hdrs = ["quiche/quic/core/congestion_control/general_loss_algorithm.h"], copts = quiche_copt, - external_deps = ["ssl"], repository = "@envoy", - visibility = ["//visibility:public"], - deps = [":quic_platform_base"], + deps = [ + ":quic_core_congestion_control_congestion_control_interface_lib", + ":quic_core_congestion_control_rtt_stats_lib", + ":quic_core_packets_lib", + ":quic_core_time_lib", + ":quic_core_unacked_packet_map_lib", + ":quic_platform_base", + ], ) envoy_cc_library( - name = "quic_core_constants_lib", - srcs = ["quiche/quic/core/quic_constants.cc"], - hdrs = ["quiche/quic/core/quic_constants.h"], + name = "quic_core_congestion_control_congestion_control_interface_lib", + hdrs = [ + "quiche/quic/core/congestion_control/loss_detection_interface.h", + "quiche/quic/core/congestion_control/send_algorithm_interface.h", + ], copts = quiche_copt, repository = "@envoy", - visibility = ["//visibility:public"], deps = [ + ":quic_core_bandwidth_lib", + ":quic_core_config_lib", + ":quic_core_connection_stats_lib", + ":quic_core_crypto_random_lib", + ":quic_core_packets_lib", + ":quic_core_time_lib", ":quic_core_types_lib", - ":quic_platform_export", + ":quic_core_unacked_packet_map_lib", + ":quic_platform", ], ) envoy_cc_library( - name = "quic_core_tag_lib", - srcs = ["quiche/quic/core/quic_tag.cc"], - hdrs = ["quiche/quic/core/quic_tag.h"], - copts = quiche_copt, - repository = "@envoy", - visibility = ["//visibility:public"], - deps = [":quic_platform_base"], + name = "quic_core_congestion_control_congestion_control_lib", + srcs = [ + "quiche/quic/core/congestion_control/send_algorithm_interface.cc", + ], + hdrs = [ + "quiche/quic/core/congestion_control/loss_detection_interface.h", + "quiche/quic/core/congestion_control/send_algorithm_interface.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_bandwidth_lib", + ":quic_core_config_lib", + ":quic_core_congestion_control_tcp_cubic_bytes_lib", + ":quic_core_connection_stats_lib", + ":quic_core_crypto_random_lib", + ":quic_core_packets_lib", + ":quic_core_time_lib", + ":quic_core_types_lib", + ":quic_core_unacked_packet_map_lib", + ":quic_platform", + ":quic_platform_bbr2_sender", + ], ) envoy_cc_library( - name = "quic_core_versions_lib", - srcs = ["quiche/quic/core/quic_versions.cc"], - hdrs = ["quiche/quic/core/quic_versions.h"], + name = "quic_core_congestion_control_pacing_sender_lib", + srcs = ["quiche/quic/core/congestion_control/pacing_sender.cc"], + hdrs = ["quiche/quic/core/congestion_control/pacing_sender.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_bandwidth_lib", + ":quic_core_config_lib", + ":quic_core_congestion_control_congestion_control_interface_lib", + ":quic_core_packets_lib", + ":quic_core_time_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_congestion_control_rtt_stats_lib", + srcs = ["quiche/quic/core/congestion_control/rtt_stats.cc"], + hdrs = ["quiche/quic/core/congestion_control/rtt_stats.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_packets_lib", + ":quic_core_time_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_congestion_control_tcp_cubic_helper", + srcs = [ + "quiche/quic/core/congestion_control/hybrid_slow_start.cc", + "quiche/quic/core/congestion_control/prr_sender.cc", + ], + hdrs = [ + "quiche/quic/core/congestion_control/hybrid_slow_start.h", + "quiche/quic/core/congestion_control/prr_sender.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_bandwidth_lib", + ":quic_core_packets_lib", + ":quic_core_time_lib", + ":quic_platform_base", + ":quic_platform_export", + ], +) + +envoy_cc_library( + name = "quic_core_congestion_control_tcp_cubic_bytes_lib", + srcs = [ + "quiche/quic/core/congestion_control/cubic_bytes.cc", + "quiche/quic/core/congestion_control/tcp_cubic_sender_bytes.cc", + ], + hdrs = [ + "quiche/quic/core/congestion_control/cubic_bytes.h", + "quiche/quic/core/congestion_control/tcp_cubic_sender_bytes.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_bandwidth_lib", + ":quic_core_congestion_control_congestion_control_interface_lib", + ":quic_core_congestion_control_rtt_stats_lib", + ":quic_core_congestion_control_tcp_cubic_helper", + ":quic_core_connection_stats_lib", + ":quic_core_constants_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_packets_lib", + ":quic_core_time_lib", + ":quic_platform", + ], +) + +envoy_cc_library( + name = "quic_core_congestion_control_uber_loss_algorithm_lib", + srcs = ["quiche/quic/core/congestion_control/uber_loss_algorithm.cc"], + hdrs = ["quiche/quic/core/congestion_control/uber_loss_algorithm.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_core_congestion_control_general_loss_algorithm_lib"], +) + +envoy_cc_library( + name = "quic_core_congestion_control_windowed_filter_lib", + hdrs = ["quiche/quic/core/congestion_control/windowed_filter.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_core_time_lib"], +) + +envoy_cc_library( + name = "quic_core_connection_lib", + srcs = ["quiche/quic/core/quic_connection.cc"], + hdrs = ["quiche/quic/core/quic_connection.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_core_alarm_factory_interface_lib", + ":quic_core_alarm_interface_lib", + ":quic_core_bandwidth_lib", + ":quic_core_blocked_writer_interface_lib", + ":quic_core_config_lib", + ":quic_core_connection_stats_lib", + ":quic_core_crypto_crypto_handshake_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_framer_lib", + ":quic_core_one_block_arena_lib", + ":quic_core_packet_creator_lib", + ":quic_core_packet_generator_lib", + ":quic_core_packet_writer_interface_lib", + ":quic_core_packets_lib", + ":quic_core_pending_retransmission_lib", + ":quic_core_proto_cached_network_parameters_proto_cc", + ":quic_core_sent_packet_manager_lib", + ":quic_core_time_lib", + ":quic_core_types_lib", + ":quic_core_uber_received_packet_manager_lib", + ":quic_core_utils_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_connection_stats_lib", + srcs = ["quiche/quic/core/quic_connection_stats.cc"], + hdrs = ["quiche/quic/core/quic_connection_stats.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_bandwidth_lib", + ":quic_core_packets_lib", + ":quic_core_time_lib", + ":quic_platform_export", + ], +) + +envoy_cc_library( + name = "quic_core_constants_lib", + srcs = ["quiche/quic/core/quic_constants.cc"], + hdrs = ["quiche/quic/core/quic_constants.h"], copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], deps = [ + ":quic_core_types_lib", + ":quic_platform_export", + ], +) + +envoy_cc_library( + name = "quic_core_crypto_crypto_handshake_lib", + srcs = [ + "quiche/quic/core/crypto/cert_compressor.cc", + "quiche/quic/core/crypto/channel_id.cc", + "quiche/quic/core/crypto/common_cert_set.cc", + "quiche/quic/core/crypto/crypto_framer.cc", + "quiche/quic/core/crypto/crypto_handshake.cc", + "quiche/quic/core/crypto/crypto_handshake_message.cc", + "quiche/quic/core/crypto/crypto_secret_boxer.cc", + "quiche/quic/core/crypto/crypto_utils.cc", + "quiche/quic/core/crypto/curve25519_key_exchange.cc", + "quiche/quic/core/crypto/key_exchange.cc", + "quiche/quic/core/crypto/p256_key_exchange.cc", + "quiche/quic/core/crypto/quic_compressed_certs_cache.cc", + "quiche/quic/core/crypto/quic_crypto_client_config.cc", + "quiche/quic/core/crypto/quic_crypto_server_config.cc", + "quiche/quic/core/crypto/transport_parameters.cc", + ], + hdrs = [ + "quiche/quic/core/crypto/cert_compressor.h", + "quiche/quic/core/crypto/channel_id.h", + "quiche/quic/core/crypto/common_cert_set.h", + "quiche/quic/core/crypto/crypto_framer.h", + "quiche/quic/core/crypto/crypto_handshake.h", + "quiche/quic/core/crypto/crypto_handshake_message.h", + "quiche/quic/core/crypto/crypto_message_parser.h", + "quiche/quic/core/crypto/crypto_secret_boxer.h", + "quiche/quic/core/crypto/crypto_utils.h", + "quiche/quic/core/crypto/curve25519_key_exchange.h", + "quiche/quic/core/crypto/key_exchange.h", + "quiche/quic/core/crypto/p256_key_exchange.h", + "quiche/quic/core/crypto/proof_verifier.h", + "quiche/quic/core/crypto/quic_compressed_certs_cache.h", + "quiche/quic/core/crypto/quic_crypto_client_config.h", + "quiche/quic/core/crypto/quic_crypto_server_config.h", + "quiche/quic/core/crypto/transport_parameters.h", + ], + copts = quiche_copt, + external_deps = [ + "ssl", + "zlib", + ], + repository = "@envoy", + tags = ["pg3"], + textual_hdrs = [ + "quiche/quic/core/crypto/common_cert_set_2.c", + "quiche/quic/core/crypto/common_cert_set_2a.inc", + "quiche/quic/core/crypto/common_cert_set_2b.inc", + "quiche/quic/core/crypto/common_cert_set_3.c", + "quiche/quic/core/crypto/common_cert_set_3a.inc", + "quiche/quic/core/crypto/common_cert_set_3b.inc", + ], + deps = [ + ":quic_core_crypto_encryption_lib", + ":quic_core_crypto_hkdf_lib", + ":quic_core_crypto_proof_source_interface_lib", + ":quic_core_crypto_random_lib", + ":quic_core_crypto_tls_handshake_lib", + ":quic_core_data_lib", + ":quic_core_error_codes_lib", + ":quic_core_lru_cache_lib", + ":quic_core_packets_lib", + ":quic_core_proto_cached_network_parameters_proto_cc", + ":quic_core_proto_crypto_server_config_proto_cc", + ":quic_core_proto_source_address_token_proto_cc", + ":quic_core_server_id_lib", + ":quic_core_socket_address_coder_lib", + ":quic_core_time_lib", + ":quic_core_types_lib", + ":quic_core_utils_lib", + ":quic_core_versions_lib", + ":quic_platform", + ], +) + +envoy_cc_library( + name = "quic_core_crypto_encryption_lib", + srcs = [ + "quiche/quic/core/crypto/aead_base_decrypter.cc", + "quiche/quic/core/crypto/aead_base_encrypter.cc", + "quiche/quic/core/crypto/aes_128_gcm_12_decrypter.cc", + "quiche/quic/core/crypto/aes_128_gcm_12_encrypter.cc", + "quiche/quic/core/crypto/aes_128_gcm_decrypter.cc", + "quiche/quic/core/crypto/aes_128_gcm_encrypter.cc", + "quiche/quic/core/crypto/aes_256_gcm_decrypter.cc", + "quiche/quic/core/crypto/aes_256_gcm_encrypter.cc", + "quiche/quic/core/crypto/aes_base_decrypter.cc", + "quiche/quic/core/crypto/aes_base_encrypter.cc", + "quiche/quic/core/crypto/chacha20_poly1305_decrypter.cc", + "quiche/quic/core/crypto/chacha20_poly1305_encrypter.cc", + "quiche/quic/core/crypto/chacha20_poly1305_tls_decrypter.cc", + "quiche/quic/core/crypto/chacha20_poly1305_tls_encrypter.cc", + "quiche/quic/core/crypto/chacha_base_decrypter.cc", + "quiche/quic/core/crypto/chacha_base_encrypter.cc", + "quiche/quic/core/crypto/null_decrypter.cc", + "quiche/quic/core/crypto/null_encrypter.cc", + "quiche/quic/core/crypto/quic_decrypter.cc", + "quiche/quic/core/crypto/quic_encrypter.cc", + ], + hdrs = [ + "quiche/quic/core/crypto/aead_base_decrypter.h", + "quiche/quic/core/crypto/aead_base_encrypter.h", + "quiche/quic/core/crypto/aes_128_gcm_12_decrypter.h", + "quiche/quic/core/crypto/aes_128_gcm_12_encrypter.h", + "quiche/quic/core/crypto/aes_128_gcm_decrypter.h", + "quiche/quic/core/crypto/aes_128_gcm_encrypter.h", + "quiche/quic/core/crypto/aes_256_gcm_decrypter.h", + "quiche/quic/core/crypto/aes_256_gcm_encrypter.h", + "quiche/quic/core/crypto/aes_base_decrypter.h", + "quiche/quic/core/crypto/aes_base_encrypter.h", + "quiche/quic/core/crypto/chacha20_poly1305_decrypter.h", + "quiche/quic/core/crypto/chacha20_poly1305_encrypter.h", + "quiche/quic/core/crypto/chacha20_poly1305_tls_decrypter.h", + "quiche/quic/core/crypto/chacha20_poly1305_tls_encrypter.h", + "quiche/quic/core/crypto/chacha_base_decrypter.h", + "quiche/quic/core/crypto/chacha_base_encrypter.h", + "quiche/quic/core/crypto/crypto_protocol.h", + "quiche/quic/core/crypto/null_decrypter.h", + "quiche/quic/core/crypto/null_encrypter.h", + "quiche/quic/core/crypto/quic_crypter.h", + "quiche/quic/core/crypto/quic_decrypter.h", + "quiche/quic/core/crypto/quic_encrypter.h", + ], + copts = quiche_copt, + external_deps = ["ssl"], + repository = "@envoy", + deps = [ + ":quic_core_crypto_hkdf_lib", + ":quic_core_data_lib", + ":quic_core_packets_lib", ":quic_core_tag_lib", ":quic_core_types_lib", + ":quic_core_utils_lib", ":quic_platform_base", ], ) envoy_cc_library( - name = "quic_core_interval_lib", - hdrs = ["quiche/quic/core/quic_interval.h"], + name = "quic_core_crypto_hkdf_lib", + srcs = ["quiche/quic/core/crypto/quic_hkdf.cc"], + hdrs = ["quiche/quic/core/crypto/quic_hkdf.h"], + external_deps = ["ssl"], + repository = "@envoy", + deps = [ + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_crypto_proof_source_interface_lib", + srcs = [ + "quiche/quic/core/crypto/proof_source.cc", + "quiche/quic/core/crypto/quic_crypto_proof.cc", + ], + hdrs = [ + "quiche/quic/core/crypto/proof_source.h", + "quiche/quic/core/crypto/quic_crypto_proof.h", + ], copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], + deps = [ + ":quic_core_packets_lib", + ":quic_core_versions_lib", + ":quic_platform_base", + ":quic_platform_export", + ], ) envoy_cc_library( - name = "quic_core_interval_set_lib", - hdrs = ["quiche/quic/core/quic_interval_set.h"], + name = "quic_core_crypto_random_lib", + srcs = ["quiche/quic/core/crypto/quic_random.cc"], + hdrs = ["quiche/quic/core/crypto/quic_random.h"], + copts = quiche_copt, + external_deps = ["ssl"], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [":quic_platform_base"], +) + +envoy_cc_library( + name = "quic_core_crypto_tls_handshake_lib", + srcs = [ + "quiche/quic/core/crypto/tls_client_connection.cc", + "quiche/quic/core/crypto/tls_connection.cc", + "quiche/quic/core/crypto/tls_server_connection.cc", + ], + hdrs = [ + "quiche/quic/core/crypto/tls_client_connection.h", + "quiche/quic/core/crypto/tls_connection.h", + "quiche/quic/core/crypto/tls_server_connection.h", + ], + copts = quiche_copt, + external_deps = ["ssl"], + repository = "@envoy", + deps = [ + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_data_lib", + srcs = [ + "quiche/quic/core/quic_data_reader.cc", + "quiche/quic/core/quic_data_writer.cc", + ], + hdrs = [ + "quiche/quic/core/quic_data_reader.h", + "quiche/quic/core/quic_data_writer.h", + ], copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":quic_core_interval_lib", + ":quic_core_constants_lib", + ":quic_core_crypto_random_lib", + ":quic_core_packets_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_error_codes_lib", + srcs = ["quiche/quic/core/quic_error_codes.cc"], + hdrs = ["quiche/quic/core/quic_error_codes.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [":quic_platform_export"], +) + +envoy_cc_library( + name = "quic_core_framer_lib", + srcs = ["quiche/quic/core/quic_framer.cc"], + hdrs = ["quiche/quic/core/quic_framer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_constants_lib", + ":quic_core_crypto_crypto_handshake_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_crypto_random_lib", + ":quic_core_data_lib", + ":quic_core_packets_lib", + ":quic_core_socket_address_coder_lib", + ":quic_core_stream_frame_data_producer_lib", + ":quic_core_types_lib", + ":quic_core_utils_lib", + ":quic_core_versions_lib", ":quic_platform_base", ], ) @@ -618,74 +1123,344 @@ envoy_cc_library( ) envoy_cc_library( - name = "quic_core_ack_listener_interface", - srcs = ["quiche/quic/core/quic_ack_listener_interface.cc"], - hdrs = ["quiche/quic/core/quic_ack_listener_interface.h"], + name = "quic_core_interval_lib", + hdrs = ["quiche/quic/core/quic_interval.h"], copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], - deps = [ - ":quic_core_time_lib", - ":quic_core_types_lib", - ":quic_platform_base", - ], ) envoy_cc_library( - name = "quic_core_bandwidth_lib", - srcs = ["quiche/quic/core/quic_bandwidth.cc"], - hdrs = ["quiche/quic/core/quic_bandwidth.h"], + name = "quic_core_interval_set_lib", + hdrs = ["quiche/quic/core/quic_interval_set.h"], copts = quiche_copt, repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":quic_core_constants_lib", - ":quic_core_time_lib", - ":quic_core_types_lib", + ":quic_core_interval_lib", ":quic_platform_base", ], ) envoy_cc_library( - name = "quic_core_data_lib", - srcs = [ - # "quiche/quic/core/quic_data_reader.cc", - "quiche/quic/core/quic_data_writer.cc", - ], - hdrs = [ - # "quiche/quic/core/quic_data_reader.h", - "quiche/quic/core/quic_data_writer.h", - ], - copts = quiche_copt, + name = "quic_core_lru_cache_lib", + hdrs = ["quiche/quic/core/quic_lru_cache.h"], + repository = "@envoy", + deps = [":quic_platform_base"], +) + +envoy_cc_library( + name = "quic_core_one_block_arena_lib", + srcs = ["quiche/quic/core/quic_one_block_arena.h"], repository = "@envoy", visibility = ["//visibility:public"], deps = [ - ":quic_core_constants_lib", - ":quic_core_crypto_random_lib", + ":quic_core_arena_scoped_ptr_lib", ":quic_core_types_lib", ":quic_platform_base", ], ) envoy_cc_library( - name = "quic_core_utils_lib", - srcs = ["quiche/quic/core/quic_utils.cc"], - hdrs = ["quiche/quic/core/quic_utils.h"], + name = "quic_core_packet_creator_lib", + srcs = ["quiche/quic/core/quic_packet_creator.cc"], + hdrs = ["quiche/quic/core/quic_packet_creator.h"], copts = quiche_copt, repository = "@envoy", - visibility = ["//visibility:public"], deps = [ ":quic_core_constants_lib", - ":quic_core_crypto_random_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_data_lib", + ":quic_core_framer_lib", + ":quic_core_packets_lib", + ":quic_core_pending_retransmission_lib", + ":quic_core_types_lib", + ":quic_core_utils_lib", + ":quic_core_versions_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_packet_generator_lib", + srcs = ["quiche/quic/core/quic_packet_generator.cc"], + hdrs = ["quiche/quic/core/quic_packet_generator.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_crypto_random_lib", + ":quic_core_packet_creator_lib", + ":quic_core_pending_retransmission_lib", + ":quic_core_sent_packet_manager_lib", + ":quic_core_types_lib", + ":quic_core_utils_lib", + ":quic_platform_base", + ":quic_platform_mem_slice_span", + ], +) + +envoy_cc_library( + name = "quic_core_packet_number_indexed_queue_lib", + hdrs = ["quiche/quic/core/packet_number_indexed_queue.h"], + repository = "@envoy", + deps = [ + ":quic_core_constants_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_packet_writer_interface_lib", + srcs = ["quiche/quic/core/quic_packet_writer_wrapper.cc"], + hdrs = [ + "quiche/quic/core/quic_packet_writer.h", + "quiche/quic/core/quic_packet_writer_wrapper.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_packets_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_packets_lib", + srcs = [ + "quiche/quic/core/quic_packets.cc", + "quiche/quic/core/quic_write_blocked_list.cc", + ], + hdrs = [ + "quiche/quic/core/quic_packets.h", + "quiche/quic/core/quic_write_blocked_list.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_ack_listener_interface_lib", + ":quic_core_bandwidth_lib", + ":quic_core_constants_lib", ":quic_core_error_codes_lib", ":quic_core_frames_frames_lib", + ":quic_core_time_lib", ":quic_core_types_lib", + ":quic_core_utils_lib", ":quic_core_versions_lib", ":quic_platform_base", ":quic_platform_socket_address", + ":spdy_core_priority_write_scheduler_lib", ], ) +envoy_cc_library( + name = "quic_core_pending_retransmission_lib", + hdrs = ["quiche/quic/core/quic_pending_retransmission.h"], + repository = "@envoy", + deps = [ + ":quic_core_frames_frames_lib", + ":quic_core_transmission_info_lib", + ":quic_core_types_lib", + ":quic_platform_export", + ], +) + +envoy_cc_library( + name = "quic_core_process_packet_interface_lib", + hdrs = ["quiche/quic/core/quic_process_packet_interface.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_packets_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_received_packet_manager_lib", + srcs = ["quiche/quic/core/quic_received_packet_manager.cc"], + hdrs = ["quiche/quic/core/quic_received_packet_manager.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_config_lib", + ":quic_core_congestion_control_rtt_stats_lib", + ":quic_core_connection_stats_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_framer_lib", + ":quic_core_packets_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_sent_packet_manager_lib", + srcs = ["quiche/quic/core/quic_sent_packet_manager.cc"], + hdrs = ["quiche/quic/core/quic_sent_packet_manager.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_congestion_control_congestion_control_lib", + ":quic_core_congestion_control_general_loss_algorithm_lib", + ":quic_core_congestion_control_pacing_sender_lib", + ":quic_core_congestion_control_rtt_stats_lib", + ":quic_core_congestion_control_uber_loss_algorithm_lib", + ":quic_core_connection_stats_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_packets_lib", + ":quic_core_pending_retransmission_lib", + ":quic_core_proto_cached_network_parameters_proto_cc", + ":quic_core_sustained_bandwidth_recorder_lib", + ":quic_core_transmission_info_lib", + ":quic_core_types_lib", + ":quic_core_unacked_packet_map_lib", + ":quic_core_utils_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_server_id_lib", + srcs = ["quiche/quic/core/quic_server_id.cc"], + hdrs = ["quiche/quic/core/quic_server_id.h"], + repository = "@envoy", + deps = [ + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_server_lib", + srcs = [ + "quiche/quic/core/chlo_extractor.cc", + "quiche/quic/core/quic_buffered_packet_store.cc", + "quiche/quic/core/quic_dispatcher.cc", + ], + hdrs = [ + "quiche/quic/core/chlo_extractor.h", + "quiche/quic/core/quic_buffered_packet_store.h", + "quiche/quic/core/quic_dispatcher.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_alarm_factory_interface_lib", + ":quic_core_alarm_interface_lib", + ":quic_core_blocked_writer_interface_lib", + ":quic_core_connection_lib", + ":quic_core_crypto_crypto_handshake_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_crypto_random_lib", + ":quic_core_framer_lib", + ":quic_core_packets_lib", + ":quic_core_process_packet_interface_lib", + ":quic_core_session_lib", + ":quic_core_time_lib", + ":quic_core_time_wait_list_manager_lib", + ":quic_core_types_lib", + ":quic_core_utils_lib", + ":quic_core_version_manager_lib", + ":quic_platform", + ], +) + +envoy_cc_library( + name = "quic_core_session_lib", + srcs = [ + "quiche/quic/core/legacy_quic_stream_id_manager.cc", + "quiche/quic/core/quic_control_frame_manager.cc", + "quiche/quic/core/quic_crypto_client_handshaker.cc", + "quiche/quic/core/quic_crypto_client_stream.cc", + "quiche/quic/core/quic_crypto_handshaker.cc", + "quiche/quic/core/quic_crypto_server_handshaker.cc", + "quiche/quic/core/quic_crypto_server_stream.cc", + "quiche/quic/core/quic_crypto_stream.cc", + "quiche/quic/core/quic_flow_controller.cc", + "quiche/quic/core/quic_session.cc", + "quiche/quic/core/quic_stream.cc", + "quiche/quic/core/quic_stream_id_manager.cc", + "quiche/quic/core/quic_stream_sequencer.cc", + "quiche/quic/core/tls_client_handshaker.cc", + "quiche/quic/core/tls_handshaker.cc", + "quiche/quic/core/tls_server_handshaker.cc", + "quiche/quic/core/uber_quic_stream_id_manager.cc", + ], + hdrs = [ + "quiche/quic/core/legacy_quic_stream_id_manager.h", + "quiche/quic/core/quic_control_frame_manager.h", + "quiche/quic/core/quic_crypto_client_handshaker.h", + "quiche/quic/core/quic_crypto_client_stream.h", + "quiche/quic/core/quic_crypto_handshaker.h", + "quiche/quic/core/quic_crypto_server_handshaker.h", + "quiche/quic/core/quic_crypto_server_stream.h", + "quiche/quic/core/quic_crypto_stream.h", + "quiche/quic/core/quic_flow_controller.h", + "quiche/quic/core/quic_session.h", + "quiche/quic/core/quic_stream.h", + "quiche/quic/core/quic_stream_id_manager.h", + "quiche/quic/core/quic_stream_sequencer.h", + "quiche/quic/core/tls_client_handshaker.h", + "quiche/quic/core/tls_handshaker.h", + "quiche/quic/core/tls_server_handshaker.h", + "quiche/quic/core/uber_quic_stream_id_manager.h", + ], + copts = quiche_copt, + external_deps = ["ssl"], + repository = "@envoy", + deps = [ + ":quic_core_config_lib", + ":quic_core_connection_lib", + ":quic_core_constants_lib", + ":quic_core_crypto_crypto_handshake_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_crypto_random_lib", + ":quic_core_crypto_tls_handshake_lib", + ":quic_core_frames_frames_lib", + ":quic_core_packet_creator_lib", + ":quic_core_packets_lib", + ":quic_core_server_id_lib", + ":quic_core_session_notifier_interface_lib", + ":quic_core_stream_frame_data_producer_lib", + ":quic_core_stream_send_buffer_lib", + ":quic_core_stream_sequencer_buffer_lib", + ":quic_core_types_lib", + ":quic_core_utils_lib", + ":quic_core_versions_lib", + ":quic_platform", + ":quic_platform_mem_slice_span", + ":spdy_core_protocol_lib", + ], +) + +envoy_cc_library( + name = "quic_core_session_notifier_interface_lib", + hdrs = ["quiche/quic/core/session_notifier_interface.h"], + repository = "@envoy", + deps = [ + ":quic_core_frames_frames_lib", + ":quic_core_time_lib", + ], +) + +envoy_cc_library( + name = "quic_core_socket_address_coder_lib", + srcs = ["quiche/quic/core/quic_socket_address_coder.cc"], + hdrs = ["quiche/quic/core/quic_socket_address_coder.h"], + repository = "@envoy", + deps = [ + ":quic_platform_base", + ":quic_platform_socket_address", + ], +) + +envoy_cc_library( + name = "quic_core_stream_frame_data_producer_lib", + hdrs = ["quiche/quic/core/quic_stream_frame_data_producer.h"], + repository = "@envoy", + deps = [":quic_core_types_lib"], +) + envoy_cc_library( name = "quic_core_stream_send_buffer_lib", srcs = ["quiche/quic/core/quic_stream_send_buffer.cc"], @@ -705,6 +1480,184 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quic_core_stream_sequencer_buffer_lib", + srcs = ["quiche/quic/core/quic_stream_sequencer_buffer.cc"], + hdrs = ["quiche/quic/core/quic_stream_sequencer_buffer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_constants_lib", + ":quic_core_interval_lib", + ":quic_core_interval_set_lib", + ":quic_core_packets_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_sustained_bandwidth_recorder_lib", + srcs = ["quiche/quic/core/quic_sustained_bandwidth_recorder.cc"], + hdrs = ["quiche/quic/core/quic_sustained_bandwidth_recorder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_bandwidth_lib", + ":quic_core_time_lib", + ":quic_platform_base", + ":quic_platform_export", + ], +) + +envoy_cc_library( + name = "quic_core_tag_lib", + srcs = ["quiche/quic/core/quic_tag.cc"], + hdrs = ["quiche/quic/core/quic_tag.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [":quic_platform_base"], +) + +envoy_cc_library( + name = "quic_core_time_lib", + srcs = ["quiche/quic/core/quic_time.cc"], + hdrs = ["quiche/quic/core/quic_time.h"], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [":quic_platform_base"], +) + +envoy_cc_library( + name = "quic_core_time_wait_list_manager_lib", + srcs = ["quiche/quic/core/quic_time_wait_list_manager.cc"], + hdrs = ["quiche/quic/core/quic_time_wait_list_manager.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_blocked_writer_interface_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_framer_lib", + ":quic_core_packet_writer_interface_lib", + ":quic_core_packets_lib", + ":quic_core_session_lib", + ":quic_core_types_lib", + ":quic_core_utils_lib", + ":quic_platform", + ], +) + +envoy_cc_library( + name = "quic_core_transmission_info_lib", + srcs = ["quiche/quic/core/quic_transmission_info.cc"], + hdrs = ["quiche/quic/core/quic_transmission_info.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_ack_listener_interface_lib", + ":quic_core_frames_frames_lib", + ":quic_core_types_lib", + ":quic_platform_export", + ], +) + +envoy_cc_library( + name = "quic_core_types_lib", + srcs = [ + "quiche/quic/core/quic_connection_id.cc", + "quiche/quic/core/quic_packet_number.cc", + "quiche/quic/core/quic_types.cc", + ], + hdrs = [ + "quiche/quic/core/quic_connection_id.h", + "quiche/quic/core/quic_packet_number.h", + "quiche/quic/core/quic_types.h", + ], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_core_error_codes_lib", + ":quic_core_time_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_uber_received_packet_manager_lib", + srcs = ["quiche/quic/core/uber_received_packet_manager.cc"], + hdrs = ["quiche/quic/core/uber_received_packet_manager.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_received_packet_manager_lib", + ":quic_core_utils_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_unacked_packet_map_lib", + srcs = ["quiche/quic/core/quic_unacked_packet_map.cc"], + hdrs = ["quiche/quic/core/quic_unacked_packet_map.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_connection_stats_lib", + ":quic_core_packets_lib", + ":quic_core_session_notifier_interface_lib", + ":quic_core_transmission_info_lib", + ":quic_core_utils_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_utils_lib", + srcs = ["quiche/quic/core/quic_utils.cc"], + hdrs = ["quiche/quic/core/quic_utils.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_core_constants_lib", + ":quic_core_crypto_random_lib", + ":quic_core_error_codes_lib", + ":quic_core_frames_frames_lib", + ":quic_core_types_lib", + ":quic_core_versions_lib", + ":quic_platform_base", + ":quic_platform_socket_address", + ], +) + +envoy_cc_library( + name = "quic_core_version_manager_lib", + srcs = ["quiche/quic/core/quic_version_manager.cc"], + hdrs = ["quiche/quic/core/quic_version_manager.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_versions_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_versions_lib", + srcs = ["quiche/quic/core/quic_versions.cc"], + hdrs = ["quiche/quic/core/quic_versions.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_core_tag_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + envoy_cc_test_library( name = "epoll_server_platform", hdrs = [ diff --git a/bazel/external/quiche.genrule_cmd b/bazel/external/quiche.genrule_cmd index 0546a6cd2c3a..70fb92af61eb 100644 --- a/bazel/external/quiche.genrule_cmd +++ b/bazel/external/quiche.genrule_cmd @@ -47,7 +47,9 @@ cat <sed_commands # Rewrite third_party includes. /^#include/ s!third_party/boringssl/src/include/!! +/^#include/ s!third_party/zlib/zlib!zlib! +/^import/ s!cached_network_parameters!quiche/quic/core/proto/cached_network_parameters! EOF for src_file in $(SRCS); do diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 92f7b5f3b594..9960796a9b88 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -242,8 +242,8 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/curl/curl/releases/download/curl-7_65_1/curl-7.65.1.tar.gz"], ), com_googlesource_quiche = dict( - # Static snapshot of https://quiche.googlesource.com/quiche/+archive/88e3e05c341147f4052e17a2769ac2722739c498.tar.gz - sha256 = "74ac2169adecfdc086a4942be2758b9ec3bf1782f75d1b421b695b8f4a19cc82", - urls = ["https://storage.googleapis.com/quiche-envoy-integration/88e3e05c341147f4052e17a2769ac2722739c498.tar.gz"], + # Static snapshot of https://quiche.googlesource.com/quiche/+archive/77ae31cbfb5bf41299c8c10a06a205a8a0d37cae.tar.gz + sha256 = "20542b8f3df505e4850c8538d747ce21275b1fde64106ccae49a19a3fd7a1ac5", + urls = ["https://storage.googleapis.com/quiche-envoy-integration/77ae31cbfb5bf41299c8c10a06a205a8a0d37cae.tar.gz"], ), ) diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 1d9baa577b87..54aacde06c81 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -15,7 +15,7 @@ envoy_cc_library( external_deps = ["quiche_quic_platform"], deps = [ "//include/envoy/event:timer_interface", - "@com_googlesource_quiche//:quic_core_alarm_interface", + "@com_googlesource_quiche//:quic_core_alarm_interface_lib", ], ) @@ -26,7 +26,7 @@ envoy_cc_library( external_deps = ["quiche_quic_platform"], deps = [ ":envoy_quic_alarm_lib", - "@com_googlesource_quiche//:quic_core_alarm_factory_interface", + "@com_googlesource_quiche//:quic_core_alarm_factory_interface_lib", "@com_googlesource_quiche//:quic_core_arena_scoped_ptr_lib", "@com_googlesource_quiche//:quic_core_one_block_arena_lib", ], diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD index f1660355d196..15880743587e 100644 --- a/source/extensions/quic_listeners/quiche/platform/BUILD +++ b/source/extensions/quic_listeners/quiche/platform/BUILD @@ -105,7 +105,10 @@ envoy_cc_library( "quic_logging_impl.h", ], visibility = ["//visibility:public"], - deps = ["//source/common/common:assert_lib"], + deps = [ + "//source/common/common:assert_lib", + "//source/common/common:stl_helpers", + ], ) envoy_cc_library( @@ -133,7 +136,6 @@ envoy_cc_library( "quic_ptr_util_impl.h", "quic_reference_counted_impl.h", "quic_server_stats_impl.h", - "quic_socket_address_impl.h", "quic_stack_trace_impl.h", "quic_str_cat_impl.h", "quic_stream_buffer_allocator_impl.h", @@ -227,6 +229,13 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quic_platform_bbr2_sender_impl_lib", + hdrs = ["quic_bbr2_sender_impl.h"], + visibility = ["//visibility:public"], + deps = ["@com_googlesource_quiche//:quic_core_congestion_control_bbr_lib"], +) + envoy_cc_library( name = "envoy_quic_clock_lib", srcs = ["envoy_quic_clock.cc"], diff --git a/source/extensions/quic_listeners/quiche/platform/flags_list.h b/source/extensions/quic_listeners/quiche/platform/flags_list.h index e334ff8f21e1..d269e7c6959b 100644 --- a/source/extensions/quic_listeners/quiche/platform/flags_list.h +++ b/source/extensions/quic_listeners/quiche/platform/flags_list.h @@ -52,10 +52,6 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_one_mss_conservation, false, "When true, ensure BBR allows at least one MSS to be sent in " "response to an ACK in packet conservation.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_slower_startup3, false, - "Add 3 connection options to decrease the pacing and CWND gain in " - "QUIC BBR STARTUP.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_slower_startup4, false, "Enables the BBQ5 connection option, which forces saved aggregation values " "to expire when the bandwidth increases more than 25% in QUIC BBR STARTUP.") @@ -134,10 +130,6 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_termination_packets, false, "If true, GFE time wait list will send termination packets based on " "current packet's encryption level.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_time_of_first_packet_sent_after_receiving, false, - "When true, fix initialization and updating of " - "|time_of_first_packet_sent_after_receiving_| in QuicConnection.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_limit_window_updates_in_traces, false, "Limits the number of window update events recorded in Tracegraf logs.") @@ -184,9 +176,6 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_optimize_inflight_check, false, "Stop checking QuicUnackedPacketMap::HasUnackedRetransmittableFrames " "and instead rely on the existing check that bytes_in_flight > 0") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_print_tag_hex, false, - "When true, non-ASCII QUIC tags are printed as hex instead of integers.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_check_toss_on_insertion_failure, false, "If true, enable the code that fixes a race condition for quic udp " "proxying in L0. See b/70036019.") @@ -292,9 +281,6 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_validate_packet_number_post_decrypti "If true, a QUIC endpoint will valid a received packet number after " "successfully decrypting the packet.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_send_version_negotiation_fixed_bit, false, - "When true, version negotiation packets sent by the server will set the fixed bit.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_eliminate_static_stream_map_3, false, "If true, static streams in a QuicSession will be stored inside dynamic stream map. " "static_stream_map will no longer be used.") @@ -310,9 +296,6 @@ QUICHE_FLAG(bool, quic_reloadable_flag_send_quic_fallback_server_config_on_leto_ QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_bbr_cwnd_in_bandwidth_resumption, true, " If true, adjust congestion window when doing bandwidth resumption in BBR.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_lumpy_pacing_at_low_bw, false, - "If true, disable lumpy pacing for low bandwidth flows.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_cwnd_and_pacing_gains, false, "If true, uses conservative cwnd gain and pacing gain.") @@ -340,13 +323,25 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_ignore_tlpr_if_sending_ping, false, QUICHE_FLAG(bool, quic_reloadable_flag_quic_terminate_gquic_connection_as_ietf, false, "If true, terminate Google QUIC connections similarly as IETF QUIC.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_44, false, + "If true, disable QUIC version 44.") + +QUICHE_FLAG( + bool, quic_reloadable_flag_quic_fix_packets_acked, false, + "If true, when detecting losses, use packets_acked of corresponding packet number space.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_ignore_tlpr_if_no_pending_stream_data, false, + "If true, ignore TLPR if there is no pending stream data") + +QUICHE_FLAG( + bool, quic_reloadable_flag_quic_drop_invalid_small_initial_connection_id, false, + "When true, QuicDispatcher will drop packets that have an initial destination connection ID " + "that is too short, instead of responding with a Version Negotiation packet to reject it.") + QUICHE_FLAG(bool, quic_restart_flag_quic_allow_loas_multipacket_chlo, false, "If true, inspects QUIC CHLOs for kLOAS and early creates sessions " "to allow multi-packet CHLOs") -QUICHE_FLAG(bool, quic_restart_flag_quic_enable_accept_random_ipn, false, - "Allow QUIC to accept initial packet numbers that are random, not 1.") - QUICHE_FLAG(bool, quic_restart_flag_quic_enable_gso_for_udp_egress, false, "If 1) flag is true, 2) UDP egress_method is used in GFE config, and " "3) UDP GSO is supported by the kernel, GFE will use UDP GSO for " @@ -394,14 +389,6 @@ QUICHE_FLAG(bool, quic_restart_flag_quic_do_not_override_connection_id, false, " When true, QuicFramer will not override connection IDs in headers and will instead " "respect the source/destination direction as expected by IETF QUIC.") -QUICHE_FLAG(bool, quic_restart_flag_quic_server_drop_version_negotiation, false, - "When true, QUIC server will drop IETF QUIC Version Negotiation packets.") - -QUICHE_FLAG( - bool, quic_restart_flag_quic_allow_variable_length_connection_id_for_negotiation, false, - "When true, allow variable length QUIC connection IDs for unsupported versions. This allows " - "performing version negotiation when the client-chosen server connection ID length is not 8") - QUICHE_FLAG(bool, quic_restart_flag_quic_no_framer_object_in_dispatcher, false, "If true, make QuicDispatcher no longer have an instance of QuicFramer.") @@ -470,6 +457,9 @@ QUICHE_FLAG(double, quic_pace_time_into_future_srtt_fraction, QUICHE_FLAG(bool, quic_export_server_num_packets_per_write_histogram, false, "If true, export number of packets written per write operation histogram.") +QUICHE_FLAG(int64_t, quic_headers_stream_id_in_v99, 0, + "QUIC version 99 will use this stream ID for the headers stream.") + QUICHE_FLAG(bool, http2_reloadable_flag_http2_testonly_default_false, false, "A testonly reloadable flag that will always default to false.") diff --git a/source/extensions/quic_listeners/quiche/platform/quic_bbr2_sender_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_bbr2_sender_impl.h new file mode 100644 index 000000000000..8995e1a443b6 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/platform/quic_bbr2_sender_impl.h @@ -0,0 +1,15 @@ +#pragma once + +// NOLINT(namespace-envoy) + +// This file is part of the QUICHE platform implementation, and is not to be +// consumed or referenced directly by other Envoy code. It serves purely as a +// porting layer for QUICHE. + +#include "quiche/quic/core/congestion_control/bbr_sender.h" + +namespace quic { + +using QuicBbr2SenderImpl = BbrSender; + +} // namespace quic diff --git a/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h index b0c98300c07a..41cd2c1b4c01 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h @@ -13,6 +13,7 @@ #include #include "common/common/logger.h" +#include "common/common/stl_helpers.h" #include "absl/base/optimization.h" #include "absl/synchronization/mutex.h" diff --git a/source/extensions/quic_listeners/quiche/platform/quic_reference_counted_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_reference_counted_impl.h index 648f742139dc..95ccdcd204b4 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_reference_counted_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_reference_counted_impl.h @@ -33,8 +33,6 @@ class QuicReferenceCountedImpl { template class QuicReferenceCountedPointerImpl { public: - static_assert(std::is_base_of::value, - "T must derive from QuicReferenceCounted"); QuicReferenceCountedPointerImpl() : refptr_(nullptr, T::destroy) {} QuicReferenceCountedPointerImpl(T* p) : refptr_(p, T::destroy) {} QuicReferenceCountedPointerImpl(std::nullptr_t) : refptr_(nullptr, T::destroy) {} diff --git a/source/extensions/quic_listeners/quiche/platform/quic_socket_address_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_socket_address_impl.h deleted file mode 100644 index 77a5a38edef4..000000000000 --- a/source/extensions/quic_listeners/quiche/platform/quic_socket_address_impl.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once - -// NOLINT(namespace-envoy) - -// This file is part of the QUICHE platform implementation, and is not to be -// consumed or referenced directly by other Envoy code. It serves purely as a -// porting layer for QUICHE. - -#include - -#include - -#include "quiche/quic/platform/api/quic_ip_address.h" - -namespace quic { - -// Implements the interface required by -// https://quiche.googlesource.com/quiche/+/refs/heads/master/quic/platform/api/quic_socket_address.h -// This is a dummy implementation which just allows its dependency to build. -// TODO(vasilvv) Remove this impl once QuicSocketAddress and QuicIpAddress are -// removed from platform API. - -class QuicSocketAddressImpl { -public: - QuicSocketAddressImpl() = default; - QuicSocketAddressImpl(QuicIpAddress, uint16_t) {} - explicit QuicSocketAddressImpl(const struct sockaddr_storage&) {} - QuicSocketAddressImpl(const struct sockaddr*, socklen_t&) {} - QuicSocketAddressImpl(const QuicSocketAddressImpl&) = default; - QuicSocketAddressImpl& operator=(const QuicSocketAddressImpl&) = default; - QuicSocketAddressImpl& operator=(QuicSocketAddressImpl&&) = default; - friend bool operator==(QuicSocketAddressImpl, QuicSocketAddressImpl) { return false; } - friend bool operator!=(QuicSocketAddressImpl, QuicSocketAddressImpl) { return true; } - - bool IsInitialized() const { return false; } - std::string ToString() const { return "Unimplemented."; } - int FromSocket(int) { return -1; } - QuicSocketAddressImpl Normalized() const { return QuicSocketAddressImpl(); } - - QuicIpAddress host() const { return QuicIpAddress(); } - uint16_t port() const { return 0; } - - sockaddr_storage generic_address() const { return sockaddr_storage{}; } -}; - -} // namespace quic From 0ce2190b0b4ee9ab47a95a386ff4af3463da3c38 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Thu, 20 Jun 2019 22:35:54 -0700 Subject: [PATCH 044/542] fix clang reference string in check_format.py (#7350) Signed-off-by: Derek Argueta --- tools/check_format.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/check_format.py b/tools/check_format.py index 94446a829471..c6cc431c7c70 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -191,7 +191,7 @@ def checkTools(): "users".format(CLANG_FORMAT_PATH)) else: error_messages.append( - "Command {} not found. If you have clang-format in version 7.x.x " + "Command {} not found. If you have clang-format in version 8.x.x " "installed, but the binary name is different or it's not available in " "PATH, please use CLANG_FORMAT environment variable to specify the path. " "Examples:\n" From 4b76a2f456e0771faa8d1e444df8e3a8f067c73b Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Thu, 20 Jun 2019 22:36:51 -0700 Subject: [PATCH 045/542] clang-tidy: performance-for-range-copy (#7349) Signed-off-by: Derek Argueta --- .clang-tidy | 2 +- source/common/config/filter_json.cc | 4 +-- source/common/config/grpc_mux_impl.cc | 4 +-- source/common/config/rds_json.cc | 26 ++++++++++--------- .../common/grpc/google_async_client_impl.cc | 2 +- source/common/http/utility.cc | 2 +- source/common/router/config_impl.cc | 2 +- .../router/metadatamatchcriteria_impl.cc | 2 +- source/common/upstream/cds_api_impl.cc | 2 +- source/common/upstream/load_stats_reporter.cc | 4 +-- .../common/upstream/outlier_detection_impl.cc | 2 +- source/common/upstream/subset_lb.cc | 2 +- .../network/http_connection_manager/config.cc | 2 +- .../transport_sockets/tls/context_impl.cc | 2 +- source/server/http/admin.cc | 8 +++--- test/common/filesystem/directory_test.cc | 4 +-- test/common/upstream/upstream_impl_test.cc | 2 +- test/config/utility.cc | 2 +- .../filters/common/ext_authz/test_common.cc | 2 +- .../network/thrift_proxy/conn_manager_test.cc | 2 +- .../metrics_service_integration_test.cc | 2 +- test/integration/http2_integration_test.cc | 2 +- test/integration/http_integration.cc | 2 +- test/integration/integration_admin_test.h | 2 +- test/server/http/admin_test.cc | 2 +- 25 files changed, 45 insertions(+), 43 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 870690b2d183..a7d317327ae3 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,7 +1,7 @@ Checks: 'clang-diagnostic-*,clang-analyzer-*,abseil-*,bugprone-*,modernize-*,performance-*,readability-redundant-*,readability-braces-around-statements,readability-container-size-empty' #TODO(lizan): grow this list, fix possible warnings and make more checks as error -WarningsAsErrors: 'bugprone-assert-side-effect,modernize-make-shared,modernize-make-unique,readability-redundant-smartptr-get,readability-braces-around-statements,readability-redundant-string-cstr,bugprone-use-after-move,readability-container-size-empty,performance-faster-string-find' +WarningsAsErrors: 'bugprone-assert-side-effect,modernize-make-shared,modernize-make-unique,readability-redundant-smartptr-get,readability-braces-around-statements,readability-redundant-string-cstr,bugprone-use-after-move,readability-container-size-empty,performance-faster-string-find,performance-for-range-copy' CheckOptions: - key: bugprone-assert-side-effect.AssertMacros diff --git a/source/common/config/filter_json.cc b/source/common/config/filter_json.cc index 87b19780c9a1..9c46f2092db5 100644 --- a/source/common/config/filter_json.cc +++ b/source/common/config/filter_json.cc @@ -284,14 +284,14 @@ void FilterJson::translateFaultFilter( JSON_UTIL_SET_DURATION_FROM_FIELD(*json_config_delay, *delay, fixed_delay, fixed_duration); } - for (const auto json_header_matcher : json_config.getObjectArray("headers", true)) { + for (const auto& json_header_matcher : json_config.getObjectArray("headers", true)) { auto* header_matcher = proto_config.mutable_headers()->Add(); RdsJson::translateHeaderMatcher(*json_header_matcher, *header_matcher); } JSON_UTIL_SET_STRING(json_config, proto_config, upstream_cluster); - for (auto json_downstream_node : json_config.getStringArray("downstream_nodes", true)) { + for (const auto& json_downstream_node : json_config.getStringArray("downstream_nodes", true)) { auto* downstream_node = proto_config.mutable_downstream_nodes()->Add(); *downstream_node = json_downstream_node; } diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 4d9b32975bf0..b18bcac597ff 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -175,7 +175,7 @@ void GrpcMuxImpl::onDiscoveryResponse( continue; } Protobuf::RepeatedPtrField found_resources; - for (auto watched_resource_name : watch->resources_) { + for (const auto& watched_resource_name : watch->resources_) { auto it = resources.find(watched_resource_name); if (it != resources.end()) { found_resources.Add()->MergeFrom(it->second); @@ -205,7 +205,7 @@ void GrpcMuxImpl::onDiscoveryResponse( void GrpcMuxImpl::onWriteable() { drainRequests(); } void GrpcMuxImpl::onStreamEstablished() { - for (const auto type_url : subscriptions_) { + for (const auto& type_url : subscriptions_) { queueDiscoveryRequest(type_url); } } diff --git a/source/common/config/rds_json.cc b/source/common/config/rds_json.cc index 658ae6b145b2..8e220f2b912f 100644 --- a/source/common/config/rds_json.cc +++ b/source/common/config/rds_json.cc @@ -56,7 +56,7 @@ void RdsJson::translateRateLimit(const Json::Object& json_rate_limit, JSON_UTIL_SET_INTEGER(json_rate_limit, rate_limit, stage); JSON_UTIL_SET_STRING(json_rate_limit, rate_limit, disable_key); const auto actions = json_rate_limit.getObjectArray("actions"); - for (const auto json_action : actions) { + for (const auto& json_action : actions) { auto* action = rate_limit.mutable_actions()->Add(); const std::string type = json_action->getString("type"); if (type == "source_cluster") { @@ -119,7 +119,7 @@ void RdsJson::translateRouteConfiguration(const Json::Object& json_route_config, envoy::api::v2::RouteConfiguration& route_config) { json_route_config.validateSchema(Json::Schema::ROUTE_CONFIGURATION_SCHEMA); - for (const auto json_virtual_host : json_route_config.getObjectArray("virtual_hosts", true)) { + for (const auto& json_virtual_host : json_route_config.getObjectArray("virtual_hosts", true)) { auto* virtual_host = route_config.mutable_virtual_hosts()->Add(); translateVirtualHost(*json_virtual_host, *virtual_host); } @@ -129,7 +129,7 @@ void RdsJson::translateRouteConfiguration(const Json::Object& json_route_config, route_config.add_internal_only_headers(header); } - for (const auto header_value : + for (const auto& header_value : json_route_config.getObjectArray("response_headers_to_add", true)) { auto* header_value_option = route_config.mutable_response_headers_to_add()->Add(); BaseJson::translateHeaderValueOption(*header_value, *header_value_option); @@ -140,7 +140,8 @@ void RdsJson::translateRouteConfiguration(const Json::Object& json_route_config, route_config.add_response_headers_to_remove(header); } - for (const auto header_value : json_route_config.getObjectArray("request_headers_to_add", true)) { + for (const auto& header_value : + json_route_config.getObjectArray("request_headers_to_add", true)) { auto* header_value_option = route_config.mutable_request_headers_to_add()->Add(); BaseJson::translateHeaderValueOption(*header_value, *header_value_option); } @@ -159,7 +160,7 @@ void RdsJson::translateVirtualHost(const Json::Object& json_virtual_host, virtual_host.add_domains(domain); } - for (const auto json_route : json_virtual_host.getObjectArray("routes", true)) { + for (const auto& json_route : json_virtual_host.getObjectArray("routes", true)) { auto* route = virtual_host.mutable_routes()->Add(); translateRoute(*json_route, *route); } @@ -169,18 +170,19 @@ void RdsJson::translateVirtualHost(const Json::Object& json_virtual_host, StringUtil::toUpper(json_virtual_host.getString("require_ssl", "")), &tls_requirement); virtual_host.set_require_tls(tls_requirement); - for (const auto json_virtual_cluster : + for (const auto& json_virtual_cluster : json_virtual_host.getObjectArray("virtual_clusters", true)) { auto* virtual_cluster = virtual_host.mutable_virtual_clusters()->Add(); translateVirtualCluster(*json_virtual_cluster, *virtual_cluster); } - for (const auto json_rate_limit : json_virtual_host.getObjectArray("rate_limits", true)) { + for (const auto& json_rate_limit : json_virtual_host.getObjectArray("rate_limits", true)) { auto* rate_limit = virtual_host.mutable_rate_limits()->Add(); translateRateLimit(*json_rate_limit, *rate_limit); } - for (const auto header_value : json_virtual_host.getObjectArray("request_headers_to_add", true)) { + for (const auto& header_value : + json_virtual_host.getObjectArray("request_headers_to_add", true)) { auto* header_value_option = virtual_host.mutable_request_headers_to_add()->Add(); BaseJson::translateHeaderValueOption(*header_value, *header_value_option); } @@ -226,12 +228,12 @@ void RdsJson::translateRoute(const Json::Object& json_route, envoy::api::v2::rou *match->mutable_runtime_fraction()); } - for (const auto json_header_matcher : json_route.getObjectArray("headers", true)) { + for (const auto& json_header_matcher : json_route.getObjectArray("headers", true)) { auto* header_matcher = match->mutable_headers()->Add(); translateHeaderMatcher(*json_header_matcher, *header_matcher); } - for (const auto json_query_parameter_matcher : + for (const auto& json_query_parameter_matcher : json_route.getObjectArray("query_parameters", true)) { auto* query_parameter_matcher = match->mutable_query_parameters()->Add(); translateQueryParameterMatcher(*json_query_parameter_matcher, *query_parameter_matcher); @@ -308,12 +310,12 @@ void RdsJson::translateRoute(const Json::Object& json_route, envoy::api::v2::rou &priority); action->set_priority(priority); - for (const auto header_value : json_route.getObjectArray("request_headers_to_add", true)) { + for (const auto& header_value : json_route.getObjectArray("request_headers_to_add", true)) { auto* header_value_option = route.mutable_request_headers_to_add()->Add(); BaseJson::translateHeaderValueOption(*header_value, *header_value_option); } - for (const auto json_rate_limit : json_route.getObjectArray("rate_limits", true)) { + for (const auto& json_rate_limit : json_route.getObjectArray("rate_limits", true)) { auto* rate_limit = action->mutable_rate_limits()->Add(); translateRateLimit(*json_rate_limit, *rate_limit); } diff --git a/source/common/grpc/google_async_client_impl.cc b/source/common/grpc/google_async_client_impl.cc index 1ae162a44898..9f89a70ab7f0 100644 --- a/source/common/grpc/google_async_client_impl.cc +++ b/source/common/grpc/google_async_client_impl.cc @@ -348,7 +348,7 @@ void GoogleAsyncStreamImpl::metadataTranslate( Http::HeaderMap& header_map) { // More painful copying, this time due to the mismatch in header // representation data structures in Envoy and Google gRPC. - for (auto it : grpc_metadata) { + for (const auto& it : grpc_metadata) { header_map.addCopy(Http::LowerCaseString(std::string(it.first.data(), it.first.size())), std::string(it.second.data(), it.second.size())); } diff --git a/source/common/http/utility.cc b/source/common/http/utility.cc index a36c50835b84..52171de11e6c 100644 --- a/source/common/http/utility.cc +++ b/source/common/http/utility.cc @@ -422,7 +422,7 @@ MessagePtr Utility::prepareHeaders(const ::envoy::api::v2::core::HttpUri& http_u std::string Utility::queryParamsToString(const QueryParams& params) { std::string out; std::string delim = "?"; - for (auto p : params) { + for (const auto& p : params) { absl::StrAppend(&out, delim, p.first, "=", p.second); delim = "&"; } diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 25cfea0397a7..6973cbc7b247 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -462,7 +462,7 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, cors_policy_ = std::make_unique(route.route().cors(), factory_context.runtime()); } - for (const auto upgrade_config : route.route().upgrade_configs()) { + for (const auto& upgrade_config : route.route().upgrade_configs()) { const bool enabled = upgrade_config.has_enabled() ? upgrade_config.enabled().value() : true; const bool success = upgrade_map_ diff --git a/source/common/router/metadatamatchcriteria_impl.cc b/source/common/router/metadatamatchcriteria_impl.cc index 714ec30cdb87..ef6008425671 100644 --- a/source/common/router/metadatamatchcriteria_impl.cc +++ b/source/common/router/metadatamatchcriteria_impl.cc @@ -20,7 +20,7 @@ MetadataMatchCriteriaImpl::extractMetadataMatchCriteria(const MetadataMatchCrite } // Add values from matches, replacing name/values copied from parent. - for (const auto it : matches.fields()) { + for (const auto& it : matches.fields()) { const auto index_it = existing.find(it.first); if (index_it != existing.end()) { v[index_it->second] = std::make_shared(it.first, it.second); diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 483a5b145d16..caea8d9f571e 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -81,7 +81,7 @@ void CdsApiImpl::onConfigUpdate( exception_msgs.push_back(fmt::format("{}: {}", cluster.name(), e.what())); } } - for (auto resource_name : removed_resources) { + for (const auto& resource_name : removed_resources) { if (cm_.removeCluster(resource_name)) { any_applied = true; ENVOY_LOG(debug, "cds: remove cluster '{}'", resource_name); diff --git a/source/common/upstream/load_stats_reporter.cc b/source/common/upstream/load_stats_reporter.cc index 336987180f24..0f98c0fe426c 100644 --- a/source/common/upstream/load_stats_reporter.cc +++ b/source/common/upstream/load_stats_reporter.cc @@ -64,7 +64,7 @@ void LoadStatsReporter::sendLoadStatsRequest() { uint64_t rq_error = 0; uint64_t rq_active = 0; uint64_t rq_issued = 0; - for (auto host : hosts) { + for (const auto& host : hosts) { rq_success += host->stats().rq_success_.latch(); rq_error += host->stats().rq_error_.latch(); rq_active += host->stats().rq_active_.value(); @@ -154,7 +154,7 @@ void LoadStatsReporter::startLoadReportPeriod() { } auto& cluster = it->second.get(); for (auto& host_set : cluster.prioritySet().hostSetsPerPriority()) { - for (auto host : host_set->hosts()) { + for (const auto& host : host_set->hosts()) { host->stats().rq_success_.latch(); host->stats().rq_error_.latch(); host->stats().rq_total_.latch(); diff --git a/source/common/upstream/outlier_detection_impl.cc b/source/common/upstream/outlier_detection_impl.cc index d9e906a0949d..f16ab7d95b81 100644 --- a/source/common/upstream/outlier_detection_impl.cc +++ b/source/common/upstream/outlier_detection_impl.cc @@ -140,7 +140,7 @@ DetectorImpl::DetectorImpl(const Cluster& cluster, } DetectorImpl::~DetectorImpl() { - for (auto host : host_monitors_) { + for (const auto& host : host_monitors_) { if (host.first->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)) { ASSERT(stats_.ejections_active_.value() > 0); stats_.ejections_active_.dec(); diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index 197c8d1ef9a2..1632e754c4c7 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -472,7 +472,7 @@ SubsetLoadBalancer::extractSubsetMetadata(const std::set& subset_ke } const auto& fields = filter_it->second.fields(); - for (const auto key : subset_keys) { + for (const auto& key : subset_keys) { const auto it = fields.find(key); if (it == fields.end()) { break; diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 07c604ace6b6..f8e6895e4940 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -311,7 +311,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( processFilter(filters[i], i, "http", filter_factories_); } - for (auto upgrade_config : config.upgrade_configs()) { + for (const auto& upgrade_config : config.upgrade_configs()) { const std::string& name = upgrade_config.upgrade_type(); const bool enabled = upgrade_config.has_enabled() ? upgrade_config.enabled().value() : true; if (findUpgradeCaseInsensitive(upgrade_filter_factories_, name) != diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index 477e97d84a68..1e358db812b0 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -191,7 +191,7 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c if (config.certificateValidationContext() != nullptr && !config.certificateValidationContext()->verifyCertificateSpkiList().empty()) { - for (auto hash : config.certificateValidationContext()->verifyCertificateSpkiList()) { + for (const auto& hash : config.certificateValidationContext()->verifyCertificateSpkiList()) { const auto decoded = Base64::decode(hash); if (decoded.size() != SHA256_DIGEST_LENGTH) { throw EnvoyException(fmt::format("Invalid base64-encoded SHA-256 {}", hash)); diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 487855d8a1fb..5d49691910f1 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -433,7 +433,7 @@ void AdminImpl::writeClustersAsText(Buffer::Instance& response) { all_stats[gauge->name()] = gauge->value(); } - for (auto stat : all_stats) { + for (const auto& stat : all_stats) { response.add(fmt::format("{}::{}::{}::{}\n", cluster.second.get().info()->name(), host->address()->asString(), stat.first, stat.second)); } @@ -757,7 +757,7 @@ Http::Code AdminImpl::handlerStats(absl::string_view url, Http::HeaderMap& respo rc = Http::Code::NotFound; } } else { // Display plain stats if format query param is not there. - for (auto stat : all_stats) { + for (const auto& stat : all_stats) { response.add(fmt::format("{}: {}\n", stat.first, stat.second)); } // TODO(ramaraochavali): See the comment in ThreadLocalStoreImpl::histograms() for why we use a @@ -769,7 +769,7 @@ Http::Code AdminImpl::handlerStats(absl::string_view url, Http::HeaderMap& respo all_histograms.emplace(histogram->name(), histogram->quantileSummary()); } } - for (auto histogram : all_histograms) { + for (const auto& histogram : all_histograms) { response.add(fmt::format("{}: {}\n", histogram.first, histogram.second)); } } @@ -894,7 +894,7 @@ AdminImpl::statsAsJson(const std::map& all_stats, document.SetObject(); rapidjson::Value stats_array(rapidjson::kArrayType); rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); - for (auto stat : all_stats) { + for (const auto& stat : all_stats) { Value stat_obj; stat_obj.SetObject(); Value stat_name; diff --git a/test/common/filesystem/directory_test.cc b/test/common/filesystem/directory_test.cc index 4c63a0d68d8c..cdc52b9121c7 100644 --- a/test/common/filesystem/directory_test.cc +++ b/test/common/filesystem/directory_test.cc @@ -75,13 +75,13 @@ using EntrySet = std::unordered_set; EntrySet getDirectoryContents(const std::string& dir_path, bool recursive) { Directory directory(dir_path); EntrySet ret; - for (const DirectoryEntry entry : directory) { + for (const DirectoryEntry& entry : directory) { ret.insert(entry); if (entry.type_ == FileType::Directory && entry.name_ != "." && entry.name_ != ".." && recursive) { std::string subdir_name = entry.name_; EntrySet subdir = getDirectoryContents(dir_path + "/" + subdir_name, recursive); - for (const DirectoryEntry entry : subdir) { + for (const DirectoryEntry& entry : subdir) { ret.insert({subdir_name + "/" + entry.name_, entry.type_}); } } diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index e2b058264d2f..9001f60c7691 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -686,7 +686,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { cluster.prioritySet().hostSetsPerPriority()[0]->healthyHostsPerLocality().get().size()); // Ensure that all host objects in the host list are unique. - for (const auto host : hosts) { + for (const auto& host : hosts) { EXPECT_EQ(1, std::count(hosts.begin(), hosts.end(), host)); } diff --git a/test/config/utility.cc b/test/config/utility.cc index 6f64c2206bba..8a90d921b1ec 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -267,7 +267,7 @@ ConfigHelper::ConfigHelper(const Network::Address::IpVersion version, Api::Api& } void ConfigHelper::applyConfigModifiers() { - for (auto config_modifier : config_modifiers_) { + for (const auto& config_modifier : config_modifiers_) { config_modifier(bootstrap_); } config_modifiers_.clear(); diff --git a/test/extensions/filters/common/ext_authz/test_common.cc b/test/extensions/filters/common/ext_authz/test_common.cc index 4557952a5ba7..a4b1dfe1ea33 100644 --- a/test/extensions/filters/common/ext_authz/test_common.cc +++ b/test/extensions/filters/common/ext_authz/test_common.cc @@ -71,7 +71,7 @@ Response TestCommon::makeAuthzResponse(CheckStatus status, Http::Code status_cod HeaderValueOptionVector TestCommon::makeHeaderValueOption(KeyValueOptionVector&& headers) { HeaderValueOptionVector header_option_vector{}; - for (auto header : headers) { + for (const auto& header : headers) { envoy::api::v2::core::HeaderValueOption header_value_option; auto* mutable_header = header_value_option.mutable_header(); mutable_header->set_key(header.key); diff --git a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc index 74688be7c7e3..a8e76e040a2c 100644 --- a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc @@ -83,7 +83,7 @@ class ThriftConnectionManagerTest : public testing::Test { // Destroy any existing filter first. filter_ = nullptr; - for (auto counter : store_.counters()) { + for (const auto& counter : store_.counters()) { counter->reset(); } diff --git a/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc b/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc index 759d142b066f..6d698a36dfc7 100644 --- a/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc +++ b/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc @@ -85,7 +85,7 @@ class MetricsServiceIntegrationTest : public Grpc::GrpcClientIntegrationParamTes const Protobuf::RepeatedPtrField<::io::prometheus::client::MetricFamily>& envoy_metrics = request_msg.envoy_metrics(); - for (::io::prometheus::client::MetricFamily metrics_family : envoy_metrics) { + for (const ::io::prometheus::client::MetricFamily& metrics_family : envoy_metrics) { if (metrics_family.name() == "cluster.cluster_0.membership_change" && metrics_family.type() == ::io::prometheus::client::MetricType::COUNTER) { known_counter_exists = true; diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index 9100cc57e194..cddc0a62530c 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -257,7 +257,7 @@ TEST_P(Http2MetadataIntegrationTest, ProxyInvalidMetadata) { } void verifyExpectedMetadata(Http::MetadataMap metadata_map, std::set keys) { - for (const auto key : keys) { + for (const auto& key : keys) { // keys are the same as their corresponding values. EXPECT_EQ(metadata_map.find(key)->second, key); } diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 452184f62f83..e4f16603652e 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -328,7 +328,7 @@ void HttpIntegrationTest::waitForNextUpstreamRequest(uint64_t upstream_index) { } void HttpIntegrationTest::addFilters(std::vector filters) { - for (const auto filter : filters) { + for (const auto& filter : filters) { config_helper_.addFilter(filter); } } diff --git a/test/integration/integration_admin_test.h b/test/integration/integration_admin_test.h index 0bc34fbe72d4..07bcfab5e0a6 100644 --- a/test/integration/integration_admin_test.h +++ b/test/integration/integration_admin_test.h @@ -30,7 +30,7 @@ class IntegrationAdminTest : public HttpProtocolIntegrationTest { Json::ObjectSharedPtr statsjson = Json::Factory::loadFromString(stats_json); EXPECT_TRUE(statsjson->hasObject("stats")); uint64_t histogram_count = 0; - for (Json::ObjectSharedPtr obj_ptr : statsjson->getObjectArray("stats")) { + for (const Json::ObjectSharedPtr& obj_ptr : statsjson->getObjectArray("stats")) { if (obj_ptr->hasObject("histograms")) { histogram_count++; const Json::ObjectSharedPtr& histograms_ptr = obj_ptr->getObject("histograms"); diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index d8ac9debd063..5955c1619350 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -1033,7 +1033,7 @@ TEST_P(AdminInstanceTest, RuntimeModifyNoArguments) { TEST_P(AdminInstanceTest, TracingStatsDisabled) { const std::string& name = admin_.tracingStats().service_forced_.name(); - for (Stats::CounterSharedPtr counter : server_.stats().counters()) { + for (const Stats::CounterSharedPtr& counter : server_.stats().counters()) { EXPECT_NE(counter->name(), name) << "Unexpected tracing stat found in server stats: " << name; } } From cad2e18928bf445016a3a3a1d200763015974e13 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Fri, 21 Jun 2019 10:30:01 -0700 Subject: [PATCH 046/542] clang-tidy: modernize-use-using (#7355) Signed-off-by: Derek Argueta --- .clang-tidy | 2 +- source/common/http/header_map_impl.h | 2 +- .../extensions/filters/http/common/aws/region_provider.h | 2 +- source/extensions/filters/http/tap/tap_config_impl.h | 8 ++++---- test/common/http/http2/metadata_encoder_decoder_test.cc | 8 ++++---- test/test_common/utility.h | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index a7d317327ae3..8c632b671fa2 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,7 +1,7 @@ Checks: 'clang-diagnostic-*,clang-analyzer-*,abseil-*,bugprone-*,modernize-*,performance-*,readability-redundant-*,readability-braces-around-statements,readability-container-size-empty' #TODO(lizan): grow this list, fix possible warnings and make more checks as error -WarningsAsErrors: 'bugprone-assert-side-effect,modernize-make-shared,modernize-make-unique,readability-redundant-smartptr-get,readability-braces-around-statements,readability-redundant-string-cstr,bugprone-use-after-move,readability-container-size-empty,performance-faster-string-find,performance-for-range-copy' +WarningsAsErrors: 'bugprone-assert-side-effect,bugprone-use-after-move,modernize-make-shared,modernize-make-unique,modernize-use-using,performance-faster-string-find,performance-for-range-copy,readability-braces-around-statements,readability-container-size-empty,readability-redundant-smartptr-get,readability-redundant-string-cstr' CheckOptions: - key: bugprone-assert-side-effect.AssertMacros diff --git a/source/common/http/header_map_impl.h b/source/common/http/header_map_impl.h index 7dcb665922a8..51b319499722 100644 --- a/source/common/http/header_map_impl.h +++ b/source/common/http/header_map_impl.h @@ -111,7 +111,7 @@ class HeaderMapImpl : public HeaderMap, NonCopyable { const LowerCaseString* key_; }; - typedef StaticLookupResponse (*EntryCb)(HeaderMapImpl&); + using EntryCb = StaticLookupResponse (*)(HeaderMapImpl&); /** * This is the static lookup table that is used to determine whether a header is one of the O(1) diff --git a/source/extensions/filters/http/common/aws/region_provider.h b/source/extensions/filters/http/common/aws/region_provider.h index 58e2ad64da74..2bd307595f11 100644 --- a/source/extensions/filters/http/common/aws/region_provider.h +++ b/source/extensions/filters/http/common/aws/region_provider.h @@ -24,7 +24,7 @@ class RegionProvider { virtual absl::optional getRegion() PURE; }; -typedef std::shared_ptr RegionProviderSharedPtr; +using RegionProviderSharedPtr = std::shared_ptr; } // namespace Aws } // namespace Common diff --git a/source/extensions/filters/http/tap/tap_config_impl.h b/source/extensions/filters/http/tap/tap_config_impl.h index 9846b6e52951..6ef19ddc7d92 100644 --- a/source/extensions/filters/http/tap/tap_config_impl.h +++ b/source/extensions/filters/http/tap/tap_config_impl.h @@ -42,10 +42,10 @@ class HttpPerRequestTapperImpl : public HttpPerRequestTapper, Logger::Loggable> ProtoList; + using ProtoList = std::list>; // Iterate through using protoEqual as ignore_ordering is true, and fields // in the sub-protos may also be out of order. ProtoList lhs_list = From ccf66c692c03224492f7e0849e3681a625f6a18f Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 21 Jun 2019 14:56:31 -0700 Subject: [PATCH 047/542] dns cache: add stats (#7347) Signed-off-by: Matt Klein --- .../dynamic_forward_proxy_filter.rst | 24 ++++++- .../intro/arch_overview/http/http_proxy.rst | 4 +- docs/root/operations/cli.rst | 6 -- include/envoy/upstream/cluster_factory.h | 2 - .../clusters/dynamic_forward_proxy/cluster.cc | 2 +- .../common/dynamic_forward_proxy/dns_cache.h | 2 +- .../dynamic_forward_proxy/dns_cache_impl.cc | 26 ++++++- .../dynamic_forward_proxy/dns_cache_impl.h | 28 +++++++- .../dns_cache_manager_impl.cc | 10 +-- .../dns_cache_manager_impl.h | 14 ++-- .../http/dynamic_forward_proxy/config.cc | 2 +- .../network/http_connection_manager/config.cc | 4 +- .../dns_cache_impl_test.cc | 69 ++++++++++++++++++- .../proxy_filter_integration_test.cc | 30 +++++++- test/test_common/test_time_system.h | 4 +- 15 files changed, 189 insertions(+), 38 deletions(-) diff --git a/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst b/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst index 407dafb5dc26..e1320a1b19ab 100644 --- a/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst +++ b/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst @@ -5,12 +5,12 @@ Dynamic forward proxy .. attention:: - HTTP dynamic forward proxy support should be considered alpha and not production ready. Stats - as well as circuit breakers are missing and will be added soon. + HTTP dynamic forward proxy support should be considered alpha and not production ready. Circuit + breakers are missing and will be added soon. * HTTP dynamic forward proxy :ref:`architecture overview ` * :ref:`v2 API reference ` -* This filter should be configured with the name *envoy.filters.http.dynamic_forward_proxy*. +* This filter should be configured with the name *envoy.filters.http.dynamic_forward_proxy* The following is a complete configuration that configures both the :ref:`dynamic forward proxy HTTP filter @@ -92,3 +92,21 @@ HTTP dynamic forward proxy. common_tls_context: validation_context: trusted_ca: {filename: /etc/ssl/certs/ca-certificates.crt} + +Statistics +---------- + +The dynamic forward proxy DNS cache outputs statistics in the dns_cache..* +namespace. + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + dns_query_attempt, Counter, Number of DNS query attempts. + dns_query_success, Counter, Number of DNS query successes. + dns_query_failure, Counter, Number of DNS query failures. + host_address_changed, Counter, Number of DNS queries that resulted in a host address change. + host_added, Counter, Number of hosts that have been added to the cache. + host_removed, Counter, Number of hosts that have been removed from the cache. + num_hosts, Gauge, Number of hosts that are currently in the cache. diff --git a/docs/root/intro/arch_overview/http/http_proxy.rst b/docs/root/intro/arch_overview/http/http_proxy.rst index f094e0c3fa01..28a835153270 100644 --- a/docs/root/intro/arch_overview/http/http_proxy.rst +++ b/docs/root/intro/arch_overview/http/http_proxy.rst @@ -5,8 +5,8 @@ HTTP dynamic forward proxy .. attention:: - HTTP dynamic forward proxy support should be considered alpha and not production ready. Stats - as well as circuit breakers are missing and will be added soon. + HTTP dynamic forward proxy support should be considered alpha and not production ready. Circuit + breakers are missing and will be added soon. Through the combination of both an :ref:`HTTP filter ` and :ref:`custom cluster `, diff --git a/docs/root/operations/cli.rst b/docs/root/operations/cli.rst index 2f9e3f976ed8..3f6494e64138 100644 --- a/docs/root/operations/cli.rst +++ b/docs/root/operations/cli.rst @@ -214,12 +214,6 @@ following are the command line options that Envoy supports. during a hot restart. See the :ref:`hot restart overview ` for more information. Defaults to 900 seconds (15 minutes). - .. attention:: - - This setting affects the output of :option:`--hot-restart-version`. If you started Envoy with this - option set to a non default value, you should use the same option (and same value) for subsequent hot - restarts. - .. option:: --disable-hot-restart *(optional)* This flag disables Envoy hot restart for builds that have it enabled. By default, hot diff --git a/include/envoy/upstream/cluster_factory.h b/include/envoy/upstream/cluster_factory.h index 11231b2d7b88..f8f5aac9cecb 100644 --- a/include/envoy/upstream/cluster_factory.h +++ b/include/envoy/upstream/cluster_factory.h @@ -98,8 +98,6 @@ class ClusterFactoryContext { virtual Ssl::ContextManager& sslContextManager() PURE; /** - * TODO(hyang): Remove this and only expose the scope, this would require refactoring - * TransportSocketFactoryContext * @return the server-wide stats store. */ virtual Stats::Store& stats() PURE; diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index f56e529bf14a..1b6d22699151 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -147,7 +147,7 @@ ClusterFactory::createClusterWithConfig( Server::Configuration::TransportSocketFactoryContext& socket_factory_context, Stats::ScopePtr&& stats_scope) { Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( - context.singletonManager(), context.dispatcher(), context.tls()); + context.singletonManager(), context.dispatcher(), context.tls(), context.stats()); auto new_cluster = std::make_shared( cluster, proto_config, context.runtime(), cache_manager_factory, context.localInfo(), socket_factory_context, std::move(stats_scope), context.addedViaApi()); diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache.h b/source/extensions/common/dynamic_forward_proxy/dns_cache.h index eb804396ef0f..8265b3b4ab04 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache.h @@ -144,7 +144,7 @@ using DnsCacheManagerSharedPtr = std::shared_ptr; */ DnsCacheManagerSharedPtr getCacheManager(Singleton::Manager& manager, Event::Dispatcher& main_thread_dispatcher, - ThreadLocal::SlotAllocator& tls); + ThreadLocal::SlotAllocator& tls, Stats::Scope& root_scope); /** * Factory for getting a DNS cache manager. diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index 6eab79ba0e3f..484b6f25142e 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -12,14 +12,16 @@ namespace DynamicForwardProxy { // TODO(mattklein123): circuit breakers / maximums on the number of hosts that the cache can // contain. -// TODO(mattklein123): stats DnsCacheImpl::DnsCacheImpl( Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls, + Stats::Scope& root_scope, const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config) : main_thread_dispatcher_(main_thread_dispatcher), dns_lookup_family_(Upstream::getDnsLookupFamilyFromEnum(config.dns_lookup_family())), resolver_(main_thread_dispatcher.createDnsResolver({})), tls_slot_(tls.allocateSlot()), + scope_(root_scope.createScope(fmt::format("dns_cache.{}.", config.name()))), + stats_{ALL_DNS_CACHE_STATS(POOL_COUNTER(*scope_), POOL_GAUGE(*scope_))}, refresh_interval_(PROTOBUF_GET_MS_OR_DEFAULT(config, dns_refresh_rate, 60000)), host_ttl_(PROTOBUF_GET_MS_OR_DEFAULT(config, host_ttl, 300000)) { tls_slot_->set([](Event::Dispatcher&) { return std::make_shared(); }); @@ -126,6 +128,7 @@ void DnsCacheImpl::startResolve(const std::string& host, PrimaryHostInfo& host_i ENVOY_LOG(debug, "starting main thread resolve for host='{}' dns='{}' port='{}'", host, host_info.host_to_resolve_, host_info.port_); ASSERT(host_info.active_query_ == nullptr); + stats_.dns_query_attempt_.inc(); host_info.active_query_ = resolver_->resolve( host_info.host_to_resolve_, dns_lookup_family_, [this, host](const std::list&& address_list) { @@ -154,6 +157,12 @@ void DnsCacheImpl::finishResolve( ? Network::Utility::getAddressWithPort(*address_list.front(), primary_host_info.port_) : nullptr; + if (address_list.empty()) { + stats_.dns_query_failure_.inc(); + } else { + stats_.dns_query_success_.inc(); + } + // Only the change the address if: // 1) The new address is valid && // 2a) The host doesn't yet have an address || @@ -168,6 +177,7 @@ void DnsCacheImpl::finishResolve( primary_host_info.host_info_->address_ = new_address; runAddUpdateCallbacks(host, primary_host_info.host_info_); address_changed = true; + stats_.host_address_changed_.inc(); } if (first_resolve || address_changed) { @@ -231,6 +241,20 @@ void DnsCacheImpl::ThreadLocalHostInfo::updateHostMap(const TlsHostMapSharedPtr& } } +DnsCacheImpl::PrimaryHostInfo::PrimaryHostInfo(DnsCacheImpl& parent, + absl::string_view host_to_resolve, uint16_t port, + const Event::TimerCb& timer_cb) + : parent_(parent), host_to_resolve_(host_to_resolve), port_(port), + refresh_timer_(parent.main_thread_dispatcher_.createTimer(timer_cb)) { + parent_.stats_.host_added_.inc(); + parent_.stats_.num_hosts_.inc(); +} + +DnsCacheImpl::PrimaryHostInfo::~PrimaryHostInfo() { + parent_.stats_.host_removed_.inc(); + parent_.stats_.num_hosts_.dec(); +} + } // namespace DynamicForwardProxy } // namespace Common } // namespace Extensions diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index 741eceb04966..387282abaa16 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -14,9 +14,29 @@ namespace Extensions { namespace Common { namespace DynamicForwardProxy { +/** + * All DNS cache stats. @see stats_macros.h + */ +#define ALL_DNS_CACHE_STATS(COUNTER, GAUGE) \ + COUNTER(dns_query_attempt) \ + COUNTER(dns_query_failure) \ + COUNTER(dns_query_success) \ + COUNTER(host_added) \ + COUNTER(host_address_changed) \ + COUNTER(host_removed) \ + GAUGE(num_hosts, NeverImport) + +/** + * Struct definition for all DNS cache stats. @see stats_macros.h + */ +struct DnsCacheStats { + ALL_DNS_CACHE_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) +}; + class DnsCacheImpl : public DnsCache, Logger::Loggable { public: DnsCacheImpl(Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls, + Stats::Scope& root_scope, const envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig& config); ~DnsCacheImpl(); @@ -68,10 +88,10 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable update_callbacks_; absl::flat_hash_map primary_hosts_; const std::chrono::milliseconds refresh_interval_; diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc index 0219746d0116..a895bf82fca1 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc @@ -26,17 +26,19 @@ DnsCacheSharedPtr DnsCacheManagerImpl::getCache( } DnsCacheSharedPtr new_cache = - std::make_shared(main_thread_dispatcher_, tls_, config); + std::make_shared(main_thread_dispatcher_, tls_, root_scope_, config); caches_.emplace(config.name(), ActiveCache{config, new_cache}); return new_cache; } DnsCacheManagerSharedPtr getCacheManager(Singleton::Manager& singleton_manager, Event::Dispatcher& main_thread_dispatcher, - ThreadLocal::SlotAllocator& tls) { + ThreadLocal::SlotAllocator& tls, + Stats::Scope& root_scope) { return singleton_manager.getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(dns_cache_manager), [&main_thread_dispatcher, &tls] { - return std::make_shared(main_thread_dispatcher, tls); + SINGLETON_MANAGER_REGISTERED_NAME(dns_cache_manager), + [&main_thread_dispatcher, &tls, &root_scope] { + return std::make_shared(main_thread_dispatcher, tls, root_scope); }); } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h index 7217028932d4..7b195f4c0883 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h @@ -11,8 +11,9 @@ namespace DynamicForwardProxy { class DnsCacheManagerImpl : public DnsCacheManager, public Singleton::Instance { public: - DnsCacheManagerImpl(Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls) - : main_thread_dispatcher_(main_thread_dispatcher), tls_(tls) {} + DnsCacheManagerImpl(Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls, + Stats::Scope& root_scope) + : main_thread_dispatcher_(main_thread_dispatcher), tls_(tls), root_scope_(root_scope) {} // DnsCacheManager DnsCacheSharedPtr @@ -30,23 +31,26 @@ class DnsCacheManagerImpl : public DnsCacheManager, public Singleton::Instance { Event::Dispatcher& main_thread_dispatcher_; ThreadLocal::SlotAllocator& tls_; + Stats::Scope& root_scope_; absl::flat_hash_map caches_; }; class DnsCacheManagerFactoryImpl : public DnsCacheManagerFactory { public: DnsCacheManagerFactoryImpl(Singleton::Manager& singleton_manager, Event::Dispatcher& dispatcher, - ThreadLocal::SlotAllocator& tls) - : singleton_manager_(singleton_manager), dispatcher_(dispatcher), tls_(tls) {} + ThreadLocal::SlotAllocator& tls, Stats::Scope& root_scope) + : singleton_manager_(singleton_manager), dispatcher_(dispatcher), tls_(tls), + root_scope_(root_scope) {} DnsCacheManagerSharedPtr get() override { - return getCacheManager(singleton_manager_, dispatcher_, tls_); + return getCacheManager(singleton_manager_, dispatcher_, tls_, root_scope_); } private: Singleton::Manager& singleton_manager_; Event::Dispatcher& dispatcher_; ThreadLocal::SlotAllocator& tls_; + Stats::Scope& root_scope_; }; } // namespace DynamicForwardProxy diff --git a/source/extensions/filters/http/dynamic_forward_proxy/config.cc b/source/extensions/filters/http/dynamic_forward_proxy/config.cc index dc4b93374186..93ae5a1fdcd6 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/config.cc +++ b/source/extensions/filters/http/dynamic_forward_proxy/config.cc @@ -12,7 +12,7 @@ Http::FilterFactoryCb DynamicForwardProxyFilterFactory::createFilterFactoryFromP const envoy::config::filter::http::dynamic_forward_proxy::v2alpha::FilterConfig& proto_config, const std::string&, Server::Configuration::FactoryContext& context) { Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory( - context.singletonManager(), context.dispatcher(), context.threadLocal()); + context.singletonManager(), context.dispatcher(), context.threadLocal(), context.scope()); ProxyFilterConfigSharedPtr filter_config(std::make_shared( proto_config, cache_manager_factory, context.clusterManager())); return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index f8e6895e4940..04edfdba79c4 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -163,9 +163,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( delayed_close_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(config, delayed_close_timeout, 1000)), normalize_path_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( config, normalize_path, - // TODO(htuch): we should have a - // boolean variant of featureEnabled() - // here. + // TODO(htuch): we should have a boolean variant of featureEnabled() here. context.runtime().snapshot().featureEnabled("http_connection_manager.normalize_path", #ifdef ENVOY_NORMALIZE_PATH_BY_DEFAULT 100 diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index a92d668f06f8..dbf1120fc274 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -28,17 +28,39 @@ makeAddressList(const std::list address_list) { class DnsCacheImplTest : public testing::Test, public Event::TestUsingSimulatedTime { public: void initialize() { + config_.set_name("foo"); config_.set_dns_lookup_family(envoy::api::v2::Cluster::V4_ONLY); EXPECT_CALL(dispatcher_, createDnsResolver(_)).WillOnce(Return(resolver_)); - dns_cache_ = std::make_unique(dispatcher_, tls_, config_); + dns_cache_ = std::make_unique(dispatcher_, tls_, store_, config_); update_callbacks_handle_ = dns_cache_->addUpdateCallbacks(update_callbacks_); } + ~DnsCacheImplTest() { + dns_cache_.reset(); + EXPECT_EQ(0, TestUtility::findGauge(store_, "dns_cache.foo.num_hosts")->value()); + } + + void checkStats(uint64_t query_attempt, uint64_t query_success, uint64_t query_failure, + uint64_t address_changed, uint64_t added, uint64_t removed, uint64_t num_hosts) { + const auto counter_value = [this](const std::string& name) { + return TestUtility::findCounter(store_, "dns_cache.foo." + name)->value(); + }; + + EXPECT_EQ(query_attempt, counter_value("dns_query_attempt")); + EXPECT_EQ(query_success, counter_value("dns_query_success")); + EXPECT_EQ(query_failure, counter_value("dns_query_failure")); + EXPECT_EQ(address_changed, counter_value("host_address_changed")); + EXPECT_EQ(added, counter_value("host_added")); + EXPECT_EQ(removed, counter_value("host_removed")); + EXPECT_EQ(num_hosts, TestUtility::findGauge(store_, "dns_cache.foo.num_hosts")->value()); + } + envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig config_; NiceMock dispatcher_; std::shared_ptr resolver_{std::make_shared()}; NiceMock tls_; + Stats::IsolatedStoreImpl store_; std::unique_ptr dns_cache_; MockUpdateCallbacks update_callbacks_; DnsCache::AddUpdateCallbacksHandlePtr update_callbacks_handle_; @@ -65,26 +87,49 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); EXPECT_NE(handle, nullptr); + checkStats(1 /* attempt */, 0 /* success */, 0 /* failure */, 0 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); resolve_cb(makeAddressList({"10.0.0.1"})); + checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + // Re-resolve timer. EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); resolve_timer->invokeCallback(); + checkStats(2 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + // Address does not change. EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); resolve_cb(makeAddressList({"10.0.0.1"})); + checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + + // Re-resolve timer. + EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + resolve_timer->invokeCallback(); + + checkStats(3 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + // Address does change. EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.2:80"))); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); resolve_cb(makeAddressList({"10.0.0.2"})); + + checkStats(3 /* attempt */, 3 /* success */, 0 /* failure */, 2 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); } // TTL purge test. @@ -100,25 +145,38 @@ TEST_F(DnsCacheImplTest, TTL) { DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); EXPECT_NE(handle, nullptr); + checkStats(1 /* attempt */, 0 /* success */, 0 /* failure */, 0 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); resolve_cb(makeAddressList({"10.0.0.1"})); + checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + // Re-resolve with ~60s passed. TTL should still be OK at default of 5 minutes. simTime().sleep(std::chrono::milliseconds(60001)); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); resolve_timer->invokeCallback(); + checkStats(2 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); resolve_cb(makeAddressList({"10.0.0.1"})); + checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); // Re-resolve with ~5m passed. This is not realistic as we would have re-resolved many times // during this period but it's good enough for the test. simTime().sleep(std::chrono::milliseconds(300000)); EXPECT_CALL(update_callbacks_, onDnsHostRemove("foo.com")); resolve_timer->invokeCallback(); + checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, + 1 /* added */, 1 /* removed */, 0 /* num hosts */); // Make sure we don't get a cache hit the next time the host is requested. resolve_timer = new Event::MockTimer(&dispatcher_); @@ -126,6 +184,8 @@ TEST_F(DnsCacheImplTest, TTL) { .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); EXPECT_NE(handle, nullptr); + checkStats(3 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, + 2 /* added */, 1 /* removed */, 1 /* num hosts */); } // TTL purge test with different refresh/TTL parameters. @@ -199,10 +259,14 @@ TEST_F(DnsCacheImplTest, ResolveFailure) { .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); EXPECT_NE(handle, nullptr); + checkStats(1 /* attempt */, 0 /* success */, 0 /* failure */, 0 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); resolve_cb(makeAddressList({})); + checkStats(1 /* attempt */, 0 /* success */, 1 /* failure */, 0 /* address changed */, + 1 /* added */, 0 /* removed */, 1 /* num hosts */); handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); EXPECT_EQ(handle, nullptr); @@ -337,7 +401,8 @@ TEST_F(DnsCacheImplTest, InvalidPort) { TEST(DnsCacheManagerImplTest, LoadViaConfig) { NiceMock dispatcher; NiceMock tls; - DnsCacheManagerImpl cache_manager(dispatcher, tls); + Stats::IsolatedStoreImpl store; + DnsCacheManagerImpl cache_manager(dispatcher, tls, store); envoy::config::common::dynamic_forward_proxy::v2alpha::DnsCacheConfig config1; config1.set_name("foo"); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 5ab1a65505e9..27249b2e33aa 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -4,6 +4,7 @@ namespace Envoy { namespace { class ProxyFilterIntegrationTest : public testing::TestWithParam, + public Event::TestUsingSimulatedTime, public HttpIntegrationTest { public: ProxyFilterIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, GetParam()) {} @@ -73,15 +74,38 @@ TEST_P(ProxyFilterIntegrationTest, RequestWithBody) { auto response = sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); checkSimpleRequestSuccess(1024, 1024, response.get()); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.dns_query_attempt")->value()); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_added")->value()); // Now send another request. This should hit the DNS cache. - // TODO(mattklein123): Verify this with stats once stats are added. response = sendRequestAndWaitForResponse(request_headers, 512, default_response_headers_, 512); checkSimpleRequestSuccess(512, 512, response.get()); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.dns_query_attempt")->value()); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_added")->value()); } -// TODO(mattklein123): Add a test for host expiration. We can do this both with simulated time -// and by checking stats. +// Verify that we expire hosts. +TEST_P(ProxyFilterIntegrationTest, RemoveHostViaTTL) { + codec_client_ = makeHttpConnection(lookupPort("http")); + Http::TestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = + sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.dns_query_attempt")->value()); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_added")->value()); + EXPECT_EQ(1, test_server_->gauge("dns_cache.foo.num_hosts")->value()); + + // > 5m + simTime().sleep(std::chrono::milliseconds(300001)); + test_server_->waitForGaugeEq("dns_cache.foo.num_hosts", 0); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_removed")->value()); +} } // namespace } // namespace Envoy diff --git a/test/test_common/test_time_system.h b/test/test_common/test_time_system.h index ccbefd942e61..bd3c184648ad 100644 --- a/test/test_common/test_time_system.h +++ b/test/test_common/test_time_system.h @@ -120,7 +120,9 @@ class DelegatingTestTimeSystem : public DelegatingTestTimeSystemBase(); }; auto time_system = dynamic_cast(&singleton_->timeSystem(make_time_system)); - RELEASE_ASSERT(time_system, "Two different types of time-systems allocated"); + RELEASE_ASSERT(time_system, + "Two different types of time-systems allocated. If deriving from " + "Event::TestUsingSimulatedTime make sure it is the first base class."); return *time_system; } From 47925a53e67645fa20ca60a5c3257084bd870994 Mon Sep 17 00:00:00 2001 From: Justin Stallard Date: Sat, 22 Jun 2019 13:30:06 -0400 Subject: [PATCH 048/542] api: fix ruby package naming conflicts (#7369) This fixes for ruby what https://github.com/envoyproxy/envoy/pull/3854 fixed for C#. Description: The existing protos yield a ruby module and class with the same name for both cluster and listener. This fixes for ruby what https://github.com/envoyproxy/envoy/pull/3854 fixed for C#. Risk Level: Low Testing: I successfully generated valid envoy configurations using the ruby code generated by these updated protos. Docs Changes: N/A Release Notes: Ruby module name overrides for data plane api proto definitions. Signed-off-by: Justin Stallard --- api/envoy/api/v2/cluster/circuit_breaker.proto | 1 + api/envoy/api/v2/cluster/outlier_detection.proto | 1 + api/envoy/api/v2/listener/listener.proto | 1 + 3 files changed, 3 insertions(+) diff --git a/api/envoy/api/v2/cluster/circuit_breaker.proto b/api/envoy/api/v2/cluster/circuit_breaker.proto index f219fa07b4fe..bc2bcf2548d2 100644 --- a/api/envoy/api/v2/cluster/circuit_breaker.proto +++ b/api/envoy/api/v2/cluster/circuit_breaker.proto @@ -7,6 +7,7 @@ option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.cluster"; option go_package = "cluster"; option csharp_namespace = "Envoy.Api.V2.ClusterNS"; +option ruby_package = "Envoy.Api.V2.ClusterNS"; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/api/v2/cluster/outlier_detection.proto b/api/envoy/api/v2/cluster/outlier_detection.proto index cd33cde1eccd..95f132242c11 100644 --- a/api/envoy/api/v2/cluster/outlier_detection.proto +++ b/api/envoy/api/v2/cluster/outlier_detection.proto @@ -6,6 +6,7 @@ option java_outer_classname = "OutlierDetectionProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.cluster"; option csharp_namespace = "Envoy.Api.V2.ClusterNS"; +option ruby_package = "Envoy.Api.V2.ClusterNS"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; diff --git a/api/envoy/api/v2/listener/listener.proto b/api/envoy/api/v2/listener/listener.proto index b9397ab7afc7..157def72132b 100644 --- a/api/envoy/api/v2/listener/listener.proto +++ b/api/envoy/api/v2/listener/listener.proto @@ -7,6 +7,7 @@ option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.listener"; option go_package = "listener"; option csharp_namespace = "Envoy.Api.V2.ListenerNS"; +option ruby_package = "Envoy::Api::V2::ListenerNS"; import "envoy/api/v2/core/address.proto"; import "envoy/api/v2/auth/cert.proto"; From b1a1f6b6e2832a33f4e4b8df7e609f9e347b2a23 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Sat, 22 Jun 2019 10:31:56 -0700 Subject: [PATCH 049/542] admin: fix compilation error (#7366) Signed-off-by: Ruslan Nigmatullin --- source/server/http/admin.cc | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 5d49691910f1..256c3542633b 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -673,22 +673,16 @@ Http::Code AdminImpl::handlerResetCounters(absl::string_view, Http::HeaderMap&, } envoy::admin::v2alpha::ServerInfo::State AdminImpl::serverState() { - envoy::admin::v2alpha::ServerInfo::State state; - switch (server_.initManager().state()) { case Init::Manager::State::Uninitialized: - state = envoy::admin::v2alpha::ServerInfo::PRE_INITIALIZING; - break; + return envoy::admin::v2alpha::ServerInfo::PRE_INITIALIZING; case Init::Manager::State::Initializing: - state = envoy::admin::v2alpha::ServerInfo::INITIALIZING; - break; + return envoy::admin::v2alpha::ServerInfo::INITIALIZING; case Init::Manager::State::Initialized: - state = server_.healthCheckFailed() ? envoy::admin::v2alpha::ServerInfo::DRAINING - : envoy::admin::v2alpha::ServerInfo::LIVE; - break; + return server_.healthCheckFailed() ? envoy::admin::v2alpha::ServerInfo::DRAINING + : envoy::admin::v2alpha::ServerInfo::LIVE; } - - return state; + NOT_REACHED_GCOVR_EXCL_LINE; } Http::Code AdminImpl::handlerServerInfo(absl::string_view, Http::HeaderMap& headers, From 87f7c44a2f7ea7a062febb8c0dba95137d8e6f93 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Sat, 22 Jun 2019 23:04:10 +0530 Subject: [PATCH 050/542] stats: flush stats even server is stuck initializing (#7298) Signed-off-by: Rama Chavali --- source/server/server.cc | 53 ++++++---- source/server/server.h | 1 + test/integration/server.h | 4 +- test/server/BUILD | 1 + test/server/server_test.cc | 139 +++++++++++++++++++++----- test/server/stats_sink_bootstrap.yaml | 16 +++ test/test_common/utility.cc | 7 ++ test/test_common/utility.h | 11 ++ 8 files changed, 185 insertions(+), 47 deletions(-) create mode 100644 test/server/stats_sink_bootstrap.yaml diff --git a/source/server/server.cc b/source/server/server.cc index 3f5205347198..a67c3e383507 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -166,27 +166,38 @@ void InstanceUtil::flushMetricsToSinks(const std::list& sinks, void InstanceImpl::flushStats() { ENVOY_LOG(debug, "flushing stats"); - // A shutdown initiated before this callback may prevent this from being called as per - // the semantics documented in ThreadLocal's runOnAllThreads method. - stats_store_.mergeHistograms([this]() -> void { - // mergeParentStatsIfAny() does nothing and returns a struct of 0s if there is no parent. - HotRestart::ServerStatsFromParent parent_stats = restarter_.mergeParentStatsIfAny(stats_store_); - - server_stats_->uptime_.set(time(nullptr) - original_start_time_); - server_stats_->memory_allocated_.set(Memory::Stats::totalCurrentlyAllocated() + - parent_stats.parent_memory_allocated_); - server_stats_->memory_heap_size_.set(Memory::Stats::totalCurrentlyReserved()); - server_stats_->parent_connections_.set(parent_stats.parent_connections_); - server_stats_->total_connections_.set(listener_manager_->numConnections() + - parent_stats.parent_connections_); - server_stats_->days_until_first_cert_expiring_.set( - sslContextManager().daysUntilFirstCertExpires()); - InstanceUtil::flushMetricsToSinks(config_.statsSinks(), stats_store_); - // TODO(ramaraochavali): consider adding different flush interval for histograms. - if (stat_flush_timer_ != nullptr) { - stat_flush_timer_->enableTimer(config_.statsFlushInterval()); - } - }); + // If Envoy is not fully initialized, workers will not be started and mergeHistograms + // completion callback is not called immediately. As a result of this server stats will + // not be updated and flushed to stat sinks. So skip mergeHistograms call if workers are + // not started yet. + if (initManager().state() == Init::Manager::State::Initialized) { + // A shutdown initiated before this callback may prevent this from being called as per + // the semantics documented in ThreadLocal's runOnAllThreads method. + stats_store_.mergeHistograms([this]() -> void { flushStatsInternal(); }); + } else { + ENVOY_LOG(debug, "Envoy is not fully initialized, skipping histogram merge and flushing stats"); + flushStatsInternal(); + } +} + +void InstanceImpl::flushStatsInternal() { + // mergeParentStatsIfAny() does nothing and returns a struct of 0s if there is no parent. + HotRestart::ServerStatsFromParent parent_stats = restarter_.mergeParentStatsIfAny(stats_store_); + + server_stats_->uptime_.set(time(nullptr) - original_start_time_); + server_stats_->memory_allocated_.set(Memory::Stats::totalCurrentlyAllocated() + + parent_stats.parent_memory_allocated_); + server_stats_->memory_heap_size_.set(Memory::Stats::totalCurrentlyReserved()); + server_stats_->parent_connections_.set(parent_stats.parent_connections_); + server_stats_->total_connections_.set(listener_manager_->numConnections() + + parent_stats.parent_connections_); + server_stats_->days_until_first_cert_expiring_.set( + sslContextManager().daysUntilFirstCertExpires()); + InstanceUtil::flushMetricsToSinks(config_.statsSinks(), stats_store_); + // TODO(ramaraochavali): consider adding different flush interval for histograms. + if (stat_flush_timer_ != nullptr) { + stat_flush_timer_->enableTimer(config_.statsFlushInterval()); + } } bool InstanceImpl::healthCheckFailed() { return server_stats_->live_.value() == 0; } diff --git a/source/server/server.h b/source/server/server.h index fe77f35cd874..b6a1f786358b 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -213,6 +213,7 @@ class InstanceImpl : Logger::Loggable, private: ProtobufTypes::MessagePtr dumpBootstrapConfig(); void flushStats(); + void flushStatsInternal(); void initialize(const Options& options, Network::Address::InstanceConstSharedPtr local_address, ComponentFactory& component_factory, ListenerHooks& hooks); void loadServerFlags(const absl::optional& flags_path); diff --git a/test/integration/server.h b/test/integration/server.h index ab21e43b3538..14a6a3191790 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -257,9 +257,7 @@ class IntegrationTestServer : public Logger::Loggable, absl::optional> process_object); void waitForCounterEq(const std::string& name, uint64_t value) override { - while (counter(name) == nullptr || counter(name)->value() != value) { - time_system_.sleep(std::chrono::milliseconds(10)); - } + TestUtility::waitForCounterEq(stat_store(), name, value, time_system_); } void waitForCounterGe(const std::string& name, uint64_t value) override { diff --git a/test/server/BUILD b/test/server/BUILD index f37d8ff8bc7e..3b3c4625015a 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -225,6 +225,7 @@ envoy_cc_test( ":node_bootstrap_without_access_log.yaml", ":runtime_bootstrap.yaml", ":runtime_test_data", + ":stats_sink_bootstrap.yaml", ":zipkin_tracing.yaml", "//test/config/integration:server.json", "//test/config/integration:server_config_files", diff --git a/test/server/server_test.cc b/test/server/server_test.cc index b91ab2bac861..b6235ca138e7 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -134,12 +134,42 @@ TEST_F(RunHelperTest, ShutdownBeforeInitManagerInit) { target.ready(); } +class InitializingInitManager : public Init::ManagerImpl { +public: + InitializingInitManager(absl::string_view name) : Init::ManagerImpl(name) {} + + State state() const override { return State::Initializing; } +}; + +class InitializingInstanceImpl : public InstanceImpl { +private: + InitializingInitManager init_manager_{"Server"}; + +public: + InitializingInstanceImpl(const Options& options, Event::TimeSystem& time_system, + Network::Address::InstanceConstSharedPtr local_address, + ListenerHooks& hooks, HotRestart& restarter, Stats::StoreRoot& store, + Thread::BasicLockable& access_log_lock, + ComponentFactory& component_factory, + Runtime::RandomGeneratorPtr&& random_generator, + ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, + Filesystem::Instance& file_system, + std::unique_ptr process_context) + : InstanceImpl(options, time_system, local_address, hooks, restarter, store, access_log_lock, + component_factory, std::move(random_generator), tls, thread_factory, + file_system, std::move(process_context)) {} + + Init::Manager& initManager() override { return init_manager_; } +}; + // Class creates minimally viable server instance for testing. class ServerInstanceImplTest : public testing::TestWithParam { protected: ServerInstanceImplTest() : version_(GetParam()) {} - void initialize(const std::string& bootstrap_path) { + void initialize(const std::string& bootstrap_path) { initialize(bootstrap_path, false); } + + void initialize(const std::string& bootstrap_path, const bool use_intializing_instance) { if (bootstrap_path.empty()) { options_.config_path_ = TestEnvironment::temporaryFileSubstitute( "test/config/integration/server.json", {{"upstream_0", 0}, {"upstream_1", 0}}, version_); @@ -151,13 +181,24 @@ class ServerInstanceImplTest : public testing::TestWithParam(*process_object_); } - server_ = std::make_unique( - options_, test_time_.timeSystem(), - Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("127.0.0.1")), - hooks_, restart_, stats_store_, fakelock_, component_factory_, - std::make_unique>(), *thread_local_, - Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), - std::move(process_context_)); + if (use_intializing_instance) { + server_ = std::make_unique( + options_, test_time_.timeSystem(), + Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("127.0.0.1")), + hooks_, restart_, stats_store_, fakelock_, component_factory_, + std::make_unique>(), *thread_local_, + Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), + std::move(process_context_)); + + } else { + server_ = std::make_unique( + options_, test_time_.timeSystem(), + Network::Address::InstanceConstSharedPtr(new Network::Address::Ipv4Instance("127.0.0.1")), + hooks_, restart_, stats_store_, fakelock_, component_factory_, + std::make_unique>(), *thread_local_, + Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), + std::move(process_context_)); + } EXPECT_TRUE(server_->api().fileSystem().fileExists("/dev/null")); } @@ -180,6 +221,27 @@ class ServerInstanceImplTest : public testing::TestWithParamapi().fileSystem().fileExists("/dev/null")); } + Thread::ThreadPtr startTestServer(const std::string& bootstrap_path, + const bool use_intializing_instance) { + absl::Notification started; + + auto server_thread = Thread::threadFactoryForTest().createThread([&] { + initialize(bootstrap_path, use_intializing_instance); + auto startup_handle = server_->registerCallback(ServerLifecycleNotifier::Stage::Startup, + [&] { started.Notify(); }); + auto shutdown_handle = server_->registerCallback(ServerLifecycleNotifier::Stage::ShutdownExit, + [&](Event::PostCb) { FAIL(); }); + shutdown_handle = nullptr; // unregister callback + server_->run(); + startup_handle = nullptr; + server_ = nullptr; + thread_local_ = nullptr; + }); + + started.WaitForNotification(); + return server_thread; + } + // Returns the server's tracer as a pointer, for use in dynamic_cast tests. Tracing::HttpTracer* tracer() { return &server_->httpContext().tracer(); }; @@ -197,27 +259,58 @@ class ServerInstanceImplTest : public testing::TestWithParam server_; }; +// Custom StatsSink that just increments a counter when flush is called. +class CustomStatsSink : public Stats::Sink { +public: + CustomStatsSink(Stats::Scope& scope) : stats_flushed_(scope.counter("stats.flushed")) {} + + // Stats::Sink + void flush(Stats::MetricSnapshot&) override { stats_flushed_.inc(); } + + void onHistogramComplete(const Stats::Histogram&, uint64_t) override {} + +private: + Stats::Counter& stats_flushed_; +}; + +// Custom StatsSinFactory that creates CustomStatsSink. +class CustomStatsSinkFactory : public Server::Configuration::StatsSinkFactory { +public: + // StatsSinkFactory + Stats::SinkPtr createStatsSink(const Protobuf::Message&, Server::Instance& server) override { + return std::make_unique(server.stats()); + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Empty()}; + } + + std::string name() override { return "envoy.custom_stats_sink"; } +}; + INSTANTIATE_TEST_SUITE_P(IpVersions, ServerInstanceImplTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); -TEST_P(ServerInstanceImplTest, EmptyShutdownLifecycleNotifications) { - absl::Notification started; +/** + * Static registration for the custom sink factory. @see RegisterFactory. + */ +REGISTER_FACTORY(CustomStatsSinkFactory, Server::Configuration::StatsSinkFactory); - auto server_thread = Thread::threadFactoryForTest().createThread([&] { - initialize("test/server/node_bootstrap.yaml"); - auto startup_handle = server_->registerCallback(ServerLifecycleNotifier::Stage::Startup, - [&] { started.Notify(); }); - auto shutdown_handle = server_->registerCallback(ServerLifecycleNotifier::Stage::ShutdownExit, - [&](Event::PostCb) { FAIL(); }); - shutdown_handle = nullptr; // unregister callback - server_->run(); - startup_handle = nullptr; - server_ = nullptr; - thread_local_ = nullptr; - }); +// Validates that server stats are flushed even when server is stuck with initialization. +TEST_P(ServerInstanceImplTest, StatsFlushWhenServerIsStillInitializing) { + auto server_thread = startTestServer("test/server/stats_sink_bootstrap.yaml", true); - started.WaitForNotification(); + // Wait till stats are flushed to custom sink and validate that the actual flush happens. + TestUtility::waitForCounterEq(stats_store_, "stats.flushed", 1, test_time_.timeSystem()); + EXPECT_EQ(Init::Manager::State::Initializing, server_->initManager().state()); + + server_->dispatcher().post([&] { server_->shutdown(); }); + server_thread->join(); +} + +TEST_P(ServerInstanceImplTest, EmptyShutdownLifecycleNotifications) { + auto server_thread = startTestServer("test/server/node_bootstrap.yaml", false); server_->dispatcher().post([&] { server_->shutdown(); }); server_thread->join(); } diff --git a/test/server/stats_sink_bootstrap.yaml b/test/server/stats_sink_bootstrap.yaml new file mode 100644 index 000000000000..922647f3c973 --- /dev/null +++ b/test/server/stats_sink_bootstrap.yaml @@ -0,0 +1,16 @@ +node: + id: bootstrap_id + cluster: bootstrap_cluster + locality: + zone: bootstrap_zone + sub_zone: bootstrap_sub_zone + build_version: should_be_ignored +admin: + access_log_path: /dev/null + address: + socket_address: + address: {{ ntop_ip_loopback_address }} + port_value: 0 +stats_sinks: +- name: envoy.custom_stats_sink +stats_flush_interval: 1s \ No newline at end of file diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 983932374d32..7f1e4e64eafd 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -153,6 +153,13 @@ Stats::GaugeSharedPtr TestUtility::findGauge(Stats::Store& store, const std::str return findByName(store.gauges(), name); } +void TestUtility::waitForCounterEq(Stats::Store& store, const std::string& name, uint64_t value, + Event::TestTimeSystem& time_system) { + while (findCounter(store, name) == nullptr || findCounter(store, name)->value() != value) { + time_system.sleep(std::chrono::milliseconds(10)); + } +} + std::list TestUtility::makeDnsResponse(const std::list& addresses) { std::list ret; diff --git a/test/test_common/utility.h b/test/test_common/utility.h index eb6c410b6980..57e9eadb7072 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -24,6 +24,7 @@ #include "test/test_common/file_system_for_test.h" #include "test/test_common/printers.h" +#include "test/test_common/test_time_system.h" #include "test/test_common/thread_factory_for_test.h" #include "absl/time/time.h" @@ -178,6 +179,16 @@ class TestUtility { */ static Stats::GaugeSharedPtr findGauge(Stats::Store& store, const std::string& name); + /** + * Wait till Counter value is equal to the passed ion value. + * @param store supplies the stats store. + * @param name supplies the name of the counter to wait for. + * @param value supplies the value of the counter. + * @param time_system the time system to use for waiting. + */ + static void waitForCounterEq(Stats::Store& store, const std::string& name, uint64_t value, + Event::TestTimeSystem& time_system); + /** * Convert a string list of IP addresses into a list of network addresses usable for DNS * response testing. From d0cefa7f071dbd4ef24399c2db8656c3a5d8c3ef Mon Sep 17 00:00:00 2001 From: danzh Date: Sat, 22 Jun 2019 13:37:01 -0400 Subject: [PATCH 051/542] quiche: switch UdplistenerImpl to use recvmsg (#7165) Signed-off-by: Dan Zhang --- bazel/envoy_internal.bzl | 4 + include/envoy/api/os_sys_calls.h | 6 + include/envoy/network/io_handle.h | 31 ++ include/envoy/network/listener.h | 3 +- source/common/api/os_sys_calls_impl.cc | 5 + source/common/api/os_sys_calls_impl.h | 1 + source/common/common/logger.h | 1 + source/common/event/dispatcher_impl.cc | 2 +- .../common/network/io_socket_handle_impl.cc | 167 +++++++ source/common/network/io_socket_handle_impl.h | 7 +- .../common/network/socket_option_factory.cc | 18 + source/common/network/socket_option_factory.h | 2 + source/common/network/socket_option_impl.h | 15 + source/common/network/udp_listener_impl.cc | 129 +++--- source/common/network/udp_listener_impl.h | 16 +- source/server/connection_handler_impl.cc | 2 +- source/server/connection_handler_impl.h | 2 +- source/server/listener_manager_impl.cc | 6 + test/common/network/BUILD | 3 +- test/common/network/listener_impl_test.cc | 2 +- test/common/network/listener_impl_test_base.h | 3 +- test/common/network/udp_listener_impl_test.cc | 417 +++++++----------- test/mocks/api/mocks.h | 2 + test/mocks/network/mocks.h | 4 +- test/server/listener_manager_impl_test.cc | 5 +- tools/spelling_dictionary.txt | 6 + 26 files changed, 517 insertions(+), 342 deletions(-) diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index b716c0f7b348..f12d52b2570e 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -52,6 +52,10 @@ def envoy_copts(repository, test = False): }) + select({ repository + "//bazel:enable_log_debug_assert_in_release": ["-DENVOY_LOG_DEBUG_ASSERT_IN_RELEASE"], "//conditions:default": [], + }) + select({ + # APPLE_USE_RFC_3542 is needed to support IPV6_PKTINFO in MAC OS. + repository + "//bazel:apple": ["-D__APPLE_USE_RFC_3542"], + "//conditions:default": [], }) + envoy_select_hot_restart(["-DENVOY_HOT_RESTART"], repository) + \ _envoy_select_perf_annotation(["-DENVOY_PERF_ANNOTATION"]) + \ envoy_select_google_grpc(["-DENVOY_GOOGLE_GRPC"], repository) + \ diff --git a/include/envoy/api/os_sys_calls.h b/include/envoy/api/os_sys_calls.h index bdece6810d02..e2213a0db0a1 100644 --- a/include/envoy/api/os_sys_calls.h +++ b/include/envoy/api/os_sys_calls.h @@ -53,6 +53,12 @@ class OsSysCalls { */ virtual SysCallSizeResult recvfrom(int sockfd, void* buffer, size_t length, int flags, struct sockaddr* addr, socklen_t* addrlen) PURE; + + /** + * @see recvmsg (man 2 recvmsg) + */ + virtual SysCallSizeResult recvmsg(int sockfd, struct msghdr* msg, int flags) PURE; + /** * Release all resources allocated for fd. * @return zero on success, -1 returned otherwise. diff --git a/include/envoy/network/io_handle.h b/include/envoy/network/io_handle.h index 49cb81e561e4..8bc0634f2ddc 100644 --- a/include/envoy/network/io_handle.h +++ b/include/envoy/network/io_handle.h @@ -77,6 +77,37 @@ class IoHandle { */ virtual Api::IoCallUint64Result sendmsg(const Buffer::RawSlice* slices, uint64_t num_slice, int flags, const Address::Instance& address) PURE; + + struct RecvMsgOutput { + /* + * @param dropped_packets points to a variable to store how many packets are + * dropped so far. If nullptr, recvmsg() won't try to get this information + * from transport header. + */ + RecvMsgOutput(uint32_t* dropped_packets) : dropped_packets_(dropped_packets) {} + + // If not nullptr, its value is the total number of packets dropped. recvmsg() will update it + // when more packets are dropped. + uint32_t* dropped_packets_; + // The destination address from transport header. + std::shared_ptr local_address_; + // The the source address from transport header. + std::shared_ptr peer_address_; + }; + + /** + * Receive a message into given slices, output overflow, source/destination + * addresses via passed-in parameters upon success. + * @param slices points to the location of receiving buffer. + * @param num_slice indicates number of slices |slices| contains. + * @param self_port the port this handle is assigned to. This is used to populate + * local_address because local port can't be retrieved from control message. + * @param output modified upon each call to return fields requested in it. + * @return a Api::IoCallUint64Result with err_ = an Api::IoError instance or + * err_ = nullptr and rc_ = the bytes received for success. + */ + virtual Api::IoCallUint64Result recvmsg(Buffer::RawSlice* slices, const uint64_t num_slice, + uint32_t self_port, RecvMsgOutput& output) PURE; }; using IoHandlePtr = std::unique_ptr; diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 0e61cd4ced4e..47f1f8a48d85 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -120,6 +120,7 @@ struct UdpRecvData { Address::InstanceConstSharedPtr local_address_; Address::InstanceConstSharedPtr peer_address_; // TODO(conquerAtapple): Fix ownership semantics. Buffer::InstancePtr buffer_; + MonotonicTime receive_time_; // TODO(conquerAtapple): // Add UdpReader here so that the callback handler can @@ -171,7 +172,7 @@ class UdpListenerCallbacks { * @param error_code ErrorCode for the error event. * @param error_number System error number. */ - virtual void onReceiveError(const ErrorCode& error_code, int error_number) PURE; + virtual void onReceiveError(const ErrorCode& error_code, Api::IoError::IoErrorCode err) PURE; }; /** diff --git a/source/common/api/os_sys_calls_impl.cc b/source/common/api/os_sys_calls_impl.cc index 24577fbccb94..75f3f373da07 100644 --- a/source/common/api/os_sys_calls_impl.cc +++ b/source/common/api/os_sys_calls_impl.cc @@ -45,6 +45,11 @@ SysCallSizeResult OsSysCallsImpl::recvfrom(int sockfd, void* buffer, size_t leng return {rc, errno}; } +SysCallSizeResult OsSysCallsImpl::recvmsg(int sockfd, struct msghdr* msg, int flags) { + const ssize_t rc = ::recvmsg(sockfd, msg, flags); + return {rc, errno}; +} + SysCallIntResult OsSysCallsImpl::ftruncate(int fd, off_t length) { const int rc = ::ftruncate(fd, length); return {rc, errno}; diff --git a/source/common/api/os_sys_calls_impl.h b/source/common/api/os_sys_calls_impl.h index 1e2706778c72..fa4ae8920401 100644 --- a/source/common/api/os_sys_calls_impl.h +++ b/source/common/api/os_sys_calls_impl.h @@ -17,6 +17,7 @@ class OsSysCallsImpl : public OsSysCalls { SysCallSizeResult recv(int socket, void* buffer, size_t length, int flags) override; SysCallSizeResult recvfrom(int sockfd, void* buffer, size_t length, int flags, struct sockaddr* addr, socklen_t* addrlen) override; + SysCallSizeResult recvmsg(int sockfd, struct msghdr* msg, int flags) override; SysCallIntResult close(int fd) override; SysCallIntResult ftruncate(int fd, off_t length) override; SysCallPtrResult mmap(void* addr, size_t length, int prot, int flags, int fd, diff --git a/source/common/common/logger.h b/source/common/common/logger.h index b01f210fa5bb..dcc83627ac07 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -40,6 +40,7 @@ namespace Logger { FUNCTION(http2) \ FUNCTION(hystrix) \ FUNCTION(init) \ + FUNCTION(io) \ FUNCTION(kafka) \ FUNCTION(lua) \ FUNCTION(main) \ diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index c7d2c0365396..4ffcf6181a82 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -130,7 +130,7 @@ DispatcherImpl::createListener(Network::Socket& socket, Network::ListenerCallbac Network::ListenerPtr DispatcherImpl::createUdpListener(Network::Socket& socket, Network::UdpListenerCallbacks& cb) { ASSERT(isThreadSafe()); - return Network::ListenerPtr{new Network::UdpListenerImpl(*this, socket, cb)}; + return Network::ListenerPtr{new Network::UdpListenerImpl(*this, socket, cb, timeSource())}; } TimerPtr DispatcherImpl::createTimer(TimerCb cb) { diff --git a/source/common/network/io_socket_handle_impl.cc b/source/common/network/io_socket_handle_impl.cc index cc9cbcdbc0b0..e17698d2a3af 100644 --- a/source/common/network/io_socket_handle_impl.cc +++ b/source/common/network/io_socket_handle_impl.cc @@ -11,6 +11,8 @@ #include "common/network/address_impl.h" #include "common/network/io_socket_error_impl.h" +#include "absl/types/optional.h" + using Envoy::Api::SysCallIntResult; using Envoy::Api::SysCallSizeResult; @@ -131,5 +133,170 @@ IoSocketHandleImpl::sysCallResultToIoCallResult(const Api::SysCallSizeResult& re : Api::IoErrorPtr(new IoSocketError(result.errno_), IoSocketError::deleteIoError))); } +Address::InstanceConstSharedPtr maybeGetDstAddressFromHeader(const struct cmsghdr& cmsg, + uint32_t self_port) { + if (cmsg.cmsg_type == IPV6_PKTINFO) { + const struct in6_pktinfo* info = reinterpret_cast(CMSG_DATA(&cmsg)); + sockaddr_in6 ipv6_addr; + memset(&ipv6_addr, 0, sizeof(sockaddr_in6)); + ipv6_addr.sin6_family = AF_INET6; + ipv6_addr.sin6_addr = info->ipi6_addr; + ipv6_addr.sin6_port = htons(self_port); + return Address::addressFromSockAddr(reinterpret_cast(ipv6_addr), + sizeof(sockaddr_in6), /*v6only=*/false); + } +#ifndef IP_RECVDSTADDR + if (cmsg.cmsg_type == IP_PKTINFO) { + const struct in_pktinfo* info = reinterpret_cast(CMSG_DATA(&cmsg)); +#else + if (cmsg.cmsg_type == IP_RECVDSTADDR) { + const struct in_addr* addr = reinterpret_cast(CMSG_DATA(&cmsg)); +#endif + sockaddr_in ipv4_addr; + memset(&ipv4_addr, 0, sizeof(sockaddr_in)); + ipv4_addr.sin_family = AF_INET; + ipv4_addr.sin_addr = +#ifndef IP_RECVDSTADDR + info->ipi_addr; +#else + *addr; +#endif + ipv4_addr.sin_port = htons(self_port); + return Address::addressFromSockAddr(reinterpret_cast(ipv4_addr), + sizeof(sockaddr_in), /*v6only=*/false); + } + return nullptr; +} + +absl::optional maybeGetPacketsDroppedFromHeader( +#ifdef SO_RXQ_OVFL + const struct cmsghdr& cmsg) { + if (cmsg.cmsg_type == SO_RXQ_OVFL) { + return *reinterpret_cast(CMSG_DATA(&cmsg)); + } +#else + const struct cmsghdr&) { +#endif + return absl::nullopt; +} + +Api::IoCallUint64Result IoSocketHandleImpl::recvmsg(Buffer::RawSlice* slices, + const uint64_t num_slice, uint32_t self_port, + RecvMsgOutput& output) { + +#ifndef __APPLE__ + // The minimum cmsg buffer size to filled in destination address and packets dropped when + // receiving a packet. It is possible for a received packet to contain both IPv4 and IPv6 + // addresses. + constexpr int cmsg_space = CMSG_SPACE(sizeof(int)) + CMSG_SPACE(sizeof(struct in_pktinfo)) + + CMSG_SPACE(sizeof(struct in6_pktinfo)); + char cbuf[cmsg_space]; +#else + // CMSG_SPACE() is supposed to be a constant expression, and most + // BSD-ish code uses it to determine the size of buffers used to + // send/receive descriptors in the control message for + // sendmsg/recvmsg. Such buffers are often automatic variables. + + // In Darwin, CMSG_SPACE uses __DARWIN_ALIGN. And __DARWIN_ALIGN(p) + // expands to + + // ((__darwin_size_t)((char *)(__darwin_size_t)(p) + __DARNWIN_ALIGNBYTES) + // &~ __DARWIN_ALIGNBYTES) + + // which Clang (to which many Apple employees contribute!) complains + // about when invoked with -pedantic -- and with our -Werror, causes + // Clang-based builds to fail. Clang says this is not a constant + // expression: + + // error: variable length array folded to constant array as an + // extension [-Werror,-pedantic] + + // Possibly true per standard definition, since that cast to (char *) + // is ugly, but actually Clang was able to convert to a compile-time + // constant... it just had to work harder. + + // As a ugly workaround, we define the following constant for use in + // automatic array declarations, and in all cases have an assert to + // verify that the value is of sufficient size. (The assert should + // constantly propagate and be dead-code eliminated in normal compiles + // and should cause a quick death if ever violated.) + constexpr int cmsg_space = 128; + char cbuf[cmsg_space]; +#endif + + STACK_ARRAY(iov, iovec, num_slice); + uint64_t num_slices_for_read = 0; + for (uint64_t i = 0; i < num_slice; i++) { + if (slices[i].mem_ != nullptr && slices[i].len_ != 0) { + iov[num_slices_for_read].iov_base = slices[i].mem_; + iov[num_slices_for_read].iov_len = slices[i].len_; + ++num_slices_for_read; + } + } + + sockaddr_storage peer_addr; + msghdr hdr; + hdr.msg_name = &peer_addr; + hdr.msg_namelen = sizeof(sockaddr_storage); + hdr.msg_iov = iov.begin(); + hdr.msg_iovlen = num_slices_for_read; + hdr.msg_flags = 0; + + struct cmsghdr* cmsg = reinterpret_cast(cbuf); + cmsg->cmsg_len = cmsg_space; + hdr.msg_control = cmsg; + hdr.msg_controllen = cmsg_space; + auto& os_sys_calls = Api::OsSysCallsSingleton::get(); + const Api::SysCallSizeResult result = os_sys_calls.recvmsg(fd_, &hdr, 0); + if (result.rc_ < 0) { + return sysCallResultToIoCallResult(result); + } + + RELEASE_ASSERT((hdr.msg_flags & MSG_CTRUNC) == 0, + fmt::format("Incorrectly set control message length: {}", hdr.msg_controllen)); + RELEASE_ASSERT(hdr.msg_namelen > 0, + fmt::format("Unable to get remote address from recvmsg() for fd: {}", fd_)); + try { + // Set v6only to false so that mapped-v6 address can be normalize to v4 + // address. Though dual stack may be disabled, it's still okay to assume the + // address is from a dual stack socket. This is because mapped-v6 address + // must come from a dual stack socket. An actual v6 address can come from + // both dual stack socket and v6 only socket. If |peer_addr| is an actual v6 + // address and the socket is actually v6 only, the returned address will be + // regarded as a v6 address from dual stack socket. However, this address is not going to be + // used to create socket. Wrong knowledge of dual stack support won't hurt. + output.peer_address_ = + Address::addressFromSockAddr(peer_addr, hdr.msg_namelen, /*v6only=*/false); + } catch (const EnvoyException& e) { + PANIC(fmt::format("Invalid remote address for fd: {}, error: {}", fd_, e.what())); + } + + // Get overflow, local and peer addresses from control message. + if (hdr.msg_controllen > 0) { + struct cmsghdr* cmsg; + for (cmsg = CMSG_FIRSTHDR(&hdr); cmsg != nullptr; cmsg = CMSG_NXTHDR(&hdr, cmsg)) { + if (output.local_address_ == nullptr) { + try { + Address::InstanceConstSharedPtr addr = maybeGetDstAddressFromHeader(*cmsg, self_port); + if (addr != nullptr) { + // This is a IP packet info message. + output.local_address_ = std::move(addr); + continue; + } + } catch (const EnvoyException& e) { + PANIC(fmt::format("Invalid destination address for fd: {}, error: {}", fd_, e.what())); + } + } + if (output.dropped_packets_ != nullptr) { + absl::optional maybe_dropped = maybeGetPacketsDroppedFromHeader(*cmsg); + if (maybe_dropped) { + *output.dropped_packets_ = *maybe_dropped; + } + } + } + } + return sysCallResultToIoCallResult(result); +} + } // namespace Network } // namespace Envoy diff --git a/source/common/network/io_socket_handle_impl.h b/source/common/network/io_socket_handle_impl.h index 3cb77a43755e..580ce942f233 100644 --- a/source/common/network/io_socket_handle_impl.h +++ b/source/common/network/io_socket_handle_impl.h @@ -4,13 +4,15 @@ #include "envoy/api/os_sys_calls.h" #include "envoy/network/io_handle.h" +#include "common/common/logger.h" + namespace Envoy { namespace Network { /** * IoHandle derivative for sockets */ -class IoSocketHandleImpl : public IoHandle { +class IoSocketHandleImpl : public IoHandle, protected Logger::Loggable { public: explicit IoSocketHandleImpl(int fd = -1) : fd_(fd) {} @@ -35,6 +37,9 @@ class IoSocketHandleImpl : public IoHandle { Api::IoCallUint64Result sendmsg(const Buffer::RawSlice* slices, uint64_t num_slice, int flags, const Address::Instance& address) override; + Api::IoCallUint64Result recvmsg(Buffer::RawSlice* slices, const uint64_t num_slice, + uint32_t self_port, RecvMsgOutput& output) override; + private: // Converts a SysCallSizeResult to IoCallUint64Result. Api::IoCallUint64Result sysCallResultToIoCallResult(const Api::SysCallSizeResult& result); diff --git a/source/common/network/socket_option_factory.cc b/source/common/network/socket_option_factory.cc index 44c113bb408b..451db09aae37 100644 --- a/source/common/network/socket_option_factory.cc +++ b/source/common/network/socket_option_factory.cc @@ -94,5 +94,23 @@ SocketOptionFactory::buildTcpFastOpenOptions(uint32_t queue_length) { return options; } +std::unique_ptr SocketOptionFactory::buildIpPacketInfoOptions() { + std::unique_ptr options = std::make_unique(); + options->push_back(std::make_shared( + envoy::api::v2::core::SocketOption::STATE_BOUND, ENVOY_RECV_IP_PKT_INFO, + ENVOY_RECV_IPV6_PKT_INFO, 1)); + return options; +} + +std::unique_ptr SocketOptionFactory::buildRxQueueOverFlowOptions() { + std::unique_ptr options = std::make_unique(); +#ifdef SO_RXQ_OVFL + options->push_back(std::make_shared( + envoy::api::v2::core::SocketOption::STATE_BOUND, + Network::SocketOptionName(std::make_pair(SOL_SOCKET, SO_RXQ_OVFL)), 1)); +#endif + return options; +} + } // namespace Network } // namespace Envoy diff --git a/source/common/network/socket_option_factory.h b/source/common/network/socket_option_factory.h index 9f7ed3a66cad..6fb2f2f5abad 100644 --- a/source/common/network/socket_option_factory.h +++ b/source/common/network/socket_option_factory.h @@ -31,6 +31,8 @@ class SocketOptionFactory : Logger::Loggable { static std::unique_ptr buildTcpFastOpenOptions(uint32_t queue_length); static std::unique_ptr buildLiteralOptions( const Protobuf::RepeatedPtrField& socket_options); + static std::unique_ptr buildIpPacketInfoOptions(); + static std::unique_ptr buildRxQueueOverFlowOptions(); }; } // namespace Network } // namespace Envoy diff --git a/source/common/network/socket_option_impl.h b/source/common/network/socket_option_impl.h index 4ff917bb70ec..a024cf6865ba 100644 --- a/source/common/network/socket_option_impl.h +++ b/source/common/network/socket_option_impl.h @@ -82,6 +82,21 @@ namespace Network { #define ENVOY_SOCKET_TCP_FASTOPEN Network::SocketOptionName() #endif +// Linux uses IP_PKTINFO for both sending source address and receiving destination +// address. +// FreeBSD uses IP_RECVDSTADDR for receiving destination address and IP_SENDSRCADDR for sending +// source address. +#ifdef IP_RECVDSTADDR +#define ENVOY_RECV_IP_PKT_INFO Network::SocketOptionName(std::make_pair(IPPROTO_IP, IP_RECVDSTADDR)) +#elif IP_PKTINFO +#define ENVOY_RECV_IP_PKT_INFO Network::SocketOptionName(std::make_pair(IPPROTO_IP, IP_PKTINFO)) +#endif + +// Both Linux and FreeBSD use IPV6_RECVPKTINFO for both sending source address and +// receiving destination address. +#define ENVOY_RECV_IPV6_PKT_INFO \ + Network::SocketOptionName(std::make_pair(IPPROTO_IPV6, IPV6_RECVPKTINFO)) + class SocketOptionImpl : public Socket::Option, Logger::Loggable { public: SocketOptionImpl(envoy::api::v2::core::SocketOption::SocketState in_state, diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index d329619f00a1..475ed94b72df 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -2,6 +2,10 @@ #include +#include +#include +#include + #include "envoy/buffer/buffer.h" #include "envoy/common/exception.h" @@ -22,9 +26,12 @@ namespace Envoy { namespace Network { +// Max UDP payload. +static const uint64_t MAX_UDP_PACKET_SIZE = 1500; + UdpListenerImpl::UdpListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, - UdpListenerCallbacks& cb) - : BaseListenerImpl(dispatcher, socket), cb_(cb) { + UdpListenerCallbacks& cb, TimeSource& time_source) + : BaseListenerImpl(dispatcher, socket), cb_(cb), time_source_(time_source) { file_event_ = dispatcher_.createFileEvent( socket.ioHandle().fd(), [this](uint32_t events) -> void { onSocketEvent(events); }, Event::FileTriggerType::Edge, Event::FileReadyType::Read | Event::FileReadyType::Write); @@ -49,36 +56,6 @@ void UdpListenerImpl::enable() { file_event_->setEnabled(Event::FileReadyType::Read | Event::FileReadyType::Write); } -UdpListenerImpl::ReceiveResult UdpListenerImpl::doRecvFrom(sockaddr_storage& peer_addr, - socklen_t& addr_len) { - constexpr uint64_t const read_length = 16384; - - Buffer::InstancePtr buffer = std::make_unique(); - - addr_len = sizeof(sockaddr_storage); - memset(&peer_addr, 0, addr_len); - - Buffer::RawSlice slice; - const uint64_t num_slices = buffer->reserve(read_length, &slice, 1); - - ASSERT(num_slices == 1); - - auto& os_sys_calls = Api::OsSysCallsSingleton::get(); - // TODO(sumukhs) - Convert this to use the IOHandle interface rather than os_sys_calls - const Api::SysCallSizeResult result = - os_sys_calls.recvfrom(socket_.ioHandle().fd(), slice.mem_, read_length, 0, - reinterpret_cast(&peer_addr), &addr_len); - if (result.rc_ < 0) { - return ReceiveResult{Api::SysCallIntResult{static_cast(result.rc_), result.errno_}, - nullptr}; - } - slice.len_ = std::min(slice.len_, static_cast(result.rc_)); - buffer->commit(&slice, 1); - - ENVOY_UDP_LOG(trace, "recvfrom bytes {}", result.rc_); - return ReceiveResult{Api::SysCallIntResult{static_cast(result.rc_), 0}, std::move(buffer)}; -} - void UdpListenerImpl::onSocketEvent(short flags) { ASSERT((flags & (Event::FileReadyType::Read | Event::FileReadyType::Write))); ENVOY_UDP_LOG(trace, "socket event: {}", flags); @@ -94,56 +71,74 @@ void UdpListenerImpl::onSocketEvent(short flags) { void UdpListenerImpl::handleReadCallback() { ENVOY_UDP_LOG(trace, "handleReadCallback"); - sockaddr_storage addr; - socklen_t addr_len = 0; - + // TODO(danzh) make this variable configurable to support jumbo frames. + const uint64_t read_buffer_length = MAX_UDP_PACKET_SIZE; do { - ReceiveResult recv_result = doRecvFrom(addr, addr_len); - if ((recv_result.result_.rc_ < 0)) { - if (recv_result.result_.errno_ != EAGAIN) { - ENVOY_UDP_LOG(error, "recvfrom result {}", recv_result.result_.errno_); + Buffer::InstancePtr buffer = std::make_unique(); + Buffer::RawSlice slice; + const uint64_t num_slices = buffer->reserve(read_buffer_length, &slice, 1); + ASSERT(num_slices == 1); + + IoHandle::RecvMsgOutput output(&packets_dropped_); + uint32_t old_packets_dropped = packets_dropped_; + MonotonicTime receive_time = time_source_.monotonicTime(); + Api::IoCallUint64Result result = socket_.ioHandle().recvmsg( + &slice, num_slices, socket_.localAddress()->ip()->port(), output); + + if (!result.ok()) { + // No more to read or encountered a system error. + if (result.err_->getErrorCode() != Api::IoError::IoErrorCode::Again) { + ENVOY_UDP_LOG(error, "recvfrom result {}: {}", + static_cast(result.err_->getErrorCode()), + result.err_->getErrorDetails()); cb_.onReceiveError(UdpListenerCallbacks::ErrorCode::SyscallError, - recv_result.result_.errno_); + result.err_->getErrorCode()); } + // Stop reading. return; } - if (recv_result.result_.rc_ == 0) { - // TODO(conqerAtapple): Is zero length packet interesting? - return; + if (result.rc_ == 0) { + // TODO(conqerAtapple): Is zero length packet interesting? If so add stats + // for it. Otherwise remove the warning log below. + ENVOY_UDP_LOG(trace, "received 0-length packet"); } - Address::InstanceConstSharedPtr local_address = socket_.localAddress(); - - RELEASE_ASSERT( - addr_len > 0, - fmt::format( - "Unable to get remote address for fd: {}, local address: {}. address length is 0 ", - socket_.ioHandle().fd(), local_address->asString())); + RELEASE_ASSERT(output.local_address_ != nullptr, "fail to get local address from IP header"); + + if (packets_dropped_ != old_packets_dropped) { + // The kernel tracks SO_RXQ_OVFL as a uint32 which can overflow to a smaller + // value. So as long as this count differs from previously recorded value, + // more packets are dropped by kernel. + uint32_t delta = (packets_dropped_ > old_packets_dropped) + ? (packets_dropped_ - old_packets_dropped) + : (packets_dropped_ + + (std::numeric_limits::max() - old_packets_dropped) + 1); + // TODO(danzh) add stats for this. + ENVOY_UDP_LOG(debug, "Kernel dropped {} more packets. Consider increase receive buffer size.", + delta); + } - Address::InstanceConstSharedPtr peer_address; + // Adjust used memory length. + slice.len_ = std::min(slice.len_, static_cast(result.rc_)); + buffer->commit(&slice, 1); - try { - peer_address = Address::addressFromSockAddr( - addr, addr_len, local_address->ip()->version() == Address::IpVersion::v6); - } catch (const EnvoyException&) { - // Intentional no-op. The assert should fail below - } + ENVOY_UDP_LOG(trace, "recvmsg bytes {}", result.rc_); - RELEASE_ASSERT((peer_address != nullptr), + RELEASE_ASSERT(output.peer_address_ != nullptr, fmt::format("Unable to get remote address for fd: {}, local address: {} ", - socket_.ioHandle().fd(), local_address->asString())); + socket_.ioHandle().fd(), socket_.localAddress()->asString())); // Unix domain sockets are not supported - RELEASE_ASSERT(peer_address->type() == Address::Type::Ip, - fmt::format("Unsupported peer address: {} local address: {}, receive size: " - "{}, address length: {}", - peer_address->asString(), local_address->asString(), - recv_result.result_.rc_, addr_len)); - - UdpRecvData recvData{local_address, peer_address, std::move(recv_result.buffer_)}; + RELEASE_ASSERT(output.peer_address_->type() == Address::Type::Ip, + fmt::format("Unsupported remote address: {} local address: {}, receive size: " + "{}", + output.peer_address_->asString(), socket_.localAddress()->asString(), + result.rc_)); + + UdpRecvData recvData{std::move(output.local_address_), std::move(output.peer_address_), + std::move(buffer), receive_time}; cb_.onData(recvData); - } while (true); } diff --git a/source/common/network/udp_listener_impl.h b/source/common/network/udp_listener_impl.h index 2a871959b3e0..b526b8034f8d 100644 --- a/source/common/network/udp_listener_impl.h +++ b/source/common/network/udp_listener_impl.h @@ -2,6 +2,8 @@ #include +#include "envoy/common/time.h" + #include "common/buffer/buffer_impl.h" #include "common/event/event_impl_base.h" #include "common/event/file_event_impl.h" @@ -18,7 +20,8 @@ class UdpListenerImpl : public BaseListenerImpl, public virtual UdpListener, protected Logger::Loggable { public: - UdpListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, UdpListenerCallbacks& cb); + UdpListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, UdpListenerCallbacks& cb, + TimeSource& time_source); ~UdpListenerImpl() override; @@ -31,22 +34,17 @@ class UdpListenerImpl : public BaseListenerImpl, const Address::InstanceConstSharedPtr& localAddress() const override; Api::IoCallUint64Result send(const UdpSendData& data) override; - struct ReceiveResult { - Api::SysCallIntResult result_; - Buffer::InstancePtr buffer_; - }; - - // Test overrides for mocking - virtual ReceiveResult doRecvFrom(sockaddr_storage& peer_addr, socklen_t& addr_len); - protected: void handleWriteCallback(); void handleReadCallback(); UdpListenerCallbacks& cb_; + uint32_t packets_dropped_{0}; private: void onSocketEvent(short flags); + + TimeSource& time_source_; Event::FileEventPtr file_event_; }; diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index d516848f50dd..b2fe7b2ae95c 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -346,7 +346,7 @@ void ConnectionHandlerImpl::ActiveUdpListener::onWriteReady(const Network::Socke } void ConnectionHandlerImpl::ActiveUdpListener::onReceiveError( - const Network::UdpListenerCallbacks::ErrorCode&, int) { + const Network::UdpListenerCallbacks::ErrorCode&, Api::IoError::IoErrorCode) { // TODO(sumukhs): Determine what to do on receive error. // Would the filters need to know on error? Can't foresee a scenario where they // would take an action diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index 35f0e67b242f..f8b83e3628f6 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -109,7 +109,7 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { void onData(Network::UdpRecvData& data) override; void onWriteReady(const Network::Socket& socket) override; void onReceiveError(const Network::UdpListenerCallbacks::ErrorCode& error_code, - int error_number) override; + Api::IoError::IoErrorCode err) override; // Network::UdpListenerFilterManager void addReadFilter(Network::UdpListenerReadFilterPtr&& filter) override; diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 66d83a53ac75..9cef398c1a7b 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -213,6 +213,12 @@ ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, const std::st addListenSocketOptions( Network::SocketOptionFactory::buildLiteralOptions(config.socket_options())); } + if (socket_type_ == Network::Address::SocketType::Datagram) { + // Needed for recvmsg to return destination address in IP header. + addListenSocketOptions(Network::SocketOptionFactory::buildIpPacketInfoOptions()); + // Needed to return receive buffer overflown indicator. + addListenSocketOptions(Network::SocketOptionFactory::buildRxQueueOverFlowOptions()); + } if (!config.listener_filters().empty()) { switch (socket_type_) { diff --git a/test/common/network/BUILD b/test/common/network/BUILD index e28b3f6a5919..0b62e822f71f 100644 --- a/test/common/network/BUILD +++ b/test/common/network/BUILD @@ -24,6 +24,7 @@ envoy_cc_test_library( "//test/mocks/server:server_mocks", "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", + "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", ], ) @@ -191,7 +192,7 @@ envoy_cc_test( "//test/mocks/server:server_mocks", "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", - "//test/test_common:test_time_lib", + "//test/test_common:threadsafe_singleton_injector_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/common/network/listener_impl_test.cc b/test/common/network/listener_impl_test.cc index 76cfb4b5092a..3cca4c1bad82 100644 --- a/test/common/network/listener_impl_test.cc +++ b/test/common/network/listener_impl_test.cc @@ -252,7 +252,7 @@ TEST_P(ListenerImplTest, DisableAndEnableListener) { timer->enableTimer(std::chrono::milliseconds(2000)); EXPECT_CALL(listener_callbacks, onAccept_(_, _)).Times(0); - + time_system_.sleep(std::chrono::milliseconds(2000)); dispatcher_->run(Event::Dispatcher::RunType::Block); // When the listener is re-enabled, the pending connection should be accepted. diff --git a/test/common/network/listener_impl_test_base.h b/test/common/network/listener_impl_test_base.h index 3720cc40559e..7c20c853a403 100644 --- a/test/common/network/listener_impl_test_base.h +++ b/test/common/network/listener_impl_test_base.h @@ -5,6 +5,7 @@ #include "common/network/utility.h" #include "test/test_common/network_utility.h" +#include "test/test_common/simulated_time_system.h" #include "test/test_common/test_time.h" #include "test/test_common/utility.h" @@ -34,8 +35,8 @@ class ListenerImplTestBase : public testing::TestWithParam { const Address::IpVersion version_; const Address::InstanceConstSharedPtr alt_address_; + Event::SimulatedTimeSystem time_system_; Api::ApiPtr api_; - DangerousDeprecatedTestTime test_time_; Event::DispatcherPtr dispatcher_; }; diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index 80bd6208fd58..04e9860e09d2 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -3,14 +3,17 @@ #include #include "common/network/address_impl.h" +#include "common/network/socket_option_factory.h" #include "common/network/udp_listener_impl.h" #include "common/network/utility.h" #include "test/common/network/listener_impl_test_base.h" +#include "test/mocks/api/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/network_utility.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -24,73 +27,100 @@ namespace Envoy { namespace Network { namespace { -class TestUdpListenerImpl : public UdpListenerImpl { +class UdpListenerImplTest : public ListenerImplTestBase { public: - TestUdpListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, UdpListenerCallbacks& cb) - : UdpListenerImpl(dispatcher, socket, cb) {} + UdpListenerImplTest() + : server_socket_(createServerSocket(true)), send_to_addr_(getServerLoopbackAddress()) { + time_system_.sleep(std::chrono::milliseconds(100)); + } - MOCK_METHOD2(doRecvFrom, - UdpListenerImpl::ReceiveResult(sockaddr_storage& peer_addr, socklen_t& addr_len)); + void SetUp() override { + // Set listening socket options. + server_socket_->addOptions(SocketOptionFactory::buildIpPacketInfoOptions()); + server_socket_->addOptions(SocketOptionFactory::buildRxQueueOverFlowOptions()); - UdpListenerImpl::ReceiveResult doRecvFrom_(sockaddr_storage& peer_addr, socklen_t& addr_len) { - return UdpListenerImpl::doRecvFrom(peer_addr, addr_len); + listener_ = std::make_unique( + dispatcherImpl(), *server_socket_, listener_callbacks_, dispatcherImpl().timeSource()); } -}; -class UdpListenerImplTest : public ListenerImplTestBase { protected: - SocketPtr getSocket(Address::SocketType type, const Address::InstanceConstSharedPtr& address, - const Network::Socket::OptionsSharedPtr& options, bool bind) { - if (type == Address::SocketType::Stream) { - using NetworkSocketTraitType = NetworkSocketTrait; - return std::make_unique>(address, options, bind); - } else if (type == Address::SocketType::Datagram) { - using NetworkSocketTraitType = NetworkSocketTrait; - return std::make_unique>(address, options, bind); + Address::Instance* getServerLoopbackAddress() { + if (version_ == Address::IpVersion::v4) { + return new Address::Ipv4Instance(Network::Test::getLoopbackAddressString(version_), + server_socket_->localAddress()->ip()->port()); } + return new Address::Ipv6Instance(Network::Test::getLoopbackAddressString(version_), + server_socket_->localAddress()->ip()->port()); + } - return nullptr; + SocketPtr createServerSocket(bool bind) { + return std::make_unique>>( + Network::Test::getAnyAddress(version_), nullptr, bind); } + + SocketPtr createClientSocket(bool bind) { + return std::make_unique>>( + Network::Test::getCanonicalLoopbackAddress(version_), nullptr, bind); + } + + // Validates receive data, source/destination address and received time. + void validateRecvCallbackParams(const UdpRecvData& data) { + ASSERT_NE(data.local_address_, nullptr); + + ASSERT_NE(data.peer_address_, nullptr); + ASSERT_NE(data.peer_address_->ip(), nullptr); + + EXPECT_EQ(data.local_address_->asString(), send_to_addr_->asString()); + + EXPECT_EQ(data.peer_address_->ip()->addressAsString(), + client_socket_->localAddress()->ip()->addressAsString()); + + EXPECT_EQ(*data.local_address_, *send_to_addr_); + EXPECT_EQ(time_system_.monotonicTime(), data.receive_time_); + // Advance time so that next onData() should have different received time. + time_system_.sleep(std::chrono::milliseconds(100)); + } + + SocketPtr server_socket_; + SocketPtr client_socket_; + Address::InstanceConstSharedPtr send_to_addr_; + MockUdpListenerCallbacks listener_callbacks_; + std::unique_ptr listener_; }; + INSTANTIATE_TEST_CASE_P(IpVersions, UdpListenerImplTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); // Test that socket options are set after the listener is setup. TEST_P(UdpListenerImplTest, UdpSetListeningSocketOptionsSuccess) { - Network::MockListenerCallbacks listener_callbacks; - Network::MockConnectionHandler connection_handler; - - Network::UdpListenSocket socket(Network::Test::getCanonicalLoopbackAddress(version_), nullptr, - true); + MockUdpListenerCallbacks listener_callbacks; + Network::UdpListenSocket socket(Network::Test::getAnyAddress(version_), nullptr, true); std::shared_ptr option = std::make_shared(); socket.addOption(option); + EXPECT_CALL(*option, setOption(_, envoy::api::v2::core::SocketOption::STATE_BOUND)) + .WillOnce(Return(true)); + UdpListenerImpl listener(dispatcherImpl(), socket, listener_callbacks, + dispatcherImpl().timeSource()); + +#ifdef SO_RXQ_OVFL + // Verify that overflow detection is enabled. + int get_overflow = 0; + auto& os_syscalls = Api::OsSysCallsSingleton::get(); + socklen_t int_size = static_cast(sizeof(get_overflow)); + const Api::SysCallIntResult result = os_syscalls.getsockopt( + server_socket_->ioHandle().fd(), SOL_SOCKET, SO_RXQ_OVFL, &get_overflow, &int_size); + EXPECT_EQ(0, result.rc_); + EXPECT_EQ(1, get_overflow); +#endif } /** * Tests UDP listener for actual destination and data. */ TEST_P(UdpListenerImplTest, UseActualDstUdp) { - // Setup server socket - SocketPtr server_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, true); - - ASSERT_NE(server_socket, nullptr); - - // Setup callback handler and listener. - Network::MockUdpListenerCallbacks listener_callbacks; - Network::TestUdpListenerImpl listener(dispatcherImpl(), *server_socket, listener_callbacks); - - EXPECT_CALL(listener, doRecvFrom(_, _)) - .WillRepeatedly(Invoke([&](sockaddr_storage& peer_addr, socklen_t& addr_len) { - return listener.doRecvFrom_(peer_addr, addr_len); - })); - // Setup client socket. - SocketPtr client_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, false); + client_socket_ = createClientSocket(false); // We send 2 packets const std::string first("first"); @@ -100,44 +130,29 @@ TEST_P(UdpListenerImplTest, UseActualDstUdp) { void_pointer = static_cast(second.c_str()); Buffer::RawSlice second_slice{const_cast(void_pointer), second.length()}; - auto send_rc = client_socket->ioHandle().sendto(first_slice, 0, *server_socket->localAddress()); + auto send_rc = client_socket_->ioHandle().sendto(first_slice, 0, *send_to_addr_); ASSERT_EQ(send_rc.rc_, first.length()); - send_rc = client_socket->ioHandle().sendto(second_slice, 0, *server_socket->localAddress()); + send_rc = client_socket_->ioHandle().sendto(second_slice, 0, *send_to_addr_); ASSERT_EQ(send_rc.rc_, second.length()); - auto validateCallParams = [&](Address::InstanceConstSharedPtr local_address, - Address::InstanceConstSharedPtr peer_address) { - ASSERT_NE(local_address, nullptr); - - ASSERT_NE(peer_address, nullptr); - ASSERT_NE(peer_address->ip(), nullptr); - - EXPECT_EQ(local_address->asString(), server_socket->localAddress()->asString()); - - EXPECT_EQ(peer_address->ip()->addressAsString(), - client_socket->localAddress()->ip()->addressAsString()); - - EXPECT_EQ(*local_address, *server_socket->localAddress()); - }; - - EXPECT_CALL(listener_callbacks, onData_(_)) + EXPECT_CALL(listener_callbacks_, onData_(_)) .WillOnce(Invoke([&](const UdpRecvData& data) -> void { - validateCallParams(data.local_address_, data.peer_address_); + validateRecvCallbackParams(data); EXPECT_EQ(data.buffer_->toString(), first); })) .WillOnce(Invoke([&](const UdpRecvData& data) -> void { - validateCallParams(data.local_address_, data.peer_address_); + validateRecvCallbackParams(data); EXPECT_EQ(data.buffer_->toString(), second); dispatcher_->exit(); })); - EXPECT_CALL(listener_callbacks, onWriteReady_(_)) + EXPECT_CALL(listener_callbacks_, onWriteReady_(_)) .WillRepeatedly(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket->ioHandle().fd()); + EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); })); dispatcher_->run(Event::Dispatcher::RunType::Block); @@ -147,26 +162,8 @@ TEST_P(UdpListenerImplTest, UseActualDstUdp) { * Tests UDP listener for read and write callbacks with actual data. */ TEST_P(UdpListenerImplTest, UdpEcho) { - // Setup server socket - SocketPtr server_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, true); - - ASSERT_NE(server_socket, nullptr); - - // Setup callback handler and listener. - Network::MockUdpListenerCallbacks listener_callbacks; - Network::TestUdpListenerImpl listener(dispatcherImpl(), *server_socket, listener_callbacks); - - EXPECT_CALL(listener, doRecvFrom(_, _)) - .WillRepeatedly(Invoke([&](sockaddr_storage& peer_addr, socklen_t& addr_len) { - return listener.doRecvFrom_(peer_addr, addr_len); - })); - // Setup client socket. - SocketPtr client_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, false); + client_socket_ = createClientSocket(false); // We send 2 packets and expect it to echo. const std::string first("first"); @@ -176,39 +173,20 @@ TEST_P(UdpListenerImplTest, UdpEcho) { void_pointer = static_cast(second.c_str()); Buffer::RawSlice second_slice{const_cast(void_pointer), second.length()}; - auto send_rc = client_socket->ioHandle().sendto(first_slice, 0, *server_socket->localAddress()); + auto send_rc = client_socket_->ioHandle().sendto(first_slice, 0, *send_to_addr_); ASSERT_EQ(send_rc.rc_, first.length()); - send_rc = client_socket->ioHandle().sendto(second_slice, 0, *server_socket->localAddress()); + send_rc = client_socket_->ioHandle().sendto(second_slice, 0, *send_to_addr_); ASSERT_EQ(send_rc.rc_, second.length()); - auto validateCallParams = [&](Address::InstanceConstSharedPtr local_address, - Address::InstanceConstSharedPtr peer_address) { - ASSERT_NE(local_address, nullptr); - - ASSERT_NE(peer_address, nullptr); - ASSERT_NE(peer_address->ip(), nullptr); - - EXPECT_EQ(local_address->asString(), server_socket->localAddress()->asString()); - - EXPECT_EQ(peer_address->ip()->addressAsString(), - client_socket->localAddress()->ip()->addressAsString()); - - EXPECT_EQ(*local_address, *server_socket->localAddress()); - }; - - Event::TimerPtr timer = dispatcher_->createTimer([&] { dispatcher_->exit(); }); - - timer->enableTimer(std::chrono::milliseconds(2000)); - // For unit test purposes, we assume that the data was received in order. Address::InstanceConstSharedPtr test_peer_address; std::vector server_received_data; - EXPECT_CALL(listener_callbacks, onData_(_)) + EXPECT_CALL(listener_callbacks_, onData_(_)) .WillOnce(Invoke([&](const UdpRecvData& data) -> void { - validateCallParams(data.local_address_, data.peer_address_); + validateRecvCallbackParams(data); test_peer_address = data.peer_address_; @@ -218,7 +196,7 @@ TEST_P(UdpListenerImplTest, UdpEcho) { server_received_data.push_back(data_str); })) .WillOnce(Invoke([&](const UdpRecvData& data) -> void { - validateCallParams(data.local_address_, data.peer_address_); + validateRecvCallbackParams(data); const std::string data_str = data.buffer_->toString(); EXPECT_EQ(data_str, second); @@ -226,38 +204,38 @@ TEST_P(UdpListenerImplTest, UdpEcho) { server_received_data.push_back(data_str); })); - EXPECT_CALL(listener_callbacks, onWriteReady_(_)) - .WillRepeatedly(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket->ioHandle().fd()); - ASSERT_NE(test_peer_address, nullptr); - - for (const auto& data : server_received_data) { - const std::string::size_type data_size = data.length() + 1; - uint64_t total_sent = 0; - const void* void_data = static_cast(data.c_str() + total_sent); - Buffer::RawSlice slice{const_cast(void_data), data_size - total_sent}; - - do { - auto send_rc = - const_cast(&socket)->ioHandle().sendto(slice, 0, *test_peer_address); - - if (send_rc.ok()) { - total_sent += send_rc.rc_; - if (total_sent >= data_size) { - break; - } - } else if (send_rc.err_->getErrorCode() != Api::IoError::IoErrorCode::Again) { - break; - } - } while (((send_rc.rc_ == 0) && - (send_rc.err_->getErrorCode() == Api::IoError::IoErrorCode::Again)) || - (total_sent < data_size)); - - EXPECT_EQ(total_sent, data_size); + EXPECT_CALL(listener_callbacks_, onWriteReady_(_)).WillOnce(Invoke([&](const Socket& socket) { + EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); + ASSERT_NE(test_peer_address, nullptr); + + for (const auto& data : server_received_data) { + const std::string::size_type data_size = data.length() + 1; + uint64_t total_sent = 0; + const void* void_data = static_cast(data.c_str() + total_sent); + Buffer::RawSlice slice{const_cast(void_data), data_size - total_sent}; + + do { + auto send_rc = + const_cast(&socket)->ioHandle().sendto(slice, 0, *test_peer_address); + + if (send_rc.ok()) { + total_sent += send_rc.rc_; + if (total_sent >= data_size) { + break; + } + } else if (send_rc.err_->getErrorCode() != Api::IoError::IoErrorCode::Again) { + break; } + } while (((send_rc.rc_ == 0) && + (send_rc.err_->getErrorCode() == Api::IoError::IoErrorCode::Again)) || + (total_sent < data_size)); - server_received_data.clear(); - })); + EXPECT_EQ(total_sent, data_size); + } + + server_received_data.clear(); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); } @@ -266,28 +244,11 @@ TEST_P(UdpListenerImplTest, UdpEcho) { * Tests UDP listener's `enable` and `disable` APIs. */ TEST_P(UdpListenerImplTest, UdpListenerEnableDisable) { - // Setup server socket - SocketPtr server_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, true); - ASSERT_NE(server_socket, nullptr); - - auto const* server_ip = server_socket->localAddress()->ip(); + auto const* server_ip = server_socket_->localAddress()->ip(); ASSERT_NE(server_ip, nullptr); - // Setup callback handler and listener. - Network::MockUdpListenerCallbacks listener_callbacks; - Network::TestUdpListenerImpl listener(dispatcherImpl(), *server_socket, listener_callbacks); - - EXPECT_CALL(listener, doRecvFrom(_, _)) - .WillRepeatedly(Invoke([&](sockaddr_storage& peer_addr, socklen_t& addr_len) { - return listener.doRecvFrom_(peer_addr, addr_len); - })); - // Setup client socket. - SocketPtr client_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, false); + client_socket_ = createClientSocket(false); // We first disable the listener and then send two packets. // - With the listener disabled, we expect that none of the callbacks will be @@ -300,55 +261,36 @@ TEST_P(UdpListenerImplTest, UdpListenerEnableDisable) { void_pointer = static_cast(second.c_str()); Buffer::RawSlice second_slice{const_cast(void_pointer), second.length()}; - listener.disable(); + listener_->disable(); - auto send_rc = client_socket->ioHandle().sendto(first_slice, 0, *server_socket->localAddress()); + auto send_rc = client_socket_->ioHandle().sendto(first_slice, 0, *send_to_addr_); ASSERT_EQ(send_rc.rc_, first.length()); - send_rc = client_socket->ioHandle().sendto(second_slice, 0, *server_socket->localAddress()); + send_rc = client_socket_->ioHandle().sendto(second_slice, 0, *send_to_addr_); ASSERT_EQ(send_rc.rc_, second.length()); - auto validateCallParams = [&](Address::InstanceConstSharedPtr local_address, - Address::InstanceConstSharedPtr peer_address) { - ASSERT_NE(local_address, nullptr); - - ASSERT_NE(peer_address, nullptr); - ASSERT_NE(peer_address->ip(), nullptr); - - EXPECT_EQ(local_address->asString(), server_socket->localAddress()->asString()); - - EXPECT_EQ(peer_address->ip()->addressAsString(), - client_socket->localAddress()->ip()->addressAsString()); - - EXPECT_EQ(*local_address, *server_socket->localAddress()); - }; + EXPECT_CALL(listener_callbacks_, onData_(_)).Times(0); - Event::TimerPtr timer = dispatcher_->createTimer([&] { dispatcher_->exit(); }); - - timer->enableTimer(std::chrono::milliseconds(2000)); - - EXPECT_CALL(listener_callbacks, onData_(_)).Times(0); - - EXPECT_CALL(listener_callbacks, onWriteReady_(_)).Times(0); + EXPECT_CALL(listener_callbacks_, onWriteReady_(_)).Times(0); dispatcher_->run(Event::Dispatcher::RunType::Block); - listener.enable(); + listener_->enable(); - EXPECT_CALL(listener_callbacks, onData_(_)) + EXPECT_CALL(listener_callbacks_, onData_(_)) .Times(2) .WillOnce(Return()) .WillOnce(Invoke([&](const UdpRecvData& data) -> void { - validateCallParams(data.local_address_, data.peer_address_); + validateRecvCallbackParams(data); EXPECT_EQ(data.buffer_->toString(), second); dispatcher_->exit(); })); - EXPECT_CALL(listener_callbacks, onWriteReady_(_)) + EXPECT_CALL(listener_callbacks_, onWriteReady_(_)) .WillRepeatedly(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket->ioHandle().fd()); + EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); })); dispatcher_->run(Event::Dispatcher::RunType::Block); @@ -357,28 +299,11 @@ TEST_P(UdpListenerImplTest, UdpListenerEnableDisable) { /** * Tests UDP listener's error callback. */ -TEST_P(UdpListenerImplTest, UdpListenerRecvFromError) { - // Setup server socket - SocketPtr server_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, true); - - ASSERT_NE(server_socket, nullptr); - - auto const* server_ip = server_socket->localAddress()->ip(); +TEST_P(UdpListenerImplTest, UdpListenerRecvMsgError) { + auto const* server_ip = server_socket_->localAddress()->ip(); ASSERT_NE(server_ip, nullptr); - // Setup callback handler and listener. - Network::MockUdpListenerCallbacks listener_callbacks; - Network::TestUdpListenerImpl listener(dispatcherImpl(), *server_socket, listener_callbacks); - - EXPECT_CALL(listener, doRecvFrom(_, _)).WillRepeatedly(Invoke([&](sockaddr_storage&, socklen_t&) { - return UdpListenerImpl::ReceiveResult{{-1, -1}, nullptr}; - })); - - SocketPtr client_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, false); + client_socket_ = createClientSocket(false); // When the `receive` system call returns an error, we expect the `onReceiveError` // callback callwed with `SyscallError` parameter. @@ -386,25 +311,30 @@ TEST_P(UdpListenerImplTest, UdpListenerRecvFromError) { const void* void_pointer = static_cast(first.c_str()); Buffer::RawSlice first_slice{const_cast(void_pointer), first.length()}; - auto send_rc = client_socket->ioHandle().sendto(first_slice, 0, *server_socket->localAddress()); + auto send_rc = client_socket_->ioHandle().sendto(first_slice, 0, *send_to_addr_); ASSERT_EQ(send_rc.rc_, first.length()); - EXPECT_CALL(listener_callbacks, onData_(_)).Times(0); + EXPECT_CALL(listener_callbacks_, onData_(_)).Times(0); - EXPECT_CALL(listener_callbacks, onWriteReady_(_)) + EXPECT_CALL(listener_callbacks_, onWriteReady_(_)) .Times(1) .WillRepeatedly(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket->ioHandle().fd()); + EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); })); - EXPECT_CALL(listener_callbacks, onReceiveError_(_, _)) + EXPECT_CALL(listener_callbacks_, onReceiveError_(_, _)) .Times(1) - .WillOnce(Invoke([&](const UdpListenerCallbacks::ErrorCode& err_code, int err) -> void { - ASSERT_EQ(err_code, UdpListenerCallbacks::ErrorCode::SyscallError); - ASSERT_EQ(err, -1); + .WillOnce(Invoke([&](const UdpListenerCallbacks::ErrorCode& err_code, + Api::IoError::IoErrorCode err) -> void { + ASSERT_EQ(UdpListenerCallbacks::ErrorCode::SyscallError, err_code); + ASSERT_EQ(Api::IoError::IoErrorCode::NoSupport, err); dispatcher_->exit(); })); + // Inject mocked OsSysCalls implementation to mock a read failure. + Api::MockOsSysCalls os_sys_calls; + TestThreadsafeSingletonInjector os_calls(&os_sys_calls); + EXPECT_CALL(os_sys_calls, recvmsg(_, _, _)).WillOnce(Return(Api::SysCallSizeResult{-1, ENOTSUP})); dispatcher_->run(Event::Dispatcher::RunType::Block); } @@ -415,37 +345,23 @@ TEST_P(UdpListenerImplTest, UdpListenerRecvFromError) { * 2. Send the data from the udp listener to the client socket and validate the contents */ TEST_P(UdpListenerImplTest, SendData) { - // Setup server socket - SocketPtr server_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, true); - ASSERT_NE(server_socket, nullptr); - - // Setup server callback handler and listener. - Network::MockUdpListenerCallbacks server_listener_callbacks; - Network::TestUdpListenerImpl server_listener(dispatcherImpl(), *server_socket, - server_listener_callbacks); - // Setup client socket. - SocketPtr client_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, true); - ASSERT_NE(client_socket, nullptr); + client_socket_ = createClientSocket(true); + ASSERT_NE(client_socket_, nullptr); const std::string payload("hello world"); Buffer::InstancePtr buffer(new Buffer::OwnedImpl()); buffer->add(payload); - UdpSendData send_data{client_socket->localAddress(), *buffer}; + UdpSendData send_data{client_socket_->localAddress(), *buffer}; - auto send_result = server_listener.send(send_data); + auto send_result = listener_->send(send_data); EXPECT_EQ(send_result.ok(), true); // This is trigerred on opening the listener on registering the event - EXPECT_CALL(server_listener_callbacks, onWriteReady_(_)) - .WillOnce(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket->ioHandle().fd()); - })); + EXPECT_CALL(listener_callbacks_, onWriteReady_(_)).WillOnce(Invoke([&](const Socket& socket) { + EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); + })); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); @@ -456,7 +372,7 @@ TEST_P(UdpListenerImplTest, SendData) { do { Api::IoCallUint64Result result = - result_buffer->read(client_socket->ioHandle(), bytes_to_read - bytes_read); + result_buffer->read(client_socket_->ioHandle(), bytes_to_read - bytes_read); if (result.ok()) { bytes_read += result.rc_; @@ -479,33 +395,24 @@ TEST_P(UdpListenerImplTest, SendData) { * The send fails because the server_socket is created with bind=false. */ TEST_P(UdpListenerImplTest, SendDataError) { - // Setup server socket - SocketPtr server_socket = - getSocket(Address::SocketType::Datagram, Network::Test::getCanonicalLoopbackAddress(version_), - nullptr, false /*do not bind the socket so that the send fails*/); - ASSERT_NE(server_socket, nullptr); - - // Setup server callback handler and listener. - Network::MockUdpListenerCallbacks server_listener_callbacks; - Network::TestUdpListenerImpl server_listener(dispatcherImpl(), *server_socket, - server_listener_callbacks); - const std::string payload("hello world"); Buffer::InstancePtr buffer(new Buffer::OwnedImpl()); buffer->add(payload); // send data to itself - UdpSendData send_data{server_socket->localAddress(), *buffer}; + UdpSendData send_data{server_socket_->localAddress(), *buffer}; // This is trigerred on opening the listener on registering the event - EXPECT_CALL(server_listener_callbacks, onWriteReady_(_)) - .WillOnce(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket->ioHandle().fd()); - })); - - auto send_result = server_listener.send(send_data); + EXPECT_CALL(listener_callbacks_, onWriteReady_(_)).WillOnce(Invoke([&](const Socket& socket) { + EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); + })); + // Inject mocked OsSysCalls implementation to mock a write failure. + Api::MockOsSysCalls os_sys_calls; + TestThreadsafeSingletonInjector os_calls(&os_sys_calls); + EXPECT_CALL(os_sys_calls, sendmsg(_, _, _)).WillOnce(Return(Api::SysCallSizeResult{-1, ENOTSUP})); + auto send_result = listener_->send(send_data); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); - EXPECT_EQ(send_result.ok(), false); - EXPECT_EQ(send_result.err_->getErrorCode(), Api::IoError::IoErrorCode::UnknownError); + EXPECT_FALSE(send_result.ok()); + EXPECT_EQ(send_result.err_->getErrorCode(), Api::IoError::IoErrorCode::NoSupport); dispatcher_->exit(); } diff --git a/test/mocks/api/mocks.h b/test/mocks/api/mocks.h index 592f7b68675f..4bda9755288a 100644 --- a/test/mocks/api/mocks.h +++ b/test/mocks/api/mocks.h @@ -61,10 +61,12 @@ class MockOsSysCalls : public OsSysCallsImpl { MOCK_METHOD3(ioctl, SysCallIntResult(int sockfd, unsigned long int request, void* argp)); MOCK_METHOD1(close, SysCallIntResult(int)); MOCK_METHOD3(writev, SysCallSizeResult(int, const iovec*, int)); + MOCK_METHOD3(sendmsg, SysCallSizeResult(int fd, const msghdr* message, int flags)); MOCK_METHOD3(readv, SysCallSizeResult(int, const iovec*, int)); MOCK_METHOD4(recv, SysCallSizeResult(int socket, void* buffer, size_t length, int flags)); MOCK_METHOD6(recvfrom, SysCallSizeResult(int sockfd, void* buffer, size_t length, int flags, struct sockaddr* addr, socklen_t* addrlen)); + MOCK_METHOD3(recvmsg, SysCallSizeResult(int socket, struct msghdr* msg, int flags)); MOCK_METHOD2(ftruncate, SysCallIntResult(int fd, off_t length)); MOCK_METHOD6(mmap, SysCallPtrResult(void* addr, size_t length, int prot, int flags, int fd, off_t offset)); diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 49e70c6afc88..80fec6e54c42 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -144,7 +144,7 @@ class MockUdpListenerCallbacks : public UdpListenerCallbacks { void onWriteReady(const Socket& socket) override { onWriteReady_(socket); } - void onReceiveError(const ErrorCode& err_code, int err) override { + void onReceiveError(const ErrorCode& err_code, Api::IoError::IoErrorCode err) override { onReceiveError_(err_code, err); } @@ -152,7 +152,7 @@ class MockUdpListenerCallbacks : public UdpListenerCallbacks { MOCK_METHOD1(onWriteReady_, void(const Socket& socket)); - MOCK_METHOD2(onReceiveError_, void(const ErrorCode& err_code, int err)); + MOCK_METHOD2(onReceiveError_, void(const ErrorCode& err_code, Api::IoError::IoErrorCode err)); }; class MockDrainDecision : public DrainDecision { diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index a35196a81fd1..be56ec99eee7 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -382,8 +382,11 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, UdpAddress) { EXPECT_CALL(server_.random_, uuid()); EXPECT_CALL(listener_factory_, createListenSocket(_, Network::Address::SocketType::Datagram, _, true)); + NiceMock os_sys_calls; + TestThreadsafeSingletonInjector os_calls(&os_sys_calls); + EXPECT_CALL(os_sys_calls, setsockopt_(_, _, _, _, _)).Times(testing::AtLeast(1)); manager_->addOrUpdateListener(listener_proto, "", true); - EXPECT_EQ(1U, manager_->listeners().size()); + EXPECT_EQ(1u, manager_->listeners().size()); } TEST_F(ListenerManagerImplWithRealFiltersTest, BadListenerConfig) { diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index e1a53572eccf..33786deb4df1 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -179,6 +179,7 @@ OSS OSX OT OU +OVFL PB PC PCC @@ -198,6 +199,8 @@ QUIC RAII RANLUX RBAC +RECVDSTADDR +RECVPKTINFO RCU RDN RDS @@ -217,12 +220,14 @@ RTDS RTTI RW RX +RXQ Runn SA SAN SANs SDK SDS +SENDSRCADDR SHA SHM SIBABRT @@ -386,6 +391,7 @@ cstring ctor ctrl customizations +darwin dbg de dechunking From 7816d5e4baa1dee7558572393c83fa1d96a5052a Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Sun, 23 Jun 2019 19:34:25 -0700 Subject: [PATCH 052/542] admin: show hostname in clusters output (#7362) Part of https://github.com/envoyproxy/envoy/issues/1606 Risk Level: Low Testing: Modified UT Docs Changes: N/A Release Notes: Added Signed-off-by: Matt Klein --- api/envoy/admin/v2alpha/clusters.proto | 3 +++ docs/root/intro/version_history.rst | 2 ++ source/server/http/admin.cc | 3 +++ test/server/http/admin_test.cc | 37 ++++++++++++++++++++++---- 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/api/envoy/admin/v2alpha/clusters.proto b/api/envoy/admin/v2alpha/clusters.proto index b74ace25b5cc..885f6b3034a6 100644 --- a/api/envoy/admin/v2alpha/clusters.proto +++ b/api/envoy/admin/v2alpha/clusters.proto @@ -65,6 +65,9 @@ message HostStatus { // The host's weight. If not configured, the value defaults to 1. uint32 weight = 5; + + // The hostname of the host, if applicable. + string hostname = 6; } // Health status for a host. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b666d151d989..1fbf7bf8b71e 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -10,6 +10,8 @@ Version history * admin: the administration interface now includes a :ref:`/ready endpoint ` for easier readiness checks. * admin: extend :ref:`/runtime_modify endpoint ` to support parameters within the request body. * admin: the :ref:`/listener endpoint ` now returns :ref:`listeners.proto` which includes listener names and ports. +* admin: the :ref:`/clusters endpoint ` now shows hostname + for each host, useful for DNS based clusters. * api: track and report requests issued since last load report. * build: releases are built with Clang and linked with LLD. * control-plane: management servers can respond with HTTP 304 to indicate that config is up to date for Envoy proxies polling a :ref:`REST API Config Type ` diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 256c3542633b..6d94bbd162c0 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -353,6 +353,7 @@ void AdminImpl::writeClustersAsJson(Buffer::Instance& response) { envoy::admin::v2alpha::HostStatus& host_status = *cluster_status.add_host_statuses(); Network::Utility::addressToProtobufAddress(*host->address(), *host_status.mutable_address()); + host_status.set_hostname(host->hostname()); std::vector sorted_counters; for (const Stats::CounterSharedPtr& counter : host->counters()) { sorted_counters.push_back(counter); @@ -438,6 +439,8 @@ void AdminImpl::writeClustersAsText(Buffer::Instance& response) { host->address()->asString(), stat.first, stat.second)); } + response.add(fmt::format("{}::{}::hostname::{}\n", cluster.second.get().info()->name(), + host->address()->asString(), host->hostname())); response.add(fmt::format("{}::{}::health_flags::{}\n", cluster.second.get().info()->name(), host->address()->asString(), Upstream::HostUtility::healthFlagsToString(*host))); diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index 5955c1619350..64639c23f9e7 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -1064,6 +1064,8 @@ TEST_P(AdminInstanceTest, ClustersJson) { Network::Address::InstanceConstSharedPtr address = Network::Utility::resolveUrl("tcp://1.2.3.4:80"); ON_CALL(*host, address()).WillByDefault(Return(address)); + const std::string hostname = "foo.com"; + ON_CALL(*host, hostname()).WillByDefault(ReturnRef(hostname)); // Add stats in random order and validate that they come in order. Stats::IsolatedStoreImpl store; @@ -1152,7 +1154,8 @@ TEST_P(AdminInstanceTest, ClustersJson) { "success_rate": { "value": 43.2 }, - "weight": 5 + "weight": 5, + "hostname": "foo.com" } ] } @@ -1167,10 +1170,34 @@ TEST_P(AdminInstanceTest, ClustersJson) { EXPECT_THAT(output_proto, ProtoEq(expected_proto)); // Ensure that the normal text format is used by default. - EXPECT_EQ(Http::Code::OK, getCallback("/clusters", header_map, response)); - std::string text_output = response.toString(); - envoy::admin::v2alpha::Clusters failed_conversion_proto; - EXPECT_THROW(TestUtility::loadFromJson(text_output, failed_conversion_proto), EnvoyException); + Buffer::OwnedImpl response2; + EXPECT_EQ(Http::Code::OK, getCallback("/clusters", header_map, response2)); + const std::string expected_text = R"EOF(fake_cluster::outlier::success_rate_average::0 +fake_cluster::outlier::success_rate_ejection_threshold::6 +fake_cluster::default_priority::max_connections::1 +fake_cluster::default_priority::max_pending_requests::1024 +fake_cluster::default_priority::max_requests::1024 +fake_cluster::default_priority::max_retries::1 +fake_cluster::high_priority::max_connections::1 +fake_cluster::high_priority::max_pending_requests::1024 +fake_cluster::high_priority::max_requests::1024 +fake_cluster::high_priority::max_retries::1 +fake_cluster::added_via_api::true +fake_cluster::1.2.3.4:80::arest_counter::5 +fake_cluster::1.2.3.4:80::atest_gauge::10 +fake_cluster::1.2.3.4:80::rest_counter::10 +fake_cluster::1.2.3.4:80::test_counter::10 +fake_cluster::1.2.3.4:80::test_gauge::11 +fake_cluster::1.2.3.4:80::hostname::foo.com +fake_cluster::1.2.3.4:80::health_flags::/failed_active_hc/failed_outlier_check/degraded_active_hc/degraded_eds_health/pending_dynamic_removal +fake_cluster::1.2.3.4:80::weight::5 +fake_cluster::1.2.3.4:80::region::test_region +fake_cluster::1.2.3.4:80::zone::test_zone +fake_cluster::1.2.3.4:80::sub_zone::test_sub_zone +fake_cluster::1.2.3.4:80::canary::false +fake_cluster::1.2.3.4:80::success_rate::43.2 +)EOF"; + EXPECT_EQ(expected_text, response2.toString()); } TEST_P(AdminInstanceTest, GetRequest) { From 400edea12239ffdb8a1e35cb13a23fb3f0494ab8 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Mon, 24 Jun 2019 19:25:56 +0530 Subject: [PATCH 053/542] refactor test methods (#7372) Signed-off-by: Rama Chavali --- test/integration/server.h | 12 +++--------- test/test_common/utility.cc | 21 +++++++++++++++++++++ test/test_common/utility.h | 30 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/test/integration/server.h b/test/integration/server.h index 14a6a3191790..1b77f511fbae 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -261,21 +261,15 @@ class IntegrationTestServer : public Logger::Loggable, } void waitForCounterGe(const std::string& name, uint64_t value) override { - while (counter(name) == nullptr || counter(name)->value() < value) { - time_system_.sleep(std::chrono::milliseconds(10)); - } + TestUtility::waitForCounterGe(stat_store(), name, value, time_system_); } void waitForGaugeGe(const std::string& name, uint64_t value) override { - while (gauge(name) == nullptr || gauge(name)->value() < value) { - time_system_.sleep(std::chrono::milliseconds(10)); - } + TestUtility::waitForGaugeGe(stat_store(), name, value, time_system_); } void waitForGaugeEq(const std::string& name, uint64_t value) override { - while (gauge(name) == nullptr || gauge(name)->value() != value) { - time_system_.sleep(std::chrono::milliseconds(10)); - } + TestUtility::waitForGaugeEq(stat_store(), name, value, time_system_); } Stats::CounterSharedPtr counter(const std::string& name) override { diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 7f1e4e64eafd..bd3392da5b0f 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -160,6 +160,27 @@ void TestUtility::waitForCounterEq(Stats::Store& store, const std::string& name, } } +void TestUtility::waitForCounterGe(Stats::Store& store, const std::string& name, uint64_t value, + Event::TestTimeSystem& time_system) { + while (findCounter(store, name) == nullptr || findCounter(store, name)->value() < value) { + time_system.sleep(std::chrono::milliseconds(10)); + } +} + +void TestUtility::waitForGaugeGe(Stats::Store& store, const std::string& name, uint64_t value, + Event::TestTimeSystem& time_system) { + while (findGauge(store, name) == nullptr || findGauge(store, name)->value() < value) { + time_system.sleep(std::chrono::milliseconds(10)); + } +} + +void TestUtility::waitForGaugeEq(Stats::Store& store, const std::string& name, uint64_t value, + Event::TestTimeSystem& time_system) { + while (findGauge(store, name) == nullptr || findGauge(store, name)->value() != value) { + time_system.sleep(std::chrono::milliseconds(10)); + } +} + std::list TestUtility::makeDnsResponse(const std::list& addresses) { std::list ret; diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 57e9eadb7072..8f1bdcc721c3 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -189,6 +189,36 @@ class TestUtility { static void waitForCounterEq(Stats::Store& store, const std::string& name, uint64_t value, Event::TestTimeSystem& time_system); + /** + * Wait for a counter to >= a given value. + * @param store supplies the stats store. + * @param name counter name. + * @param value target value. + * @param time_system the time system to use for waiting. + */ + static void waitForCounterGe(Stats::Store& store, const std::string& name, uint64_t value, + Event::TestTimeSystem& time_system); + + /** + * Wait for a gauge to >= a given value. + * @param store supplies the stats store. + * @param name gauge name. + * @param value target value. + * @param time_system the time system to use for waiting. + */ + static void waitForGaugeGe(Stats::Store& store, const std::string& name, uint64_t value, + Event::TestTimeSystem& time_system); + + /** + * Wait for a gauge to == a given value. + * @param store supplies the stats store. + * @param name gauge name. + * @param value target value. + * @param time_system the time system to use for waiting. + */ + static void waitForGaugeEq(Stats::Store& store, const std::string& name, uint64_t value, + Event::TestTimeSystem& time_system); + /** * Convert a string list of IP addresses into a list of network addresses usable for DNS * response testing. From cca7c80d4991964e84b656ab10ff244246b6d988 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 24 Jun 2019 11:25:22 -0400 Subject: [PATCH 054/542] fuzz: remove invalid characters from cluster header names (#7301) Signed-off-by: Asra Ali --- ...-testcase-route_fuzz_test-5088096376324096 | 275 ++++++++++++++++++ test/common/router/route_fuzz_test.cc | 33 ++- 2 files changed, 302 insertions(+), 6 deletions(-) create mode 100644 test/common/router/route_corpus/clusterfuzz-testcase-route_fuzz_test-5088096376324096 diff --git a/test/common/router/route_corpus/clusterfuzz-testcase-route_fuzz_test-5088096376324096 b/test/common/router/route_corpus/clusterfuzz-testcase-route_fuzz_test-5088096376324096 new file mode 100644 index 000000000000..4359d84c5ebe --- /dev/null +++ b/test/common/router/route_corpus/clusterfuzz-testcase-route_fuzz_test-5088096376324096 @@ -0,0 +1,275 @@ +config { + virtual_hosts { + name: "p" + domains: "b" + routes { + match { + path: "z" + } + route { + cluster_header: "\000\000\001\003" + prefix_rewrite: " " + cors { + allow_origin: "" + allow_headers: "b" + } + } + } + cors { + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "e" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "j" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "e" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + allow_origin: "b" + } + } +} diff --git a/test/common/router/route_fuzz_test.cc b/test/common/router/route_fuzz_test.cc index 0d9b6f7d0486..865c30f3f8aa 100644 --- a/test/common/router/route_fuzz_test.cc +++ b/test/common/router/route_fuzz_test.cc @@ -31,18 +31,39 @@ template T replaceInvalidHeaders(const T& config) { // Removes invalid headers from the RouteConfiguration as well as in each of the virtual hosts. envoy::api::v2::RouteConfiguration cleanRouteConfig(envoy::api::v2::RouteConfiguration route_config) { + // A route config contains a list of HTTP headers that should be added and/or removed to each + // request and/or response the connection manager routes. This removes invalid characters the + // headers. envoy::api::v2::RouteConfiguration clean_config = replaceInvalidHeaders(route_config); + // Replace invalid characters from auto internal_only_headers = clean_config.mutable_internal_only_headers(); std::for_each(internal_only_headers->begin(), internal_only_headers->end(), [](std::string& n) { n = Fuzz::replaceInvalidCharacters(n); }); - // Remove invalid characters in each virtual host. auto virtual_hosts = clean_config.mutable_virtual_hosts(); - std::for_each(virtual_hosts->begin(), virtual_hosts->end(), - [](envoy::api::v2::route::VirtualHost& virtual_host) { - virtual_host = - replaceInvalidHeaders(virtual_host); - }); + std::for_each( + virtual_hosts->begin(), virtual_hosts->end(), + [](envoy::api::v2::route::VirtualHost& virtual_host) { + // Each virtual host in the routing configuration contains a list of headers to add and/or + // remove from each request and response that get routed through it. This replaces invalid + // header characters in these fields. + virtual_host = replaceInvalidHeaders(virtual_host); + // Envoy can determine the cluster to route to by reading the HTTP header named by the + // cluster_header from the request header. Because these cluster_headers are destined to be + // added to a Header Map, we iterate through each route in and remove invalid characters + // from their cluster headers. + std::for_each(virtual_host.mutable_routes()->begin(), virtual_host.mutable_routes()->end(), + [](envoy::api::v2::route::Route& route) { + if (route.has_route()) { + route.mutable_route()->set_cluster_header( + Fuzz::replaceInvalidCharacters(route.route().cluster_header())); + if (route.route().cluster_header().empty()) { + route.mutable_route()->set_cluster_header("not-empty"); + } + } + }); + }); + return clean_config; } From ad57ed8511b636869afb3eef3c21b52890d71890 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Mon, 24 Jun 2019 09:28:37 -0700 Subject: [PATCH 055/542] clang-tidy: performance-unnecessary-copy-initialization (#7378) Signed-off-by: Derek Argueta --- .clang-tidy | 2 +- source/common/router/config_impl.cc | 2 +- source/common/router/config_utility.cc | 2 +- source/common/upstream/cluster_manager_impl.cc | 2 +- source/extensions/filters/common/lua/wrappers.cc | 4 ++-- .../http/grpc_json_transcoder/json_transcoder_filter.cc | 2 +- source/extensions/tracers/dynamic_ot/config.cc | 2 +- source/extensions/transport_sockets/alts/config.cc | 2 +- source/server/overload_manager_impl.cc | 2 +- test/common/network/address_impl_test.cc | 2 +- test/common/network/resolver_impl_test.cc | 2 +- test/common/upstream/load_balancer_impl_test.cc | 2 +- test/common/upstream/subset_lb_test.cc | 2 +- test/integration/load_stats_integration_test.cc | 4 ++-- test/test_common/logging.h | 2 +- test/tools/router_check/router.cc | 4 ++-- 16 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 8c632b671fa2..101180b43834 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,7 +1,7 @@ Checks: 'clang-diagnostic-*,clang-analyzer-*,abseil-*,bugprone-*,modernize-*,performance-*,readability-redundant-*,readability-braces-around-statements,readability-container-size-empty' #TODO(lizan): grow this list, fix possible warnings and make more checks as error -WarningsAsErrors: 'bugprone-assert-side-effect,bugprone-use-after-move,modernize-make-shared,modernize-make-unique,modernize-use-using,performance-faster-string-find,performance-for-range-copy,readability-braces-around-statements,readability-container-size-empty,readability-redundant-smartptr-get,readability-redundant-string-cstr' +WarningsAsErrors: 'bugprone-assert-side-effect,bugprone-use-after-move,modernize-make-shared,modernize-make-unique,modernize-use-using,performance-faster-string-find,performance-for-range-copy,performance-unnecessary-copy-initialization,readability-braces-around-statements,readability-container-size-empty,readability-redundant-smartptr-get,readability-redundant-string-cstr' CheckOptions: - key: bugprone-assert-side-effect.AssertMacros diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 6973cbc7b247..8fc29afc398d 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -81,7 +81,7 @@ RetryPolicyImpl::RetryPolicyImpl(const envoy::api::v2::route::RetryPolicy& retry retry_host_predicate_configs_.emplace_back(host_predicate.name(), std::move(config)); } - const auto retry_priority = retry_policy.retry_priority(); + const auto& retry_priority = retry_policy.retry_priority(); if (!retry_priority.name().empty()) { auto& factory = Envoy::Config::Utility::getAndCheckFactory( retry_priority.name()); diff --git a/source/common/router/config_utility.cc b/source/common/router/config_utility.cc index 2fb3fa682b57..2ca55c665e33 100644 --- a/source/common/router/config_utility.cc +++ b/source/common/router/config_utility.cc @@ -82,7 +82,7 @@ std::string ConfigUtility::parseDirectResponseBody(const envoy::api::v2::route:: return EMPTY_STRING; } const auto& body = route.direct_response().body(); - const std::string filename = body.filename(); + const std::string& filename = body.filename(); if (!filename.empty()) { if (!api.fileSystem().fileExists(filename)) { throw EnvoyException(fmt::format("response body file {} does not exist", filename)); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index e8d23e95b2c8..2a293afe4d5f 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -450,7 +450,7 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::api::v2::Cluster& clust // We don't allow updates to statically configured clusters in the main configuration. We check // both the warming clusters and the active clusters to see if we need an update or the update // should be blocked. - const std::string cluster_name = cluster.name(); + const std::string& cluster_name = cluster.name(); const auto existing_active_cluster = active_clusters_.find(cluster_name); const auto existing_warming_cluster = warming_clusters_.find(cluster_name); const uint64_t new_hash = MessageUtil::hash(cluster); diff --git a/source/extensions/filters/common/lua/wrappers.cc b/source/extensions/filters/common/lua/wrappers.cc index 399155a253e3..7b49f926a246 100644 --- a/source/extensions/filters/common/lua/wrappers.cc +++ b/source/extensions/filters/common/lua/wrappers.cc @@ -40,7 +40,7 @@ void MetadataMapHelper::setValue(lua_State* state, const ProtobufWkt::Value& val return lua_pushboolean(state, value.bool_value()); case ProtobufWkt::Value::kStringValue: { - const auto string_value = value.string_value(); + const auto& string_value = value.string_value(); return lua_pushstring(state, string_value.c_str()); } @@ -49,7 +49,7 @@ void MetadataMapHelper::setValue(lua_State* state, const ProtobufWkt::Value& val } case ProtobufWkt::Value::kListValue: { - const auto list = value.list_value(); + const auto& list = value.list_value(); const int values_size = list.values_size(); lua_createtable(state, values_size, 0); diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc index e21977ce5a80..7b00de163760 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc @@ -154,7 +154,7 @@ JsonTranscoderConfig::JsonTranscoderConfig( Protobuf::util::NewTypeResolverForDescriptorPool(Grpc::Common::typeUrlPrefix(), &descriptor_pool_)); - const auto print_config = proto_config.print_options(); + const auto& print_config = proto_config.print_options(); print_options_.add_whitespace = print_config.add_whitespace(); print_options_.always_print_primitive_fields = print_config.always_print_primitive_fields(); print_options_.always_print_enums_as_ints = print_config.always_print_enums_as_ints(); diff --git a/source/extensions/tracers/dynamic_ot/config.cc b/source/extensions/tracers/dynamic_ot/config.cc index fa4fc998f461..f5b4098a298e 100644 --- a/source/extensions/tracers/dynamic_ot/config.cc +++ b/source/extensions/tracers/dynamic_ot/config.cc @@ -18,7 +18,7 @@ DynamicOpenTracingTracerFactory::DynamicOpenTracingTracerFactory() Tracing::HttpTracerPtr DynamicOpenTracingTracerFactory::createHttpTracerTyped( const envoy::config::trace::v2::DynamicOtConfig& proto_config, Server::Instance& server) { - const std::string library = proto_config.library(); + const std::string& library = proto_config.library(); const std::string config = MessageUtil::getJsonStringFromMessage(proto_config.config()); Tracing::DriverPtr dynamic_driver = std::make_unique(server.stats(), library, config); diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index dd41e6fefc17..1549a2a33d77 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -82,7 +82,7 @@ Network::TransportSocketFactoryPtr createTransportSocketFactoryHelper( message); HandshakeValidator validator = createHandshakeValidator(config); - const std::string handshaker_service = config.handshaker_service(); + const std::string& handshaker_service = config.handshaker_service(); HandshakerFactory factory = [handshaker_service, is_upstream, alts_shared_state](Event::Dispatcher& dispatcher, diff --git a/source/server/overload_manager_impl.cc b/source/server/overload_manager_impl.cc index 39cf33921748..8de9f2ac61b5 100644 --- a/source/server/overload_manager_impl.cc +++ b/source/server/overload_manager_impl.cc @@ -120,7 +120,7 @@ OverloadManagerImpl::OverloadManagerImpl( } for (const auto& trigger : action.triggers()) { - const std::string resource = trigger.name(); + const std::string& resource = trigger.name(); if (resources_.find(resource) == resources_.end()) { throw EnvoyException( diff --git a/test/common/network/address_impl_test.cc b/test/common/network/address_impl_test.cc index b6aa8adfac9d..584b5906d722 100644 --- a/test/common/network/address_impl_test.cc +++ b/test/common/network/address_impl_test.cc @@ -451,7 +451,7 @@ class MixedAddressTest : public testing::TestWithParam<::testing::tuple(GetParam()); - TestCase rhs_case = ::testing::get<1>(GetParam()); + const TestCase& rhs_case = ::testing::get<1>(GetParam()); InstanceConstSharedPtr lhs = testCaseToInstance(lhs_case); InstanceConstSharedPtr rhs = testCaseToInstance(rhs_case); if (lhs_case == rhs_case) { diff --git a/test/common/network/resolver_impl_test.cc b/test/common/network/resolver_impl_test.cc index 250adf540ddf..f5fa8daf536a 100644 --- a/test/common/network/resolver_impl_test.cc +++ b/test/common/network/resolver_impl_test.cc @@ -85,7 +85,7 @@ class TestResolver : public Resolver { public: InstanceConstSharedPtr resolve(const envoy::api::v2::core::SocketAddress& socket_address) override { - const std::string logical = socket_address.address(); + const std::string& logical = socket_address.address(); const std::string physical = getPhysicalName(logical); const std::string port = getPort(socket_address); return InstanceConstSharedPtr{new MockResolvedAddress(fmt::format("{}:{}", logical, port), diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 5050fc4e1560..e9d26eb922cc 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -1175,7 +1175,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNoLocalLocality) { HostsPerLocalitySharedPtr upstream_hosts_per_locality = makeHostsPerLocality( {{makeTestHost(info_, "tcp://127.0.0.1:80")}, {makeTestHost(info_, "tcp://127.0.0.1:81")}}, true); - HostsPerLocalitySharedPtr local_hosts_per_locality = upstream_hosts_per_locality; + const HostsPerLocalitySharedPtr& local_hosts_per_locality = upstream_hosts_per_locality; hostSet().healthy_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index 44b64701f4f6..9ac37eca4514 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -38,7 +38,7 @@ class SubsetLoadBalancerDescribeMetadataTester { using MetadataVector = std::vector>; void test(std::string expected, const MetadataVector& metadata) { - SubsetLoadBalancer::SubsetMetadata subset_metadata(metadata); + const SubsetLoadBalancer::SubsetMetadata& subset_metadata(metadata); EXPECT_EQ(expected, lb_.get()->describeMetadata(subset_metadata)); } diff --git a/test/integration/load_stats_integration_test.cc b/test/integration/load_stats_integration_test.cc index de9a3e3434cb..00235f30478b 100644 --- a/test/integration/load_stats_integration_test.cc +++ b/test/integration/load_stats_integration_test.cc @@ -173,14 +173,14 @@ class LoadStatsIntegrationTest : public testing::TestWithParamset_total_dropped_requests(cluster_stats->total_dropped_requests() + local_cluster_stats.total_dropped_requests()); for (int i = 0; i < local_cluster_stats.upstream_locality_stats_size(); ++i) { - auto local_upstream_locality_stats = local_cluster_stats.upstream_locality_stats(i); + const auto& local_upstream_locality_stats = local_cluster_stats.upstream_locality_stats(i); bool copied = false; for (int j = 0; j < cluster_stats->upstream_locality_stats_size(); ++j) { auto* upstream_locality_stats = cluster_stats->mutable_upstream_locality_stats(j); diff --git a/test/test_common/logging.h b/test/test_common/logging.h index b56161c96979..c7e1fed70c9a 100644 --- a/test/test_common/logging.h +++ b/test/test_common/logging.h @@ -160,7 +160,7 @@ using ExpectedLogMessages = std::vector; Envoy::LogLevelSetter save_levels(spdlog::level::trace); \ Envoy::LogRecordingSink log_recorder(Envoy::Logger::Registry::getSink()); \ stmt; \ - const std::vector logs = log_recorder.messages(); \ + const std::vector& logs = log_recorder.messages(); \ ASSERT_EQ(0, logs.size()) << " Logs:\n " << absl::StrJoin(logs, " "); \ } while (false) diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index 081c1faa57cb..a8359d4a6cb4 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -169,11 +169,11 @@ bool RouterCheckTool::compareEntries(const std::string& expected_routes) { ToolConfig tool_config = ToolConfig::create(check_config); tool_config.route_ = config_->route(*tool_config.headers_, tool_config.random_value_); - std::string test_name = check_config.test_name(); + const std::string& test_name = check_config.test_name(); if (details_) { std::cout << test_name << std::endl; } - envoy::RouterCheckToolSchema::ValidationAssert validate = check_config.validate(); + const envoy::RouterCheckToolSchema::ValidationAssert& validate = check_config.validate(); using checkerFunc = std::function; From 62cf5d8abf94e7c3d0faafbc1cf2c372b3a60e63 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 24 Jun 2019 16:04:56 -0400 Subject: [PATCH 056/542] fuzz: fix filesystem crash (#7360) Possible fix for filesystem crash in server fuzz test due to #7339 Testing: Add corpus entry Issue https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=15414 Signed-off-by: Asra Ali --- ...clusterfuzz-testcase-server_fuzz_test-5691106634760192 | 0 test/test_common/environment.cc | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5691106634760192 diff --git a/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5691106634760192 b/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5691106634760192 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index 70050aaab309..8afd806cd331 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -43,7 +43,7 @@ std::string makeTempDir(char* name_template) { name_template, strerror(errno))); #ifdef __cpp_lib_filesystem std::filesystem::create_directories(dirname); -#elifdef __cpp_lib_experimental_filesystem +#elif defined __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories(dirname); #endif #else @@ -88,7 +88,7 @@ void TestEnvironment::createPath(const std::string& path) { // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. std::filesystem::create_directories(std::filesystem::path(path)); -#elifdef __cpp_lib_experimental_filesystem +#elif defined __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories(std::experimental::filesystem::path(path)); #else // No support on this system for std::filesystem or std::experimental::filesystem. @@ -101,7 +101,7 @@ void TestEnvironment::createParentPath(const std::string& path) { // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. std::filesystem::create_directories(std::filesystem::path(path).parent_path()); -#elifdef __cpp_lib_experimental_filesystem +#elif defined __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories( std::experimental::filesystem::path(path).parent_path()); #else @@ -119,7 +119,7 @@ void TestEnvironment::removePath(const std::string& path) { return; } std::filesystem::remove_all(std::filesystem::path(path)); -#elifdef __cpp_lib_experimental_filesystem +#elif defined __cpp_lib_experimental_filesystem if (!std::experimental::filesystem::exists(path)) { return; } From 2f287b84a4ca5b1539b90523e5c7393621cae012 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Tue, 25 Jun 2019 01:36:55 +0530 Subject: [PATCH 057/542] server: add server state stat (#7371) Signed-off-by: Rama Chavali --- docs/root/configuration/statistics.rst | 3 ++- docs/root/intro/version_history.rst | 1 + source/server/http/admin.cc | 33 ++++++++++++++------------ source/server/http/admin.h | 5 ++++ source/server/server.cc | 3 +++ source/server/server.h | 1 + test/server/server_test.cc | 1 + 7 files changed, 31 insertions(+), 16 deletions(-) diff --git a/docs/root/configuration/statistics.rst b/docs/root/configuration/statistics.rst index 5829244ed82f..f9ddd2005a60 100644 --- a/docs/root/configuration/statistics.rst +++ b/docs/root/configuration/statistics.rst @@ -15,8 +15,9 @@ Server related statistics are rooted at *server.* with following statistics: uptime, Gauge, Current server uptime in seconds concurrency, Gauge, Number of worker threads memory_allocated, Gauge, Current amount of allocated memory in bytes. Total of both new and old Envoy processes on hot restart. - memory_heap_size, Gauge, Current reserved heap size in bytes. New Envoy process heap size on hot restart. + memory_heap_size, Gauge, Current reserved heap size in bytes. New Envoy process heap size on hot restart. live, Gauge, "1 if the server is not currently draining, 0 otherwise" + state, Gauge, Current :ref:`State ` of the Server. parent_connections, Gauge, Total connections of the old Envoy process on hot restart total_connections, Gauge, Total connections of both new and old Envoy processes version, Gauge, Integer represented version number based on SCM revision diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 1fbf7bf8b71e..198cef464973 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -70,6 +70,7 @@ Version history * sandbox: added :ref:`CSRF sandbox `. * server: ``--define manual_stamp=manual_stamp`` was added to allow server stamping outside of binary rules. more info in the `bazel docs `_. +* server: added :ref:`Server State ` statistic. * tool: added :repo:`proto ` support for :ref:`router check tool ` tests. * tracing: add trace sampling configuration to the route, to override the route level. * upstream: added :ref:`upstream_cx_pool_overflow ` for the connection pool circuit breaker. diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 6d94bbd162c0..07c0bdc3232c 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -675,25 +675,13 @@ Http::Code AdminImpl::handlerResetCounters(absl::string_view, Http::HeaderMap&, return Http::Code::OK; } -envoy::admin::v2alpha::ServerInfo::State AdminImpl::serverState() { - switch (server_.initManager().state()) { - case Init::Manager::State::Uninitialized: - return envoy::admin::v2alpha::ServerInfo::PRE_INITIALIZING; - case Init::Manager::State::Initializing: - return envoy::admin::v2alpha::ServerInfo::INITIALIZING; - case Init::Manager::State::Initialized: - return server_.healthCheckFailed() ? envoy::admin::v2alpha::ServerInfo::DRAINING - : envoy::admin::v2alpha::ServerInfo::LIVE; - } - NOT_REACHED_GCOVR_EXCL_LINE; -} - Http::Code AdminImpl::handlerServerInfo(absl::string_view, Http::HeaderMap& headers, Buffer::Instance& response, AdminStream&) { time_t current_time = time(nullptr); envoy::admin::v2alpha::ServerInfo server_info; server_info.set_version(VersionInfo::version()); - server_info.set_state(serverState()); + server_info.set_state( + Utility::serverState(server_.initManager().state(), server_.healthCheckFailed())); server_info.mutable_uptime_current_epoch()->set_seconds(current_time - server_.startTimeCurrentEpoch()); @@ -709,7 +697,8 @@ Http::Code AdminImpl::handlerServerInfo(absl::string_view, Http::HeaderMap& head Http::Code AdminImpl::handlerReady(absl::string_view, Http::HeaderMap&, Buffer::Instance& response, AdminStream&) { - const envoy::admin::v2alpha::ServerInfo::State state = serverState(); + const envoy::admin::v2alpha::ServerInfo::State state = + Utility::serverState(server_.initManager().state(), server_.healthCheckFailed()); response.add(envoy::admin::v2alpha::ServerInfo_State_Name(state) + "\n"); Http::Code code = state == envoy::admin::v2alpha::ServerInfo_State_LIVE @@ -1436,5 +1425,19 @@ void AdminImpl::addListenerToHandler(Network::ConnectionHandler* handler) { } } +envoy::admin::v2alpha::ServerInfo::State Utility::serverState(Init::Manager::State state, + bool health_check_failed) { + switch (state) { + case Init::Manager::State::Uninitialized: + return envoy::admin::v2alpha::ServerInfo::PRE_INITIALIZING; + case Init::Manager::State::Initializing: + return envoy::admin::v2alpha::ServerInfo::INITIALIZING; + case Init::Manager::State::Initialized: + return health_check_failed ? envoy::admin::v2alpha::ServerInfo::DRAINING + : envoy::admin::v2alpha::ServerInfo::LIVE; + } + NOT_REACHED_GCOVR_EXCL_LINE; +} + } // namespace Server } // namespace Envoy diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 24477cd55385..2cc4b0a06427 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -38,6 +38,11 @@ namespace Envoy { namespace Server { +namespace Utility { +envoy::admin::v2alpha::ServerInfo::State serverState(Init::Manager::State state, + bool health_check_failed); +} // namespace Utility + class AdminInternalAddressConfig : public Http::InternalAddressConfig { bool isInternalAddress(const Network::Address::Instance&) const override { return false; } }; diff --git a/source/server/server.cc b/source/server/server.cc index a67c3e383507..8d2b5a58c8b6 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -22,6 +22,7 @@ #include "common/api/api_impl.h" #include "common/api/os_sys_calls_impl.h" #include "common/buffer/buffer_impl.h" +#include "common/common/enum_to_int.h" #include "common/common/mutex_tracer_impl.h" #include "common/common/utility.h" #include "common/common/version.h" @@ -193,6 +194,8 @@ void InstanceImpl::flushStatsInternal() { parent_stats.parent_connections_); server_stats_->days_until_first_cert_expiring_.set( sslContextManager().daysUntilFirstCertExpires()); + server_stats_->state_.set( + enumToInt(Utility::serverState(initManager().state(), !healthCheckFailed()))); InstanceUtil::flushMetricsToSinks(config_.statsSinks(), stats_store_); // TODO(ramaraochavali): consider adding different flush interval for histograms. if (stat_flush_timer_ != nullptr) { diff --git a/source/server/server.h b/source/server/server.h index b6a1f786358b..0c1614d42f22 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -58,6 +58,7 @@ namespace Server { GAUGE(memory_allocated, Accumulate) \ GAUGE(memory_heap_size, Accumulate) \ GAUGE(parent_connections, Accumulate) \ + GAUGE(state, NeverImport) \ GAUGE(total_connections, Accumulate) \ GAUGE(uptime, Accumulate) \ GAUGE(version, NeverImport) diff --git a/test/server/server_test.cc b/test/server/server_test.cc index b6235ca138e7..48ce4bcc7d11 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -303,6 +303,7 @@ TEST_P(ServerInstanceImplTest, StatsFlushWhenServerIsStillInitializing) { // Wait till stats are flushed to custom sink and validate that the actual flush happens. TestUtility::waitForCounterEq(stats_store_, "stats.flushed", 1, test_time_.timeSystem()); + EXPECT_EQ(3L, TestUtility::findGauge(stats_store_, "server.state")->value()); EXPECT_EQ(Init::Manager::State::Initializing, server_->initManager().state()); server_->dispatcher().post([&] { server_->shutdown(); }); From aeb3e95f1b204b3299b1983f220d8bb0e9ffc0af Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Mon, 24 Jun 2019 14:08:50 -0700 Subject: [PATCH 058/542] HCM: save 8 or 16 bytes for each http filter (#7375) Use bit field and adjust member order. Signed-off-by: Yuchen Dai --- source/common/http/conn_manager_impl.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 66f819d3abc3..1e5b1901421e 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -96,9 +96,9 @@ class ConnectionManagerImpl : Logger::Loggable, */ struct ActiveStreamFilterBase : public virtual StreamFilterCallbacks { ActiveStreamFilterBase(ActiveStream& parent, bool dual_filter) - : iteration_state_(IterationState::Continue), iterate_from_current_filter_(false), - parent_(parent), headers_continued_(false), continue_headers_continued_(false), - end_stream_(false), dual_filter_(dual_filter) {} + : parent_(parent), iteration_state_(IterationState::Continue), + iterate_from_current_filter_(false), headers_continued_(false), + continue_headers_continued_(false), end_stream_(false), dual_filter_(dual_filter) {} // Functions in the following block are called after the filter finishes processing // corresponding data. Those functions handle state updates and data storage (if needed) @@ -159,13 +159,13 @@ class ConnectionManagerImpl : Logger::Loggable, StopAllWatermark, // Iteration has stopped for all frame types, and following data should // be buffered until high watermark is reached. }; + ActiveStream& parent_; IterationState iteration_state_; // If the filter resumes iteration from a StopAllBuffer/Watermark state, the current filter // hasn't parsed data and trailers. As a result, the filter iteration should start with the // current filter instead of the next one. If true, filter iteration starts with the current // filter. Otherwise, starts with the next filter in the chain. - bool iterate_from_current_filter_; - ActiveStream& parent_; + bool iterate_from_current_filter_ : 1; bool headers_continued_ : 1; bool continue_headers_continued_ : 1; // If true, end_stream is called for this filter. From 3be027b1a661136ed2d9597f2e00ce035b6e3f1b Mon Sep 17 00:00:00 2001 From: Kun Zhang Date: Mon, 24 Jun 2019 14:59:15 -0700 Subject: [PATCH 059/542] orca: split cost and utilization into separate maps. (#7254) The cost and utilization are fundamentally different in the way they are defined. There is no benefit to stuff them together, and only causes confusion and difficulties in aggregation should anyone ever try to do it. Signed-off-by: Kun Zhang --- api/udpa/data/orca/v1/orca_load_report.proto | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/api/udpa/data/orca/v1/orca_load_report.proto b/api/udpa/data/orca/v1/orca_load_report.proto index f33f11dda950..5c2eda6cd267 100644 --- a/api/udpa/data/orca/v1/orca_load_report.proto +++ b/api/udpa/data/orca/v1/orca_load_report.proto @@ -14,22 +14,23 @@ import "validate/validate.proto"; message OrcaLoadReport { // CPU utilization expressed as a fraction of available CPU resources. This - // should be derived from a sample or measurement taken during the request. + // should be derived from the latest sample or measurement. double cpu_utilization = 1 [(validate.rules).double.gte = 0, (validate.rules).double.lte = 1]; // Memory utilization expressed as a fraction of available memory - // resources. This should be derived from a sample or measurement taken - // during the request. + // resources. This should be derived from the latest sample or measurement. double mem_utilization = 2 [(validate.rules).double.gte = 0, (validate.rules).double.lte = 1]; // Total RPS being served by an endpoint. This should cover all services that an endpoint is // responsible for. uint64 rps = 3; - // Application specific requests costs. Each value may be an absolute cost (e.g. - // 3487 bytes of storage) or utilization associated with the request, - // expressed as a fraction of total resources available. Utilization - // metrics should be derived from a sample or measurement taken - // during the request. - map request_cost_or_utilization = 4; + // Application specific requests costs. Each value is an absolute cost (e.g. 3487 bytes of + // storage) associated with the request. + map request_cost = 4; + + // Resource utilization values. Each value is expressed as a fraction of total resources + // available, derived from the latest sample or measurement. + map utilization = 5 + [(validate.rules).map.values.double.gte = 0, (validate.rules).map.values.double.lte = 1]; } From 01c7caedb85ea4d29227e9f9cf197ca98b0e4e31 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 24 Jun 2019 18:04:19 -0400 Subject: [PATCH 060/542] Make Envoy::Grpc::AsyncClient class mock friendly by virtualizing its methods (#7348) Envoy::Grpc::AsyncClient template class is currently not very mock friendly since its interface is not virtual. There is no downside for making it virtual and allowing mocking of gRPC clients that use this class. Risk Level: Low Testing: unit tests Docs Changes: N/A Release Notes: N/A Fixes #7346 Signed-off-by: Yan Avlasov --- source/common/grpc/typed_async_client.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/common/grpc/typed_async_client.h b/source/common/grpc/typed_async_client.h index 8b3df8efc3b8..910ba7c5731f 100644 --- a/source/common/grpc/typed_async_client.h +++ b/source/common/grpc/typed_async_client.h @@ -95,16 +95,17 @@ template class AsyncClient /* : public Raw public: AsyncClient() {} AsyncClient(RawAsyncClientPtr&& client) : client_(std::move(client)) {} + virtual ~AsyncClient() = default; - AsyncRequest* send(const Protobuf::MethodDescriptor& service_method, - const Protobuf::Message& request, AsyncRequestCallbacks& callbacks, - Tracing::Span& parent_span, - const absl::optional& timeout) { + virtual AsyncRequest* send(const Protobuf::MethodDescriptor& service_method, + const Protobuf::Message& request, + AsyncRequestCallbacks& callbacks, Tracing::Span& parent_span, + const absl::optional& timeout) { return Internal::sendUntyped(client_.get(), service_method, request, callbacks, parent_span, timeout); } - AsyncStream start(const Protobuf::MethodDescriptor& service_method, - AsyncStreamCallbacks& callbacks) { + virtual AsyncStream start(const Protobuf::MethodDescriptor& service_method, + AsyncStreamCallbacks& callbacks) { return AsyncStream(Internal::startUntyped(client_.get(), service_method, callbacks)); } From 4c344ca21f3f29e0e52f550db83e1225421ed351 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 25 Jun 2019 06:35:50 +0800 Subject: [PATCH 061/542] ci: use ci script and remote cache in Azure for Linux (#7334) Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 19 +++++++++++++++++-- ci/build_setup.sh | 4 ++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index 844e75387ccc..f770c57e925e 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -4,11 +4,26 @@ resources: image: envoyproxy/envoy-build:ec38ecb88fd1abe55ab1daa2f6bd239ffccc9d98 jobs: -- job: BuildEnvoy +- job: EnvoyFullTest timeoutInMinutes: 360 pool: vmImage: 'Ubuntu 16.04' container: envoy-build steps: - - script: bazel build //source/exe:envoy-static + # bazel.dev isn't for CI purpose but we use it here to experiment with Azure Pipeline + # as it builds faster, before Azure can provision larger VM or we can use RBE. + - script: ci/do_ci.sh bazel.dev + env: + ENVOY_SRCDIR: $(Build.SourcesDirectory) + BUILD_DIR: $(Build.StagingDirectory) + BAZEL_EXTRA_TEST_OPTIONS: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only" + BAZEL_REMOTE_CACHE: remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/testlogs/**/test.xml' + testRunTitle: 'bazel.dev' + searchFolder: $(Build.StagingDirectory) + condition: always() diff --git a/ci/build_setup.sh b/ci/build_setup.sh index a8dc27e51a6a..951dd56ed2ff 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -32,7 +32,7 @@ mkdir -p "${FAKE_HOME}" export HOME="${FAKE_HOME}" export PYTHONUSERBASE="${FAKE_HOME}" -export BUILD_DIR=/build +export BUILD_DIR=${BUILD_DIR:-/build} if [[ ! -d "${BUILD_DIR}" ]] then echo "${BUILD_DIR} mount missing - did you forget -v :${BUILD_DIR}? Creating." @@ -57,7 +57,7 @@ rm -f "${SENTINEL}" # Environment setup. export USER=bazel -export TEST_TMPDIR=/build/tmp +export TEST_TMPDIR=${BUILD_DIR}/tmp export BAZEL="bazel" if [[ -f "/etc/redhat-release" ]] From c390bbd11ff41b9a3f929f38ec56ee54673be37a Mon Sep 17 00:00:00 2001 From: cclauss Date: Tue, 25 Jun 2019 00:38:47 +0200 Subject: [PATCH 062/542] Use print() function in both Python 2 and Python 3 (#7383) Related to #4552 Legacy __print__ statements are syntax errors in Python 3 but __print()__ function works as expected in both Python 2 and Python 3. Signed-off-by: Christian Clauss cclauss@me.com Signed-off-by: cclauss --- tools/stack_decode.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/stack_decode.py b/tools/stack_decode.py index eeb4143fc471..3e3c7bd87e7a 100755 --- a/tools/stack_decode.py +++ b/tools/stack_decode.py @@ -11,6 +11,7 @@ # In each case this script will add file and line information to any backtrace log # lines found and echo back all non-Backtrace lines untouched. +from __future__ import print_function import collections import re import subprocess @@ -73,6 +74,6 @@ def trim_proc_cwd(file_and_line_number): rununder.wait() sys.exit(rununder.returncode) # Pass back test pass/fail result else: - print "Usage (execute subprocess): stack_decode.py executable_file [additional args]" - print "Usage (read from stdin): stack_decode.py -s executable_file" + print("Usage (execute subprocess): stack_decode.py executable_file [additional args]") + print("Usage (read from stdin): stack_decode.py -s executable_file") sys.exit(1) From 2a31ba2834f3934bada62494ccb8a3b4345f5fb1 Mon Sep 17 00:00:00 2001 From: cclauss Date: Tue, 25 Jun 2019 00:48:29 +0200 Subject: [PATCH 063/542] Use print() function in both Python 2 and Python 3 (#7384) Related to #4552 Legacy __print__ statements are syntax errors in Python 3 but __print()__ function works as expected in both Python 2 and Python 3. Signed-off-by: Christian Clauss cclauss@me.com Signed-off-by: cclauss --- tools/deprecate_features/deprecate_features.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/deprecate_features/deprecate_features.py b/tools/deprecate_features/deprecate_features.py index 3c88b1f2215e..cf9f89f945fc 100644 --- a/tools/deprecate_features/deprecate_features.py +++ b/tools/deprecate_features/deprecate_features.py @@ -1,5 +1,6 @@ # A simple script to snag deprecated proto fields and add them to runtime_features.h +from __future__ import print_function import re import subprocess import fileinput @@ -19,7 +20,7 @@ def deprecate_proto(): if match: filenames_and_fields.add(tuple([match.group(1), match.group(2)])) else: - print 'no match in ' + line + ' please address manually!' + print('no match in ' + line + ' please address manually!') # Now discard any deprecated features already listed in runtime_features exiting_deprecated_regex = re.compile(r'.*"envoy.deprecated_features.(.*):(.*)",.*') @@ -59,7 +60,7 @@ def flip_runtime_features(): if match: features_to_flip.add(match.group(1)) else: - print 'no match in ' + line + ' please address manually!' + print('no match in ' + line + ' please address manually!') # Exempt the two test flags. features_to_flip.remove('envoy.reloadable_features.my_feature_name') @@ -85,8 +86,8 @@ def flip_runtime_features(): email = ('The Envoy maintainer team is cutting the next Envoy release. In the new release ' + runtime_email + deprecate_email) -print '\n\nSuggested envoy-announce email: \n' -print email +print('\n\nSuggested envoy-announce email: \n') +print(email) if not raw_input('Apply relevant runtime changes? [yN] ').strip().lower() in ('y', 'yes'): exit(1) @@ -96,6 +97,6 @@ def flip_runtime_features(): line = line.replace(line, line + runtime_features_code) if 'envoy.deprecated_features.deprecated.proto:is_deprecated_fatal' in line: line = line.replace(line, line + deprecate_code) - print line, + print(line, end='') -print '\nChanges applied. Please send the email above to envoy-announce.\n' +print('\nChanges applied. Please send the email above to envoy-announce.\n') From 5ee70cb19ca3bb6d817bae497b63c2a250d2d16b Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 25 Jun 2019 07:05:54 +0800 Subject: [PATCH 064/542] grpc: simplify GoogleGrpcUtils::makeBufferInstance (#7290) Simplifying GoogleGrpcUtils::makeBufferInstance by leveraging ref count of grpc_slice. Risk Level: Low Testing: existing tests Docs Changes: Release Notes: Fixes #7266. Signed-off-by: Lizan Zhou --- source/common/grpc/google_grpc_utils.cc | 39 +++++++++---------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/source/common/grpc/google_grpc_utils.cc b/source/common/grpc/google_grpc_utils.cc index 484c68290e63..cba62fc4722a 100644 --- a/source/common/grpc/google_grpc_utils.cc +++ b/source/common/grpc/google_grpc_utils.cc @@ -62,12 +62,17 @@ grpc::ByteBuffer GoogleGrpcUtils::makeByteBuffer(Buffer::InstancePtr&& buffer_in return {&slices[0], slices.size()}; } -struct ByteBufferContainer { - ByteBufferContainer(int ref_count) : ref_count_(ref_count) {} - ~ByteBufferContainer() { ::free(fragments_); } - uint32_t ref_count_; - Buffer::BufferFragmentImpl* fragments_ = nullptr; - std::vector slices_; +class GrpcSliceBufferFragmentImpl : public Buffer::BufferFragment { +public: + explicit GrpcSliceBufferFragmentImpl(grpc::Slice&& slice) : slice_(std::move(slice)) {} + + // Buffer::BufferFragment + const void* data() const override { return slice_.begin(); } + size_t size() const override { return slice_.size(); } + void done() override { delete this; } + +private: + const grpc::Slice slice_; }; Buffer::InstancePtr GoogleGrpcUtils::makeBufferInstance(const grpc::ByteBuffer& byte_buffer) { @@ -81,28 +86,10 @@ Buffer::InstancePtr GoogleGrpcUtils::makeBufferInstance(const grpc::ByteBuffer& if (!byte_buffer.Dump(&slices).ok()) { return nullptr; } - auto* container = new ByteBufferContainer(static_cast(slices.size())); - std::function releaser = - [container](const void*, size_t, const Buffer::BufferFragmentImpl*) { - container->ref_count_--; - if (container->ref_count_ <= 0) { - delete container; - } - }; - // NB: addBufferFragment takes a pointer alias to the BufferFragmentImpl which is passed in so we - // need to ensure that the lifetime of those objects exceeds that of the Buffer::Instance. - RELEASE_ASSERT(!::posix_memalign(reinterpret_cast(&container->fragments_), - alignof(Buffer::BufferFragmentImpl), - sizeof(Buffer::BufferFragmentImpl) * slices.size()), - "posix_memalign failure"); - for (size_t i = 0; i < slices.size(); i++) { - new (&container->fragments_[i]) - Buffer::BufferFragmentImpl(slices[i].begin(), slices[i].size(), releaser); - } + for (size_t i = 0; i < slices.size(); i++) { - buffer->addBufferFragment(container->fragments_[i]); + buffer->addBufferFragment(*new GrpcSliceBufferFragmentImpl(std::move(slices[i]))); } - container->slices_ = std::move(slices); return buffer; } From 41ecbb3e4bd48b425483e7c3aae17509f2ef3a80 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Tue, 25 Jun 2019 12:18:30 -0400 Subject: [PATCH 065/542] subset: allow matching metadata criteria with any element in list (#7047) This makes it possible to configure the subset LB to match metadata match criterias with any of the values specified in a list value on an endpoint. This allows endpoints to have multiple values for a given metadata key. To accomplish this the invariants of the subset trie construction changed: a host can now be associated with multiple subsets for a set of subset selectors. To support this the trie construction had to change to traverse all possible paths for each host. Fixes #6921 Signed-off-by: Snow Pettersen --- api/envoy/api/v2/cds.proto | 5 + docs/root/intro/version_history.rst | 3 + include/envoy/upstream/load_balancer_type.h | 6 + source/common/upstream/load_balancer_impl.h | 4 +- source/common/upstream/subset_lb.cc | 104 ++++++++++---- source/common/upstream/subset_lb.h | 4 +- test/common/upstream/subset_lb_test.cc | 144 ++++++++++++++++++++ test/mocks/upstream/cluster_info.h | 1 + tools/spelling_dictionary.txt | 1 + 9 files changed, 242 insertions(+), 30 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 4f79fda5ab53..004e9ddc4a31 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -428,6 +428,11 @@ message Cluster { // subset might become empty. With this option enabled, if that happens the LB will attempt // to select a host from the entire cluster. bool panic_mode_any = 6; + + // If true, metadata specified for a metadata key will be matched against the corresponding + // endpoint metadata if the endpoint metadata matches the value exactly OR it is a list value + // and any of the elements in the list matches the criteria. + bool list_as_any = 7; } // Configuration for load balancing subsetting. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 198cef464973..736b2fb4e2f5 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -70,6 +70,9 @@ Version history * sandbox: added :ref:`CSRF sandbox `. * server: ``--define manual_stamp=manual_stamp`` was added to allow server stamping outside of binary rules. more info in the `bazel docs `_. +* subset: added :ref:`list_as_any` option to + the subset lb which allows matching metadata against any of the values in a list value + on the endpoints. * server: added :ref:`Server State ` statistic. * tool: added :repo:`proto ` support for :ref:`router check tool ` tests. * tracing: add trace sampling configuration to the route, to override the route level. diff --git a/include/envoy/upstream/load_balancer_type.h b/include/envoy/upstream/load_balancer_type.h index 91848df3e34a..40fe0683da02 100644 --- a/include/envoy/upstream/load_balancer_type.h +++ b/include/envoy/upstream/load_balancer_type.h @@ -80,6 +80,12 @@ class LoadBalancerSubsetInfo { * selection from the fallback subset fails. */ virtual bool panicModeAny() const PURE; + + /* + * @return bool whether matching metadata should attempt to match against any of the + * elements in a list value defined in endpoint metadata. + */ + virtual bool listAsAny() const PURE; }; } // namespace Upstream diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 2f5d0de1360b..1ee7e2b02672 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -485,7 +485,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { default_subset_(subset_config.default_subset()), locality_weight_aware_(subset_config.locality_weight_aware()), scale_locality_weight_(subset_config.scale_locality_weight()), - panic_mode_any_(subset_config.panic_mode_any()) { + panic_mode_any_(subset_config.panic_mode_any()), list_as_any_(subset_config.list_as_any()) { for (const auto& subset : subset_config.subset_selectors()) { if (!subset.keys().empty()) { subset_selectors_.emplace_back(std::make_shared( @@ -507,6 +507,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { bool localityWeightAware() const override { return locality_weight_aware_; } bool scaleLocalityWeight() const override { return scale_locality_weight_; } bool panicModeAny() const override { return panic_mode_any_; } + bool listAsAny() const override { return list_as_any_; } private: const bool enabled_; @@ -516,6 +517,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { const bool locality_weight_aware_; const bool scale_locality_weight_; const bool panic_mode_any_; + const bool list_as_any_; }; } // namespace Upstream diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index 1632e754c4c7..f6fb7212bb34 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -32,7 +32,7 @@ SubsetLoadBalancer::SubsetLoadBalancer( subset_selectors_(subsets.subsetSelectors()), original_priority_set_(priority_set), original_local_priority_set_(local_priority_set), locality_weight_aware_(subsets.localityWeightAware()), - scale_locality_weight_(subsets.scaleLocalityWeight()) { + scale_locality_weight_(subsets.scaleLocalityWeight()), list_as_any_(subsets.listAsAny()) { ASSERT(subsets.isEnabled()); if (fallback_policy_ != envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK) { @@ -360,23 +360,25 @@ void SubsetLoadBalancer::processSubsets( const auto& keys = subset_selector->selector_keys_; // For each host, for each subset key, attempt to extract the metadata corresponding to the // key from the host. - SubsetMetadata kvs = extractSubsetMetadata(keys, *host); - if (!kvs.empty()) { + std::vector all_kvs = extractSubsetMetadata(keys, *host); + for (const auto& kvs : all_kvs) { // The host has metadata for each key, find or create its subset. - LbSubsetEntryPtr entry = findOrCreateSubset(subsets_, kvs, 0); - if (subsets_modified.find(entry) != subsets_modified.end()) { - // We've already invoked the callback for this entry. - continue; - } - subsets_modified.emplace(entry); - - if (entry->initialized()) { - update_cb(entry); - } else { - HostPredicate predicate = [this, kvs](const Host& host) -> bool { - return hostMatches(kvs, host); - }; - new_cb(entry, predicate, kvs, adding_hosts); + auto entry = findOrCreateSubset(subsets_, kvs, 0); + if (entry != nullptr) { + if (subsets_modified.find(entry) != subsets_modified.end()) { + // We've already invoked the callback for this entry. + continue; + } + subsets_modified.emplace(entry); + + if (entry->initialized()) { + update_cb(entry); + } else { + HostPredicate predicate = [this, kvs](const Host& host) -> bool { + return hostMatches(kvs, host); + }; + new_cb(entry, predicate, kvs, adding_hosts); + } } } } @@ -450,7 +452,19 @@ bool SubsetLoadBalancer::hostMatches(const SubsetMetadata& kvs, const Host& host return false; } - if (!ValueUtil::equal(entry_it->second, kv.second)) { + if (list_as_any_ && entry_it->second.kind_case() == ProtobufWkt::Value::kListValue) { + bool any_match = false; + for (const auto& v : entry_it->second.list_value().values()) { + if (ValueUtil::equal(v, kv.second)) { + any_match = true; + break; + } + } + + if (!any_match) { + return false; + } + } else if (!ValueUtil::equal(entry_it->second, kv.second)) { return false; } } @@ -460,31 +474,63 @@ bool SubsetLoadBalancer::hostMatches(const SubsetMetadata& kvs, const Host& host // Iterates over subset_keys looking up values from the given host's metadata. Each key-value pair // is appended to kvs. Returns a non-empty value if the host has a value for each key. -SubsetLoadBalancer::SubsetMetadata +std::vector SubsetLoadBalancer::extractSubsetMetadata(const std::set& subset_keys, const Host& host) { - SubsetMetadata kvs; + std::vector all_kvs; const envoy::api::v2::core::Metadata& metadata = *host.metadata(); const auto& filter_it = metadata.filter_metadata().find(Config::MetadataFilters::get().ENVOY_LB); if (filter_it == metadata.filter_metadata().end()) { - return kvs; + return all_kvs; } const auto& fields = filter_it->second.fields(); for (const auto& key : subset_keys) { const auto it = fields.find(key); if (it == fields.end()) { + all_kvs.clear(); break; } - kvs.emplace_back(std::pair(key, it->second)); - } - if (kvs.size() != subset_keys.size()) { - kvs.clear(); + if (list_as_any_ && it->second.kind_case() == ProtobufWkt::Value::kListValue) { + // If the list of kvs is empty, we initialize one kvs for each value in the list. + // Otherwise, we branch the list of kvs by generating one new kvs per old kvs per + // new value. + // + // For example, two kvs (, ) joined with the kv foo=[bar,baz] results in four kvs: + // + // + // + // + if (all_kvs.empty()) { + for (const auto& v : it->second.list_value().values()) { + all_kvs.emplace_back(SubsetMetadata({make_pair(key, v)})); + } + } else { + std::vector new_kvs; + for (const auto& kvs : all_kvs) { + for (const auto& v : it->second.list_value().values()) { + auto kv_copy = kvs; + kv_copy.emplace_back(make_pair(key, v)); + new_kvs.emplace_back(kv_copy); + } + } + all_kvs = new_kvs; + } + + } else { + if (all_kvs.empty()) { + all_kvs.emplace_back(SubsetMetadata({std::make_pair(key, it->second)})); + } else { + for (auto& kvs : all_kvs) { + kvs.emplace_back(std::make_pair(key, it->second)); + } + } + } } - return kvs; + return all_kvs; } std::string SubsetLoadBalancer::describeMetadata(const SubsetLoadBalancer::SubsetMetadata& kvs) { @@ -517,9 +563,11 @@ SubsetLoadBalancer::findOrCreateSubset(LbSubsetMap& subsets, const SubsetMetadat const std::string& name = kvs[idx].first; const ProtobufWkt::Value& pb_value = kvs[idx].second; const HashedValue value(pb_value); + LbSubsetEntryPtr entry; const auto& kv_it = subsets.find(name); + if (kv_it != subsets.end()) { ValueSubsetMap& value_subset_map = kv_it->second; const auto vs_it = value_subset_map.find(value); @@ -711,8 +759,8 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, } } - // Since the removed hosts would not be present in the list of all hosts, we need to evaluate the - // predicate directly for these hosts. + // Since the removed hosts would not be present in the list of all hosts, we need to evaluate + // the predicate directly for these hosts. HostVector filtered_removed; for (const auto& host : hosts_removed) { if (predicate(*host)) { diff --git a/source/common/upstream/subset_lb.h b/source/common/upstream/subset_lb.h index 02b6108d56a4..5e03ce84765e 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/common/upstream/subset_lb.h @@ -175,7 +175,8 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable cb); - SubsetMetadata extractSubsetMetadata(const std::set& subset_keys, const Host& host); + std::vector extractSubsetMetadata(const std::set& subset_keys, + const Host& host); std::string describeMetadata(const SubsetMetadata& kvs); const LoadBalancerType lb_type_; @@ -209,6 +210,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable { } using HostMetadata = std::map; + using HostListMetadata = std::map>; using HostURLMetadataMap = std::map; void init() { @@ -237,6 +238,18 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { return makeTestHost(info_, url, m); } + HostSharedPtr makeHost(const std::string& url, const HostListMetadata& metadata) { + envoy::api::v2::core::Metadata m; + for (const auto& m_it : metadata) { + auto& metadata = Config::Metadata::mutableMetadataValue( + m, Config::MetadataFilters::get().ENVOY_LB, m_it.first); + for (const auto& value : m_it.second) { + metadata.mutable_list_value()->add_values()->set_string_value(value); + } + } + + return makeTestHost(info_, url, m); + } ProtobufWkt::Struct makeDefaultSubset(HostMetadata metadata) { ProtobufWkt::Struct default_subset; @@ -634,6 +647,137 @@ TEST_P(SubsetLoadBalancerTest, BalancesSubsetAfterUpdate) { EXPECT_EQ(3U, stats_.lb_subsets_created_.value()); } +TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabled) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); + + init({}); + modifyHosts( + {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {}, {}, 0); + + { + TestLoadBalancerContext context({{"version", "1.0"}}); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + } + { + TestLoadBalancerContext context({{"version", "1.2"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + } + TestLoadBalancerContext context({{"version", "1.2.1"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabledMultipleLists) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); + + init({}); + modifyHosts( + {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), + makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.2", "1.2"}}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {}, {}, 0); + + { + TestLoadBalancerContext context({{"version", "1.0"}}); + EXPECT_TRUE(host_set_.hosts()[2] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[2] == lb_->chooseHost(&context)); + } + { + // This should LB between both hosts marked with version 1.2. + TestLoadBalancerContext context({{"version", "1.2"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + } + { + // Choose a host multiple times to ensure that hosts()[0] is the *only* + // thing selected for this subset. + TestLoadBalancerContext context({{"version", "1.2.1"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + } + + TestLoadBalancerContext context({{"version", "1.2.2"}}); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, ListAsAnyEnabledMultipleListsForSingleHost) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = {std::make_shared( + SubsetSelector{{"version", "hardware"}, + envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + EXPECT_CALL(subset_info_, listAsAny()).WillRepeatedly(Return(true)); + + init({}); + modifyHosts( + {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}, + {"hardware", std::vector{"a", "b"}}}), + makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.1", "1.1.1"}}, + {"hardware", std::vector{"b", "c"}}})}, + {}, {}, 0); + + { + TestLoadBalancerContext context({{"version", "1.2"}, {"hardware", "a"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + } + + { + TestLoadBalancerContext context({{"version", "1.1"}, {"hardware", "b"}}); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + } + + { + TestLoadBalancerContext context({{"version", "1.1"}, {"hardware", "a"}}); + EXPECT_TRUE(nullptr == lb_->chooseHost(&context)); + } + + TestLoadBalancerContext context({{"version", "1.2.1"}, {"hardware", "b"}}); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); + EXPECT_TRUE(host_set_.hosts()[0] == lb_->chooseHost(&context)); +} + +TEST_P(SubsetLoadBalancerTest, ListAsAnyDisable) { + EXPECT_CALL(subset_info_, fallbackPolicy()) + .WillRepeatedly(Return(envoy::api::v2::Cluster::LbSubsetConfig::NO_FALLBACK)); + + std::vector subset_selectors = { + std::make_shared(SubsetSelector{ + {"version"}, envoy::api::v2::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED})}; + EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); + + init({}); + modifyHosts( + {makeHost("tcp://127.0.0.1:8000", {{"version", std::vector{"1.2.1", "1.2"}}}), + makeHost("tcp://127.0.0.1:8001", {{"version", "1.0"}})}, + {}, {}, 0); + + { + TestLoadBalancerContext context({{"version", "1.0"}}); + EXPECT_TRUE(host_set_.hosts()[1] == lb_->chooseHost(&context)); + } + TestLoadBalancerContext context({{"version", "1.2"}}); + EXPECT_TRUE(nullptr == lb_->chooseHost(&context)); +} + // Test that adding backends to a failover group causes no problems. TEST_P(SubsetLoadBalancerTest, UpdateFailover) { EXPECT_CALL(subset_info_, fallbackPolicy()) diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index c55a31a42585..46b318c3f36d 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -37,6 +37,7 @@ class MockLoadBalancerSubsetInfo : public LoadBalancerSubsetInfo { MOCK_CONST_METHOD0(localityWeightAware, bool()); MOCK_CONST_METHOD0(scaleLocalityWeight, bool()); MOCK_CONST_METHOD0(panicModeAny, bool()); + MOCK_CONST_METHOD0(listAsAny, bool()); std::vector subset_selectors_; }; diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 33786deb4df1..c2129e477708 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -540,6 +540,7 @@ keepalives ketama kqueue kubernetes +kv kvs lala latencies From 815c506c96ef441d99341775af2125d58d644b8f Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Tue, 25 Jun 2019 10:41:36 -0700 Subject: [PATCH 066/542] admin: add host priority to cluster response (#7317) Add host priority to cluster response in admin server. Risk Level: low Testing: unit test Docs Changes: N/A Release Notes: updated Signed-off-by: Yan Xue --- api/envoy/admin/v2alpha/clusters.proto | 3 +++ docs/root/intro/version_history.rst | 1 + source/server/http/admin.cc | 4 ++++ test/server/http/admin_test.cc | 5 ++++- 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/api/envoy/admin/v2alpha/clusters.proto b/api/envoy/admin/v2alpha/clusters.proto index 885f6b3034a6..03a85d8f36b0 100644 --- a/api/envoy/admin/v2alpha/clusters.proto +++ b/api/envoy/admin/v2alpha/clusters.proto @@ -68,6 +68,9 @@ message HostStatus { // The hostname of the host, if applicable. string hostname = 6; + + // The host's priority. If not configured, the value defaults to 0 (highest priority). + uint32 priority = 7; } // Health status for a host. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 736b2fb4e2f5..b427780db79e 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -10,6 +10,7 @@ Version history * admin: the administration interface now includes a :ref:`/ready endpoint ` for easier readiness checks. * admin: extend :ref:`/runtime_modify endpoint ` to support parameters within the request body. * admin: the :ref:`/listener endpoint ` now returns :ref:`listeners.proto` which includes listener names and ports. +* admin: added host priority to :http:get:`/clusters` and :http:get:`/clusters?format=json` end point response * admin: the :ref:`/clusters endpoint ` now shows hostname for each host, useful for DNS based clusters. * api: track and report requests issued since last load report. diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 07c0bdc3232c..f3cafc6a1b89 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -402,6 +402,8 @@ void AdminImpl::writeClustersAsJson(Buffer::Instance& response) { } host_status.set_weight(host->weight()); + + host_status.set_priority(host->priority()); } } } @@ -457,6 +459,8 @@ void AdminImpl::writeClustersAsText(Buffer::Instance& response) { response.add(fmt::format("{}::{}::success_rate::{}\n", cluster.second.get().info()->name(), host->address()->asString(), host->outlierDetector().successRate())); + response.add(fmt::format("{}::{}::priority::{}\n", cluster.second.get().info()->name(), + host->address()->asString(), host->priority())); } } } diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index 64639c23f9e7..feca903e1b91 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -1092,6 +1092,7 @@ TEST_P(AdminInstanceTest, ClustersJson) { ON_CALL(host->outlier_detector_, successRate()).WillByDefault(Return(43.2)); ON_CALL(*host, weight()).WillByDefault(Return(5)); + ON_CALL(*host, priority()).WillByDefault(Return(6)); Buffer::OwnedImpl response; Http::HeaderMapImpl header_map; @@ -1155,7 +1156,8 @@ TEST_P(AdminInstanceTest, ClustersJson) { "value": 43.2 }, "weight": 5, - "hostname": "foo.com" + "hostname": "foo.com", + "priority": 6 } ] } @@ -1196,6 +1198,7 @@ fake_cluster::1.2.3.4:80::zone::test_zone fake_cluster::1.2.3.4:80::sub_zone::test_sub_zone fake_cluster::1.2.3.4:80::canary::false fake_cluster::1.2.3.4:80::success_rate::43.2 +fake_cluster::1.2.3.4:80::priority::6 )EOF"; EXPECT_EQ(expected_text, response2.toString()); } From e868a35dc8f159fd9b3e4010012b9e8e926c64af Mon Sep 17 00:00:00 2001 From: Sriduth Jayhari Date: Wed, 26 Jun 2019 01:48:13 +0530 Subject: [PATCH 067/542] docs: fix typo in http_connection_management.rst introduced in PR #7230 (#7394) Signed-off-by: Sriduth Jayhari --- .../intro/arch_overview/http/http_connection_management.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/intro/arch_overview/http/http_connection_management.rst b/docs/root/intro/arch_overview/http/http_connection_management.rst index 6a5df8b7c066..dd7fc29fb8eb 100644 --- a/docs/root/intro/arch_overview/http/http_connection_management.rst +++ b/docs/root/intro/arch_overview/http/http_connection_management.rst @@ -61,7 +61,7 @@ can be used to modify this behavior, and they fall into two categories: hosts that have already been attempted. * *envoy.retry_host_predicates.omit_canary_hosts*: This will reject any host that is a marked as canary host. - Hosts are marked by setting ``canary: true`` for the ```envoy.lb`` filter in the endpoint's filter metadata. + Hosts are marked by setting ``canary: true`` for the ``envoy.lb`` filter in the endpoint's filter metadata. See :ref:`LbEndpoint ` for more details. * :ref:`Priority Predicates`: These predicates can From d63aa4d05c0968eb335a891c1b1218d2675beac7 Mon Sep 17 00:00:00 2001 From: Adam Kotwasinski Date: Tue, 25 Jun 2019 13:19:48 -0700 Subject: [PATCH 068/542] Kafka codec: response encoder and decoder (#6993) Signed-off-by: Adam Kotwasinski --- api/bazel/repositories.bzl | 8 + source/extensions/filters/network/kafka/BUILD | 79 +++++++- .../filters/network/kafka/kafka_response.h | 107 +++++++++++ .../network/kafka/kafka_response_parser.cc | 47 +++++ .../network/kafka/kafka_response_parser.h | 176 ++++++++++++++++++ .../complex_type_template.j2 | 8 +- .../kafka_generator.py | 124 ++++++------ .../kafka_request_resolver_cc.j2 | 6 +- .../kafka_response_resolver_cc.j2 | 38 ++++ .../request_codec_request_test_cc.j2 | 38 ++-- .../requests_test_cc.j2 | 36 ++-- .../response_codec_response_test_cc.j2 | 78 ++++++++ .../response_parser.j2 | 20 ++ .../protocol_code_generator/responses_h.j2 | 36 ++++ .../responses_test_cc.j2 | 79 ++++++++ .../filters/network/kafka/response_codec.cc | 112 +++++++++++ .../filters/network/kafka/response_codec.h | 148 +++++++++++++++ .../serialization_composite_test_cc.j2 | 6 +- test/extensions/filters/network/kafka/BUILD | 99 +++++++++- .../filters/network/kafka/buffer_based_test.h | 57 ++++++ .../kafka/kafka_request_parser_test.cc | 24 +-- .../kafka/kafka_response_parser_test.cc | 170 +++++++++++++++++ .../kafka/request_codec_integration_test.cc | 24 +-- .../network/kafka/request_codec_unit_test.cc | 55 +++--- .../kafka/response_codec_integration_test.cc | 79 ++++++++ .../network/kafka/response_codec_unit_test.cc | 172 +++++++++++++++++ .../network/kafka/serialization_utilities.cc | 21 +-- .../network/kafka/serialization_utilities.h | 22 ++- 28 files changed, 1661 insertions(+), 208 deletions(-) create mode 100644 source/extensions/filters/network/kafka/kafka_response.h create mode 100644 source/extensions/filters/network/kafka/kafka_response_parser.cc create mode 100644 source/extensions/filters/network/kafka/kafka_response_parser.h create mode 100644 source/extensions/filters/network/kafka/protocol_code_generator/kafka_response_resolver_cc.j2 create mode 100644 source/extensions/filters/network/kafka/protocol_code_generator/response_codec_response_test_cc.j2 create mode 100644 source/extensions/filters/network/kafka/protocol_code_generator/response_parser.j2 create mode 100644 source/extensions/filters/network/kafka/protocol_code_generator/responses_h.j2 create mode 100644 source/extensions/filters/network/kafka/protocol_code_generator/responses_test_cc.j2 create mode 100644 source/extensions/filters/network/kafka/response_codec.cc create mode 100644 source/extensions/filters/network/kafka/response_codec.h create mode 100644 test/extensions/filters/network/kafka/buffer_based_test.h create mode 100644 test/extensions/filters/network/kafka/kafka_response_parser_test.cc create mode 100644 test/extensions/filters/network/kafka/response_codec_integration_test.cc create mode 100644 test/extensions/filters/network/kafka/response_codec_unit_test.cc diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index 490e72491756..67d38c4fb57a 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -314,4 +314,12 @@ filegroup( visibility = ["//visibility:public"], ) +filegroup( + name = "response_protocol_files", + srcs = glob([ + "*Response.json", + ]), + visibility = ["//visibility:public"], +) + """ diff --git a/source/extensions/filters/network/kafka/BUILD b/source/extensions/filters/network/kafka/BUILD index 37bfa7a5daba..8eb906e15b5b 100644 --- a/source/extensions/filters/network/kafka/BUILD +++ b/source/extensions/filters/network/kafka/BUILD @@ -11,16 +11,26 @@ load( envoy_package() +envoy_cc_library( + name = "abstract_codec_lib", + srcs = [], + hdrs = [ + "codec.h", + ], + deps = [ + "//source/common/buffer:buffer_lib", + ], +) + envoy_cc_library( name = "kafka_request_codec_lib", srcs = ["request_codec.cc"], hdrs = [ - "codec.h", "request_codec.h", ], deps = [ + ":abstract_codec_lib", ":kafka_request_parser_lib", - "//source/common/buffer:buffer_lib", ], ) @@ -55,7 +65,7 @@ envoy_cc_library( ) genrule( - name = "kafka_generated_source", + name = "kafka_request_generated_source", srcs = [ "@kafka_source//:request_protocol_files", ], @@ -64,7 +74,7 @@ genrule( "external/kafka_request_resolver.cc", ], cmd = """ - ./$(location :kafka_code_generator) generate-source \ + ./$(location :kafka_code_generator) generate-source request \ $(location external/requests.h) $(location external/kafka_request_resolver.cc) \ $(SRCS) """, @@ -73,6 +83,67 @@ genrule( ], ) +envoy_cc_library( + name = "kafka_response_codec_lib", + srcs = ["response_codec.cc"], + hdrs = [ + "response_codec.h", + ], + deps = [ + ":abstract_codec_lib", + ":kafka_response_parser_lib", + ], +) + +envoy_cc_library( + name = "kafka_response_parser_lib", + srcs = [ + "external/kafka_response_resolver.cc", + "kafka_response_parser.cc", + ], + hdrs = [ + "external/responses.h", + "kafka_response_parser.h", + ], + deps = [ + ":kafka_response_lib", + ":parser_lib", + "//source/common/common:assert_lib", + "//source/common/common:minimal_logger_lib", + ], +) + +envoy_cc_library( + name = "kafka_response_lib", + srcs = [ + ], + hdrs = [ + "kafka_response.h", + ], + deps = [ + ":serialization_lib", + ], +) + +genrule( + name = "kafka_response_generated_source", + srcs = [ + "@kafka_source//:response_protocol_files", + ], + outs = [ + "external/responses.h", + "external/kafka_response_resolver.cc", + ], + cmd = """ + ./$(location :kafka_code_generator) generate-source response \ + $(location external/responses.h) $(location external/kafka_response_resolver.cc) \ + $(SRCS) + """, + tools = [ + ":kafka_code_generator", + ], +) + py_binary( name = "kafka_code_generator", srcs = ["protocol_code_generator/kafka_generator.py"], diff --git a/source/extensions/filters/network/kafka/kafka_response.h b/source/extensions/filters/network/kafka/kafka_response.h new file mode 100644 index 000000000000..8e6a8e5b35c1 --- /dev/null +++ b/source/extensions/filters/network/kafka/kafka_response.h @@ -0,0 +1,107 @@ +#pragma once + +#include "extensions/filters/network/kafka/external/serialization_composite.h" +#include "extensions/filters/network/kafka/serialization.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { + +/** + * Represents Kafka response metadata: expected api key, version and correlation id. + * @see http://kafka.apache.org/protocol.html#protocol_messages + */ +struct ResponseMetadata { + ResponseMetadata(const int16_t api_key, const int16_t api_version, const int32_t correlation_id) + : api_key_{api_key}, api_version_{api_version}, correlation_id_{correlation_id} {}; + + bool operator==(const ResponseMetadata& rhs) const { + return api_key_ == rhs.api_key_ && api_version_ == rhs.api_version_ && + correlation_id_ == rhs.correlation_id_; + }; + + const int16_t api_key_; + const int16_t api_version_; + const int32_t correlation_id_; +}; + +using ResponseMetadataSharedPtr = std::shared_ptr; + +/** + * Abstract response object, carrying data related to every response. + * @see http://kafka.apache.org/protocol.html#protocol_messages + */ +class AbstractResponse { +public: + virtual ~AbstractResponse() = default; + + /** + * Constructs a request with given metadata. + * @param metadata response metadata. + */ + AbstractResponse(const ResponseMetadata& metadata) : metadata_{metadata} {}; + + /** + * Computes the size of this response, if it were to be serialized. + * @return serialized size of response. + */ + virtual uint32_t computeSize() const PURE; + + /** + * Encode the contents of this response into a given buffer. + * @param dst buffer instance to keep serialized message. + */ + virtual uint32_t encode(Buffer::Instance& dst) const PURE; + + /** + * Response's metadata. + */ + const ResponseMetadata metadata_; +}; + +using AbstractResponseSharedPtr = std::shared_ptr; + +/** + * Concrete response that carries data particular to given response type. + * @param Data concrete response data type. + */ +template class Response : public AbstractResponse { +public: + Response(const ResponseMetadata& metadata, const Data& data) + : AbstractResponse{metadata}, data_{data} {}; + + /** + * Compute the size of response, which includes both the response header (correlation id) and + * real data. + */ + uint32_t computeSize() const override { + const EncodingContext context{metadata_.api_version_}; + return context.computeSize(metadata_.correlation_id_) + context.computeSize(data_); + } + + /** + * Encodes given response into a buffer, with any extra configuration carried by the context. + */ + uint32_t encode(Buffer::Instance& dst) const override { + EncodingContext context{metadata_.api_version_}; + uint32_t written{0}; + // Encode correlation id (api key / version are not present in responses). + written += context.encode(metadata_.correlation_id_, dst); + // Encode response-specific data. + written += context.encode(data_, dst); + return written; + } + + bool operator==(const Response& rhs) const { + return metadata_ == rhs.metadata_ && data_ == rhs.data_; + }; + +private: + const Data data_; +}; + +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/kafka/kafka_response_parser.cc b/source/extensions/filters/network/kafka/kafka_response_parser.cc new file mode 100644 index 000000000000..88af19527cb7 --- /dev/null +++ b/source/extensions/filters/network/kafka/kafka_response_parser.cc @@ -0,0 +1,47 @@ +#include "extensions/filters/network/kafka/kafka_response_parser.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { + +const ResponseParserResolver& ResponseParserResolver::getDefaultInstance() { + CONSTRUCT_ON_FIRST_USE(ResponseParserResolver); +} + +ResponseParseResponse ResponseHeaderParser::parse(absl::string_view& data) { + length_deserializer_.feed(data); + if (!length_deserializer_.ready()) { + return ResponseParseResponse::stillWaiting(); + } + + correlation_id_deserializer_.feed(data); + if (!correlation_id_deserializer_.ready()) { + return ResponseParseResponse::stillWaiting(); + } + + context_->remaining_response_size_ = length_deserializer_.get(); + context_->remaining_response_size_ -= sizeof(context_->correlation_id_); + context_->correlation_id_ = correlation_id_deserializer_.get(); + + auto next_parser = parser_resolver_.createParser(context_); + return ResponseParseResponse::nextParser(next_parser); +} + +ResponseParseResponse SentinelResponseParser::parse(absl::string_view& data) { + const uint32_t min = std::min(context_->remaining_response_size_, data.size()); + data = {data.data() + min, data.size() - min}; + context_->remaining_response_size_ -= min; + if (0 == context_->remaining_response_size_) { + const auto failure_data = std::make_shared( + context_->api_key_, context_->api_version_, context_->correlation_id_); + return ResponseParseResponse::parseFailure(failure_data); + } else { + return ResponseParseResponse::stillWaiting(); + } +} + +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/kafka/kafka_response_parser.h b/source/extensions/filters/network/kafka/kafka_response_parser.h new file mode 100644 index 000000000000..a39a9487d521 --- /dev/null +++ b/source/extensions/filters/network/kafka/kafka_response_parser.h @@ -0,0 +1,176 @@ +#pragma once + +#include + +#include "extensions/filters/network/kafka/kafka_response.h" +#include "extensions/filters/network/kafka/parser.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { + +using ResponseParseResponse = ParseResponse; +using ResponseParser = Parser; +using ResponseParserSharedPtr = std::shared_ptr; + +/** + * Context that is shared between parsers that are handling the same single message. + */ +struct ResponseContext { + + /** + * Creates a context for parsing a message with given expected metadata. + * @param metadata expected response metadata. + */ + ResponseContext(const int16_t api_key, const int16_t api_version) + : api_key_{api_key}, api_version_{api_version} {}; + + /** + * Api key of response that's being parsed. + */ + const int16_t api_key_; + + /** + * Api version of response that's being parsed. + */ + const int16_t api_version_; + + /** + * Bytes left to process. + */ + int32_t remaining_response_size_; + + /** + * Response's correlation id. + */ + int32_t correlation_id_; +}; + +using ResponseContextSharedPtr = std::shared_ptr; + +/** + * Response decoder configuration object. + * Resolves the parser that will be responsible for consuming the response. + * In other words: provides (api_key, api_version) -> Parser function. + */ +class ResponseParserResolver { +public: + virtual ~ResponseParserResolver() = default; + + /** + * Creates a parser that is going to process data specific for given response. + * @param metadata expected response metadata. + * @return parser that is capable of processing response. + */ + virtual ResponseParserSharedPtr createParser(ResponseContextSharedPtr metadata) const; + + /** + * Return default resolver, that uses response's api key and version to provide a matching parser. + */ + static const ResponseParserResolver& getDefaultInstance(); +}; + +/** + * Response parser responsible for consuming response header (payload length and correlation id) and + * setting up context with this data. + * @see http://kafka.apache.org/protocol.html#protocol_common + */ +class ResponseHeaderParser : public ResponseParser { +public: + /** + * Creates a parser with given context and parser resolver. + */ + ResponseHeaderParser(ResponseContextSharedPtr context, + const ResponseParserResolver& parser_resolver) + : context_{context}, parser_resolver_{parser_resolver} {}; + + /** + * Consumes 8 bytes (2 x INT32) as response length and correlation id and updates the context with + * that value, then creates the following payload parser depending on metadata provided. + * @return ResponseParser instance to process the response payload. + */ + ResponseParseResponse parse(absl::string_view& data) override; + + const ResponseContextSharedPtr contextForTest() const { return context_; } + +private: + ResponseContextSharedPtr context_; + const ResponseParserResolver& parser_resolver_; + + Int32Deserializer length_deserializer_; + Int32Deserializer correlation_id_deserializer_; +}; + +/** + * Sentinel parser that is responsible for consuming message bytes for messages that had unsupported + * api_key & api_version. It does not attempt to capture any data, just throws it away until end of + * message. + */ +class SentinelResponseParser : public ResponseParser { +public: + SentinelResponseParser(ResponseContextSharedPtr context) : context_{context} {}; + + ResponseParseResponse parse(absl::string_view& data) override; + + const ResponseContextSharedPtr contextForTest() const { return context_; } + +private: + ResponseContextSharedPtr context_; +}; + +/** + * Response parser uses a single deserializer to construct a response object. + * This parser is responsible for consuming response-specific data (e.g. topic names) and always + * returns a parsed message. + * @param ResponseType response class. + * @param DeserializerType deserializer type corresponding to response class (should be subclass of + * Deserializer). + */ +template +class ResponseDataParser : public ResponseParser { +public: + /** + * Create a parser for given response metadata. + * @param metadata expected message metadata. + */ + ResponseDataParser(ResponseContextSharedPtr context) : context_{context} {}; + + /** + * Consume enough data to fill in deserializer and receive the parsed response. + * Fill in response's header with data stored in context. + * @param data data to process. + */ + ResponseParseResponse parse(absl::string_view& data) override { + context_->remaining_response_size_ -= deserializer_.feed(data); + + if (deserializer_.ready()) { + if (0 == context_->remaining_response_size_) { + // After a successful parse, there should be nothing left - we have consumed all the bytes. + const ResponseMetadata metadata = {context_->api_key_, context_->api_version_, + context_->correlation_id_}; + const AbstractResponseSharedPtr response = + std::make_shared>(metadata, deserializer_.get()); + return ResponseParseResponse::parsedMessage(response); + } else { + // The message makes no sense, the deserializer that matches the schema consumed all + // necessary data, but there are still bytes in this message. + return ResponseParseResponse::nextParser( + std::make_shared(context_)); + } + } else { + return ResponseParseResponse::stillWaiting(); + } + } + + const ResponseContextSharedPtr contextForTest() const { return context_; } + +private: + ResponseContextSharedPtr context_; + DeserializerType deserializer_; // Underlying response-specific deserializer. +}; + +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/complex_type_template.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/complex_type_template.j2 index 5a8cc2c534b2..a60533777970 100644 --- a/source/extensions/filters/network/kafka/protocol_code_generator/complex_type_template.j2 +++ b/source/extensions/filters/network/kafka/protocol_code_generator/complex_type_template.j2 @@ -1,10 +1,12 @@ {# - Template for structure representing a composite entity in Kafka protocol (e.g. FetchRequest). - Rendered templates for each structure in Kafka protocol will be put into 'requests.h' file. + Template for structure representing a composite entity in Kafka protocol (request or response). + Rendered templates for each structure in Kafka protocol will be put into 'requests.h' + or 'responses.h'. Each structure is capable of holding all versions of given entity (what means its fields are actually a superset of union of all versions' fields). Each version has a dedicated deserializer - (named $requestV$versionDeserializer), which calls the matching constructor. + (named "${name}V${version}Deserializer" e.g. ProduceRequestV0Deserializer or + FetchResponseV1Deserializer), which calls the matching constructor. To serialize, it is necessary to pass the encoding context (that contains the version that's being serialized). Depending on the version, the fields will be written to the buffer. diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_generator.py b/source/extensions/filters/network/kafka/protocol_code_generator/kafka_generator.py index aed29b906a0d..c3a0f286e3d9 100755 --- a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_generator.py +++ b/source/extensions/filters/network/kafka/protocol_code_generator/kafka_generator.py @@ -9,114 +9,126 @@ def main(): Can generate both main source code, as well as test code. Usage: - kafka_generator.py COMMAND OUTPUT FILES INPUT_FILES + kafka_generator.py COMMAND MESSAGE_TYPE OUTPUT_FILES INPUT_FILES where: COMMAND : 'generate-source', to generate source files, 'generate-test', to generate test files. - OUTPUT_FILES : if generate-source: location of 'requests.h' and 'kafka_request_resolver.cc', - if generate-test: location of 'requests_test.cc', 'request_codec_request_test.cc'. + MESSAGE_TYPE : 'request' or 'response' + OUTPUT_FILES : if generate-source: location of 'requests.h'/'responses.h' and + 'kafka_request_resolver.cc' / 'kafka_response_resolver.cc', + if generate-test: location of 'requests_test.cc'/'responses_test.cc', + 'request_codec_request_test.cc' / 'response_codec_response_test.cc'. INPUT_FILES: Kafka protocol json files to be processed. Kafka spec files are provided in Kafka clients jar file. When generating source code, it creates: - - requests.h - definition of all the structures/deserializers/parsers related to Kafka requests, - - kafka_request_resolver.cc - resolver that binds api_key & api_version to parsers from - requests.h. + - ${MESSAGE_TYPE}s.h - definition of all the structures/deserializers/parsers related to Kafka + requests/responses, + - kafka_${MESSAGE_TYPE}_resolver.cc - resolver that is responsible for creation of parsers + defined in ${MESSAGE_TYPE}s.h. When generating test code, it creates: - - requests_test.cc - serialization/deserialization tests for kafka structures, - - request_codec_request_test.cc - test for all request operations using the codec API. + - ${MESSAGE_TYPE}s_test.cc - serialization/deserialization tests for kafka structures, + - ${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test.cc - test for all request/response operations + using the codec API. Templates used are: - - to create 'requests.h': requests_h.j2, complex_type_template.j2, request_parser.j2, - - to create 'kafka_request_resolver.cc': kafka_request_resolver_cc.j2, - - to create 'requests_test.cc': requests_test_cc.j2, - - to create 'request_codec_request_test.cc' - request_codec_request_test_cc.j2. + - to create '${MESSAGE_TYPE}.h': ${MESSAGE_TYPE}_h.j2, complex_type_template.j2, + request_parser.j2, + - to create 'kafka_${MESSAGE_TYPE}_resolver.cc': kafka_${MESSAGE_TYPE}_resolver_cc.j2, + - to create '${MESSAGE_TYPE}s_test.cc': ${MESSAGE_TYPE}s_test_cc.j2, + - to create '${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test.cc' - + ${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test_cc.j2. """ import sys import os command = sys.argv[1] + type = sys.argv[2] + if 'generate-source' == command: - requests_h_file = os.path.abspath(sys.argv[2]) - kafka_request_resolver_cc_file = os.path.abspath(sys.argv[3]) - input_files = sys.argv[4:] + main_header_file = os.path.abspath(sys.argv[3]) + resolver_cc_file = os.path.abspath(sys.argv[4]) + input_files = sys.argv[5:] elif 'generate-test' == command: - requests_test_cc_file = os.path.abspath(sys.argv[2]) - request_codec_request_test_cc_file = os.path.abspath(sys.argv[3]) - input_files = sys.argv[4:] + header_test_cc_file = os.path.abspath(sys.argv[3]) + codec_test_cc_file = os.path.abspath(sys.argv[4]) + input_files = sys.argv[5:] else: raise ValueError('invalid command: ' + command) import re import json - requests = [] + messages = [] - # For each request specification file, remove comments, and parse the remains. + # For each specification file, remove comments, and parse the remains. for input_file in input_files: with open(input_file, 'r') as fd: raw_contents = fd.read() without_comments = re.sub(r'//.*\n', '', raw_contents) - request_spec = json.loads(without_comments) - request = parse_request(request_spec) - requests.append(request) + message_spec = json.loads(without_comments) + message = parse_top_level_element(message_spec) + messages.append(message) - # Sort requests by api_key. - requests.sort(key=lambda x: x.get_extra('api_key')) + # Sort messages by api_key. + messages.sort(key=lambda x: x.get_extra('api_key')) # Generate main source code. if 'generate-source' == command: complex_type_template = RenderingHelper.get_template('complex_type_template.j2') - request_parsers_template = RenderingHelper.get_template('request_parser.j2') + parsers_template = RenderingHelper.get_template("%s_parser.j2" % type) - requests_h_contents = '' + main_header_contents = '' - for request in requests: - # For each child structure that is used by request, render its corresponding C++ code. - for dependency in request.declaration_chain: - requests_h_contents += complex_type_template.render(complex_type=dependency) - # Each top-level structure (e.g. FetchRequest) is going to have corresponding parsers. - requests_h_contents += request_parsers_template.render(complex_type=request) + for message in messages: + # For each child structure that is used by request/response, render its matching C++ code. + for dependency in message.declaration_chain: + main_header_contents += complex_type_template.render(complex_type=dependency) + # Each top-level structure (e.g. FetchRequest/FetchResponse) needs corresponding parsers. + main_header_contents += parsers_template.render(complex_type=message) # Full file with headers, namespace declaration etc. - template = RenderingHelper.get_template('requests_h.j2') - contents = template.render(contents=requests_h_contents) + template = RenderingHelper.get_template("%ss_h.j2" % type) + contents = template.render(contents=main_header_contents) - with open(requests_h_file, 'w') as fd: + # Generate main header file. + with open(main_header_file, 'w') as fd: fd.write(contents) - template = RenderingHelper.get_template('kafka_request_resolver_cc.j2') - contents = template.render(request_types=requests) + template = RenderingHelper.get_template("kafka_%s_resolver_cc.j2" % type) + contents = template.render(message_types=messages) - with open(kafka_request_resolver_cc_file, 'w') as fd: + # Generate ...resolver.cc file. + with open(resolver_cc_file, 'w') as fd: fd.write(contents) # Generate test code. if 'generate-test' == command: - template = RenderingHelper.get_template('requests_test_cc.j2') - contents = template.render(request_types=requests) + template = RenderingHelper.get_template("%ss_test_cc.j2" % type) + contents = template.render(message_types=messages) - with open(requests_test_cc_file, 'w') as fd: + # Generate header-test file. + with open(header_test_cc_file, 'w') as fd: fd.write(contents) - template = RenderingHelper.get_template('request_codec_request_test_cc.j2') - contents = template.render(request_types=requests) + template = RenderingHelper.get_template("%s_codec_%s_test_cc.j2" % (type, type)) + contents = template.render(message_types=messages) - with open(request_codec_request_test_cc_file, 'w') as fd: + # Generate codec-test file. + with open(codec_test_cc_file, 'w') as fd: fd.write(contents) -def parse_request(spec): +def parse_top_level_element(spec): """ - Parse a given structure into a request. - Request is just a complex type, that has name & version information kept in differently named - fields, compared to sub-structures in a request. + Parse a given structure into a request/response. + Request/response is just a complex type, that has name & version information kept in differently + named fields, compared to sub-structures in a message. """ - request_type_name = spec['name'] - request_versions = Statics.parse_version_string(spec['validVersions'], 2 << 16 - 1) - return parse_complex_type(request_type_name, spec, request_versions).with_extra( - 'api_key', spec['apiKey']) + type_name = spec['name'] + versions = Statics.parse_version_string(spec['validVersions'], 2 << 16 - 1) + return parse_complex_type(type_name, spec, versions).with_extra('api_key', spec['apiKey']) def parse_complex_type(type_name, field_spec, versions): @@ -182,7 +194,7 @@ def parse_version_string(raw_versions, highest_possible_version): class FieldList: """ - List of fields used by given entity (request or child structure) in given request version + List of fields used by given entity (request or child structure) in given message version (as fields get added or removed across versions). """ @@ -308,7 +320,7 @@ class TypeSpecification: def deserializer_name_in_version(self, version): """ - Renders the deserializer name of given type, in request with given version. + Renders the deserializer name of given type, in message with given version. """ raise NotImplementedError() @@ -345,7 +357,7 @@ def deserializer_name_in_version(self, version): self.underlying.deserializer_name_in_version(version)) def default_value(self): - return '{}' + return 'std::vector<%s>{}' % (self.underlying.name) def example_value_for_test(self, version): return 'std::vector<%s>{ %s }' % (self.underlying.name, diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_request_resolver_cc.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/kafka_request_resolver_cc.j2 index d73f76955adc..e754248fc113 100644 --- a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_request_resolver_cc.j2 +++ b/source/extensions/filters/network/kafka/protocol_code_generator/kafka_request_resolver_cc.j2 @@ -23,10 +23,10 @@ namespace Kafka { RequestParserSharedPtr RequestParserResolver::createParser(int16_t api_key, int16_t api_version, RequestContextSharedPtr context) const { -{% for request_type in request_types %}{% for field_list in request_type.compute_field_lists() %} - if ({{ request_type.get_extra('api_key') }} == api_key +{% for message_type in message_types %}{% for field_list in message_type.compute_field_lists() %} + if ({{ message_type.get_extra('api_key') }} == api_key && {{ field_list.version }} == api_version) { - return std::make_shared<{{ request_type.name }}V{{ field_list.version }}Parser>(context); + return std::make_shared<{{ message_type.name }}V{{ field_list.version }}Parser>(context); }{% endfor %}{% endfor %} return std::make_shared(context); } diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_response_resolver_cc.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/kafka_response_resolver_cc.j2 new file mode 100644 index 000000000000..bc107a18dd41 --- /dev/null +++ b/source/extensions/filters/network/kafka/protocol_code_generator/kafka_response_resolver_cc.j2 @@ -0,0 +1,38 @@ +{# + Template for 'kafka_response_resolver.cc'. + Defines default Kafka response resolver, that uses response parsers in (also generated) + 'responses.h'. +#} +#include "extensions/filters/network/kafka/external/responses.h" +#include "extensions/filters/network/kafka/kafka_response_parser.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { + +/** + * Creates a parser that is going to process data specific for given response. + * If corresponding parser cannot be found (what means a newer version of Kafka protocol), + * a sentinel parser is returned. + * @param context parse context (carries the expected message type information). + * @return parser that is capable of properly consuming response bytes. + */ +ResponseParserSharedPtr ResponseParserResolver::createParser( + ResponseContextSharedPtr context) const { + + const int16_t api_key = context->api_key_; + const int16_t api_version = context->api_version_; + +{% for message_type in message_types %}{% for field_list in message_type.compute_field_lists() %} + if ({{ message_type.get_extra('api_key') }} == api_key + && {{ field_list.version }} == api_version) { + return std::make_shared<{{ message_type.name }}V{{ field_list.version }}Parser>(context); + }{% endfor %}{% endfor %} + return std::make_shared(context); +} + +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/request_codec_request_test_cc.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/request_codec_request_test_cc.j2 index c853563f8f8a..c4f872e253d5 100644 --- a/source/extensions/filters/network/kafka/protocol_code_generator/request_codec_request_test_cc.j2 +++ b/source/extensions/filters/network/kafka/protocol_code_generator/request_codec_request_test_cc.j2 @@ -12,6 +12,7 @@ #include "extensions/filters/network/kafka/external/requests.h" #include "extensions/filters/network/kafka/request_codec.h" +#include "test/extensions/filters/network/kafka/buffer_based_test.h" #include "test/extensions/filters/network/kafka/serialization_utilities.h" #include "test/mocks/server/mocks.h" @@ -23,31 +24,29 @@ namespace NetworkFilters { namespace Kafka { namespace RequestCodecRequestTest { -class RequestCodecRequestTest : public testing::Test { -protected: - template void putInBuffer(T arg); +class RequestCodecRequestTest : public testing::Test, public MessageBasedTest {}; - Buffer::OwnedImpl buffer_; -}; +using RequestCapturingCallback = CapturingCallback; -{% for request_type in request_types %} +{% for message_type in message_types %} -// Integration test for {{ request_type.name }} messages. +// Integration test for {{ message_type.name }} messages. -TEST_F(RequestCodecRequestTest, shouldHandle{{ request_type.name }}Messages) { +TEST_F(RequestCodecRequestTest, shouldHandle{{ message_type.name }}Messages) { // given - using RequestUnderTest = Request<{{ request_type.name }}>; + using RequestUnderTest = Request<{{ message_type.name }}>; std::vector sent; int32_t correlation = 0; - {% for field_list in request_type.compute_field_lists() %} + {% for field_list in message_type.compute_field_lists() %} for (int i = 0; i < 100; ++i ) { const RequestHeader header = - { {{ request_type.get_extra('api_key') }}, {{ field_list.version }}, correlation++, "id" }; - const {{ request_type.name }} data = { {{ field_list.example_value() }} }; + { {{ message_type.get_extra('api_key') }}, {{ field_list.version }}, correlation++, "id" }; + const {{ message_type.name }} data = { {{ field_list.example_value() }} }; const RequestUnderTest request = {header, data}; - putInBuffer(request); + putMessageIntoBuffer(request); sent.push_back(request); } {% endfor %} @@ -55,16 +54,15 @@ TEST_F(RequestCodecRequestTest, shouldHandle{{ request_type.name }}Messages) { const InitialParserFactory& initial_parser_factory = InitialParserFactory::getDefaultInstance(); const RequestParserResolver& request_parser_resolver = RequestParserResolver::getDefaultInstance(); - const CapturingRequestCallbackSharedPtr request_callback = - std::make_shared(); + const auto callback = std::make_shared(); - RequestDecoder testee{initial_parser_factory, request_parser_resolver, {request_callback}}; + RequestDecoder testee{initial_parser_factory, request_parser_resolver, {callback}}; // when testee.onData(buffer_); // then - const std::vector& received = request_callback->getCaptured(); + const std::vector& received = callback->getCapturedMessages(); ASSERT_EQ(received.size(), sent.size()); for (size_t i = 0; i < received.size(); ++i) { @@ -76,12 +74,6 @@ TEST_F(RequestCodecRequestTest, shouldHandle{{ request_type.name }}Messages) { } {% endfor %} -template -void RequestCodecRequestTest::putInBuffer(const T arg) { - RequestEncoder encoder{buffer_}; - encoder.encode(arg); -} - } // namespace RequestCodecRequestTest } // namespace Kafka } // namespace NetworkFilters diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/requests_test_cc.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/requests_test_cc.j2 index d7ec7ae98ca4..3232dd0b01cb 100644 --- a/source/extensions/filters/network/kafka/protocol_code_generator/requests_test_cc.j2 +++ b/source/extensions/filters/network/kafka/protocol_code_generator/requests_test_cc.j2 @@ -6,6 +6,7 @@ #include "extensions/filters/network/kafka/external/requests.h" #include "extensions/filters/network/kafka/request_codec.h" +#include "test/extensions/filters/network/kafka/buffer_based_test.h" #include "test/mocks/server/mocks.h" #include "gmock/gmock.h" @@ -17,11 +18,9 @@ namespace NetworkFilters { namespace Kafka { namespace RequestTest { -class RequestTest : public testing::Test { -public: - Buffer::OwnedImpl buffer_; - - template std::shared_ptr serializeAndDeserialize(T request); +class RequestTest : public testing::Test, public MessageBasedTest { +protected: + template std::shared_ptr serializeAndDeserialize(T message); }; class MockMessageListener : public RequestCallback { @@ -35,40 +34,39 @@ public: * Takes an instance of a request, serializes it, then deserializes it. * This method gets executed for every request * version pair. */ -template std::shared_ptr RequestTest::serializeAndDeserialize(T request) { - RequestEncoder encoder{buffer_}; - encoder.encode(request); +template std::shared_ptr RequestTest::serializeAndDeserialize(T message) { + putMessageIntoBuffer(message); std::shared_ptr mock_listener = std::make_shared(); RequestDecoder testee{RequestParserResolver::getDefaultInstance(), {mock_listener}}; - AbstractRequestSharedPtr receivedMessage; + AbstractRequestSharedPtr received_message; EXPECT_CALL(*mock_listener, onMessage(testing::_)) - .WillOnce(testing::SaveArg<0>(&receivedMessage)); + .WillOnce(testing::SaveArg<0>(&received_message)); testee.onData(buffer_); - return std::dynamic_pointer_cast(receivedMessage); + return std::dynamic_pointer_cast(received_message); }; {# - Concrete tests for each request_type and version (field_list). + Concrete tests for each message_type and version (field_list). Each request is naively constructed using some default values (put "string" as std::string, 32 as uint32_t, etc.). #} -{% for request_type in request_types %}{% for field_list in request_type.compute_field_lists() %} -TEST_F(RequestTest, shouldParse{{ request_type.name }}V{{ field_list.version }}) { +{% for message_type in message_types %}{% for field_list in message_type.compute_field_lists() %} +TEST_F(RequestTest, shouldParse{{ message_type.name }}V{{ field_list.version }}) { // given - {{ request_type.name }} data = { {{ field_list.example_value() }} }; - Request<{{ request_type.name }}> request = { { - {{ request_type.get_extra('api_key') }}, {{ field_list.version }}, 0, absl::nullopt }, data }; + {{ message_type.name }} data = { {{ field_list.example_value() }} }; + Request<{{ message_type.name }}> message = { { + {{ message_type.get_extra('api_key') }}, {{ field_list.version }}, 0, absl::nullopt }, data }; // when - auto received = serializeAndDeserialize(request); + auto received = serializeAndDeserialize(message); // then ASSERT_NE(received, nullptr); - ASSERT_EQ(*received, request); + ASSERT_EQ(*received, message); } {% endfor %}{% endfor %} diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/response_codec_response_test_cc.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/response_codec_response_test_cc.j2 new file mode 100644 index 000000000000..ca5a20d1bee6 --- /dev/null +++ b/source/extensions/filters/network/kafka/protocol_code_generator/response_codec_response_test_cc.j2 @@ -0,0 +1,78 @@ +{# + Template for 'response_codec_response_test.cc'. + + Provides integration tests using Kafka codec. + The tests do the following: + - create the message, + - serialize the message into buffer, + - pass the buffer to the codec, + - capture messages received in callback, + - verify that captured messages are identical to the ones sent. +#} +#include "extensions/filters/network/kafka/external/responses.h" +#include "extensions/filters/network/kafka/response_codec.h" + +#include "test/extensions/filters/network/kafka/buffer_based_test.h" +#include "test/extensions/filters/network/kafka/serialization_utilities.h" +#include "test/mocks/server/mocks.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace ResponseCodecResponseTest { + +class ResponseCodecResponseTest : public testing::Test, public MessageBasedTest {}; + +using ResponseCapturingCallback = CapturingCallback; + +{% for message_type in message_types %} + +// Integration test for {{ message_type.name }} messages. + +TEST_F(ResponseCodecResponseTest, shouldHandle{{ message_type.name }}Messages) { + // given + const auto callback = std::make_shared(); + ResponseDecoder testee{ {callback} }; + + using ResponseUnderTest = Response<{{ message_type.name }}>; + + std::vector sent; + int32_t correlation = 0; + + {% for field_list in message_type.compute_field_lists() %} + for (int i = 0; i < 100; ++i ) { + const ResponseMetadata metadata = + { {{ message_type.get_extra('api_key') }}, {{ field_list.version }}, correlation++ }; + const {{ message_type.name }} data = { {{ field_list.example_value() }} }; + const ResponseUnderTest response = {metadata, data}; + putMessageIntoBuffer(response); + testee.expectResponse({{ message_type.get_extra('api_key') }}, {{ field_list.version }}); + sent.push_back(response); + } + {% endfor %} + + // when + testee.onData(buffer_); + + // then + const std::vector& received = callback->getCapturedMessages(); + ASSERT_EQ(received.size(), sent.size()); + + for (size_t i = 0; i < received.size(); ++i) { + const std::shared_ptr response = + std::dynamic_pointer_cast(received[i]); + ASSERT_NE(response, nullptr); + ASSERT_EQ(*response, sent[i]); + } +} +{% endfor %} + +} // namespace ResponseCodecResponseTest +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/response_parser.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/response_parser.j2 new file mode 100644 index 000000000000..f0bb932c612e --- /dev/null +++ b/source/extensions/filters/network/kafka/protocol_code_generator/response_parser.j2 @@ -0,0 +1,20 @@ +{# + Template for top-level structure representing a response in Kafka protocol + (e.g. ProduceResponse). + Rendered templates for each response in Kafka protocol will be put into 'responses.h' file. + + This template handles binding the top-level structure deserializer + (e.g. ProduceResponseV0Deserializer) with ResponseDataParser. + These parsers are then used by ResponseParserResolver instance depending on received Kafka + api key & api version (see 'kafka_response_resolver_cc.j2'). +#} + +{% for version in complex_type.versions %}class {{ complex_type.name }}V{{ version }}Parser: + public ResponseDataParser< + {{ complex_type.name }}, {{ complex_type.name }}V{{ version }}Deserializer>{ +public: + {{ complex_type.name }}V{{ version }}Parser(ResponseContextSharedPtr context): + ResponseDataParser{context} {}; +}; + +{% endfor %} \ No newline at end of file diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/responses_h.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/responses_h.j2 new file mode 100644 index 000000000000..d7a3d6a63391 --- /dev/null +++ b/source/extensions/filters/network/kafka/protocol_code_generator/responses_h.j2 @@ -0,0 +1,36 @@ +{# + Main template for 'responses.h' file. + Gets filled in (by 'contents') with Kafka response structures, deserializers, and parsers. + + For each response we have the following: + - 1 top-level structure corresponding to the response (e.g. `struct FetchResponse`), + - N deserializers for top-level structure, one for each response version, + - N parsers binding each deserializer with parser, + - 0+ child structures (e.g. `struct FetchableTopicResponse`) that are used by the response's + top-level structure, + - deserializers for each child structure. + + So for example, for FetchResponse we have: + - struct FetchResponse, + - FetchResponseV0Deserializer, FetchResponseV1Deserializer, FetchResponseV2Deserializer, etc., + - FetchResponseV0Parser, FetchResponseV1Parser, FetchResponseV2Parser, etc., + - struct FetchableTopicResponse, + - FetchableTopicResponseV0Deserializer, FetchableTopicResponseV1Deserializer, etc. + (because topic data is present in every FetchResponse version), + - struct FetchablePartitionResponse, + - FetchablePartitionResponseV0Deserializer, FetchablePartitionResponseV1Deserializer, etc. + (because partition data is present in every FetchableTopicResponse version). + - AbortedTransaction & its Deserializers (starting with version 4). +#} +#pragma once +#include "extensions/filters/network/kafka/kafka_response.h" +#include "extensions/filters/network/kafka/kafka_response_parser.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { + +{{ contents }} + +}}}} diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/responses_test_cc.j2 b/source/extensions/filters/network/kafka/protocol_code_generator/responses_test_cc.j2 new file mode 100644 index 000000000000..89e841dae336 --- /dev/null +++ b/source/extensions/filters/network/kafka/protocol_code_generator/responses_test_cc.j2 @@ -0,0 +1,79 @@ +{# + Template for response serialization/deserialization tests. + For every response, we want to check if it can be serialized and deserialized properly. +#} + +#include "extensions/filters/network/kafka/external/responses.h" +#include "extensions/filters/network/kafka/response_codec.h" + +#include "test/extensions/filters/network/kafka/buffer_based_test.h" +#include "test/mocks/server/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace ResponseTest { + +class ResponseTest : public testing::Test, public MessageBasedTest { +protected: + template std::shared_ptr serializeAndDeserialize(T message); +}; + +class MockMessageListener : public ResponseCallback { +public: + MOCK_METHOD1(onMessage, void(AbstractResponseSharedPtr)); + MOCK_METHOD1(onFailedParse, void(ResponseMetadataSharedPtr)); +}; + +/** + * Helper method. + * Takes an instance of a response, serializes it, then deserializes it. + * This method gets executed for every response * version pair. + */ +template std::shared_ptr ResponseTest::serializeAndDeserialize(T message) { + putMessageIntoBuffer(message); + + std::shared_ptr mock_listener = std::make_shared(); + ResponseDecoder testee{ {mock_listener} }; + const ResponseMetadata& metadata = message.metadata_; + testee.expectResponse(metadata.api_key_, metadata.api_version_); + + AbstractResponseSharedPtr received_message; + EXPECT_CALL(*mock_listener, onMessage(testing::_)) + .WillOnce(testing::SaveArg<0>(&received_message)); + + testee.onData(buffer_); + + return std::dynamic_pointer_cast(received_message); +}; + +{# + Concrete tests for each message_type and version (field_list). + Each response is naively constructed using some default values + (put "string" as std::string, 32 as uint32_t, etc.). +#} +{% for message_type in message_types %}{% for field_list in message_type.compute_field_lists() %} +TEST_F(ResponseTest, shouldParse{{ message_type.name }}V{{ field_list.version }}) { + // given + {{ message_type.name }} data = { {{ field_list.example_value() }} }; + Response<{{ message_type.name }}> message = { { + {{ message_type.get_extra('api_key') }}, {{ field_list.version }}, 0 }, data }; + + // when + auto received = serializeAndDeserialize(message); + + // then + ASSERT_NE(received, nullptr); + ASSERT_EQ(*received, message); +} +{% endfor %}{% endfor %} + +} // namespace ResponseTest +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/kafka/response_codec.cc b/source/extensions/filters/network/kafka/response_codec.cc new file mode 100644 index 000000000000..7b6dd20c97f6 --- /dev/null +++ b/source/extensions/filters/network/kafka/response_codec.cc @@ -0,0 +1,112 @@ +#include "extensions/filters/network/kafka/response_codec.h" + +#include "common/common/stack_array.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { + +void ResponseInitialParserFactory::expectResponse(const int16_t api_key, + const int16_t api_version) { + expected_responses_.push({api_key, api_version}); +} + +ResponseParserSharedPtr +ResponseInitialParserFactory::create(const ResponseParserResolver& parser_resolver) { + const ExpectedResponseSpec spec = getNextResponseSpec(); + const auto context = std::make_shared(spec.first, spec.second); + return std::make_shared(context, parser_resolver); +} + +ExpectedResponseSpec ResponseInitialParserFactory::getNextResponseSpec() { + if (!expected_responses_.empty()) { + const ExpectedResponseSpec spec = expected_responses_.front(); + expected_responses_.pop(); + return spec; + } else { + // Response data should always be present in expected responses before response is to be parsed. + throw EnvoyException("attempted to create a response parser while no responses are expected"); + } +} + +void ResponseDecoder::expectResponse(const int16_t api_key, const int16_t api_version) { + factory_->expectResponse(api_key, api_version); +}; + +void ResponseDecoder::onData(Buffer::Instance& data) { + // Convert buffer to slices and pass them to `doParse`. + uint64_t num_slices = data.getRawSlices(nullptr, 0); + STACK_ARRAY(slices, Buffer::RawSlice, num_slices); + data.getRawSlices(slices.begin(), num_slices); + for (const Buffer::RawSlice& slice : slices) { + doParse(slice); + } +} + +/** + * Main parse loop: + * - initialize parser, if it is not present (using information stored in `factory_`) + * - forward data to current parser, + * - receive parser response: + * -- if still waiting, do nothing (we wait for more data), + * -- if a parser is given, replace current parser with the new one, and it the rest of the data + * -- if a message is given: + * --- notify callbacks, + * --- clean current parser. + */ +void ResponseDecoder::doParse(const Buffer::RawSlice& slice) { + const char* bytes = reinterpret_cast(slice.mem_); + absl::string_view data = {bytes, slice.len_}; + + while (!data.empty()) { + + // Re-initialize the parser. + if (!current_parser_) { + current_parser_ = factory_->create(response_parser_resolver_); + } + + // Feed the data to the parser. + ResponseParseResponse result = current_parser_->parse(data); + // This loop guarantees that parsers consuming 0 bytes also get processed in this invocation. + while (result.hasData()) { + if (!result.next_parser_) { + + // Next parser is not present, so we have finished parsing a message. + // Depending on whether the parse was successful, invoke the correct callback. + if (result.message_) { + for (auto& callback : callbacks_) { + callback->onMessage(result.message_); + } + } else { + for (auto& callback : callbacks_) { + callback->onFailedParse(result.failure_data_); + } + } + + // As we finished parsing this response, return to outer loop. + // If there is more data, the parser will be re-initialized. + current_parser_ = nullptr; + break; + } else { + + // The next parser that's supposed to consume the rest of payload was given. + current_parser_ = result.next_parser_; + } + + // Keep parsing the data. + result = current_parser_->parse(data); + } + } +} + +void ResponseEncoder::encode(const AbstractResponse& message) { + const uint32_t size = htobe32(message.computeSize()); + output_.add(&size, sizeof(size)); // Encode data length. + message.encode(output_); // Encode data. +} + +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/kafka/response_codec.h b/source/extensions/filters/network/kafka/response_codec.h new file mode 100644 index 000000000000..55d1e2ac58a5 --- /dev/null +++ b/source/extensions/filters/network/kafka/response_codec.h @@ -0,0 +1,148 @@ +#pragma once + +#include + +#include "extensions/filters/network/kafka/codec.h" +#include "extensions/filters/network/kafka/kafka_response_parser.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { + +/** + * Callback invoked when response is successfully decoded. + */ +class ResponseCallback { +public: + virtual ~ResponseCallback() = default; + + /** + * Callback method invoked when response is successfully decoded. + * @param response response that has been decoded. + */ + virtual void onMessage(AbstractResponseSharedPtr response) PURE; + + /** + * Callback method invoked when response could not be decoded. + * Invoked after all response's bytes have been consumed. + */ + virtual void onFailedParse(ResponseMetadataSharedPtr failure_data) PURE; +}; + +using ResponseCallbackSharedPtr = std::shared_ptr; + +// Helper container for data stored in ResponseInitialParserFactory. +using ExpectedResponseSpec = std::pair; + +/** + * Provides initial parser for responses. + * Response information needs to be registered with this factory beforehand, as payloads do not + * carry message type information. + */ +class ResponseInitialParserFactory { +public: + virtual ~ResponseInitialParserFactory() = default; + + /** + * Creates parser with given context. + */ + virtual ResponseParserSharedPtr create(const ResponseParserResolver& parser_resolver); + + /** + * Registers next expected message. + * @param api_key response's api key. + * @param api_version response's api version. + */ + void expectResponse(const int16_t api_key, const int16_t api_version); + +private: + ExpectedResponseSpec getNextResponseSpec(); + + std::queue expected_responses_; +}; + +using ResponseInitialParserFactorySharedPtr = std::shared_ptr; + +/** + * Decoder that decodes Kafka responses. + * When a response is decoded, the callbacks are notified, in order. + * + * This decoder uses chain of parsers to parse fragments of a response. + * Each parser along the line returns the fully parsed message or the next parser. + * Stores parse state (as large message's payload can be provided through multiple `onData` calls). + * + * As Kafka protocol does not carry response type data, it is necessary to register expected message + * type beforehand with `expectResponse`. + */ +class ResponseDecoder : public MessageDecoder, public Logger::Loggable { +public: + /** + * Creates a decoder that will notify provided callbacks. + * @param callbacks callbacks to be invoked (in order). + */ + ResponseDecoder(const std::vector callbacks) + : ResponseDecoder{std::make_shared(), + ResponseParserResolver::getDefaultInstance(), callbacks} {}; + + /** + * Visible for testing. + * Allows injecting parser resolver. + */ + ResponseDecoder(const ResponseInitialParserFactorySharedPtr factory, + const ResponseParserResolver& response_parser_resolver, + const std::vector callbacks) + : factory_{factory}, response_parser_resolver_{response_parser_resolver}, callbacks_{ + callbacks} {}; + + /** + * Registers an expected message. + * After all the previous expected responses have been parsed, the coded will use this data to + * create a parser for next message. + * @param api_key api key of the next response to be parsed. + * @param api_version api version of the next response to be parsed. + */ + void expectResponse(const int16_t api_key, const int16_t api_version); + + /** + * Consumes all data present in a buffer. + * If a response can be successfully parsed, then callbacks get notified with parsed response. + * Updates decoder state. + * Can throw if data is received, but the decoder is not expecting any response. + * Impl note: similar to redis codec, which also keeps state. + */ + void onData(Buffer::Instance& data) override; + +private: + void doParse(const Buffer::RawSlice& slice); + + ResponseInitialParserFactorySharedPtr factory_; + const ResponseParserResolver& response_parser_resolver_; + const std::vector callbacks_; + + ResponseParserSharedPtr current_parser_; +}; + +/** + * Encodes responses into underlying buffer. + */ +class ResponseEncoder : public MessageEncoder { +public: + /** + * Wraps buffer with encoder. + */ + ResponseEncoder(Buffer::Instance& output) : output_(output) {} + + /** + * Encodes response into wrapped buffer. + */ + void encode(const AbstractResponse& message) override; + +private: + Buffer::Instance& output_; +}; + +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 b/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 index 04e1a55beb85..84a27de86c3d 100644 --- a/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 +++ b/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 @@ -64,10 +64,10 @@ struct CompositeResultWith{{ field_count }}Fields { } }; -typedef CompositeDeserializerWith{{ field_count }}Delegates< +using TestCompositeDeserializer{{ field_count }} = + CompositeDeserializerWith{{ field_count }}Delegates< CompositeResultWith{{ field_count }}Fields - {% for field in range(1, field_count + 1) %}, StringDeserializer{% endfor %} -> TestCompositeDeserializer{{ field_count }}; + {% for field in range(1, field_count + 1) %}, StringDeserializer{% endfor %}>; TEST(CompositeDeserializerWith{{ field_count }}Delegates, EmptyBufferShouldNotBeReady) { // given diff --git a/test/extensions/filters/network/kafka/BUILD b/test/extensions/filters/network/kafka/BUILD index 9bb39da43407..00a2d0f4d70e 100644 --- a/test/extensions/filters/network/kafka/BUILD +++ b/test/extensions/filters/network/kafka/BUILD @@ -12,13 +12,22 @@ load( envoy_package() +envoy_cc_test_library( + name = "buffer_based_test_lib", + srcs = [], + hdrs = ["buffer_based_test.h"], + deps = [ + "//source/common/buffer:buffer_lib", + "//source/extensions/filters/network/kafka:serialization_lib", + ], +) + envoy_cc_test_library( name = "serialization_utilities_lib", srcs = ["serialization_utilities.cc"], hdrs = ["serialization_utilities.h"], deps = [ "//source/common/buffer:buffer_lib", - "//source/extensions/filters/network/kafka:kafka_request_codec_lib", "//source/extensions/filters/network/kafka:serialization_lib", ], ) @@ -63,6 +72,7 @@ envoy_extension_cc_test( srcs = ["kafka_request_parser_test.cc"], extension_name = "envoy.filters.network.kafka", deps = [ + ":buffer_based_test_lib", ":serialization_utilities_lib", "//source/extensions/filters/network/kafka:kafka_request_parser_lib", "//test/mocks/server:server_mocks", @@ -74,6 +84,7 @@ envoy_extension_cc_test( srcs = ["request_codec_unit_test.cc"], extension_name = "envoy.filters.network.kafka", deps = [ + ":buffer_based_test_lib", "//source/extensions/filters/network/kafka:kafka_request_codec_lib", "//test/mocks/server:server_mocks", ], @@ -84,6 +95,7 @@ envoy_extension_cc_test( srcs = ["request_codec_integration_test.cc"], extension_name = "envoy.filters.network.kafka", deps = [ + ":buffer_based_test_lib", ":serialization_utilities_lib", "//source/extensions/filters/network/kafka:kafka_request_codec_lib", "//test/mocks/server:server_mocks", @@ -95,6 +107,7 @@ envoy_extension_cc_test( srcs = ["external/request_codec_request_test.cc"], extension_name = "envoy.filters.network.kafka", deps = [ + ":buffer_based_test_lib", ":serialization_utilities_lib", "//source/extensions/filters/network/kafka:kafka_request_codec_lib", "//test/mocks/server:server_mocks", @@ -106,13 +119,14 @@ envoy_extension_cc_test( srcs = ["external/requests_test.cc"], extension_name = "envoy.filters.network.kafka", deps = [ + ":buffer_based_test_lib", "//source/extensions/filters/network/kafka:kafka_request_codec_lib", "//test/mocks/server:server_mocks", ], ) genrule( - name = "requests_test_generator", + name = "request_test_generator", srcs = [ "@kafka_source//:request_protocol_files", ], @@ -121,7 +135,8 @@ genrule( "external/request_codec_request_test.cc", ], cmd = """ - ./$(location //source/extensions/filters/network/kafka:kafka_code_generator) generate-test \ + ./$(location //source/extensions/filters/network/kafka:kafka_code_generator) \ + generate-test request \ $(location external/requests_test.cc) $(location external/request_codec_request_test.cc) \ $(SRCS) """, @@ -129,3 +144,81 @@ genrule( "//source/extensions/filters/network/kafka:kafka_code_generator", ], ) + +envoy_extension_cc_test( + name = "kafka_response_parser_test", + srcs = ["kafka_response_parser_test.cc"], + extension_name = "envoy.filters.network.kafka", + deps = [ + ":buffer_based_test_lib", + ":serialization_utilities_lib", + "//source/extensions/filters/network/kafka:kafka_response_parser_lib", + "//test/mocks/server:server_mocks", + ], +) + +envoy_extension_cc_test( + name = "response_codec_unit_test", + srcs = ["response_codec_unit_test.cc"], + extension_name = "envoy.filters.network.kafka", + deps = [ + ":buffer_based_test_lib", + "//source/extensions/filters/network/kafka:kafka_response_codec_lib", + "//test/mocks/server:server_mocks", + ], +) + +envoy_extension_cc_test( + name = "response_codec_integration_test", + srcs = ["response_codec_integration_test.cc"], + extension_name = "envoy.filters.network.kafka", + deps = [ + ":buffer_based_test_lib", + ":serialization_utilities_lib", + "//source/extensions/filters/network/kafka:kafka_response_codec_lib", + "//test/mocks/server:server_mocks", + ], +) + +envoy_extension_cc_test( + name = "response_codec_response_test", + srcs = ["external/response_codec_response_test.cc"], + extension_name = "envoy.filters.network.kafka", + deps = [ + ":buffer_based_test_lib", + ":serialization_utilities_lib", + "//source/extensions/filters/network/kafka:kafka_response_codec_lib", + "//test/mocks/server:server_mocks", + ], +) + +envoy_extension_cc_test( + name = "responses_test", + srcs = ["external/responses_test.cc"], + extension_name = "envoy.filters.network.kafka", + deps = [ + ":buffer_based_test_lib", + "//source/extensions/filters/network/kafka:kafka_response_codec_lib", + "//test/mocks/server:server_mocks", + ], +) + +genrule( + name = "response_test_generator", + srcs = [ + "@kafka_source//:response_protocol_files", + ], + outs = [ + "external/responses_test.cc", + "external/response_codec_response_test.cc", + ], + cmd = """ + ./$(location //source/extensions/filters/network/kafka:kafka_code_generator) \ + generate-test response \ + $(location external/responses_test.cc) $(location external/response_codec_response_test.cc) \ + $(SRCS) + """, + tools = [ + "//source/extensions/filters/network/kafka:kafka_code_generator", + ], +) diff --git a/test/extensions/filters/network/kafka/buffer_based_test.h b/test/extensions/filters/network/kafka/buffer_based_test.h new file mode 100644 index 000000000000..b61dc33a6ec4 --- /dev/null +++ b/test/extensions/filters/network/kafka/buffer_based_test.h @@ -0,0 +1,57 @@ +#pragma once + +#include "common/buffer/buffer_impl.h" +#include "common/common/stack_array.h" + +#include "extensions/filters/network/kafka/serialization.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { + +// Common utilities for various Kafka buffer-related tests. + +/** + * Utility superclass that keeps a buffer that can be played with during the test. + */ +class BufferBasedTest { +protected: + const char* getBytes() { + uint64_t num_slices = buffer_.getRawSlices(nullptr, 0); + STACK_ARRAY(slices, Buffer::RawSlice, num_slices); + buffer_.getRawSlices(slices.begin(), num_slices); + return reinterpret_cast((slices[0]).mem_); + } + + template uint32_t putIntoBuffer(const T& arg) { + EncodingContext encoder_{-1}; // Context's api_version is not used when serializing primitives. + return encoder_.encode(arg, buffer_); + } + + absl::string_view putGarbageIntoBuffer(uint32_t size = 1024) { + putIntoBuffer(Bytes(size)); + return {getBytes(), size}; + } + + Buffer::OwnedImpl buffer_; +}; + +/** + * Utility superclass that keeps a buffer and can put messages into buffer. + * @param Encoder class used for encoding messages into buffer + */ +template class MessageBasedTest : public BufferBasedTest { +protected: + template void putMessageIntoBuffer(const T& arg) { + Encoder encoder{buffer_}; + encoder.encode(arg); + } +}; + +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/kafka/kafka_request_parser_test.cc b/test/extensions/filters/network/kafka/kafka_request_parser_test.cc index 949e306e42f9..a70797742806 100644 --- a/test/extensions/filters/network/kafka/kafka_request_parser_test.cc +++ b/test/extensions/filters/network/kafka/kafka_request_parser_test.cc @@ -1,5 +1,6 @@ #include "extensions/filters/network/kafka/kafka_request_parser.h" +#include "test/extensions/filters/network/kafka/buffer_based_test.h" #include "test/extensions/filters/network/kafka/serialization_utilities.h" #include "test/mocks/server/mocks.h" @@ -16,28 +17,7 @@ namespace KafkaRequestParserTest { const int32_t FAILED_DESERIALIZER_STEP = 13; -class KafkaRequestParserTest : public testing::Test { -public: - const char* getBytes() { - uint64_t num_slices = buffer_.getRawSlices(nullptr, 0); - STACK_ARRAY(slices, Buffer::RawSlice, num_slices); - buffer_.getRawSlices(slices.begin(), num_slices); - return reinterpret_cast((slices[0]).mem_); - } - - template uint32_t putIntoBuffer(const T arg) { - EncodingContext encoder_{-1}; // Context's api_version is not used when serializing primitives. - return encoder_.encode(arg, buffer_); - } - - absl::string_view putGarbageIntoBuffer(uint32_t size = 10000) { - putIntoBuffer(Bytes(size)); - return {getBytes(), size}; - } - -protected: - Buffer::OwnedImpl buffer_; -}; +class KafkaRequestParserTest : public testing::Test, public BufferBasedTest {}; class MockRequestParserResolver : public RequestParserResolver { public: diff --git a/test/extensions/filters/network/kafka/kafka_response_parser_test.cc b/test/extensions/filters/network/kafka/kafka_response_parser_test.cc new file mode 100644 index 000000000000..de1ab3edf013 --- /dev/null +++ b/test/extensions/filters/network/kafka/kafka_response_parser_test.cc @@ -0,0 +1,170 @@ +#include "extensions/filters/network/kafka/kafka_response_parser.h" + +#include "test/extensions/filters/network/kafka/buffer_based_test.h" +#include "test/extensions/filters/network/kafka/serialization_utilities.h" +#include "test/mocks/server/mocks.h" + +#include "gmock/gmock.h" + +using testing::_; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace KafkaResponseParserTest { + +const int32_t FAILED_DESERIALIZER_STEP = 13; + +class KafkaResponseParserTest : public testing::Test, public BufferBasedTest {}; + +class MockResponseParserResolver : public ResponseParserResolver { +public: + MockResponseParserResolver(){}; + MOCK_CONST_METHOD1(createParser, ResponseParserSharedPtr(ResponseContextSharedPtr)); +}; + +class MockParser : public ResponseParser { +public: + ResponseParseResponse parse(absl::string_view&) override { + throw new EnvoyException("should not be invoked"); + } +}; + +TEST_F(KafkaResponseParserTest, ResponseHeaderParserShouldExtractHeaderAndResolveNextParser) { + // given + const MockResponseParserResolver parser_resolver; + const ResponseParserSharedPtr parser{new MockParser{}}; + EXPECT_CALL(parser_resolver, createParser(_)).WillOnce(Return(parser)); + + ResponseContextSharedPtr context = std::make_shared(0, 0); + ResponseHeaderParser testee{context, parser_resolver}; + + const int32_t payload_length = 100; + const int32_t correlation_id = 1234; + uint32_t header_len = 0; + header_len += putIntoBuffer(payload_length); + header_len += putIntoBuffer(correlation_id); // Insert correlation id. + + const absl::string_view orig_data = putGarbageIntoBuffer(); + absl::string_view data = orig_data; + + // when + const ResponseParseResponse result = testee.parse(data); + + // then + ASSERT_EQ(result.hasData(), true); + ASSERT_EQ(result.next_parser_, parser); + ASSERT_EQ(result.message_, nullptr); + ASSERT_EQ(result.failure_data_, nullptr); + + ASSERT_EQ(testee.contextForTest()->correlation_id_, correlation_id); + ASSERT_EQ(testee.contextForTest()->remaining_response_size_, + payload_length - sizeof(correlation_id)); + + assertStringViewIncrement(data, orig_data, header_len); +} + +TEST_F(KafkaResponseParserTest, ResponseDataParserShoulRethrowDeserializerExceptionsDuringFeeding) { + // given + + // This deserializer throws during feeding. + class ThrowingDeserializer : public Deserializer { + public: + uint32_t feed(absl::string_view&) override { + // Move some pointers to simulate data consumption. + throw EnvoyException("feed"); + }; + + bool ready() const override { throw std::runtime_error("should not be invoked at all"); }; + + int32_t get() const override { throw std::runtime_error("should not be invoked at all"); }; + }; + + ResponseContextSharedPtr context = std::make_shared(0, 0); + ResponseDataParser testee{context}; + + absl::string_view data = putGarbageIntoBuffer(); + + // when + bool caught = false; + try { + testee.parse(data); + } catch (EnvoyException& e) { + caught = true; + } + + // then + ASSERT_EQ(caught, true); +} + +// This deserializer consumes FAILED_DESERIALIZER_STEP bytes and returns 0 +class SomeBytesDeserializer : public Deserializer { +public: + uint32_t feed(absl::string_view& data) override { + data = {data.data() + FAILED_DESERIALIZER_STEP, data.size() - FAILED_DESERIALIZER_STEP}; + return FAILED_DESERIALIZER_STEP; + }; + + bool ready() const override { return true; }; + + int32_t get() const override { return 0; }; +}; + +TEST_F(KafkaResponseParserTest, + ResponseDataParserShouldHandleDeserializerReturningReadyButLeavingData) { + // given + const int32_t message_size = 1024; // There are still 1024 bytes to read to complete the message. + ResponseContextSharedPtr context = std::make_shared(0, 0); + context->remaining_response_size_ = message_size; + + ResponseDataParser testee{context}; + + const absl::string_view orig_data = putGarbageIntoBuffer(); + absl::string_view data = orig_data; + + // when + const ResponseParseResponse result = testee.parse(data); + + // then + ASSERT_EQ(result.hasData(), true); + ASSERT_NE(std::dynamic_pointer_cast(result.next_parser_), nullptr); + ASSERT_EQ(result.message_, nullptr); + ASSERT_EQ(result.failure_data_, nullptr); + + ASSERT_EQ(testee.contextForTest()->remaining_response_size_, + message_size - FAILED_DESERIALIZER_STEP); + + assertStringViewIncrement(data, orig_data, FAILED_DESERIALIZER_STEP); +} + +TEST_F(KafkaResponseParserTest, SentinelResponseParserShouldConsumeDataUntilEndOfMessage) { + // given + const int32_t response_len = 1000; + ResponseContextSharedPtr context = std::make_shared(0, 0); + context->remaining_response_size_ = response_len; + SentinelResponseParser testee{context}; + + const absl::string_view orig_data = putGarbageIntoBuffer(response_len * 2); + absl::string_view data = orig_data; + + // when + const ResponseParseResponse result = testee.parse(data); + + // then + ASSERT_EQ(result.hasData(), true); + ASSERT_EQ(result.next_parser_, nullptr); + ASSERT_EQ(result.message_, nullptr); + ASSERT_NE(std::dynamic_pointer_cast(result.failure_data_), nullptr); + + ASSERT_EQ(testee.contextForTest()->remaining_response_size_, 0); + + assertStringViewIncrement(data, orig_data, response_len); +} + +} // namespace KafkaResponseParserTest +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/kafka/request_codec_integration_test.cc b/test/extensions/filters/network/kafka/request_codec_integration_test.cc index 6d087f310466..74dd54afe02e 100644 --- a/test/extensions/filters/network/kafka/request_codec_integration_test.cc +++ b/test/extensions/filters/network/kafka/request_codec_integration_test.cc @@ -1,5 +1,6 @@ #include "extensions/filters/network/kafka/request_codec.h" +#include "test/extensions/filters/network/kafka/buffer_based_test.h" #include "test/extensions/filters/network/kafka/serialization_utilities.h" #include "test/mocks/server/mocks.h" @@ -11,12 +12,11 @@ namespace NetworkFilters { namespace Kafka { namespace RequestCodecIntegrationTest { -class RequestCodecIntegrationTest : public testing::Test { -protected: - template void putInBuffer(T arg); +class RequestCodecIntegrationTest : public testing::Test, + public MessageBasedTest {}; - Buffer::OwnedImpl buffer_; -}; +using RequestCapturingCallback = + CapturingCallback; // Other request types are tested in (generated) 'request_codec_request_test.cc'. TEST_F(RequestCodecIntegrationTest, shouldProduceAbortedMessageOnUnknownData) { @@ -29,15 +29,15 @@ TEST_F(RequestCodecIntegrationTest, shouldProduceAbortedMessageOnUnknownData) { const int16_t api_key = static_cast(base_api_key + i); const RequestHeader header = {api_key, 0, 0, "client-id"}; const std::vector data = std::vector(1024); - putInBuffer(Request>{header, data}); + const auto message = Request>{header, data}; + putMessageIntoBuffer(message); sent_headers.push_back(header); } const InitialParserFactory& initial_parser_factory = InitialParserFactory::getDefaultInstance(); const RequestParserResolver& request_parser_resolver = RequestParserResolver::getDefaultInstance(); - const CapturingRequestCallbackSharedPtr request_callback = - std::make_shared(); + const auto request_callback = std::make_shared(); RequestDecoder testee{initial_parser_factory, request_parser_resolver, {request_callback}}; @@ -45,7 +45,7 @@ TEST_F(RequestCodecIntegrationTest, shouldProduceAbortedMessageOnUnknownData) { testee.onData(buffer_); // then - ASSERT_EQ(request_callback->getCaptured().size(), 0); + ASSERT_EQ(request_callback->getCapturedMessages().size(), 0); const std::vector& parse_failures = request_callback->getParseFailures(); @@ -59,12 +59,6 @@ TEST_F(RequestCodecIntegrationTest, shouldProduceAbortedMessageOnUnknownData) { } } -// Helper function. -template void RequestCodecIntegrationTest::putInBuffer(T arg) { - RequestEncoder encoder{buffer_}; - encoder.encode(arg); -} - } // namespace RequestCodecIntegrationTest } // namespace Kafka } // namespace NetworkFilters diff --git a/test/extensions/filters/network/kafka/request_codec_unit_test.cc b/test/extensions/filters/network/kafka/request_codec_unit_test.cc index d26c0dfa08a5..ecd1eaf88e4f 100644 --- a/test/extensions/filters/network/kafka/request_codec_unit_test.cc +++ b/test/extensions/filters/network/kafka/request_codec_unit_test.cc @@ -1,5 +1,6 @@ #include "extensions/filters/network/kafka/request_codec.h" +#include "test/extensions/filters/network/kafka/buffer_based_test.h" #include "test/mocks/server/mocks.h" #include "gmock/gmock.h" @@ -45,12 +46,8 @@ class MockRequestCallback : public RequestCallback { using MockRequestCallbackSharedPtr = std::shared_ptr; -class RequestCodecUnitTest : public testing::Test { +class RequestCodecUnitTest : public testing::Test, public BufferBasedTest { protected: - template void putInBuffer(T arg); - - Buffer::OwnedImpl buffer_; - MockParserFactory initial_parser_factory_{}; MockRequestParserResolver parser_resolver_{}; MockRequestCallbackSharedPtr request_callback_{std::make_shared()}; @@ -61,27 +58,30 @@ RequestParseResponse consumeOneByte(absl::string_view& data) { return RequestParseResponse::stillWaiting(); } -TEST_F(RequestCodecUnitTest, shouldDoNothingIfParserNeverReturnsMessage) { +TEST_F(RequestCodecUnitTest, shouldDoNothingIfParserReturnsWaiting) { // given - putInBuffer(Request{{}, 0}); + putGarbageIntoBuffer(); MockParserSharedPtr parser = std::make_shared(); EXPECT_CALL(*parser, parse(_)).Times(AnyNumber()).WillRepeatedly(Invoke(consumeOneByte)); EXPECT_CALL(initial_parser_factory_, create(_)).WillOnce(Return(parser)); + EXPECT_CALL(*request_callback_, onMessage(_)).Times(0); + EXPECT_CALL(*request_callback_, onFailedParse(_)).Times(0); + RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; // when testee.onData(buffer_); // then - // There were no interactions with `request_callback`. + // There were no interactions with `request_callback_`. } TEST_F(RequestCodecUnitTest, shouldUseNewParserAsResponse) { // given - putInBuffer(Request{{}, 0}); + putGarbageIntoBuffer(); MockParserSharedPtr parser1 = std::make_shared(); MockParserSharedPtr parser2 = std::make_shared(); @@ -92,24 +92,26 @@ TEST_F(RequestCodecUnitTest, shouldUseNewParserAsResponse) { EXPECT_CALL(initial_parser_factory_, create(_)).WillOnce(Return(parser1)); + EXPECT_CALL(*request_callback_, onMessage(_)).Times(0); + EXPECT_CALL(*request_callback_, onFailedParse(_)).Times(0); + RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; // when testee.onData(buffer_); // then - // There were no interactions with `request_callback`. + // There were no interactions with `request_callback_`. } TEST_F(RequestCodecUnitTest, shouldPassParsedMessageToCallbackAndReinitialize) { // given - putInBuffer(Request{{}, 0}); + putGarbageIntoBuffer(); + + AbstractRequestSharedPtr message = std::make_shared>(RequestHeader(), 0); MockParserSharedPtr parser1 = std::make_shared(); - RequestParseFailureSharedPtr failure_data = - std::make_shared(RequestHeader()); - EXPECT_CALL(*parser1, parse(_)) - .WillOnce(Return(RequestParseResponse::parseFailure(failure_data))); + EXPECT_CALL(*parser1, parse(_)).WillOnce(Return(RequestParseResponse::parsedMessage(message))); MockParserSharedPtr parser2 = std::make_shared(); EXPECT_CALL(*parser2, parse(_)).Times(AnyNumber()).WillRepeatedly(Invoke(consumeOneByte)); @@ -118,7 +120,8 @@ TEST_F(RequestCodecUnitTest, shouldPassParsedMessageToCallbackAndReinitialize) { .WillOnce(Return(parser1)) .WillOnce(Return(parser2)); - EXPECT_CALL(*request_callback_, onFailedParse(failure_data)); + EXPECT_CALL(*request_callback_, onMessage(message)); + EXPECT_CALL(*request_callback_, onFailedParse(_)).Times(0); RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; @@ -126,16 +129,17 @@ TEST_F(RequestCodecUnitTest, shouldPassParsedMessageToCallbackAndReinitialize) { testee.onData(buffer_); // then - // There was only one message sent to `request_callback`. + // `request_callback_` had `onFailedParse` invoked once with matching argument. } TEST_F(RequestCodecUnitTest, shouldPassParseFailureDataToCallbackAndReinitialize) { // given - putInBuffer(Request{{}, 0}); + putGarbageIntoBuffer(); - MockParserSharedPtr parser1 = std::make_shared(); RequestParseFailureSharedPtr failure_data = std::make_shared(RequestHeader()); + + MockParserSharedPtr parser1 = std::make_shared(); EXPECT_CALL(*parser1, parse(_)) .WillOnce(Return(RequestParseResponse::parseFailure(failure_data))); @@ -146,6 +150,7 @@ TEST_F(RequestCodecUnitTest, shouldPassParseFailureDataToCallbackAndReinitialize .WillOnce(Return(parser1)) .WillOnce(Return(parser2)); + EXPECT_CALL(*request_callback_, onMessage(_)).Times(0); EXPECT_CALL(*request_callback_, onFailedParse(failure_data)); RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; @@ -154,12 +159,12 @@ TEST_F(RequestCodecUnitTest, shouldPassParseFailureDataToCallbackAndReinitialize testee.onData(buffer_); // then - // `request_callback` had `onFailedParse` invoked once with matching argument. + // `request_callback_` had `onFailedParse` invoked once with matching argument. } TEST_F(RequestCodecUnitTest, shouldInvokeParsersEvenIfTheyDoNotConsumeZeroBytes) { // given - putInBuffer(Request{{}, 0}); + putGarbageIntoBuffer(); MockParserSharedPtr parser1 = std::make_shared(); MockParserSharedPtr parser2 = std::make_shared(); @@ -194,16 +199,10 @@ TEST_F(RequestCodecUnitTest, shouldInvokeParsersEvenIfTheyDoNotConsumeZeroBytes) testee.onData(buffer_); // then - // `request_callback` was invoked only once. + // `request_callback_` had `onFailedParse` invoked once with matching argument. // After that, `parser3` was created and passed remaining data (that should have been empty). } -// Helper function. -template void RequestCodecUnitTest::putInBuffer(T arg) { - RequestEncoder encoder{buffer_}; - encoder.encode(arg); -} - } // namespace RequestCodecUnitTest } // namespace Kafka } // namespace NetworkFilters diff --git a/test/extensions/filters/network/kafka/response_codec_integration_test.cc b/test/extensions/filters/network/kafka/response_codec_integration_test.cc new file mode 100644 index 000000000000..fb34e942b831 --- /dev/null +++ b/test/extensions/filters/network/kafka/response_codec_integration_test.cc @@ -0,0 +1,79 @@ +#include "extensions/filters/network/kafka/response_codec.h" + +#include "test/extensions/filters/network/kafka/buffer_based_test.h" +#include "test/extensions/filters/network/kafka/serialization_utilities.h" +#include "test/mocks/server/mocks.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace ResponseCodecIntegrationTest { + +class ResponseCodecIntegrationTest : public testing::Test, + public MessageBasedTest {}; + +using ResponseCapturingCallback = + CapturingCallback; + +// Other response types are tested in (generated) 'response_codec_response_test.cc'. +TEST_F(ResponseCodecIntegrationTest, shouldProduceAbortedMessageOnUnknownData) { + // given + const auto callback = std::make_shared(); + ResponseDecoder testee{{callback}}; + + // As real api keys have values below 100, the messages generated in this loop should not be + // recognized by the codec. + const int16_t base_api_key = 100; + std::vector sent; + for (int16_t i = 0; i < 1000; ++i) { + const int16_t api_key = static_cast(base_api_key + i); + const int16_t api_version = 0; + const ResponseMetadata metadata = {api_key, api_version, 0}; + const std::vector data = std::vector(1024); + const auto message = Response>{metadata, data}; + putMessageIntoBuffer(message); + sent.push_back(metadata); + // We need to register the response, so the parser knows what to expect. + testee.expectResponse(api_key, api_version); + } + + // when + testee.onData(buffer_); + + // then + ASSERT_EQ(callback->getCapturedMessages().size(), 0); + + const std::vector& parse_failures = callback->getParseFailures(); + ASSERT_EQ(parse_failures.size(), sent.size()); + for (size_t i = 0; i < parse_failures.size(); ++i) { + ASSERT_EQ(*(parse_failures[i]), sent[i]); + } +} + +TEST_F(ResponseCodecIntegrationTest, shouldThrowIfAttemptingToParseResponseButNothingIsExpected) { + // given + const auto callback = std::make_shared(); + ResponseDecoder testee{{callback}}; + + putGarbageIntoBuffer(); + + // when + bool caught = false; + try { + testee.onData(buffer_); + } catch (EnvoyException& e) { + caught = true; + } + + // then + ASSERT_EQ(caught, true); +} + +} // namespace ResponseCodecIntegrationTest +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/kafka/response_codec_unit_test.cc b/test/extensions/filters/network/kafka/response_codec_unit_test.cc new file mode 100644 index 000000000000..c8fcd016cce3 --- /dev/null +++ b/test/extensions/filters/network/kafka/response_codec_unit_test.cc @@ -0,0 +1,172 @@ +#include "extensions/filters/network/kafka/response_codec.h" + +#include "test/extensions/filters/network/kafka/buffer_based_test.h" +#include "test/mocks/server/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::AnyNumber; +using testing::Eq; +using testing::Invoke; +using testing::ResultOf; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace ResponseCodecUnitTest { + +class MockResponseInitialParserFactory : public ResponseInitialParserFactory { +public: + MOCK_METHOD1(create, ResponseParserSharedPtr(const ResponseParserResolver&)); +}; + +using MockResponseInitialParserFactorySharedPtr = std::shared_ptr; + +class MockParser : public ResponseParser { +public: + MOCK_METHOD1(parse, ResponseParseResponse(absl::string_view&)); +}; + +using MockParserSharedPtr = std::shared_ptr; + +class MockResponseParserResolver : public ResponseParserResolver { +public: + MockResponseParserResolver() : ResponseParserResolver({}){}; + MOCK_CONST_METHOD1(createParser, ResponseParserSharedPtr(ResponseContextSharedPtr)); +}; + +class MockResponseCallback : public ResponseCallback { +public: + MOCK_METHOD1(onMessage, void(AbstractResponseSharedPtr)); + MOCK_METHOD1(onFailedParse, void(ResponseMetadataSharedPtr)); +}; + +using MockResponseCallbackSharedPtr = std::shared_ptr; + +class ResponseCodecUnitTest : public testing::Test, public BufferBasedTest { +protected: + MockResponseInitialParserFactorySharedPtr factory_{ + std::make_shared()}; + MockResponseParserResolver parser_resolver_{}; + MockResponseCallbackSharedPtr callback_{std::make_shared()}; +}; + +ResponseParseResponse consumeOneByte(absl::string_view& data) { + data = {data.data() + 1, data.size() - 1}; + return ResponseParseResponse::stillWaiting(); +} + +TEST_F(ResponseCodecUnitTest, shouldDoNothingIfParserReturnsWaiting) { + // given + putGarbageIntoBuffer(); + + MockParserSharedPtr parser = std::make_shared(); + EXPECT_CALL(*parser, parse(_)).Times(AnyNumber()).WillRepeatedly(Invoke(consumeOneByte)); + + EXPECT_CALL(*factory_, create(_)).WillOnce(Return(parser)); + EXPECT_CALL(parser_resolver_, createParser(_)).Times(0); + + EXPECT_CALL(*callback_, onMessage(_)).Times(0); + EXPECT_CALL(*callback_, onFailedParse(_)).Times(0); + + ResponseDecoder testee{factory_, parser_resolver_, {callback_}}; + + // when + testee.onData(buffer_); + + // then + // There were no interactions with `callback_`. +} + +TEST_F(ResponseCodecUnitTest, shouldUseNewParserAsResponse) { + // given + putGarbageIntoBuffer(); + + MockParserSharedPtr parser1 = std::make_shared(); + MockParserSharedPtr parser2 = std::make_shared(); + MockParserSharedPtr parser3 = std::make_shared(); + EXPECT_CALL(*parser1, parse(_)).WillOnce(Return(ResponseParseResponse::nextParser(parser2))); + EXPECT_CALL(*parser2, parse(_)).WillOnce(Return(ResponseParseResponse::nextParser(parser3))); + EXPECT_CALL(*parser3, parse(_)).Times(AnyNumber()).WillRepeatedly(Invoke(consumeOneByte)); + + EXPECT_CALL(*factory_, create(_)).WillOnce(Return(parser1)); + EXPECT_CALL(parser_resolver_, createParser(_)).Times(0); + + EXPECT_CALL(*callback_, onMessage(_)).Times(0); + EXPECT_CALL(*callback_, onFailedParse(_)).Times(0); + + ResponseDecoder testee{factory_, parser_resolver_, {callback_}}; + + // when + testee.onData(buffer_); + + // then + // There were no interactions with `callback_`. +} + +TEST_F(ResponseCodecUnitTest, shouldPassParsedMessageToCallback) { + // given + putGarbageIntoBuffer(); + + const AbstractResponseSharedPtr parsed_message = + std::make_shared>(ResponseMetadata{0, 0, 0}, 0); + + MockParserSharedPtr parser = std::make_shared(); + auto consume_and_return = [&parsed_message](absl::string_view& data) -> ResponseParseResponse { + data = {data.data() + data.size(), 0}; + return ResponseParseResponse::parsedMessage(parsed_message); + }; + EXPECT_CALL(*parser, parse(_)).WillOnce(Invoke(consume_and_return)); + + EXPECT_CALL(*factory_, create(_)).WillOnce(Return(parser)); + EXPECT_CALL(parser_resolver_, createParser(_)).Times(0); + + EXPECT_CALL(*callback_, onMessage(parsed_message)); + EXPECT_CALL(*callback_, onFailedParse(_)).Times(0); + + ResponseDecoder testee{factory_, parser_resolver_, {callback_}}; + + // when + testee.onData(buffer_); + + // then + // `callback_` had `onMessage` invoked once with matching argument. +} + +TEST_F(ResponseCodecUnitTest, shouldPassParseFailureDataToCallback) { + // given + putGarbageIntoBuffer(); + + const ResponseMetadataSharedPtr failure_data = std::make_shared(0, 0, 0); + + MockParserSharedPtr parser = std::make_shared(); + auto consume_and_return = [&failure_data](absl::string_view& data) -> ResponseParseResponse { + data = {data.data() + data.size(), 0}; + return ResponseParseResponse::parseFailure(failure_data); + }; + EXPECT_CALL(*parser, parse(_)).WillOnce(Invoke(consume_and_return)); + + EXPECT_CALL(*factory_, create(_)).WillOnce(Return(parser)); + EXPECT_CALL(parser_resolver_, createParser(_)).Times(0); + + EXPECT_CALL(*callback_, onMessage(_)).Times(0); + EXPECT_CALL(*callback_, onFailedParse(failure_data)); + + ResponseDecoder testee{factory_, parser_resolver_, {callback_}}; + + // when + testee.onData(buffer_); + + // then + // `callback_` had `onFailedParse` invoked once with matching argument. +} + +} // namespace ResponseCodecUnitTest +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/network/kafka/serialization_utilities.cc b/test/extensions/filters/network/kafka/serialization_utilities.cc index 95129da2e400..8b867bfcbe77 100644 --- a/test/extensions/filters/network/kafka/serialization_utilities.cc +++ b/test/extensions/filters/network/kafka/serialization_utilities.cc @@ -5,8 +5,8 @@ namespace Extensions { namespace NetworkFilters { namespace Kafka { -void assertStringViewIncrement(absl::string_view incremented, absl::string_view original, - size_t difference) { +void assertStringViewIncrement(const absl::string_view incremented, + const absl::string_view original, const size_t difference) { ASSERT_EQ(incremented.data(), original.data() + difference); ASSERT_EQ(incremented.size(), original.size() - difference); @@ -19,23 +19,6 @@ const char* getRawData(const Buffer::OwnedImpl& buffer) { return reinterpret_cast((slices[0]).mem_); } -void CapturingRequestCallback::onMessage(AbstractRequestSharedPtr message) { - captured_.push_back(message); -} - -void CapturingRequestCallback::onFailedParse(RequestParseFailureSharedPtr failure_data) { - parse_failures_.push_back(failure_data); -} - -const std::vector& CapturingRequestCallback::getCaptured() const { - return captured_; -} - -const std::vector& -CapturingRequestCallback::getParseFailures() const { - return parse_failures_; -} - } // namespace Kafka } // namespace NetworkFilters } // namespace Extensions diff --git a/test/extensions/filters/network/kafka/serialization_utilities.h b/test/extensions/filters/network/kafka/serialization_utilities.h index 3063bda95525..caadecb8b69d 100644 --- a/test/extensions/filters/network/kafka/serialization_utilities.h +++ b/test/extensions/filters/network/kafka/serialization_utilities.h @@ -3,7 +3,6 @@ #include "common/buffer/buffer_impl.h" #include "common/common/stack_array.h" -#include "extensions/filters/network/kafka/request_codec.h" #include "extensions/filters/network/kafka/serialization.h" #include "absl/strings/string_view.h" @@ -114,30 +113,33 @@ template void serializeThenDeserializeAndCheckEqualit } /** - * Request callback that captures the messages. + * Message callback that captures the messages. */ -class CapturingRequestCallback : public RequestCallback { +template class CapturingCallback : public Base { public: /** * Stores the message. */ - virtual void onMessage(AbstractRequestSharedPtr request) override; + virtual void onMessage(Message message) override { captured_messages_.push_back(message); } /** * Returns the stored messages. */ - const std::vector& getCaptured() const; + const std::vector& getCapturedMessages() const { return captured_messages_; } - virtual void onFailedParse(RequestParseFailureSharedPtr failure_data) override; + virtual void onFailedParse(Failure failure_data) override { + parse_failures_.push_back(failure_data); + } - const std::vector& getParseFailures() const; + const std::vector& getParseFailures() const { return parse_failures_; } private: - std::vector captured_; - std::vector parse_failures_; + std::vector captured_messages_; + std::vector parse_failures_; }; -using CapturingRequestCallbackSharedPtr = std::shared_ptr; +template +using CapturingCallbackSharedPtr = std::shared_ptr>; } // namespace Kafka } // namespace NetworkFilters From eefcd0e6fcbeba446454bd5396a34c69348338eb Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 25 Jun 2019 13:20:44 -0700 Subject: [PATCH 069/542] Listener: Filter chain manager refactor cont (#7246) Signed-off-by: Yuchen Dai --- source/server/BUILD | 4 +- source/server/filter_chain_manager_impl.cc | 85 ++++++-- source/server/filter_chain_manager_impl.h | 27 +-- source/server/listener_manager_impl.cc | 184 +++++++----------- source/server/listener_manager_impl.h | 23 ++- test/server/BUILD | 31 +++ test/server/filter_chain_manager_impl_test.cc | 145 ++++++++++++++ test/server/listener_manager_impl_test.cc | 2 +- 8 files changed, 359 insertions(+), 142 deletions(-) create mode 100644 test/server/filter_chain_manager_impl_test.cc diff --git a/source/server/BUILD b/source/server/BUILD index 168e7aa62795..d9cd108fd7b0 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -269,7 +269,6 @@ envoy_cc_library( "//source/common/network:utility_lib", "//source/common/protobuf:utility_lib", "//source/extensions/filters/listener:well_known_names", - "//source/extensions/filters/network:well_known_names", "//source/extensions/transport_sockets:well_known_names", "//source/extensions/transport_sockets/tls:context_config_lib", "//source/extensions/transport_sockets/tls:context_lib", @@ -284,9 +283,12 @@ envoy_cc_library( hdrs = ["filter_chain_manager_impl.h"], deps = [ "//include/envoy/server:listener_manager_interface", + "//include/envoy/server:transport_socket_config_interface", "//source/common/common:empty_string", + "//source/common/config:utility_lib", "//source/common/network:cidr_range_lib", "//source/common/network:lc_trie_lib", + "//source/server:configuration_lib", ], ) diff --git a/source/server/filter_chain_manager_impl.cc b/source/server/filter_chain_manager_impl.cc index 94a6c07a8577..a412c21c8ccf 100644 --- a/source/server/filter_chain_manager_impl.cc +++ b/source/server/filter_chain_manager_impl.cc @@ -1,6 +1,11 @@ #include "server/filter_chain_manager_impl.h" #include "common/common/empty_string.h" +#include "common/common/fmt.h" +#include "common/config/utility.h" +#include "common/protobuf/utility.h" + +#include "server/configuration_impl.h" #include "absl/strings/match.h" #include "absl/strings/str_cat.h" @@ -23,19 +28,67 @@ bool FilterChainManagerImpl::isWildcardServerName(const std::string& name) { } void FilterChainManagerImpl::addFilterChain( - uint16_t destination_port, const std::vector& destination_ips, - const std::vector& server_names, const std::string& transport_protocol, - const std::vector& application_protocols, - const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, - const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, - Network::TransportSocketFactoryPtr&& transport_socket_factory, - std::vector filters_factory) { - const auto filter_chain = std::make_shared(std::move(transport_socket_factory), - std::move(filters_factory)); - addFilterChainForDestinationPorts(destination_ports_map_, destination_port, destination_ips, - server_names, transport_protocol, application_protocols, - source_type, source_ips, source_ports, filter_chain); + absl::Span filter_chain_span, + FilterChainFactoryBuilder& filter_chain_factory_builder) { + std::unordered_set + filter_chains; + for (const auto& filter_chain : filter_chain_span) { + const auto& filter_chain_match = filter_chain->filter_chain_match(); + if (!filter_chain_match.address_suffix().empty() || filter_chain_match.has_suffix_len()) { + throw EnvoyException(fmt::format("error adding listener '{}': contains filter chains with " + "unimplemented fields", + address_->asString())); + } + if (filter_chains.find(filter_chain_match) != filter_chains.end()) { + throw EnvoyException(fmt::format("error adding listener '{}': multiple filter chains with " + "the same matching rules are defined", + address_->asString())); + } + filter_chains.insert(filter_chain_match); + + // Validate IP addresses. + std::vector destination_ips; + destination_ips.reserve(filter_chain_match.prefix_ranges().size()); + for (const auto& destination_ip : filter_chain_match.prefix_ranges()) { + const auto& cidr_range = Network::Address::CidrRange::create(destination_ip); + destination_ips.push_back(cidr_range.asString()); + } + + std::vector source_ips; + source_ips.reserve(filter_chain_match.source_prefix_ranges().size()); + for (const auto& source_ip : filter_chain_match.source_prefix_ranges()) { + const auto& cidr_range = Network::Address::CidrRange::create(source_ip); + source_ips.push_back(cidr_range.asString()); + } + + // Reject partial wildcards, we don't match on them. + for (const auto& server_name : filter_chain_match.server_names()) { + if (server_name.find('*') != std::string::npos && + !FilterChainManagerImpl::isWildcardServerName(server_name)) { + throw EnvoyException( + fmt::format("error adding listener '{}': partial wildcards are not supported in " + "\"server_names\"", + address_->asString())); + } + } + + std::vector server_names(filter_chain_match.server_names().begin(), + filter_chain_match.server_names().end()); + + std::vector application_protocols( + filter_chain_match.application_protocols().begin(), + filter_chain_match.application_protocols().end()); + + // TODO(silentdai): use absl::Span to avoid vector construction at server_names and alpn + addFilterChainForDestinationPorts( + destination_ports_map_, + PROTOBUF_GET_WRAPPED_OR_DEFAULT(filter_chain_match, destination_port, 0), destination_ips, + server_names, filter_chain_match.transport_protocol(), application_protocols, + filter_chain_match.source_type(), source_ips, filter_chain_match.source_ports(), + std::shared_ptr( + filter_chain_factory_builder.buildFilterChain(*filter_chain))); + } + convertIPsToTries(); } void FilterChainManagerImpl::addFilterChainForDestinationPorts( @@ -168,9 +221,9 @@ void FilterChainManagerImpl::addFilterChainForSourcePorts( // If we got here and found already configured branch, then it means that this FilterChainMatch // is a duplicate, and that there is some overlap in the repeated fields with already processed // FilterChainMatches. - // TODO(lambdai): bring back the address in exception - throw EnvoyException(fmt::format("error adding listener: multiple filter chains with " - "overlapping matching rules are defined")); + throw EnvoyException(fmt::format("error adding listener '{}': multiple filter chains with " + "overlapping matching rules are defined", + address_->asString())); } } diff --git a/source/server/filter_chain_manager_impl.h b/source/server/filter_chain_manager_impl.h index ff98953c8413..34175170aa76 100644 --- a/source/server/filter_chain_manager_impl.h +++ b/source/server/filter_chain_manager_impl.h @@ -3,6 +3,7 @@ #include #include "envoy/api/v2/listener/listener.pb.h" +#include "envoy/server/transport_socket_config.h" #include "common/common/logger.h" #include "common/network/cidr_range.h" @@ -13,26 +14,29 @@ namespace Envoy { namespace Server { +class FilterChainFactoryBuilder { +public: + virtual ~FilterChainFactoryBuilder() = default; + virtual std::unique_ptr + buildFilterChain(const ::envoy::api::v2::listener::FilterChain& filter_chain) const PURE; +}; + /** * Implementation of FilterChainManager. */ class FilterChainManagerImpl : public Network::FilterChainManager, Logger::Loggable { public: + explicit FilterChainManagerImpl(const Network::Address::InstanceConstSharedPtr& address) + : address_(address) {} + // Network::FilterChainManager const Network::FilterChain* findFilterChain(const Network::ConnectionSocket& socket) const override; + void - addFilterChain(uint16_t destination_port, const std::vector& destination_ips, - const std::vector& server_names, - const std::string& transport_protocol, - const std::vector& application_protocols, - const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, - const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, - Network::TransportSocketFactoryPtr&& transport_socket_factory, - std::vector filters_factory); - void finishFilterChain() { convertIPsToTries(); } + addFilterChain(absl::Span filter_chain_span, + FilterChainFactoryBuilder& b); static bool isWildcardServerName(const std::string& name); private: @@ -122,12 +126,13 @@ class FilterChainManagerImpl : public Network::FilterChainManager, // Mapping of FilterChain's configured destination ports, IPs, server names, transport protocols // and application protocols, using structures defined above. DestinationPortsMap destination_ports_map_; + const Network::Address::InstanceConstSharedPtr address_; }; class FilterChainImpl : public Network::FilterChain { public: FilterChainImpl(Network::TransportSocketFactoryPtr&& transport_socket_factory, - std::vector filters_factory) + std::vector&& filters_factory) : transport_socket_factory_(std::move(transport_socket_factory)), filters_factory_(std::move(filters_factory)) {} diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 9cef398c1a7b..0b1f58741655 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -1,5 +1,7 @@ #include "server/listener_manager_impl.h" +#include + #include "envoy/admin/v2alpha/config_dump.pb.h" #include "envoy/registry/registry.h" #include "envoy/server/transport_socket_config.h" @@ -23,7 +25,6 @@ #include "server/transport_socket_config_impl.h" #include "extensions/filters/listener/well_known_names.h" -#include "extensions/filters/network/well_known_names.h" #include "extensions/transport_sockets/well_known_names.h" #include "absl/strings/match.h" @@ -185,6 +186,7 @@ ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, const std::st ListenerManagerImpl& parent, const std::string& name, bool modifiable, bool workers_started, uint64_t hash) : parent_(parent), address_(Network::Address::resolveProtoAddress(config.address())), + filter_chain_manager_(address_), socket_type_(Network::Utility::protobufAddressSocketType(config.address())), global_scope_(parent_.server_.stats().createScope("")), listener_scope_( @@ -273,120 +275,43 @@ ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, const std::st factory.createFilterFactoryFromProto(Envoy::ProtobufWkt::Empty(), *this)); } - bool need_tls_inspector = false; - std::unordered_set - filter_chains; - - // TODO(lambdai): move the trie construction to FilterChainManagerImpl - for (const auto& filter_chain : config.filter_chains()) { - const auto& filter_chain_match = filter_chain.filter_chain_match(); - if (!filter_chain_match.address_suffix().empty() || filter_chain_match.has_suffix_len()) { - throw EnvoyException(fmt::format("error adding listener '{}': contains filter chains with " - "unimplemented fields", - address_->asString())); - } - if (filter_chains.find(filter_chain_match) != filter_chains.end()) { - throw EnvoyException(fmt::format("error adding listener '{}': multiple filter chains with " - "the same matching rules are defined", - address_->asString())); - } - filter_chains.insert(filter_chain_match); - - // If the cluster doesn't have transport socket configured, then use the default "raw_buffer" - // transport socket or BoringSSL-based "tls" transport socket if TLS settings are configured. - // We copy by value first then override if necessary. - auto transport_socket = filter_chain.transport_socket(); - if (!filter_chain.has_transport_socket()) { - if (filter_chain.has_tls_context()) { - transport_socket.set_name(Extensions::TransportSockets::TransportSocketNames::get().Tls); - MessageUtil::jsonConvert(filter_chain.tls_context(), *transport_socket.mutable_config()); - } else { - transport_socket.set_name( - Extensions::TransportSockets::TransportSocketNames::get().RawBuffer); - } - } - - auto& config_factory = Config::Utility::getAndCheckFactory< - Server::Configuration::DownstreamTransportSocketConfigFactory>(transport_socket.name()); - ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( - transport_socket, parent_.server_.messageValidationVisitor(), config_factory); - - // Validate IP addresses. - std::vector destination_ips; - destination_ips.reserve(filter_chain_match.prefix_ranges().size()); - for (const auto& destination_ip : filter_chain_match.prefix_ranges()) { - const auto& cidr_range = Network::Address::CidrRange::create(destination_ip); - destination_ips.push_back(cidr_range.asString()); - } - - std::vector server_names(filter_chain_match.server_names().begin(), - filter_chain_match.server_names().end()); - - // Reject partial wildcards, we don't match on them. - for (const auto& server_name : server_names) { - if (server_name.find('*') != std::string::npos && - !FilterChainManagerImpl::isWildcardServerName(server_name)) { - throw EnvoyException( - fmt::format("error adding listener '{}': partial wildcards are not supported in " - "\"server_names\"", - address_->asString())); - } - } - - std::vector source_ips; - source_ips.reserve(filter_chain_match.source_prefix_ranges().size()); - for (const auto& source_ip : filter_chain_match.source_prefix_ranges()) { - const auto& cidr_range = Network::Address::CidrRange::create(source_ip); - source_ips.push_back(cidr_range.asString()); - } - - std::vector application_protocols( - filter_chain_match.application_protocols().begin(), - filter_chain_match.application_protocols().end()); - Server::Configuration::TransportSocketFactoryContextImpl factory_context( - parent_.server_.admin(), parent_.server_.sslContextManager(), *listener_scope_, - parent_.server_.clusterManager(), parent_.server_.localInfo(), parent_.server_.dispatcher(), - parent_.server_.random(), parent_.server_.stats(), parent_.server_.singletonManager(), - parent_.server_.threadLocal(), parent_.server_.messageValidationVisitor(), - parent_.server_.api()); - factory_context.setInitManager(initManager()); - filter_chain_manager_.addFilterChain( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(filter_chain_match, destination_port, 0), destination_ips, - server_names, filter_chain_match.transport_protocol(), application_protocols, - filter_chain_match.source_type(), source_ips, filter_chain_match.source_ports(), - config_factory.createTransportSocketFactory(*message, factory_context, server_names), - parent_.factory_.createNetworkFilterFactoryList(filter_chain.filters(), *this)); - - need_tls_inspector |= filter_chain_match.transport_protocol() == "tls" || - (filter_chain_match.transport_protocol().empty() && - (!server_names.empty() || !application_protocols.empty())); - } - - // Convert both destination and source IP CIDRs to tries for faster lookups. - filter_chain_manager_.finishFilterChain(); - + Server::Configuration::TransportSocketFactoryContextImpl factory_context( + parent_.server_.admin(), parent_.server_.sslContextManager(), *listener_scope_, + parent_.server_.clusterManager(), parent_.server_.localInfo(), parent_.server_.dispatcher(), + parent_.server_.random(), parent_.server_.stats(), parent_.server_.singletonManager(), + parent_.server_.threadLocal(), parent_.server_.messageValidationVisitor(), + parent_.server_.api()); + factory_context.setInitManager(initManager()); + ListenerFilterChainFactoryBuilder builder(*this, factory_context); + filter_chain_manager_.addFilterChain(config.filter_chains(), builder); + const bool need_tls_inspector = + std::any_of( + config.filter_chains().begin(), config.filter_chains().end(), + [](const auto& filter_chain) { + const auto& matcher = filter_chain.filter_chain_match(); + return matcher.transport_protocol() == "tls" || + (matcher.transport_protocol().empty() && + (!matcher.server_names().empty() || !matcher.application_protocols().empty())); + }) && + not std::any_of(config.listener_filters().begin(), config.listener_filters().end(), + [](const auto& filter) { + return filter.name() == + Extensions::ListenerFilters::ListenerFilterNames::get().TlsInspector; + }); // Automatically inject TLS Inspector if it wasn't configured explicitly and it's needed. if (need_tls_inspector) { - for (const auto& filter : config.listener_filters()) { - if (filter.name() == Extensions::ListenerFilters::ListenerFilterNames::get().TlsInspector) { - need_tls_inspector = false; - break; - } - } - if (need_tls_inspector) { - const std::string message = - fmt::format("adding listener '{}': filter chain match rules require TLS Inspector " - "listener filter, but it isn't configured, trying to inject it " - "(this might fail if Envoy is compiled without it)", - address_->asString()); - ENVOY_LOG(warn, "{}", message); + const std::string message = + fmt::format("adding listener '{}': filter chain match rules require TLS Inspector " + "listener filter, but it isn't configured, trying to inject it " + "(this might fail if Envoy is compiled without it)", + address_->asString()); + ENVOY_LOG(warn, "{}", message); - auto& factory = - Config::Utility::getAndCheckFactory( - Extensions::ListenerFilters::ListenerFilterNames::get().TlsInspector); - listener_filter_factories_.push_back( - factory.createFilterFactoryFromProto(Envoy::ProtobufWkt::Empty(), *this)); - } + auto& factory = + Config::Utility::getAndCheckFactory( + Extensions::ListenerFilters::ListenerFilterNames::get().TlsInspector); + listener_filter_factories_.push_back( + factory.createFilterFactoryFromProto(Envoy::ProtobufWkt::Empty(), *this)); } } @@ -859,5 +784,40 @@ void ListenerManagerImpl::stopWorkers() { } } +ListenerFilterChainFactoryBuilder::ListenerFilterChainFactoryBuilder( + ListenerImpl& listener, + Server::Configuration::TransportSocketFactoryContextImpl& factory_context) + : parent_(listener), factory_context_(factory_context) {} + +std::unique_ptr ListenerFilterChainFactoryBuilder::buildFilterChain( + const ::envoy::api::v2::listener::FilterChain& filter_chain) const { + // If the cluster doesn't have transport socket configured, then use the default "raw_buffer" + // transport socket or BoringSSL-based "tls" transport socket if TLS settings are configured. + // We copy by value first then override if necessary. + auto transport_socket = filter_chain.transport_socket(); + if (!filter_chain.has_transport_socket()) { + if (filter_chain.has_tls_context()) { + transport_socket.set_name(Extensions::TransportSockets::TransportSocketNames::get().Tls); + MessageUtil::jsonConvert(filter_chain.tls_context(), *transport_socket.mutable_config()); + } else { + transport_socket.set_name( + Extensions::TransportSockets::TransportSocketNames::get().RawBuffer); + } + } + + auto& config_factory = Config::Utility::getAndCheckFactory< + Server::Configuration::DownstreamTransportSocketConfigFactory>(transport_socket.name()); + ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( + transport_socket, parent_.messageValidationVisitor(), config_factory); + + std::vector server_names(filter_chain.filter_chain_match().server_names().begin(), + filter_chain.filter_chain_match().server_names().end()); + + return std::make_unique( + config_factory.createTransportSocketFactory(*message, factory_context_, + std::move(server_names)), + parent_.parent_.factory_.createNetworkFilterFactoryList(filter_chain.filters(), parent_)); +} + } // namespace Server } // namespace Envoy diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 80cb5ae4b356..f302ff7391db 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -22,6 +22,12 @@ namespace Envoy { namespace Server { +namespace Configuration { +class TransportSocketFactoryContextImpl; +} + +class ListenerFilterChainFactoryBuilder; + /** * Prod implementation of ListenerComponentFactory that creates real sockets and attempts to fetch * sockets from the parent process via the hot restarter. The filter factory list is created from @@ -331,8 +337,9 @@ class ListenerImpl : public Network::ListenerConfig, private: ListenerManagerImpl& parent_; - FilterChainManagerImpl filter_chain_manager_; Network::Address::InstanceConstSharedPtr address_; + FilterChainManagerImpl filter_chain_manager_; + Network::Address::SocketType socket_type_; Network::SocketSharedPtr socket_; Stats::ScopePtr global_scope_; // Stats with global named scope, but needed for LDS cleanup. @@ -361,6 +368,20 @@ class ListenerImpl : public Network::ListenerConfig, const std::string version_info_; Network::Socket::OptionsSharedPtr listen_socket_options_; const std::chrono::milliseconds listener_filters_timeout_; + // to access ListenerManagerImpl::factory_. + friend class ListenerFilterChainFactoryBuilder; +}; + +class ListenerFilterChainFactoryBuilder : public FilterChainFactoryBuilder { +public: + ListenerFilterChainFactoryBuilder( + ListenerImpl& listener, Configuration::TransportSocketFactoryContextImpl& factory_context); + std::unique_ptr + buildFilterChain(const ::envoy::api::v2::listener::FilterChain& filter_chain) const override; + +private: + ListenerImpl& parent_; + Configuration::TransportSocketFactoryContextImpl& factory_context_; }; } // namespace Server diff --git a/test/server/BUILD b/test/server/BUILD index 3b3c4625015a..233a1d794f91 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -188,6 +188,37 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "filter_chain_manager_impl_test", + srcs = ["filter_chain_manager_impl_test.cc"], + data = ["//test/extensions/transport_sockets/tls/test_data:certs"], + deps = [ + ":utility_lib", + "//source/common/api:os_sys_calls_lib", + "//source/common/config:metadata_lib", + "//source/common/network:addr_family_aware_socket_option_lib", + "//source/common/network:listen_socket_lib", + "//source/common/network:socket_option_lib", + "//source/common/network:utility_lib", + "//source/common/protobuf", + "//source/extensions/filters/listener/original_dst:config", + "//source/extensions/filters/listener/tls_inspector:config", + "//source/extensions/filters/network/http_connection_manager:config", + "//source/extensions/transport_sockets/raw_buffer:config", + "//source/extensions/transport_sockets/tls:config", + "//source/extensions/transport_sockets/tls:ssl_socket_lib", + "//source/server:filter_chain_manager_lib", + "//source/server:listener_manager_lib", + "//test/mocks/network:network_mocks", + "//test/mocks/server:server_mocks", + "//test/test_common:environment_lib", + "//test/test_common:registry_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_time_lib", + "//test/test_common:threadsafe_singleton_injector_lib", + ], +) + envoy_cc_fuzz_test( name = "server_fuzz_test", srcs = ["server_fuzz_test.cc"], diff --git a/test/server/filter_chain_manager_impl_test.cc b/test/server/filter_chain_manager_impl_test.cc new file mode 100644 index 000000000000..3a6f3caee1a8 --- /dev/null +++ b/test/server/filter_chain_manager_impl_test.cc @@ -0,0 +1,145 @@ +#include +#include +#include +#include +#include + +#include "envoy/admin/v2alpha/config_dump.pb.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "common/api/os_sys_calls_impl.h" +#include "common/config/metadata.h" +#include "common/network/address_impl.h" +#include "common/network/io_socket_handle_impl.h" +#include "common/network/listen_socket_impl.h" +#include "common/network/socket_option_impl.h" +#include "common/network/utility.h" +#include "common/protobuf/protobuf.h" + +#include "server/configuration_impl.h" +#include "server/filter_chain_manager_impl.h" +#include "server/listener_manager_impl.h" + +#include "extensions/filters/listener/original_dst/original_dst.h" +#include "extensions/transport_sockets/tls/ssl_socket.h" + +#include "test/mocks/network/mocks.h" +#include "test/mocks/server/mocks.h" +#include "test/server/utility.h" +#include "test/test_common/environment.h" +#include "test/test_common/registry.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/threadsafe_singleton_injector.h" +#include "test/test_common/utility.h" + +#include "absl/strings/escaping.h" +#include "absl/strings/match.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::InSequence; +using testing::Invoke; +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; +using testing::Throw; + +namespace Envoy { +namespace Server { + +class MockFilterChainFactoryBuilder : public FilterChainFactoryBuilder { + + std::unique_ptr + buildFilterChain(const ::envoy::api::v2::listener::FilterChain&) const override { + // Won't dereference but requires not nullptr. + return std::make_unique(); + } +}; + +class FilterChainManagerImplTest : public testing::Test { +public: + void SetUp() override { + local_address_ = std::make_shared("127.0.0.1", 1234); + remote_address_ = std::make_shared("127.0.0.1", 1234); + TestUtility::loadFromYaml( + TestEnvironment::substitute(filter_chain_yaml, Network::Address::IpVersion::v4), + filter_chain_template_); + } + + const Network::FilterChain* + findFilterChainHelper(uint16_t destination_port, const std::string& destination_address, + const std::string& server_name, const std::string& transport_protocol, + const std::vector& application_protocols, + const std::string& source_address, uint16_t source_port) { + auto mock_socket = std::make_shared>(); + sockets_.push_back(mock_socket); + + if (absl::StartsWith(destination_address, "/")) { + local_address_ = std::make_shared(destination_address); + } else { + local_address_ = + Network::Utility::parseInternetAddress(destination_address, destination_port); + } + ON_CALL(*mock_socket, localAddress()).WillByDefault(ReturnRef(local_address_)); + + ON_CALL(*mock_socket, requestedServerName()) + .WillByDefault(Return(absl::string_view(server_name))); + ON_CALL(*mock_socket, detectedTransportProtocol()) + .WillByDefault(Return(absl::string_view(transport_protocol))); + ON_CALL(*mock_socket, requestedApplicationProtocols()) + .WillByDefault(ReturnRef(application_protocols)); + + if (absl::StartsWith(source_address, "/")) { + remote_address_ = std::make_shared(source_address); + } else { + remote_address_ = Network::Utility::parseInternetAddress(source_address, source_port); + } + ON_CALL(*mock_socket, remoteAddress()).WillByDefault(ReturnRef(remote_address_)); + return filter_chain_manager_.findFilterChain(*mock_socket); + } + + void addSingleFilterChainHelper(const envoy::api::v2::listener::FilterChain& filter_chain) { + filter_chain_manager_.addFilterChain( + std::vector{&filter_chain}, + filter_chain_factory_builder_); + } + + // Intermedia states. + Network::Address::InstanceConstSharedPtr local_address_; + Network::Address::InstanceConstSharedPtr remote_address_; + std::vector> sockets_; + + // Reuseable template. + const std::string filter_chain_yaml = R"EOF( + filter_chain_match: + destination_port: 10000 + tls_context: + common_tls_context: + tls_certificates: + - certificate_chain: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_multiple_dns_cert.pem" } + private_key: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_multiple_dns_key.pem" } + session_ticket_keys: + keys: + - filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ticket_key_a" + )EOF"; + envoy::api::v2::listener::FilterChain filter_chain_template_; + MockFilterChainFactoryBuilder filter_chain_factory_builder_; + + // Test target. + FilterChainManagerImpl filter_chain_manager_{ + std::make_shared("127.0.0.1", 1234)}; +}; + +TEST_F(FilterChainManagerImplTest, FilterChainMatchNothing) { + auto filter_chain = findFilterChainHelper(10000, "127.0.0.1", "", "tls", {}, "8.8.8.8", 111); + EXPECT_EQ(filter_chain, nullptr); +} + +TEST_F(FilterChainManagerImplTest, AddSingleFilterChain) { + addSingleFilterChainHelper(filter_chain_template_); + auto* filter_chain = findFilterChainHelper(10000, "127.0.0.1", "", "tls", {}, "8.8.8.8", 111); + EXPECT_NE(filter_chain, nullptr); +} +} // namespace Server +} // namespace Envoy \ No newline at end of file diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index be56ec99eee7..cbdeb23e8c6b 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -2352,7 +2352,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithOverlappi EXPECT_THROW_WITH_MESSAGE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(yaml), "", true), EnvoyException, - "error adding listener: multiple filter chains with " + "error adding listener '127.0.0.1:1234': multiple filter chains with " "overlapping matching rules are defined"); } From d09ce412f96eb6c7a689ed66927879b5983a27d4 Mon Sep 17 00:00:00 2001 From: Xin Date: Tue, 25 Jun 2019 18:36:21 -0400 Subject: [PATCH 070/542] [router] Add scope key builder impl and tests. (#7234) Add scope key builder implementation + unit tests. Risk Level: LOW (not fully implemented) Testing: Unit test Docs Changes: comment of hcm proto updated. Signed-off-by: Xin Zhuang --- .../v2/http_connection_manager.proto | 6 + source/common/router/scoped_config_impl.cc | 101 ++++++ source/common/router/scoped_config_impl.h | 124 ++++++- source/common/router/scoped_rds.cc | 4 +- test/common/router/BUILD | 12 + test/common/router/scoped_config_impl_test.cc | 322 ++++++++++++++++++ 6 files changed, 565 insertions(+), 4 deletions(-) create mode 100644 test/common/router/scoped_config_impl_test.cc diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 4cee964e1d2c..f0c7b0f59e1e 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -490,6 +490,10 @@ message ScopedRoutes { // Specifies a header field's key value pair to match on. message KvElement { // The separator between key and value (e.g., '=' separates 'k=v;...'). + // If an element is an empty string, the element is ignored. + // If an element contains no separator, the whole element is parsed as key and the + // fragment value is an empty string. + // If there are multiple values for a matched key, the first value is returned. string separator = 1 [(validate.rules).string.min_bytes = 1]; // The key to match on. @@ -498,6 +502,8 @@ message ScopedRoutes { oneof extract_type { // Specifies the zero based index of the element to extract. + // Note Envoy concatenates multiple values of the same header key into a comma separated + // string, the splitting always happens after the concatenation. uint32 index = 3; // Specifies the key value pair to extract the value from. diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 0274381ae850..ee3ce1318568 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -3,6 +3,107 @@ namespace Envoy { namespace Router { +bool ScopeKey::operator!=(const ScopeKey& other) const { return !(*this == other); } + +bool ScopeKey::operator==(const ScopeKey& other) const { + if (fragments_.empty() || other.fragments_.empty()) { + // An empty key equals to nothing, "NULL" != "NULL". + return false; + } + return std::equal(fragments_.begin(), fragments_.end(), other.fragments_.begin(), + other.fragments_.end(), + [](const std::unique_ptr& left, + const std::unique_ptr& right) -> bool { + // Both should be non-NULL now. + return *left == *right; + }); +} + +HeaderValueExtractorImpl::HeaderValueExtractorImpl( + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder&& config) + : FragmentBuilderBase(std::move(config)), + header_value_extractor_config_(config_.header_value_extractor()) { + ASSERT(config_.type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::kHeaderValueExtractor, + "header_value_extractor is not set."); + if (header_value_extractor_config_.extract_type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kIndex) { + if (header_value_extractor_config_.index() != 0 && + header_value_extractor_config_.element_separator().empty()) { + throw ProtoValidationException("Index > 0 for empty string element separator.", + header_value_extractor_config_); + } + } + if (header_value_extractor_config_.extract_type_case() == + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::EXTRACT_TYPE_NOT_SET) { + throw ProtoValidationException("HeaderValueExtractor extract_type not set.", + header_value_extractor_config_); + } +} + +std::unique_ptr +HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const { + const Envoy::Http::HeaderEntry* header_entry = + headers.get(Envoy::Http::LowerCaseString(header_value_extractor_config_.name())); + if (header_entry == nullptr) { + return nullptr; + } + + std::vector elements{header_entry->value().getStringView()}; + if (header_value_extractor_config_.element_separator().length() > 0) { + elements = absl::StrSplit(header_entry->value().getStringView(), + header_value_extractor_config_.element_separator()); + } + switch (header_value_extractor_config_.extract_type_case()) { + case ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kElement: + for (const auto& element : elements) { + std::pair key_value = absl::StrSplit( + element, absl::MaxSplits(header_value_extractor_config_.element().separator(), 1)); + if (key_value.first == header_value_extractor_config_.element().key()) { + return std::make_unique(key_value.second); + } + } + break; + case ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor::kIndex: + if (header_value_extractor_config_.index() < elements.size()) { + return std::make_unique(elements[header_value_extractor_config_.index()]); + } + break; + default: // EXTRACT_TYPE_NOT_SET + NOT_REACHED_GCOVR_EXCL_LINE; // Caught in constructor already. + } + + return nullptr; +} + +ScopeKeyBuilderImpl::ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder&& config) + : ScopeKeyBuilderBase(std::move(config)) { + for (const auto& fragment_builder : config_.fragments()) { + switch (fragment_builder.type_case()) { + case ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::kHeaderValueExtractor: + fragment_builders_.emplace_back(std::make_unique( + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder(fragment_builder))); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + } +} + +std::unique_ptr +ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { + ScopeKey key; + for (const auto& builder : fragment_builders_) { + // returns nullopt if a null fragment is found. + std::unique_ptr fragment = builder->computeFragment(headers); + if (fragment == nullptr) { + return nullptr; + } + key.addFragment(std::move(fragment)); + } + return std::make_unique(std::move(key)); +} + void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 94d376b2a354..808b442838c4 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -1,17 +1,137 @@ #pragma once +#include + #include "envoy/api/v2/srds.pb.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" #include "envoy/router/router.h" #include "envoy/router/scopes.h" #include "envoy/thread_local/thread_local.h" +#include "common/protobuf/utility.h" #include "common/router/config_impl.h" #include "common/router/scoped_config_manager.h" namespace Envoy { namespace Router { +using envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes; + +/** + * Scope key fragment base class. + */ +class ScopeKeyFragmentBase { +public: + bool operator!=(const ScopeKeyFragmentBase& other) const { return !(*this == other); } + + bool operator==(const ScopeKeyFragmentBase& other) const { + if (typeid(*this) == typeid(other)) { + return equals(other); + } + return false; + } + virtual ~ScopeKeyFragmentBase() = default; + +private: + // Returns true if the two fragments equal else false. + virtual bool equals(const ScopeKeyFragmentBase&) const PURE; +}; + +/** + * Scope Key is composed of non-null fragments. + **/ +class ScopeKey { +public: + ScopeKey() = default; + ScopeKey(ScopeKey&& other) = default; + + // Scopekey is not copy-assignable and copy-constructible as it contains unique_ptr inside itself. + ScopeKey(const ScopeKey&) = delete; + ScopeKey operator=(const ScopeKey&) = delete; + + // Caller should guarantee the fragment is not nullptr. + void addFragment(std::unique_ptr&& fragment) { + ASSERT(fragment != nullptr, "null fragment not allowed in ScopeKey."); + fragments_.emplace_back(std::move(fragment)); + } + + bool operator!=(const ScopeKey& other) const; + + bool operator==(const ScopeKey& other) const; + +private: + std::vector> fragments_; +}; + +// String fragment. +class StringKeyFragment : public ScopeKeyFragmentBase { +public: + explicit StringKeyFragment(absl::string_view value) : value_(value) {} + +private: + bool equals(const ScopeKeyFragmentBase& other) const override { + return value_ == static_cast(other).value_; + } + + const std::string value_; +}; + +/** + * Base class for fragment builders. + */ +class FragmentBuilderBase { +public: + explicit FragmentBuilderBase(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder&& config) + : config_(std::move(config)) {} + virtual ~FragmentBuilderBase() = default; + + // Returns a fragment if the fragment rule applies, a nullptr indicates no fragment could be + // generated from the headers. + virtual std::unique_ptr + computeFragment(const Http::HeaderMap& headers) const PURE; + +protected: + const ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config_; +}; + +class HeaderValueExtractorImpl : public FragmentBuilderBase { +public: + explicit HeaderValueExtractorImpl(ScopedRoutes::ScopeKeyBuilder::FragmentBuilder&& config); + + std::unique_ptr + computeFragment(const Http::HeaderMap& headers) const override; + +private: + const ScopedRoutes::ScopeKeyBuilder::FragmentBuilder::HeaderValueExtractor& + header_value_extractor_config_; +}; + +/** + * Base class for ScopeKeyBuilder implementations. + */ +class ScopeKeyBuilderBase { +public: + explicit ScopeKeyBuilderBase(ScopedRoutes::ScopeKeyBuilder&& config) + : config_(std::move(config)) {} + virtual ~ScopeKeyBuilderBase() = default; + + // Computes scope key for given headers, returns nullptr if a key can't be computed. + virtual std::unique_ptr computeScopeKey(const Http::HeaderMap& headers) const PURE; + +protected: + const ScopedRoutes::ScopeKeyBuilder config_; +}; + +class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { +public: + explicit ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder&& config); + + std::unique_ptr computeScopeKey(const Http::HeaderMap& headers) const override; + +private: + std::vector> fragment_builders_; +}; + /** * TODO(AndresGuedez): implement scoped routing logic. * @@ -23,9 +143,7 @@ namespace Router { */ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { public: - ThreadLocalScopedConfigImpl( - envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder - scope_key_builder) + ThreadLocalScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) : scope_key_builder_(std::move(scope_key_builder)) {} void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 0ea1c4ed5a2d..86e1d1dc88b8 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -159,7 +159,9 @@ ScopedRdsConfigProvider::ScopedRdsConfigProvider( MutableConfigProviderCommonBase::subscription_.get())), rds_config_source_(std::move(rds_config_source)) { initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared(scope_key_builder); + return std::make_shared( + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( + scope_key_builder)); }); } diff --git a/test/common/router/BUILD b/test/common/router/BUILD index c436d35207ac..eb07c6558972 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -77,6 +77,18 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "scoped_config_impl_test", + srcs = ["scoped_config_impl_test.cc"], + external_deps = [ + "abseil_strings", + ], + deps = [ + "//source/common/router:scoped_config_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "scoped_rds_test", srcs = ["scoped_rds_test.cc"], diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc new file mode 100644 index 000000000000..9c9a06a25934 --- /dev/null +++ b/test/common/router/scoped_config_impl_test.cc @@ -0,0 +1,322 @@ +#include + +#include "common/router/scoped_config_impl.h" + +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Router { +namespace { + +using ::Envoy::Http::TestHeaderMapImpl; + +class FooFragment : public ScopeKeyFragmentBase { +private: + bool equals(const ScopeKeyFragmentBase&) const override { return true; }; +}; + +TEST(ScopeKeyFragmentBaseTest, EqualSign) { + FooFragment foo; + StringKeyFragment bar("a random string"); + + EXPECT_NE(foo, bar); +} + +TEST(StringKeyFragmentTest, Empty) { + StringKeyFragment a(""); + StringKeyFragment b(""); + EXPECT_EQ(a, b); + + StringKeyFragment non_empty("ABC"); + + EXPECT_NE(a, non_empty); +} + +TEST(StringKeyFragmentTest, Normal) { + StringKeyFragment str("Abc"); + + StringKeyFragment same_str("Abc"); + EXPECT_EQ(str, same_str); + + StringKeyFragment upper_cased_str("ABC"); + EXPECT_NE(str, upper_cased_str); + + StringKeyFragment another_str("DEF"); + EXPECT_NE(str, another_str); +} + +TEST(HeaderValueExtractorImplDeathTest, InvalidConfig) { + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; + // Type not set, ASSERT only fails in debug mode. +#if !defined(NDEBUG) + EXPECT_DEATH(HeaderValueExtractorImpl(std::move(config)), "header_value_extractor is not set."); +#else + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl(std::move(config)), ProtoValidationException, + "HeaderValueExtractor extract_type not set.+"); +#endif // !defined(NDEBUG) + + // Index non-zero when element separator is an empty string. + std::string yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: '' + index: 1 +)EOF"; + TestUtility::loadFromYaml(yaml_plain, config); + + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl(std::move(config)), ProtoValidationException, + "Index > 0 for empty string element separator."); + // extract_type not set. + yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: '' +)EOF"; + TestUtility::loadFromYaml(yaml_plain, config); + + EXPECT_THROW_WITH_REGEX(HeaderValueExtractorImpl(std::move(config)), ProtoValidationException, + "HeaderValueExtractor extract_type not set.+"); +} + +TEST(HeaderValueExtractorImplTest, HeaderExtractionByIndex) { + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; + std::string yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: ',' + index: 1 +)EOF"; + + TestUtility::loadFromYaml(yaml_plain, config); + HeaderValueExtractorImpl extractor(std::move(config)); + std::unique_ptr fragment = + extractor.computeFragment(TestHeaderMapImpl{{"foo_header", "part-0,part-1:value_bluh"}}); + + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{"part-1:value_bluh"}); + + // No such header. + fragment = extractor.computeFragment(TestHeaderMapImpl{{"bar_header", "part-0"}}); + EXPECT_EQ(fragment, nullptr); + + // Empty header value. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", ""}, + }); + EXPECT_EQ(fragment, nullptr); + + // Index out of bound. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "part-0"}, + }); + EXPECT_EQ(fragment, nullptr); + + // Element is empty. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "part-0,,,bluh"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment("")); +} + +TEST(HeaderValueExtractorImplTest, HeaderExtractionByKey) { + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; + std::string yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: ';' + element: + key: 'bar' + separator: '=>' +)EOF"; + + TestUtility::loadFromYaml(yaml_plain, config); + HeaderValueExtractorImpl extractor(std::move(config)); + std::unique_ptr fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "part-0;bar=>bluh;foo=>foo_value"}, + }); + + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{"bluh"}); + + // No such header. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"bluh", "part-0;"}, + }); + EXPECT_EQ(fragment, nullptr); + + // Empty header value. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", ""}, + }); + EXPECT_EQ(fragment, nullptr); + + // No such key. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "part-0"}, + }); + EXPECT_EQ(fragment, nullptr); + + // Empty value. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "bluh;;bar=>;foo=>last_value"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{""}); + + // Duplicate values, the first value returned. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "bluh;;bar=>value1;bar=>value2;bluh;;bar=>last_value"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{"value1"}); + + // No separator in the element, value is set to empty string. + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "bluh;;bar;bar=>value2;bluh;;bar=>last_value"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{""}); +} + +TEST(HeaderValueExtractorImplTest, ElementSeparatorEmpty) { + ScopedRoutes::ScopeKeyBuilder::FragmentBuilder config; + std::string yaml_plain = R"EOF( + header_value_extractor: + name: 'foo_header' + element_separator: '' + element: + key: 'bar' + separator: '=' +)EOF"; + + TestUtility::loadFromYaml(yaml_plain, config); + HeaderValueExtractorImpl extractor(std::move(config)); + std::unique_ptr fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "bar=b;c=d;e=f"}, + }); + EXPECT_NE(fragment, nullptr); + EXPECT_EQ(*fragment, StringKeyFragment{"b;c=d;e=f"}); + + fragment = extractor.computeFragment(TestHeaderMapImpl{ + {"foo_header", "a=b;bar=d;e=f"}, + }); + EXPECT_EQ(fragment, nullptr); +} + +// Helper function which makes a ScopeKey from a list of strings. +ScopeKey makeKey(const std::vector& parts) { + ScopeKey key; + for (const auto& part : parts) { + key.addFragment(std::make_unique(part)); + } + return key; +} + +TEST(ScopeKeyDeathTest, AddNullFragment) { + ScopeKey key; + EXPECT_DEBUG_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); +} + +TEST(ScopeKeyTest, Unmatches) { + ScopeKey key1; + ScopeKey key2; + // Empty key != empty key. + EXPECT_NE(key1, key2); + + // Empty key != non-empty key. + EXPECT_NE(key1, makeKey({""})); + + EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); + + // Two keys of different length won't match. + EXPECT_NE(makeKey({"a", "b"}), makeKey({"a", "b", "c"})); + + // Case sensitive. + EXPECT_NE(makeKey({"a", "b"}), makeKey({"A", "b"})); +} + +TEST(ScopeKeyTest, Matches) { + // An empty string fragment equals another. + EXPECT_EQ(makeKey({"", ""}), makeKey({"", ""})); + EXPECT_EQ(makeKey({"a", "", ""}), makeKey({"a", "", ""})); + + // Non empty fragments comparison. + EXPECT_EQ(makeKey({"A", "b"}), makeKey({"A", "b"})); +} + +TEST(ScopeKeyBuilderImplTest, Parse) { + std::string yaml_plain = R"EOF( + fragments: + - header_value_extractor: + name: 'foo_header' + element_separator: ',' + element: + key: 'bar' + separator: '=' + - header_value_extractor: + name: 'bar_header' + element_separator: ';' + index: 2 +)EOF"; + + ScopedRoutes::ScopeKeyBuilder config; + TestUtility::loadFromYaml(yaml_plain, config); + ScopeKeyBuilderImpl key_builder(std::move(config)); + + std::unique_ptr key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,bar=bar_value,e=f"}, + {"bar_header", "a=b;bar=bar_value;index2"}, + }); + EXPECT_NE(key, nullptr); + EXPECT_EQ(*key, makeKey({"bar_value", "index2"})); + + // Empty string fragment is fine. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,bar,e=f"}, + {"bar_header", "a=b;bar=bar_value;"}, + }); + EXPECT_NE(key, nullptr); + EXPECT_EQ(*key, makeKey({"", ""})); + + // Key not found. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,meh,e=f"}, + {"bar_header", "a=b;bar=bar_value;"}, + }); + EXPECT_EQ(key, nullptr); + + // Index out of bound. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,bar=bar_value,e=f"}, + {"bar_header", "a=b;bar=bar_value"}, + }); + EXPECT_EQ(key, nullptr); + + // Header missing. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,bar=bar_value,e=f"}, + {"foobar_header", "a=b;bar=bar_value;index2"}, + }); + EXPECT_EQ(key, nullptr); + + // Header value empty. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", ""}, + {"bar_header", "a=b;bar=bar_value;index2"}, + }); + EXPECT_EQ(key, nullptr); + + // Case sensitive. + key = key_builder.computeScopeKey(TestHeaderMapImpl{ + {"foo_header", "a=b,Bar=bar_value,e=f"}, + {"bar_header", "a=b;bar=bar_value;index2"}, + }); + EXPECT_EQ(key, nullptr); +} + +} // namespace +} // namespace Router +} // namespace Envoy From e4bdb73598d7388651a9af7fd5b0ce793b979554 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Tue, 25 Jun 2019 16:03:14 -0700 Subject: [PATCH 071/542] clang-tidy: performance-noexcept-move-constructor (#7393) performance-noexcept-move-constructor checks for move constructors that aren't declared noexcept, the reason being that STL containers will use the copy constructor for operations like emplace_back if the move constructor is not declared noexcept due to exception safety guarantees. http://www.hlsl.co.uk/blog/2017/12/1/c-noexcept-and-move-constructors-effect-on-performance-in-stl-containers https://stackoverflow.com/a/32225460 Risk Level: low Testing: existing Docs Changes: N/A Release Notes: N/A Relates to #4863 Signed-off-by: Derek Argueta --- .clang-tidy | 2 +- include/envoy/http/header_map.h | 6 ++++-- source/common/http/header_map_impl.cc | 2 +- source/common/stats/symbol_table_impl.h | 2 +- test/mocks/http/mocks.h | 8 ++++---- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 101180b43834..76764919891d 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,7 +1,7 @@ Checks: 'clang-diagnostic-*,clang-analyzer-*,abseil-*,bugprone-*,modernize-*,performance-*,readability-redundant-*,readability-braces-around-statements,readability-container-size-empty' #TODO(lizan): grow this list, fix possible warnings and make more checks as error -WarningsAsErrors: 'bugprone-assert-side-effect,bugprone-use-after-move,modernize-make-shared,modernize-make-unique,modernize-use-using,performance-faster-string-find,performance-for-range-copy,performance-unnecessary-copy-initialization,readability-braces-around-statements,readability-container-size-empty,readability-redundant-smartptr-get,readability-redundant-string-cstr' +WarningsAsErrors: 'bugprone-assert-side-effect,bugprone-use-after-move,modernize-make-shared,modernize-make-unique,modernize-use-using,performance-faster-string-find,performance-for-range-copy,performance-noexcept-move-constructor,performance-unnecessary-copy-initialization,readability-braces-around-statements,readability-container-size-empty,readability-redundant-smartptr-get,readability-redundant-string-cstr' CheckOptions: - key: bugprone-assert-side-effect.AssertMacros diff --git a/include/envoy/http/header_map.h b/include/envoy/http/header_map.h index a9c0ea95058f..8af3b10e0b8d 100644 --- a/include/envoy/http/header_map.h +++ b/include/envoy/http/header_map.h @@ -46,7 +46,9 @@ static inline bool validHeaderString(absl::string_view s) { */ class LowerCaseString { public: - LowerCaseString(LowerCaseString&& rhs) : string_(std::move(rhs.string_)) { ASSERT(valid()); } + LowerCaseString(LowerCaseString&& rhs) noexcept : string_(std::move(rhs.string_)) { + ASSERT(valid()); + } LowerCaseString(const LowerCaseString& rhs) : string_(rhs.string_) { ASSERT(valid()); } explicit LowerCaseString(const std::string& new_string) : string_(new_string) { ASSERT(valid()); @@ -113,7 +115,7 @@ class HeaderString { */ explicit HeaderString(const std::string& ref_value); - HeaderString(HeaderString&& move_value); + HeaderString(HeaderString&& move_value) noexcept; ~HeaderString(); /** diff --git a/source/common/http/header_map_impl.cc b/source/common/http/header_map_impl.cc index 90ed21144ddb..2ceafa694186 100644 --- a/source/common/http/header_map_impl.cc +++ b/source/common/http/header_map_impl.cc @@ -54,7 +54,7 @@ HeaderString::HeaderString(const std::string& ref_value) : type_(Type::Reference ASSERT(valid()); } -HeaderString::HeaderString(HeaderString&& move_value) { +HeaderString::HeaderString(HeaderString&& move_value) noexcept { type_ = move_value.type_; string_length_ = move_value.string_length_; switch (move_value.type_) { diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index 3d6ce5284b82..259e011f6b41 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -261,7 +261,7 @@ class StatNameStorage { // Move constructor; needed for using StatNameStorage as an // absl::flat_hash_map value. - StatNameStorage(StatNameStorage&& src) : bytes_(std::move(src.bytes_)) {} + StatNameStorage(StatNameStorage&& src) noexcept : bytes_(std::move(src.bytes_)) {} // Obtains new backing storage for an already existing StatName. Used to // record a computed StatName held in a temp into a more persistent data diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 6dd46f28b347..50a9c5256cae 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -489,7 +489,7 @@ class IsSubsetOfHeadersMatcherImpl : public testing::MatcherInterface(other.expected_headers_)) {} IsSubsetOfHeadersMatcher(const IsSubsetOfHeadersMatcher& other) @@ -544,7 +544,7 @@ class IsSupersetOfHeadersMatcherImpl : public testing::MatcherInterface(other.expected_headers_)) {} IsSupersetOfHeadersMatcher(const IsSupersetOfHeadersMatcher& other) From a659c5ecfd93a9d3a6580cbc968aa3fcca8eb480 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Tue, 25 Jun 2019 20:40:55 -0400 Subject: [PATCH 072/542] forward proxy: add limits and circuit breakers (#7361) Part of https://github.com/envoyproxy/envoy/issues/1606 Signed-off-by: Matt Klein --- .../v2alpha/dns_cache.proto | 10 ++ .../dynamic_forward_proxy_filter.rst | 3 +- .../intro/arch_overview/http/http_proxy.rst | 21 ++- include/envoy/upstream/resource_manager.h | 15 +++ .../clusters/dynamic_forward_proxy/cluster.cc | 6 + .../common/dynamic_forward_proxy/dns_cache.h | 39 ++++-- .../dynamic_forward_proxy/dns_cache_impl.cc | 26 ++-- .../dynamic_forward_proxy/dns_cache_impl.h | 20 +-- .../dynamic_forward_proxy/proxy_filter.cc | 64 ++++++++-- .../http/dynamic_forward_proxy/proxy_filter.h | 13 +- .../dns_cache_impl_test.cc | 120 +++++++++++------- .../common/dynamic_forward_proxy/mocks.cc | 8 +- .../common/dynamic_forward_proxy/mocks.h | 31 +++-- .../proxy_filter_integration_test.cc | 59 +++++++-- .../proxy_filter_test.cc | 81 ++++++++++-- .../reverse_bridge_test.cc | 1 + test/mocks/http/mocks.cc | 23 ++++ test/mocks/http/mocks.h | 25 ++-- 18 files changed, 413 insertions(+), 152 deletions(-) diff --git a/api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto index 33803393fcea..f7c796fe90f6 100644 --- a/api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto +++ b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto @@ -9,6 +9,7 @@ option java_package = "io.envoyproxy.envoy.config.common.dynamic_forward_proxy.v import "envoy/api/v2/cds.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; @@ -55,4 +56,13 @@ message DnsCacheConfig { // // The TTL has no relation to DNS TTL and is only used to control Envoy's resource usage. google.protobuf.Duration host_ttl = 4 [(validate.rules).duration.gt = {}]; + + // The maximum number of hosts that the cache will hold. If not specified defaults to 1024. + // + // .. note: + // + // The implementation is approximate and enforced independently on each worker thread, thus + // it is possible for the maximum hosts in the cache to go slightly above the configured + // value depending on timing. This is similar to how other circuit breakers work. + google.protobuf.UInt32Value max_hosts = 5 [(validate.rules).uint32.gt = 0]; } diff --git a/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst b/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst index e1320a1b19ab..80c569ae5a3e 100644 --- a/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst +++ b/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst @@ -5,8 +5,7 @@ Dynamic forward proxy .. attention:: - HTTP dynamic forward proxy support should be considered alpha and not production ready. Circuit - breakers are missing and will be added soon. + HTTP dynamic forward proxy support should be considered alpha and not production ready. * HTTP dynamic forward proxy :ref:`architecture overview ` * :ref:`v2 API reference ` diff --git a/docs/root/intro/arch_overview/http/http_proxy.rst b/docs/root/intro/arch_overview/http/http_proxy.rst index 28a835153270..496feab8c570 100644 --- a/docs/root/intro/arch_overview/http/http_proxy.rst +++ b/docs/root/intro/arch_overview/http/http_proxy.rst @@ -5,8 +5,7 @@ HTTP dynamic forward proxy .. attention:: - HTTP dynamic forward proxy support should be considered alpha and not production ready. Circuit - breakers are missing and will be added soon. + HTTP dynamic forward proxy support should be considered alpha and not production ready. Through the combination of both an :ref:`HTTP filter ` and :ref:`custom cluster `, @@ -40,11 +39,6 @@ For further configuration information see the :ref:`HTTP filter configuration do Memory usage details -------------------- -.. attention:: - - HTTP dynamic forward proxy support currently can use unbounded memory and is not ready for - production use. Circuit breakers and other limits will be added soon. - Memory usage detail's for Envoy's dynamic forward proxy support are as follows: * Each resolved host/port pair uses a fixed amount of memory global to the server and shared @@ -52,3 +46,16 @@ Memory usage detail's for Envoy's dynamic forward proxy support are as follows: * Address changes are performed inline using read/write locks and require no host reallocations. * Hosts removed via TTL are purged once all active connections stop referring to them and all used memory is regained. +* The :ref:`max_hosts + ` field can + be used to limit the number of hosts that the DNS cache will store at any given time. +* The cluster's :ref:`max_pending_requests + ` circuit breaker can + be used to limit the number of requests that are pending waiting for the DNS cache to load + a host. +* Long lived upstream connections can have the underlying logical host expire via TTL while the + connection is still open. Upstream requests and connections are still bound by other cluster + circuit breakers such as :ref:`max_requests + `. The current assumption is that + host data shared between connections uses a marginal amount of memory compared to the connections + and requests themselves, making it not worth controlling independently. diff --git a/include/envoy/upstream/resource_manager.h b/include/envoy/upstream/resource_manager.h index e439a3af7294..4fe7681aaaa3 100644 --- a/include/envoy/upstream/resource_manager.h +++ b/include/envoy/upstream/resource_manager.h @@ -2,6 +2,7 @@ #include #include +#include #include "envoy/common/pure.h" @@ -48,6 +49,20 @@ class Resource { virtual uint64_t max() PURE; }; +/** + * RAII wrapper that increments a resource on construction and decrements it on destruction. + */ +class ResourceAutoIncDec { +public: + ResourceAutoIncDec(Resource& resource) : resource_(resource) { resource_.inc(); } + ~ResourceAutoIncDec() { resource_.dec(); } + +private: + Resource& resource_; +}; + +using ResourceAutoIncDecPtr = std::unique_ptr; + /** * Global resource manager that loosely synchronizes maximum connections, pending requests, etc. * NOTE: Currently this is used on a per cluster basis. In the future we may consider also chaining diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index 1b6d22699151..e67066f9a444 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -49,6 +49,12 @@ void Cluster::onDnsHostAddOrUpdate( // We should never get a host with no address from the cache. ASSERT(host_info->address() != nullptr); + // NOTE: Right now we allow a DNS cache to be shared between multiple clusters. Though we have + // connection/request circuit breakers on the cluster, we don't have any way to control the + // maximum hosts on a cluster. We currently assume that host data shared via shared pointer is a + // marginal memory cost above that already used by connections and requests, so relying on + // connection/request circuit breakers is sufficient. We may have to revisit this in the future. + HostInfoMapSharedPtr current_map = getCurrentHostMap(); const auto host_map_it = current_map->find(host); if (host_map_it != current_map->end()) { diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache.h b/source/extensions/common/dynamic_forward_proxy/dns_cache.h index 8265b3b4ab04..96ee2d5a7473 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache.h @@ -39,11 +39,11 @@ using DnsHostInfoSharedPtr = std::shared_ptr; class DnsCache { public: /** - * Callbacks used in the loadDnsCache() method. + * Callbacks used in the loadDnsCacheEntry() method. */ - class LoadDnsCacheCallbacks { + class LoadDnsCacheEntryCallbacks { public: - virtual ~LoadDnsCacheCallbacks() = default; + virtual ~LoadDnsCacheEntryCallbacks() = default; /** * Called when the DNS cache load is complete (or failed). @@ -52,14 +52,15 @@ class DnsCache { }; /** - * Handle returned from loadDnsCache(). Destruction of the handle will cancel any future callback. + * Handle returned from loadDnsCacheEntry(). Destruction of the handle will cancel any future + * callback. */ - class LoadDnsCacheHandle { + class LoadDnsCacheEntryHandle { public: - virtual ~LoadDnsCacheHandle() = default; + virtual ~LoadDnsCacheEntryHandle() = default; }; - using LoadDnsCacheHandlePtr = std::unique_ptr; + using LoadDnsCacheEntryHandlePtr = std::unique_ptr; /** * Update callbacks that can be registered in the addUpdateCallbacks() method. @@ -104,12 +105,26 @@ class DnsCache { * target ports. * @param default_port supplies the port to use if the host does not have a port embedded in it. * @param callbacks supplies the cache load callbacks to invoke if async processing is needed. - * @return a cache load handle, in which case the callbacks will be invoked at a later time, - * or nullptr if the cache is already loaded. In this case, callbacks will never be - * called. + * @return a cache load result which includes both a status and handle. If the handle is non-null + * the callbacks will be invoked at a later time, otherwise consult the status for the + * reason the cache is not loading. In this case, callbacks will never be called. */ - virtual LoadDnsCacheHandlePtr loadDnsCache(absl::string_view host, uint16_t default_port, - LoadDnsCacheCallbacks& callbacks) PURE; + enum class LoadDnsCacheEntryStatus { + // The cache entry is already loaded. Callbacks will not be called. + InCache, + // The cache entry is loading. Callbacks will be called at a later time unless cancelled. + Loading, + // The cache is full and the requested host is not in cache. Callbacks will not be called. + Overflow + }; + + struct LoadDnsCacheEntryResult { + LoadDnsCacheEntryStatus status_; + LoadDnsCacheEntryHandlePtr handle_; + }; + + virtual LoadDnsCacheEntryResult loadDnsCacheEntry(absl::string_view host, uint16_t default_port, + LoadDnsCacheEntryCallbacks& callbacks) PURE; /** * Add update callbacks to the cache. diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index 484b6f25142e..81dbc25805ca 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -10,9 +10,6 @@ namespace Extensions { namespace Common { namespace DynamicForwardProxy { -// TODO(mattklein123): circuit breakers / maximums on the number of hosts that the cache can -// contain. - DnsCacheImpl::DnsCacheImpl( Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls, Stats::Scope& root_scope, @@ -23,7 +20,8 @@ DnsCacheImpl::DnsCacheImpl( scope_(root_scope.createScope(fmt::format("dns_cache.{}.", config.name()))), stats_{ALL_DNS_CACHE_STATS(POOL_COUNTER(*scope_), POOL_GAUGE(*scope_))}, refresh_interval_(PROTOBUF_GET_MS_OR_DEFAULT(config, dns_refresh_rate, 60000)), - host_ttl_(PROTOBUF_GET_MS_OR_DEFAULT(config, host_ttl, 300000)) { + host_ttl_(PROTOBUF_GET_MS_OR_DEFAULT(config, host_ttl, 300000)), + max_hosts_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_hosts, 1024)) { tls_slot_->set([](Event::Dispatcher&) { return std::make_shared(); }); updateTlsHostsMap(); } @@ -40,21 +38,29 @@ DnsCacheImpl::~DnsCacheImpl() { } } -DnsCacheImpl::LoadDnsCacheHandlePtr DnsCacheImpl::loadDnsCache(absl::string_view host, - uint16_t default_port, - LoadDnsCacheCallbacks& callbacks) { +DnsCacheImpl::LoadDnsCacheEntryResult +DnsCacheImpl::loadDnsCacheEntry(absl::string_view host, uint16_t default_port, + LoadDnsCacheEntryCallbacks& callbacks) { ENVOY_LOG(debug, "thread local lookup for host '{}'", host); auto& tls_host_info = tls_slot_->getTyped(); auto tls_host = tls_host_info.host_map_->find(host); if (tls_host != tls_host_info.host_map_->end()) { ENVOY_LOG(debug, "thread local hit for host '{}'", host); - return nullptr; + return {LoadDnsCacheEntryStatus::InCache, nullptr}; + } else if (tls_host_info.host_map_->size() >= max_hosts_) { + // Given that we do this check in thread local context, it's possible for two threads to race + // and potentially go slightly above the configured max hosts. This is an OK given compromise + // given how much simpler the implementation is. + ENVOY_LOG(debug, "DNS cache overflow for host '{}'", host); + stats_.host_overflow_.inc(); + return {LoadDnsCacheEntryStatus::Overflow, nullptr}; } else { ENVOY_LOG(debug, "thread local miss for host '{}', posting to main thread", host); main_thread_dispatcher_.post( [this, host = std::string(host), default_port]() { startCacheLoad(host, default_port); }); - return std::make_unique(tls_host_info.pending_resolutions_, host, - callbacks); + return {LoadDnsCacheEntryStatus::Loading, + std::make_unique(tls_host_info.pending_resolutions_, host, + callbacks)}; } } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index 387282abaa16..7b6cd6940c79 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -23,6 +23,7 @@ namespace DynamicForwardProxy { COUNTER(dns_query_success) \ COUNTER(host_added) \ COUNTER(host_address_changed) \ + COUNTER(host_overflow) \ COUNTER(host_removed) \ GAUGE(num_hosts, NeverImport) @@ -41,23 +42,23 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable; using TlsHostMapSharedPtr = std::shared_ptr; - struct LoadDnsCacheHandleImpl : public LoadDnsCacheHandle, - RaiiListElement { - LoadDnsCacheHandleImpl(std::list& parent, absl::string_view host, - LoadDnsCacheCallbacks& callbacks) - : RaiiListElement(parent, this), host_(host), + struct LoadDnsCacheEntryHandleImpl : public LoadDnsCacheEntryHandle, + RaiiListElement { + LoadDnsCacheEntryHandleImpl(std::list& parent, + absl::string_view host, LoadDnsCacheEntryCallbacks& callbacks) + : RaiiListElement(parent, this), host_(host), callbacks_(callbacks) {} const std::string host_; - LoadDnsCacheCallbacks& callbacks_; + LoadDnsCacheEntryCallbacks& callbacks_; }; // Per-thread DNS cache info including the currently known hosts as well as any pending callbacks. @@ -66,7 +67,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable pending_resolutions_; + std::list pending_resolutions_; }; struct DnsHostInfoImpl : public DnsHostInfo { @@ -129,6 +130,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable primary_hosts_; const std::chrono::milliseconds refresh_interval_; const std::chrono::milliseconds host_ttl_; + const uint32_t max_hosts_; }; } // namespace DynamicForwardProxy diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc index f0df762ac708..696bc32fffa2 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.cc @@ -7,6 +7,15 @@ namespace Extensions { namespace HttpFilters { namespace DynamicForwardProxy { +struct ResponseStringValues { + const std::string DnsCacheOverflow = "DNS cache overflow"; + const std::string PendingRequestOverflow = "Dynamic forward proxy pending request overflow"; +}; + +using ResponseStrings = ConstSingleton; + +using LoadDnsCacheEntryStatus = Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryStatus; + ProxyFilterConfig::ProxyFilterConfig( const envoy::config::filter::http::dynamic_forward_proxy::v2alpha::FilterConfig& proto_config, Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory& cache_manager_factory, @@ -19,43 +28,78 @@ void ProxyFilter::onDestroy() { // Make sure we destroy any active cache load handle in case we are getting reset and deferred // deleted. cache_load_handle_.reset(); + circuit_breaker_.reset(); } Http::FilterHeadersStatus ProxyFilter::decodeHeaders(Http::HeaderMap& headers, bool) { Router::RouteConstSharedPtr route = decoder_callbacks_->route(); - if (!route || !route->routeEntry()) { + const Router::RouteEntry* route_entry; + if (!route || !(route_entry = route->routeEntry())) { return Http::FilterHeadersStatus::Continue; } - Upstream::ThreadLocalCluster* cluster = - config_->clusterManager().get(route->routeEntry()->clusterName()); + Upstream::ThreadLocalCluster* cluster = config_->clusterManager().get(route_entry->clusterName()); if (!cluster) { return Http::FilterHeadersStatus::Continue; } + cluster_info_ = cluster->info(); + + auto& resource = cluster_info_->resourceManager(route_entry->priority()).pendingRequests(); + if (!resource.canCreate()) { + ENVOY_STREAM_LOG(debug, "pending request overflow", *decoder_callbacks_); + cluster_info_->stats().upstream_rq_pending_overflow_.inc(); + decoder_callbacks_->sendLocalReply( + Http::Code::ServiceUnavailable, ResponseStrings::get().PendingRequestOverflow, nullptr, + absl::nullopt, ResponseStrings::get().PendingRequestOverflow); + return Http::FilterHeadersStatus::StopIteration; + } + circuit_breaker_ = std::make_unique(resource); uint16_t default_port = 80; - if (cluster->info()->transportSocketFactory().implementsSecureTransport()) { + if (cluster_info_->transportSocketFactory().implementsSecureTransport()) { default_port = 443; } - // See the comments in dns_cache.h for how loadDnsCache() handles hosts with embedded ports. + // See the comments in dns_cache.h for how loadDnsCacheEntry() handles hosts with embedded ports. // TODO(mattklein123): Because the filter and cluster have independent configuration, it is // not obvious to the user if something is misconfigured. We should see if // we can do better here, perhaps by checking the cache to see if anything // else is attached to it or something else? - cache_load_handle_ = - config_->cache().loadDnsCache(headers.Host()->value().getStringView(), default_port, *this); + auto result = config_->cache().loadDnsCacheEntry(headers.Host()->value().getStringView(), + default_port, *this); + cache_load_handle_ = std::move(result.handle_); if (cache_load_handle_ == nullptr) { - ENVOY_STREAM_LOG(debug, "DNS cache already loaded, continuing", *decoder_callbacks_); + circuit_breaker_.reset(); + } + + switch (result.status_) { + case LoadDnsCacheEntryStatus::InCache: { + ASSERT(cache_load_handle_ == nullptr); + ENVOY_STREAM_LOG(debug, "DNS cache entry already loaded, continuing", *decoder_callbacks_); return Http::FilterHeadersStatus::Continue; } + case LoadDnsCacheEntryStatus::Loading: { + ASSERT(cache_load_handle_ != nullptr); + ENVOY_STREAM_LOG(debug, "waiting to load DNS cache entry", *decoder_callbacks_); + return Http::FilterHeadersStatus::StopAllIterationAndWatermark; + } + case LoadDnsCacheEntryStatus::Overflow: { + ASSERT(cache_load_handle_ == nullptr); + ENVOY_STREAM_LOG(debug, "DNS cache overflow", *decoder_callbacks_); + decoder_callbacks_->sendLocalReply(Http::Code::ServiceUnavailable, + ResponseStrings::get().DnsCacheOverflow, nullptr, + absl::nullopt, ResponseStrings::get().DnsCacheOverflow); + return Http::FilterHeadersStatus::StopIteration; + } + } - ENVOY_STREAM_LOG(debug, "waiting to load DNS cache", *decoder_callbacks_); - return Http::FilterHeadersStatus::StopAllIterationAndWatermark; + NOT_REACHED_GCOVR_EXCL_LINE; } void ProxyFilter::onLoadDnsCacheComplete() { ENVOY_STREAM_LOG(debug, "load DNS cache complete, continuing", *decoder_callbacks_); + ASSERT(circuit_breaker_ != nullptr); + circuit_breaker_.reset(); decoder_callbacks_->continueDecoding(); } diff --git a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h index 83f431eb1d93..16ab1fa44003 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h +++ b/source/extensions/filters/http/dynamic_forward_proxy/proxy_filter.h @@ -29,9 +29,10 @@ class ProxyFilterConfig { using ProxyFilterConfigSharedPtr = std::shared_ptr; -class ProxyFilter : public Http::PassThroughDecoderFilter, - public Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheCallbacks, - Logger::Loggable { +class ProxyFilter + : public Http::PassThroughDecoderFilter, + public Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryCallbacks, + Logger::Loggable { public: ProxyFilter(const ProxyFilterConfigSharedPtr& config) : config_(config) {} @@ -39,12 +40,14 @@ class ProxyFilter : public Http::PassThroughDecoderFilter, Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap& headers, bool end_stream) override; void onDestroy() override; - // Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheCallbacks + // Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryCallbacks void onLoadDnsCacheComplete() override; private: const ProxyFilterConfigSharedPtr config_; - Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheHandlePtr cache_load_handle_; + Upstream::ClusterInfoConstSharedPtr cluster_info_; + Upstream::ResourceAutoIncDecPtr circuit_breaker_; + Extensions::Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryHandlePtr cache_load_handle_; }; } // namespace DynamicForwardProxy diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index dbf1120fc274..799a65b7a4bb 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -79,13 +79,14 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); checkStats(1 /* attempt */, 0 /* success */, 0 /* failure */, 0 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -137,13 +138,14 @@ TEST_F(DnsCacheImplTest, TTL) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); checkStats(1 /* attempt */, 0 /* success */, 0 /* failure */, 0 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -182,8 +184,9 @@ TEST_F(DnsCacheImplTest, TTL) { resolve_timer = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_NE(handle, nullptr); + result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); checkStats(3 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, 2 /* added */, 1 /* removed */, 1 /* num hosts */); } @@ -195,13 +198,14 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); @@ -228,11 +232,12 @@ TEST_F(DnsCacheImplTest, InlineResolve) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Event::PostCb post_cb; EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb)); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("localhost", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("localhost", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*resolver_, resolve("localhost", _, _)) @@ -253,12 +258,13 @@ TEST_F(DnsCacheImplTest, ResolveFailure) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); checkStats(1 /* attempt */, 0 /* success */, 0 /* failure */, 0 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -268,8 +274,9 @@ TEST_F(DnsCacheImplTest, ResolveFailure) { checkStats(1 /* attempt */, 0 /* success */, 1 /* failure */, 0 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); - handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_EQ(handle, nullptr); + result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::InCache, result.status_); + EXPECT_EQ(result.handle_, nullptr); } // Cancel a cache load before the resolve completes. @@ -277,14 +284,15 @@ TEST_F(DnsCacheImplTest, CancelResolve) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); - handle.reset(); + result.handle_.reset(); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); resolve_cb(makeAddressList({"10.0.0.1"})); @@ -296,16 +304,18 @@ TEST_F(DnsCacheImplTest, MultipleResolveSameHost) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks1; + MockLoadDnsCacheEntryCallbacks callbacks1; Network::DnsResolver::ResolveCb resolve_cb; EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle1 = dns_cache_->loadDnsCache("foo.com", 80, callbacks1); - EXPECT_NE(handle1, nullptr); + auto result1 = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks1); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result1.status_); + EXPECT_NE(result1.handle_, nullptr); - MockLoadDnsCacheCallbacks callbacks2; - DnsCache::LoadDnsCacheHandlePtr handle2 = dns_cache_->loadDnsCache("foo.com", 80, callbacks2); - EXPECT_NE(handle2, nullptr); + MockLoadDnsCacheEntryCallbacks callbacks2; + auto result2 = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks2); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result2.status_); + EXPECT_NE(result2.handle_, nullptr); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); @@ -319,19 +329,21 @@ TEST_F(DnsCacheImplTest, MultipleResolveDifferentHost) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks1; + MockLoadDnsCacheEntryCallbacks callbacks1; Network::DnsResolver::ResolveCb resolve_cb1; EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb1), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle1 = dns_cache_->loadDnsCache("foo.com", 80, callbacks1); - EXPECT_NE(handle1, nullptr); + auto result1 = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks1); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result1.status_); + EXPECT_NE(result1.handle_, nullptr); - MockLoadDnsCacheCallbacks callbacks2; + MockLoadDnsCacheEntryCallbacks callbacks2; Network::DnsResolver::ResolveCb resolve_cb2; EXPECT_CALL(*resolver_, resolve("bar.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb2), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle2 = dns_cache_->loadDnsCache("bar.com", 443, callbacks2); - EXPECT_NE(handle2, nullptr); + auto result2 = dns_cache_->loadDnsCacheEntry("bar.com", 443, callbacks2); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result2.status_); + EXPECT_NE(result2.handle_, nullptr); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("bar.com", SharedAddressEquals("10.0.0.1:443"))); @@ -349,19 +361,22 @@ TEST_F(DnsCacheImplTest, CacheHit) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); resolve_cb(makeAddressList({"10.0.0.1"})); - EXPECT_EQ(nullptr, dns_cache_->loadDnsCache("foo.com", 80, callbacks)); + result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::InCache, result.status_); + EXPECT_EQ(result.handle_, nullptr); } // Make sure we destroy active queries if the cache goes away. @@ -369,12 +384,13 @@ TEST_F(DnsCacheImplTest, CancelActiveQueriesOnDestroy) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); EXPECT_CALL(resolver_->active_query_, cancel()); dns_cache_.reset(); @@ -385,18 +401,32 @@ TEST_F(DnsCacheImplTest, InvalidPort) { initialize(); InSequence s; - MockLoadDnsCacheCallbacks callbacks; + MockLoadDnsCacheEntryCallbacks callbacks; Network::DnsResolver::ResolveCb resolve_cb; EXPECT_CALL(*resolver_, resolve("foo.com:abc", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); - DnsCache::LoadDnsCacheHandlePtr handle = dns_cache_->loadDnsCache("foo.com:abc", 80, callbacks); - EXPECT_NE(handle, nullptr); + auto result = dns_cache_->loadDnsCacheEntry("foo.com:abc", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); resolve_cb(makeAddressList({})); } +// Max host overflow. +TEST_F(DnsCacheImplTest, MaxHostOverflow) { + config_.mutable_max_hosts()->set_value(0); + initialize(); + InSequence s; + + MockLoadDnsCacheEntryCallbacks callbacks; + auto result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Overflow, result.status_); + EXPECT_EQ(result.handle_, nullptr); + EXPECT_EQ(1, TestUtility::findCounter(store_, "dns_cache.foo.host_overflow")->value()); +} + // DNS cache manager config tests. TEST(DnsCacheManagerImplTest, LoadViaConfig) { NiceMock dispatcher; diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.cc b/test/extensions/common/dynamic_forward_proxy/mocks.cc index 2ec514837de3..f2d6c207c009 100644 --- a/test/extensions/common/dynamic_forward_proxy/mocks.cc +++ b/test/extensions/common/dynamic_forward_proxy/mocks.cc @@ -12,8 +12,8 @@ namespace DynamicForwardProxy { MockDnsCache::MockDnsCache() = default; MockDnsCache::~MockDnsCache() = default; -MockLoadDnsCacheHandle::MockLoadDnsCacheHandle() = default; -MockLoadDnsCacheHandle::~MockLoadDnsCacheHandle() { onDestroy(); } +MockLoadDnsCacheEntryHandle::MockLoadDnsCacheEntryHandle() = default; +MockLoadDnsCacheEntryHandle::~MockLoadDnsCacheEntryHandle() { onDestroy(); } MockDnsCacheManager::MockDnsCacheManager() { ON_CALL(*this, getCache(_)).WillByDefault(Return(dns_cache_)); @@ -28,8 +28,8 @@ MockDnsHostInfo::~MockDnsHostInfo() = default; MockUpdateCallbacks::MockUpdateCallbacks() = default; MockUpdateCallbacks::~MockUpdateCallbacks() = default; -MockLoadDnsCacheCallbacks::MockLoadDnsCacheCallbacks() = default; -MockLoadDnsCacheCallbacks::~MockLoadDnsCacheCallbacks() = default; +MockLoadDnsCacheEntryCallbacks::MockLoadDnsCacheEntryCallbacks() = default; +MockLoadDnsCacheEntryCallbacks::~MockLoadDnsCacheEntryCallbacks() = default; } // namespace DynamicForwardProxy } // namespace Common diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.h b/test/extensions/common/dynamic_forward_proxy/mocks.h index 5fd19e34f481..0d2bfad3f1c7 100644 --- a/test/extensions/common/dynamic_forward_proxy/mocks.h +++ b/test/extensions/common/dynamic_forward_proxy/mocks.h @@ -14,13 +14,20 @@ class MockDnsCache : public DnsCache { MockDnsCache(); ~MockDnsCache(); - LoadDnsCacheHandlePtr loadDnsCache(absl::string_view host, uint16_t default_port, - LoadDnsCacheCallbacks& callbacks) override { - return LoadDnsCacheHandlePtr{loadDnsCache_(host, default_port, callbacks)}; + struct MockLoadDnsCacheEntryResult { + LoadDnsCacheEntryStatus status_; + LoadDnsCacheEntryHandle* handle_; + }; + + LoadDnsCacheEntryResult loadDnsCacheEntry(absl::string_view host, uint16_t default_port, + LoadDnsCacheEntryCallbacks& callbacks) override { + MockLoadDnsCacheEntryResult result = loadDnsCacheEntry_(host, default_port, callbacks); + return {result.status_, LoadDnsCacheEntryHandlePtr{result.handle_}}; } - MOCK_METHOD3(loadDnsCache_, - DnsCache::LoadDnsCacheHandle*(absl::string_view host, uint16_t default_port, - LoadDnsCacheCallbacks& callbacks)); + MOCK_METHOD3(loadDnsCacheEntry_, + MockLoadDnsCacheEntryResult(absl::string_view host, uint16_t default_port, + LoadDnsCacheEntryCallbacks& callbacks)); + AddUpdateCallbacksHandlePtr addUpdateCallbacks(UpdateCallbacks& callbacks) override { return AddUpdateCallbacksHandlePtr{addUpdateCallbacks_(callbacks)}; } @@ -28,10 +35,10 @@ class MockDnsCache : public DnsCache { DnsCache::AddUpdateCallbacksHandle*(UpdateCallbacks& callbacks)); }; -class MockLoadDnsCacheHandle : public DnsCache::LoadDnsCacheHandle { +class MockLoadDnsCacheEntryHandle : public DnsCache::LoadDnsCacheEntryHandle { public: - MockLoadDnsCacheHandle(); - ~MockLoadDnsCacheHandle(); + MockLoadDnsCacheEntryHandle(); + ~MockLoadDnsCacheEntryHandle(); MOCK_METHOD0(onDestroy, void()); }; @@ -70,10 +77,10 @@ class MockUpdateCallbacks : public DnsCache::UpdateCallbacks { MOCK_METHOD1(onDnsHostRemove, void(const std::string& host)); }; -class MockLoadDnsCacheCallbacks : public DnsCache::LoadDnsCacheCallbacks { +class MockLoadDnsCacheEntryCallbacks : public DnsCache::LoadDnsCacheEntryCallbacks { public: - MockLoadDnsCacheCallbacks(); - ~MockLoadDnsCacheCallbacks(); + MockLoadDnsCacheEntryCallbacks(); + ~MockLoadDnsCacheEntryCallbacks(); MOCK_METHOD0(onLoadDnsCacheComplete, void()); }; diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 27249b2e33aa..ae7d16374682 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -21,7 +21,7 @@ class ProxyFilterIntegrationTest : public testing::TestWithParammutable_clusters(0); - cluster_0->clear_hosts(); - cluster_0->set_lb_policy(envoy::api::v2::Cluster::CLUSTER_PROVIDED); + config_helper_.addConfigModifier( + [max_hosts](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters(0); + cluster_0->clear_hosts(); + cluster_0->set_lb_policy(envoy::api::v2::Cluster::CLUSTER_PROVIDED); - const std::string cluster_type_config = fmt::format(R"EOF( + const std::string cluster_type_config = + fmt::format(R"EOF( name: envoy.clusters.dynamic_forward_proxy typed_config: "@type": type.googleapis.com/envoy.config.cluster.dynamic_forward_proxy.v2alpha.ClusterConfig dns_cache_config: name: foo dns_lookup_family: {} + max_hosts: {} )EOF", - ipVersionToDnsFamily(GetParam())); + ipVersionToDnsFamily(GetParam()), max_hosts); - TestUtility::loadFromYaml(cluster_type_config, *cluster_0->mutable_cluster_type()); - }); + TestUtility::loadFromYaml(cluster_type_config, *cluster_0->mutable_cluster_type()); + }); HttpIntegrationTest::initialize(); } @@ -63,8 +67,9 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, ProxyFilterIntegrationTest, // A basic test where we pause a request to lookup localhost, and then do another request which // should hit the TLS cache. TEST_P(ProxyFilterIntegrationTest, RequestWithBody) { + setup(); codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestHeaderMapImpl request_headers{ + const Http::TestHeaderMapImpl request_headers{ {":method", "POST"}, {":path", "/test/long/url"}, {":scheme", "http"}, @@ -86,8 +91,9 @@ TEST_P(ProxyFilterIntegrationTest, RequestWithBody) { // Verify that we expire hosts. TEST_P(ProxyFilterIntegrationTest, RemoveHostViaTTL) { + setup(); codec_client_ = makeHttpConnection(lookupPort("http")); - Http::TestHeaderMapImpl request_headers{ + const Http::TestHeaderMapImpl request_headers{ {":method", "POST"}, {":path", "/test/long/url"}, {":scheme", "http"}, @@ -100,6 +106,7 @@ TEST_P(ProxyFilterIntegrationTest, RemoveHostViaTTL) { EXPECT_EQ(1, test_server_->counter("dns_cache.foo.dns_query_attempt")->value()); EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_added")->value()); EXPECT_EQ(1, test_server_->gauge("dns_cache.foo.num_hosts")->value()); + cleanupUpstreamAndDownstream(); // > 5m simTime().sleep(std::chrono::milliseconds(300001)); @@ -107,5 +114,33 @@ TEST_P(ProxyFilterIntegrationTest, RemoveHostViaTTL) { EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_removed")->value()); } +// Test DNS cache host overflow. +TEST_P(ProxyFilterIntegrationTest, DNSCacheHostOverflow) { + setup(1); + + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = + sendRequestAndWaitForResponse(request_headers, 1024, default_response_headers_, 1024); + checkSimpleRequestSuccess(1024, 1024, response.get()); + + // Send another request, this should lead to a response directly from the filter. + const Http::TestHeaderMapImpl request_headers2{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", fmt::format("localhost2", fake_upstreams_[0]->localAddress()->ip()->port())}}; + response = codec_client_->makeHeaderOnlyRequest(request_headers2); + response->waitForEndStream(); + EXPECT_EQ("503", response->headers().Status()->value().getStringView()); + EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_overflow")->value()); +} + } // namespace } // namespace Envoy diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc index 9fae4430601f..4718dee34720 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc @@ -15,6 +15,10 @@ namespace HttpFilters { namespace DynamicForwardProxy { namespace { +using LoadDnsCacheEntryStatus = Common::DynamicForwardProxy::DnsCache::LoadDnsCacheEntryStatus; +using MockLoadDnsCacheEntryResult = + Common::DynamicForwardProxy::MockDnsCache::MockLoadDnsCacheEntryResult; + class ProxyFilterTest : public testing::Test, public Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactory { public: @@ -31,6 +35,14 @@ class ProxyFilterTest : public testing::Test, // Allow for an otherwise strict mock. EXPECT_CALL(callbacks_, connection()).Times(AtLeast(0)); EXPECT_CALL(callbacks_, streamId()).Times(AtLeast(0)); + + // Configure max pending to 1 so we can test circuit breaking. + cm_.thread_local_cluster_.cluster_.info_->resetResourceManager(0, 1, 0, 0, 0); + } + + ~ProxyFilterTest() { + EXPECT_TRUE( + cm_.thread_local_cluster_.cluster_.info_->resource_manager_->pendingRequests().canCreate()); } Extensions::Common::DynamicForwardProxy::DnsCacheManagerSharedPtr get() override { @@ -55,10 +67,10 @@ TEST_F(ProxyFilterTest, HttpDefaultPort) { EXPECT_CALL(callbacks_, route()); EXPECT_CALL(cm_, get(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(false)); - Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheHandle* handle = - new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheHandle(); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCache_(Eq("foo"), 80, _)) - .WillOnce(Return(handle)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("foo"), 80, _)) + .WillOnce(Return(MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle})); EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, filter_->decodeHeaders(request_headers_, false)); @@ -73,13 +85,66 @@ TEST_F(ProxyFilterTest, HttpsDefaultPort) { EXPECT_CALL(callbacks_, route()); EXPECT_CALL(cm_, get(_)); EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); - Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheHandle* handle = - new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheHandle(); - EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCache_(Eq("foo"), 443, _)) - .WillOnce(Return(handle)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("foo"), 443, _)) + .WillOnce(Return(MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle})); + EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, + filter_->decodeHeaders(request_headers_, false)); + + EXPECT_CALL(*handle, onDestroy()); + filter_->onDestroy(); +} + +// Cache overflow. +TEST_F(ProxyFilterTest, CacheOverflow) { + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(cm_, get(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("foo"), 443, _)) + .WillOnce(Return(MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Overflow, nullptr})); + EXPECT_CALL(callbacks_, sendLocalReply(Http::Code::ServiceUnavailable, Eq("DNS cache overflow"), + _, _, Eq("DNS cache overflow"))); + EXPECT_CALL(callbacks_, encodeHeaders_(_, false)); + EXPECT_CALL(callbacks_, encodeData(_, true)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers_, false)); + + filter_->onDestroy(); +} + +// Circuit breaker overflow +TEST_F(ProxyFilterTest, CircuitBreakerOverflow) { + InSequence s; + + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(cm_, get(_)); + EXPECT_CALL(*transport_socket_factory_, implementsSecureTransport()).WillOnce(Return(true)); + Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle* handle = + new Extensions::Common::DynamicForwardProxy::MockLoadDnsCacheEntryHandle(); + EXPECT_CALL(*dns_cache_manager_->dns_cache_, loadDnsCacheEntry_(Eq("foo"), 443, _)) + .WillOnce(Return(MockLoadDnsCacheEntryResult{LoadDnsCacheEntryStatus::Loading, handle})); EXPECT_EQ(Http::FilterHeadersStatus::StopAllIterationAndWatermark, filter_->decodeHeaders(request_headers_, false)); + // Create a second filter for a 2nd request. + auto filter2 = std::make_unique(filter_config_); + filter2->setDecoderFilterCallbacks(callbacks_); + EXPECT_CALL(callbacks_, route()); + EXPECT_CALL(cm_, get(_)); + EXPECT_CALL(callbacks_, sendLocalReply(Http::Code::ServiceUnavailable, + Eq("Dynamic forward proxy pending request overflow"), _, _, + Eq("Dynamic forward proxy pending request overflow"))); + EXPECT_CALL(callbacks_, encodeHeaders_(_, false)); + EXPECT_CALL(callbacks_, encodeData(_, true)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter2->decodeHeaders(request_headers_, false)); + + EXPECT_EQ(1, + cm_.thread_local_cluster_.cluster_.info_->stats_.upstream_rq_pending_overflow_.value()); + filter2->onDestroy(); EXPECT_CALL(*handle, onDestroy()); filter_->onDestroy(); } diff --git a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc index 7a7adf663bf8..137970bb6bb6 100644 --- a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc +++ b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc @@ -65,6 +65,7 @@ TEST_F(ReverseBridgeTest, InvalidGrpcRequest) { // We should remove the first five bytes. Envoy::Buffer::OwnedImpl buffer; buffer.add("abc", 3); + EXPECT_CALL(decoder_callbacks_, sendLocalReply(_, _, _, _, _)); EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, _)).WillOnce(Invoke([](auto& headers, auto) { EXPECT_THAT(headers, HeaderValueOf(Http::Headers::get().Status, "200")); EXPECT_THAT(headers, HeaderValueOf(Http::Headers::get().GrpcStatus, "2")); diff --git a/test/mocks/http/mocks.cc b/test/mocks/http/mocks.cc index d1dab5be2225..c30c9bc400ea 100644 --- a/test/mocks/http/mocks.cc +++ b/test/mocks/http/mocks.cc @@ -65,10 +65,33 @@ MockStreamDecoderFilterCallbacks::MockStreamDecoderFilterCallbacks() { ON_CALL(*this, activeSpan()).WillByDefault(ReturnRef(active_span_)); ON_CALL(*this, tracingConfig()).WillByDefault(ReturnRef(tracing_config_)); + ON_CALL(*this, sendLocalReply(_, _, _, _, _)) + .WillByDefault(Invoke([this](Code code, absl::string_view body, + std::function modify_headers, + const absl::optional grpc_status, + absl::string_view details) { + sendLocalReply_(code, body, modify_headers, grpc_status, details); + })); } MockStreamDecoderFilterCallbacks::~MockStreamDecoderFilterCallbacks() = default; +void MockStreamDecoderFilterCallbacks::sendLocalReply_( + Code code, absl::string_view body, std::function modify_headers, + const absl::optional grpc_status, absl::string_view details) { + details_ = std::string(details); + Utility::sendLocalReply( + is_grpc_request_, + [this, modify_headers](HeaderMapPtr&& headers, bool end_stream) -> void { + if (modify_headers != nullptr) { + modify_headers(*headers); + } + encodeHeaders(std::move(headers), end_stream); + }, + [this](Buffer::Instance& data, bool end_stream) -> void { encodeData(data, end_stream); }, + stream_destroyed_, code, body, grpc_status, is_head_request_); +} + MockStreamEncoderFilterCallbacks::MockStreamEncoderFilterCallbacks() { initializeMockStreamFilterCallbacks(*this); ON_CALL(*this, encodingBuffer()).WillByDefault(Invoke(&buffer_, &Buffer::InstancePtr::get)); diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 50a9c5256cae..64ebef385c5f 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -152,22 +152,11 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, MOCK_CONST_METHOD0(getUpstreamSocketOptions, Network::Socket::OptionsSharedPtr()); // Http::StreamDecoderFilterCallbacks - void sendLocalReply(Code code, absl::string_view body, - std::function modify_headers, - const absl::optional grpc_status, - absl::string_view details) override { - details_ = std::string(details); - Utility::sendLocalReply( - is_grpc_request_, - [this, modify_headers](HeaderMapPtr&& headers, bool end_stream) -> void { - if (modify_headers != nullptr) { - modify_headers(*headers); - } - encodeHeaders(std::move(headers), end_stream); - }, - [this](Buffer::Instance& data, bool end_stream) -> void { encodeData(data, end_stream); }, - stream_destroyed_, code, body, grpc_status, is_head_request_); - } + void sendLocalReply_(Code code, absl::string_view body, + std::function modify_headers, + const absl::optional grpc_status, + absl::string_view details); + void encode100ContinueHeaders(HeaderMapPtr&& headers) override { encode100ContinueHeaders_(*headers); } @@ -190,6 +179,10 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, MOCK_METHOD2(encodeData, void(Buffer::Instance& data, bool end_stream)); MOCK_METHOD1(encodeTrailers_, void(HeaderMap& trailers)); MOCK_METHOD1(encodeMetadata_, void(MetadataMapPtr metadata_map)); + MOCK_METHOD5(sendLocalReply, void(Code code, absl::string_view body, + std::function modify_headers, + const absl::optional grpc_status, + absl::string_view details)); Buffer::InstancePtr buffer_; std::list callbacks_{}; From 136f1dafb30c71314e270d35be021ef49b92174b Mon Sep 17 00:00:00 2001 From: cclauss Date: Wed, 26 Jun 2019 06:49:59 +0200 Subject: [PATCH 073/542] Define cmp() in Python 3 (#7385) Related to #4552 __cmp()__ was removed in Python 3 so redefine it in accordance with https://portingguide.readthedocs.io/en/latest/comparisons.html#the-cmp-function Signed-off-by: Christian Clauss cclauss@me.com Signed-off-by: cclauss --- tools/check_spelling_pedantic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/check_spelling_pedantic.py b/tools/check_spelling_pedantic.py index b8aaca0ae2a4..393674c65c21 100755 --- a/tools/check_spelling_pedantic.py +++ b/tools/check_spelling_pedantic.py @@ -17,6 +17,14 @@ except NameError: pass +try: + cmp +except NameError: + + def cmp(x, y): + return (x > y) - (x < y) + + TOOLS_DIR = os.path.dirname(os.path.realpath(__file__)) # Single line comments: // comment OR /* comment */ From 1faaed85740a97533484db3232796aef7973677f Mon Sep 17 00:00:00 2001 From: htuch Date: Wed, 26 Jun 2019 00:50:25 -0400 Subject: [PATCH 074/542] opencensus: tweak enum in configuration API. (#7280) Since this API is still experimental, tweaking to match best proto practices. Risk level: Low Signed-off-by: Harvey Tuch --- api/envoy/config/trace/v2/trace.proto | 9 ++++++--- .../tracers/opencensus/opencensus_tracer_impl.cc | 12 ++++++------ test/extensions/tracers/opencensus/tracer_test.cc | 14 ++++++++------ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/api/envoy/config/trace/v2/trace.proto b/api/envoy/config/trace/v2/trace.proto index 21fb8d8ddf21..ac2a9cbb7649 100644 --- a/api/envoy/config/trace/v2/trace.proto +++ b/api/envoy/config/trace/v2/trace.proto @@ -134,14 +134,17 @@ message OpenCensusConfig { string zipkin_service_name = 7; enum TraceContext { + // No-op default, no trace context is utilized. + NONE = 0; + // W3C Trace-Context format "traceparent:" header. - trace_context = 0; + TRACE_CONTEXT = 1; // Binary "grpc-trace-bin:" header. - grpc_trace_bin = 1; + GRPC_TRACE_BIN = 2; // "X-Cloud-Trace-Context:" header. - cloud_trace_context = 2; + CLOUD_TRACE_CONTEXT = 3; } // List of incoming trace context headers we will accept. First one found diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 5badb14f4faf..6a93dbbffd90 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -69,7 +69,7 @@ startSpanHelper(const std::string& name, bool traced, const Http::HeaderMap& req for (const auto& incoming : oc_config.incoming_trace_context()) { bool found = false; switch (incoming) { - case OpenCensusConfig::trace_context: { + case OpenCensusConfig::TRACE_CONTEXT: { const Http::HeaderEntry* header = request_headers.get(Constants::get().TRACEPARENT); if (header != nullptr) { found = true; @@ -79,7 +79,7 @@ startSpanHelper(const std::string& name, bool traced, const Http::HeaderMap& req break; } - case OpenCensusConfig::grpc_trace_bin: { + case OpenCensusConfig::GRPC_TRACE_BIN: { const Http::HeaderEntry* header = request_headers.get(Constants::get().GRPC_TRACE_BIN); if (header != nullptr) { found = true; @@ -89,7 +89,7 @@ startSpanHelper(const std::string& name, bool traced, const Http::HeaderMap& req break; } - case OpenCensusConfig::cloud_trace_context: { + case OpenCensusConfig::CLOUD_TRACE_CONTEXT: { const Http::HeaderEntry* header = request_headers.get(Constants::get().X_CLOUD_TRACE_CONTEXT); if (header != nullptr) { found = true; @@ -154,20 +154,20 @@ void Span::injectContext(Http::HeaderMap& request_headers) { using OpenCensusConfig = envoy::config::trace::v2::OpenCensusConfig; for (const auto& outgoing : oc_config_.outgoing_trace_context()) { switch (outgoing) { - case OpenCensusConfig::trace_context: + case OpenCensusConfig::TRACE_CONTEXT: request_headers.setReferenceKey( Constants::get().TRACEPARENT, ::opencensus::trace::propagation::ToTraceParentHeader(span_.context())); break; - case OpenCensusConfig::grpc_trace_bin: { + case OpenCensusConfig::GRPC_TRACE_BIN: { std::string val = ::opencensus::trace::propagation::ToGrpcTraceBinHeader(span_.context()); val = Base64::encode(val.data(), val.size(), /*add_padding=*/false); request_headers.setReferenceKey(Constants::get().GRPC_TRACE_BIN, val); break; } - case OpenCensusConfig::cloud_trace_context: + case OpenCensusConfig::CLOUD_TRACE_CONTEXT: request_headers.setReferenceKey( Constants::get().X_CLOUD_TRACE_CONTEXT, ::opencensus::trace::propagation::ToCloudTraceContextHeader(span_.context())); diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index b6b69c938ebf..a223b08e3be5 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -166,12 +166,14 @@ TEST(OpenCensusTracerTest, PropagateTraceContext) { // The test calls the helper with each kind of incoming context in turn. auto helper = [](const std::string& header, const std::string& value) { OpenCensusConfig oc_config; - oc_config.add_incoming_trace_context(OpenCensusConfig::trace_context); - oc_config.add_incoming_trace_context(OpenCensusConfig::grpc_trace_bin); - oc_config.add_incoming_trace_context(OpenCensusConfig::cloud_trace_context); - oc_config.add_outgoing_trace_context(OpenCensusConfig::trace_context); - oc_config.add_outgoing_trace_context(OpenCensusConfig::grpc_trace_bin); - oc_config.add_outgoing_trace_context(OpenCensusConfig::cloud_trace_context); + oc_config.add_incoming_trace_context(OpenCensusConfig::NONE); + oc_config.add_incoming_trace_context(OpenCensusConfig::TRACE_CONTEXT); + oc_config.add_incoming_trace_context(OpenCensusConfig::GRPC_TRACE_BIN); + oc_config.add_incoming_trace_context(OpenCensusConfig::CLOUD_TRACE_CONTEXT); + oc_config.add_outgoing_trace_context(OpenCensusConfig::NONE); + oc_config.add_outgoing_trace_context(OpenCensusConfig::TRACE_CONTEXT); + oc_config.add_outgoing_trace_context(OpenCensusConfig::GRPC_TRACE_BIN); + oc_config.add_outgoing_trace_context(OpenCensusConfig::CLOUD_TRACE_CONTEXT); std::unique_ptr driver(new OpenCensus::Driver(oc_config)); NiceMock config; Http::TestHeaderMapImpl request_headers{ From c37b3ab5e7e2c17a2303e9af4a2a6afd980b5da5 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 26 Jun 2019 11:54:23 -0700 Subject: [PATCH 075/542] bazel: Update to 0.27.0 (#7391) Signed-off-by: Keith Smiley --- .azure-pipelines/linux.yml | 2 +- .circleci/config.yml | 2 +- api/tools/BUILD | 2 ++ bazel/external/gcovr.BUILD | 1 + bazel/gen_compilation_database.sh | 3 +-- bazel/repository_locations.bzl | 6 +++--- ci/do_ci.sh | 2 +- tools/protodoc/BUILD | 1 + tools/socket_passing.py | 8 ++++---- 9 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index f770c57e925e..4bb64ea59faa 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -1,7 +1,7 @@ resources: containers: - container: envoy-build - image: envoyproxy/envoy-build:ec38ecb88fd1abe55ab1daa2f6bd239ffccc9d98 + image: envoyproxy/envoy-build:d0cefa7f071dbd4ef24399c2db8656c3a5d8c3ef jobs: - job: EnvoyFullTest diff --git a/.circleci/config.yml b/.circleci/config.yml index 76df3f26138f..d3458d108886 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ executors: ubuntu-build: description: "A regular build executor based on ubuntu image" docker: - - image: envoyproxy/envoy-build:ec38ecb88fd1abe55ab1daa2f6bd239ffccc9d98 + - image: envoyproxy/envoy-build:d0cefa7f071dbd4ef24399c2db8656c3a5d8c3ef resource_class: xlarge working_directory: /source diff --git a/api/tools/BUILD b/api/tools/BUILD index b2848aee130d..e90cfa0eef29 100644 --- a/api/tools/BUILD +++ b/api/tools/BUILD @@ -4,6 +4,7 @@ py_binary( name = "tap2pcap", srcs = ["tap2pcap.py"], licenses = ["notice"], # Apache 2 + python_version = "PY2", visibility = ["//visibility:public"], deps = [ "//envoy/data/tap/v2alpha:wrapper_py", @@ -17,6 +18,7 @@ py_test( "data/tap2pcap_h2_ipv4.pb_text", "data/tap2pcap_h2_ipv4.txt", ], + python_version = "PY2", # Don't run this by default, since we don't want to force local dependency on Wireshark/tshark, # will explicitly invoke in CI. tags = ["manual"], diff --git a/bazel/external/gcovr.BUILD b/bazel/external/gcovr.BUILD index 57ee8e7fb971..0215a25da848 100644 --- a/bazel/external/gcovr.BUILD +++ b/bazel/external/gcovr.BUILD @@ -10,6 +10,7 @@ par_binary( name = "gcovr", srcs = [":renamed_gcovr.py"], main = ":renamed_gcovr.py", + python_version = "PY2", visibility = ["//visibility:public"], ) diff --git a/bazel/gen_compilation_database.sh b/bazel/gen_compilation_database.sh index 29afd5aba8f1..d642ddfd285d 100755 --- a/bazel/gen_compilation_database.sh +++ b/bazel/gen_compilation_database.sh @@ -1,7 +1,6 @@ #!/bin/bash -#TODO(lizan): revert to released version once new version is released -RELEASE_VERSION=d5a0ee259aa356886618eafae17ca05ebf79d6c2 +RELEASE_VERSION=0.3.5 if [[ ! -d bazel-compilation-database-${RELEASE_VERSION} ]]; then curl -L https://github.com/grailbio/bazel-compilation-database/archive/${RELEASE_VERSION}.tar.gz | tar -xz diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 9960796a9b88..5818d9108a59 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -63,9 +63,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/gabime/spdlog/archive/v1.3.1.tar.gz"], ), com_github_gcovr_gcovr = dict( - sha256 = "8a60ba6242d67a58320e9e16630d80448ef6d5284fda5fb3eff927b63c8b04a2", - strip_prefix = "gcovr-3.3", - urls = ["https://github.com/gcovr/gcovr/archive/3.3.tar.gz"], + sha256 = "1c52a71f245adfe1b45e30fbe5015337fe66546f17f40038b3969b7b42acceed", + strip_prefix = "gcovr-3.4", + urls = ["https://github.com/gcovr/gcovr/archive/3.4.tar.gz"], ), com_github_google_libprotobuf_mutator = dict( sha256 = "97b3639630040f41c45f45838ab00b78909e6b4cb69c8028e01302bea5b79495", diff --git a/ci/do_ci.sh b/ci/do_ci.sh index d55381c86de0..e048a8abe920 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -246,7 +246,7 @@ elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then # gcovr is a pain to run with `bazel run`, so package it up into a # relocatable and hermetic-ish .par file. - bazel build @com_github_gcovr_gcovr//:gcovr.par + bazel build --python_version=PY2 @com_github_gcovr_gcovr//:gcovr.par export GCOVR="/tmp/gcovr.par" cp -f "${ENVOY_SRCDIR}/bazel-bin/external/com_github_gcovr_gcovr/gcovr.par" ${GCOVR} diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index 55a5fd529f9d..8e428b5d24fd 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -3,6 +3,7 @@ licenses(["notice"]) # Apache 2 py_binary( name = "protodoc", srcs = ["protodoc.py"], + python_version = "PY2", visibility = ["//visibility:public"], deps = [ "@com_envoyproxy_protoc_gen_validate//validate:validate_py", diff --git a/tools/socket_passing.py b/tools/socket_passing.py index 22f57cf59aeb..ee30db34ad01 100755 --- a/tools/socket_passing.py +++ b/tools/socket_passing.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 # This tool is a helper script that queries the admin address for all listener # addresses after envoy startup. (The admin address is written out to a file by @@ -10,7 +10,7 @@ from collections import OrderedDict import argparse -import httplib +import http.client import json import os.path import re @@ -29,12 +29,12 @@ def GenerateNewConfig(original_yaml, admin_address, updated_json): with open(original_yaml, 'r') as original_file: sys.stdout.write('Admin address is ' + admin_address + '\n') try: - admin_conn = httplib.HTTPConnection(admin_address) + admin_conn = http.client.HTTPConnection(admin_address) admin_conn.request('GET', '/listeners?format=json') admin_response = admin_conn.getresponse() if not admin_response.status == 200: return False - discovered_listeners = json.loads(admin_response.read()) + discovered_listeners = json.loads(admin_response.read().decode('utf-8')) except Exception as e: sys.stderr.write('Cannot connect to admin: %s\n' % e) return False From 89eb31bcbe2308bf1e9073620e843bf472363495 Mon Sep 17 00:00:00 2001 From: Michael Rebello Date: Wed, 26 Jun 2019 19:53:51 -0700 Subject: [PATCH 076/542] tools: include .m/.mm files when formatting (#7406) Including these file types will allow this script to be reused in other repos like Envoy Mobile for linting. https://github.com/lyft/envoy-mobile/issues/191 Risk Level: Low Testing: Locally Docs Changes: No Signed-off-by: Michael Rebello --- tools/check_format.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/check_format.py b/tools/check_format.py index c6cc431c7c70..8a1de0b673aa 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -17,7 +17,8 @@ EXCLUDED_PREFIXES = ("./generated/", "./thirdparty/", "./build", "./.git/", "./bazel-", "./.cache", "./source/extensions/extensions_build_config.bzl", "./tools/testdata/check_format/", "./tools/pyformat/") -SUFFIXES = (".cc", ".h", "BUILD", "WORKSPACE", ".bzl", ".java", ".md", ".rst", ".proto") +SUFFIXES = ("BUILD", "WORKSPACE", ".bzl", ".cc", ".h", ".java", ".m", ".md", ".mm", ".proto", + ".rst") DOCS_SUFFIX = (".md", ".rst") PROTO_SUFFIX = (".proto") From 9d2068f31f71e44d8dc1333771c1b7a1ca6eb745 Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Thu, 27 Jun 2019 22:04:16 +0800 Subject: [PATCH 077/542] dns: switch from ares_gethostbyname to ares_getaddrinfo (#7395) Upgrade c-ares. Part of #6975 Description: Upgrade c-ares. Switch from ares_gethostbyname to ares_getaddrinfo Risk Level: Med Testing: Unit test Docs Changes: N/A Release Notes: Signed-off-by: crazyxy --- bazel/repository_locations.bzl | 10 +++-- source/common/network/dns_impl.cc | 71 ++++++++++++++++++------------- source/common/network/dns_impl.h | 12 +++--- tools/spelling_dictionary.txt | 2 + 4 files changed, 57 insertions(+), 38 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 5818d9108a59..cdb4961edadf 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -27,9 +27,13 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://files.pythonhosted.org/packages/c6/b4/510617906f8e0c5660e7d96fbc5585113f83ad547a3989b80297ac72a74c/thrift-0.11.0.tar.gz"], ), com_github_c_ares_c_ares = dict( - sha256 = "7deb7872cbd876c29036d5f37e30c4cbc3cc068d59d8b749ef85bb0736649f04", - strip_prefix = "c-ares-cares-1_15_0", - urls = ["https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz"], + sha256 = "96edccdb19d79f6bc48c2c0e5916346c8f0507efa72e76bd146a1b9d05f93c2a", + strip_prefix = "c-ares-5dd3629bc93449840c36dd635ea6cce606b8c366", + # 2019-06-19 + # 21 new commits from release-1.15.0. Upgrade for commit 7d3591ee8a1a63e7748e68e6d880bd1763a32885 "getaddrinfo enhancements" + # Use getaddrinfo to query DNS record and TTL. + # TODO(crazyxy): Update to release-1.16.0 when it is released. + urls = ["https://github.com/c-ares/c-ares/archive/5dd3629bc93449840c36dd635ea6cce606b8c366.tar.gz"], ), com_github_circonus_labs_libcircllhist = dict( sha256 = "8165aa25e529d7d4b9ae849d3bf30371255a99d6db0421516abcff23214cdc2c", diff --git a/source/common/network/dns_impl.cc b/source/common/network/dns_impl.cc index 3702210c50ce..29e2b48042fa 100644 --- a/source/common/network/dns_impl.cc +++ b/source/common/network/dns_impl.cc @@ -67,8 +67,8 @@ void DnsResolverImpl::initializeChannel(ares_options* options, int optmask) { ares_init_options(&channel_, options, optmask | ARES_OPT_SOCK_STATE_CB); } -void DnsResolverImpl::PendingResolution::onAresHostCallback(int status, int timeouts, - hostent* hostent) { +void DnsResolverImpl::PendingResolution::onAresGetAddrInfoCallback(int status, int timeouts, + ares_addrinfo* addrinfo) { // We receive ARES_EDESTRUCTION when destructing with pending queries. if (status == ARES_EDESTRUCTION) { ASSERT(owned_); @@ -81,30 +81,34 @@ void DnsResolverImpl::PendingResolution::onAresHostCallback(int status, int time std::list address_list; if (status == ARES_SUCCESS) { - if (hostent->h_addrtype == AF_INET) { - for (int i = 0; hostent->h_addr_list[i] != nullptr; ++i) { - ASSERT(hostent->h_length == sizeof(in_addr)); - sockaddr_in address; - memset(&address, 0, sizeof(address)); - address.sin_family = AF_INET; - address.sin_port = 0; - address.sin_addr = *reinterpret_cast(hostent->h_addr_list[i]); - address_list.emplace_back(new Address::Ipv4Instance(&address)); - } - } else if (hostent->h_addrtype == AF_INET6) { - for (int i = 0; hostent->h_addr_list[i] != nullptr; ++i) { - ASSERT(hostent->h_length == sizeof(in6_addr)); - sockaddr_in6 address; - memset(&address, 0, sizeof(address)); - address.sin6_family = AF_INET6; - address.sin6_port = 0; - address.sin6_addr = *reinterpret_cast(hostent->h_addr_list[i]); - address_list.emplace_back(new Address::Ipv6Instance(address)); + if (addrinfo != nullptr && addrinfo->nodes != nullptr) { + if (addrinfo->nodes->ai_family == AF_INET) { + for (const ares_addrinfo_node* ai = addrinfo->nodes; ai != nullptr; ai = ai->ai_next) { + sockaddr_in address; + memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_port = 0; + address.sin_addr = reinterpret_cast(ai->ai_addr)->sin_addr; + address_list.emplace_back(new Address::Ipv4Instance(&address)); + } + } else if (addrinfo->nodes->ai_family == AF_INET6) { + for (const ares_addrinfo_node* ai = addrinfo->nodes; ai != nullptr; ai = ai->ai_next) { + sockaddr_in6 address; + memset(&address, 0, sizeof(address)); + address.sin6_family = AF_INET6; + address.sin6_port = 0; + address.sin6_addr = reinterpret_cast(ai->ai_addr)->sin6_addr; + address_list.emplace_back(new Address::Ipv6Instance(address)); + } } } + if (!address_list.empty()) { completed_ = true; } + + ASSERT(addrinfo != nullptr); + ares_freeaddrinfo(addrinfo); } if (timeouts > 0) { @@ -134,7 +138,7 @@ void DnsResolverImpl::PendingResolution::onAresHostCallback(int status, int time if (!completed_ && fallback_if_failed_) { fallback_if_failed_ = false; - getHostByName(AF_INET); + getAddrInfo(AF_INET); // Note: Nothing can follow this call to getHostByName due to deletion of this // object upon synchronous resolution. return; @@ -195,9 +199,9 @@ ActiveDnsQuery* DnsResolverImpl::resolve(const std::string& dns_name, } if (dns_lookup_family == DnsLookupFamily::V4Only) { - pending_resolution->getHostByName(AF_INET); + pending_resolution->getAddrInfo(AF_INET); } else { - pending_resolution->getHostByName(AF_INET6); + pending_resolution->getAddrInfo(AF_INET6); } if (pending_resolution->completed_) { @@ -215,11 +219,20 @@ ActiveDnsQuery* DnsResolverImpl::resolve(const std::string& dns_name, } } -void DnsResolverImpl::PendingResolution::getHostByName(int family) { - ares_gethostbyname( - channel_, dns_name_.c_str(), family, - [](void* arg, int status, int timeouts, hostent* hostent) { - static_cast(arg)->onAresHostCallback(status, timeouts, hostent); +void DnsResolverImpl::PendingResolution::getAddrInfo(int family) { + struct ares_addrinfo_hints hints = {}; + hints.ai_family = family; + + /** + * ARES_AI_NOSORT result addresses will not be sorted and no connections to resolved addresses + * will be attempted + */ + hints.ai_flags = ARES_AI_NOSORT; + + ares_getaddrinfo( + channel_, dns_name_.c_str(), /* service */ nullptr, &hints, + [](void* arg, int status, int timeouts, ares_addrinfo* addrinfo) { + static_cast(arg)->onAresGetAddrInfoCallback(status, timeouts, addrinfo); }, this); } diff --git a/source/common/network/dns_impl.h b/source/common/network/dns_impl.h index 1dad26e11a01..096c2c71178f 100644 --- a/source/common/network/dns_impl.h +++ b/source/common/network/dns_impl.h @@ -50,17 +50,17 @@ class DnsResolverImpl : public DnsResolver, protected Logger::Loggable Date: Thu, 27 Jun 2019 16:06:24 +0200 Subject: [PATCH 078/542] Run flake8 on Python instead of legacy Python (#7388) Blocked by: #7383, #7384, and #7385 Related to: #4552 #7322 Signed-off-by: cclauss cclauss@me.com Also adds more critical flake8 tests. Signed-off-by: cclauss --- tools/deprecate_features/deprecate_features.py | 3 ++- tools/format_python_tools.sh | 8 +++++--- tools/requirements.txt | 2 +- tools/shell_utils.sh | 2 +- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/deprecate_features/deprecate_features.py b/tools/deprecate_features/deprecate_features.py index cf9f89f945fc..0f67d3e280b9 100644 --- a/tools/deprecate_features/deprecate_features.py +++ b/tools/deprecate_features/deprecate_features.py @@ -4,6 +4,7 @@ import re import subprocess import fileinput +from six.moves import input # Sorts out the list of deprecated proto fields which should be disallowed and returns a tuple of @@ -89,7 +90,7 @@ def flip_runtime_features(): print('\n\nSuggested envoy-announce email: \n') print(email) -if not raw_input('Apply relevant runtime changes? [yN] ').strip().lower() in ('y', 'yes'): +if not input('Apply relevant runtime changes? [yN] ').strip().lower() in ('y', 'yes'): exit(1) for line in fileinput.FileInput('source/common/runtime/runtime_features.cc', inplace=1): diff --git a/tools/format_python_tools.sh b/tools/format_python_tools.sh index c58dccfb763e..91b34feb2122 100755 --- a/tools/format_python_tools.sh +++ b/tools/format_python_tools.sh @@ -9,10 +9,12 @@ cd "$SCRIPTPATH" source_venv "$VENV_DIR" echo "Installing requirements..." -pip install -r requirements.txt +pip3 install --upgrade pip +pip3 install -r requirements.txt echo "Running Python format check..." -python format_python_tools.py $1 +python3 format_python_tools.py $1 echo "Running Python3 flake8 check..." -flake8 . --exclude=*/venv/* --count --select=E901,E999,F821,F822,F823 --show-source --statistics +python3 -m flake8 --version +python3 -m flake8 . --exclude=*/venv/* --count --select=E9,F63,F72,F82 --show-source --statistics diff --git a/tools/requirements.txt b/tools/requirements.txt index b0e19802c518..b6920fa69873 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,2 +1,2 @@ -flake8==3.6.0 +flake8==3.7.7 yapf==0.25.0 diff --git a/tools/shell_utils.sh b/tools/shell_utils.sh index 8aa65eed8f0b..a0c7c60d8def 100644 --- a/tools/shell_utils.sh +++ b/tools/shell_utils.sh @@ -2,7 +2,7 @@ source_venv() { VENV_DIR=$1 if [[ "$VIRTUAL_ENV" == "" ]]; then if [[ ! -d "${VENV_DIR}"/venv ]]; then - virtualenv "${VENV_DIR}"/venv --no-site-packages --python=python2.7 + virtualenv "${VENV_DIR}"/venv --no-site-packages --python=python3.5 fi source "${VENV_DIR}"/venv/bin/activate else From 89d81e65df2a6b364bcf0af4c251241c2dd5866e Mon Sep 17 00:00:00 2001 From: Christoph Pakulski Date: Thu, 27 Jun 2019 14:38:18 -0400 Subject: [PATCH 079/542] upstream: outlier detector - distinguish upstream from internal errors (#4822) Signed-off-by: Christoph Pakulski --- api/envoy/admin/v2alpha/clusters.proto | 51 +- .../api/v2/cluster/outlier_detection.proto | 41 +- .../v2alpha/outlier_detection_event.proto | 29 +- .../cluster_manager/cluster_runtime.rst | 15 + .../cluster_manager/cluster_stats.rst | 8 +- .../intro/arch_overview/upstream/outlier.rst | 94 ++- docs/root/intro/version_history.rst | 1 + include/envoy/upstream/outlier_detection.h | 51 +- source/common/router/router.cc | 24 +- source/common/router/router.h | 3 +- source/common/tcp_proxy/tcp_proxy.cc | 8 +- source/common/upstream/load_balancer_impl.cc | 3 +- .../common/upstream/outlier_detection_impl.cc | 249 ++++++-- .../common/upstream/outlier_detection_impl.h | 212 +++++-- .../network/common/redis/client_impl.cc | 8 +- source/server/http/admin.cc | 55 +- test/common/router/router_test.cc | 140 ++++- .../common/router/router_upstream_log_test.cc | 4 +- test/common/tcp_proxy/tcp_proxy_test.cc | 12 +- .../upstream/outlier_detection_impl_test.cc | 547 +++++++++++++++++- .../network/common/redis/client_impl_test.cc | 68 ++- test/mocks/upstream/host.h | 12 +- test/server/http/admin_test.cc | 30 +- 23 files changed, 1412 insertions(+), 253 deletions(-) diff --git a/api/envoy/admin/v2alpha/clusters.proto b/api/envoy/admin/v2alpha/clusters.proto index 03a85d8f36b0..cc2c95110c6a 100644 --- a/api/envoy/admin/v2alpha/clusters.proto +++ b/api/envoy/admin/v2alpha/clusters.proto @@ -28,9 +28,15 @@ message ClusterStatus { // Denotes whether this cluster was added via API or configured statically. bool added_via_api = 2; - // The success rate threshold used in the last interval. The threshold is used to eject hosts - // based on their success rate. See - // :ref:`Cluster outlier detection ` statistics + // The success rate threshold used in the last interval. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *false*, all errors: externally and locally generated were used to calculate the threshold. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *true*, only externally generated errors were used to calculate the threshold. + // The threshold is used to eject hosts based on their success rate. See + // :ref:`Cluster outlier detection ` documentation for details. // // Note: this field may be omitted in any of the three following cases: // @@ -43,6 +49,23 @@ message ClusterStatus { // Mapping from host address to the host's current status. repeated HostStatus host_statuses = 4; + + // The success rate threshold used in the last interval when only locally originated failures were + // taken into account and externally originated errors were treated as success. + // This field should be interpretted only when + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *true*. The threshold is used to eject hosts based on their success rate. + // See :ref:`Cluster outlier detection ` documentation for + // details. + // + // Note: this field may be omitted in any of the three following cases: + // + // 1. There were not enough hosts with enough request volume to proceed with success rate based + // outlier ejection. + // 2. The threshold is computed to be < 0 because a negative value implies that there was no + // threshold for that interval. + // 3. Outlier detection is not enabled for this cluster. + envoy.type.Percent local_origin_success_rate_ejection_threshold = 5; } // Current state of a particular host. @@ -57,6 +80,14 @@ message HostStatus { HostHealthStatus health_status = 3; // Request success rate for this host over the last calculated interval. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *false*, all errors: externally and locally generated were used in success rate + // calculation. If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *true*, only externally generated errors were used in success rate calculation. + // See :ref:`Cluster outlier detection ` documentation for + // details. // // Note: the message will not be present if host did not have enough request volume to calculate // success rate or the cluster did not have enough hosts to run through success rate outlier @@ -71,6 +102,20 @@ message HostStatus { // The host's priority. If not configured, the value defaults to 0 (highest priority). uint32 priority = 7; + + // Request success rate for this host over the last calculated + // interval when only locally originated errors are taken into account and externally originated + // errors were treated as success. + // This field should be interpretted only when + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *true*. + // See :ref:`Cluster outlier detection ` documentation for + // details. + // + // Note: the message will not be present if host did not have enough request volume to calculate + // success rate or the cluster did not have enough hosts to run through success rate outlier + // ejection. + envoy.type.Percent local_origin_success_rate = 8; } // Health status for a host. diff --git a/api/envoy/api/v2/cluster/outlier_detection.proto b/api/envoy/api/v2/cluster/outlier_detection.proto index 95f132242c11..69feb60e2325 100644 --- a/api/envoy/api/v2/cluster/outlier_detection.proto +++ b/api/envoy/api/v2/cluster/outlier_detection.proto @@ -21,7 +21,8 @@ option (gogoproto.equal_all) = true; // See the :ref:`architecture overview ` for // more information on outlier detection. message OutlierDetection { - // The number of consecutive 5xx responses before a consecutive 5xx ejection + // The number of consecutive 5xx responses or local origin errors that are mapped + // to 5xx error codes before a consecutive 5xx ejection // occurs. Defaults to 5. google.protobuf.UInt32Value consecutive_5xx = 1; @@ -71,9 +72,8 @@ message OutlierDetection { // be 1900. Defaults to 1900. google.protobuf.UInt32Value success_rate_stdev_factor = 9; - // The number of consecutive gateway failures (502, 503, 504 status or - // connection errors that are mapped to one of those status codes) before a - // consecutive gateway failure ejection occurs. Defaults to 5. + // The number of consecutive gateway failures (502, 503, 504 status codes) + // before a consecutive gateway failure ejection occurs. Defaults to 5. google.protobuf.UInt32Value consecutive_gateway_failure = 10; // The % chance that a host will be actually ejected when an outlier status @@ -81,4 +81,37 @@ message OutlierDetection { // used to disable ejection or to ramp it up slowly. Defaults to 0. google.protobuf.UInt32Value enforcing_consecutive_gateway_failure = 11 [(validate.rules).uint32.lte = 100]; + + // Determines whether to distinguish local origin failures from external errors. If set to true + // the following configuration parameters are taken into account: + // :ref:`consecutive_local_origin_failure`, + // :ref:`enforcing_consecutive_local_origin_failure` + // and + // :ref:`enforcing_local_origin_success_rate`. + // Defaults to false. + bool split_external_local_origin_errors = 12; + + // The number of consecutive locally originated failures before ejection + // occurs. Defaults to 5. Parameter takes effect only when + // :ref:`split_external_local_origin_errors` + // is set to true. + google.protobuf.UInt32Value consecutive_local_origin_failure = 13; + + // The % chance that a host will be actually ejected when an outlier status + // is detected through consecutive locally originated failures. This setting can be + // used to disable ejection or to ramp it up slowly. Defaults to 100. + // Parameter takes effect only when + // :ref:`split_external_local_origin_errors` + // is set to true. + google.protobuf.UInt32Value enforcing_consecutive_local_origin_failure = 14 + [(validate.rules).uint32.lte = 100]; + + // The % chance that a host will be actually ejected when an outlier status + // is detected through success rate statistics for locally originated errors. + // This setting can be used to disable ejection or to ramp it up slowly. Defaults to 100. + // Parameter takes effect only when + // :ref:`split_external_local_origin_errors` + // is set to true. + google.protobuf.UInt32Value enforcing_local_origin_success_rate = 15 + [(validate.rules).uint32.lte = 100]; } diff --git a/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto b/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto index ba3f0289942f..65559abc5485 100644 --- a/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto +++ b/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto @@ -47,12 +47,37 @@ message OutlierDetectionEvent { // Type of ejection that took place enum OutlierEjectionType { - // In case upstream host returns certain number of consecutive 5xx + // In case upstream host returns certain number of consecutive 5xx. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *false*, all type of errors are treated as HTTP 5xx errors. + // See :ref:`Cluster outlier detection ` documentation for + // details. CONSECUTIVE_5XX = 0; // In case upstream host returns certain number of consecutive gateway errors CONSECUTIVE_GATEWAY_FAILURE = 1; // Runs over aggregated success rate statistics from every host in cluster + // and selects hosts for which ratio of successful replies deviates from other hosts + // in the cluster. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *false*, all errors (externally and locally generated) are used to calculate success rate + // statistics. See :ref:`Cluster outlier detection ` + // documentation for details. SUCCESS_RATE = 2; + // Consecutive local origin failures: Connection failures, resets, timeouts, etc + // This type of ejection happens only when + // :ref:`outlier_detection.split_external_local_origin_errors` + // is set to *true*. + // See :ref:`Cluster outlier detection ` documentation for + CONSECUTIVE_LOCAL_ORIGIN_FAILURE = 3; + // Runs over aggregated success rate statistics for local origin failures + // for all hosts in the cluster and selects hosts for which success rate deviates from other + // hosts in the cluster. This type of ejection happens only when + // :ref:`outlier_detection.split_external_local_origin_errors` + // is set to *true*. + // See :ref:`Cluster outlier detection ` documentation for + SUCCESS_RATE_LOCAL_ORIGIN = 4; } // Represents possible action applied to upstream host @@ -74,4 +99,4 @@ message OutlierEjectSuccessRate { } message OutlierEjectConsecutive { -} \ No newline at end of file +} diff --git a/docs/root/configuration/cluster_manager/cluster_runtime.rst b/docs/root/configuration/cluster_manager/cluster_runtime.rst index 2ac83ab011ec..a64750cf6625 100644 --- a/docs/root/configuration/cluster_manager/cluster_runtime.rst +++ b/docs/root/configuration/cluster_manager/cluster_runtime.rst @@ -42,6 +42,11 @@ outlier_detection.consecutive_gateway_failure ` setting in outlier detection +outlier_detection.consecutive_local_origin_failure + :ref:`consecutive_local_origin_failure + ` + setting in outlier detection + outlier_detection.interval_ms :ref:`interval_ms ` @@ -67,11 +72,21 @@ outlier_detection.enforcing_consecutive_gateway_failure ` setting in outlier detection +outlier_detection.enforcing_consecutive_local_origin_failure + :ref:`enforcing_consecutive_local_origin_failure + ` + setting in outlier detection + outlier_detection.enforcing_success_rate :ref:`enforcing_success_rate ` setting in outlier detection +outlier_detection.enforcing_local_origin_success_rate + :ref:`enforcing_local_origin_success_rate + ` + setting in outlier detection + outlier_detection.success_rate_minimum_hosts :ref:`success_rate_minimum_hosts ` diff --git a/docs/root/configuration/cluster_manager/cluster_stats.rst b/docs/root/configuration/cluster_manager/cluster_stats.rst index b5b6554be7b6..f4f1890baf8d 100644 --- a/docs/root/configuration/cluster_manager/cluster_stats.rst +++ b/docs/root/configuration/cluster_manager/cluster_stats.rst @@ -133,10 +133,14 @@ statistics will be rooted at *cluster..outlier_detection.* and contain the ejections_overflow, Counter, Number of ejections aborted due to the max ejection % ejections_enforced_consecutive_5xx, Counter, Number of enforced consecutive 5xx ejections ejections_detected_consecutive_5xx, Counter, Number of detected consecutive 5xx ejections (even if unenforced) - ejections_enforced_success_rate, Counter, Number of enforced success rate outlier ejections - ejections_detected_success_rate, Counter, Number of detected success rate outlier ejections (even if unenforced) + ejections_enforced_success_rate, Counter, Number of enforced success rate outlier ejections. Exact meaning of this counter depends on :ref:`outlier_detection.split_external_local_origin_errors` config item. Refer to :ref:`Outlier Detection documentation` for details. + ejections_detected_success_rate, Counter, Number of detected success rate outlier ejections (even if unenforced). Exact meaning of this counter depends on :ref:`outlier_detection.split_external_local_origin_errors` config item. Refer to :ref:`Outlier Detection documentation` for details. ejections_enforced_consecutive_gateway_failure, Counter, Number of enforced consecutive gateway failure ejections ejections_detected_consecutive_gateway_failure, Counter, Number of detected consecutive gateway failure ejections (even if unenforced) + ejections_enforced_consecutive_local_origin_failure, Counter, Number of enforced consecutive local origin failure ejections + ejections_detected_consecutive_local_origin_failure, Counter, Number of detected consecutive local origin failure ejections (even if unenforced) + ejections_enforced_local_origin_success_rate, Counter, Number of enforced success rate outlier ejections for locally originated failures + ejections_detected_local_origin_success_rate, Counter, Number of detected success rate outlier ejections for locally originated failures (even if unenforced) ejections_total, Counter, Deprecated. Number of ejections due to any outlier type (even if unenforced) ejections_consecutive_5xx, Counter, Deprecated. Number of consecutive 5xx ejections (even if unenforced) diff --git a/docs/root/intro/arch_overview/upstream/outlier.rst b/docs/root/intro/arch_overview/upstream/outlier.rst index ad4bc1a410e0..b8b4fce4b7a1 100644 --- a/docs/root/intro/arch_overview/upstream/outlier.rst +++ b/docs/root/intro/arch_overview/upstream/outlier.rst @@ -10,6 +10,43 @@ such as consecutive failures, temporal success rate, temporal latency, etc. Outl form of *passive* health checking. Envoy also supports :ref:`active health checking `. *Passive* and *active* health checking can be enabled together or independently, and form the basis for an overall upstream health checking solution. +Outlier detection is part of :ref:`cluster configuration ` +and it needs filters to report errors, timeouts, resets. Currently the following filters support +outlier detection: :ref:`http router `, +:ref:`tcp proxy ` and :ref:`redis proxy `. + +Detected errors fall into two categories: externally and locally originated errors. Externally generated errors +are transaction specific and occur on the upstream server in response to the received request. For example, HTTP server returning error code 500 or redis server returning payload which cannot be decoded. Those errors are generated on the upstream host after Envoy has successfully connected to it. +Locally originated errors are generated by Envoy in response to an event which interrupted or prevented communication with the upstream host. Examples of locally originated errors are timeout, TCP reset, inability to connect to a specified port, etc. + +Type of detected errors depends on filter type. :ref:`http router ` filter for example +detects locally originated errors (timeouts, resets - errors related to connection to upstream host) and because it +also understands HTTP protocol it reports +errors returned by HTTP server (externally generated errors). In such scenario, even when connection to upstream HTTP server is successful, +transaction with the server may fail. +On the contrary, :ref:`tcp proxy ` filter does not understand any protocol above +TCP layer and reports only locally originated errors. + +In default configuration (:ref:`outlier_detection.split_external_local_origin_errors` is *false*) +locally originated errors are not distinguished from externally generated (transaction) errors and all end up +in the same bucket and are compared against +:ref:`outlier_detection.consecutive_5xx`, +:ref:`outlier_detection.consecutive_gateway_failure` and +:ref:`outlier_detection.success_rate_stdev_factor` +configuration items. For example, if connection to an upstream HTTP server fails twice because of timeout and +then, after successful connection, the server returns error code 500, the total error count will be 3. + +Outlier detection may also be configured to distinguish locally originated errors from externally originated (transaction) errors. +It is done via +:ref:`outlier_detection.split_external_local_origin_errors` configuration item. +In that mode locally originated errors are tracked by separate counters than externally originated +(transaction) errors and +the outlier detector may be configured to react to locally originated errors and ignore externally originated errors +or vice-versa. + +It is important to understand that a cluster may be shared among several filter chains. If one filter chain +ejects a host based on its outlier detection type, other filter chains will be also affected even though their +outlier detection type would not eject that host. Ejection algorithm ------------------ @@ -42,22 +79,47 @@ Envoy supports the following outlier detection types: Consecutive 5xx ^^^^^^^^^^^^^^^ -If an upstream host returns some number of consecutive 5xx, it will be ejected. Note that in this -case a 5xx means an actual 5xx respond code, or an event that would cause the HTTP router to return -one on the upstream's behalf (reset, connection failure, etc.). The number of consecutive 5xx -required for ejection is controlled by the :ref:`outlier_detection.consecutive_5xx -` value. +In default mode (:ref:`outlier_detection.split_external_local_origin_errors` is *false*) this detection type takes into account all generated errors: locally +originated and externally originated (transaction) type of errors. +Errors generated by non-HTTP filters, like :ref:`tcp proxy ` or +:ref:`redis proxy ` are internally mapped to HTTP 5xx codes and treated as such. + +In split mode (:ref:`outlier_detection.split_external_local_origin_errors` is *true*) this detection type takes into account only externally originated (transaction) errors ignoring locally originated errors. +If an upstream host is HTTP-server, only 5xx types of error are taken into account (see :ref:`Consecutive Gateway Failure` for exceptions). +For redis servers, served via +:ref:`redis proxy ` only malformed responses from the server are taken into account. +Properly formatted responses, even when they carry operational error (like index not found, access denied) are not taken into account. + +If an upstream host returns some number of errors which are treated as consecutive 5xx type errors, it will be ejected. +The number of consecutive 5xx required for ejection is controlled by +the :ref:`outlier_detection.consecutive_5xx` value. + +.. _consecutive_gateway_failure: Consecutive Gateway Failure ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +This detection type takes into account subset of 5xx errors, called "gateway errors" (502, 503 or 504 status code) +and is supported only by :ref:`http router `. + If an upstream host returns some number of consecutive "gateway errors" (502, 503 or 504 status -code), it will be ejected. Note that this includes events that would cause the HTTP router to -return one of these status codes on the upstream's behalf (reset, connection failure, etc.). The -number of consecutive gateway failures required for ejection is controlled by +code), it will be ejected. +The number of consecutive gateway failures required for ejection is controlled by the :ref:`outlier_detection.consecutive_gateway_failure ` value. +Consecutive Local Origin Failure +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This detection type is enabled only when :ref:`outlier_detection.split_external_local_origin_errors` is *true* and takes into account only locally originated errors (timeout, reset, etc). +If Envoy repeatedly cannot connect to an upstream host or communication with the upstream host is repeatedly interrupted, it will be ejected. +Various locally originated problems are detected: timeout, TCP reset, ICMP errors, etc. The number of consecutive +locally originated failures required for ejection is controlled +by the :ref:`outlier_detection.consecutive_local_origin_failure +` value. +This detection type is supported by :ref:`http router `, +:ref:`tcp proxy ` and :ref:`redis proxy `. + Success Rate ^^^^^^^^^^^^ @@ -68,7 +130,21 @@ calculated for a host if its request volume over the aggregation interval is les value. Moreover, detection will not be performed for a cluster if the number of hosts with the minimum required request volume in an interval is less than the :ref:`outlier_detection.success_rate_minimum_hosts` -value. +value. + +In default configuration mode (:ref:`outlier_detection.split_external_local_origin_errors` is *false*) +this detection type takes into account all type of errors: locally and externally originated. +:ref:`outlier_detection.enforcing_local_origin_success` config item is ignored. + +In split mode (:ref:`outlier_detection.split_external_local_origin_errors` is *true*), +locally originated errors and externally originated (transaction) errors are counted and treated separately. +Most configuration items, namely +:ref:`outlier_detection.success_rate_minimum_hosts`, +:ref:`outlier_detection.success_rate_request_volume`, +:ref:`outlier_detection.success_rate_stdev_factor` apply to both +types of errors, but :ref:`outlier_detection.enforcing_success_rate` applies +to externally originated errors only and :ref:`outlier_detection.enforcing_local_origin_success_rate` applies to locally originated errors only. + .. _arch_overview_outlier_detection_logging: diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b427780db79e..453ea130d782 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -36,6 +36,7 @@ Version history * http: changed `sendLocalReply` to send percent-encoded `GrpcMessage`. * http: added :ref:`dynamic forward proxy ` support. * jwt_authn: make filter's parsing of JWT more flexible, allowing syntax like ``jwt=eyJhbGciOiJS...ZFnFIw,extra=7,realm=123`` +* outlier_detector: added configuration :ref:`outlier_detection.split_external_local_origin_errors` to distinguish locally and externally generated errors. See :ref:`arch_overview_outlier_detection` for full details. * listener: added :ref:`source IP ` and :ref:`source port ` filter chain matching. diff --git a/include/envoy/upstream/outlier_detection.h b/include/envoy/upstream/outlier_detection.h index d8ffa2badc0a..fdf93ffae39a 100644 --- a/include/envoy/upstream/outlier_detection.h +++ b/include/envoy/upstream/outlier_detection.h @@ -26,15 +26,25 @@ namespace Outlier { * Non-HTTP result of requests/operations. */ enum class Result { - SUCCESS, // Successfully established a connection or completed a request. - TIMEOUT, // Timed out while connecting or executing a request. - CONNECT_FAILED, // Remote host rejected the connection. + // Local origin errors detected by Envoy. + LOCAL_ORIGIN_TIMEOUT, // Timed out while connecting or executing a request. + LOCAL_ORIGIN_CONNECT_FAILED, // Remote host rejected the connection. + LOCAL_ORIGIN_CONNECT_SUCCESS, // Successfully established a connection to upstream host. + // Use this code when there is another protocol on top of + // transport protocol. For example HTTP runs on top of tcp. + // The same for redis. It first establishes TCP and then runs + // a transaction. + LOCAL_ORIGIN_CONNECT_SUCCESS_FINAL, // Successfully established a connection to upstream host + // Use this code when there is no other protocol on top of the + // protocol used by a filter. For example tcp_proxy filter + // serves only tcp level. There is no other protocol on top of + // tcp which the tcp_proxy filter is aware of. // The entries below only make sense when Envoy understands requests/responses for the // protocol being proxied. They do not make sense for TcpProxy, for example. - - REQUEST_FAILED, // Request was not completed successfully. - SERVER_FAILURE, // The server indicated it cannot process a request. + // External origin errors. + EXT_ORIGIN_REQUEST_FAILED, // The server indicated it cannot process a request + EXT_ORIGIN_REQUEST_SUCCESS // Request was completed successfully. }; /** @@ -42,6 +52,9 @@ enum class Result { */ class DetectorHostMonitor { public: + // Types of Success Rate monitors. + enum class SuccessRateMonitorType { ExternalOrigin, LocalOrigin }; + virtual ~DetectorHostMonitor() = default; /** @@ -56,8 +69,16 @@ class DetectorHostMonitor { /** * Add a non-HTTP result for a host. + * Some non-HTTP codes like TIMEOUT may require special mapping to HTTP code + * and such code may be passed as optional parameter. + */ + virtual void putResult(Result result, absl::optional code) PURE; + + /** + * Wrapper around putResult with 2 params when mapping to HTTP code is not + * required. */ - virtual void putResult(Result result) PURE; + void putResult(Result result) { putResult(result, absl::nullopt); } /** * Add a response time for a host (in this case response time is generic and might be used for @@ -81,8 +102,14 @@ class DetectorHostMonitor { * @return the success rate of the host in the last calculated interval, in the range 0-100. * -1 means that the host did not have enough request volume to calculate success rate * or the cluster did not have enough hosts to run through success rate outlier ejection. + * @param type specifies for which Success Rate Monitor the success rate value should be returned. + * If the outlier detector is configured not to split external and local origin errors, + * ExternalOrigin type returns success rate for all types of errors: external and local + * origin and LocalOrigin type returns -1. If the outlier detector is configured to split external + * and local origin errors, ExternalOrigin type returns success rate for external origin errors + * and LocalOrigin type returns success rate for local origin errors. */ - virtual double successRate() const PURE; + virtual double successRate(SuccessRateMonitorType type) const PURE; }; using DetectorHostMonitorPtr = std::unique_ptr; @@ -111,8 +138,9 @@ class Detector { * interval. * @return the average success rate, or -1 if there were not enough hosts with enough request * volume to proceed with success rate based outlier ejection. + * @param type - see DetectorHostMonitor::successRate. */ - virtual double successRateAverage() const PURE; + virtual double successRateAverage(DetectorHostMonitor::SuccessRateMonitorType) const PURE; /** * Returns the success rate threshold used in the last interval. The threshold is used to eject @@ -120,13 +148,12 @@ class Detector { * @return the threshold, or -1 if there were not enough hosts with enough request volume to * proceed with success rate based outlier ejection. */ - virtual double successRateEjectionThreshold() const PURE; + virtual double + successRateEjectionThreshold(DetectorHostMonitor::SuccessRateMonitorType) const PURE; }; using DetectorSharedPtr = std::shared_ptr; -enum class EjectionType { Consecutive5xx, SuccessRate, ConsecutiveGatewayFailure }; - /** * Sink for outlier detection event logs. */ diff --git a/source/common/router/router.cc b/source/common/router/router.cc index dbdf35a19708..7503d1e660a5 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -665,7 +665,8 @@ void Filter::onResponseTimeout() { // If this upstream request already hit a "soft" timeout, then it // already recorded a timeout into outlier detection. Don't do it again. if (!upstream_request->outlier_detection_timeout_recorded_) { - updateOutlierDetection(timeout_response_code_, *upstream_request); + updateOutlierDetection(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, *upstream_request, + absl::optional(enumToInt(timeout_response_code_))); } chargeUpstreamAbort(timeout_response_code_, false, *upstream_request); @@ -682,7 +683,8 @@ void Filter::onResponseTimeout() { void Filter::onSoftPerTryTimeout(UpstreamRequest& upstream_request) { // Track this as a timeout for outlier detection purposes even though we didn't // cancel the request yet and might get a 2xx later. - updateOutlierDetection(timeout_response_code_, upstream_request); + updateOutlierDetection(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, upstream_request, + absl::optional(enumToInt(timeout_response_code_))); upstream_request.outlier_detection_timeout_recorded_ = true; if (!downstream_response_started_ && retry_state_) { @@ -719,7 +721,8 @@ void Filter::onPerTryTimeout(UpstreamRequest& upstream_request) { upstream_request.resetStream(); - updateOutlierDetection(timeout_response_code_, upstream_request); + updateOutlierDetection(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, upstream_request, + absl::optional(enumToInt(timeout_response_code_))); if (maybeRetryReset(Http::StreamResetReason::LocalReset, upstream_request)) { return; @@ -733,9 +736,11 @@ void Filter::onPerTryTimeout(UpstreamRequest& upstream_request) { StreamInfo::ResponseCodeDetails::get().UpstreamPerTryTimeout); } -void Filter::updateOutlierDetection(Http::Code code, UpstreamRequest& upstream_request) { +void Filter::updateOutlierDetection(Upstream::Outlier::Result result, + UpstreamRequest& upstream_request, + absl::optional code) { if (upstream_request.upstream_host_) { - upstream_request.upstream_host_->outlierDetector().putHttpResponseCode(enumToInt(code)); + upstream_request.upstream_host_->outlierDetector().putResult(result, code); } } @@ -825,7 +830,12 @@ void Filter::onUpstreamReset(Http::StreamResetReason reset_reason, ENVOY_STREAM_LOG(debug, "upstream reset: reset reason {}", *callbacks_, Http::Utility::resetReasonToString(reset_reason)); - updateOutlierDetection(Http::Code::ServiceUnavailable, upstream_request); + // TODO: The reset may also come from upstream over the wire. In this case it should be + // treated as external origin error and distinguished from local origin error. + // This matters only when running OutlierDetection with split_external_local_origin_errors config + // param set to true. + updateOutlierDetection(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, upstream_request, + absl::nullopt); if (maybeRetryReset(reset_reason, upstream_request)) { return; @@ -1455,6 +1465,8 @@ void Filter::UpstreamRequest::onPoolReady(Http::StreamEncoder& request_encoder, Upstream::HostDescriptionConstSharedPtr host) { ENVOY_STREAM_LOG(debug, "pool ready", *parent_.callbacks_); + host->outlierDetector().putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS); + // TODO(ggreenway): set upstream local address in the StreamInfo. onUpstreamHostSelected(host); request_encoder.getStream().addCallbacks(*this); diff --git a/source/common/router/router.h b/source/common/router/router.h index 0fc6f816332e..b50c94aeecb3 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -467,7 +467,8 @@ class Filter : Logger::Loggable, void sendNoHealthyUpstreamResponse(); bool setupRetry(); bool setupRedirect(const Http::HeaderMap& headers, UpstreamRequest& upstream_request); - void updateOutlierDetection(Http::Code code, UpstreamRequest& upstream_request); + void updateOutlierDetection(Upstream::Outlier::Result result, UpstreamRequest& upstream_request, + absl::optional code); void doRetry(); // Called immediately after a non-5xx header is received from upstream, performs stats accounting // and handle difference between gRPC and non-gRPC requests. diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index 07758e5982d7..ce466c2512c6 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -14,6 +14,7 @@ #include "common/access_log/access_log_impl.h" #include "common/common/assert.h" #include "common/common/empty_string.h" +#include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/macros.h" #include "common/common/utility.h" @@ -447,7 +448,8 @@ void Filter::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr&& conn_data, void Filter::onConnectTimeout() { ENVOY_CONN_LOG(debug, "connect timeout", read_callbacks_->connection()); - read_callbacks_->upstreamHost()->outlierDetector().putResult(Upstream::Outlier::Result::TIMEOUT); + read_callbacks_->upstreamHost()->outlierDetector().putResult( + Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT); getStreamInfo().setResponseFlag(StreamInfo::ResponseFlag::UpstreamConnectionFailure); // Raise LocalClose, which will trigger a reconnect if needed/configured. @@ -520,7 +522,7 @@ void Filter::onUpstreamEvent(Network::ConnectionEvent event) { if (event == Network::ConnectionEvent::RemoteClose) { getStreamInfo().setResponseFlag(StreamInfo::ResponseFlag::UpstreamConnectionFailure); read_callbacks_->upstreamHost()->outlierDetector().putResult( - Upstream::Outlier::Result::CONNECT_FAILED); + Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED); } initializeUpstreamConnection(); @@ -535,7 +537,7 @@ void Filter::onUpstreamEvent(Network::ConnectionEvent event) { read_callbacks_->connection().readDisable(false); read_callbacks_->upstreamHost()->outlierDetector().putResult( - Upstream::Outlier::Result::SUCCESS); + Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS_FINAL); getStreamInfo().setRequestedServerName(read_callbacks_->connection().requestedServerName()); ENVOY_LOG(debug, "TCP:onUpstreamEvent(), requestedServerName: {}", diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index ea3dad58b5fb..3969ed5dc308 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -115,8 +115,7 @@ LoadBalancerBase::LoadBalancerBase(const PrioritySet& priority_set, ClusterStats // - normalized total health is = 100%. It means there are enough healthy hosts to handle the load. // Do not enter panic mode, even if a specific priority has low number of healthy hosts. // - normalized total health is < 100%. There are not enough healthy hosts to handle the load. -// Continue -// distributing the load among priority sets, but turn on panic mode for a given priority +// Continue distributing the load among priority sets, but turn on panic mode for a given priority // if # of healthy hosts in priority set is low. // - normalized total health is 0%. All hosts are down. Redirect 100% of traffic to P=0 and enable // panic mode. diff --git a/source/common/upstream/outlier_detection_impl.cc b/source/common/upstream/outlier_detection_impl.cc index f16ab7d95b81..812d72ab1575 100644 --- a/source/common/upstream/outlier_detection_impl.cc +++ b/source/common/upstream/outlier_detection_impl.cc @@ -33,6 +33,20 @@ DetectorSharedPtr DetectorImplFactory::createForCluster( } } +DetectorHostMonitorImpl::DetectorHostMonitorImpl(std::shared_ptr detector, + HostSharedPtr host) + : detector_(detector), host_(host), + // add Success Rate monitors + external_origin_SR_monitor_(envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE), + local_origin_SR_monitor_( + envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN) { + // Setup method to call when putResult is invoked. Depending on the config's + // split_external_local_origin_errors_ boolean value different method is called. + put_result_func_ = detector->config().splitExternalLocalOriginErrors() + ? &DetectorHostMonitorImpl::putResultWithLocalExternalSplit + : &DetectorHostMonitorImpl::putResultNoLocalExternalSplit; +} + void DetectorHostMonitorImpl::eject(MonotonicTime ejection_time) { ASSERT(!host_.lock()->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); host_.lock()->healthFlagSet(Host::HealthFlag::FAILED_OUTLIER_CHECK); @@ -45,11 +59,12 @@ void DetectorHostMonitorImpl::uneject(MonotonicTime unejection_time) { } void DetectorHostMonitorImpl::updateCurrentSuccessRateBucket() { - success_rate_accumulator_bucket_.store(success_rate_accumulator_.updateCurrentWriter()); + external_origin_SR_monitor_.updateCurrentSuccessRateBucket(); + local_origin_SR_monitor_.updateCurrentSuccessRateBucket(); } void DetectorHostMonitorImpl::putHttpResponseCode(uint64_t response_code) { - success_rate_accumulator_bucket_.load()->total_request_counter_++; + external_origin_SR_monitor_.incTotalReqCounter(); if (Http::CodeUtility::is5xx(response_code)) { std::shared_ptr detector = detector_.lock(); if (!detector) { @@ -72,38 +87,123 @@ void DetectorHostMonitorImpl::putHttpResponseCode(uint64_t response_code) { detector->onConsecutive5xx(host_.lock()); } } else { - success_rate_accumulator_bucket_.load()->success_request_counter_++; + external_origin_SR_monitor_.incSuccessReqCounter(); consecutive_5xx_ = 0; consecutive_gateway_failure_ = 0; } } -Http::Code DetectorHostMonitorImpl::resultToHttpCode(Result result) { +absl::optional DetectorHostMonitorImpl::resultToHttpCode(Result result) { Http::Code http_code = Http::Code::InternalServerError; switch (result) { - case Result::SUCCESS: + case Result::EXT_ORIGIN_REQUEST_SUCCESS: + case Result::LOCAL_ORIGIN_CONNECT_SUCCESS_FINAL: http_code = Http::Code::OK; break; - case Result::TIMEOUT: + case Result::LOCAL_ORIGIN_TIMEOUT: http_code = Http::Code::GatewayTimeout; break; - case Result::CONNECT_FAILED: + case Result::LOCAL_ORIGIN_CONNECT_FAILED: http_code = Http::Code::ServiceUnavailable; break; - case Result::REQUEST_FAILED: + case Result::EXT_ORIGIN_REQUEST_FAILED: http_code = Http::Code::InternalServerError; break; - case Result::SERVER_FAILURE: - http_code = Http::Code::ServiceUnavailable; + // LOCAL_ORIGIN_CONNECT_SUCCESS is used is 2-layer protocols, like HTTP. + // First connection is established and then higher level protocol runs. + // If error happens in higher layer protocol, it will be mapped to + // HTTP code indicating error. In order not to intervene with result of + // higher layer protocol, this code is not mapped to HTTP code. + case Result::LOCAL_ORIGIN_CONNECT_SUCCESS: + return absl::nullopt; + } + + return absl::optional(http_code); +} + +// Method is called by putResult when external and local origin errors +// are not treated differently. All errors are mapped to HTTP codes. +// Depending on the value of the parameter *code* the function behaves differently: +// - if the *code* is not defined, mapping uses resultToHttpCode method to do mapping. +// - if *code* is defined, it is taken as HTTP code and reported as such to outlier detector. +void DetectorHostMonitorImpl::putResultNoLocalExternalSplit(Result result, + absl::optional code) { + if (code) { + putHttpResponseCode(code.value()); + } else { + absl::optional http_code = resultToHttpCode(result); + if (http_code) { + putHttpResponseCode(enumToInt(http_code.value())); + } + } +} + +// Method is called by putResult when external and local origin errors +// are treated separately. Local origin errors have separate counters and +// separate success rate monitor. +void DetectorHostMonitorImpl::putResultWithLocalExternalSplit(Result result, + absl::optional) { + switch (result) { + // SUCCESS is used to report success for connection level. Server may still respond with + // error, but connection to server was OK. + case Result::LOCAL_ORIGIN_CONNECT_SUCCESS: + case Result::LOCAL_ORIGIN_CONNECT_SUCCESS_FINAL: + return localOriginNoFailure(); + // Connectivity related errors. + case Result::LOCAL_ORIGIN_TIMEOUT: + case Result::LOCAL_ORIGIN_CONNECT_FAILED: + return localOriginFailure(); + // EXT_ORIGIN_REQUEST_FAILED is used when connection to server was successful, but transaction on + // server level failed. Since it it similar to HTTP 5xx, map it to 5xx handler. + case Result::EXT_ORIGIN_REQUEST_FAILED: + // map it to http code and call http handler. + return putHttpResponseCode(enumToInt(Http::Code::ServiceUnavailable)); + // EXT_ORIGIN_REQUEST_SUCCESS is used to report that transaction with non-http server was + // completed successfully. This means that connection and server level transactions were + // successful. Map it to http code 200 OK and indicate that there was no errors on connection + // level. + case Result::EXT_ORIGIN_REQUEST_SUCCESS: + putHttpResponseCode(enumToInt(Http::Code::OK)); + localOriginNoFailure(); break; } +} - return http_code; +// Method is used by other components to reports success or error. +// It calls putResultWithLocalExternalSplit or put putResultNoLocalExternalSplit via +// std::function. The setting happens in constructor based on split_external_local_origin_errors +// config parameter. +void DetectorHostMonitorImpl::putResult(Result result, absl::optional code) { + put_result_func_(this, result, code); } -void DetectorHostMonitorImpl::putResult(Result result) { - putHttpResponseCode(enumToInt(resultToHttpCode(result))); +void DetectorHostMonitorImpl::localOriginFailure() { + std::shared_ptr detector = detector_.lock(); + if (!detector) { + // It's possible for the cluster/detector to go away while we still have a host in use. + return; + } + local_origin_SR_monitor_.incTotalReqCounter(); + if (++consecutive_local_origin_failure_ == + detector->runtime().snapshot().getInteger( + "outlier_detection.consecutive_local_origin_failure", + detector->config().consecutiveLocalOriginFailure())) { + detector->onConsecutiveLocalOriginFailure(host_.lock()); + } +} + +void DetectorHostMonitorImpl::localOriginNoFailure() { + std::shared_ptr detector = detector_.lock(); + if (!detector) { + // It's possible for the cluster/detector to go away while we still have a host in use. + return; + } + + local_origin_SR_monitor_.incTotalReqCounter(); + local_origin_SR_monitor_.incSuccessReqCounter(); + + resetConsecutiveLocalOriginFailure(); } DetectorConfig::DetectorConfig(const envoy::api::v2::cluster::OutlierDetection& config) @@ -127,7 +227,15 @@ DetectorConfig::DetectorConfig(const envoy::api::v2::cluster::OutlierDetection& enforcing_consecutive_gateway_failure_(static_cast( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_consecutive_gateway_failure, 0))), enforcing_success_rate_(static_cast( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_success_rate, 100))) {} + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_success_rate, 100))), + split_external_local_origin_errors_(config.split_external_local_origin_errors()), + consecutive_local_origin_failure_(static_cast( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, consecutive_local_origin_failure, 5))), + enforcing_consecutive_local_origin_failure_( + static_cast(PROTOBUF_GET_WRAPPED_OR_DEFAULT( + config, enforcing_consecutive_local_origin_failure, 100))), + enforcing_local_origin_success_rate_(static_cast( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_local_origin_success_rate, 100))) {} DetectorImpl::DetectorImpl(const Cluster& cluster, const envoy::api::v2::cluster::OutlierDetection& config, @@ -136,7 +244,10 @@ DetectorImpl::DetectorImpl(const Cluster& cluster, : config_(config), dispatcher_(dispatcher), runtime_(runtime), time_source_(time_source), stats_(generateStats(cluster.info()->statsScope())), interval_timer_(dispatcher.createTimer([this]() -> void { onIntervalTimer(); })), - event_logger_(event_logger), success_rate_average_(-1), success_rate_ejection_threshold_(-1) { + event_logger_(event_logger) { + // Insert success rate initial numbers for each type of SR detector + external_origin_SR_num_ = {-1, -1}; + local_origin_SR_num_ = {-1, -1}; } DetectorImpl::~DetectorImpl() { @@ -156,6 +267,7 @@ DetectorImpl::create(const Cluster& cluster, std::shared_ptr detector( new DetectorImpl(cluster, config, dispatcher, runtime, time_source, event_logger)); detector->initialize(cluster); + return detector; } @@ -235,6 +347,14 @@ bool DetectorImpl::enforceEjection(envoy::data::cluster::v2alpha::OutlierEjectio case envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE: return runtime_.snapshot().featureEnabled("outlier_detection.enforcing_success_rate", config_.enforcingSuccessRate()); + case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE: + return runtime_.snapshot().featureEnabled( + "outlier_detection.enforcing_consecutive_local_origin_failure", + config_.enforcingConsecutiveLocalOriginFailure()); + case envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN: + return runtime_.snapshot().featureEnabled( + "outlier_detection.enforcing_local_origin_success_rate", + config_.enforcingLocalOriginSuccessRate()); default: // Checked by schema. NOT_REACHED_GCOVR_EXCL_LINE; @@ -256,6 +376,36 @@ void DetectorImpl::updateEnforcedEjectionStats( case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE: stats_.ejections_enforced_consecutive_gateway_failure_.inc(); break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE: + stats_.ejections_enforced_consecutive_local_origin_failure_.inc(); + break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN: + stats_.ejections_enforced_local_origin_success_rate_.inc(); + break; + default: + // Checked by schema. + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +void DetectorImpl::updateDetectedEjectionStats( + envoy::data::cluster::v2alpha::OutlierEjectionType type) { + switch (type) { + case envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE: + stats_.ejections_detected_success_rate_.inc(); + break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX: + stats_.ejections_detected_consecutive_5xx_.inc(); + break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE: + stats_.ejections_detected_consecutive_gateway_failure_.inc(); + break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE: + stats_.ejections_detected_consecutive_local_origin_failure_.inc(); + break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN: + stats_.ejections_detected_local_origin_success_rate_.inc(); + break; default: // Checked by schema. NOT_REACHED_GCOVR_EXCL_LINE; @@ -331,6 +481,11 @@ void DetectorImpl::onConsecutiveGatewayFailure(HostSharedPtr host) { host, envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE); } +void DetectorImpl::onConsecutiveLocalOriginFailure(HostSharedPtr host) { + notifyMainThreadConsecutiveError( + host, envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE); +} + void DetectorImpl::onConsecutiveErrorWorker( HostSharedPtr host, envoy::data::cluster::v2alpha::OutlierEjectionType type) { // Ejections come in cross thread. There is a chance that the host has already been removed from @@ -344,26 +499,28 @@ void DetectorImpl::onConsecutiveErrorWorker( // We also reset the appropriate counter here to allow the monitor to detect a bout of consecutive // error responses even if the monitor is not charged with an interleaved non-error code. + updateDetectedEjectionStats(type); + ejectHost(host, type); + + // reset counters switch (type) { case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX: stats_.ejections_consecutive_5xx_.inc(); // Deprecated - stats_.ejections_detected_consecutive_5xx_.inc(); - ejectHost(host, envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX); host_monitors_[host]->resetConsecutive5xx(); break; case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE: - stats_.ejections_detected_consecutive_gateway_failure_.inc(); - ejectHost(host, - envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE); host_monitors_[host]->resetConsecutiveGatewayFailure(); break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE: + host_monitors_[host]->resetConsecutiveLocalOriginFailure(); + break; default: // Checked by schema. NOT_REACHED_GCOVR_EXCL_LINE; } } -Utility::EjectionPair Utility::successRateEjectionThreshold( +DetectorImpl::EjectionPair DetectorImpl::successRateEjectionThreshold( double success_rate_sum, const std::vector& valid_success_rate_hosts, double success_rate_stdev_factor) { // This function is using mean and standard deviation as statistical measures for outlier @@ -393,7 +550,8 @@ Utility::EjectionPair Utility::successRateEjectionThreshold( return {mean, (mean - (success_rate_stdev_factor * stdev))}; } -void DetectorImpl::processSuccessRateEjections() { +void DetectorImpl::processSuccessRateEjections( + DetectorHostMonitor::SuccessRateMonitorType monitor_type) { uint64_t success_rate_minimum_hosts = runtime_.snapshot().getInteger( "outlier_detection.success_rate_minimum_hosts", config_.successRateMinimumHosts()); uint64_t success_rate_request_volume = runtime_.snapshot().getInteger( @@ -402,8 +560,7 @@ void DetectorImpl::processSuccessRateEjections() { double success_rate_sum = 0; // Reset the Detector's success rate mean and stdev. - success_rate_average_ = -1; - success_rate_ejection_threshold_ = -1; + getSRNums(monitor_type) = {-1, -1}; // Exit early if there are not enough hosts. if (host_monitors_.size() < success_rate_minimum_hosts) { @@ -416,34 +573,37 @@ void DetectorImpl::processSuccessRateEjections() { for (const auto& host : host_monitors_) { // Don't do work if the host is already ejected. if (!host.first->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)) { - absl::optional host_success_rate = - host.second->successRateAccumulator().getSuccessRate(success_rate_request_volume); + absl::optional host_success_rate = host.second->getSRMonitor(monitor_type) + .successRateAccumulator() + .getSuccessRate(success_rate_request_volume); if (host_success_rate) { valid_success_rate_hosts.emplace_back( HostSuccessRatePair(host.first, host_success_rate.value())); success_rate_sum += host_success_rate.value(); - host.second->successRate(host_success_rate.value()); + host.second->successRate(monitor_type, host_success_rate.value()); } } } if (!valid_success_rate_hosts.empty() && valid_success_rate_hosts.size() >= success_rate_minimum_hosts) { - double success_rate_stdev_factor = + const double success_rate_stdev_factor = runtime_.snapshot().getInteger("outlier_detection.success_rate_stdev_factor", config_.successRateStdevFactor()) / 1000.0; - Utility::EjectionPair ejection_pair = Utility::successRateEjectionThreshold( + getSRNums(monitor_type) = successRateEjectionThreshold( success_rate_sum, valid_success_rate_hosts, success_rate_stdev_factor); - success_rate_average_ = ejection_pair.success_rate_average_; - success_rate_ejection_threshold_ = ejection_pair.ejection_threshold_; + const double success_rate_ejection_threshold = getSRNums(monitor_type).ejection_threshold_; for (const auto& host_success_rate_pair : valid_success_rate_hosts) { - if (host_success_rate_pair.success_rate_ < success_rate_ejection_threshold_) { + if (host_success_rate_pair.success_rate_ < success_rate_ejection_threshold) { stats_.ejections_success_rate_.inc(); // Deprecated. - stats_.ejections_detected_success_rate_.inc(); - ejectHost(host_success_rate_pair.host_, - envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE); + const envoy::data::cluster::v2alpha::OutlierEjectionType type = + host_monitors_[host_success_rate_pair.host_] + ->getSRMonitor(monitor_type) + .getEjectionType(); + updateDetectedEjectionStats(type); + ejectHost(host_success_rate_pair.host_, type); } } } @@ -459,10 +619,12 @@ void DetectorImpl::onIntervalTimer() { host.second->updateCurrentSuccessRateBucket(); // Refresh host success rate stat for the /clusters endpoint. If there is a new valid value, it // will get updated in processSuccessRateEjections(). - host.second->successRate(-1); + host.second->successRate(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin, -1); + host.second->successRate(DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin, -1); } - processSuccessRateEjections(); + processSuccessRateEjections(DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin); + processSuccessRateEjections(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin); armIntervalTimer(); } @@ -486,13 +648,18 @@ void EventLoggerImpl::logEject(const HostDescriptionConstSharedPtr& host, Detect event.set_enforced(enforced); - if (type == envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE) { + if ((type == envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE) || + (type == envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN)) { + const DetectorHostMonitor::SuccessRateMonitorType monitor_type = + (type == envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE) + ? DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin + : DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin; event.mutable_eject_success_rate_event()->set_cluster_average_success_rate( - detector.successRateAverage()); + detector.successRateAverage(monitor_type)); event.mutable_eject_success_rate_event()->set_cluster_success_rate_ejection_threshold( - detector.successRateEjectionThreshold()); + detector.successRateEjectionThreshold(monitor_type)); event.mutable_eject_success_rate_event()->set_host_success_rate( - host->outlierDetector().successRate()); + host->outlierDetector().successRate(monitor_type)); } else { event.mutable_eject_consecutive_event(); } diff --git a/source/common/upstream/outlier_detection_impl.h b/source/common/upstream/outlier_detection_impl.h index 4fc644d86648..3a03a7556c00 100644 --- a/source/common/upstream/outlier_detection_impl.h +++ b/source/common/upstream/outlier_detection_impl.h @@ -31,11 +31,11 @@ class DetectorHostMonitorNullImpl : public DetectorHostMonitor { // Upstream::Outlier::DetectorHostMonitor uint32_t numEjections() override { return 0; } void putHttpResponseCode(uint64_t) override {} - void putResult(Result) override {} + void putResult(Result, absl::optional) override {} void putResponseTime(std::chrono::milliseconds) override {} const absl::optional& lastEjectionTime() override { return time_; } const absl::optional& lastUnejectionTime() override { return time_; } - double successRate() const override { return -1; } + double successRate(SuccessRateMonitorType) const override { return -1; } private: const absl::optional time_; @@ -99,6 +99,35 @@ class SuccessRateAccumulator { std::unique_ptr backup_success_rate_bucket_; }; +class SuccessRateMonitor { +public: + SuccessRateMonitor(envoy::data::cluster::v2alpha::OutlierEjectionType ejection_type) + : ejection_type_(ejection_type), success_rate_(-1) { + // Point the success_rate_accumulator_bucket_ pointer to a bucket. + updateCurrentSuccessRateBucket(); + } + double getSuccessRate() const { return success_rate_; } + SuccessRateAccumulator& successRateAccumulator() { return success_rate_accumulator_; } + void setSuccessRate(double new_success_rate) { success_rate_ = new_success_rate; } + void updateCurrentSuccessRateBucket() { + success_rate_accumulator_bucket_.store(success_rate_accumulator_.updateCurrentWriter()); + } + void incTotalReqCounter() { success_rate_accumulator_bucket_.load()->total_request_counter_++; } + void incSuccessReqCounter() { + success_rate_accumulator_bucket_.load()->success_request_counter_++; + } + + envoy::data::cluster::v2alpha::OutlierEjectionType getEjectionType() const { + return ejection_type_; + } + +private: + SuccessRateAccumulator success_rate_accumulator_; + std::atomic success_rate_accumulator_bucket_; + envoy::data::cluster::v2alpha::OutlierEjectionType ejection_type_; + double success_rate_; +}; + class DetectorImpl; /** @@ -106,43 +135,75 @@ class DetectorImpl; */ class DetectorHostMonitorImpl : public DetectorHostMonitor { public: - DetectorHostMonitorImpl(std::shared_ptr detector, HostSharedPtr host) - : detector_(detector), host_(host), success_rate_(-1) { - // Point the success_rate_accumulator_bucket_ pointer to a bucket. - updateCurrentSuccessRateBucket(); - } + DetectorHostMonitorImpl(std::shared_ptr detector, HostSharedPtr host); void eject(MonotonicTime ejection_time); void uneject(MonotonicTime ejection_time); - void updateCurrentSuccessRateBucket(); - SuccessRateAccumulator& successRateAccumulator() { return success_rate_accumulator_; } - void successRate(double new_success_rate) { success_rate_ = new_success_rate; } + void resetConsecutive5xx() { consecutive_5xx_ = 0; } void resetConsecutiveGatewayFailure() { consecutive_gateway_failure_ = 0; } - static Http::Code resultToHttpCode(Result result); + void resetConsecutiveLocalOriginFailure() { consecutive_local_origin_failure_ = 0; } + static absl::optional resultToHttpCode(Result result); // Upstream::Outlier::DetectorHostMonitor uint32_t numEjections() override { return num_ejections_; } void putHttpResponseCode(uint64_t response_code) override; - void putResult(Result result) override; + void putResult(Result result, absl::optional code) override; void putResponseTime(std::chrono::milliseconds) override {} const absl::optional& lastEjectionTime() override { return last_ejection_time_; } const absl::optional& lastUnejectionTime() override { return last_unejection_time_; } - double successRate() const override { return success_rate_; } + + const SuccessRateMonitor& getSRMonitor(SuccessRateMonitorType type) const { + return (SuccessRateMonitorType::ExternalOrigin == type) ? external_origin_SR_monitor_ + : local_origin_SR_monitor_; + } + + SuccessRateMonitor& getSRMonitor(SuccessRateMonitorType type) { + // Call const version of the same method + return const_cast( + const_cast(this)->getSRMonitor(type)); + } + + double successRate(SuccessRateMonitorType type) const override { + return getSRMonitor(type).getSuccessRate(); + } + void updateCurrentSuccessRateBucket(); + void successRate(SuccessRateMonitorType type, double new_success_rate) { + getSRMonitor(type).setSuccessRate(new_success_rate); + } + + // handlers for reporting local origin errors + void localOriginFailure(); + void localOriginNoFailure(); private: std::weak_ptr detector_; std::weak_ptr host_; - std::atomic consecutive_5xx_{0}; - std::atomic consecutive_gateway_failure_{0}; absl::optional last_ejection_time_; absl::optional last_unejection_time_; uint32_t num_ejections_{}; - SuccessRateAccumulator success_rate_accumulator_; - std::atomic success_rate_accumulator_bucket_; - double success_rate_; + + // counters for externally generated failures + std::atomic consecutive_5xx_{0}; + std::atomic consecutive_gateway_failure_{0}; + + // counters for local origin failures + std::atomic consecutive_local_origin_failure_{0}; + + // success rate monitors: + // - external_origin: for all events when external/local are not split + // and for external origin failures when external/local events are split + // - local origin: for local events when external/local events are split and + // not used when external/local events are not split. + SuccessRateMonitor external_origin_SR_monitor_; + SuccessRateMonitor local_origin_SR_monitor_; + + void putResultNoLocalExternalSplit(Result result, absl::optional code); + void putResultWithLocalExternalSplit(Result result, absl::optional code); + std::function code)> + put_result_func_; }; /** @@ -156,6 +217,10 @@ class DetectorHostMonitorImpl : public DetectorHostMonitor { COUNTER(ejections_enforced_consecutive_5xx) \ COUNTER(ejections_enforced_consecutive_gateway_failure) \ COUNTER(ejections_enforced_success_rate) \ + COUNTER(ejections_detected_consecutive_local_origin_failure) \ + COUNTER(ejections_enforced_consecutive_local_origin_failure) \ + COUNTER(ejections_detected_local_origin_success_rate) \ + COUNTER(ejections_enforced_local_origin_success_rate) \ COUNTER(ejections_enforced_total) \ COUNTER(ejections_overflow) \ COUNTER(ejections_success_rate) \ @@ -176,17 +241,25 @@ class DetectorConfig { public: DetectorConfig(const envoy::api::v2::cluster::OutlierDetection& config); - uint64_t intervalMs() { return interval_ms_; } - uint64_t baseEjectionTimeMs() { return base_ejection_time_ms_; } - uint64_t consecutive5xx() { return consecutive_5xx_; } - uint64_t consecutiveGatewayFailure() { return consecutive_gateway_failure_; } - uint64_t maxEjectionPercent() { return max_ejection_percent_; } - uint64_t successRateMinimumHosts() { return success_rate_minimum_hosts_; } - uint64_t successRateRequestVolume() { return success_rate_request_volume_; } - uint64_t successRateStdevFactor() { return success_rate_stdev_factor_; } - uint64_t enforcingConsecutive5xx() { return enforcing_consecutive_5xx_; } - uint64_t enforcingConsecutiveGatewayFailure() { return enforcing_consecutive_gateway_failure_; } - uint64_t enforcingSuccessRate() { return enforcing_success_rate_; } + uint64_t intervalMs() const { return interval_ms_; } + uint64_t baseEjectionTimeMs() const { return base_ejection_time_ms_; } + uint64_t consecutive5xx() const { return consecutive_5xx_; } + uint64_t consecutiveGatewayFailure() const { return consecutive_gateway_failure_; } + uint64_t maxEjectionPercent() const { return max_ejection_percent_; } + uint64_t successRateMinimumHosts() const { return success_rate_minimum_hosts_; } + uint64_t successRateRequestVolume() const { return success_rate_request_volume_; } + uint64_t successRateStdevFactor() const { return success_rate_stdev_factor_; } + uint64_t enforcingConsecutive5xx() const { return enforcing_consecutive_5xx_; } + uint64_t enforcingConsecutiveGatewayFailure() const { + return enforcing_consecutive_gateway_failure_; + } + uint64_t enforcingSuccessRate() const { return enforcing_success_rate_; } + bool splitExternalLocalOriginErrors() const { return split_external_local_origin_errors_; } + uint64_t consecutiveLocalOriginFailure() const { return consecutive_local_origin_failure_; } + uint64_t enforcingConsecutiveLocalOriginFailure() const { + return enforcing_consecutive_local_origin_failure_; + } + uint64_t enforcingLocalOriginSuccessRate() const { return enforcing_local_origin_success_rate_; } private: const uint64_t interval_ms_; @@ -200,6 +273,10 @@ class DetectorConfig { const uint64_t enforcing_consecutive_5xx_; const uint64_t enforcing_consecutive_gateway_failure_; const uint64_t enforcing_success_rate_; + const bool split_external_local_origin_errors_; + const uint64_t consecutive_local_origin_failure_; + const uint64_t enforcing_consecutive_local_origin_failure_; + const uint64_t enforcing_local_origin_success_rate_; }; /** @@ -217,13 +294,38 @@ class DetectorImpl : public Detector, public std::enable_shared_from_this& valid_success_rate_hosts, + double success_rate_stdev_factor); private: DetectorImpl(const Cluster& cluster, const envoy::api::v2::cluster::OutlierDetection& config, @@ -244,7 +346,8 @@ class DetectorImpl : public Detector, public std::enable_shared_from_this callbacks_; std::unordered_map host_monitors_; EventLoggerSharedPtr event_logger_; - double success_rate_average_; - double success_rate_ejection_threshold_; + + // EjectionPair for external and local origin events. + // When external/local origin events are not split, external_origin_SR_num_ are used for + // both types of events: external and local. local_origin_SR_num_ is not used. + // When external/local origin events are split, external_origin_SR_num_ are used only + // for external events and local_origin_SR_num_ is used for local origin events. + EjectionPair external_origin_SR_num_; + EjectionPair local_origin_SR_num_; + + const EjectionPair& getSRNums(DetectorHostMonitor::SuccessRateMonitorType monitor_type) const { + return (DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin == monitor_type) + ? external_origin_SR_num_ + : local_origin_SR_num_; + } + EjectionPair& getSRNums(DetectorHostMonitor::SuccessRateMonitorType monitor_type) { + return const_cast( + static_cast(*this).getSRNums(monitor_type)); + } }; class EventLoggerImpl : public EventLogger { @@ -280,31 +399,6 @@ class EventLoggerImpl : public EventLogger { TimeSource& time_source_; }; -/** - * Utilities for Outlier Detection. - */ -class Utility { -public: - struct EjectionPair { - double success_rate_average_; - double ejection_threshold_; - }; - - /** - * This function returns an EjectionPair for success rate outlier detection. The pair contains - * the average success rate of all valid hosts in the cluster and the ejection threshold. - * If a host's success rate is under this threshold, the host is an outlier. - * @param success_rate_sum is the sum of the data in the success_rate_data vector. - * @param valid_success_rate_hosts is the vector containing the individual success rate data - * points. - * @return EjectionPair. - */ - static EjectionPair - successRateEjectionThreshold(double success_rate_sum, - const std::vector& valid_success_rate_hosts, - double success_rate_stdev_factor); -}; - } // namespace Outlier } // namespace Upstream } // namespace Envoy diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index e204301ce7e9..462a7ca2157e 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -92,7 +92,7 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca } void ClientImpl::onConnectOrOpTimeout() { - putOutlierEvent(Upstream::Outlier::Result::TIMEOUT); + putOutlierEvent(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT); if (connected_) { host_->cluster().stats().upstream_rq_timeout_.inc(); host_->stats().rq_timeout_.inc(); @@ -108,7 +108,7 @@ void ClientImpl::onData(Buffer::Instance& data) { try { decoder_->decode(data); } catch (ProtocolError&) { - putOutlierEvent(Upstream::Outlier::Result::REQUEST_FAILED); + putOutlierEvent(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_FAILED); host_->cluster().stats().upstream_cx_protocol_error_.inc(); host_->stats().rq_error_.inc(); connection_->close(Network::ConnectionCloseType::NoFlush); @@ -127,7 +127,7 @@ void ClientImpl::onEvent(Network::ConnectionEvent event) { if (!pending_requests_.empty()) { host_->cluster().stats().upstream_cx_destroy_with_active_rq_.inc(); if (event == Network::ConnectionEvent::RemoteClose) { - putOutlierEvent(Upstream::Outlier::Result::SERVER_FAILURE); + putOutlierEvent(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED); host_->cluster().stats().upstream_cx_destroy_remote_with_active_rq_.inc(); } if (event == Network::ConnectionEvent::LocalClose) { @@ -195,7 +195,7 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { connect_or_op_timer_->enableTimer(config_.opTimeout()); } - putOutlierEvent(Upstream::Outlier::Result::SUCCESS); + putOutlierEvent(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS); } ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks) diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index f3cafc6a1b89..1423a99c9068 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -311,10 +311,22 @@ void AdminImpl::addOutlierInfo(const std::string& cluster_name, const Upstream::Outlier::Detector* outlier_detector, Buffer::Instance& response) { if (outlier_detector) { - response.add(fmt::format("{}::outlier::success_rate_average::{}\n", cluster_name, - outlier_detector->successRateAverage())); - response.add(fmt::format("{}::outlier::success_rate_ejection_threshold::{}\n", cluster_name, - outlier_detector->successRateEjectionThreshold())); + response.add(fmt::format( + "{}::outlier::success_rate_average::{}\n", cluster_name, + outlier_detector->successRateAverage( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin))); + response.add(fmt::format( + "{}::outlier::success_rate_ejection_threshold::{}\n", cluster_name, + outlier_detector->successRateEjectionThreshold( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin))); + response.add(fmt::format( + "{}::outlier::local_origin_success_rate_average::{}\n", cluster_name, + outlier_detector->successRateAverage( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin))); + response.add(fmt::format( + "{}::outlier::local_origin_success_rate_ejection_threshold::{}\n", cluster_name, + outlier_detector->successRateEjectionThreshold( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin))); } } @@ -341,9 +353,19 @@ void AdminImpl::writeClustersAsJson(Buffer::Instance& response) { cluster_status.set_name(cluster_info->name()); const Upstream::Outlier::Detector* outlier_detector = cluster.outlierDetector(); - if (outlier_detector != nullptr && outlier_detector->successRateEjectionThreshold() > 0.0) { + if (outlier_detector != nullptr && + outlier_detector->successRateEjectionThreshold( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin) > 0.0) { cluster_status.mutable_success_rate_ejection_threshold()->set_value( - outlier_detector->successRateEjectionThreshold()); + outlier_detector->successRateEjectionThreshold( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + } + if (outlier_detector != nullptr && + outlier_detector->successRateEjectionThreshold( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin) > 0.0) { + cluster_status.mutable_local_origin_success_rate_ejection_threshold()->set_value( + outlier_detector->successRateEjectionThreshold( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); } cluster_status.set_added_via_api(cluster_info->addedViaApi()); @@ -396,7 +418,8 @@ void AdminImpl::writeClustersAsJson(Buffer::Instance& response) { HEALTH_FLAG_ENUM_VALUES(SET_HEALTH_FLAG) #undef SET_HEALTH_FLAG - double success_rate = host->outlierDetector().successRate(); + double success_rate = host->outlierDetector().successRate( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin); if (success_rate >= 0.0) { host_status.mutable_success_rate()->set_value(success_rate); } @@ -404,6 +427,11 @@ void AdminImpl::writeClustersAsJson(Buffer::Instance& response) { host_status.set_weight(host->weight()); host_status.set_priority(host->priority()); + success_rate = host->outlierDetector().successRate( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin); + if (success_rate >= 0.0) { + host_status.mutable_local_origin_success_rate()->set_value(success_rate); + } } } } @@ -456,11 +484,18 @@ void AdminImpl::writeClustersAsText(Buffer::Instance& response) { host->address()->asString(), host->locality().sub_zone())); response.add(fmt::format("{}::{}::canary::{}\n", cluster.second.get().info()->name(), host->address()->asString(), host->canary())); - response.add(fmt::format("{}::{}::success_rate::{}\n", cluster.second.get().info()->name(), - host->address()->asString(), - host->outlierDetector().successRate())); response.add(fmt::format("{}::{}::priority::{}\n", cluster.second.get().info()->name(), host->address()->asString(), host->priority())); + response.add(fmt::format( + "{}::{}::success_rate::{}\n", cluster.second.get().info()->name(), + host->address()->asString(), + host->outlierDetector().successRate( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin))); + response.add(fmt::format( + "{}::{}::local_origin_success_rate::{}\n", cluster.second.get().info()->name(), + host->address()->asString(), + host->outlierDetector().successRate( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin))); } } } diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 256bb7926fc3..d5b6c75701ed 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -1092,7 +1092,12 @@ TEST_F(RouterTest, ResetDuringEncodeHeaders) { Http::TestHeaderMapImpl headers; HttpTestUtility::addDefaultHeaders(headers); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + // First connection is successful and reset happens later on. + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); router_.decodeHeaders(headers, true); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); } @@ -1128,7 +1133,8 @@ TEST_F(RouterTest, UpstreamTimeout) { EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), false)); EXPECT_CALL(callbacks_, encodeData(_, true)); EXPECT_CALL(*router_.retry_state_, shouldRetryReset(_, _)).Times(0); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); response_timeout_->callback_(); EXPECT_EQ(1U, @@ -1259,7 +1265,8 @@ TEST_F(RouterTest, GrpcReset) { EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(200)); response_decoder->decodeHeaders(std::move(response_headers), false); EXPECT_TRUE(verifyHostUpstreamStats(0, 0)); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); EXPECT_EQ(1UL, stats_store_.counter("test.rq_reset_after_downstream_response_started").value()); @@ -1347,7 +1354,9 @@ TEST_F(RouterTest, UpstreamTimeoutWithAltResponse) { Http::TestHeaderMapImpl response_headers{{":status", "204"}}; EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); EXPECT_CALL(*router_.retry_state_, shouldRetryReset(_, _)).Times(0); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(204)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(204))); response_timeout_->callback_(); EXPECT_EQ(1U, @@ -1393,7 +1402,9 @@ TEST_F(RouterTest, UpstreamPerTryTimeout) { {":status", "504"}, {"content-length", "24"}, {"content-type", "text/plain"}}; EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), false)); EXPECT_CALL(callbacks_, encodeData(_, true)); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); per_try_timeout_->callback_(); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ @@ -1443,7 +1454,8 @@ TEST_F(RouterTest, UpstreamPerTryTimeoutDelayedPoolReady) { {":status", "504"}, {"content-length", "24"}, {"content-type", "text/plain"}}; EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), false)); EXPECT_CALL(callbacks_, encodeData(_, true)); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); per_try_timeout_->callback_(); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ @@ -1489,7 +1501,8 @@ TEST_F(RouterTest, UpstreamPerTryTimeoutExcludesNewStream) { pool_callbacks->onPoolReady(encoder, cm_.conn_pool_.host_); EXPECT_CALL(encoder.stream_, resetStream(Http::StreamResetReason::LocalReset)); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); EXPECT_CALL(*per_try_timeout_, disableTimer()); EXPECT_CALL(*response_timeout_, disableTimer()); EXPECT_CALL(callbacks_.stream_info_, @@ -1523,6 +1536,10 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); return nullptr; })); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(2); expectPerTryTimerCreate(); expectResponseTimerCreate(); @@ -1530,9 +1547,10 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { HttpTestUtility::addDefaultHeaders(headers); router_.decodeHeaders(headers, true); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); - NiceMock encoder2; Http::StreamDecoder* response_decoder2 = nullptr; router_.retry_state_->expectHedgedPerTryTimeoutRetry(); @@ -1596,6 +1614,11 @@ TEST_F(RouterTest, HedgedPerTryTimeoutThirdRequestSucceeds) { EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); Http::HeaderMapPtr response_headers1(new Http::TestHeaderMapImpl{{":status", "500"}}); + // Local origin connect success happens for first and third try. + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(2); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(500)); EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); @@ -1619,7 +1642,9 @@ TEST_F(RouterTest, HedgedPerTryTimeoutThirdRequestSucceeds) { // Now trigger a per try timeout on the 2nd request, expect a 3rd router_.retry_state_->expectHedgedPerTryTimeoutRetry(); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); NiceMock encoder3; Http::StreamDecoder* response_decoder3 = nullptr; EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) @@ -1672,6 +1697,10 @@ TEST_F(RouterTest, RetryOnlyOnceForSameUpstreamRequest) { callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); return nullptr; })); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(2); expectPerTryTimerCreate(); expectResponseTimerCreate(); @@ -1681,7 +1710,9 @@ TEST_F(RouterTest, RetryOnlyOnceForSameUpstreamRequest) { EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); per_try_timeout_->callback_(); @@ -1706,7 +1737,9 @@ TEST_F(RouterTest, RetryOnlyOnceForSameUpstreamRequest) { EXPECT_CALL(*router_.retry_state_, wouldRetryFromHeaders(_)).WillOnce(Return(true)); response_decoder1->decodeHeaders(std::move(response_headers1), true); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); response_timeout_->callback_(); } @@ -1728,6 +1761,10 @@ TEST_F(RouterTest, BadHeadersDroppedIfPreviousRetryScheduled) { callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); return nullptr; })); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(2); expectPerTryTimerCreate(); expectResponseTimerCreate(); @@ -1737,7 +1774,9 @@ TEST_F(RouterTest, BadHeadersDroppedIfPreviousRetryScheduled) { EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); per_try_timeout_->callback_(); @@ -1798,7 +1837,8 @@ TEST_F(RouterTest, RetryRequestNotComplete) { router_.decodeHeaders(headers, false); router_.retry_state_->expectResetRetry(); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); } @@ -1818,6 +1858,10 @@ TEST_F(RouterTest, HedgedPerTryTimeoutGlobalTimeout) { callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); return nullptr; })); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(2); expectPerTryTimerCreate(); expectResponseTimerCreate(); @@ -1825,7 +1869,9 @@ TEST_F(RouterTest, HedgedPerTryTimeoutGlobalTimeout) { HttpTestUtility::addDefaultHeaders(headers); router_.decodeHeaders(headers, true); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); @@ -1849,7 +1895,9 @@ TEST_F(RouterTest, HedgedPerTryTimeoutGlobalTimeout) { // Now trigger global timeout, expect everything to be reset EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(1); EXPECT_CALL(encoder2.stream_, resetStream(_)).Times(1); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) .WillOnce(Invoke([&](Http::HeaderMap& headers, bool) -> void { @@ -1876,6 +1924,10 @@ TEST_F(RouterTest, HedgingRetriesExhaustedBadResponse) { callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); return nullptr; })); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(1); expectPerTryTimerCreate(); expectResponseTimerCreate(); @@ -1883,7 +1935,9 @@ TEST_F(RouterTest, HedgingRetriesExhaustedBadResponse) { HttpTestUtility::addDefaultHeaders(headers); router_.decodeHeaders(headers, true); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); @@ -1899,6 +1953,10 @@ TEST_F(RouterTest, HedgingRetriesExhaustedBadResponse) { callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); return nullptr; })); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(1); expectPerTryTimerCreate(); router_.retry_state_->callback_(); @@ -1946,6 +2004,14 @@ TEST_F(RouterTest, HedgingRetriesProceedAfterReset) { callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); return nullptr; })); + // First is reset + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)) + .Times(1); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(2); expectPerTryTimerCreate(); expectResponseTimerCreate(); @@ -1953,7 +2019,9 @@ TEST_F(RouterTest, HedgingRetriesProceedAfterReset) { HttpTestUtility::addDefaultHeaders(headers); router_.decodeHeaders(headers, true); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); @@ -1975,7 +2043,6 @@ TEST_F(RouterTest, HedgingRetriesProceedAfterReset) { EXPECT_TRUE(verifyHostUpstreamStats(0, 0)); // Now trigger an upstream reset in response to the first request. - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); EXPECT_CALL(encoder1.stream_, resetStream(_)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); @@ -2016,6 +2083,10 @@ TEST_F(RouterTest, HedgingRetryImmediatelyReset) { callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); return nullptr; })); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(1); Http::TestHeaderMapImpl headers{{"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; HttpTestUtility::addDefaultHeaders(headers); @@ -2029,7 +2100,9 @@ TEST_F(RouterTest, HedgingRetryImmediatelyReset) { router_.retry_state_->expectHedgedPerTryTimeoutRetry(); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, router_.decodeData(*body_data, true)); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); EXPECT_CALL(encoder.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); per_try_timeout_->callback_(); @@ -2039,7 +2112,8 @@ TEST_F(RouterTest, HedgingRetryImmediatelyReset) { .WillOnce(Invoke([&](Http::StreamDecoder&, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); callbacks.onPoolFailure(Http::ConnectionPool::PoolFailureReason::ConnectionFailure, absl::string_view(), cm_.conn_pool_.host_); return nullptr; @@ -2089,7 +2163,8 @@ TEST_F(RouterTest, RetryNoneHealthy) { router_.decodeHeaders(headers, true); router_.retry_state_->expectResetRetry(); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); encoder1.stream_.resetStream(Http::StreamResetReason::LocalReset); EXPECT_CALL(cm_, httpConnPoolForCluster(_, _, _, _)).WillOnce(Return(nullptr)); @@ -2120,7 +2195,8 @@ TEST_F(RouterTest, RetryUpstreamReset) { router_.decodeHeaders(headers, true); router_.retry_state_->expectResetRetry(); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); // We expect this reset to kick off a new request. @@ -2129,6 +2205,9 @@ TEST_F(RouterTest, RetryUpstreamReset) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))); callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); return nullptr; })); @@ -2167,7 +2246,8 @@ TEST_F(RouterTest, RetryUpstreamPerTryTimeout) { router_.decodeHeaders(headers, true); router_.retry_state_->expectResetRetry(); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(504)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); per_try_timeout_->callback_(); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); @@ -2178,6 +2258,9 @@ TEST_F(RouterTest, RetryUpstreamPerTryTimeout) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))); callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); return nullptr; })); @@ -2294,7 +2377,8 @@ TEST_F(RouterTest, RetryUpstreamResetResponseStarted) { response_decoder->decodeHeaders(std::move(response_headers), false); absl::string_view rc_details2 = "upstream_reset_after_response_started{remote reset}"; EXPECT_CALL(callbacks_.stream_info_, setResponseCodeDetails(rc_details2)); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); // For normal HTTP, once we have a 200 we consider this a success, even if a // later reset occurs. @@ -2327,7 +2411,8 @@ TEST_F(RouterTest, RetryUpstreamReset100ContinueResponseStarted) { EXPECT_EQ( 1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_.counter("upstream_rq_100").value()); - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); } @@ -4006,7 +4091,8 @@ TEST_F(WatermarkTest, RetryRequestNotComplete) { router_.decodeData(data, false); // This should not trigger a retry as the retry state has been deleted. - EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); EXPECT_EQ(callbacks_.details_, "upstream_reset_before_response_started{remote reset}"); } diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index e07b2ef6d822..17e9a4c978d1 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -174,7 +174,7 @@ class RouterUpstreamLogTest : public testing::Test { router_->retry_state_->expectResetRetry(); EXPECT_CALL(context_.cluster_manager_.conn_pool_.host_->outlier_detector_, - putHttpResponseCode(504)); + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); per_try_timeout_->callback_(); // We expect this reset to kick off a new request. @@ -184,6 +184,8 @@ class RouterUpstreamLogTest : public testing::Test { [&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; + EXPECT_CALL(context_.cluster_manager_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, _)); callbacks.onPoolReady(encoder2, context_.cluster_manager_.conn_pool_.host_); return nullptr; })); diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index d07fb9bb1582..ec747aa3dca3 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -617,11 +617,11 @@ TEST_F(TcpProxyTest, ConnectAttemptsLimit) { setup(3, config); EXPECT_CALL(upstream_hosts_.at(0)->outlier_detector_, - putResult(Upstream::Outlier::Result::TIMEOUT)); + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); EXPECT_CALL(upstream_hosts_.at(1)->outlier_detector_, - putResult(Upstream::Outlier::Result::CONNECT_FAILED)); + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); EXPECT_CALL(upstream_hosts_.at(2)->outlier_detector_, - putResult(Upstream::Outlier::Result::CONNECT_FAILED)); + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); @@ -643,16 +643,16 @@ TEST_F(TcpProxyTest, OutlierDetection) { setup(3, config); EXPECT_CALL(upstream_hosts_.at(0)->outlier_detector_, - putResult(Upstream::Outlier::Result::TIMEOUT)); + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); raiseEventUpstreamConnectFailed(0, Tcp::ConnectionPool::PoolFailureReason::Timeout); EXPECT_CALL(upstream_hosts_.at(1)->outlier_detector_, - putResult(Upstream::Outlier::Result::CONNECT_FAILED)); + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); raiseEventUpstreamConnectFailed(1, Tcp::ConnectionPool::PoolFailureReason::RemoteConnectionFailure); EXPECT_CALL(upstream_hosts_.at(2)->outlier_detector_, - putResult(Upstream::Outlier::Result::SUCCESS)); + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS_FINAL, _)); raiseEventUpstreamConnected(2); } diff --git a/test/common/upstream/outlier_detection_impl_test.cc b/test/common/upstream/outlier_detection_impl_test.cc index f49b7c397146..bccf463c5e91 100644 --- a/test/common/upstream/outlier_detection_impl_test.cc +++ b/test/common/upstream/outlier_detection_impl_test.cc @@ -66,6 +66,16 @@ class OutlierDetectorImplTest : public testing::Test { .WillByDefault(Return(true)); ON_CALL(runtime_.snapshot_, featureEnabled("outlier_detection.enforcing_success_rate", 100)) .WillByDefault(Return(true)); + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_consecutive_local_origin_failure_", 100)) + .WillByDefault(Return(true)); + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_local_origin_success_rate", 100)) + .WillByDefault(Return(true)); + + // Prepare separate config with split_external_local_origin_errors set to true. + // It will be used for tests with split external and local origin errors. + outlier_detection_split_.set_split_external_local_origin_errors(true); } void addHosts(std::vector urls, bool primary = true) { @@ -75,9 +85,9 @@ class OutlierDetectorImplTest : public testing::Test { } } - void loadRq(HostVector& hosts, int num_rq, int http_code) { + template void loadRq(HostVector& hosts, int num_rq, T code) { for (uint64_t i = 0; i < hosts.size(); i++) { - loadRq(hosts[i], num_rq, http_code); + loadRq(hosts[i], num_rq, code); } } @@ -103,6 +113,7 @@ class OutlierDetectorImplTest : public testing::Test { Event::SimulatedTimeSystem time_system_; std::shared_ptr event_logger_{new MockEventLogger()}; envoy::api::v2::cluster::OutlierDetection empty_outlier_detection_; + envoy::api::v2::cluster::OutlierDetection outlier_detection_split_; Stats::Gauge& outlier_detection_ejections_active_; }; @@ -152,7 +163,7 @@ TEST_F(OutlierDetectorImplTest, DestroyWithActive) { cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); - loadRq(hosts_[0], 4, Result::REQUEST_FAILED); + loadRq(hosts_[0], 4, 500); time_system_.setMonotonicTime(std::chrono::milliseconds(0)); EXPECT_CALL(checker_, check(hosts_[0])); EXPECT_CALL(*event_logger_, @@ -189,7 +200,11 @@ TEST_F(OutlierDetectorImplTest, DestroyHostInUse) { loadRq(hosts_[0], 5, 500); } -TEST_F(OutlierDetectorImplTest, BasicFlow5xx) { +/* + Tests scenario when connect errors are reported by Non-http codes and success is reported by + http codes. (this happens in http router). +*/ +TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodes) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); @@ -200,7 +215,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xx) { addHosts({"tcp://127.0.0.1:81"}); cluster_.prioritySet().getMockHostSet(0)->runCallbacks({hosts_[1]}, {}); - // Cause a consecutive 5xx error. + // Cause a consecutive 5xx error on host[0] by reporting HTTP codes. loadRq(hosts_[0], 1, 500); loadRq(hosts_[0], 1, 200); hosts_[0]->outlierDetector().putResponseTime(std::chrono::milliseconds(5)); @@ -257,6 +272,138 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xx) { .value()); } +/* Test verifies the LOCAL_ORIGIN_CONNECT_SUCCESS with optional HTTP code 200, + cancels LOCAL_ORIGIN_CONNECT_FAILED event. +*/ +TEST_F(OutlierDetectorImplTest, ConnectSuccessWithOptionalHTTP_OK) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({"tcp://127.0.0.1:80"}); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + std::shared_ptr detector(DetectorImpl::create( + cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + // Make sure that in non-split mode LOCAL_ORIGIN_CONNECT_SUCCESS with optional HTTP code 200 + // cancels LOCAL_ORIGIN_CONNECT_FAILED. + // such scenario is used by tcp_proxy. + for (auto i = 0; i < 100; i++) { + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(enumToInt(Http::Code::OK))); + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_CONNECT_FAILED); + } + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); +} + +/* Test verifies the EXT_ORIGIN_REQUEST_SUCCESS cancels EXT_ORIGIN_REQUEST_FAILED event in non-split + * mode. + * EXT_ORIGIN_REQUEST_FAILED is mapped to 5xx code and EXT_ORIGIN_REQUEST_SUCCESS is mapped to 200 + * code. + */ +TEST_F(OutlierDetectorImplTest, ExternalOriginEventsNonSplit) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({"tcp://127.0.0.1:80"}); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + std::shared_ptr detector(DetectorImpl::create( + cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + // Make sure that EXT_ORIGIN_REQUEST_SUCCESS cancels EXT_ORIGIN_REQUEST_FAILED + // such scenario is used by redis filter. + for (auto i = 0; i < 100; i++) { + hosts_[0]->outlierDetector().putResult(Result::EXT_ORIGIN_REQUEST_FAILED); + hosts_[0]->outlierDetector().putResult(Result::EXT_ORIGIN_REQUEST_SUCCESS); + } + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + + // Now make sure that EXT_ORIGIN_REQUEST_FAILED ejects the host + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, true)); + for (auto i = 0; i < 100; i++) { + hosts_[0]->outlierDetector().putResult(Result::EXT_ORIGIN_REQUEST_FAILED); + } + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); +} + +TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaNonHttpCodes) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({"tcp://127.0.0.1:80"}); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + std::shared_ptr detector(DetectorImpl::create( + cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + addHosts({"tcp://127.0.0.1:81"}); + cluster_.prioritySet().getMockHostSet(0)->runCallbacks({hosts_[1]}, {}); + + // Cause a consecutive 5xx error on host[0] by reporting Non-HTTP codes. + loadRq(hosts_[0], 1, Result::LOCAL_ORIGIN_CONNECT_FAILED); + loadRq(hosts_[0], 1, 200); + hosts_[0]->outlierDetector().putResponseTime(std::chrono::milliseconds(5)); + loadRq(hosts_[0], 4, Result::LOCAL_ORIGIN_CONNECT_FAILED); + + time_system_.setMonotonicTime(std::chrono::milliseconds(0)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE, + false)); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, true)); + loadRq(hosts_[0], 1, Result::LOCAL_ORIGIN_CONNECT_FAILED); + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Interval that doesn't bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + interval_timer_->callback_(); + EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); + + // Interval that does bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(30001)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[0]))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + interval_timer_->callback_(); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); + + // Eject host again to cause an ejection after an unejection has taken place + hosts_[0]->outlierDetector().putResponseTime(std::chrono::milliseconds(5)); + loadRq(hosts_[0], 4, Result::LOCAL_ORIGIN_CONNECT_FAILED); + + time_system_.setMonotonicTime(std::chrono::milliseconds(40000)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE, + false)); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, true)); + loadRq(hosts_[0], 1, Result::LOCAL_ORIGIN_CONNECT_FAILED); + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + cluster_.prioritySet().getMockHostSet(0)->runCallbacks({}, hosts_); + + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + EXPECT_EQ(2UL, cluster_.info_->stats_store_.counter("outlier_detection.ejections_total").value()); + EXPECT_EQ( + 2UL, + cluster_.info_->stats_store_.counter("outlier_detection.ejections_consecutive_5xx").value()); + EXPECT_EQ(0UL, cluster_.info_->stats_store_ + .counter("outlier_detection.ejections_consecutive_gateway_failure") + .value()); +} + /** * Test that the consecutive gateway failure detector correctly fires, and also successfully * retriggers after uneject. This will also ensure that the stats counters end up with the expected @@ -356,6 +503,159 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailure) { .value()); } +/* + * Test passing of optional HTTP code with Result:: LOCAL_ORIGIN_TIMEOUT + */ +TEST_F(OutlierDetectorImplTest, TimeoutWithHttpCode) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({ + "tcp://127.0.0.1:80", + "tcp://127.0.0.1:81", + "tcp://127.0.0.1:84", + }); + + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + std::shared_ptr detector(DetectorImpl::create( + cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + // Report several LOCAL_ORIGIN_TIMEOUT with optional Http code 500. Host should be ejected. + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, true)); + // Get the configured number of failures and simulate than number of connect failures. + uint32_t n = runtime_.snapshot_.getInteger("outlier_detection.consecutive_5xx", + detector->config().consecutive5xx()); + while (n--) { + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_TIMEOUT, + absl::optional(500)); + } + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + + // Wait until it is unejected + time_system_.setMonotonicTime(std::chrono::milliseconds(50001)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[0]))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + interval_timer_->callback_(); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + + // Report several LOCAL_ORIGIN_TIMEOUT with HTTP code other that 500. Node should not be ejected. + EXPECT_CALL(checker_, check(hosts_[0])).Times(0); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, true)) + .Times(0); + // Get the configured number of failures and simulate than number of connect failures. + n = runtime_.snapshot_.getInteger("outlier_detection.consecutive_5xx", + detector->config().consecutive5xx()); + while (n--) { + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_TIMEOUT, + absl::optional(200)); + } + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + + // Report LOCAL_ORIGIN_TIMEOUT without explicit HTTP code mapping. It should be implicitly mapped + // to 5xx code and the node should be ejected. + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, true)); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE, + false)); + // Get the configured number of failures and simulate than number of connect failures. + n = runtime_.snapshot_.getInteger("outlier_detection.consecutive_gateway_failure", + detector->config().consecutiveGatewayFailure()); + while (n--) { + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_TIMEOUT); + } + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); +} + +/** + * Set of tests to verify ejecting and unejecting nodes when local/connect failures are reported. + */ +TEST_F(OutlierDetectorImplTest, BasicFlowLocalOriginFailure) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({"tcp://127.0.0.1:80"}, true); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + std::shared_ptr detector(DetectorImpl::create( + cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, event_logger_)); + + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_consecutive_local_origin_failure", 100)) + .WillByDefault(Return(true)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + // When connect failure is detected the following methods should be called. + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE, + true)); + time_system_.setMonotonicTime(std::chrono::milliseconds(0)); + + // Get the configured number of failures and simulate than number of connect failures. + uint32_t n = runtime_.snapshot_.getInteger("outlier_detection.consecutive_local_origin_failure", + detector->config().consecutiveLocalOriginFailure()); + while (n--) { + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_CONNECT_FAILED); + } + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Wait short time - not enough to be unejected + time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + interval_timer_->callback_(); + EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); + + // Interval that does bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(30001)); + EXPECT_CALL(checker_, check(hosts_[0])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[0]))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + interval_timer_->callback_(); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + + // Simulate few connect failures, not enough for ejection and then simulate connect success + // and again few failures not enough for ejection. + n = runtime_.snapshot_.getInteger("outlier_detection.consecutive_local_origin_failure", + detector->config().consecutiveLocalOriginFailure()); + n--; // make sure that this is not enough for ejection. + while (n--) { + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_CONNECT_FAILED); + } + // now success and few failures + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_CONNECT_SUCCESS); + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_CONNECT_FAILED); + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_CONNECT_FAILED); + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); + + // Check stats + EXPECT_EQ( + 1UL, + cluster_.info_->stats_store_.counter("outlier_detection.ejections_enforced_total").value()); + EXPECT_EQ(1UL, + cluster_.info_->stats_store_ + .counter("outlier_detection.ejections_detected_consecutive_local_origin_failure") + .value()); + EXPECT_EQ(1UL, + cluster_.info_->stats_store_ + .counter("outlier_detection.ejections_enforced_consecutive_local_origin_failure") + .value()); +} + /** * Test the interaction between the consecutive gateway failure and 5xx detectors. * This will first trigger a consecutive gateway failure with 503s, and then trigger 5xx with a mix @@ -454,7 +754,57 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailureAnd5xx) { .value()); } -TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRate) { +// Test mapping of Non-Http codes to Http. This happens when split between external and local +// origin errors is turned off. +TEST_F(OutlierDetectorImplTest, BasicFlowNonHttpCodesExternalOrigin) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({"tcp://127.0.0.1:80"}); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + std::shared_ptr detector(DetectorImpl::create( + cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + addHosts({"tcp://127.0.0.1:81"}); + cluster_.prioritySet().getMockHostSet(0)->runCallbacks({hosts_[1]}, {}); + + ON_CALL(runtime_.snapshot_, featureEnabled("outlier_detection.enforcing_consecutive_5xx", 100)) + .WillByDefault(Return(true)); + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_consecutive_gateway_failure", 0)) + .WillByDefault(Return(false)); + + // Make sure that EXT_ORIGIN_REQUEST_SUCCESS cancels LOCAL_ORIGIN_CONNECT_FAILED + for (auto i = 0; i < 100; i++) { + loadRq(hosts_[0], 1, Result::LOCAL_ORIGIN_CONNECT_FAILED); + loadRq(hosts_[0], 1, Result::EXT_ORIGIN_REQUEST_SUCCESS); + } + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + + // Cause a consecutive 5xx error. This situation happens in router filter. + // Make sure that one CONNECT_SUCCESS with optional code zero, does not + // interrupt sequence of LOCAL_ORIGIN_CONNECT_FAILED. + loadRq(hosts_[0], 1, Result::LOCAL_ORIGIN_CONNECT_FAILED); + hosts_[0]->outlierDetector().putResult(Result::LOCAL_ORIGIN_CONNECT_SUCCESS); + hosts_[0]->outlierDetector().putResponseTime(std::chrono::milliseconds(5)); + loadRq(hosts_[0], 3, Result::LOCAL_ORIGIN_CONNECT_FAILED); + + time_system_.setMonotonicTime(std::chrono::milliseconds(0)); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, true)); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE, + false)); + EXPECT_CALL(checker_, check(hosts_[0])); + loadRq(hosts_[0], 1, Result::LOCAL_ORIGIN_CONNECT_FAILED); + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); +} + +TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({ "tcp://127.0.0.1:80", @@ -501,9 +851,19 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRate) { ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_stdev_factor", 1900)) .WillByDefault(Return(1900)); interval_timer_->callback_(); - EXPECT_EQ(50, hosts_[4]->outlierDetector().successRate()); - EXPECT_EQ(90, detector->successRateAverage()); - EXPECT_EQ(52, detector->successRateEjectionThreshold()); + EXPECT_EQ(50, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(90, detector->successRateAverage( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(52, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + // Make sure that local origin success rate monitor is not affected + EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(-1, + detector->successRateAverage(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(-1, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); @@ -545,9 +905,142 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRate) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); interval_timer_->callback_(); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); - EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate()); - EXPECT_EQ(-1, detector->successRateAverage()); - EXPECT_EQ(-1, detector->successRateEjectionThreshold()); + EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(-1, detector->successRateAverage( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(-1, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); +} + +// Test verifies that EXT_ORIGIN_REQUEST_FAILED and EXT_ORIGIN_REQUEST_SUCCESS cancel +// each other in split mode. +TEST_F(OutlierDetectorImplTest, ExternalOriginEventsWithSplit) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({"tcp://127.0.0.1:80"}, true); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + std::shared_ptr detector(DetectorImpl::create( + cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, event_logger_)); + + for (auto i = 0; i < 100; i++) { + hosts_[0]->outlierDetector().putResult(Result::EXT_ORIGIN_REQUEST_FAILED); + hosts_[0]->outlierDetector().putResult(Result::EXT_ORIGIN_REQUEST_SUCCESS); + } + EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + + // Now make sure that EXT_ORIGIN_REQUEST_FAILED ejects the host + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, true)); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[0]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE, + false)); + for (auto i = 0; i < 100; i++) { + hosts_[0]->outlierDetector().putResult(Result::EXT_ORIGIN_REQUEST_FAILED); + } + EXPECT_TRUE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); +} + +TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({ + "tcp://127.0.0.1:80", + "tcp://127.0.0.1:81", + "tcp://127.0.0.1:82", + "tcp://127.0.0.1:83", + "tcp://127.0.0.1:84", + }); + + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + std::shared_ptr detector(DetectorImpl::create( + cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, event_logger_)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + // Turn off detecting consecutive local origin failures. + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_consecutive_local_origin_failure", 100)) + .WillByDefault(Return(false)); + // Expect non-enforcing logging to happen every time the consecutive_ counter + // gets saturated (every 5 times). + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE, + false)) + .Times(40); + // Cause a SR error on one host. First have 4 of the hosts have perfect SR. + loadRq(hosts_, 200, Result::LOCAL_ORIGIN_CONNECT_SUCCESS); + loadRq(hosts_[4], 200, Result::LOCAL_ORIGIN_CONNECT_FAILED); + + time_system_.setMonotonicTime(std::chrono::milliseconds(10000)); + EXPECT_CALL(checker_, check(hosts_[4])); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN, + true)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_stdev_factor", 1900)) + .WillByDefault(Return(1900)); + interval_timer_->callback_(); + EXPECT_EQ(50, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(90, + detector->successRateAverage(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(52, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + // Make sure that external origin success rate monitor is not affected + EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(-1, detector->successRateAverage( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(-1, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Interval that doesn't bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(19999)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + interval_timer_->callback_(); + EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Interval that does bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(50001)); + EXPECT_CALL(checker_, check(hosts_[4])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[4]))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + interval_timer_->callback_(); + EXPECT_FALSE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + + // Expect non-enforcing logging to happen every time the consecutive_ counter + // gets saturated (every 5 times). + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE, + false)) + .Times(5); + + // Give 4 hosts enough request volume but not to the 5th. Should not cause an ejection. + loadRq(hosts_, 25, Result::LOCAL_ORIGIN_CONNECT_SUCCESS); + loadRq(hosts_[4], 25, Result::LOCAL_ORIGIN_CONNECT_FAILED); + + time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + interval_timer_->callback_(); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(-1, + detector->successRateAverage(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(-1, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); } // Validate that empty hosts doesn't crash success rate handling when success_rate_minimum_hosts is @@ -821,9 +1314,15 @@ TEST(OutlierDetectionEventLoggerImplTest, All) { StringViewSaver log3; EXPECT_CALL(host->outlier_detector_, lastUnejectionTime()).WillOnce(ReturnRef(monotonic_time)); - EXPECT_CALL(host->outlier_detector_, successRate()).WillOnce(Return(0)); - EXPECT_CALL(detector, successRateAverage()).WillOnce(Return(0)); - EXPECT_CALL(detector, successRateEjectionThreshold()).WillOnce(Return(0)); + EXPECT_CALL(host->outlier_detector_, + successRate(DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)) + .WillOnce(Return(0)); + EXPECT_CALL(detector, + successRateAverage(DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)) + .WillOnce(Return(0)); + EXPECT_CALL(detector, successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)) + .WillOnce(Return(0)); EXPECT_CALL(*file, write(absl::string_view( "{\"type\":\"SUCCESS_RATE\",\"cluster_name\":\"fake_cluster\"," @@ -858,20 +1357,10 @@ TEST(OutlierUtility, SRThreshold) { }; double sum = 450; - Utility::EjectionPair ejection_pair = Utility::successRateEjectionThreshold(sum, data, 1.9); - EXPECT_EQ(52.0, ejection_pair.ejection_threshold_); - EXPECT_EQ(90.0, ejection_pair.success_rate_average_); -} - -TEST(DetectorHostMonitorImpl, resultToHttpCode) { - EXPECT_EQ(Http::Code::OK, DetectorHostMonitorImpl::resultToHttpCode(Result::SUCCESS)); - EXPECT_EQ(Http::Code::GatewayTimeout, DetectorHostMonitorImpl::resultToHttpCode(Result::TIMEOUT)); - EXPECT_EQ(Http::Code::ServiceUnavailable, - DetectorHostMonitorImpl::resultToHttpCode(Result::CONNECT_FAILED)); - EXPECT_EQ(Http::Code::InternalServerError, - DetectorHostMonitorImpl::resultToHttpCode(Result::REQUEST_FAILED)); - EXPECT_EQ(Http::Code::ServiceUnavailable, - DetectorHostMonitorImpl::resultToHttpCode(Result::SERVER_FAILURE)); + DetectorImpl::EjectionPair success_rate_nums = + DetectorImpl::successRateEjectionThreshold(sum, data, 1.9); + EXPECT_EQ(90.0, success_rate_nums.success_rate_average_); // average success rate + EXPECT_EQ(52.0, success_rate_nums.ejection_threshold_); // ejection threshold } } // namespace diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 5b974d6f0924..444ef060931f 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -136,7 +136,8 @@ TEST_F(RedisClientImplTest, BatchWithZeroBufferAndTimeout) { Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); })); upstream_read_filter_->onData(fake_data, false); @@ -187,7 +188,8 @@ TEST_F(RedisClientImplTest, BatchWithTimerFiring) { Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); })); upstream_read_filter_->onData(fake_data, false); @@ -230,13 +232,15 @@ TEST_F(RedisClientImplTest, BatchWithTimerCancelledByBufferFlush) { Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); Common::Redis::RespValuePtr response2(new Common::Redis::RespValue()); EXPECT_CALL(callbacks2, onResponse_(Ref(response2))); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response2)); })); upstream_read_filter_->onData(fake_data, false); @@ -278,13 +282,15 @@ TEST_F(RedisClientImplTest, Basic) { Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); Common::Redis::RespValuePtr response2(new Common::Redis::RespValue()); EXPECT_CALL(callbacks2, onResponse_(Ref(response2))); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response2)); })); upstream_read_filter_->onData(fake_data, false); @@ -324,13 +330,15 @@ TEST_F(RedisClientImplTest, Cancel) { Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(callbacks1, onResponse_(_)).Times(0); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); Common::Redis::RespValuePtr response2(new Common::Redis::RespValue()); EXPECT_CALL(callbacks2, onResponse_(Ref(response2))); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response2)); })); upstream_read_filter_->onData(fake_data, false); @@ -359,7 +367,8 @@ TEST_F(RedisClientImplTest, FailAll) { onConnected(); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SERVER_FAILURE)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); EXPECT_CALL(callbacks1, onFailure()); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); EXPECT_CALL(connection_callbacks, onEvent(Network::ConnectionEvent::RemoteClose)); @@ -415,7 +424,8 @@ TEST_F(RedisClientImplTest, ProtocolError) { EXPECT_CALL(*decoder_, decode(Ref(fake_data))).WillOnce(Invoke([&](Buffer::Instance&) -> void { throw Common::Redis::ProtocolError("error"); })); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::REQUEST_FAILED)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_FAILED, _)); EXPECT_CALL(*upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); EXPECT_CALL(callbacks1, onFailure()); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); @@ -437,7 +447,8 @@ TEST_F(RedisClientImplTest, ConnectFail) { PoolRequest* handle1 = client_->makeRequest(request1, callbacks1); EXPECT_NE(nullptr, handle1); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SERVER_FAILURE)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); EXPECT_CALL(callbacks1, onFailure()); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); upstream_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); @@ -469,7 +480,7 @@ TEST_F(RedisClientImplTest, OutlierDisabled) { PoolRequest* handle1 = client_->makeRequest(request1, callbacks1); EXPECT_NE(nullptr, handle1); - EXPECT_CALL(host_->outlier_detector_, putResult(_)).Times(0); + EXPECT_CALL(host_->outlier_detector_, putResult(_, _)).Times(0); EXPECT_CALL(callbacks1, onFailure()); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); upstream_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); @@ -490,7 +501,8 @@ TEST_F(RedisClientImplTest, ConnectTimeout) { PoolRequest* handle1 = client_->makeRequest(request1, callbacks1); EXPECT_NE(nullptr, handle1); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::TIMEOUT)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); EXPECT_CALL(*upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); EXPECT_CALL(callbacks1, onFailure()); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); @@ -519,7 +531,8 @@ TEST_F(RedisClientImplTest, OpTimeout) { EXPECT_CALL(callbacks1, onResponse_(_)); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); respond(); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_rq_total_.value()); @@ -531,7 +544,8 @@ TEST_F(RedisClientImplTest, OpTimeout) { handle1 = client_->makeRequest(request1, callbacks1); EXPECT_NE(nullptr, handle1); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::TIMEOUT)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); EXPECT_CALL(*upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); EXPECT_CALL(callbacks1, onFailure()); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); @@ -580,7 +594,8 @@ TEST_F(RedisClientImplTest, AskRedirection) { EXPECT_CALL(callbacks1, onRedirection(Ref(*response1))).WillOnce(Return(false)); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_internal_redirect_failed_total_.value()); @@ -591,7 +606,8 @@ TEST_F(RedisClientImplTest, AskRedirection) { response2->asString() = "ASK 2222 10.1.2.4:4321"; EXPECT_CALL(callbacks2, onRedirection(Ref(*response2))).WillOnce(Return(true)); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response2)); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_internal_redirect_succeeded_total_.value()); @@ -640,7 +656,8 @@ TEST_F(RedisClientImplTest, MovedRedirection) { EXPECT_CALL(callbacks1, onRedirection(Ref(*response1))).WillOnce(Return(false)); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_internal_redirect_failed_total_.value()); @@ -651,7 +668,8 @@ TEST_F(RedisClientImplTest, MovedRedirection) { response2->asString() = "MOVED 2222 10.1.2.4:4321"; EXPECT_CALL(callbacks2, onRedirection(Ref(*response2))).WillOnce(Return(true)); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response2)); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_internal_redirect_succeeded_total_.value()); @@ -699,7 +717,8 @@ TEST_F(RedisClientImplTest, AskRedirectionNotEnabled) { // Simulate redirection failure. EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); EXPECT_EQ(0UL, host_->cluster_.stats_.upstream_internal_redirect_failed_total_.value()); @@ -711,7 +730,8 @@ TEST_F(RedisClientImplTest, AskRedirectionNotEnabled) { response2->asString() = "ASK 2222 10.1.2.4:4321"; EXPECT_CALL(callbacks2, onResponse_(Ref(response2))); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response2)); EXPECT_EQ(0UL, host_->cluster_.stats_.upstream_internal_redirect_failed_total_.value()); @@ -759,7 +779,8 @@ TEST_F(RedisClientImplTest, MovedRedirectionNotEnabled) { response1->asString() = "MOVED 1111 10.1.2.3:4321"; EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); EXPECT_EQ(0UL, host_->cluster_.stats_.upstream_internal_redirect_succeeded_total_.value()); @@ -771,7 +792,8 @@ TEST_F(RedisClientImplTest, MovedRedirectionNotEnabled) { response2->asString() = "MOVED 2222 10.1.2.4:4321"; EXPECT_CALL(callbacks2, onResponse_(Ref(response2))); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::SUCCESS)); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response2)); EXPECT_EQ(0UL, host_->cluster_.stats_.upstream_internal_redirect_succeeded_total_.value()); diff --git a/test/mocks/upstream/host.h b/test/mocks/upstream/host.h index 0388a083c729..f4c43a2fd06b 100644 --- a/test/mocks/upstream/host.h +++ b/test/mocks/upstream/host.h @@ -26,12 +26,13 @@ class MockDetectorHostMonitor : public DetectorHostMonitor { MOCK_METHOD0(numEjections, uint32_t()); MOCK_METHOD1(putHttpResponseCode, void(uint64_t code)); - MOCK_METHOD1(putResult, void(Result result)); + MOCK_METHOD2(putResult, void(Result result, absl::optional code)); MOCK_METHOD1(putResponseTime, void(std::chrono::milliseconds time)); MOCK_METHOD0(lastEjectionTime, const absl::optional&()); MOCK_METHOD0(lastUnejectionTime, const absl::optional&()); - MOCK_CONST_METHOD0(successRate, double()); - MOCK_METHOD1(successRate, void(double new_success_rate)); + MOCK_CONST_METHOD1(successRate, double(DetectorHostMonitor::SuccessRateMonitorType type)); + MOCK_METHOD2(successRate, + void(DetectorHostMonitor::SuccessRateMonitorType type, double new_success_rate)); }; class MockEventLogger : public EventLogger { @@ -57,8 +58,9 @@ class MockDetector : public Detector { } MOCK_METHOD1(addChangedStateCb, void(ChangeStateCb cb)); - MOCK_CONST_METHOD0(successRateAverage, double()); - MOCK_CONST_METHOD0(successRateEjectionThreshold, double()); + MOCK_CONST_METHOD1(successRateAverage, double(DetectorHostMonitor::SuccessRateMonitorType)); + MOCK_CONST_METHOD1(successRateEjectionThreshold, + double(DetectorHostMonitor::SuccessRateMonitorType)); std::list callbacks_; }; diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index feca903e1b91..e90a0954fbd8 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -1047,7 +1047,14 @@ TEST_P(AdminInstanceTest, ClustersJson) { NiceMock outlier_detector; ON_CALL(Const(cluster), outlierDetector()).WillByDefault(Return(&outlier_detector)); - ON_CALL(outlier_detector, successRateEjectionThreshold()).WillByDefault(Return(6.0)); + ON_CALL(outlier_detector, + successRateEjectionThreshold( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)) + .WillByDefault(Return(6.0)); + ON_CALL(outlier_detector, + successRateEjectionThreshold( + Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)) + .WillByDefault(Return(9.0)); ON_CALL(*cluster.info_, addedViaApi()).WillByDefault(Return(true)); @@ -1090,8 +1097,14 @@ TEST_P(AdminInstanceTest, ClustersJson) { ON_CALL(*host, healthFlagGet(Upstream::Host::HealthFlag::PENDING_DYNAMIC_REMOVAL)) .WillByDefault(Return(true)); - ON_CALL(host->outlier_detector_, successRate()).WillByDefault(Return(43.2)); + ON_CALL( + host->outlier_detector_, + successRate(Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)) + .WillByDefault(Return(43.2)); ON_CALL(*host, weight()).WillByDefault(Return(5)); + ON_CALL(host->outlier_detector_, + successRate(Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)) + .WillByDefault(Return(93.2)); ON_CALL(*host, priority()).WillByDefault(Return(6)); Buffer::OwnedImpl response; @@ -1108,6 +1121,9 @@ TEST_P(AdminInstanceTest, ClustersJson) { "success_rate_ejection_threshold": { "value": 6 }, + "local_origin_success_rate_ejection_threshold": { + "value": 9 + }, "added_via_api": true, "host_statuses": [ { @@ -1157,7 +1173,10 @@ TEST_P(AdminInstanceTest, ClustersJson) { }, "weight": 5, "hostname": "foo.com", - "priority": 6 + "priority": 6, + "local_origin_success_rate": { + "value": 93.2 + } } ] } @@ -1176,6 +1195,8 @@ TEST_P(AdminInstanceTest, ClustersJson) { EXPECT_EQ(Http::Code::OK, getCallback("/clusters", header_map, response2)); const std::string expected_text = R"EOF(fake_cluster::outlier::success_rate_average::0 fake_cluster::outlier::success_rate_ejection_threshold::6 +fake_cluster::outlier::local_origin_success_rate_average::0 +fake_cluster::outlier::local_origin_success_rate_ejection_threshold::9 fake_cluster::default_priority::max_connections::1 fake_cluster::default_priority::max_pending_requests::1024 fake_cluster::default_priority::max_requests::1024 @@ -1197,8 +1218,9 @@ fake_cluster::1.2.3.4:80::region::test_region fake_cluster::1.2.3.4:80::zone::test_zone fake_cluster::1.2.3.4:80::sub_zone::test_sub_zone fake_cluster::1.2.3.4:80::canary::false -fake_cluster::1.2.3.4:80::success_rate::43.2 fake_cluster::1.2.3.4:80::priority::6 +fake_cluster::1.2.3.4:80::success_rate::43.2 +fake_cluster::1.2.3.4:80::local_origin_success_rate::93.2 )EOF"; EXPECT_EQ(expected_text, response2.toString()); } From 9b096c31d36ee908888c071d2073d126693dc3c0 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Thu, 27 Jun 2019 22:35:57 -0700 Subject: [PATCH 080/542] clang-tidy: use empty() (#7419) readability-container-size-empty is already enabled as an error in clang-tidy, this diff just cleans up the existing errors. Risk Level: Low Testing: Existing Signed-off-by: Derek Argueta --- source/common/upstream/cluster_manager_impl.cc | 2 +- source/common/upstream/subset_lb.cc | 2 +- source/extensions/common/tap/tap_config_base.cc | 2 +- .../filters/network/thrift_proxy/header_transport_impl.cc | 2 +- source/extensions/transport_sockets/tls/context_impl.cc | 2 +- test/common/grpc/grpc_client_integration_test.cc | 8 ++++---- test/common/network/address_impl_test.cc | 2 +- test/common/router/config_impl_test.cc | 2 +- test/common/upstream/cluster_manager_impl_test.cc | 2 +- test/common/upstream/subset_lb_test.cc | 2 +- test/extensions/filters/http/squash/squash_filter_test.cc | 2 +- .../filters/network/mysql_proxy/mysql_command_test.cc | 4 ++-- test/integration/fake_upstream.h | 2 +- 13 files changed, 17 insertions(+), 17 deletions(-) diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 2a293afe4d5f..d4a666f97af4 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -344,7 +344,7 @@ void ClusterManagerImpl::onClusterInit(Cluster& cluster) { const auto merge_timeout = PROTOBUF_GET_MS_OR_DEFAULT(cluster.info()->lbConfig(), update_merge_window, 1000); // Remember: we only merge updates with no adds/removes — just hc/weight/metadata changes. - const bool is_mergeable = !hosts_added.size() && !hosts_removed.size(); + const bool is_mergeable = hosts_added.empty() && hosts_removed.empty(); if (merge_timeout > 0) { // If this is not mergeable, we should cancel any scheduled updates since diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index f6fb7212bb34..e867e4f39ced 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -71,7 +71,7 @@ SubsetLoadBalancer::SubsetLoadBalancer( // Configure future updates. original_priority_set_callback_handle_ = priority_set.addPriorityUpdateCb( [this](uint32_t priority, const HostVector& hosts_added, const HostVector& hosts_removed) { - if (!hosts_added.size() && !hosts_removed.size()) { + if (hosts_added.empty() && hosts_removed.empty()) { // It's possible that metadata changed, without hosts being added nor removed. // If so we need to add any new subsets, remove unused ones, and regroup hosts into // the right subsets. diff --git a/source/extensions/common/tap/tap_config_base.cc b/source/extensions/common/tap/tap_config_base.cc index 85aa6bf1f69c..5db293de3a84 100644 --- a/source/extensions/common/tap/tap_config_base.cc +++ b/source/extensions/common/tap/tap_config_base.cc @@ -74,7 +74,7 @@ TapConfigBaseImpl::TapConfigBaseImpl(envoy::service::tap::v2alpha::TapConfig&& p } const Matcher& TapConfigBaseImpl::rootMatcher() const { - ASSERT(matchers_.size() >= 1); + ASSERT(!matchers_.empty()); return *matchers_[0]; } diff --git a/source/extensions/filters/network/thrift_proxy/header_transport_impl.cc b/source/extensions/filters/network/thrift_proxy/header_transport_impl.cc index f95457d456b2..e7eda453c9f6 100644 --- a/source/extensions/filters/network/thrift_proxy/header_transport_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/header_transport_impl.cc @@ -198,7 +198,7 @@ void HeaderTransportImpl::encodeFrame(Buffer::Instance& buffer, const MessageMet } BufferHelper::writeVarIntI32(header_buffer, 0); // num transforms - if (headers.size() > 0) { + if (!headers.empty()) { // Info ID 1 header_buffer.writeByte(1); diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index 1e358db812b0..11ffa4d5bb09 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -895,7 +895,7 @@ int ServerContextImpl::sessionTicketProcess(SSL*, uint8_t* key_name, uint8_t* iv if (encrypt == 1) { // Encrypt - RELEASE_ASSERT(session_ticket_keys_.size() >= 1, ""); + RELEASE_ASSERT(!session_ticket_keys_.empty(), ""); // TODO(ggreenway): validate in SDS that session_ticket_keys_ cannot be empty, // or if we allow it to be emptied, reconfigure the context so this callback // isn't set. diff --git a/test/common/grpc/grpc_client_integration_test.cc b/test/common/grpc/grpc_client_integration_test.cc index 930c163de8cd..5044cace4d40 100644 --- a/test/common/grpc/grpc_client_integration_test.cc +++ b/test/common/grpc/grpc_client_integration_test.cc @@ -396,8 +396,8 @@ class GrpcAccessTokenClientIntegrationTest : public GrpcSslClientIntegrationTest AssertionResult result = fake_stream.waitForHeadersComplete(); RELEASE_ASSERT(result, result.message()); Http::TestHeaderMapImpl stream_headers(fake_stream.headers()); - if (access_token_value_ != "") { - if (access_token_value_2_ == "") { + if (!access_token_value_.empty()) { + if (access_token_value_2_.empty()) { EXPECT_EQ("Bearer " + access_token_value_, stream_headers.get_("authorization")); } else { EXPECT_EQ("Bearer " + access_token_value_ + ",Bearer " + access_token_value_2_, @@ -414,10 +414,10 @@ class GrpcAccessTokenClientIntegrationTest : public GrpcSslClientIntegrationTest ssl_creds->mutable_root_certs()->set_filename( TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); google_grpc->add_call_credentials()->set_access_token(access_token_value_); - if (access_token_value_2_ != "") { + if (!access_token_value_2_.empty()) { google_grpc->add_call_credentials()->set_access_token(access_token_value_2_); } - if (refresh_token_value_ != "") { + if (!refresh_token_value_.empty()) { google_grpc->add_call_credentials()->set_google_refresh_token(refresh_token_value_); } return config; diff --git a/test/common/network/address_impl_test.cc b/test/common/network/address_impl_test.cc index 584b5906d722..27298c383d9d 100644 --- a/test/common/network/address_impl_test.cc +++ b/test/common/network/address_impl_test.cc @@ -431,7 +431,7 @@ class MixedAddressTest : public testing::TestWithParam<::testing::tuplemutable_routes(0) ->mutable_route() ->mutable_hash_policy(); - if (hash_policies->size() > 0) { + if (!hash_policies->empty()) { return hash_policies->Mutable(0); } else { return hash_policies->Add(); diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 47f469f73001..69870436d991 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -290,7 +290,7 @@ class ClusterManagerImplTest : public testing::Test { envoy::api::v2::core::Metadata buildMetadata(const std::string& version) const { envoy::api::v2::core::Metadata metadata; - if (version != "") { + if (!version.empty()) { Envoy::Config::Metadata::mutableMetadataValue( metadata, Config::MetadataFilters::get().ENVOY_LB, "version") .set_string_value(version); diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index 50ec11fcc5f3..c18dc9a4f8f0 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -381,7 +381,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { bool is_default = false) const { envoy::api::v2::core::Metadata metadata; - if (version != "") { + if (!version.empty()) { Envoy::Config::Metadata::mutableMetadataValue( metadata, Config::MetadataFilters::get().ENVOY_LB, "version") .set_string_value(version); diff --git a/test/extensions/filters/http/squash/squash_filter_test.cc b/test/extensions/filters/http/squash/squash_filter_test.cc index ac0c30f1516b..03c19f42d95b 100644 --- a/test/extensions/filters/http/squash/squash_filter_test.cc +++ b/test/extensions/filters/http/squash/squash_filter_test.cc @@ -244,7 +244,7 @@ class SquashFilterTest : public testing::Test { } Envoy::Http::AsyncClient::Callbacks* popPendingCallback() { - if (0 == callbacks_.size()) { + if (callbacks_.empty()) { // Can't use ASSERT_* as this is not a test function throw std::underflow_error("empty deque"); } diff --git a/test/extensions/filters/network/mysql_proxy/mysql_command_test.cc b/test/extensions/filters/network/mysql_proxy/mysql_command_test.cc index 76a7f35a7a0f..a5b1648dadfa 100644 --- a/test/extensions/filters/network/mysql_proxy/mysql_command_test.cc +++ b/test/extensions/filters/network/mysql_proxy/mysql_command_test.cc @@ -71,7 +71,7 @@ class MySQLCommandTest : public testing::Test, public MySQLTestUtils { std::string buildCreate(enum TestResource res, std::string option, bool if_not_exists, std::string res_name, std::string value) { std::string command("CREATE "); - if (option != "") { + if (!option.empty()) { command.append(option); command.append(SPACE); } @@ -159,7 +159,7 @@ class MySQLCommandTest : public testing::Test, public MySQLTestUtils { //"INSERT INTO ... std::string buildInsert(std::string option, bool into, std::string table, std::string values) { std::string command("INSERT "); - if (option != "") { + if (!option.empty()) { command.append(option); command.append(SPACE); } diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 7fa348ee5a52..38d66690e41e 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -126,7 +126,7 @@ class FakeStream : public Http::StreamDecoder, << "Couldn't decode gRPC data frame: " << body().toString(); } } - if (decoded_grpc_frames_.size() < 1) { + if (decoded_grpc_frames_.empty()) { timeout = std::chrono::duration_cast(end_time - timeSystem().monotonicTime()); if (!waitForData(client_dispatcher, grpc_decoder_.length(), timeout)) { From 245285f1ec8e18e6c02edca2e1d42d7c4474b134 Mon Sep 17 00:00:00 2001 From: Adam Kotwasinski Date: Sat, 29 Jun 2019 09:34:58 -0700 Subject: [PATCH 081/542] =?UTF-8?q?Kafka=20codec:=20reorganize=20code=20ge?= =?UTF-8?q?nerators,=20so=20that=20test=20sources=20are=20pre=E2=80=A6=20(?= =?UTF-8?q?#7405)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adam Kotwasinski --- source/extensions/filters/network/kafka/BUILD | 48 +++-- .../complex_type_template.j2 | 0 .../generator.py} | 175 ++++++++---------- .../kafka_request_resolver_cc.j2 | 0 .../kafka_response_resolver_cc.j2 | 0 .../network/kafka/protocol/launcher.py | 46 +++++ .../request_parser.j2 | 0 .../requests_h.j2 | 0 .../response_parser.j2 | 0 .../responses_h.j2 | 0 .../network/kafka/serialization/generator.py | 57 ++++++ .../network/kafka/serialization/launcher.py | 33 ++++ .../serialization_composite_h.j2 | 0 .../serialization_composite_generator.py | 78 -------- test/extensions/filters/network/kafka/BUILD | 47 +++-- .../network/kafka/protocol/launcher.py | 44 +++++ .../request_codec_request_test_cc.j2 | 0 .../kafka/protocol}/requests_test_cc.j2 | 0 .../response_codec_response_test_cc.j2 | 0 .../kafka/protocol}/responses_test_cc.j2 | 0 .../network/kafka/serialization/launcher.py | 32 ++++ .../serialization_composite_test_cc.j2 | 0 tools/check_format.py | 4 +- 23 files changed, 354 insertions(+), 210 deletions(-) rename source/extensions/filters/network/kafka/{protocol_code_generator => protocol}/complex_type_template.j2 (100%) rename source/extensions/filters/network/kafka/{protocol_code_generator/kafka_generator.py => protocol/generator.py} (78%) rename source/extensions/filters/network/kafka/{protocol_code_generator => protocol}/kafka_request_resolver_cc.j2 (100%) rename source/extensions/filters/network/kafka/{protocol_code_generator => protocol}/kafka_response_resolver_cc.j2 (100%) create mode 100644 source/extensions/filters/network/kafka/protocol/launcher.py rename source/extensions/filters/network/kafka/{protocol_code_generator => protocol}/request_parser.j2 (100%) rename source/extensions/filters/network/kafka/{protocol_code_generator => protocol}/requests_h.j2 (100%) rename source/extensions/filters/network/kafka/{protocol_code_generator => protocol}/response_parser.j2 (100%) rename source/extensions/filters/network/kafka/{protocol_code_generator => protocol}/responses_h.j2 (100%) create mode 100755 source/extensions/filters/network/kafka/serialization/generator.py create mode 100644 source/extensions/filters/network/kafka/serialization/launcher.py rename source/extensions/filters/network/kafka/{serialization_code_generator => serialization}/serialization_composite_h.j2 (100%) delete mode 100755 source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_generator.py create mode 100644 test/extensions/filters/network/kafka/protocol/launcher.py rename {source/extensions/filters/network/kafka/protocol_code_generator => test/extensions/filters/network/kafka/protocol}/request_codec_request_test_cc.j2 (100%) rename {source/extensions/filters/network/kafka/protocol_code_generator => test/extensions/filters/network/kafka/protocol}/requests_test_cc.j2 (100%) rename {source/extensions/filters/network/kafka/protocol_code_generator => test/extensions/filters/network/kafka/protocol}/response_codec_response_test_cc.j2 (100%) rename {source/extensions/filters/network/kafka/protocol_code_generator => test/extensions/filters/network/kafka/protocol}/responses_test_cc.j2 (100%) create mode 100644 test/extensions/filters/network/kafka/serialization/launcher.py rename {source/extensions/filters/network/kafka/serialization_code_generator => test/extensions/filters/network/kafka/serialization}/serialization_composite_test_cc.j2 (100%) diff --git a/source/extensions/filters/network/kafka/BUILD b/source/extensions/filters/network/kafka/BUILD index 8eb906e15b5b..8ff831a0274c 100644 --- a/source/extensions/filters/network/kafka/BUILD +++ b/source/extensions/filters/network/kafka/BUILD @@ -74,12 +74,12 @@ genrule( "external/kafka_request_resolver.cc", ], cmd = """ - ./$(location :kafka_code_generator) generate-source request \ + ./$(location :kafka_protocol_code_generator_bin) request \ $(location external/requests.h) $(location external/kafka_request_resolver.cc) \ $(SRCS) """, tools = [ - ":kafka_code_generator", + ":kafka_protocol_code_generator_bin", ], ) @@ -135,21 +135,29 @@ genrule( "external/kafka_response_resolver.cc", ], cmd = """ - ./$(location :kafka_code_generator) generate-source response \ + ./$(location :kafka_protocol_code_generator_bin) response \ $(location external/responses.h) $(location external/kafka_response_resolver.cc) \ $(SRCS) """, tools = [ - ":kafka_code_generator", + ":kafka_protocol_code_generator_bin", ], ) py_binary( - name = "kafka_code_generator", - srcs = ["protocol_code_generator/kafka_generator.py"], - data = glob(["protocol_code_generator/*.j2"]), - main = "protocol_code_generator/kafka_generator.py", - deps = ["@com_github_pallets_jinja//:jinja2"], + name = "kafka_protocol_code_generator_bin", + srcs = ["protocol/launcher.py"], + data = glob(["protocol/*.j2"]), + main = "protocol/launcher.py", + deps = [ + ":kafka_protocol_generator_lib", + "@com_github_pallets_jinja//:jinja2", + ], +) + +py_library( + name = "kafka_protocol_generator_lib", + srcs = ["protocol/generator.py"], ) envoy_cc_library( @@ -183,20 +191,28 @@ genrule( "external/serialization_composite.h", ], cmd = """ - ./$(location :serialization_composite_generator) generate-source \ + ./$(location :serialization_composite_code_generator_bin) \ $(location external/serialization_composite.h) """, tools = [ - ":serialization_composite_generator", + ":serialization_composite_code_generator_bin", ], ) py_binary( - name = "serialization_composite_generator", - srcs = ["serialization_code_generator/serialization_composite_generator.py"], - data = glob(["serialization_code_generator/*.j2"]), - main = "serialization_code_generator/serialization_composite_generator.py", - deps = ["@com_github_pallets_jinja//:jinja2"], + name = "serialization_composite_code_generator_bin", + srcs = ["serialization/launcher.py"], + data = glob(["serialization/*.j2"]), + main = "serialization/launcher.py", + deps = [ + ":serialization_composite_generator_lib", + "@com_github_pallets_jinja//:jinja2", + ], +) + +py_library( + name = "serialization_composite_generator_lib", + srcs = ["serialization/generator.py"], ) envoy_cc_library( diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/complex_type_template.j2 b/source/extensions/filters/network/kafka/protocol/complex_type_template.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/complex_type_template.j2 rename to source/extensions/filters/network/kafka/protocol/complex_type_template.j2 diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_generator.py b/source/extensions/filters/network/kafka/protocol/generator.py similarity index 78% rename from source/extensions/filters/network/kafka/protocol_code_generator/kafka_generator.py rename to source/extensions/filters/network/kafka/protocol/generator.py index c3a0f286e3d9..a0cad7e89ffd 100755 --- a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_generator.py +++ b/source/extensions/filters/network/kafka/protocol/generator.py @@ -1,67 +1,85 @@ #!/usr/bin/python +# Main library file containing all the protocol generation logic. -def main(): + +def generate_main_code(type, main_header_file, resolver_cc_file, input_files): """ - Kafka header generator script - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Generates C++ headers from Kafka protocol specification. - Can generate both main source code, as well as test code. - - Usage: - kafka_generator.py COMMAND MESSAGE_TYPE OUTPUT_FILES INPUT_FILES - where: - COMMAND : 'generate-source', to generate source files, - 'generate-test', to generate test files. - MESSAGE_TYPE : 'request' or 'response' - OUTPUT_FILES : if generate-source: location of 'requests.h'/'responses.h' and - 'kafka_request_resolver.cc' / 'kafka_response_resolver.cc', - if generate-test: location of 'requests_test.cc'/'responses_test.cc', - 'request_codec_request_test.cc' / 'response_codec_response_test.cc'. - INPUT_FILES: Kafka protocol json files to be processed. - - Kafka spec files are provided in Kafka clients jar file. - When generating source code, it creates: - - ${MESSAGE_TYPE}s.h - definition of all the structures/deserializers/parsers related to Kafka - requests/responses, - - kafka_${MESSAGE_TYPE}_resolver.cc - resolver that is responsible for creation of parsers - defined in ${MESSAGE_TYPE}s.h. - When generating test code, it creates: - - ${MESSAGE_TYPE}s_test.cc - serialization/deserialization tests for kafka structures, - - ${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test.cc - test for all request/response operations - using the codec API. - - Templates used are: - - to create '${MESSAGE_TYPE}.h': ${MESSAGE_TYPE}_h.j2, complex_type_template.j2, - request_parser.j2, - - to create 'kafka_${MESSAGE_TYPE}_resolver.cc': kafka_${MESSAGE_TYPE}_resolver_cc.j2, - - to create '${MESSAGE_TYPE}s_test.cc': ${MESSAGE_TYPE}s_test_cc.j2, - - to create '${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test.cc' - - ${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test_cc.j2. + Main code generator. + + Takes input files and processes them into structures representing a Kafka message (request or + response). + + These responses are then used to create: + - main_header_file - contains definitions of Kafka structures and their deserializers + - resolver_cc_file - contains request api key & version mapping to deserializer (from header file) """ + # Parse provided input files. + messages = parse_messages(input_files) - import sys - import os + complex_type_template = RenderingHelper.get_template('complex_type_template.j2') + parsers_template = RenderingHelper.get_template("%s_parser.j2" % type) - command = sys.argv[1] - type = sys.argv[2] + main_header_contents = '' + + for message in messages: + # For each child structure that is used by request/response, render its matching C++ code. + for dependency in message.declaration_chain: + main_header_contents += complex_type_template.render(complex_type=dependency) + # Each top-level structure (e.g. FetchRequest/FetchResponse) needs corresponding parsers. + main_header_contents += parsers_template.render(complex_type=message) + + # Full file with headers, namespace declaration etc. + template = RenderingHelper.get_template("%ss_h.j2" % type) + contents = template.render(contents=main_header_contents) + + # Generate main header file. + with open(main_header_file, 'w') as fd: + fd.write(contents) + + template = RenderingHelper.get_template("kafka_%s_resolver_cc.j2" % type) + contents = template.render(message_types=messages) + + # Generate ...resolver.cc file. + with open(resolver_cc_file, 'w') as fd: + fd.write(contents) + + +def generate_test_code(type, header_test_cc_file, codec_test_cc_file, input_files): + """ + Test code generator. + + Takes input files and processes them into structures representing a Kafka message (request or + response). + + These responses are then used to create: + - header_test_cc_file - tests for basic message serialization deserialization + - codec_test_cc_file - tests involving codec and Request/ResponseParserResolver + """ + # Parse provided input files. + messages = parse_messages(input_files) + + # Generate header-test file. + template = RenderingHelper.get_template("%ss_test_cc.j2" % type) + contents = template.render(message_types=messages) + with open(header_test_cc_file, 'w') as fd: + fd.write(contents) + + # Generate codec-test file. + template = RenderingHelper.get_template("%s_codec_%s_test_cc.j2" % (type, type)) + contents = template.render(message_types=messages) + with open(codec_test_cc_file, 'w') as fd: + fd.write(contents) - if 'generate-source' == command: - main_header_file = os.path.abspath(sys.argv[3]) - resolver_cc_file = os.path.abspath(sys.argv[4]) - input_files = sys.argv[5:] - elif 'generate-test' == command: - header_test_cc_file = os.path.abspath(sys.argv[3]) - codec_test_cc_file = os.path.abspath(sys.argv[4]) - input_files = sys.argv[5:] - else: - raise ValueError('invalid command: ' + command) +def parse_messages(input_files): + """ + Parse request/response structures from provided input files. + """ import re import json messages = [] - # For each specification file, remove comments, and parse the remains. for input_file in input_files: with open(input_file, 'r') as fd: @@ -73,51 +91,7 @@ def main(): # Sort messages by api_key. messages.sort(key=lambda x: x.get_extra('api_key')) - - # Generate main source code. - if 'generate-source' == command: - complex_type_template = RenderingHelper.get_template('complex_type_template.j2') - parsers_template = RenderingHelper.get_template("%s_parser.j2" % type) - - main_header_contents = '' - - for message in messages: - # For each child structure that is used by request/response, render its matching C++ code. - for dependency in message.declaration_chain: - main_header_contents += complex_type_template.render(complex_type=dependency) - # Each top-level structure (e.g. FetchRequest/FetchResponse) needs corresponding parsers. - main_header_contents += parsers_template.render(complex_type=message) - - # Full file with headers, namespace declaration etc. - template = RenderingHelper.get_template("%ss_h.j2" % type) - contents = template.render(contents=main_header_contents) - - # Generate main header file. - with open(main_header_file, 'w') as fd: - fd.write(contents) - - template = RenderingHelper.get_template("kafka_%s_resolver_cc.j2" % type) - contents = template.render(message_types=messages) - - # Generate ...resolver.cc file. - with open(resolver_cc_file, 'w') as fd: - fd.write(contents) - - # Generate test code. - if 'generate-test' == command: - template = RenderingHelper.get_template("%ss_test_cc.j2" % type) - contents = template.render(message_types=messages) - - # Generate header-test file. - with open(header_test_cc_file, 'w') as fd: - fd.write(contents) - - template = RenderingHelper.get_template("%s_codec_%s_test_cc.j2" % (type, type)) - contents = template.render(message_types=messages) - - # Generate codec-test file. - with open(codec_test_cc_file, 'w') as fd: - fd.write(contents) + return messages def parse_top_level_element(spec): @@ -535,10 +509,9 @@ class RenderingHelper: def get_template(template): import jinja2 import os + import sys + # Templates are resolved relatively to main start script, due to main & test templates being + # stored in different directories. env = jinja2.Environment( - loader=jinja2.FileSystemLoader(searchpath=os.path.dirname(os.path.abspath(__file__)))) + loader=jinja2.FileSystemLoader(searchpath=os.path.dirname(os.path.abspath(sys.argv[0])))) return env.get_template(template) - - -if __name__ == "__main__": - main() diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_request_resolver_cc.j2 b/source/extensions/filters/network/kafka/protocol/kafka_request_resolver_cc.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/kafka_request_resolver_cc.j2 rename to source/extensions/filters/network/kafka/protocol/kafka_request_resolver_cc.j2 diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/kafka_response_resolver_cc.j2 b/source/extensions/filters/network/kafka/protocol/kafka_response_resolver_cc.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/kafka_response_resolver_cc.j2 rename to source/extensions/filters/network/kafka/protocol/kafka_response_resolver_cc.j2 diff --git a/source/extensions/filters/network/kafka/protocol/launcher.py b/source/extensions/filters/network/kafka/protocol/launcher.py new file mode 100644 index 000000000000..fd913c3e25c0 --- /dev/null +++ b/source/extensions/filters/network/kafka/protocol/launcher.py @@ -0,0 +1,46 @@ +#!/usr/bin/python + +# Launcher for generating Kafka protocol code. + +import source.extensions.filters.network.kafka.protocol.generator as generator +import sys +import os + + +def main(): + """ + Kafka code generator script + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Generates C++ code from Kafka protocol specification for Kafka codec. + + Usage: + launcher.py MESSAGE_TYPE OUTPUT_FILES INPUT_FILES + where: + MESSAGE_TYPE : 'request' or 'response' + OUTPUT_FILES : location of 'requests.h'/'responses.h' and + 'kafka_request_resolver.cc'/'kafka_response_resolver.cc', + INPUT_FILES: Kafka protocol json files to be processed. + + Kafka spec files are provided in Kafka clients jar file. + + Files created are: + - ${MESSAGE_TYPE}s.h - definition of all the structures/deserializers/parsers related to Kafka + requests/responses, + - kafka_${MESSAGE_TYPE}_resolver.cc - resolver that is responsible for creation of parsers + defined in ${MESSAGE_TYPE}s.h (it maps request's api key & version to matching parser). + + Templates used are: + - to create '${MESSAGE_TYPE}.h': ${MESSAGE_TYPE}_h.j2, complex_type_template.j2, + request_parser.j2, + - to create 'kafka_${MESSAGE_TYPE}_resolver.cc': kafka_${MESSAGE_TYPE}_resolver_cc.j2, + """ + + type = sys.argv[1] + main_header_file = os.path.abspath(sys.argv[2]) + resolver_cc_file = os.path.abspath(sys.argv[3]) + input_files = sys.argv[4:] + generator.generate_main_code(type, main_header_file, resolver_cc_file, input_files) + + +if __name__ == "__main__": + main() diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/request_parser.j2 b/source/extensions/filters/network/kafka/protocol/request_parser.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/request_parser.j2 rename to source/extensions/filters/network/kafka/protocol/request_parser.j2 diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/requests_h.j2 b/source/extensions/filters/network/kafka/protocol/requests_h.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/requests_h.j2 rename to source/extensions/filters/network/kafka/protocol/requests_h.j2 diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/response_parser.j2 b/source/extensions/filters/network/kafka/protocol/response_parser.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/response_parser.j2 rename to source/extensions/filters/network/kafka/protocol/response_parser.j2 diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/responses_h.j2 b/source/extensions/filters/network/kafka/protocol/responses_h.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/responses_h.j2 rename to source/extensions/filters/network/kafka/protocol/responses_h.j2 diff --git a/source/extensions/filters/network/kafka/serialization/generator.py b/source/extensions/filters/network/kafka/serialization/generator.py new file mode 100755 index 000000000000..574aa3d66de3 --- /dev/null +++ b/source/extensions/filters/network/kafka/serialization/generator.py @@ -0,0 +1,57 @@ +#!/usr/bin/python + +# Main library file containing all the composite deserializer logic. + + +def generate_main_code(serialization_composite_h_file): + """ + Main code generator. + Renders the header file for serialization composites. + The location of output file is provided as argument. + """ + generate_code('serialization_composite_h.j2', serialization_composite_h_file) + + +def generate_test_code(serialization_composite_test_cc_file): + """ + Test code generator. + Renders the test file for serialization composites. + The location of output file is provided as argument. + """ + generate_code('serialization_composite_test_cc.j2', serialization_composite_test_cc_file) + + +def generate_code(template_name, output_file): + """ + Gets definition of structures to render. + Then renders these structures using template provided into provided output file. + """ + field_counts = get_field_counts() + template = RenderingHelper.get_template(template_name) + contents = template.render(counts=field_counts) + with open(output_file, 'w') as fd: + fd.write(contents) + + +def get_field_counts(): + """ + Generate argument counts that should be processed by composite deserializers. + """ + return range(1, 10) + + +class RenderingHelper: + """ + Helper for jinja templates. + """ + + @staticmethod + def get_template(template): + import jinja2 + import os + import sys + # Templates are resolved relatively to main start script, due to main & test templates being + # stored in different directories. + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(searchpath=os.path.dirname(os.path.abspath(sys.argv[0])))) + return env.get_template(template) diff --git a/source/extensions/filters/network/kafka/serialization/launcher.py b/source/extensions/filters/network/kafka/serialization/launcher.py new file mode 100644 index 000000000000..2f63ffb45b96 --- /dev/null +++ b/source/extensions/filters/network/kafka/serialization/launcher.py @@ -0,0 +1,33 @@ +#!/usr/bin/python + +# Launcher for generating composite serializer code. + +import source.extensions.filters.network.kafka.serialization.generator as generator +import sys +import os + + +def main(): + """ + Serialization composite code generator + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Generates main source code files for composite deserializers. + The files are generated, as they are extremely repetitive (composite deserializer for 0..9 + sub-deserializers). + + Usage: + launcher.py LOCATION_OF_OUTPUT_FILE + where: + LOCATION_OF_OUTPUT_FILE : location of 'serialization_composite.h'. + + Creates 'serialization_composite.h' - header with declarations of + CompositeDeserializerWith???Delegates classes. + + Template used: 'serialization_composite_h.j2'. + """ + serialization_composite_h_file = os.path.abspath(sys.argv[1]) + generator.generate_main_code(serialization_composite_h_file) + + +if __name__ == "__main__": + main() diff --git a/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_h.j2 b/source/extensions/filters/network/kafka/serialization/serialization_composite_h.j2 similarity index 100% rename from source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_h.j2 rename to source/extensions/filters/network/kafka/serialization/serialization_composite_h.j2 diff --git a/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_generator.py b/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_generator.py deleted file mode 100755 index 100bf7593c71..000000000000 --- a/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_generator.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/python - - -def main(): - """ - Serialization composite generator script - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Generates main&test source code files for composite deserializers. - The files are generated, as they are extremely repetitive (composite deserializer for 0..9 - sub-deserializers). - - Usage: - serialization_composite_generator.py COMMAND LOCATION_OF_OUTPUT_FILE - where: - COMMAND : 'generate-source', to generate source files, - 'generate-test', to generate test files. - LOCATION_OF_OUTPUT_FILE : if generate-source: location of 'serialization_composite.h', - if generate-test: location of 'serialization_composite_test.cc'. - - When generating source code, it creates: - - serialization_composite.h - header with declarations of CompositeDeserializerWith???Delegates - classes. - When generating test code, it creates: - - serialization_composite_test.cc - tests for these classes. - - Templates used are: - - to create 'serialization_composite.h': serialization_composite_h.j2, - - to create 'serialization_composite_test.cc': serialization_composite_test_cc.j2. - """ - - import sys - import os - - command = sys.argv[1] - if 'generate-source' == command: - serialization_composite_h_file = os.path.abspath(sys.argv[2]) - elif 'generate-test' == command: - serialization_composite_test_cc_file = os.path.abspath(sys.argv[2]) - else: - raise ValueError('invalid command: ' + command) - - import re - import json - - # Number of fields deserialized by each deserializer class. - field_counts = range(1, 10) - - # Generate main source code. - if 'generate-source' == command: - template = RenderingHelper.get_template('serialization_composite_h.j2') - contents = template.render(counts=field_counts) - with open(serialization_composite_h_file, 'w') as fd: - fd.write(contents) - - # Generate test code. - if 'generate-test' == command: - template = RenderingHelper.get_template('serialization_composite_test_cc.j2') - contents = template.render(counts=field_counts) - with open(serialization_composite_test_cc_file, 'w') as fd: - fd.write(contents) - - -class RenderingHelper: - """ - Helper for jinja templates. - """ - - @staticmethod - def get_template(template): - import jinja2 - import os - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(searchpath=os.path.dirname(os.path.abspath(__file__)))) - return env.get_template(template) - - -if __name__ == "__main__": - main() diff --git a/test/extensions/filters/network/kafka/BUILD b/test/extensions/filters/network/kafka/BUILD index 00a2d0f4d70e..3041acfe2d91 100644 --- a/test/extensions/filters/network/kafka/BUILD +++ b/test/extensions/filters/network/kafka/BUILD @@ -55,15 +55,26 @@ envoy_extension_cc_test( ) genrule( - name = "serialization_composite_test_generator", + name = "serialization_composite_generated_tests", srcs = [], outs = ["external/serialization_composite_test.cc"], cmd = """ - ./$(location //source/extensions/filters/network/kafka:serialization_composite_generator) \ - generate-test $(location external/serialization_composite_test.cc) + ./$(location :serialization_composite_test_generator_bin) \ + $(location external/serialization_composite_test.cc) """, tools = [ - "//source/extensions/filters/network/kafka:serialization_composite_generator", + ":serialization_composite_test_generator_bin", + ], +) + +py_binary( + name = "serialization_composite_test_generator_bin", + srcs = ["serialization/launcher.py"], + data = glob(["serialization/*.j2"]), + main = "serialization/launcher.py", + deps = [ + "//source/extensions/filters/network/kafka:serialization_composite_generator_lib", + "@com_github_pallets_jinja//:jinja2", ], ) @@ -126,7 +137,7 @@ envoy_extension_cc_test( ) genrule( - name = "request_test_generator", + name = "request_generated_tests", srcs = [ "@kafka_source//:request_protocol_files", ], @@ -135,13 +146,12 @@ genrule( "external/request_codec_request_test.cc", ], cmd = """ - ./$(location //source/extensions/filters/network/kafka:kafka_code_generator) \ - generate-test request \ + ./$(location :kafka_protocol_test_generator_bin) request \ $(location external/requests_test.cc) $(location external/request_codec_request_test.cc) \ $(SRCS) """, tools = [ - "//source/extensions/filters/network/kafka:kafka_code_generator", + ":kafka_protocol_test_generator_bin", ], ) @@ -204,7 +214,7 @@ envoy_extension_cc_test( ) genrule( - name = "response_test_generator", + name = "response_generated_tests", srcs = [ "@kafka_source//:response_protocol_files", ], @@ -213,12 +223,23 @@ genrule( "external/response_codec_response_test.cc", ], cmd = """ - ./$(location //source/extensions/filters/network/kafka:kafka_code_generator) \ - generate-test response \ - $(location external/responses_test.cc) $(location external/response_codec_response_test.cc) \ + ./$(location :kafka_protocol_test_generator_bin) response \ + $(location external/responses_test.cc) \ + $(location external/response_codec_response_test.cc) \ $(SRCS) """, tools = [ - "//source/extensions/filters/network/kafka:kafka_code_generator", + ":kafka_protocol_test_generator_bin", + ], +) + +py_binary( + name = "kafka_protocol_test_generator_bin", + srcs = ["protocol/launcher.py"], + data = glob(["protocol/*.j2"]), + main = "protocol/launcher.py", + deps = [ + "//source/extensions/filters/network/kafka:kafka_protocol_generator_lib", + "@com_github_pallets_jinja//:jinja2", ], ) diff --git a/test/extensions/filters/network/kafka/protocol/launcher.py b/test/extensions/filters/network/kafka/protocol/launcher.py new file mode 100644 index 000000000000..737859a14429 --- /dev/null +++ b/test/extensions/filters/network/kafka/protocol/launcher.py @@ -0,0 +1,44 @@ +#!/usr/bin/python + +# Launcher for generating Kafka protocol tests. + +import source.extensions.filters.network.kafka.protocol.generator as generator +import sys +import os + + +def main(): + """ + Kafka test generator script + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Generates tests from Kafka protocol specification. + + Usage: + launcher.py MESSAGE_TYPE OUTPUT_FILES INPUT_FILES + where: + MESSAGE_TYPE : 'request' or 'response' + OUTPUT_FILES : location of 'requests_test.cc'/'responses_test.cc' and + 'request_codec_request_test.cc' / 'response_codec_response_test.cc'. + INPUT_FILES: Kafka protocol json files to be processed. + + Kafka spec files are provided in Kafka clients jar file. + + Files created are: + - ${MESSAGE_TYPE}s_test.cc - serialization/deserialization tests for kafka structures, + - ${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test.cc - integration tests involving coded for all + request/response operations. + + Templates used are: + - to create '${MESSAGE_TYPE}s_test.cc': ${MESSAGE_TYPE}s_test_cc.j2, + - to create '${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test.cc' - + ${MESSAGE_TYPE}_codec_${MESSAGE_TYPE}_test_cc.j2. + """ + type = sys.argv[1] + header_test_cc_file = os.path.abspath(sys.argv[2]) + codec_test_cc_file = os.path.abspath(sys.argv[3]) + input_files = sys.argv[4:] + generator.generate_test_code(type, header_test_cc_file, codec_test_cc_file, input_files) + + +if __name__ == "__main__": + main() diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/request_codec_request_test_cc.j2 b/test/extensions/filters/network/kafka/protocol/request_codec_request_test_cc.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/request_codec_request_test_cc.j2 rename to test/extensions/filters/network/kafka/protocol/request_codec_request_test_cc.j2 diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/requests_test_cc.j2 b/test/extensions/filters/network/kafka/protocol/requests_test_cc.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/requests_test_cc.j2 rename to test/extensions/filters/network/kafka/protocol/requests_test_cc.j2 diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/response_codec_response_test_cc.j2 b/test/extensions/filters/network/kafka/protocol/response_codec_response_test_cc.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/response_codec_response_test_cc.j2 rename to test/extensions/filters/network/kafka/protocol/response_codec_response_test_cc.j2 diff --git a/source/extensions/filters/network/kafka/protocol_code_generator/responses_test_cc.j2 b/test/extensions/filters/network/kafka/protocol/responses_test_cc.j2 similarity index 100% rename from source/extensions/filters/network/kafka/protocol_code_generator/responses_test_cc.j2 rename to test/extensions/filters/network/kafka/protocol/responses_test_cc.j2 diff --git a/test/extensions/filters/network/kafka/serialization/launcher.py b/test/extensions/filters/network/kafka/serialization/launcher.py new file mode 100644 index 000000000000..223e56ef3e90 --- /dev/null +++ b/test/extensions/filters/network/kafka/serialization/launcher.py @@ -0,0 +1,32 @@ +#!/usr/bin/python + +# Launcher for generating composite serializer tests. + +import source.extensions.filters.network.kafka.serialization.generator as generator +import sys +import os + + +def main(): + """ + Serialization composite test generator + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Generates test source files for composite deserializers. + The files are generated, as they are extremely repetitive (tests for composite deserializer + for 0..9 sub-deserializers). + + Usage: + launcher.py LOCATION_OF_OUTPUT_FILE + where: + LOCATION_OF_OUTPUT_FILE : location of 'serialization_composite_test.cc'. + + Creates 'serialization_composite_test.cc' - tests composite deserializers. + + Template used is 'serialization_composite_test_cc.j2'. + """ + serialization_composite_test_cc_file = os.path.abspath(sys.argv[1]) + generator.generate_test_code(serialization_composite_test_cc_file) + + +if __name__ == "__main__": + main() diff --git a/source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 b/test/extensions/filters/network/kafka/serialization/serialization_composite_test_cc.j2 similarity index 100% rename from source/extensions/filters/network/kafka/serialization_code_generator/serialization_composite_test_cc.j2 rename to test/extensions/filters/network/kafka/serialization/serialization_composite_test_cc.j2 diff --git a/tools/check_format.py b/tools/check_format.py index 8a1de0b673aa..40770ff3b7f6 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -148,8 +148,8 @@ "extensions/filters/network/ext_authz", "extensions/filters/network/redis_proxy", "extensions/filters/network/kafka", - "extensions/filters/network/kafka/protocol_code_generator", - "extensions/filters/network/kafka/serialization_code_generator", + "extensions/filters/network/kafka/protocol", + "extensions/filters/network/kafka/serialization", "extensions/filters/network/mongo_proxy", "extensions/filters/network/common", "extensions/filters/network/common/redis", From fea49ad7ed9887893bcd00c7cd53705022313582 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Sat, 29 Jun 2019 09:51:48 -0700 Subject: [PATCH 082/542] build: avoid referring to //source/... (#7424) Using //source/... instead of //source/exe:envoy-static results in all targets under //source/... and their dependencies being built, even when disabled on a given platform and/or using various flags. Signed-off-by: Piotr Sikora --- bazel/README.md | 8 ++++---- ci/do_ci.sh | 2 +- ci/mac_ci_steps.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bazel/README.md b/bazel/README.md index 0eba43900fa5..357a1b802f2f 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -483,7 +483,7 @@ resources, you can override Bazel's default job parallelism determination with `--jobs=N` to restrict the build to at most `N` simultaneous jobs, e.g.: ``` -bazel build --jobs=2 //source/... +bazel build --jobs=2 //source/exe:envoy-static ``` # Debugging the Bazel build @@ -492,19 +492,19 @@ When trying to understand what Bazel is doing, the `-s` and `--explain` options are useful. To have Bazel provide verbose output on which commands it is executing: ``` -bazel build -s //source/... +bazel build -s //source/exe:envoy-static ``` To have Bazel emit to a text file the rationale for rebuilding a target: ``` -bazel build --explain=file.txt //source/... +bazel build --explain=file.txt //source/exe:envoy-static ``` To get more verbose explanations: ``` -bazel build --explain=file.txt --verbose_explanations //source/... +bazel build --explain=file.txt --verbose_explanations //source/exe:envoy-static ``` # Resolving paths in bazel build output diff --git a/ci/do_ci.sh b/ci/do_ci.sh index e048a8abe920..ccbe79643107 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -107,7 +107,7 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then if [[ "$TEST_TARGETS" == "//test/..." ]]; then # We have various test binaries in the test directory such as tools, benchmarks, etc. We # run a build pass to make sure they compile. - bazel build ${BAZEL_BUILD_OPTIONS} -c opt //include/... //source/... //test/... + bazel build ${BAZEL_BUILD_OPTIONS} -c opt //include/... //source/exe:envoy-static //test/... fi # Now run all of the tests which should already be compiled. bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c opt ${TEST_TARGETS} diff --git a/ci/mac_ci_steps.sh b/ci/mac_ci_steps.sh index 351282757be7..7d322dafee2a 100755 --- a/ci/mac_ci_steps.sh +++ b/ci/mac_ci_steps.sh @@ -29,5 +29,5 @@ else TEST_TARGETS=//test/... fi -bazel build ${BAZEL_BUILD_OPTIONS} //source/... ${TEST_TARGETS} +bazel build ${BAZEL_BUILD_OPTIONS} //source/exe:envoy-static ${TEST_TARGETS} bazel test ${BAZEL_BUILD_OPTIONS} ${TEST_TARGETS} From ae02dc6bdd5c5ea61c3869395d81689e34988156 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Sun, 30 Jun 2019 21:38:11 -0400 Subject: [PATCH 083/542] stats: replace std::shared_ptr usage for stats with a custom version that uses the internal ref-count we already store. (#7364) * stats: replace std::shared_ptr usage for stats with a custom version that uses the internal ref-count we already store. (#7364) Signed-off-by: Joshua Marantz --- include/envoy/stats/BUILD | 7 + include/envoy/stats/histogram.h | 7 +- include/envoy/stats/refcount_ptr.h | 163 ++++++++++++++++++ include/envoy/stats/stats.h | 10 +- source/common/stats/heap_stat_data.cc | 91 ++++++---- source/common/stats/heap_stat_data.h | 23 ++- source/common/stats/isolated_store_impl.cc | 7 +- source/common/stats/isolated_store_impl.h | 23 +-- source/common/stats/null_counter.h | 8 + source/common/stats/null_gauge.h | 8 + source/common/stats/thread_local_store.cc | 23 +-- source/common/stats/thread_local_store.h | 16 +- source/server/http/admin.cc | 14 +- source/server/http/admin.h | 14 +- test/common/stats/BUILD | 6 + test/common/stats/heap_stat_data_test.cc | 18 +- test/common/stats/isolated_store_impl_test.cc | 8 +- test/common/stats/refcount_ptr_test.cc | 73 ++++++++ test/common/stats/thread_local_store_test.cc | 14 +- test/integration/stats_integration_test.cc | 13 +- test/mocks/stats/mocks.h | 38 ++-- test/server/http/admin_test.cc | 17 +- tools/spelling_dictionary.txt | 2 + 23 files changed, 472 insertions(+), 131 deletions(-) create mode 100644 include/envoy/stats/refcount_ptr.h create mode 100644 test/common/stats/refcount_ptr_test.cc diff --git a/include/envoy/stats/BUILD b/include/envoy/stats/BUILD index aeef2282b823..2dcf68988def 100644 --- a/include/envoy/stats/BUILD +++ b/include/envoy/stats/BUILD @@ -8,6 +8,12 @@ load( envoy_package() +envoy_cc_library( + name = "refcount_ptr_interface", + hdrs = ["refcount_ptr.h"], + deps = ["//source/common/common:assert_lib"], +) + # TODO(jmarantz): atomize the build rules to match the include files. envoy_cc_library( name = "stats_interface", @@ -24,6 +30,7 @@ envoy_cc_library( "tag_producer.h", ], deps = [ + ":refcount_ptr_interface", ":symbol_table_interface", "//include/envoy/common:interval_set_interface", ], diff --git a/include/envoy/stats/histogram.h b/include/envoy/stats/histogram.h index 6aa312a62572..6ff5505fc6fa 100644 --- a/include/envoy/stats/histogram.h +++ b/include/envoy/stats/histogram.h @@ -5,6 +5,7 @@ #include #include "envoy/common/pure.h" +#include "envoy/stats/refcount_ptr.h" #include "envoy/stats/stats.h" namespace Envoy { @@ -71,7 +72,7 @@ class HistogramStatistics { * class (Sinks, in particular) will need to explicitly differentiate between histograms * representing durations and histograms representing other types of data. */ -class Histogram : public virtual Metric { +class Histogram : public virtual Metric, public RefcountHelper { public: ~Histogram() override = default; @@ -81,7 +82,7 @@ class Histogram : public virtual Metric { virtual void recordValue(uint64_t value) PURE; }; -using HistogramSharedPtr = std::shared_ptr; +using HistogramSharedPtr = RefcountPtr; /** * A histogram that is stored in main thread and provides summary view of the histogram. @@ -117,7 +118,7 @@ class ParentHistogram : public virtual Histogram { virtual const std::string bucketSummary() const PURE; }; -using ParentHistogramSharedPtr = std::shared_ptr; +using ParentHistogramSharedPtr = RefcountPtr; } // namespace Stats } // namespace Envoy diff --git a/include/envoy/stats/refcount_ptr.h b/include/envoy/stats/refcount_ptr.h new file mode 100644 index 000000000000..437f7b40cce2 --- /dev/null +++ b/include/envoy/stats/refcount_ptr.h @@ -0,0 +1,163 @@ +#pragma once + +#include + +#include "envoy/common/pure.h" + +#include "common/common/assert.h" + +namespace Envoy { +namespace Stats { + +// Implements a reference-counted pointer to a class, so that its external usage +// model is identical to std::shared_ptr, but the reference count itself is held +// in the class. The class is expected to implement three methods: +// void incRefCount() +// bool decRefCount() -- returns true if the reference count goes to zero. +// uint32_t use_count() +// It may implement them by delegating to RefcountHelper (see below), or by +// inheriting from RefcountInterface (see below). +// +// TODO(jmarantz): replace this with an absl or std implementation when +// available. See https://github.com/abseil/abseil-cpp/issues/344, issued June +// 26, 2019, and http://wg21.link/p0406, issued 2017. Note that the turnaround +// time for getting an absl API added is measurable in months, and for a std +// API in years. +template class RefcountPtr { +public: + RefcountPtr() : ptr_(nullptr) {} + + // Constructing a reference-counted object from a pointer; this is safe to + // do when the reference-count is held in the object. For example, this code + // crashes: + // { + // std::shared_ptr a = std::make_shared("x"); + // std::shared_ptr b(a.get()); + // } + // whereas the analogous code for RefcountPtr works fine. + RefcountPtr(T* ptr) : ptr_(ptr) { + if (ptr_ != nullptr) { + ptr_->incRefCount(); + } + } + + RefcountPtr(const RefcountPtr& src) : RefcountPtr(src.get()) {} + + // Constructor for up-casting reference-counted pointers. This doesn't change + // the underlying object; it just upcasts the DerivedClass* in src.ptr_ to a + // BaseClass* for assignment to this->ptr_. For usage of this to compile, + // DerivedClass* must be assignable to BaseClass* without explicit casts. + template + RefcountPtr(const RefcountPtr& src) : RefcountPtr(src.get()) {} + + // Move-construction is used by absl::flat_hash_map during resizes. + RefcountPtr(RefcountPtr&& src) noexcept : ptr_(src.ptr_) { src.ptr_ = nullptr; } + + RefcountPtr& operator=(const RefcountPtr& src) { + if (&src != this && src.ptr_ != ptr_) { + resetInternal(); + ptr_ = src.get(); + if (ptr_ != nullptr) { + ptr_->incRefCount(); + } + } + return *this; + } + + // Move-assignment is used during std::vector resizes. + RefcountPtr& operator=(RefcountPtr&& src) noexcept { + if (&src != this && src.ptr_ != ptr_) { + resetInternal(); + ptr_ = src.ptr_; + src.ptr_ = nullptr; + } + return *this; + } + + ~RefcountPtr() { resetInternal(); } + + // Implements required subset of the shared_ptr API; + // see https://en.cppreference.com/w/cpp/memory/shared_ptr for details. + T* operator->() const { return ptr_; } + T* get() const { return ptr_; } + T& operator*() const { return *ptr_; } + operator bool() const { return ptr_ != nullptr; } + bool operator==(const T* ptr) const { return ptr_ == ptr; } + bool operator!=(const T* ptr) const { return ptr_ != ptr; } + bool operator==(const RefcountPtr& a) const { return ptr_ == a.ptr_; } + bool operator!=(const RefcountPtr& a) const { return ptr_ != a.ptr_; } + uint32_t use_count() const { return ptr_->use_count(); } + void reset() { + resetInternal(); + ptr_ = nullptr; + } + +private: + // Like reset() but does not bother to clear ptr_, as it is about to be + // overwritten or destroyed. + void resetInternal() { + if (ptr_ != nullptr && ptr_->decRefCount()) { + delete ptr_; + } + } + + T* ptr_; +}; + +template static bool operator==(std::nullptr_t, const RefcountPtr& a) { + return a == nullptr; +} +template static bool operator!=(std::nullptr_t, const RefcountPtr& a) { + return a != nullptr; +} + +// Helper interface for classes to derive from, enabling implementation of the +// three methods as part of derived classes. It is not necessary to inherit from +// this interface to wrap a class in RefcountPtr; instead the class can just +// implement the same method. +class RefcountInterface { +public: + virtual ~RefcountInterface() = default; + + /** + * Increments the reference count. + */ + virtual void incRefCount() PURE; + + /** + * Decrements the reference count. + * @return true if the reference count has gone to zero, so the object should be freed. + */ + virtual bool decRefCount() PURE; + + /** + * @return the number of references to the object. + */ + virtual uint32_t use_count() const PURE; +}; + +// Delegation helper for RefcountPtr. This can be instantiated in a class, but +// explicit delegation will be needed for each of the three methods. +struct RefcountHelper { + // Implements the RefcountInterface API. + void incRefCount() { + // Note: The ++ref_count_ here and --ref_count_ below have implicit memory + // orderings of sequentially consistent. Relaxed on addition and + // acquire/release on subtraction is the typical use for reference + // counting. On x86, the difference in instruction count is minimal, but the + // savings are greater on other platforms. + // + // https://www.boost.org/doc/libs/1_69_0/doc/html/atomic/usage_examples.html#boost_atomic.usage_examples.example_reference_counters + ++ref_count_; + } + bool decRefCount() { + ASSERT(ref_count_ >= 1); + return --ref_count_ == 0; + } + uint32_t use_count() const { return ref_count_; } + + std::atomic ref_count_{0}; +}; + +} // namespace Stats +} // namespace Envoy diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index ac302a08bb76..116fdd86028f 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -6,6 +6,7 @@ #include #include "envoy/common/pure.h" +#include "envoy/stats/refcount_ptr.h" #include "envoy/stats/symbol_table.h" #include "absl/strings/string_view.h" @@ -109,9 +110,10 @@ class Metric { * global counter as well as periodic counter. Calling latch() returns the periodic counter and * clears it. */ -class Counter : public virtual Metric { +class Counter : public virtual Metric, public RefcountInterface { public: ~Counter() override = default; + virtual void add(uint64_t amount) PURE; virtual void inc() PURE; virtual uint64_t latch() PURE; @@ -119,12 +121,12 @@ class Counter : public virtual Metric { virtual uint64_t value() const PURE; }; -using CounterSharedPtr = std::shared_ptr; +using CounterSharedPtr = RefcountPtr; /** * A gauge that can both increment and decrement. */ -class Gauge : public virtual Metric { +class Gauge : public virtual Metric, public RefcountInterface { public: enum class ImportMode { Uninitialized, // Gauge was discovered during hot-restart transfer. @@ -158,7 +160,7 @@ class Gauge : public virtual Metric { virtual void mergeImportMode(ImportMode import_mode) PURE; }; -using GaugeSharedPtr = std::shared_ptr; +using GaugeSharedPtr = RefcountPtr; } // namespace Stats } // namespace Envoy diff --git a/source/common/stats/heap_stat_data.cc b/source/common/stats/heap_stat_data.cc index 8a26ceb2dd07..228d8331de9b 100644 --- a/source/common/stats/heap_stat_data.cc +++ b/source/common/stats/heap_stat_data.cc @@ -20,7 +20,10 @@ namespace Envoy { namespace Stats { -HeapStatDataAllocator::~HeapStatDataAllocator() { ASSERT(stats_.empty()); } +HeapStatDataAllocator::~HeapStatDataAllocator() { + ASSERT(counters_.empty()); + ASSERT(gauges_.empty()); +} HeapStatData* HeapStatData::alloc(StatName stat_name, SymbolTable& symbol_table) { symbol_table.incRefCount(stat_name); @@ -28,47 +31,31 @@ HeapStatData* HeapStatData::alloc(StatName stat_name, SymbolTable& symbol_table) } void HeapStatData::free(SymbolTable& symbol_table) { + ASSERT(ref_count_ == 0); symbol_table.free(statName()); delete this; } -HeapStatData& HeapStatDataAllocator::alloc(StatName name) { - using HeapStatDataFreeFn = std::function; - std::unique_ptr data_ptr( - HeapStatData::alloc(name, symbolTable()), - [this](HeapStatData* d) { d->free(symbolTable()); }); - Thread::ReleasableLockGuard lock(mutex_); - auto ret = stats_.insert(data_ptr.get()); - HeapStatData* existing_data = *ret.first; - lock.release(); - - if (ret.second) { - return *data_ptr.release(); - } - ++existing_data->ref_count_; - return *existing_data; +void HeapStatDataAllocator::removeCounterFromSet(Counter* counter) { + Thread::LockGuard lock(mutex_); + const size_t count = counters_.erase(counter->statName()); + ASSERT(count == 1); } -void HeapStatDataAllocator::free(HeapStatData& data) { - ASSERT(data.ref_count_ > 0); - if (--data.ref_count_ > 0) { - return; - } - - { - Thread::LockGuard lock(mutex_); - size_t key_removed = stats_.erase(&data); - ASSERT(key_removed == 1); - } - - data.free(symbolTable()); +void HeapStatDataAllocator::removeGaugeFromSet(Gauge* gauge) { + Thread::LockGuard lock(mutex_); + const size_t count = gauges_.erase(gauge->statName()); + ASSERT(count == 1); } #ifndef ENVOY_CONFIG_COVERAGE void HeapStatDataAllocator::debugPrint() { Thread::LockGuard lock(mutex_); - for (HeapStatData* heap_stat_data : stats_) { - ENVOY_LOG_MISC(info, "{}", symbolTable().toString(heap_stat_data->statName())); + for (Counter* counter : counters_) { + ENVOY_LOG_MISC(info, "counter: {}", symbolTable().toString(counter->statName())); + } + for (Gauge* gauge : gauges_) { + ENVOY_LOG_MISC(info, "gauge: {}", symbolTable().toString(gauge->statName())); } } #endif @@ -85,7 +72,8 @@ class CounterImpl : public Counter, public MetricImpl { // alternative would be to store the SymbolTable reference in the // MetricImpl, costing 8 bytes per stat. MetricImpl::clear(); - alloc_.free(data_); + alloc_.removeCounterFromSet(this); + data_.free(symbolTable()); } // Stats::Counter @@ -103,6 +91,14 @@ class CounterImpl : public Counter, public MetricImpl { SymbolTable& symbolTable() override { return alloc_.symbolTable(); } StatName statName() const override { return data_.statName(); } + // RefcountInterface + void incRefCount() override { ++data_.ref_count_; } + bool decRefCount() override { + ASSERT(data_.ref_count_ >= 1); + return --data_.ref_count_ == 0; + } + uint32_t use_count() const override { return data_.ref_count_; } + private: HeapStatData& data_; HeapStatDataAllocator& alloc_; @@ -135,7 +131,8 @@ class GaugeImpl : public Gauge, public MetricImpl { // alternative would be to store the SymbolTable reference in the // MetricImpl, costing 8 bytes per stat. MetricImpl::clear(); - alloc_.free(data_); + alloc_.removeGaugeFromSet(this); + data_.free(symbolTable()); } // Stats::Gauge @@ -196,6 +193,14 @@ class GaugeImpl : public Gauge, public MetricImpl { SymbolTable& symbolTable() override { return alloc_.symbolTable(); } StatName statName() const override { return data_.statName(); } + // RefcountInterface + void incRefCount() override { ++data_.ref_count_; } + bool decRefCount() override { + ASSERT(data_.ref_count_ >= 1); + return --data_.ref_count_ == 0; + } + uint32_t use_count() const override { return data_.ref_count_; } + private: HeapStatData& data_; HeapStatDataAllocator& alloc_; @@ -204,13 +209,29 @@ class GaugeImpl : public Gauge, public MetricImpl { CounterSharedPtr HeapStatDataAllocator::makeCounter(StatName name, absl::string_view tag_extracted_name, const std::vector& tags) { - return std::make_shared(alloc(name), *this, tag_extracted_name, tags); + Thread::LockGuard lock(mutex_); + auto iter = counters_.find(name); + if (iter != counters_.end()) { + return CounterSharedPtr(*iter); + } + auto counter = CounterSharedPtr( + new CounterImpl(*HeapStatData::alloc(name, symbolTable()), *this, tag_extracted_name, tags)); + counters_.insert(counter.get()); + return counter; } GaugeSharedPtr HeapStatDataAllocator::makeGauge(StatName name, absl::string_view tag_extracted_name, const std::vector& tags, Gauge::ImportMode import_mode) { - return std::make_shared(alloc(name), *this, tag_extracted_name, tags, import_mode); + Thread::LockGuard lock(mutex_); + auto iter = gauges_.find(name); + if (iter != gauges_.end()) { + return GaugeSharedPtr(*iter); + } + auto gauge = GaugeSharedPtr(new GaugeImpl(*HeapStatData::alloc(name, symbolTable()), *this, + tag_extracted_name, tags, import_mode)); + gauges_.insert(gauge.get()); + return gauge; } } // namespace Stats diff --git a/source/common/stats/heap_stat_data.h b/source/common/stats/heap_stat_data.h index 9536d0e00849..0be761b14095 100644 --- a/source/common/stats/heap_stat_data.h +++ b/source/common/stats/heap_stat_data.h @@ -37,7 +37,7 @@ struct HeapStatData : public InlineStorage { std::atomic value_{0}; std::atomic pending_increment_{0}; std::atomic flags_{0}; - std::atomic ref_count_{1}; + std::atomic ref_count_{0}; SymbolTable::Storage symbol_storage_; // This is a 'using' nickname for uint8_t[]. }; @@ -47,7 +47,8 @@ class HeapStatDataAllocator : public StatDataAllocator { ~HeapStatDataAllocator() override; HeapStatData& alloc(StatName name); - void free(HeapStatData& data); + void removeCounterFromSet(Counter* counter); + void removeGaugeFromSet(Gauge* gauge); // StatDataAllocator CounterSharedPtr makeCounter(StatName name, absl::string_view tag_extracted_name, @@ -63,17 +64,27 @@ class HeapStatDataAllocator : public StatDataAllocator { private: struct HeapStatHash { - size_t operator()(const HeapStatData* a) const { return a->hash(); } + using is_transparent = void; + size_t operator()(const Metric* a) const { return a->statName().hash(); } + size_t operator()(StatName a) const { return a.hash(); } }; + struct HeapStatCompare { - bool operator()(const HeapStatData* a, const HeapStatData* b) const { return *a == *b; } + using is_transparent = void; + bool operator()(const Metric* a, const Metric* b) const { + return a->statName() == b->statName(); + } + bool operator()(StatName a, const Metric* b) const { return a == b->statName(); } + bool operator()(const Metric* a, StatName b) const { return a->statName() == b; } }; // An unordered set of HeapStatData pointers which keys off the key() // field in each object. This necessitates a custom comparator and hasher, which key off of the // StatNamePtr's own StatNamePtrHash and StatNamePtrCompare operators. - using StatSet = absl::flat_hash_set; - StatSet stats_ GUARDED_BY(mutex_); + template + using StatSet = absl::flat_hash_set; + StatSet counters_ GUARDED_BY(mutex_); + StatSet gauges_ GUARDED_BY(mutex_); SymbolTable& symbol_table_; diff --git a/source/common/stats/isolated_store_impl.cc b/source/common/stats/isolated_store_impl.cc index dcbc0eb338de..83a99379c587 100644 --- a/source/common/stats/isolated_store_impl.cc +++ b/source/common/stats/isolated_store_impl.cc @@ -32,10 +32,11 @@ IsolatedStoreImpl::IsolatedStoreImpl(SymbolTable& symbol_table) import_mode); }), histograms_([this](StatName name) -> HistogramSharedPtr { - return std::make_shared(name, *this, alloc_.symbolTable().toString(name), - std::vector()); + return HistogramSharedPtr(new HistogramImpl( + name, *this, alloc_.symbolTable().toString(name), std::vector())); }), - null_gauge_(symbol_table) {} + null_counter_(new NullCounterImpl(symbol_table)), + null_gauge_(new NullGaugeImpl(symbol_table)) {} ScopePtr IsolatedStoreImpl::createScope(const std::string& name) { return std::make_unique(name, *this); diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index 867946e4382c..ba492db0c040 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -10,6 +10,7 @@ #include "common/common/utility.h" #include "common/stats/heap_stat_data.h" +#include "common/stats/null_counter.h" #include "common/stats/null_gauge.h" #include "common/stats/store_impl.h" #include "common/stats/symbol_table_impl.h" @@ -25,8 +26,8 @@ namespace Stats { */ template class IsolatedStatsCache { public: - using Allocator = std::function(StatName name)>; - using AllocatorImportMode = std::function(StatName, Gauge::ImportMode)>; + using Allocator = std::function(StatName name)>; + using AllocatorImportMode = std::function(StatName, Gauge::ImportMode)>; IsolatedStatsCache(Allocator alloc) : alloc_(alloc) {} IsolatedStatsCache(AllocatorImportMode alloc) : alloc_import_(alloc) {} @@ -37,7 +38,7 @@ template class IsolatedStatsCache { return *stat->second; } - std::shared_ptr new_stat = alloc_(name); + RefcountPtr new_stat = alloc_(name); stats_.emplace(new_stat->statName(), new_stat); return *new_stat; } @@ -48,13 +49,13 @@ template class IsolatedStatsCache { return *stat->second; } - std::shared_ptr new_stat = alloc_import_(name, import_mode); + RefcountPtr new_stat = alloc_import_(name, import_mode); stats_.emplace(new_stat->statName(), new_stat); return *new_stat; } - std::vector> toVector() const { - std::vector> vec; + std::vector> toVector() const { + std::vector> vec; vec.reserve(stats_.size()); for (auto& stat : stats_) { vec.push_back(stat.second); @@ -71,10 +72,10 @@ template class IsolatedStatsCache { if (stat == stats_.end()) { return absl::nullopt; } - return std::cref(*stat->second.get()); + return std::cref(*stat->second); } - StatNameHashMap> stats_; + StatNameHashMap> stats_; Allocator alloc_; AllocatorImportMode alloc_import_; }; @@ -93,7 +94,8 @@ class IsolatedStoreImpl : public StoreImpl { gauge.mergeImportMode(import_mode); return gauge; } - NullGaugeImpl& nullGauge(const std::string&) override { return null_gauge_; } + NullCounterImpl& nullCounter() { return *null_counter_; } + NullGaugeImpl& nullGauge(const std::string&) override { return *null_gauge_; } Histogram& histogramFromStatName(StatName name) override { Histogram& histogram = histograms_.get(name); return histogram; @@ -144,7 +146,8 @@ class IsolatedStoreImpl : public StoreImpl { IsolatedStatsCache counters_; IsolatedStatsCache gauges_; IsolatedStatsCache histograms_; - NullGaugeImpl null_gauge_; + RefcountPtr null_counter_; + RefcountPtr null_gauge_; }; } // namespace Stats diff --git a/source/common/stats/null_counter.h b/source/common/stats/null_counter.h index f0ca00163cc3..777e6a34c509 100644 --- a/source/common/stats/null_counter.h +++ b/source/common/stats/null_counter.h @@ -27,6 +27,14 @@ class NullCounterImpl : public Counter, NullMetricImpl { uint64_t latch() override { return 0; } void reset() override {} uint64_t value() const override { return 0; } + + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + +private: + RefcountHelper refcount_helper_; }; } // namespace Stats diff --git a/source/common/stats/null_gauge.h b/source/common/stats/null_gauge.h index 568f0d70510e..f8d231dcf19e 100644 --- a/source/common/stats/null_gauge.h +++ b/source/common/stats/null_gauge.h @@ -30,6 +30,14 @@ class NullGaugeImpl : public Gauge, NullMetricImpl { uint64_t value() const override { return 0; } ImportMode importMode() const override { return ImportMode::NeverImport; } void mergeImportMode(ImportMode /* import_mode */) override {} + + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + +private: + RefcountHelper refcount_helper_; }; } // namespace Stats diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 334f018666d9..741a4caf006b 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -313,9 +313,9 @@ bool ThreadLocalStoreImpl::checkAndRememberRejection(StatName name, template StatType& ThreadLocalStoreImpl::ScopeImpl::safeMakeStat( - StatName name, StatMap>& central_cache_map, + StatName name, StatMap>& central_cache_map, StatNameStorageSet& central_rejected_stats, MakeStatFn make_stat, - StatMap>* tls_cache, StatNameHashSet* tls_rejected_stats, + StatMap>* tls_cache, StatNameHashSet* tls_rejected_stats, StatType& null_stat) { if (tls_rejected_stats != nullptr && @@ -335,7 +335,7 @@ StatType& ThreadLocalStoreImpl::ScopeImpl::safeMakeStat( // central store location. It might contain nothing. In this case, we allocate a new stat. Thread::LockGuard lock(parent_.lock_); auto iter = central_cache_map.find(name); - std::shared_ptr* central_ref = nullptr; + RefcountPtr* central_ref = nullptr; if (iter != central_cache_map.end()) { central_ref = &(iter->second); } else if (parent_.checkAndRememberRejection(name, central_rejected_stats, tls_rejected_stats)) { @@ -343,7 +343,7 @@ StatType& ThreadLocalStoreImpl::ScopeImpl::safeMakeStat( return null_stat; } else { TagExtraction extraction(parent_, name); - std::shared_ptr stat = + RefcountPtr stat = make_stat(parent_.alloc_, name, extraction.tagExtractedName(), extraction.tags()); ASSERT(stat != nullptr); central_ref = ¢ral_cache_map[stat->statName()]; @@ -362,7 +362,7 @@ StatType& ThreadLocalStoreImpl::ScopeImpl::safeMakeStat( template absl::optional> ThreadLocalStoreImpl::ScopeImpl::findStatLockHeld( - StatName name, StatMap>& central_cache_map) const { + StatName name, StatMap>& central_cache_map) const { auto iter = central_cache_map.find(name); if (iter == central_cache_map.end()) { return absl::nullopt; @@ -501,8 +501,9 @@ Histogram& ThreadLocalStoreImpl::ScopeImpl::histogramFromStatName(StatName name) return parent_.null_histogram_; } else { TagExtraction extraction(parent_, final_stat_name); - auto stat = std::make_shared( - final_stat_name, parent_, *this, extraction.tagExtractedName(), extraction.tags()); + + RefcountPtr stat(new ParentHistogramImpl( + final_stat_name, parent_, *this, extraction.tagExtractedName(), extraction.tags())); central_ref = ¢ral_cache_.histograms_[stat->statName()]; *central_ref = stat; } @@ -530,8 +531,8 @@ ThreadLocalStoreImpl::ScopeImpl::findHistogram(StatName name) const { return absl::nullopt; } - std::shared_ptr histogram_ref(iter->second); - return std::cref(*histogram_ref.get()); + RefcountPtr histogram_ref(iter->second); + return std::cref(*histogram_ref); } Histogram& ThreadLocalStoreImpl::ScopeImpl::tlsHistogram(StatName name, @@ -554,8 +555,8 @@ Histogram& ThreadLocalStoreImpl::ScopeImpl::tlsHistogram(StatName name, std::vector tags; std::string tag_extracted_name = parent_.tagProducer().produceTags(symbolTable().toString(name), tags); - TlsHistogramSharedPtr hist_tls_ptr = - std::make_shared(name, tag_extracted_name, tags, symbolTable()); + TlsHistogramSharedPtr hist_tls_ptr( + new ThreadLocalHistogramImpl(name, tag_extracted_name, tags, symbolTable())); parent.addTlsHistogram(hist_tls_ptr); diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 97b19109602b..1e679646e50c 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -64,7 +64,7 @@ class ThreadLocalHistogramImpl : public Histogram, public MetricImpl { SymbolTable& symbol_table_; }; -using TlsHistogramSharedPtr = std::shared_ptr; +using TlsHistogramSharedPtr = RefcountPtr; class TlsScope; @@ -115,7 +115,7 @@ class ParentHistogramImpl : public ParentHistogram, public MetricImpl { StatNameStorage name_; }; -using ParentHistogramImplSharedPtr = std::shared_ptr; +using ParentHistogramImplSharedPtr = RefcountPtr; /** * Class used to create ThreadLocalHistogram in the scope. @@ -278,9 +278,9 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo findHistogram(StatName name) const override; template - using MakeStatFn = std::function(StatDataAllocator&, StatName name, - absl::string_view tag_extracted_name, - const std::vector& tags)>; + using MakeStatFn = std::function(StatDataAllocator&, StatName name, + absl::string_view tag_extracted_name, + const std::vector& tags)>; /** * Makes a stat either by looking it up in the central cache, @@ -294,10 +294,10 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo * used if non-empty, or filled in if empty (and non-null). */ template - StatType& safeMakeStat(StatName name, StatMap>& central_cache_map, + StatType& safeMakeStat(StatName name, StatMap>& central_cache_map, StatNameStorageSet& central_rejected_stats, MakeStatFn make_stat, - StatMap>* tls_cache, + StatMap>* tls_cache, StatNameHashSet* tls_rejected_stats, StatType& null_stat); /** @@ -311,7 +311,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo */ template absl::optional> - findStatLockHeld(StatName name, StatMap>& central_cache_map) const; + findStatLockHeld(StatName name, StatMap>& central_cache_map) const; void extractTagsAndTruncate(StatName& name, std::unique_ptr& truncated_name_storage, diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 1423a99c9068..746b5c7c1e48 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -756,13 +756,13 @@ Http::Code AdminImpl::handlerStats(absl::string_view url, Http::HeaderMap& respo std::map all_stats; for (const Stats::CounterSharedPtr& counter : server_.stats().counters()) { - if (shouldShowMetric(counter, used_only, regex)) { + if (shouldShowMetric(*counter, used_only, regex)) { all_stats.emplace(counter->name(), counter->value()); } } for (const Stats::GaugeSharedPtr& gauge : server_.stats().gauges()) { - if (shouldShowMetric(gauge, used_only, regex)) { + if (shouldShowMetric(*gauge, used_only, regex)) { ASSERT(gauge->importMode() != Stats::Gauge::ImportMode::Uninitialized); all_stats.emplace(gauge->name(), gauge->value()); } @@ -790,7 +790,7 @@ Http::Code AdminImpl::handlerStats(absl::string_view url, Http::HeaderMap& respo // implemented this can be switched back to a normal map. std::multimap all_histograms; for (const Stats::ParentHistogramSharedPtr& histogram : server_.stats().histograms()) { - if (shouldShowMetric(histogram, used_only, regex)) { + if (shouldShowMetric(*histogram, used_only, regex)) { all_histograms.emplace(histogram->name(), histogram->quantileSummary()); } } @@ -845,7 +845,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( const bool used_only, const absl::optional& regex) { std::unordered_set metric_type_tracker; for (const auto& counter : counters) { - if (!shouldShowMetric(counter, used_only, regex)) { + if (!shouldShowMetric(*counter, used_only, regex)) { continue; } @@ -859,7 +859,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( } for (const auto& gauge : gauges) { - if (!shouldShowMetric(gauge, used_only, regex)) { + if (!shouldShowMetric(*gauge, used_only, regex)) { continue; } @@ -873,7 +873,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( } for (const auto& histogram : histograms) { - if (!shouldShowMetric(histogram, used_only, regex)) { + if (!shouldShowMetric(*histogram, used_only, regex)) { continue; } @@ -941,7 +941,7 @@ AdminImpl::statsAsJson(const std::map& all_stats, rapidjson::Value histogram_array(rapidjson::kArrayType); for (const Stats::ParentHistogramSharedPtr& histogram : all_histograms) { - if (shouldShowMetric(histogram, used_only, regex)) { + if (shouldShowMetric(*histogram, used_only, regex)) { if (!found_used_histogram) { // It is not possible for the supported quantiles to differ across histograms, so it is ok // to send them once. diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 2cc4b0a06427..2991515cfef3 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -223,10 +223,11 @@ class AdminImpl : public Admin, void writeListenersAsJson(Buffer::Instance& response); void writeListenersAsText(Buffer::Instance& response); - static bool shouldShowMetric(const std::shared_ptr& metric, const bool used_only, + template + static bool shouldShowMetric(const StatType& metric, const bool used_only, const absl::optional& regex) { - return ((!used_only || metric->used()) && - (!regex.has_value() || std::regex_search(metric->name(), regex.value()))); + return ((!used_only || metric.used()) && + (!regex.has_value() || std::regex_search(metric.name(), regex.value()))); } static std::string statsAsJson(const std::map& all_stats, const std::vector& all_histograms, @@ -449,10 +450,11 @@ class PrometheusStatsFormatter { * Determine whether a metric has never been emitted and choose to * not show it if we only wanted used metrics. */ - static bool shouldShowMetric(const std::shared_ptr& metric, const bool used_only, + template + static bool shouldShowMetric(const StatType& metric, const bool used_only, const absl::optional& regex) { - return ((!used_only || metric->used()) && - (!regex.has_value() || std::regex_search(metric->name(), regex.value()))); + return ((!used_only || metric.used()) && + (!regex.has_value() || std::regex_search(metric.name(), regex.value()))); } }; diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 49f3a6168aaf..76e4e949a2a1 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -74,6 +74,12 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "refcount_ptr_test", + srcs = ["refcount_ptr_test.cc"], + deps = ["//include/envoy/stats:refcount_ptr_interface"], +) + envoy_cc_test( name = "symbol_table_impl_test", srcs = ["symbol_table_impl_test.cc"], diff --git a/test/common/stats/heap_stat_data_test.cc b/test/common/stats/heap_stat_data_test.cc index 68b5e13a82e8..315c6b685c3f 100644 --- a/test/common/stats/heap_stat_data_test.cc +++ b/test/common/stats/heap_stat_data_test.cc @@ -37,24 +37,24 @@ TEST_F(HeapStatDataTest, HeapNoTruncate) { const std::string long_string(128, 'A'); StatName stat_name = makeStat(long_string); HeapStatData* stat{}; - EXPECT_NO_LOGS(stat = &alloc_.alloc(stat_name)); + EXPECT_NO_LOGS(stat = HeapStatData::alloc(stat_name, symbol_table_)); EXPECT_EQ(stat->statName(), stat_name); - alloc_.free(*stat); + stat->free(symbol_table_); }; TEST_F(HeapStatDataTest, HeapAlloc) { - HeapStatData* stat_1 = &alloc_.alloc(makeStat("ref_name")); + HeapStatData* stat_1 = HeapStatData::alloc(makeStat("ref_name"), symbol_table_); ASSERT_NE(stat_1, nullptr); - HeapStatData* stat_2 = &alloc_.alloc(makeStat("ref_name")); + HeapStatData* stat_2 = HeapStatData::alloc(makeStat("ref_name"), symbol_table_); ASSERT_NE(stat_2, nullptr); - HeapStatData* stat_3 = &alloc_.alloc(makeStat("not_ref_name")); + HeapStatData* stat_3 = HeapStatData::alloc(makeStat("not_ref_name"), symbol_table_); ASSERT_NE(stat_3, nullptr); - EXPECT_EQ(stat_1, stat_2); + EXPECT_NE(stat_1, stat_2); EXPECT_NE(stat_1, stat_3); EXPECT_NE(stat_2, stat_3); - alloc_.free(*stat_1); - alloc_.free(*stat_2); - alloc_.free(*stat_3); + stat_1->free(symbol_table_); + stat_2->free(symbol_table_); + stat_3->free(symbol_table_); } } // namespace diff --git a/test/common/stats/isolated_store_impl_test.cc b/test/common/stats/isolated_store_impl_test.cc index a91f01f93c51..b74b9fb1a840 100644 --- a/test/common/stats/isolated_store_impl_test.cc +++ b/test/common/stats/isolated_store_impl_test.cc @@ -186,9 +186,11 @@ TEST_F(StatsIsolatedStoreImplTest, StatsMacros) { } TEST_F(StatsIsolatedStoreImplTest, NullImplCoverage) { - NullCounterImpl c(store_.symbolTable()); - EXPECT_EQ(0, c.latch()); - NullGaugeImpl g(store_.symbolTable()); + NullCounterImpl& c = store_.nullCounter(); + c.inc(); + EXPECT_EQ(0, c.value()); + NullGaugeImpl& g = store_.nullGauge(""); + g.inc(); EXPECT_EQ(0, g.value()); } diff --git a/test/common/stats/refcount_ptr_test.cc b/test/common/stats/refcount_ptr_test.cc new file mode 100644 index 000000000000..ff729b09ecbf --- /dev/null +++ b/test/common/stats/refcount_ptr_test.cc @@ -0,0 +1,73 @@ +#include + +#include "envoy/stats/refcount_ptr.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Stats { + +class RefcountedString : public std::string, public RefcountHelper { +public: + explicit RefcountedString(const std::string& s) : std::string(s) {} +}; +using SharedString = RefcountPtr; + +class DerivedRefcountedString : public RefcountedString {}; +using DerivedSharedString = RefcountPtr; + +TEST(RefcountPtr, Constructors) { + SharedString rp1; // Default constructor. + EXPECT_FALSE(rp1); + rp1 = new RefcountedString("Hello"); // Assign from pointer. + EXPECT_EQ(1, rp1.use_count()); + SharedString rp2(rp1); // Copy-constructor. + EXPECT_EQ(2, rp1.use_count()); + EXPECT_EQ(2, rp2.use_count()); + EXPECT_EQ(rp1, rp2); + EXPECT_EQ(*rp1, *rp2); + *rp1 += ", World!"; // Object is shared, so mutations are shared. + EXPECT_EQ(rp1, rp2); + EXPECT_EQ(*rp1, *rp2); + EXPECT_EQ("Hello, World!", *rp2); + SharedString rp3(std::move(rp2)); // Move-constructor. + EXPECT_EQ(2, rp3.use_count()); + EXPECT_EQ("Hello, World!", *rp3); + EXPECT_NE(rp2, rp3); // NOLINT -- intentionally testing what happens to a variable post-move. + EXPECT_EQ(nullptr, rp2); // NOLINT -- ditto + EXPECT_NE(rp1, rp2); // NOLINT -- ditto + EXPECT_EQ(rp1, rp3); + EXPECT_FALSE(rp2); // NOLINT -- ditto + EXPECT_TRUE(rp3); + EXPECT_TRUE(rp1); + SharedString rp4(new RefcountedString("Hello, World!")); // Construct from pointer. + EXPECT_EQ(*rp4, *rp3); + EXPECT_NE(rp4, rp3); + DerivedSharedString rp5(rp4); // Construct across hierarchies. + EXPECT_EQ(rp5, rp4); + EXPECT_EQ(*rp5, *rp4); + SharedString rp6; + rp6 = std::move(rp4); // move-assign. + EXPECT_EQ(nullptr, rp4); // NOLINT -- intentionally testing what happens to a variable post-move. + EXPECT_EQ(rp5, rp6); +} + +TEST(RefcountPtr, Operators) { + RefcountedString* ptr = new RefcountedString("Hello, World!"); + SharedString shared(ptr); + EXPECT_TRUE(shared); + EXPECT_EQ(13, shared->size()); + RefcountedString& ref = *shared; + EXPECT_EQ(&ref, ptr); + SharedString shared2(new RefcountedString("Hello, World!")); + EXPECT_NE(&ref, shared2.get()); + SharedString shared3(shared2.get()); + EXPECT_EQ(shared2, shared3); + EXPECT_EQ(2, shared2.use_count()); + shared2.reset(); + EXPECT_EQ(nullptr, shared2); + EXPECT_EQ(1, shared3.use_count()); +} + +} // namespace Stats +} // namespace Envoy diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index cc7bcf242490..6b23b106c6aa 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -387,7 +387,7 @@ TEST_F(StatsThreadLocalStoreTest, NestedScopes) { ScopePtr scope2 = scope1->createScope("foo."); Counter& c2 = scope2->counter("bar"); - EXPECT_NE(&c1, &c2); + EXPECT_EQ(&c1, &c2); EXPECT_EQ("scope1.foo.bar", c2.name()); StatNameManagedStorage c2_name("scope1.foo.bar", symbol_table_); auto found_counter2 = store_->findCounter(c2_name.statName()); @@ -417,7 +417,7 @@ TEST_F(StatsThreadLocalStoreTest, OverlappingScopes) { // We will call alloc twice, but they should point to the same backing storage. Counter& c1 = scope1->counter("c"); Counter& c2 = scope2->counter("c"); - EXPECT_NE(&c1, &c2); + EXPECT_EQ(&c1, &c2); c1.inc(); EXPECT_EQ(1UL, c1.value()); EXPECT_EQ(1UL, c2.value()); @@ -431,7 +431,7 @@ TEST_F(StatsThreadLocalStoreTest, OverlappingScopes) { // Gauges should work the same way. Gauge& g1 = scope1->gauge("g", Gauge::ImportMode::Accumulate); Gauge& g2 = scope2->gauge("g", Gauge::ImportMode::Accumulate); - EXPECT_NE(&g1, &g2); + EXPECT_EQ(&g1, &g2); g1.set(5); EXPECT_EQ(5UL, g1.value()); EXPECT_EQ(5UL, g2.value()); @@ -861,8 +861,8 @@ TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTls) { TestUtil::forEachSampleStat( 1000, [&store](absl::string_view name) { store.counter(std::string(name)); }); const size_t million = 1000 * 1000; - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 19602128); // June 13, 2019 - EXPECT_MEMORY_LE(memory_test.consumedBytes(), 20 * million); + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 17432336); // June 29, 2019 + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 18 * million); } TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithTls) { @@ -881,8 +881,8 @@ TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithTls) { TestUtil::forEachSampleStat( 1000, [&store](absl::string_view name) { store.counter(std::string(name)); }); const size_t million = 1000 * 1000; - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 22879216); - EXPECT_MEMORY_LE(memory_test.consumedBytes(), 23 * million); + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 19660848); // June 29, 2019 + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 20 * million); store.shutdownThreading(); tls.shutdownThread(); } diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 5bdc4b751de8..3818dc87d306 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -207,9 +207,18 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // 2019/06/03 7199 49393 absl update // 2019/06/06 7208 49650 make memory targets approximate // 2019/06/17 7243 49412 49700 macros for exact/upper-bound memory checks + // 2019/06/29 7364 45685 46000 combine 2 levels of stat ref-counting into 1 - EXPECT_MEMORY_EQ(m_per_cluster, 49412); - EXPECT_MEMORY_LE(m_per_cluster, 49700); + // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI + // 'release' builds, where we control the platform and tool-chain. So you + // will need to find the correct value only after failing CI and looking + // at the logs. + // + // On a local clang8/libstdc++/linux flow, the memory usage was observed in + // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may + // vary. + EXPECT_MEMORY_EQ(m_per_cluster, 45685); + EXPECT_MEMORY_LE(m_per_cluster, 46000); } } // namespace diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index 7f255dcd87c0..dca7e8130482 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -29,7 +29,7 @@ namespace Stats { class MockMetric : public virtual Metric { public: MockMetric(); - ~MockMetric(); + ~MockMetric() override; // This bit of C++ subterfuge allows us to support the wealth of tests that // do metric->name_ = "foo" even though names are more complex now. Note @@ -83,7 +83,7 @@ class MockMetric : public virtual Metric { class MockCounter : public Counter, public MockMetric { public: MockCounter(); - ~MockCounter(); + ~MockCounter() override; MOCK_METHOD1(add, void(uint64_t amount)); MOCK_METHOD0(inc, void()); @@ -95,12 +95,20 @@ class MockCounter : public Counter, public MockMetric { bool used_; uint64_t value_; uint64_t latch_; + + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + +private: + RefcountHelper refcount_helper_; }; -class MockGauge : public Gauge, public MockMetric { +class MockGauge : public Gauge, public MockMetric, public RefcountHelper { public: MockGauge(); - ~MockGauge(); + ~MockGauge() override; MOCK_METHOD1(add, void(uint64_t amount)); MOCK_METHOD0(dec, void()); @@ -116,12 +124,20 @@ class MockGauge : public Gauge, public MockMetric { bool used_; uint64_t value_; ImportMode import_mode_; + + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + +private: + RefcountHelper refcount_helper_; }; class MockHistogram : public Histogram, public MockMetric { public: MockHistogram(); - ~MockHistogram(); + ~MockHistogram() override; MOCK_METHOD1(recordValue, void(uint64_t value)); MOCK_CONST_METHOD0(used, bool()); @@ -132,7 +148,7 @@ class MockHistogram : public Histogram, public MockMetric { class MockParentHistogram : public ParentHistogram, public MockMetric { public: MockParentHistogram(); - ~MockParentHistogram(); + ~MockParentHistogram() override; void merge() override {} const std::string quantileSummary() const override { return ""; }; @@ -152,7 +168,7 @@ class MockParentHistogram : public ParentHistogram, public MockMetric { class MockMetricSnapshot : public MetricSnapshot { public: MockMetricSnapshot(); - ~MockMetricSnapshot(); + ~MockMetricSnapshot() override; MOCK_METHOD0(counters, const std::vector&()); MOCK_METHOD0(gauges, const std::vector>&()); @@ -166,7 +182,7 @@ class MockMetricSnapshot : public MetricSnapshot { class MockSink : public Sink { public: MockSink(); - ~MockSink(); + ~MockSink() override; MOCK_METHOD1(flush, void(MetricSnapshot& snapshot)); MOCK_METHOD2(onHistogramComplete, void(const Histogram& histogram, uint64_t value)); @@ -180,7 +196,7 @@ class SymbolTableProvider { class MockStore : public SymbolTableProvider, public StoreImpl { public: MockStore(); - ~MockStore(); + ~MockStore() override; ScopePtr createScope(const std::string& name) override { return ScopePtr{createScope_(name)}; } @@ -222,7 +238,7 @@ class MockIsolatedStatsStore : private Test::Global, public IsolatedStoreImpl { public: MockIsolatedStatsStore(); - ~MockIsolatedStatsStore(); + ~MockIsolatedStatsStore() override; MOCK_METHOD2(deliverHistogramToSinks, void(const Histogram& histogram, uint64_t value)); }; @@ -230,7 +246,7 @@ class MockIsolatedStatsStore : private Test::Global, class MockStatsMatcher : public StatsMatcher { public: MockStatsMatcher(); - ~MockStatsMatcher(); + ~MockStatsMatcher() override; MOCK_CONST_METHOD1(rejects, bool(const std::string& name)); bool acceptsAll() const override { return accepts_all_; } bool rejectsAll() const override { return rejects_all_; } diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index e90a0954fbd8..c72496089e7e 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -1389,6 +1389,11 @@ class PrometheusStatsFormatterTest : public testing::Test { histograms_.push_back(histogram); } + using MockHistogramSharedPtr = Stats::RefcountPtr>; + MockHistogramSharedPtr makeHistogram() { + return MockHistogramSharedPtr(new NiceMock()); + } + Stats::FakeSymbolTableImpl symbol_table_; Stats::HeapStatDataAllocator alloc_; std::vector counters_; @@ -1473,7 +1478,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { h1_cumulative.setHistogramValues(std::vector(0)); Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - auto histogram = std::make_shared>(); + auto histogram = makeHistogram(); histogram->name_ = "histogram1"; histogram->used_ = true; ON_CALL(*histogram, cumulativeStatistics()) @@ -1526,7 +1531,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) { Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - auto histogram = std::make_shared>(); + auto histogram = makeHistogram(); histogram->name_ = "histogram1"; histogram->used_ = true; ON_CALL(*histogram, cumulativeStatistics()) @@ -1578,7 +1583,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { h1_cumulative.setHistogramValues(h1_values); Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - auto histogram1 = std::make_shared>(); + auto histogram1 = makeHistogram(); histogram1->name_ = "cluster.test_1.upstream_rq_time"; histogram1->used_ = true; histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); @@ -1638,7 +1643,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { h1_cumulative.setHistogramValues(h1_values); Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - auto histogram1 = std::make_shared>(); + auto histogram1 = makeHistogram(); histogram1->name_ = "cluster.test_1.upstream_rq_time"; histogram1->used_ = true; histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); @@ -1685,7 +1690,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { h1_cumulative.setHistogramValues(h1_values); Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - auto histogram1 = std::make_shared>(); + auto histogram1 = makeHistogram(); histogram1->name_ = "cluster.test_1.upstream_rq_time"; histogram1->used_ = false; histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); @@ -1724,7 +1729,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) { h1_cumulative.setHistogramValues(h1_values); Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); - auto histogram1 = std::make_shared>(); + auto histogram1 = makeHistogram(); histogram1->name_ = "cluster.test_1.upstream_rq_time"; histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}); addHistogram(histogram1); diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index cd4b375e2302..982232db171f 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -290,8 +290,10 @@ WS Welford's Werror XDS +decRefCount getaddrinfo sendto +upcasts vip xDSes XFCC From f36973d057eddb8076cbd18f4d4e783808f81320 Mon Sep 17 00:00:00 2001 From: Mike Schore Date: Mon, 1 Jul 2019 08:26:21 -0700 Subject: [PATCH 084/542] check_format: allow paths to be excluded from build_fixer (#7412) Signed-off-by: Mike Schore --- tools/check_format.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tools/check_format.py b/tools/check_format.py index 40770ff3b7f6..57204eea5e06 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -354,6 +354,13 @@ def isWorkspaceFile(file_path): return os.path.basename(file_path) == "WORKSPACE" +def isBuildFixerExcludedFile(file_path): + for excluded_path in build_fixer_check_excluded_paths: + if file_path.startswith(excluded_path): + return True + return False + + def hasInvalidAngleBracketDirectory(line): if not line.startswith(INCLUDE_ANGLE): return False @@ -558,8 +565,10 @@ def fixBuildPath(file_path): sys.stdout.write(fixBuildLine(line, file_path)) error_messages = [] + # TODO(htuch): Add API specific BUILD fixer script. - if not isApiFile(file_path) and not isSkylarkFile(file_path) and not isWorkspaceFile(file_path): + if not isBuildFixerExcludedFile(file_path) and not isApiFile(file_path) and not isSkylarkFile( + file_path) and not isWorkspaceFile(file_path): if os.system("%s %s %s" % (ENVOY_BUILD_FIXER_PATH, file_path, file_path)) != 0: error_messages += ["envoy_build_fixer rewrite failed for file: %s" % file_path] @@ -570,7 +579,9 @@ def fixBuildPath(file_path): def checkBuildPath(file_path): error_messages = [] - if not isApiFile(file_path) and not isSkylarkFile(file_path) and not isWorkspaceFile(file_path): + + if not isBuildFixerExcludedFile(file_path) and not isApiFile(file_path) and not isSkylarkFile( + file_path) and not isWorkspaceFile(file_path): command = "%s %s | diff %s -" % (ENVOY_BUILD_FIXER_PATH, file_path, file_path) error_messages += executeCommand(command, "envoy_build_fixer check failed", file_path) @@ -795,6 +806,12 @@ def checkErrorMessages(error_messages): nargs="+", default=[], help="exclude paths from the namespace_check.") + parser.add_argument( + "--build_fixer_check_excluded_paths", + type=str, + nargs="+", + default=[], + help="exclude paths from envoy_build_fixer check.") parser.add_argument( "--include_dir_order", type=str, @@ -807,6 +824,7 @@ def checkErrorMessages(error_messages): envoy_build_rule_check = not args.skip_envoy_build_rule_check namespace_check = args.namespace_check namespace_check_excluded_paths = args.namespace_check_excluded_paths + build_fixer_check_excluded_paths = args.build_fixer_check_excluded_paths include_dir_order = args.include_dir_order if args.add_excluded_prefixes: EXCLUDED_PREFIXES += tuple(args.add_excluded_prefixes) From 188ae9f19467a0a7c2260b0c4543def2c2a96c46 Mon Sep 17 00:00:00 2001 From: ivitjuk <875244+ivitjuk@users.noreply.github.com> Date: Mon, 1 Jul 2019 08:59:34 -0700 Subject: [PATCH 085/542] Added a set of AWS credential providers. (#7281) - EnvironmentCredentialsProvider fetches credentials from the environment - InstanceProfileCredentialsProvider fetches credentials from the instance metadata - TaskRoleCredentialsProvider fetches credentials from the ECS task metadata Also added DefaultCredentialsProviderChain which is able to pick the correct credential provider according to the execution environment it is running in. Signed-off-by: Ivan Vitjuk --- .../extensions/filters/http/common/aws/BUILD | 15 + .../http/common/aws/credentials_provider.h | 43 +- .../common/aws/credentials_provider_impl.cc | 220 +++++++++ .../common/aws/credentials_provider_impl.h | 170 +++++++ test/extensions/filters/http/common/aws/BUILD | 21 + .../aws/credentials_provider_impl_test.cc | 457 ++++++++++++++++++ .../common/aws/credentials_provider_test.cc | 57 +++ .../filters/http/common/aws/mocks.cc | 2 +- .../filters/http/common/aws/mocks.h | 16 + 9 files changed, 990 insertions(+), 11 deletions(-) create mode 100644 source/extensions/filters/http/common/aws/credentials_provider_impl.cc create mode 100644 source/extensions/filters/http/common/aws/credentials_provider_impl.h create mode 100644 test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc create mode 100644 test/extensions/filters/http/common/aws/credentials_provider_test.cc diff --git a/source/extensions/filters/http/common/aws/BUILD b/source/extensions/filters/http/common/aws/BUILD index 0a0c543fb426..db1057ca56d0 100644 --- a/source/extensions/filters/http/common/aws/BUILD +++ b/source/extensions/filters/http/common/aws/BUILD @@ -40,6 +40,21 @@ envoy_cc_library( external_deps = ["abseil_optional"], ) +envoy_cc_library( + name = "credentials_provider_impl_lib", + srcs = ["credentials_provider_impl.cc"], + hdrs = ["credentials_provider_impl.h"], + external_deps = ["abseil_time"], + deps = [ + ":credentials_provider_interface", + "//include/envoy/api:api_interface", + "//source/common/common:logger_lib", + "//source/common/common:thread_lib", + "//source/common/http:utility_lib", + "//source/common/json:json_loader_lib", + ], +) + envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], diff --git a/source/extensions/filters/http/common/aws/credentials_provider.h b/source/extensions/filters/http/common/aws/credentials_provider.h index e761b915ba4e..04ad9d8b101c 100644 --- a/source/extensions/filters/http/common/aws/credentials_provider.h +++ b/source/extensions/filters/http/common/aws/credentials_provider.h @@ -13,17 +13,27 @@ namespace HttpFilters { namespace Common { namespace Aws { +/** + * AWS credentials container + * + * If a credential component was not found in the execution environment, it's getter method will + * return absl::nullopt. Credential components with the empty string value are treated as not found. + */ class Credentials { public: - Credentials() = default; - - Credentials(absl::string_view access_key_id, absl::string_view secret_access_key) - : access_key_id_(access_key_id), secret_access_key_(secret_access_key) {} - - Credentials(absl::string_view access_key_id, absl::string_view secret_access_key, - absl::string_view session_token) - : access_key_id_(access_key_id), secret_access_key_(secret_access_key), - session_token_(session_token) {} + Credentials(absl::string_view access_key_id = absl::string_view(), + absl::string_view secret_access_key = absl::string_view(), + absl::string_view session_token = absl::string_view()) { + if (!access_key_id.empty()) { + access_key_id_ = std::string(access_key_id); + if (!secret_access_key.empty()) { + secret_access_key_ = std::string(secret_access_key); + if (!session_token.empty()) { + session_token_ = std::string(session_token); + } + } + } + } const absl::optional& accessKeyId() const { return access_key_id_; } @@ -31,16 +41,29 @@ class Credentials { const absl::optional& sessionToken() const { return session_token_; } + bool operator==(const Credentials& other) const { + return access_key_id_ == other.access_key_id_ && + secret_access_key_ == other.secret_access_key_ && session_token_ == other.session_token_; + } + private: absl::optional access_key_id_; absl::optional secret_access_key_; absl::optional session_token_; }; +/** + * Interface for classes able to fetch AWS credentials from the execution environment. + */ class CredentialsProvider { public: virtual ~CredentialsProvider() = default; + /** + * Get credentials from the environment. + * + * @return AWS credentials + */ virtual Credentials getCredentials() PURE; }; @@ -50,4 +73,4 @@ using CredentialsProviderSharedPtr = std::shared_ptr; } // namespace Common } // namespace HttpFilters } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/extensions/filters/http/common/aws/credentials_provider_impl.cc b/source/extensions/filters/http/common/aws/credentials_provider_impl.cc new file mode 100644 index 000000000000..43cde8c1ddd8 --- /dev/null +++ b/source/extensions/filters/http/common/aws/credentials_provider_impl.cc @@ -0,0 +1,220 @@ +#include "extensions/filters/http/common/aws/credentials_provider_impl.h" + +#include "envoy/common/exception.h" + +#include "common/common/lock_guard.h" +#include "common/http/utility.h" +#include "common/json/json_loader.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Common { +namespace Aws { + +constexpr static char AWS_ACCESS_KEY_ID[] = "AWS_ACCESS_KEY_ID"; +constexpr static char AWS_SECRET_ACCESS_KEY[] = "AWS_SECRET_ACCESS_KEY"; +constexpr static char AWS_SESSION_TOKEN[] = "AWS_SESSION_TOKEN"; + +constexpr static char ACCESS_KEY_ID[] = "AccessKeyId"; +constexpr static char SECRET_ACCESS_KEY[] = "SecretAccessKey"; +constexpr static char TOKEN[] = "Token"; +constexpr static char EXPIRATION[] = "Expiration"; +constexpr static char EXPIRATION_FORMAT[] = "%E4Y%m%dT%H%M%S%z"; +constexpr static char TRUE[] = "true"; + +constexpr static char AWS_CONTAINER_CREDENTIALS_RELATIVE_URI[] = + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; +constexpr static char AWS_CONTAINER_CREDENTIALS_FULL_URI[] = "AWS_CONTAINER_CREDENTIALS_FULL_URI"; +constexpr static char AWS_CONTAINER_AUTHORIZATION_TOKEN[] = "AWS_CONTAINER_AUTHORIZATION_TOKEN"; +constexpr static char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED"; + +constexpr static std::chrono::hours REFRESH_INTERVAL{1}; +constexpr static std::chrono::seconds REFRESH_GRACE_PERIOD{5}; +constexpr static char EC2_METADATA_HOST[] = "169.254.169.254:80"; +constexpr static char CONTAINER_METADATA_HOST[] = "169.254.170.2:80"; +constexpr static char SECURITY_CREDENTIALS_PATH[] = "/latest/meta-data/iam/security-credentials"; + +Credentials EnvironmentCredentialsProvider::getCredentials() { + ENVOY_LOG(debug, "Getting AWS credentials from the environment"); + + const auto access_key_id = std::getenv(AWS_ACCESS_KEY_ID); + if (access_key_id == nullptr) { + return Credentials(); + } + + const auto secret_access_key = std::getenv(AWS_SECRET_ACCESS_KEY); + const auto session_token = std::getenv(AWS_SESSION_TOKEN); + + ENVOY_LOG(debug, "Found following AWS credentials in the environment: {}={}, {}={}, {}={}", + AWS_ACCESS_KEY_ID, access_key_id ? access_key_id : "", AWS_SECRET_ACCESS_KEY, + secret_access_key ? "*****" : "", AWS_SESSION_TOKEN, session_token ? "*****" : ""); + + return Credentials(access_key_id, secret_access_key, session_token); +} + +void MetadataCredentialsProviderBase::refreshIfNeeded() { + const Thread::LockGuard lock(lock_); + if (needsRefresh()) { + refresh(); + } +} + +bool InstanceProfileCredentialsProvider::needsRefresh() { + return api_.timeSource().systemTime() - last_updated_ > REFRESH_INTERVAL; +} + +void InstanceProfileCredentialsProvider::refresh() { + ENVOY_LOG(debug, "Getting AWS credentials from the instance metadata"); + + // First discover the Role of this instance + const auto instance_role_string = + metadata_fetcher_(EC2_METADATA_HOST, SECURITY_CREDENTIALS_PATH, ""); + if (!instance_role_string) { + ENVOY_LOG(error, "Could not retrieve credentials listing from the instance metadata"); + return; + } + + const auto instance_role_list = + StringUtil::splitToken(StringUtil::trim(instance_role_string.value()), "\n"); + if (instance_role_list.empty()) { + ENVOY_LOG(error, "No AWS credentials were found in the instance metadata"); + return; + } + ENVOY_LOG(debug, "AWS credentials list:\n{}", instance_role_string.value()); + + // Only one Role can be associated with an instance: + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + const auto credential_path = + std::string(SECURITY_CREDENTIALS_PATH) + "/" + + std::string(instance_role_list[0].data(), instance_role_list[0].size()); + ENVOY_LOG(debug, "AWS credentials path: {}", credential_path); + + // Then fetch and parse the credentials + const auto credential_document = metadata_fetcher_(EC2_METADATA_HOST, credential_path, ""); + if (!credential_document) { + ENVOY_LOG(error, "Could not load AWS credentials document from the instance metadata"); + return; + } + + Json::ObjectSharedPtr document_json; + try { + document_json = Json::Factory::loadFromString(credential_document.value()); + } catch (EnvoyException& e) { + ENVOY_LOG(error, "Could not parse AWS credentials document: {}", e.what()); + return; + } + + const auto access_key_id = document_json->getString(ACCESS_KEY_ID, ""); + const auto secret_access_key = document_json->getString(SECRET_ACCESS_KEY, ""); + const auto session_token = document_json->getString(TOKEN, ""); + + ENVOY_LOG(debug, "Found following AWS credentials in the instance metadata: {}={}, {}={}, {}={}", + AWS_ACCESS_KEY_ID, access_key_id, AWS_SECRET_ACCESS_KEY, + secret_access_key.empty() ? "" : "*****", AWS_SESSION_TOKEN, + session_token.empty() ? "" : "*****"); + + cached_credentials_ = Credentials(access_key_id, secret_access_key, session_token); + last_updated_ = api_.timeSource().systemTime(); +} + +bool TaskRoleCredentialsProvider::needsRefresh() { + const auto now = api_.timeSource().systemTime(); + return (now - last_updated_ > REFRESH_INTERVAL) || + (expiration_time_ - now < REFRESH_GRACE_PERIOD); +} + +void TaskRoleCredentialsProvider::refresh() { + ENVOY_LOG(debug, "Getting AWS credentials from the task role at URI: {}", credential_uri_); + + absl::string_view host; + absl::string_view path; + Http::Utility::extractHostPathFromUri(credential_uri_, host, path); + const auto credential_document = + metadata_fetcher_(std::string(host.data(), host.size()), + std::string(path.data(), path.size()), authorization_token_); + if (!credential_document) { + ENVOY_LOG(error, "Could not load AWS credentials document from the task role"); + return; + } + + Json::ObjectSharedPtr document_json; + try { + document_json = Json::Factory::loadFromString(credential_document.value()); + } catch (EnvoyException& e) { + ENVOY_LOG(error, "Could not parse AWS credentials document from the task role: {}", e.what()); + return; + } + + const auto access_key_id = document_json->getString(ACCESS_KEY_ID, ""); + const auto secret_access_key = document_json->getString(SECRET_ACCESS_KEY, ""); + const auto session_token = document_json->getString(TOKEN, ""); + + ENVOY_LOG(debug, "Found following AWS credentials in the task role: {}={}, {}={}, {}={}", + AWS_ACCESS_KEY_ID, access_key_id, AWS_SECRET_ACCESS_KEY, + secret_access_key.empty() ? "" : "*****", AWS_SESSION_TOKEN, + session_token.empty() ? "" : "*****"); + + const auto expiration_str = document_json->getString(EXPIRATION, ""); + if (!expiration_str.empty()) { + absl::Time expiration_time; + if (absl::ParseTime(EXPIRATION_FORMAT, expiration_str, &expiration_time, nullptr)) { + ENVOY_LOG(debug, "Task role AWS credentials expiration time: {}", expiration_str); + expiration_time_ = absl::ToChronoTime(expiration_time); + } + } + + last_updated_ = api_.timeSource().systemTime(); + cached_credentials_ = Credentials(access_key_id, secret_access_key, session_token); +} + +Credentials CredentialsProviderChain::getCredentials() { + for (auto& provider : providers_) { + const auto credentials = provider->getCredentials(); + if (credentials.accessKeyId() && credentials.secretAccessKey()) { + return credentials; + } + } + + ENVOY_LOG(debug, "No AWS credentials found, using anonymous credentials"); + return Credentials(); +} + +DefaultCredentialsProviderChain::DefaultCredentialsProviderChain( + Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + const CredentialsProviderChainFactories& factories) { + ENVOY_LOG(debug, "Using environment credentials provider"); + add(factories.createEnvironmentCredentialsProvider()); + + const auto relative_uri = std::getenv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI); + const auto full_uri = std::getenv(AWS_CONTAINER_CREDENTIALS_FULL_URI); + const auto metadata_disabled = std::getenv(AWS_EC2_METADATA_DISABLED); + + if (relative_uri != nullptr) { + const auto uri = std::string(CONTAINER_METADATA_HOST) + relative_uri; + ENVOY_LOG(debug, "Using task role credentials provider with URI: {}", uri); + add(factories.createTaskRoleCredentialsProvider(api, metadata_fetcher, uri)); + } else if (full_uri != nullptr) { + const auto authorization_token = std::getenv(AWS_CONTAINER_AUTHORIZATION_TOKEN); + if (authorization_token != nullptr) { + ENVOY_LOG(debug, + "Using task role credentials provider with URI: " + "{} and authorization token", + full_uri); + add(factories.createTaskRoleCredentialsProvider(api, metadata_fetcher, full_uri, + authorization_token)); + } else { + ENVOY_LOG(debug, "Using task role credentials provider with URI: {}", full_uri); + add(factories.createTaskRoleCredentialsProvider(api, metadata_fetcher, full_uri)); + } + } else if (metadata_disabled == nullptr || strncmp(metadata_disabled, TRUE, strlen(TRUE)) != 0) { + ENVOY_LOG(debug, "Using instance profile credentials provider"); + add(factories.createInstanceProfileCredentialsProvider(api, metadata_fetcher)); + } +} + +} // namespace Aws +} // namespace Common +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/common/aws/credentials_provider_impl.h b/source/extensions/filters/http/common/aws/credentials_provider_impl.h new file mode 100644 index 000000000000..31e4d25d5d87 --- /dev/null +++ b/source/extensions/filters/http/common/aws/credentials_provider_impl.h @@ -0,0 +1,170 @@ +#pragma once + +#include + +#include "envoy/api/api.h" +#include "envoy/event/timer.h" + +#include "common/common/logger.h" +#include "common/common/thread.h" + +#include "extensions/filters/http/common/aws/credentials_provider.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Common { +namespace Aws { + +/** + * Retrieve AWS credentials from the environment variables. + * + * Adheres to conventions specified in: + * https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html + */ +class EnvironmentCredentialsProvider : public CredentialsProvider, + public Logger::Loggable { +public: + Credentials getCredentials() override; +}; + +class MetadataCredentialsProviderBase : public CredentialsProvider, + public Logger::Loggable { +public: + using MetadataFetcher = std::function( + const std::string& host, const std::string& path, const std::string& auth_token)>; + + MetadataCredentialsProviderBase(Api::Api& api, const MetadataFetcher& metadata_fetcher) + : api_(api), metadata_fetcher_(metadata_fetcher) {} + + Credentials getCredentials() override { + refreshIfNeeded(); + return cached_credentials_; + } + +protected: + Api::Api& api_; + MetadataFetcher metadata_fetcher_; + SystemTime last_updated_; + Credentials cached_credentials_; + Thread::MutexBasicLockable lock_; + + void refreshIfNeeded(); + + virtual bool needsRefresh() PURE; + virtual void refresh() PURE; +}; + +/** + * Retrieve AWS credentials from the instance metadata. + * + * https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials + */ +class InstanceProfileCredentialsProvider : public MetadataCredentialsProviderBase { +public: + InstanceProfileCredentialsProvider(Api::Api& api, const MetadataFetcher& metadata_fetcher) + : MetadataCredentialsProviderBase(api, metadata_fetcher) {} + +private: + bool needsRefresh() override; + void refresh() override; +}; + +/** + * Retrieve AWS credentials from the task metadata. + * + * https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html#enable_task_iam_roles + */ +class TaskRoleCredentialsProvider : public MetadataCredentialsProviderBase { +public: + TaskRoleCredentialsProvider(Api::Api& api, const MetadataFetcher& metadata_fetcher, + const std::string& credential_uri, + const std::string& authorization_token = std::string()) + : MetadataCredentialsProviderBase(api, metadata_fetcher), credential_uri_(credential_uri), + authorization_token_(authorization_token) {} + +private: + SystemTime expiration_time_; + std::string credential_uri_; + std::string authorization_token_; + + bool needsRefresh() override; + void refresh() override; +}; + +/** + * AWS credentials provider chain, able to fallback between multiple credential providers. + */ +class CredentialsProviderChain : public CredentialsProvider, + public Logger::Loggable { +public: + virtual ~CredentialsProviderChain() = default; + + void add(const CredentialsProviderSharedPtr& credentials_provider) { + providers_.emplace_back(credentials_provider); + } + + Credentials getCredentials() override; + +protected: + std::list providers_; +}; + +class CredentialsProviderChainFactories { +public: + virtual ~CredentialsProviderChainFactories() = default; + + virtual CredentialsProviderSharedPtr createEnvironmentCredentialsProvider() const PURE; + + virtual CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( + Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + const std::string& credential_uri, + const std::string& authorization_token = std::string()) const PURE; + + virtual CredentialsProviderSharedPtr createInstanceProfileCredentialsProvider( + Api::Api& api, + const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher) const PURE; +}; + +/** + * Default AWS credentials provider chain. + * + * Reference implementation: + * https://github.com/aws/aws-sdk-cpp/blob/master/aws-cpp-sdk-core/source/auth/AWSCredentialsProviderChain.cpp#L44 + */ +class DefaultCredentialsProviderChain : public CredentialsProviderChain, + public CredentialsProviderChainFactories { +public: + DefaultCredentialsProviderChain( + Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher) + : DefaultCredentialsProviderChain(api, metadata_fetcher, *this) {} + + DefaultCredentialsProviderChain( + Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + const CredentialsProviderChainFactories& factories); + +private: + CredentialsProviderSharedPtr createEnvironmentCredentialsProvider() const override { + return std::make_shared(); + } + + CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( + Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + const std::string& credential_uri, + const std::string& authorization_token = std::string()) const override { + return std::make_shared(api, metadata_fetcher, credential_uri, + authorization_token); + } + + CredentialsProviderSharedPtr createInstanceProfileCredentialsProvider( + Api::Api& api, + const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher) const override { + return std::make_shared(api, metadata_fetcher); + } +}; + +} // namespace Aws +} // namespace Common +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/common/aws/BUILD b/test/extensions/filters/http/common/aws/BUILD index 67bc0a150c6e..1f8f81417d0a 100644 --- a/test/extensions/filters/http/common/aws/BUILD +++ b/test/extensions/filters/http/common/aws/BUILD @@ -49,3 +49,24 @@ envoy_cc_test( "//test/test_common:environment_lib", ], ) + +envoy_cc_test( + name = "credentials_provider_impl_test", + srcs = ["credentials_provider_impl_test.cc"], + deps = [ + "//source/extensions/filters/http/common/aws:credentials_provider_impl_lib", + "//test/extensions/filters/http/common/aws:aws_mocks", + "//test/mocks/api:api_mocks", + "//test/mocks/event:event_mocks", + "//test/test_common:environment_lib", + "//test/test_common:simulated_time_system_lib", + ], +) + +envoy_cc_test( + name = "credentials_provider_test", + srcs = ["credentials_provider_test.cc"], + deps = [ + "//source/extensions/filters/http/common/aws:credentials_provider_interface", + ], +) diff --git a/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc b/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc new file mode 100644 index 000000000000..d4c22e6215af --- /dev/null +++ b/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc @@ -0,0 +1,457 @@ +#include "extensions/filters/http/common/aws/credentials_provider_impl.h" + +#include "test/extensions/filters/http/common/aws/mocks.h" +#include "test/mocks/api/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/test_common/environment.h" +#include "test/test_common/simulated_time_system.h" + +using testing::_; +using testing::InSequence; +using testing::NiceMock; +using testing::Ref; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Common { +namespace Aws { + +class EvironmentCredentialsProviderTest : public testing::Test { +public: + ~EvironmentCredentialsProviderTest() { + TestEnvironment::unsetEnvVar("AWS_ACCESS_KEY_ID"); + TestEnvironment::unsetEnvVar("AWS_SECRET_ACCESS_KEY"); + TestEnvironment::unsetEnvVar("AWS_SESSION_TOKEN"); + } + + EnvironmentCredentialsProvider provider_; +}; + +TEST_F(EvironmentCredentialsProviderTest, AllEnvironmentVars) { + TestEnvironment::setEnvVar("AWS_ACCESS_KEY_ID", "akid", 1); + TestEnvironment::setEnvVar("AWS_SECRET_ACCESS_KEY", "secret", 1); + TestEnvironment::setEnvVar("AWS_SESSION_TOKEN", "token", 1); + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); +} + +TEST_F(EvironmentCredentialsProviderTest, NoEnvironmentVars) { + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(EvironmentCredentialsProviderTest, MissingAccessKeyId) { + TestEnvironment::setEnvVar("AWS_SECRET_ACCESS_KEY", "secret", 1); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(EvironmentCredentialsProviderTest, NoSessionToken) { + TestEnvironment::setEnvVar("AWS_ACCESS_KEY_ID", "akid", 1); + TestEnvironment::setEnvVar("AWS_SECRET_ACCESS_KEY", "secret", 1); + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +class InstanceProfileCredentialsProviderTest : public testing::Test { +public: + InstanceProfileCredentialsProviderTest() + : api_(Api::createApiForTest(time_system_)), + provider_(*api_, + [this](const std::string& host, const std::string& path, + const std::string& auth_token) -> absl::optional { + return this->fetcher_.fetch(host, path, auth_token); + }) {} + + void expectCredentialListing(const absl::optional& listing) { + EXPECT_CALL(fetcher_, + fetch("169.254.169.254:80", "/latest/meta-data/iam/security-credentials", _)) + .WillOnce(Return(listing)); + } + + void expectDocument(const absl::optional& document) { + EXPECT_CALL(fetcher_, + fetch("169.254.169.254:80", "/latest/meta-data/iam/security-credentials/doc1", _)) + .WillOnce(Return(document)); + } + + Event::SimulatedTimeSystem time_system_; + Api::ApiPtr api_; + NiceMock fetcher_; + InstanceProfileCredentialsProvider provider_; +}; + +TEST_F(InstanceProfileCredentialsProviderTest, FailedCredentailListing) { + expectCredentialListing(absl::optional()); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, EmptyCredentialListing) { + expectCredentialListing(""); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, MissingDocument) { + expectCredentialListing("doc1\ndoc2\ndoc3"); + expectDocument(absl::optional()); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, MalformedDocumenet) { + expectCredentialListing("doc1"); + expectDocument(R"EOF( +not json +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, EmptyValues) { + expectCredentialListing("doc1"); + expectDocument(R"EOF( +{ + "AccessKeyId": "", + "SecretAccessKey": "", + "Token": "" +} +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, FullCachedCredentials) { + expectCredentialListing("doc1"); + expectDocument(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token" +} +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + const auto cached_credentials = provider_.getCredentials(); + EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("token", cached_credentials.sessionToken().value()); +} + +TEST_F(InstanceProfileCredentialsProviderTest, CredentialExpiration) { + InSequence sequence; + expectCredentialListing("doc1"); + expectDocument(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token" +} +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + time_system_.sleep(std::chrono::hours(2)); + expectCredentialListing("doc1"); + expectDocument(R"EOF( +{ + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "Token": "new_token" +} +)EOF"); + const auto new_credentials = provider_.getCredentials(); + EXPECT_EQ("new_akid", new_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", new_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", new_credentials.sessionToken().value()); +} + +class TaskRoleCredentialsProviderTest : public testing::Test { +public: + TaskRoleCredentialsProviderTest() + : api_(Api::createApiForTest(time_system_)), + provider_( + *api_, + [this](const std::string& host, const std::string& path, + const absl::optional& auth_token) -> absl::optional { + return this->fetcher_.fetch(host, path, auth_token); + }, + "169.254.170.2:80/path/to/doc", "auth_token") { + // Tue Jan 2 03:04:05 UTC 2018 + time_system_.setSystemTime(std::chrono::milliseconds(1514862245000)); + } + + void expectDocument(const absl::optional& document) { + EXPECT_CALL(fetcher_, fetch("169.254.170.2:80", "/path/to/doc", _)).WillOnce(Return(document)); + } + + Event::SimulatedTimeSystem time_system_; + Api::ApiPtr api_; + NiceMock fetcher_; + TaskRoleCredentialsProvider provider_; +}; + +TEST_F(TaskRoleCredentialsProviderTest, FailedFetchingDocument) { + expectDocument(absl::optional()); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(TaskRoleCredentialsProviderTest, MalformedDocumenet) { + expectDocument(R"EOF( +not json +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(TaskRoleCredentialsProviderTest, EmptyValues) { + expectDocument(R"EOF( +{ + "AccessKeyId": "", + "SecretAccessKey": "", + "Token": "", + "Expiration": "" +} +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_FALSE(credentials.accessKeyId().has_value()); + EXPECT_FALSE(credentials.secretAccessKey().has_value()); + EXPECT_FALSE(credentials.sessionToken().has_value()); +} + +TEST_F(TaskRoleCredentialsProviderTest, FullCachedCredentials) { + expectDocument(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token", + "Expiration": "20180102T030500Z" +} +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + const auto cached_credentials = provider_.getCredentials(); + EXPECT_EQ("akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("token", cached_credentials.sessionToken().value()); +} + +TEST_F(TaskRoleCredentialsProviderTest, NormalCredentialExpiration) { + InSequence sequence; + expectDocument(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token", + "Expiration": "20190102T030405Z" +} +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + time_system_.sleep(std::chrono::hours(2)); + expectDocument(R"EOF( +{ + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "Token": "new_token", + "Expiration": "20190102T030405Z" +} +)EOF"); + const auto cached_credentials = provider_.getCredentials(); + EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); +} + +TEST_F(TaskRoleCredentialsProviderTest, TimestampCredentialExpiration) { + InSequence sequence; + expectDocument(R"EOF( +{ + "AccessKeyId": "akid", + "SecretAccessKey": "secret", + "Token": "token", + "Expiration": "20180102T030405Z" +} +)EOF"); + const auto credentials = provider_.getCredentials(); + EXPECT_EQ("akid", credentials.accessKeyId().value()); + EXPECT_EQ("secret", credentials.secretAccessKey().value()); + EXPECT_EQ("token", credentials.sessionToken().value()); + expectDocument(R"EOF( +{ + "AccessKeyId": "new_akid", + "SecretAccessKey": "new_secret", + "Token": "new_token", + "Expiration": "20190102T030405Z" +} +)EOF"); + const auto cached_credentials = provider_.getCredentials(); + EXPECT_EQ("new_akid", cached_credentials.accessKeyId().value()); + EXPECT_EQ("new_secret", cached_credentials.secretAccessKey().value()); + EXPECT_EQ("new_token", cached_credentials.sessionToken().value()); +} + +class DefaultCredentialsProviderChainTest : public testing::Test { +public: + DefaultCredentialsProviderChainTest() : api_(Api::createApiForTest(time_system_)) { + EXPECT_CALL(factories_, createEnvironmentCredentialsProvider()); + } + + ~DefaultCredentialsProviderChainTest() { + TestEnvironment::unsetEnvVar("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); + TestEnvironment::unsetEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI"); + TestEnvironment::unsetEnvVar("AWS_CONTAINER_AUTHORIZATION_TOKEN"); + TestEnvironment::unsetEnvVar("AWS_EC2_METADATA_DISABLED"); + } + + class MockCredentialsProviderChainFactories : public CredentialsProviderChainFactories { + public: + MOCK_CONST_METHOD0(createEnvironmentCredentialsProvider, CredentialsProviderSharedPtr()); + MOCK_CONST_METHOD4(createTaskRoleCredentialsProviderMock, + CredentialsProviderSharedPtr( + Api::Api&, const MetadataCredentialsProviderBase::MetadataFetcher&, + const std::string&, const std::string&)); + MOCK_CONST_METHOD2(createInstanceProfileCredentialsProvider, + CredentialsProviderSharedPtr( + Api::Api&, + const MetadataCredentialsProviderBase::MetadataFetcher& fetcher)); + + virtual CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( + Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, + const std::string& credential_uri, const std::string& authorization_token) const { + return createTaskRoleCredentialsProviderMock(api, metadata_fetcher, credential_uri, + authorization_token); + } + }; + + Event::SimulatedTimeSystem time_system_; + Api::ApiPtr api_; + NiceMock factories_; +}; + +TEST_F(DefaultCredentialsProviderChainTest, NoEnvironmentVars) { + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)); + DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, MetadataDisabled) { + TestEnvironment::setEnvVar("AWS_EC2_METADATA_DISABLED", "true", 1); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)).Times(0); + DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, MetadataNotDisabled) { + TestEnvironment::setEnvVar("AWS_EC2_METADATA_DISABLED", "false", 1); + EXPECT_CALL(factories_, createInstanceProfileCredentialsProvider(Ref(*api_), _)); + DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, RelativeUri) { + TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "/path/to/creds", 1); + EXPECT_CALL(factories_, createTaskRoleCredentialsProviderMock( + Ref(*api_), _, "169.254.170.2:80/path/to/creds", "")); + DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, FullUriNoAuthorizationToken) { + TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://host/path/to/creds", 1); + EXPECT_CALL(factories_, createTaskRoleCredentialsProviderMock(Ref(*api_), _, + "http://host/path/to/creds", "")); + DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); +} + +TEST_F(DefaultCredentialsProviderChainTest, FullUriWithAuthorizationToken) { + TestEnvironment::setEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI", "http://host/path/to/creds", 1); + TestEnvironment::setEnvVar("AWS_CONTAINER_AUTHORIZATION_TOKEN", "auth_token", 1); + EXPECT_CALL(factories_, createTaskRoleCredentialsProviderMock( + Ref(*api_), _, "http://host/path/to/creds", "auth_token")); + DefaultCredentialsProviderChain chain(*api_, DummyMetadataFetcher(), factories_); +} + +TEST(CredentialsProviderChainTest, getCredentials_noCredentials) { + auto mock_provider1 = std::make_shared(); + auto mock_provider2 = std::make_shared(); + + EXPECT_CALL(*mock_provider1, getCredentials()).Times(1); + EXPECT_CALL(*mock_provider2, getCredentials()).Times(1); + + CredentialsProviderChain chain; + chain.add(mock_provider1); + chain.add(mock_provider2); + + const Credentials creds = chain.getCredentials(); + EXPECT_EQ(Credentials(), creds); +} + +TEST(CredentialsProviderChainTest, getCredentials_firstProviderReturns) { + auto mock_provider1 = std::make_shared(); + auto mock_provider2 = std::make_shared(); + + const Credentials creds("access_key", "secret_key"); + + EXPECT_CALL(*mock_provider1, getCredentials()).WillOnce(Return(creds)); + EXPECT_CALL(*mock_provider2, getCredentials()).Times(0); + + CredentialsProviderChain chain; + chain.add(mock_provider1); + chain.add(mock_provider2); + + const Credentials ret_creds = chain.getCredentials(); + EXPECT_EQ(creds, ret_creds); +} + +TEST(CredentialsProviderChainTest, getCredentials_secondProviderReturns) { + auto mock_provider1 = std::make_shared(); + auto mock_provider2 = std::make_shared(); + + const Credentials creds("access_key", "secret_key"); + + EXPECT_CALL(*mock_provider1, getCredentials()).Times(1); + EXPECT_CALL(*mock_provider2, getCredentials()).WillOnce(Return(creds)); + + CredentialsProviderChain chain; + chain.add(mock_provider1); + chain.add(mock_provider2); + + const Credentials ret_creds = chain.getCredentials(); + EXPECT_EQ(creds, ret_creds); +} + +} // namespace Aws +} // namespace Common +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/common/aws/credentials_provider_test.cc b/test/extensions/filters/http/common/aws/credentials_provider_test.cc new file mode 100644 index 000000000000..a492ec9b2a87 --- /dev/null +++ b/test/extensions/filters/http/common/aws/credentials_provider_test.cc @@ -0,0 +1,57 @@ +#include "extensions/filters/http/common/aws/credentials_provider.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Common { +namespace Aws { + +TEST(Credentials, Default) { + const auto c = Credentials(); + EXPECT_FALSE(c.accessKeyId().has_value()); + EXPECT_FALSE(c.secretAccessKey().has_value()); + EXPECT_FALSE(c.sessionToken().has_value()); +} + +TEST(Credentials, AllNull) { + const auto c = Credentials(nullptr, nullptr, nullptr); + EXPECT_FALSE(c.accessKeyId().has_value()); + EXPECT_FALSE(c.secretAccessKey().has_value()); + EXPECT_FALSE(c.sessionToken().has_value()); +} + +TEST(Credentials, AllEmpty) { + const auto c = Credentials("", "", ""); + EXPECT_FALSE(c.accessKeyId().has_value()); + EXPECT_FALSE(c.secretAccessKey().has_value()); + EXPECT_FALSE(c.sessionToken().has_value()); +} + +TEST(Credentials, OnlyAccessKeyId) { + const auto c = Credentials("access_key", "", ""); + EXPECT_EQ("access_key", c.accessKeyId()); + EXPECT_FALSE(c.secretAccessKey().has_value()); + EXPECT_FALSE(c.sessionToken().has_value()); +} + +TEST(Credentials, AccessKeyIdAndSecretKey) { + const auto c = Credentials("access_key", "secret_key", ""); + EXPECT_EQ("access_key", c.accessKeyId()); + EXPECT_EQ("secret_key", c.secretAccessKey()); + EXPECT_FALSE(c.sessionToken().has_value()); +} + +TEST(Credentials, AllNonEmpty) { + const auto c = Credentials("access_key", "secret_key", "session_token"); + EXPECT_EQ("access_key", c.accessKeyId()); + EXPECT_EQ("secret_key", c.secretAccessKey()); + EXPECT_EQ("session_token", c.sessionToken()); +} + +} // namespace Aws +} // namespace Common +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/common/aws/mocks.cc b/test/extensions/filters/http/common/aws/mocks.cc index cdc43b1ed1c3..ee5048203bb1 100644 --- a/test/extensions/filters/http/common/aws/mocks.cc +++ b/test/extensions/filters/http/common/aws/mocks.cc @@ -18,4 +18,4 @@ MockSigner::~MockSigner() {} } // namespace Common } // namespace HttpFilters } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/test/extensions/filters/http/common/aws/mocks.h b/test/extensions/filters/http/common/aws/mocks.h index 857ed433ac2b..a19f906f238b 100644 --- a/test/extensions/filters/http/common/aws/mocks.h +++ b/test/extensions/filters/http/common/aws/mocks.h @@ -27,6 +27,22 @@ class MockSigner : public Signer { MOCK_METHOD2(sign, void(Http::Message&, bool)); }; +class MockMetadataFetcher { +public: + virtual ~MockMetadataFetcher(){}; + + MOCK_CONST_METHOD3(fetch, absl::optional(const std::string&, const std::string&, + const absl::optional&)); +}; + +class DummyMetadataFetcher { +public: + absl::optional operator()(const std::string&, const std::string&, + const absl::optional&) { + return absl::nullopt; + } +}; + } // namespace Aws } // namespace Common } // namespace HttpFilters From 965bc9388192f85d113ba9396731d840f786ad49 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 1 Jul 2019 13:49:35 -0400 Subject: [PATCH 086/542] fuzz: catch newline characters in request / response header access logs (#7299) Signed-off-by: Asra Ali --- source/common/access_log/access_log_formatter.cc | 11 +++++++++-- ...ed-access_log_formatter_fuzz_test-5630958620901376 | 9 +++++++++ test/common/access_log/access_log_formatter_test.cc | 2 ++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 test/common/access_log/access_log_formatter_corpus/clusterfuzz-testcase-minimized-access_log_formatter_fuzz_test-5630958620901376 diff --git a/source/common/access_log/access_log_formatter.cc b/source/common/access_log/access_log_formatter.cc index 47752f08efcd..899ae27ff8d1 100644 --- a/source/common/access_log/access_log_formatter.cc +++ b/source/common/access_log/access_log_formatter.cc @@ -24,7 +24,9 @@ static const std::string UnspecifiedValueString = "-"; namespace { // Matches newline pattern in a StartTimeFormatter format string. -const std::regex& getNewlinePattern(){CONSTRUCT_ON_FIRST_USE(std::regex, "%[-_0^#]*[1-9]*n")}; +const std::regex& getStartTimeNewlinePattern(){ + CONSTRUCT_ON_FIRST_USE(std::regex, "%[-_0^#]*[1-9]*n")}; +const std::regex& getNewlinePattern(){CONSTRUCT_ON_FIRST_USE(std::regex, "\n")}; // Helper that handles the case when the ConnectionInfo is missing or if the desired value is // empty. @@ -169,6 +171,11 @@ void AccessLogFormatParser::parseCommandHeader(const std::string& token, const s } else { alternative_header = ""; } + // The main and alternative header should not contain invalid characters {NUL, LR, CF}. + if (std::regex_search(main_header, getNewlinePattern()) || + std::regex_search(alternative_header, getNewlinePattern())) { + throw EnvoyException("Invalid header configuration. Format string contains newline."); + } } void AccessLogFormatParser::parseCommand(const std::string& token, const size_t start, @@ -280,7 +287,7 @@ std::vector AccessLogFormatParser::parse(const std::string : ""; // Validate the input specifier here. The formatted string may be destined for a header, and // should not contain invalid characters {NUL, LR, CF}. - if (std::regex_search(args, getNewlinePattern())) { + if (std::regex_search(args, getStartTimeNewlinePattern())) { throw EnvoyException("Invalid header configuration. Format string contains newline."); } formatters.emplace_back(FormatterProviderPtr{new StartTimeFormatter(args)}); diff --git a/test/common/access_log/access_log_formatter_corpus/clusterfuzz-testcase-minimized-access_log_formatter_fuzz_test-5630958620901376 b/test/common/access_log/access_log_formatter_corpus/clusterfuzz-testcase-minimized-access_log_formatter_fuzz_test-5630958620901376 new file mode 100644 index 000000000000..d2d4428a327a --- /dev/null +++ b/test/common/access_log/access_log_formatter_corpus/clusterfuzz-testcase-minimized-access_log_formatter_fuzz_test-5630958620901376 @@ -0,0 +1,9 @@ +format: "%RESP(\n )% %REQ(\n)%" +response_trailers { +} +stream_info { + start_time: 7313138969067071779 + response_code { + value: 262144 + } +} diff --git a/test/common/access_log/access_log_formatter_test.cc b/test/common/access_log/access_log_formatter_test.cc index 994560d4a9c5..d10665a034f4 100644 --- a/test/common/access_log/access_log_formatter_test.cc +++ b/test/common/access_log/access_log_formatter_test.cc @@ -1043,6 +1043,8 @@ TEST(AccessLogFormatterTest, ParserFailures) { "%protocol%", "%REQ(TEST):%", "%REQ(TEST):3q4%", + "%REQ(\n)%", + "%REQ(?\n)%", "%RESP(TEST):%", "%RESP(X?Y):%", "%RESP(X?Y):343o24%", From cf1954eca15b32d9ff0a37422ad059bc3433eba7 Mon Sep 17 00:00:00 2001 From: danzh Date: Mon, 1 Jul 2019 13:50:52 -0400 Subject: [PATCH 087/542] quiche: add fake implemention of QuicProofSource and QuicProofVerifier (#7328) Signed-off-by: Dan Zhang --- bazel/external/quiche.BUILD | 116 ++++++++++++++++-- bazel/external/quiche.genrule_cmd | 5 + bazel/repository_locations.bzl | 6 +- ci/do_ci.sh | 4 +- source/extensions/quic_listeners/quiche/BUILD | 24 ++++ .../quiche/envoy_quic_fake_proof_source.h | 88 +++++++++++++ .../quiche/envoy_quic_fake_proof_verifier.h | 62 ++++++++++ .../quiche/platform/flags_list.h | 6 + test/extensions/quic_listeners/quiche/BUILD | 13 ++ .../quiche/envoy_quic_proof_source_test.cc | 75 +++++++++++ .../quic_listeners/quiche/platform/BUILD | 5 + .../quiche/platform/spdy_test_impl.h | 10 ++ tools/spelling_dictionary.txt | 2 + 13 files changed, 403 insertions(+), 13 deletions(-) create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h create mode 100644 test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc create mode 100644 test/extensions/quic_listeners/quiche/platform/spdy_test_impl.h diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 75c2c90572c0..250488e50400 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -53,6 +53,7 @@ genrule( # These options are only used to suppress errors in brought-in QUICHE tests. # Use #pragma GCC diagnostic ignored in integration code to suppress these errors. quiche_copt = [ + # Remove these after upstream fix. "-Wno-unused-parameter", # quic_inlined_frame.h uses offsetof() to optimize memory usage in frames. "-Wno-invalid-offsetof", @@ -134,6 +135,13 @@ envoy_cc_library( deps = [":spdy_platform"], ) +envoy_cc_test_library( + name = "spdy_platform_test", + hdrs = ["quiche/spdy/platform/api/spdy_test.h"], + repository = "@envoy", + deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:spdy_platform_test_impl_lib"], +) + envoy_cc_test_library( name = "spdy_platform_test_helpers", hdrs = ["quiche/spdy/platform/api/spdy_test_helpers.h"], @@ -195,6 +203,7 @@ envoy_cc_library( envoy_cc_library( name = "spdy_core_protocol_lib", + srcs = ["quiche/spdy/core/spdy_protocol.cc"], hdrs = [ "quiche/spdy/core/spdy_bitmasks.h", "quiche/spdy/core/spdy_protocol.h", @@ -230,6 +239,7 @@ envoy_cc_test_library( ":spdy_core_headers_handler_interface_lib", ":spdy_core_protocol_lib", ":spdy_platform", + ":spdy_platform_test", ], ) @@ -426,6 +436,13 @@ cc_proto_library( deps = [":quic_core_proto_cached_network_parameters_proto"], ) +envoy_cc_library( + name = "quic_core_proto_cached_network_parameters_proto_header", + hdrs = ["quiche/quic/core/proto/cached_network_parameters_proto.h"], + repository = "@envoy", + deps = [":quic_core_proto_cached_network_parameters_proto_cc"], +) + proto_library( name = "quic_core_proto_source_address_token_proto", srcs = ["quiche/quic/core/proto/source_address_token.proto"], @@ -437,6 +454,13 @@ cc_proto_library( deps = [":quic_core_proto_source_address_token_proto"], ) +envoy_cc_library( + name = "quic_core_proto_source_address_token_proto_header", + hdrs = ["quiche/quic/core/proto/source_address_token_proto.h"], + repository = "@envoy", + deps = [":quic_core_proto_source_address_token_proto_cc"], +) + proto_library( name = "quic_core_proto_crypto_server_config_proto", srcs = ["quiche/quic/core/proto/crypto_server_config.proto"], @@ -447,12 +471,20 @@ cc_proto_library( deps = [":quic_core_proto_crypto_server_config_proto"], ) +envoy_cc_library( + name = "quic_core_proto_crypto_server_config_proto_header", + hdrs = ["quiche/quic/core/proto/crypto_server_config_proto.h"], + repository = "@envoy", + deps = [":quic_core_proto_crypto_server_config_proto_cc"], +) + envoy_cc_library( name = "quic_core_ack_listener_interface_lib", srcs = ["quiche/quic/core/quic_ack_listener_interface.cc"], hdrs = ["quiche/quic/core/quic_ack_listener_interface.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_time_lib", @@ -466,6 +498,7 @@ envoy_cc_library( srcs = ["quiche/quic/core/quic_alarm.cc"], hdrs = ["quiche/quic/core/quic_alarm.h"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_arena_scoped_ptr_lib", @@ -477,6 +510,7 @@ envoy_cc_library( name = "quic_core_alarm_factory_interface_lib", hdrs = ["quiche/quic/core/quic_alarm_factory.h"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_alarm_interface_lib", @@ -490,6 +524,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_bandwidth.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_constants_lib", @@ -503,6 +538,7 @@ envoy_cc_library( name = "quic_core_blocked_writer_interface_lib", hdrs = ["quiche/quic/core/quic_blocked_writer_interface.h"], repository = "@envoy", + tags = ["nofips"], deps = [":quic_platform_export"], ) @@ -510,6 +546,7 @@ envoy_cc_library( name = "quic_core_arena_scoped_ptr_lib", hdrs = ["quiche/quic/core/quic_arena_scoped_ptr.h"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [":quic_platform_base"], ) @@ -535,6 +572,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_config.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_constants_lib", @@ -554,6 +592,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/congestion_control/bandwidth_sampler.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_packet_number_indexed_queue_lib", @@ -570,6 +609,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/congestion_control/bbr_sender.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_bandwidth_lib", @@ -592,6 +632,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/congestion_control/general_loss_algorithm.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_congestion_control_congestion_control_interface_lib", ":quic_core_congestion_control_rtt_stats_lib", @@ -610,6 +651,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_config_lib", @@ -634,6 +676,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_config_lib", @@ -655,6 +698,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/congestion_control/pacing_sender.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_config_lib", @@ -671,6 +715,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/congestion_control/rtt_stats.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_packets_lib", ":quic_core_time_lib", @@ -690,6 +735,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_packets_lib", @@ -711,6 +757,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_congestion_control_congestion_control_interface_lib", @@ -731,6 +778,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/congestion_control/uber_loss_algorithm.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_congestion_control_general_loss_algorithm_lib"], ) @@ -739,6 +787,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/congestion_control/windowed_filter.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_time_lib"], ) @@ -748,6 +797,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_connection.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_alarm_factory_interface_lib", @@ -765,7 +815,7 @@ envoy_cc_library( ":quic_core_packet_writer_interface_lib", ":quic_core_packets_lib", ":quic_core_pending_retransmission_lib", - ":quic_core_proto_cached_network_parameters_proto_cc", + ":quic_core_proto_cached_network_parameters_proto_header", ":quic_core_sent_packet_manager_lib", ":quic_core_time_lib", ":quic_core_types_lib", @@ -781,6 +831,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_connection_stats.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_packets_lib", @@ -795,6 +846,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_constants.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_types_lib", @@ -846,7 +898,10 @@ envoy_cc_library( "zlib", ], repository = "@envoy", - tags = ["pg3"], + tags = [ + "nofips", + "pg3", + ], textual_hdrs = [ "quiche/quic/core/crypto/common_cert_set_2.c", "quiche/quic/core/crypto/common_cert_set_2a.inc", @@ -855,6 +910,7 @@ envoy_cc_library( "quiche/quic/core/crypto/common_cert_set_3a.inc", "quiche/quic/core/crypto/common_cert_set_3b.inc", ], + visibility = ["//visibility:public"], deps = [ ":quic_core_crypto_encryption_lib", ":quic_core_crypto_hkdf_lib", @@ -865,9 +921,9 @@ envoy_cc_library( ":quic_core_error_codes_lib", ":quic_core_lru_cache_lib", ":quic_core_packets_lib", - ":quic_core_proto_cached_network_parameters_proto_cc", - ":quic_core_proto_crypto_server_config_proto_cc", - ":quic_core_proto_source_address_token_proto_cc", + ":quic_core_proto_cached_network_parameters_proto_header", + ":quic_core_proto_crypto_server_config_proto_header", + ":quic_core_proto_source_address_token_proto_header", ":quic_core_server_id_lib", ":quic_core_socket_address_coder_lib", ":quic_core_time_lib", @@ -929,6 +985,7 @@ envoy_cc_library( copts = quiche_copt, external_deps = ["ssl"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_crypto_hkdf_lib", ":quic_core_data_lib", @@ -946,6 +1003,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/crypto/quic_hkdf.h"], external_deps = ["ssl"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_platform_base", ], @@ -963,6 +1021,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_packets_lib", @@ -979,6 +1038,7 @@ envoy_cc_library( copts = quiche_copt, external_deps = ["ssl"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [":quic_platform_base"], ) @@ -998,6 +1058,7 @@ envoy_cc_library( copts = quiche_copt, external_deps = ["ssl"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_types_lib", ":quic_platform_base", @@ -1016,6 +1077,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_constants_lib", @@ -1032,6 +1094,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_error_codes.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [":quic_platform_export"], ) @@ -1042,6 +1105,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_framer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_constants_lib", ":quic_core_crypto_crypto_handshake_lib", @@ -1110,6 +1174,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_buffer_allocator_lib", @@ -1127,6 +1192,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_interval.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], ) @@ -1135,6 +1201,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_interval_set.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_interval_lib", @@ -1146,6 +1213,7 @@ envoy_cc_library( name = "quic_core_lru_cache_lib", hdrs = ["quiche/quic/core/quic_lru_cache.h"], repository = "@envoy", + tags = ["nofips"], deps = [":quic_platform_base"], ) @@ -1153,6 +1221,7 @@ envoy_cc_library( name = "quic_core_one_block_arena_lib", srcs = ["quiche/quic/core/quic_one_block_arena.h"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_arena_scoped_ptr_lib", @@ -1167,6 +1236,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_packet_creator.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_constants_lib", ":quic_core_crypto_encryption_lib", @@ -1187,6 +1257,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_packet_generator.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_crypto_random_lib", ":quic_core_packet_creator_lib", @@ -1203,6 +1274,7 @@ envoy_cc_library( name = "quic_core_packet_number_indexed_queue_lib", hdrs = ["quiche/quic/core/packet_number_indexed_queue.h"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_constants_lib", ":quic_core_types_lib", @@ -1219,6 +1291,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_packets_lib", ":quic_core_types_lib", @@ -1238,6 +1311,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_ack_listener_interface_lib", ":quic_core_bandwidth_lib", @@ -1248,7 +1322,7 @@ envoy_cc_library( ":quic_core_types_lib", ":quic_core_utils_lib", ":quic_core_versions_lib", - ":quic_platform_base", + ":quic_platform", ":quic_platform_socket_address", ":spdy_core_priority_write_scheduler_lib", ], @@ -1258,6 +1332,7 @@ envoy_cc_library( name = "quic_core_pending_retransmission_lib", hdrs = ["quiche/quic/core/quic_pending_retransmission.h"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_frames_frames_lib", ":quic_core_transmission_info_lib", @@ -1271,6 +1346,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_process_packet_interface.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_packets_lib", ":quic_platform_base", @@ -1283,6 +1359,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_received_packet_manager.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_config_lib", ":quic_core_congestion_control_rtt_stats_lib", @@ -1300,6 +1377,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_sent_packet_manager.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_congestion_control_congestion_control_lib", ":quic_core_congestion_control_general_loss_algorithm_lib", @@ -1310,7 +1388,7 @@ envoy_cc_library( ":quic_core_crypto_encryption_lib", ":quic_core_packets_lib", ":quic_core_pending_retransmission_lib", - ":quic_core_proto_cached_network_parameters_proto_cc", + ":quic_core_proto_cached_network_parameters_proto_header", ":quic_core_sustained_bandwidth_recorder_lib", ":quic_core_transmission_info_lib", ":quic_core_types_lib", @@ -1325,6 +1403,7 @@ envoy_cc_library( srcs = ["quiche/quic/core/quic_server_id.cc"], hdrs = ["quiche/quic/core/quic_server_id.h"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_platform_base", ], @@ -1344,6 +1423,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_alarm_factory_interface_lib", ":quic_core_alarm_interface_lib", @@ -1408,6 +1488,7 @@ envoy_cc_library( copts = quiche_copt, external_deps = ["ssl"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_config_lib", ":quic_core_connection_lib", @@ -1437,6 +1518,7 @@ envoy_cc_library( name = "quic_core_session_notifier_interface_lib", hdrs = ["quiche/quic/core/session_notifier_interface.h"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_frames_frames_lib", ":quic_core_time_lib", @@ -1448,6 +1530,7 @@ envoy_cc_library( srcs = ["quiche/quic/core/quic_socket_address_coder.cc"], hdrs = ["quiche/quic/core/quic_socket_address_coder.h"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_platform_base", ":quic_platform_socket_address", @@ -1458,6 +1541,7 @@ envoy_cc_library( name = "quic_core_stream_frame_data_producer_lib", hdrs = ["quiche/quic/core/quic_stream_frame_data_producer.h"], repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_types_lib"], ) @@ -1467,6 +1551,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_stream_send_buffer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_data_lib", @@ -1486,6 +1571,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_stream_sequencer_buffer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_constants_lib", ":quic_core_interval_lib", @@ -1502,6 +1588,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_sustained_bandwidth_recorder.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", ":quic_core_time_lib", @@ -1516,6 +1603,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_tag.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [":quic_platform_base"], ) @@ -1535,6 +1623,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_time_wait_list_manager.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_blocked_writer_interface_lib", ":quic_core_crypto_encryption_lib", @@ -1554,6 +1643,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_transmission_info.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_ack_listener_interface_lib", ":quic_core_frames_frames_lib", @@ -1576,6 +1666,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_error_codes_lib", @@ -1590,6 +1681,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/uber_received_packet_manager.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_received_packet_manager_lib", ":quic_core_utils_lib", @@ -1603,6 +1695,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_unacked_packet_map.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_connection_stats_lib", ":quic_core_packets_lib", @@ -1619,6 +1712,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_utils.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_constants_lib", @@ -1638,6 +1732,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_version_manager.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_versions_lib", ":quic_platform_base", @@ -1650,8 +1745,10 @@ envoy_cc_library( hdrs = ["quiche/quic/core/quic_versions.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ + ":quic_core_crypto_random_lib", ":quic_core_tag_lib", ":quic_core_types_lib", ":quic_platform_base", @@ -1762,7 +1859,10 @@ envoy_cc_test( name = "spdy_platform_api_test", srcs = ["quiche/spdy/platform/api/spdy_string_utils_test.cc"], repository = "@envoy", - deps = [":spdy_platform"], + deps = [ + ":spdy_platform", + ":spdy_platform_test", + ], ) envoy_cc_library( diff --git a/bazel/external/quiche.genrule_cmd b/bazel/external/quiche.genrule_cmd index 70fb92af61eb..3a6921d52ea3 100644 --- a/bazel/external/quiche.genrule_cmd +++ b/bazel/external/quiche.genrule_cmd @@ -31,6 +31,7 @@ cat <sed_commands /^#include/ s!net/quic/platform/impl/quic_thread_impl.h!test/extensions/quic_listeners/quiche/platform/quic_thread_impl.h! /^#include/ s!net/quiche/common/platform/impl/quiche_test_impl.h!test/extensions/quic_listeners/quiche/platform/quiche_test_impl.h! /^#include/ s!net/spdy/platform/impl/spdy_test_helpers_impl.h!test/extensions/quic_listeners/quiche/platform/spdy_test_helpers_impl.h! +/^#include/ s!net/spdy/platform/impl/spdy_test_impl.h!test/extensions/quic_listeners/quiche/platform/spdy_test_impl.h! # Rewrite include directives for platform impl files. /^#include/ s!net/(http2|spdy|quic|quiche/common)/platform/impl/!extensions/quic_listeners/quiche/platform/! @@ -50,6 +51,10 @@ cat <sed_commands /^#include/ s!third_party/zlib/zlib!zlib! /^import/ s!cached_network_parameters!quiche/quic/core/proto/cached_network_parameters! + +# Rewrite #pragma clang +/^#pragma/ s!clang!GCC! +/^#pragma/ s!-Weverything!-Wall! EOF for src_file in $(SRCS); do diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index cdb4961edadf..c0a699b12752 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -246,8 +246,8 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/curl/curl/releases/download/curl-7_65_1/curl-7.65.1.tar.gz"], ), com_googlesource_quiche = dict( - # Static snapshot of https://quiche.googlesource.com/quiche/+archive/77ae31cbfb5bf41299c8c10a06a205a8a0d37cae.tar.gz - sha256 = "20542b8f3df505e4850c8538d747ce21275b1fde64106ccae49a19a3fd7a1ac5", - urls = ["https://storage.googleapis.com/quiche-envoy-integration/77ae31cbfb5bf41299c8c10a06a205a8a0d37cae.tar.gz"], + # Static snapshot of https://quiche.googlesource.com/quiche/+archive/8f3a576515ada564d3e2b560220649f49a21dec1.tar.gz + sha256 = "ae2f3ecd7a87154ae4668f7b1876846c1cf4efa8abc987511809433f73914a68", + urls = ["https://storage.googleapis.com/quiche-envoy-integration/8f3a576515ada564d3e2b560220649f49a21dec1.tar.gz"], ), ) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index ccbe79643107..ec4d05496390 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -205,9 +205,9 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then echo "bazel with different compiletime options build with tests..." # Building all the dependencies from scratch to link them against libc++. echo "Building..." - bazel build ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg //source/exe:envoy-static + bazel build ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg //source/exe:envoy-static --build_tag_filters=-nofips echo "Building and testing ${TEST_TARGETS}" - bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg ${TEST_TARGETS} + bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c dbg ${TEST_TARGETS} --test_tag_filters=-nofips --build_tests_only # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only # these tests under "-c opt" to save time in CI. diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 54aacde06c81..408364c76301 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -13,6 +13,7 @@ envoy_cc_library( srcs = ["envoy_quic_alarm.cc"], hdrs = ["envoy_quic_alarm.h"], external_deps = ["quiche_quic_platform"], + tags = ["nofips"], deps = [ "//include/envoy/event:timer_interface", "@com_googlesource_quiche//:quic_core_alarm_interface_lib", @@ -24,6 +25,7 @@ envoy_cc_library( srcs = ["envoy_quic_alarm_factory.cc"], hdrs = ["envoy_quic_alarm_factory.h"], external_deps = ["quiche_quic_platform"], + tags = ["nofips"], deps = [ ":envoy_quic_alarm_lib", "@com_googlesource_quiche//:quic_core_alarm_factory_interface_lib", @@ -31,3 +33,25 @@ envoy_cc_library( "@com_googlesource_quiche//:quic_core_one_block_arena_lib", ], ) + +envoy_cc_library( + name = "envoy_quic_proof_source_lib", + hdrs = ["envoy_quic_fake_proof_source.h"], + external_deps = ["quiche_quic_platform"], + tags = ["nofips"], + deps = [ + "@com_googlesource_quiche//:quic_core_crypto_proof_source_interface_lib", + "@com_googlesource_quiche//:quic_core_versions_lib", + ], +) + +envoy_cc_library( + name = "envoy_quic_proof_verifier_lib", + hdrs = ["envoy_quic_fake_proof_verifier.h"], + external_deps = ["quiche_quic_platform"], + tags = ["nofips"], + deps = [ + "@com_googlesource_quiche//:quic_core_crypto_crypto_handshake_lib", + "@com_googlesource_quiche//:quic_core_versions_lib", + ], +) diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h new file mode 100644 index 000000000000..455c76aebc41 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h @@ -0,0 +1,88 @@ +#pragma once + +#include + +#include "common/common/assert.h" + +#include "absl/strings/str_cat.h" + +#pragma GCC diagnostic push + +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +#include "quiche/quic/core/crypto/proof_source.h" +#include "quiche/quic/core/quic_versions.h" + +#pragma GCC diagnostic pop + +#include "quiche/quic/platform/api/quic_reference_counted.h" +#include "quiche/quic/platform/api/quic_socket_address.h" +#include "quiche/quic/platform/api/quic_string_piece.h" + +namespace Envoy { +namespace Quic { + +// A fake implementation of quic::ProofSource which returns a fake cert and +// a fake signature for a given QUIC server config. +class EnvoyQuicFakeProofSource : public quic::ProofSource { +public: + ~EnvoyQuicFakeProofSource() override = default; + + // quic::ProofSource + // Returns a fake certs chain and its fake SCT "Fake timestamp" and fake TLS signature wrapped + // in QuicCryptoProof. + void GetProof(const quic::QuicSocketAddress& server_address, const std::string& hostname, + const std::string& server_config, quic::QuicTransportVersion /*transport_version*/, + quic::QuicStringPiece /*chlo_hash*/, + std::unique_ptr callback) override { + quic::QuicReferenceCountedPointer chain = + GetCertChain(server_address, hostname); + quic::QuicCryptoProof proof; + bool success = false; + auto signature_callback = std::make_unique(success, proof.signature); + ComputeTlsSignature(server_address, hostname, 0, server_config, std::move(signature_callback)); + ASSERT(success); + proof.leaf_cert_scts = "Fake timestamp"; + callback->Run(true, chain, proof, nullptr /* details */); + } + + // Returns a certs chain with a fake certificate "Fake cert from [host_name]". + quic::QuicReferenceCountedPointer + GetCertChain(const quic::QuicSocketAddress& /*server_address*/, + const std::string& hostname) override { + std::vector certs; + certs.push_back(absl::StrCat("Fake cert from ", hostname)); + return quic::QuicReferenceCountedPointer( + new quic::ProofSource::Chain(certs)); + } + + // Always call callback with a signature "Fake signature for { [server_config] }". + void + ComputeTlsSignature(const quic::QuicSocketAddress& /*server_address*/, + const std::string& /*hostname*/, uint16_t /*signature_algorithm*/, + quic::QuicStringPiece in, + std::unique_ptr callback) override { + callback->Run(true, absl::StrCat("Fake signature for { ", in, " }")); + } + +private: + // Used by GetProof() to get fake signature. + class FakeSignatureCallback : public quic::ProofSource::SignatureCallback { + public: + FakeSignatureCallback(bool& success, std::string& signature) + : success_(success), signature_(signature) {} + + // quic::ProofSource::SignatureCallback + void Run(bool ok, std::string signature) override { + success_ = ok; + signature_ = signature; + } + + private: + bool& success_; + std::string& signature_; + }; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h new file mode 100644 index 000000000000..09cbf73b4a38 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h @@ -0,0 +1,62 @@ +#pragma once + +#include "absl/strings/str_cat.h" + +#pragma GCC diagnostic push + +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" + +#include "quiche/quic/core/crypto/proof_verifier.h" +#include "quiche/quic/core/quic_versions.h" + +#pragma GCC diagnostic pop + +namespace Envoy { +namespace Quic { + +// A fake implementation of quic::ProofVerifier which approves the certs and +// signature produced by EnvoyQuicFakeProofSource. +class EnvoyQuicFakeProofVerifier : public quic::ProofVerifier { +public: + ~EnvoyQuicFakeProofVerifier() override = default; + + // quic::ProofVerifier + // Return success if the certs chain is valid and signature is "Fake signature for { + // [server_config] }". Otherwise failure. + quic::QuicAsyncStatus + VerifyProof(const std::string& hostname, const uint16_t /*port*/, + const std::string& server_config, quic::QuicTransportVersion /*quic_version*/, + absl::string_view /*chlo_hash*/, const std::vector& certs, + const std::string& cert_sct, const std::string& signature, + const quic::ProofVerifyContext* context, std::string* error_details, + std::unique_ptr* details, + std::unique_ptr callback) override { + if (VerifyCertChain(hostname, certs, "", cert_sct, context, error_details, details, + std::move(callback)) == quic::QUIC_SUCCESS && + signature == absl::StrCat("Fake signature for { ", server_config, " }")) { + return quic::QUIC_SUCCESS; + } + return quic::QUIC_FAILURE; + } + + // Return success if the certs chain has only one fake certificate "Fake cert from [host_name]" + // and its SCT is "Fake timestamp". Otherwise failure. + quic::QuicAsyncStatus + VerifyCertChain(const std::string& hostname, const std::vector& certs, + const std::string& /*ocsp_response*/, const std::string& cert_sct, + const quic::ProofVerifyContext* /*context*/, std::string* /*error_details*/, + std::unique_ptr* /*details*/, + std::unique_ptr /*callback*/) override { + std::string cert = absl::StrCat("Fake cert from ", hostname); + if (cert_sct == "Fake timestamp" && certs.size() == 1 && certs[0] == cert) { + return quic::QUIC_SUCCESS; + } + return quic::QUIC_FAILURE; + } + + std::unique_ptr CreateDefaultContext() override { return nullptr; } +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/platform/flags_list.h b/source/extensions/quic_listeners/quiche/platform/flags_list.h index d269e7c6959b..6de7e5b36d6c 100644 --- a/source/extensions/quic_listeners/quiche/platform/flags_list.h +++ b/source/extensions/quic_listeners/quiche/platform/flags_list.h @@ -338,6 +338,9 @@ QUICHE_FLAG( "When true, QuicDispatcher will drop packets that have an initial destination connection ID " "that is too short, instead of responding with a Version Negotiation packet to reject it.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_version_negotiation_grease, false, + "When true, QUIC Version Negotiation packets will randomly include fake versions.") + QUICHE_FLAG(bool, quic_restart_flag_quic_allow_loas_multipacket_chlo, false, "If true, inspects QUIC CHLOs for kLOAS and early creates sessions " "to allow multi-packet CHLOs") @@ -460,6 +463,9 @@ QUICHE_FLAG(bool, quic_export_server_num_packets_per_write_histogram, false, QUICHE_FLAG(int64_t, quic_headers_stream_id_in_v99, 0, "QUIC version 99 will use this stream ID for the headers stream.") +QUICHE_FLAG(bool, quic_disable_version_negotiation_grease_randomness, false, + "If true, use predictable version negotiation versions.") + QUICHE_FLAG(bool, http2_reloadable_flag_http2_testonly_default_false, false, "A testonly reloadable flag that will always default to false.") diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index ce334636c88e..2e3bd787c694 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -16,6 +16,7 @@ envoy_cc_test( name = "envoy_quic_alarm_test", srcs = ["envoy_quic_alarm_test.cc"], external_deps = ["quiche_quic_platform"], + tags = ["nofips"], deps = [ "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_factory_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_lib", @@ -23,3 +24,15 @@ envoy_cc_test( "//test/test_common:simulated_time_system_lib", ], ) + +envoy_cc_test( + name = "envoy_quic_proof_source_test", + srcs = ["envoy_quic_proof_source_test.cc"], + external_deps = ["quiche_quic_platform"], + tags = ["nofips"], + deps = [ + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_source_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_verifier_lib", + "@com_googlesource_quiche//:quic_core_versions_lib", + ], +) diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc new file mode 100644 index 000000000000..e548fe72ce72 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc @@ -0,0 +1,75 @@ +#include +#include + +#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h" +#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { + +namespace Quic { + +class TestGetProofCallback : public quic::ProofSource::Callback { +public: + TestGetProofCallback(bool& called, std::string signature, std::string leaf_cert_scts, + std::vector certs) + : called_(called), expected_signature_(std::move(signature)), + expected_leaf_certs_scts_(std::move(leaf_cert_scts)), expected_certs_(std::move(certs)) {} + + // quic::ProofSource::Callback + void Run(bool ok, const quic::QuicReferenceCountedPointer& chain, + const quic::QuicCryptoProof& proof, + std::unique_ptr details) override { + EXPECT_TRUE(ok); + EXPECT_EQ(expected_signature_, proof.signature); + EXPECT_EQ(expected_leaf_certs_scts_, proof.leaf_cert_scts); + EXPECT_EQ(expected_certs_, chain->certs); + EXPECT_EQ(nullptr, details); + called_ = true; + } + +private: + bool& called_; + std::string expected_signature_; + std::string expected_leaf_certs_scts_; + std::vector expected_certs_; +}; + +class EnvoyQuicFakeProofSourceTest : public ::testing::Test { +protected: + std::string hostname_{"www.fake.com"}; + quic::QuicSocketAddress server_address_; + quic::QuicTransportVersion version_{quic::QUIC_VERSION_UNSUPPORTED}; + quic::QuicStringPiece chlo_hash_{""}; + std::string server_config_{"Server Config"}; + std::vector expected_certs_{absl::StrCat("Fake cert from ", hostname_)}; + std::string expected_signature_{absl::StrCat("Fake signature for { ", server_config_, " }")}; + EnvoyQuicFakeProofSource proof_source_; + EnvoyQuicFakeProofVerifier proof_verifier_; +}; + +TEST_F(EnvoyQuicFakeProofSourceTest, TestGetProof) { + bool called = false; + auto callback = std::make_unique(called, expected_signature_, + "Fake timestamp", expected_certs_); + proof_source_.GetProof(server_address_, hostname_, server_config_, version_, chlo_hash_, + std::move(callback)); + EXPECT_TRUE(called); +} + +TEST_F(EnvoyQuicFakeProofSourceTest, TestVerifyProof) { + EXPECT_EQ(quic::QUIC_SUCCESS, + proof_verifier_.VerifyProof(hostname_, /*port=*/0, server_config_, version_, chlo_hash_, + expected_certs_, "Fake timestamp", expected_signature_, + nullptr, nullptr, nullptr, nullptr)); + std::vector wrong_certs{"wrong cert"}; + EXPECT_EQ(quic::QUIC_FAILURE, + proof_verifier_.VerifyProof(hostname_, /*port=*/0, server_config_, version_, chlo_hash_, + wrong_certs, "Fake timestamp", expected_signature_, nullptr, + nullptr, nullptr, nullptr)); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/platform/BUILD b/test/extensions/quic_listeners/quiche/platform/BUILD index 346c76ce24b7..654ddcd98160 100644 --- a/test/extensions/quic_listeners/quiche/platform/BUILD +++ b/test/extensions/quic_listeners/quiche/platform/BUILD @@ -209,6 +209,11 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "spdy_platform_test_impl_lib", + hdrs = ["spdy_test_impl.h"], +) + envoy_cc_test( name = "envoy_quic_clock_test", srcs = ["envoy_quic_clock_test.cc"], diff --git a/test/extensions/quic_listeners/quiche/platform/spdy_test_impl.h b/test/extensions/quic_listeners/quiche/platform/spdy_test_impl.h new file mode 100644 index 000000000000..e351735c3ab1 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/platform/spdy_test_impl.h @@ -0,0 +1,10 @@ +#pragma once + +// NOLINT(namespace-envoy) + +// This file is part of the QUICHE platform implementation, and is not to be +// consumed or referenced directly by other Envoy code. It serves purely as a +// porting layer for QUICHE. + +#include "gmock/gmock.h" +#include "gtest/gtest.h" diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 982232db171f..7fdc7d95db96 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -18,6 +18,7 @@ CB CBs CDS CHACHA +CHLO CIDR CLA CLI @@ -226,6 +227,7 @@ Runn SA SAN SANs +SCT SDK SDS SENDSRCADDR From 3c8a1ef9d128bf11ccc179b2e171e180a0861332 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 1 Jul 2019 14:40:38 -0400 Subject: [PATCH 088/542] api: use unique outer class names for tap proto files (#7434) When building protos using the Java protoc, multiple input files mapping to the same output file causes an erorr. This updates the name of the generated files for the tap proto files to be unique. Signed-off-by: Snow Pettersen --- api/envoy/service/tap/v2alpha/tap.proto | 2 +- api/envoy/service/tap/v2alpha/tapds.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/service/tap/v2alpha/tap.proto b/api/envoy/service/tap/v2alpha/tap.proto index c4cc919c275c..9dccbb314bab 100644 --- a/api/envoy/service/tap/v2alpha/tap.proto +++ b/api/envoy/service/tap/v2alpha/tap.proto @@ -7,7 +7,7 @@ package envoy.service.tap.v2alpha; import "validate/validate.proto"; -option java_outer_classname = "CommonProto"; +option java_outer_classname = "TapProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.tap.v2alpha"; diff --git a/api/envoy/service/tap/v2alpha/tapds.proto b/api/envoy/service/tap/v2alpha/tapds.proto index a7aea7de9e7c..2f61074ca131 100644 --- a/api/envoy/service/tap/v2alpha/tapds.proto +++ b/api/envoy/service/tap/v2alpha/tapds.proto @@ -8,7 +8,7 @@ package envoy.service.tap.v2alpha; import "google/api/annotations.proto"; -option java_outer_classname = "CommonProto"; +option java_outer_classname = "TapDsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.tap.v2alpha"; From e0e7628c3bc4227245f15c4f047ddad04912351c Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Mon, 1 Jul 2019 14:58:39 -0700 Subject: [PATCH 089/542] dns: adjust DNS refresh rate respecting to record TTL (#6975) Signed-off-by: Yan Xue --- api/envoy/api/v2/cds.proto | 7 +- .../upstream/service_discovery.rst | 12 + docs/root/intro/version_history.rst | 1 + include/envoy/network/dns.h | 16 +- source/common/network/dns_impl.cc | 11 +- source/common/upstream/logical_dns_cluster.cc | 18 +- source/common/upstream/logical_dns_cluster.h | 1 + source/common/upstream/strict_dns_cluster.cc | 26 +- source/common/upstream/strict_dns_cluster.h | 1 + .../clusters/redis/redis_cluster.cc | 13 +- .../extensions/clusters/redis/redis_cluster.h | 4 +- .../dynamic_forward_proxy/dns_cache_impl.cc | 29 +- .../dynamic_forward_proxy/dns_cache_impl.h | 3 +- test/common/network/dns_impl_test.cc | 348 +++++++++++------- test/common/upstream/upstream_impl_test.cc | 35 ++ .../clusters/redis/redis_cluster_test.cc | 5 +- .../dns_cache_impl_test.cc | 40 +- test/test_common/utility.cc | 9 +- test/test_common/utility.h | 5 +- tools/spelling_dictionary.txt | 1 + 20 files changed, 371 insertions(+), 214 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 004e9ddc4a31..83fe4329e15b 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -51,7 +51,7 @@ service ClusterDiscoveryService { // [#protodoc-title: Clusters] // Configuration for a single upstream cluster. -// [#comment:next free field: 39] +// [#comment:next free field: 40] message Cluster { // Supplies the name of the cluster which must be unique across all clusters. // The cluster name is used when emitting @@ -274,6 +274,11 @@ message Cluster { google.protobuf.Duration dns_refresh_rate = 16 [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + // Optional configuration for setting cluster's DNS refresh rate. If the value is set to true, + // cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS + // resolution. + bool respect_dns_ttl = 39; + // When V4_ONLY is selected, the DNS resolver will only perform a lookup for // addresses in the IPv4 family. If V6_ONLY is selected, the DNS resolver will // only perform a lookup for addresses in the IPv6 family. If AUTO is diff --git a/docs/root/intro/arch_overview/upstream/service_discovery.rst b/docs/root/intro/arch_overview/upstream/service_discovery.rst index 4d00f638dbdc..3f6bd5d7b4f4 100644 --- a/docs/root/intro/arch_overview/upstream/service_discovery.rst +++ b/docs/root/intro/arch_overview/upstream/service_discovery.rst @@ -39,6 +39,12 @@ This means that care should be taken if active health checking is used with DNS to the same IPs: if an IP is repeated many times between DNS names it might cause undue load on the upstream host. +If :ref:`respect_dns_ttl ` is enabled, DNS record TTLs and +:ref:`dns_refresh_rate ` are used to control DNS refresh rate. +For strict DNS cluster, if the minimum of all record TTLs is 0, :ref:`dns_refresh_rate ` +will be used as the cluster's DNS refresh rate. :ref:`dns_refresh_rate ` +defaults to 5000ms if not specified. + .. _arch_overview_service_discovery_types_logical_dns: Logical DNS @@ -58,6 +64,12 @@ When interacting with large scale web services, this is the best of all possible asynchronous/eventually consistent DNS resolution, long lived connections, and zero blocking in the forwarding path. +If :ref:`respect_dns_ttl ` is enabled, DNS record TTLs and +:ref:`dns_refresh_rate ` are used to control DNS refresh rate. +For logical DNS cluster, if the TTL of first record is 0, :ref:`dns_refresh_rate ` +will be used as the cluster's DNS refresh rate. :ref:`dns_refresh_rate ` +defaults to 5000ms if not specified. + .. _arch_overview_service_discovery_types_original_destination: Original destination diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 453ea130d782..ecd9a14ee2bb 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -17,6 +17,7 @@ Version history * build: releases are built with Clang and linked with LLD. * control-plane: management servers can respond with HTTP 304 to indicate that config is up to date for Envoy proxies polling a :ref:`REST API Config Type ` * csrf: added support for whitelisting additional source origins. +* dns: added support for getting DNS record TTL which is used by STRICT_DNS/LOGICAL_DNS cluster as DNS refresh rate. * dubbo_proxy: support the :ref:`Dubbo proxy filter `. * eds: added support to specify max time for which endpoints can be used :ref:`gRPC filter `. * event: added :ref:`loop duration and poll delay statistics `. diff --git a/include/envoy/network/dns.h b/include/envoy/network/dns.h index 6e898ee9e4e5..26f1f3cbfaa2 100644 --- a/include/envoy/network/dns.h +++ b/include/envoy/network/dns.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -24,6 +25,17 @@ class ActiveDnsQuery { virtual void cancel() PURE; }; +/** + * DNS response. + */ +struct DnsResponse { + DnsResponse(const Address::InstanceConstSharedPtr& address, const std::chrono::seconds ttl) + : address_(address), ttl_(ttl) {} + + const Address::InstanceConstSharedPtr address_; + const std::chrono::seconds ttl_; +}; + enum class DnsLookupFamily { V4Only, V6Only, Auto }; /** @@ -35,10 +47,10 @@ class DnsResolver { /** * Called when a resolution attempt is complete. - * @param address_list supplies the list of resolved IP addresses. The list will be empty if + * @param response supplies the list of resolved IP addresses and TTLs. The list will be empty if * the resolution failed. */ - using ResolveCb = std::function&& address_list)>; + using ResolveCb = std::function&& response)>; /** * Initiate an async DNS resolution. diff --git a/source/common/network/dns_impl.cc b/source/common/network/dns_impl.cc index 29e2b48042fa..c4409d473b6b 100644 --- a/source/common/network/dns_impl.cc +++ b/source/common/network/dns_impl.cc @@ -79,7 +79,7 @@ void DnsResolverImpl::PendingResolution::onAresGetAddrInfoCallback(int status, i completed_ = true; } - std::list address_list; + std::list address_list; if (status == ARES_SUCCESS) { if (addrinfo != nullptr && addrinfo->nodes != nullptr) { if (addrinfo->nodes->ai_family == AF_INET) { @@ -89,7 +89,10 @@ void DnsResolverImpl::PendingResolution::onAresGetAddrInfoCallback(int status, i address.sin_family = AF_INET; address.sin_port = 0; address.sin_addr = reinterpret_cast(ai->ai_addr)->sin_addr; - address_list.emplace_back(new Address::Ipv4Instance(&address)); + + address_list.emplace_back( + DnsResponse(std::make_shared(&address), + std::chrono::seconds(ai->ai_ttl))); } } else if (addrinfo->nodes->ai_family == AF_INET6) { for (const ares_addrinfo_node* ai = addrinfo->nodes; ai != nullptr; ai = ai->ai_next) { @@ -98,7 +101,9 @@ void DnsResolverImpl::PendingResolution::onAresGetAddrInfoCallback(int status, i address.sin6_family = AF_INET6; address.sin6_port = 0; address.sin6_addr = reinterpret_cast(ai->ai_addr)->sin6_addr; - address_list.emplace_back(new Address::Ipv6Instance(address)); + address_list.emplace_back( + DnsResponse(std::make_shared(address), + std::chrono::seconds(ai->ai_ttl))); } } } diff --git a/source/common/upstream/logical_dns_cluster.cc b/source/common/upstream/logical_dns_cluster.cc index 3ae00a4dd17b..88ce5e02da5b 100644 --- a/source/common/upstream/logical_dns_cluster.cc +++ b/source/common/upstream/logical_dns_cluster.cc @@ -26,6 +26,7 @@ LogicalDnsCluster::LogicalDnsCluster( dns_resolver_(dns_resolver), dns_refresh_rate_ms_( std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate, 5000))), + respect_dns_ttl_(cluster.respect_dns_ttl()), resolve_timer_( factory_context.dispatcher().createTimer([this]() -> void { startResolve(); })), local_info_(factory_context.localInfo()), @@ -70,18 +71,23 @@ void LogicalDnsCluster::startResolve() { active_dns_query_ = dns_resolver_->resolve( dns_address, dns_lookup_family_, - [this, - dns_address](std::list&& address_list) -> void { + [this, dns_address](std::list&& response) -> void { active_dns_query_ = nullptr; ENVOY_LOG(debug, "async DNS resolution complete for {}", dns_address); info_->stats().update_success_.inc(); - if (!address_list.empty()) { + std::chrono::milliseconds refresh_rate = dns_refresh_rate_ms_; + if (!response.empty()) { // TODO(mattklein123): Move port handling into the DNS interface. - ASSERT(address_list.front() != nullptr); + ASSERT(response.front().address_ != nullptr); Network::Address::InstanceConstSharedPtr new_address = - Network::Utility::getAddressWithPort(*address_list.front(), + Network::Utility::getAddressWithPort(*(response.front().address_), Network::Utility::portFromTcpUrl(dns_url_)); + + if (respect_dns_ttl_ && response.front().ttl_ != std::chrono::seconds(0)) { + refresh_rate = response.front().ttl_; + } + if (!logical_host_) { logical_host_.reset( new LogicalHost(info_, hostname_, new_address, localityLbEndpoint(), lbEndpoint())); @@ -107,7 +113,7 @@ void LogicalDnsCluster::startResolve() { } onPreInitComplete(); - resolve_timer_->enableTimer(dns_refresh_rate_ms_); + resolve_timer_->enableTimer(refresh_rate); }); } diff --git a/source/common/upstream/logical_dns_cluster.h b/source/common/upstream/logical_dns_cluster.h index 07829f286533..68e786c80af6 100644 --- a/source/common/upstream/logical_dns_cluster.h +++ b/source/common/upstream/logical_dns_cluster.h @@ -62,6 +62,7 @@ class LogicalDnsCluster : public ClusterImplBase { Network::DnsResolverSharedPtr dns_resolver_; const std::chrono::milliseconds dns_refresh_rate_ms_; + const bool respect_dns_ttl_; Network::DnsLookupFamily dns_lookup_family_; Event::TimerPtr resolve_timer_; std::string dns_url_; diff --git a/source/common/upstream/strict_dns_cluster.cc b/source/common/upstream/strict_dns_cluster.cc index e06bc79a87b1..71aa2b3c71e0 100644 --- a/source/common/upstream/strict_dns_cluster.cc +++ b/source/common/upstream/strict_dns_cluster.cc @@ -12,7 +12,8 @@ StrictDnsClusterImpl::StrictDnsClusterImpl( added_via_api), local_info_(factory_context.localInfo()), dns_resolver_(dns_resolver), dns_refresh_rate_ms_( - std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate, 5000))) { + std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate, 5000))), + respect_dns_ttl_(cluster.respect_dns_ttl()) { std::list resolve_targets; const envoy::api::v2::ClusterLoadAssignment load_assignment( cluster.has_load_assignment() ? cluster.load_assignment() @@ -90,24 +91,28 @@ void StrictDnsClusterImpl::ResolveTarget::startResolve() { active_query_ = parent_.dns_resolver_->resolve( dns_address_, parent_.dns_lookup_family_, - [this](std::list&& address_list) -> void { + [this](std::list&& response) -> void { active_query_ = nullptr; ENVOY_LOG(trace, "async DNS resolution complete for {}", dns_address_); parent_.info_->stats().update_success_.inc(); std::unordered_map updated_hosts; HostVector new_hosts; - for (const Network::Address::InstanceConstSharedPtr& address : address_list) { + std::chrono::seconds ttl_refresh_rate = std::chrono::seconds::max(); + for (const auto& resp : response) { // TODO(mattklein123): Currently the DNS interface does not consider port. We need to // make a new address that has port in it. We need to both support IPv6 as well as // potentially move port handling into the DNS interface itself, which would work better // for SRV. - ASSERT(address != nullptr); + ASSERT(resp.address_ != nullptr); new_hosts.emplace_back(new HostImpl( - parent_.info_, dns_address_, Network::Utility::getAddressWithPort(*address, port_), + parent_.info_, dns_address_, + Network::Utility::getAddressWithPort(*(resp.address_), port_), lb_endpoint_.metadata(), lb_endpoint_.load_balancing_weight().value(), locality_lb_endpoint_.locality(), lb_endpoint_.endpoint().health_check_config(), locality_lb_endpoint_.priority(), lb_endpoint_.health_status())); + + ttl_refresh_rate = min(ttl_refresh_rate, resp.ttl_); } HostVector hosts_added; @@ -130,7 +135,16 @@ void StrictDnsClusterImpl::ResolveTarget::startResolve() { // completes. This is not perfect but is easier to code and unclear if the extra // complexity is needed so will start with this. parent_.onPreInitComplete(); - resolve_timer_->enableTimer(parent_.dns_refresh_rate_ms_); + + std::chrono::milliseconds final_refresh_rate = parent_.dns_refresh_rate_ms_; + + if (parent_.respect_dns_ttl_ && ttl_refresh_rate != std::chrono::seconds(0)) { + final_refresh_rate = ttl_refresh_rate; + ENVOY_LOG(debug, "DNS refresh rate reset for {}, refresh rate {} ms", dns_address_, + final_refresh_rate.count()); + } + + resolve_timer_->enableTimer(final_refresh_rate); }); } diff --git a/source/common/upstream/strict_dns_cluster.h b/source/common/upstream/strict_dns_cluster.h index 27b3fc5f3062..2b0c8f4f135d 100644 --- a/source/common/upstream/strict_dns_cluster.h +++ b/source/common/upstream/strict_dns_cluster.h @@ -52,6 +52,7 @@ class StrictDnsClusterImpl : public BaseDynamicClusterImpl { Network::DnsResolverSharedPtr dns_resolver_; std::list resolve_targets_; const std::chrono::milliseconds dns_refresh_rate_ms_; + const bool respect_dns_ttl_; Network::DnsLookupFamily dns_lookup_family_; uint32_t overprovisioning_factor_; }; diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index 70cf9e2d9051..06c0ac9d6b0d 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -128,10 +128,10 @@ void RedisCluster::DnsDiscoveryResolveTarget::startResolve() { active_query_ = parent_.dns_resolver_->resolve( dns_address_, parent_.dns_lookup_family_, - [this](std::list&& address_list) -> void { + [this](std::list&& response) -> void { active_query_ = nullptr; ENVOY_LOG(trace, "async DNS resolution complete for {}", dns_address_); - parent_.redis_discovery_session_.registerDiscoveryAddress(address_list, port_); + parent_.redis_discovery_session_.registerDiscoveryAddress(std::move(response), port_); parent_.redis_discovery_session_.startResolve(); }); } @@ -190,13 +190,12 @@ void RedisCluster::RedisDiscoveryClient::onEvent(Network::ConnectionEvent event) } void RedisCluster::RedisDiscoverySession::registerDiscoveryAddress( - const std::list& address_list, - const uint32_t port) { + std::list&& response, const uint32_t port) { // Since the address from DNS does not have port, we need to make a new address that has port in // it. - for (const Network::Address::InstanceConstSharedPtr& address : address_list) { - ASSERT(address != nullptr); - discovery_address_list_.push_back(Network::Utility::getAddressWithPort(*address, port)); + for (const Network::DnsResponse& res : response) { + ASSERT(res.address_ != nullptr); + discovery_address_list_.push_back(Network::Utility::getAddressWithPort(*(res.address_), port)); } } diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index fad26cafcaa6..d35dc0dfcd39 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -195,9 +195,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { ~RedisDiscoverySession() override; - void registerDiscoveryAddress( - const std::list& address_list, - const uint32_t port); + void registerDiscoveryAddress(std::list&& response, const uint32_t port); // Start discovery against a random host from existing hosts void startResolve(); diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index 81dbc25805ca..b8173e2dfd00 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -134,19 +134,18 @@ void DnsCacheImpl::startResolve(const std::string& host, PrimaryHostInfo& host_i ENVOY_LOG(debug, "starting main thread resolve for host='{}' dns='{}' port='{}'", host, host_info.host_to_resolve_, host_info.port_); ASSERT(host_info.active_query_ == nullptr); + stats_.dns_query_attempt_.inc(); - host_info.active_query_ = resolver_->resolve( - host_info.host_to_resolve_, dns_lookup_family_, - [this, host](const std::list&& address_list) { - finishResolve(host, address_list); - }); + host_info.active_query_ = + resolver_->resolve(host_info.host_to_resolve_, dns_lookup_family_, + [this, host](std::list&& response) { + finishResolve(host, std::move(response)); + }); } -void DnsCacheImpl::finishResolve( - const std::string& host, - const std::list& address_list) { - ENVOY_LOG(debug, "main thread resolve complete for host '{}'. {} results", host, - address_list.size()); +void DnsCacheImpl::finishResolve(const std::string& host, + std::list&& response) { + ENVOY_LOG(debug, "main thread resolve complete for host '{}'. {} results", host, response.size()); const auto primary_host_it = primary_hosts_.find(host); ASSERT(primary_host_it != primary_hosts_.end()); @@ -158,12 +157,12 @@ void DnsCacheImpl::finishResolve( std::make_shared(main_thread_dispatcher_.timeSource()); } - const auto new_address = - !address_list.empty() - ? Network::Utility::getAddressWithPort(*address_list.front(), primary_host_info.port_) - : nullptr; + const auto new_address = !response.empty() + ? Network::Utility::getAddressWithPort(*(response.front().address_), + primary_host_info.port_) + : nullptr; - if (address_list.empty()) { + if (response.empty()) { stats_.dns_query_failure_.inc(); } else { stats_.dns_query_success_.inc(); diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index 7b6cd6940c79..7ecccdf3a8d2 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -113,8 +113,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable& address_list); + void finishResolve(const std::string& host, std::list&& response); void runAddUpdateCallbacks(const std::string& host, const DnsHostInfoSharedPtr& host_info); void runRemoveCallbacks(const std::string& host); void updateTlsHostsMap(); diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index bc2727363c95..78ad6b566524 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -55,9 +55,9 @@ enum record_type { A, AAAA }; class TestDnsServerQuery { public: TestDnsServerQuery(ConnectionPtr connection, const HostMap& hosts_A, const HostMap& hosts_AAAA, - const CNameMap& cnames) + const CNameMap& cnames, const std::chrono::seconds& record_ttl) : connection_(std::move(connection)), hosts_A_(hosts_A), hosts_AAAA_(hosts_AAAA), - cnames_(cnames) { + cnames_(cnames), record_ttl_(record_ttl) { connection_->addReadFilter(Network::ReadFilterSharedPtr{new ReadFilter(*this)}); } @@ -199,7 +199,7 @@ class TestDnsServerQuery { DNS_RR_SET_TYPE(cname_rr_fixed, T_CNAME); DNS_RR_SET_LEN(cname_rr_fixed, encodedCname.size() + 1); DNS_RR_SET_CLASS(cname_rr_fixed, C_IN); - DNS_RR_SET_TTL(cname_rr_fixed, 0); + DNS_RR_SET_TTL(cname_rr_fixed, parent_.record_ttl_.count()); write_buffer.add(question, name_len); write_buffer.add(cname_rr_fixed, RRFIXEDSZ); write_buffer.add(encodedCname.c_str(), encodedCname.size() + 1); @@ -215,7 +215,7 @@ class TestDnsServerQuery { DNS_RR_SET_LEN(response_rr_fixed, sizeof(in6_addr)); } DNS_RR_SET_CLASS(response_rr_fixed, C_IN); - DNS_RR_SET_TTL(response_rr_fixed, 0); + DNS_RR_SET_TTL(response_rr_fixed, parent_.record_ttl_.count()); if (ips != nullptr) { for (const auto& it : *ips) { write_buffer.add(ip_question, ip_name_len); @@ -253,11 +253,12 @@ class TestDnsServerQuery { const HostMap& hosts_A_; const HostMap& hosts_AAAA_; const CNameMap& cnames_; + const std::chrono::seconds& record_ttl_; }; class TestDnsServer : public ListenerCallbacks { public: - TestDnsServer(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} + TestDnsServer(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher), record_ttl_(0) {} void onAccept(ConnectionSocketPtr&& socket, bool) override { Network::ConnectionPtr new_connection = dispatcher_.createServerConnection( @@ -266,8 +267,8 @@ class TestDnsServer : public ListenerCallbacks { } void onNewConnection(ConnectionPtr&& new_connection) override { - TestDnsServerQuery* query = - new TestDnsServerQuery(std::move(new_connection), hosts_A_, hosts_AAAA_, cnames_); + TestDnsServerQuery* query = new TestDnsServerQuery(std::move(new_connection), hosts_A_, + hosts_AAAA_, cnames_, record_ttl_); queries_.emplace_back(query); } @@ -283,12 +284,15 @@ class TestDnsServer : public ListenerCallbacks { cnames_[hostname] = cname; } + void setRecordTtl(const std::chrono::seconds& ttl) { record_ttl_ = ttl; } + private: Event::Dispatcher& dispatcher_; HostMap hosts_A_; HostMap hosts_AAAA_; CNameMap cnames_; + std::chrono::seconds record_ttl_; // All queries are tracked so we can do resource reclamation when the test is // over. std::vector> queries_; @@ -426,6 +430,15 @@ class DnsImplTest : public testing::TestWithParam { server_.reset(); } + std::list + getAddressList(const std::list& response) { + std::list address; + + for_each(response.begin(), response.end(), + [&](DnsResponse resp) { address.emplace_back(resp.address_); }); + return address; + } + protected: // Should the DnsResolverImpl use a zero timeout for c-ares queries? virtual bool zero_timeout() const { return false; } @@ -459,12 +472,11 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, DnsImplTest, // development, where segfaults were encountered due to callback invocations on // destruction. TEST_P(DnsImplTest, DestructPending) { - EXPECT_NE(nullptr, - resolver_->resolve("", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - FAIL(); - UNREFERENCED_PARAMETER(results); - })); + EXPECT_NE(nullptr, resolver_->resolve("", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + FAIL(); + UNREFERENCED_PARAMETER(results); + })); // Also validate that pending events are around to exercise the resource // reclamation path. EXPECT_GT(peer_->events().size(), 0U); @@ -475,23 +487,21 @@ TEST_P(DnsImplTest, DestructPending) { // asynchronous behavior or network events. TEST_P(DnsImplTest, LocalLookup) { std::list address_list; - EXPECT_NE(nullptr, - resolver_->resolve("", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(address_list.empty()); if (GetParam() == Address::IpVersion::v4) { // EXPECT_CALL(dispatcher_, post(_)); - EXPECT_EQ(nullptr, - resolver_->resolve("localhost", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - })); + EXPECT_EQ(nullptr, resolver_->resolve("localhost", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + })); EXPECT_TRUE(hasAddress(address_list, "127.0.0.1")); EXPECT_FALSE(hasAddress(address_list, "::1")); } @@ -500,20 +510,18 @@ TEST_P(DnsImplTest, LocalLookup) { const std::string error_msg = "Synchronous DNS IPv6 localhost resolution failed. Please verify localhost resolves to ::1 " "in /etc/hosts, since this misconfiguration is a common cause of these failures."; - EXPECT_EQ(nullptr, - resolver_->resolve("localhost", DnsLookupFamily::V6Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - })) + EXPECT_EQ(nullptr, resolver_->resolve("localhost", DnsLookupFamily::V6Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + })) << error_msg; EXPECT_TRUE(hasAddress(address_list, "::1")) << error_msg; EXPECT_FALSE(hasAddress(address_list, "127.0.0.1")); - EXPECT_EQ(nullptr, - resolver_->resolve("localhost", DnsLookupFamily::Auto, - [&](std::list&& results) -> void { - address_list = std::move(results); - })) + EXPECT_EQ(nullptr, resolver_->resolve("localhost", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + })) << error_msg; EXPECT_FALSE(hasAddress(address_list, "127.0.0.1")); EXPECT_TRUE(hasAddress(address_list, "::1")) << error_msg; @@ -523,32 +531,29 @@ TEST_P(DnsImplTest, LocalLookup) { TEST_P(DnsImplTest, DnsIpAddressVersionV6) { std::list address_list; server_->addHosts("some.good.domain", {"1::2"}, AAAA); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "1::2")); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_FALSE(hasAddress(address_list, "1::2")); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::V6Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V6Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "1::2")); @@ -560,18 +565,18 @@ TEST_P(DnsImplTest, CallbackException) { // state providing regression coverage for #4307. EXPECT_EQ(nullptr, resolver_->resolve( "1.2.3.4", DnsLookupFamily::V4Only, - [&](std::list && + [&](const std::list & /*results*/) -> void { throw EnvoyException("Envoy exception"); })); EXPECT_THROW_WITH_MESSAGE(dispatcher_->run(Event::Dispatcher::RunType::Block), EnvoyException, "Envoy exception"); EXPECT_EQ(nullptr, resolver_->resolve( "1.2.3.4", DnsLookupFamily::V4Only, - [&](std::list && + [&](const std::list & /*results*/) -> void { throw std::runtime_error("runtime error"); })); EXPECT_THROW_WITH_MESSAGE(dispatcher_->run(Event::Dispatcher::RunType::Block), EnvoyException, "runtime error"); EXPECT_EQ(nullptr, resolver_->resolve("1.2.3.4", DnsLookupFamily::V4Only, - [&](std::list && + [&](const std::list & /*results*/) -> void { throw std::string(); })); EXPECT_THROW_WITH_MESSAGE(dispatcher_->run(Event::Dispatcher::RunType::Block), EnvoyException, "unknown"); @@ -580,32 +585,29 @@ TEST_P(DnsImplTest, CallbackException) { TEST_P(DnsImplTest, DnsIpAddressVersion) { std::list address_list; server_->addHosts("some.good.domain", {"1.2.3.4"}, A); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "1.2.3.4")); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "1.2.3.4")); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::V6Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V6Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_FALSE(hasAddress(address_list, "1.2.3.4")); @@ -616,22 +618,20 @@ TEST_P(DnsImplTest, DnsIpAddressVersion) { TEST_P(DnsImplTest, RemoteAsyncLookup) { server_->addHosts("some.good.domain", {"201.134.56.7"}, A); std::list address_list; - EXPECT_NE(nullptr, - resolver_->resolve("some.bad.domain", DnsLookupFamily::Auto, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.bad.domain", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(address_list.empty()); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "201.134.56.7")); @@ -641,12 +641,11 @@ TEST_P(DnsImplTest, RemoteAsyncLookup) { TEST_P(DnsImplTest, MultiARecordLookup) { server_->addHosts("some.good.domain", {"201.134.56.7", "123.4.5.6", "6.5.4.3"}, A); std::list address_list; - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "201.134.56.7")); @@ -658,12 +657,11 @@ TEST_P(DnsImplTest, CNameARecordLookupV4) { server_->addCName("root.cnam.domain", "result.cname.domain"); server_->addHosts("result.cname.domain", {"201.134.56.7"}, A); std::list address_list; - EXPECT_NE(nullptr, - resolver_->resolve("root.cnam.domain", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("root.cnam.domain", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "201.134.56.7")); @@ -673,12 +671,11 @@ TEST_P(DnsImplTest, CNameARecordLookupWithV6) { server_->addCName("root.cnam.domain", "result.cname.domain"); server_->addHosts("result.cname.domain", {"201.134.56.7"}, A); std::list address_list; - EXPECT_NE(nullptr, - resolver_->resolve("root.cnam.domain", DnsLookupFamily::Auto, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("root.cnam.domain", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "201.134.56.7")); @@ -688,36 +685,33 @@ TEST_P(DnsImplTest, MultiARecordLookupWithV6) { server_->addHosts("some.good.domain", {"201.134.56.7", "123.4.5.6", "6.5.4.3"}, A); server_->addHosts("some.good.domain", {"1::2", "1::2:3", "1::2:3:4"}, AAAA); std::list address_list; - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "201.134.56.7")); EXPECT_TRUE(hasAddress(address_list, "123.4.5.6")); EXPECT_TRUE(hasAddress(address_list, "6.5.4.3")); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "1::2")); EXPECT_TRUE(hasAddress(address_list, "1::2:3")); EXPECT_TRUE(hasAddress(address_list, "1::2:3:4")); - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::V6Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V6Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(hasAddress(address_list, "1::2")); @@ -729,17 +723,15 @@ TEST_P(DnsImplTest, MultiARecordLookupWithV6) { TEST_P(DnsImplTest, Cancel) { server_->addHosts("some.good.domain", {"201.134.56.7"}, A); - ActiveDnsQuery* query = resolver_->resolve( - "some.domain", DnsLookupFamily::Auto, - [](const std::list &&) -> void { FAIL(); }); + ActiveDnsQuery* query = resolver_->resolve("some.domain", DnsLookupFamily::Auto, + [](std::list &&) -> void { FAIL(); }); std::list address_list; - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); ASSERT_NE(nullptr, query); query->cancel(); @@ -748,6 +740,89 @@ TEST_P(DnsImplTest, Cancel) { EXPECT_TRUE(hasAddress(address_list, "201.134.56.7")); } +// Validate working of querying ttl of resource record. +TEST_P(DnsImplTest, RecordTtlLookup) { + if (GetParam() == Address::IpVersion::v4) { + EXPECT_EQ(nullptr, resolver_->resolve("localhost", DnsLookupFamily::V4Only, + [](std::list&& results) -> void { + for (auto address : results) { + EXPECT_EQ(address.ttl_, std::chrono::seconds(0)); + } + })); + } + + if (GetParam() == Address::IpVersion::v6) { + EXPECT_EQ(nullptr, resolver_->resolve("localhost", DnsLookupFamily::V6Only, + [](std::list&& results) -> void { + for (auto address : results) { + EXPECT_EQ(address.ttl_, std::chrono::seconds(0)); + } + })); + + EXPECT_EQ(nullptr, resolver_->resolve("localhost", DnsLookupFamily::Auto, + [](std::list&& results) -> void { + for (auto address : results) { + EXPECT_EQ(address.ttl_, std::chrono::seconds(0)); + } + })); + } + + server_->addHosts("some.good.domain", {"201.134.56.7", "123.4.5.6", "6.5.4.3"}, A); + server_->addHosts("some.good.domain", {"1::2", "1::2:3", "1::2:3:4"}, AAAA); + server_->setRecordTtl(std::chrono::seconds(300)); + + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + for (auto address : results) { + EXPECT_EQ(address.ttl_, std::chrono::seconds(300)); + } + + dispatcher_->exit(); + })); + + dispatcher_->run(Event::Dispatcher::RunType::Block); + + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::Auto, + [&](std::list&& results) -> void { + for (auto address : results) { + EXPECT_EQ(address.ttl_, std::chrono::seconds(300)); + } + dispatcher_->exit(); + })); + + dispatcher_->run(Event::Dispatcher::RunType::Block); + + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V6Only, + [&](std::list&& results) -> void { + for (auto address : results) { + EXPECT_EQ(address.ttl_, std::chrono::seconds(300)); + } + dispatcher_->exit(); + })); + + dispatcher_->run(Event::Dispatcher::RunType::Block); + + std::list address_list; + + server_->addHosts("domain.onion", {"1.2.3.4"}, A); + server_->addHosts("domain.onion.", {"2.3.4.5"}, A); + + // test onion domain + EXPECT_EQ(nullptr, resolver_->resolve("domain.onion", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); + EXPECT_TRUE(address_list.empty()); + + EXPECT_EQ(nullptr, resolver_->resolve("domain.onion.", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); + EXPECT_TRUE(address_list.empty()); +} + class DnsImplZeroTimeoutTest : public DnsImplTest { protected: bool zero_timeout() const override { return true; } @@ -762,12 +837,11 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, DnsImplZeroTimeoutTest, TEST_P(DnsImplZeroTimeoutTest, Timeout) { server_->addHosts("some.good.domain", {"201.134.56.7"}, A); std::list address_list; - EXPECT_NE(nullptr, - resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, - [&](std::list&& results) -> void { - address_list = std::move(results); - dispatcher_->exit(); - })); + EXPECT_NE(nullptr, resolver_->resolve("some.good.domain", DnsLookupFamily::V4Only, + [&](std::list&& results) -> void { + address_list = getAddressList(results); + dispatcher_->exit(); + })); dispatcher_->run(Event::Dispatcher::RunType::Block); EXPECT_TRUE(address_list.empty()); @@ -785,7 +859,7 @@ TEST(DnsImplUnitTest, PendingTimerEnable) { EXPECT_CALL(dispatcher, createFileEvent_(_, _, _, _)).WillOnce(Return(file_event)); EXPECT_CALL(*timer, enableTimer(_)); EXPECT_NE(nullptr, resolver.resolve("some.bad.domain.invalid", DnsLookupFamily::V4Only, - [&](std::list&& results) { + [&](std::list&& results) { UNREFERENCED_PARAMETER(results); })); } diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 9001f60c7691..cc94979718b3 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -886,6 +886,41 @@ TEST_F(StrictDnsClusterImplTest, CustomResolverFails) { EnvoyException, "STRICT_DNS clusters must NOT have a custom resolver name set"); } +TEST_F(StrictDnsClusterImplTest, RecordTtlAsDnsRefreshRate) { + ResolverData resolver(*dns_resolver_, dispatcher_); + + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + dns_refresh_rate: 4s + respect_dns_ttl: true + hosts: [{ socket_address: { address: localhost1, port_value: 11001 }}] + )EOF"; + + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); + Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( + "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() + : cluster_config.alt_stat_name())); + Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( + admin_, ssl_context_manager_, *scope, cm_, local_info_, dispatcher_, random_, stats_, + singleton_manager_, tls_, validation_visitor_, *api_); + StrictDnsClusterImpl cluster(cluster_config, runtime_, dns_resolver_, factory_context, + std::move(scope), false); + ReadyWatcher membership_updated; + cluster.prioritySet().addPriorityUpdateCb( + [&](uint32_t, const HostVector&, const HostVector&) -> void { membership_updated.ready(); }); + + cluster.initialize([] {}); + + EXPECT_CALL(membership_updated, ready()); + + EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(5000))); + resolver.dns_callback_( + TestUtility::makeDnsResponse({"192.168.1.1", "192.168.1.2"}, std::chrono::seconds(5))); +} + TEST(HostImplTest, HostCluster) { MockClusterMockPrioritySet cluster; HostSharedPtr host = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", 1); diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 7e69602477bc..c035354cb101 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -430,8 +430,9 @@ class RedisClusterTest : public testing::Test, void testRedisResolve() { EXPECT_CALL(dispatcher_, createTimer_(_)); RedisCluster::RedisDiscoverySession discovery_session(*cluster_, *this); - discovery_session.registerDiscoveryAddress( - TestUtility::makeDnsResponse(std::list({"127.0.0.1", "127.0.0.2"})), 22120); + auto dns_response = + TestUtility::makeDnsResponse(std::list({"127.0.0.1", "127.0.0.2"})); + discovery_session.registerDiscoveryAddress(std::move(dns_response), 22120); expectRedisResolve(true); discovery_session.startResolve(); diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index 799a65b7a4bb..3fe6ddad32d3 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -5,6 +5,7 @@ #include "test/mocks/network/mocks.h" #include "test/mocks/thread_local/mocks.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" using testing::InSequence; using testing::Return; @@ -16,15 +17,6 @@ namespace Common { namespace DynamicForwardProxy { namespace { -std::list -makeAddressList(const std::list address_list) { - std::list ret; - for (const auto& address : address_list) { - ret.emplace_back(Network::Utility::parseInternetAddress(address)); - } - return ret; -} - class DnsCacheImplTest : public testing::Test, public Event::TestUsingSimulatedTime { public: void initialize() { @@ -95,7 +87,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -110,7 +102,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { // Address does not change. EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -127,7 +119,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.2:80"))); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); - resolve_cb(makeAddressList({"10.0.0.2"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.2"})); checkStats(3 /* attempt */, 3 /* success */, 0 /* failure */, 2 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -154,7 +146,7 @@ TEST_F(DnsCacheImplTest, TTL) { onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"}, std::chrono::seconds(0))); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -168,7 +160,7 @@ TEST_F(DnsCacheImplTest, TTL) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -211,7 +203,7 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000))); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"}, std::chrono::seconds(0))); // Re-resolve with ~30s passed. TTL should still be OK at 60s. simTime().sleep(std::chrono::milliseconds(30001)); @@ -219,7 +211,7 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); resolve_timer->invokeCallback(); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000))); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); // Re-resolve with ~30s passed. TTL should expire. simTime().sleep(std::chrono::milliseconds(30001)); @@ -243,7 +235,7 @@ TEST_F(DnsCacheImplTest, InlineResolve) { EXPECT_CALL(*resolver_, resolve("localhost", _, _)) .WillOnce(Invoke([](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb callback) { - callback(makeAddressList({"127.0.0.1"})); + callback(TestUtility::makeDnsResponse({"127.0.0.1"})); return nullptr; })); EXPECT_CALL(update_callbacks_, @@ -270,7 +262,7 @@ TEST_F(DnsCacheImplTest, ResolveFailure) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - resolve_cb(makeAddressList({})); + resolve_cb(TestUtility::makeDnsResponse({})); checkStats(1 /* attempt */, 0 /* success */, 1 /* failure */, 0 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -295,7 +287,7 @@ TEST_F(DnsCacheImplTest, CancelResolve) { result.handle_.reset(); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); } // Two cache loads that are trying to resolve the same host. Make sure we only do a single resolve @@ -321,7 +313,7 @@ TEST_F(DnsCacheImplTest, MultipleResolveSameHost) { onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); EXPECT_CALL(callbacks2, onLoadDnsCacheComplete()); EXPECT_CALL(callbacks1, onLoadDnsCacheComplete()); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); } // Two cache loads that are resolving different hosts. @@ -348,12 +340,12 @@ TEST_F(DnsCacheImplTest, MultipleResolveDifferentHost) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("bar.com", SharedAddressEquals("10.0.0.1:443"))); EXPECT_CALL(callbacks2, onLoadDnsCacheComplete()); - resolve_cb2(makeAddressList({"10.0.0.1"})); + resolve_cb2(TestUtility::makeDnsResponse({"10.0.0.1"})); EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.2:80"))); EXPECT_CALL(callbacks1, onLoadDnsCacheComplete()); - resolve_cb1(makeAddressList({"10.0.0.2"})); + resolve_cb1(TestUtility::makeDnsResponse({"10.0.0.2"})); } // A successful resolve followed by a cache hit. @@ -372,7 +364,7 @@ TEST_F(DnsCacheImplTest, CacheHit) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - resolve_cb(makeAddressList({"10.0.0.1"})); + resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); result = dns_cache_->loadDnsCacheEntry("foo.com", 80, callbacks); EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::InCache, result.status_); @@ -411,7 +403,7 @@ TEST_F(DnsCacheImplTest, InvalidPort) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate(_, _)).Times(0); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - resolve_cb(makeAddressList({})); + resolve_cb(TestUtility::makeDnsResponse({})); } // Max host overflow. diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index bd3392da5b0f..02b950cb102d 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -181,11 +181,12 @@ void TestUtility::waitForGaugeEq(Stats::Store& store, const std::string& name, u } } -std::list -TestUtility::makeDnsResponse(const std::list& addresses) { - std::list ret; +std::list +TestUtility::makeDnsResponse(const std::list& addresses, std::chrono::seconds ttl) { + std::list ret; for (const auto& address : addresses) { - ret.emplace_back(Network::Utility::parseInternetAddress(address)); + + ret.emplace_back(Network::DnsResponse(Network::Utility::parseInternetAddress(address), ttl)); } return ret; } diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 8f1bdcc721c3..7e33eeff7d1e 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -223,8 +223,9 @@ class TestUtility { * Convert a string list of IP addresses into a list of network addresses usable for DNS * response testing. */ - static std::list - makeDnsResponse(const std::list& addresses); + static std::list + makeDnsResponse(const std::list& addresses, + std::chrono::seconds = std::chrono::seconds(0)); /** * List files in a given directory path diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 7fdc7d95db96..21ef6280e234 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -269,6 +269,7 @@ TPROXY TSAN TSI TTL +TTLs TX TXT UA From 7d5e6b2c14b0718a1002b2f8a1c77c80fa90eed5 Mon Sep 17 00:00:00 2001 From: Matt Fowles Kulukundis Date: Mon, 1 Jul 2019 15:34:59 -0700 Subject: [PATCH 090/542] Switch ThreadId to an opaque value type. (#7330) ThreadId does not need to be an abstract class and that just interferes with using them as keys in hashtables. Signed-off-by: Matt Kulukundis --- bazel/repositories.bzl | 4 ++ include/envoy/server/guarddog.h | 4 +- include/envoy/server/watchdog.h | 2 +- include/envoy/thread/thread.h | 26 +++++++--- source/common/common/posix/thread_impl.cc | 12 +---- source/common/common/posix/thread_impl.h | 14 +----- source/common/common/win32/thread_impl.cc | 11 ++-- source/common/common/win32/thread_impl.h | 14 +----- source/common/event/dispatcher_impl.cc | 2 +- source/common/event/dispatcher_impl.h | 6 ++- source/common/singleton/BUILD | 1 + source/common/singleton/manager_impl.cc | 3 +- source/common/singleton/manager_impl.h | 10 ++-- source/server/config_validation/server.cc | 2 +- source/server/guarddog_impl.cc | 2 +- source/server/guarddog_impl.h | 2 +- source/server/server.cc | 2 +- source/server/watchdog_impl.h | 9 ++-- test/common/common/BUILD | 10 ++++ test/common/common/thread_id_test.cc | 50 +++++++++++++++++++ test/common/singleton/manager_impl_test.cc | 4 +- .../upstream/cluster_factory_impl_test.cc | 2 +- .../upstream/cluster_manager_impl_test.cc | 2 +- test/common/upstream/eds_test.cc | 2 +- test/common/upstream/hds_test.cc | 2 +- .../upstream/logical_dns_cluster_test.cc | 2 +- .../upstream/original_dst_cluster_test.cc | 2 +- test/common/upstream/upstream_impl_test.cc | 4 +- .../dynamic_forward_proxy/cluster_test.cc | 2 +- .../clusters/redis/redis_cluster_test.cc | 2 +- .../transport_sockets/alts/config_test.cc | 4 +- test/mocks/server/mocks.cc | 7 ++- test/mocks/server/mocks.h | 44 ++++++++-------- .../config_validation/cluster_manager_test.cc | 2 +- test/server/guarddog_impl_test.cc | 2 +- test/test_common/only_one_thread.cc | 4 +- test/test_common/only_one_thread.h | 2 +- 37 files changed, 160 insertions(+), 115 deletions(-) create mode 100644 test/common/common/thread_id_test.cc diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 72c5c6e9d87c..be0825a10619 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -419,6 +419,10 @@ def _com_google_absl(): name = "abseil_hash", actual = "@com_google_absl//absl/hash:hash", ) + native.bind( + name = "abseil_hash_testing", + actual = "@com_google_absl//absl/hash:hash_testing", + ) native.bind( name = "abseil_inlined_vector", actual = "@com_google_absl//absl/container:inlined_vector", diff --git a/include/envoy/server/guarddog.h b/include/envoy/server/guarddog.h index b17c55379aac..4386aa7f9051 100644 --- a/include/envoy/server/guarddog.h +++ b/include/envoy/server/guarddog.h @@ -26,9 +26,9 @@ class GuardDog { * to avoid triggering the GuardDog. If no longer needed use the * stopWatching() method to remove it from the list of watched objects. * - * @param thread_id a Thread::ThreadIdPtr containing the system thread id + * @param thread_id a Thread::ThreadId containing the system thread id */ - virtual WatchDogSharedPtr createWatchDog(Thread::ThreadIdPtr&& thread_id) PURE; + virtual WatchDogSharedPtr createWatchDog(Thread::ThreadId thread_id) PURE; /** * Tell the GuardDog to forget about this WatchDog. diff --git a/include/envoy/server/watchdog.h b/include/envoy/server/watchdog.h index f60755784582..d230ab48f6fb 100644 --- a/include/envoy/server/watchdog.h +++ b/include/envoy/server/watchdog.h @@ -36,7 +36,7 @@ class WatchDog { * This can be used if this is later used on a thread where there is no dispatcher. */ virtual void touch() PURE; - virtual const Thread::ThreadId& threadId() const PURE; + virtual Thread::ThreadId threadId() const PURE; virtual MonotonicTime lastTouchTime() const PURE; }; diff --git a/include/envoy/thread/thread.h b/include/envoy/thread/thread.h index 41dd7291ee07..8d630d8ac134 100644 --- a/include/envoy/thread/thread.h +++ b/include/envoy/thread/thread.h @@ -1,7 +1,9 @@ #pragma once #include +#include #include +#include #include "envoy/common/pure.h" @@ -10,16 +12,26 @@ namespace Envoy { namespace Thread { +/** + * An id for a thread. + */ class ThreadId { public: - virtual ~ThreadId() = default; - - virtual std::string debugString() const PURE; - virtual bool isCurrentThreadId() const PURE; + ThreadId() : id_(std::numeric_limits::min()) {} + explicit ThreadId(int64_t id) : id_(id) {} + + std::string debugString() const { return std::to_string(id_); } + bool isEmpty() const { return *this == ThreadId(); } + friend bool operator==(ThreadId lhs, ThreadId rhs) { return lhs.id_ == rhs.id_; } + friend bool operator!=(ThreadId lhs, ThreadId rhs) { return lhs.id_ != rhs.id_; } + template friend H AbslHashValue(H h, ThreadId id) { + return H::combine(std::move(h), id.id_); + } + +private: + int64_t id_; }; -using ThreadIdPtr = std::unique_ptr; - class Thread { public: virtual ~Thread() = default; @@ -48,7 +60,7 @@ class ThreadFactory { /** * Return the current system thread ID */ - virtual ThreadIdPtr currentThreadId() PURE; + virtual ThreadId currentThreadId() PURE; }; /** diff --git a/source/common/common/posix/thread_impl.cc b/source/common/common/posix/thread_impl.cc index 463ae4745ef7..324230ade176 100644 --- a/source/common/common/posix/thread_impl.cc +++ b/source/common/common/posix/thread_impl.cc @@ -24,14 +24,8 @@ int64_t getCurrentThreadId() { } // namespace -ThreadIdImplPosix::ThreadIdImplPosix(int64_t id) : id_(id) {} - -std::string ThreadIdImplPosix::debugString() const { return std::to_string(id_); } - -bool ThreadIdImplPosix::isCurrentThreadId() const { return id_ == getCurrentThreadId(); } - ThreadImplPosix::ThreadImplPosix(std::function thread_routine) - : thread_routine_(thread_routine) { + : thread_routine_(std::move(thread_routine)) { RELEASE_ASSERT(Logger::Registry::initialized(), ""); const int rc = pthread_create( &thread_handle_, nullptr, @@ -52,9 +46,7 @@ ThreadPtr ThreadFactoryImplPosix::createThread(std::function thread_rout return std::make_unique(thread_routine); } -ThreadIdPtr ThreadFactoryImplPosix::currentThreadId() { - return std::make_unique(getCurrentThreadId()); -} +ThreadId ThreadFactoryImplPosix::currentThreadId() { return ThreadId(getCurrentThreadId()); } } // namespace Thread } // namespace Envoy diff --git a/source/common/common/posix/thread_impl.h b/source/common/common/posix/thread_impl.h index aecc59e05b09..81c81d3be3fc 100755 --- a/source/common/common/posix/thread_impl.h +++ b/source/common/common/posix/thread_impl.h @@ -9,18 +9,6 @@ namespace Envoy { namespace Thread { -class ThreadIdImplPosix : public ThreadId { -public: - ThreadIdImplPosix(int64_t id); - - // Thread::ThreadId - std::string debugString() const override; - bool isCurrentThreadId() const override; - -private: - int64_t id_; -}; - /** * Wrapper for a pthread thread. We don't use std::thread because it eats exceptions and leads to * unusable stack traces. @@ -44,7 +32,7 @@ class ThreadFactoryImplPosix : public ThreadFactory { public: // Thread::ThreadFactory ThreadPtr createThread(std::function thread_routine) override; - ThreadIdPtr currentThreadId() override; + ThreadId currentThreadId() override; }; } // namespace Thread diff --git a/source/common/common/win32/thread_impl.cc b/source/common/common/win32/thread_impl.cc index bee7b9f2f979..1d3eca968957 100644 --- a/source/common/common/win32/thread_impl.cc +++ b/source/common/common/win32/thread_impl.cc @@ -6,12 +6,6 @@ namespace Envoy { namespace Thread { -ThreadIdImplWin32::ThreadIdImplWin32(DWORD id) : id_(id) {} - -std::string ThreadIdImplWin32::debugString() const { return std::to_string(id_); } - -bool ThreadIdImplWin32::isCurrentThreadId() const { return id_ == ::GetCurrentThreadId(); } - ThreadImplWin32::ThreadImplWin32(std::function thread_routine) : thread_routine_(thread_routine) { RELEASE_ASSERT(Logger::Registry::initialized(), ""); @@ -36,8 +30,9 @@ ThreadPtr ThreadFactoryImplWin32::createThread(std::function thread_rout return std::make_unique(thread_routine); } -ThreadIdPtr ThreadFactoryImplWin32::currentThreadId() { - return std::make_unique(::GetCurrentThreadId()); +ThreadId ThreadFactoryImplWin32::currentThreadId() { + // TODO(mhoran): test this in windows please. + return ThreadId(static_cast(::GetCurrentThreadId())); } } // namespace Thread diff --git a/source/common/common/win32/thread_impl.h b/source/common/common/win32/thread_impl.h index cd3821900f85..a8c74eb5d21a 100644 --- a/source/common/common/win32/thread_impl.h +++ b/source/common/common/win32/thread_impl.h @@ -13,18 +13,6 @@ namespace Envoy { namespace Thread { -class ThreadIdImplWin32 : public ThreadId { -public: - ThreadIdImplWin32(DWORD id); - - // Thread::ThreadId - std::string debugString() const override; - bool isCurrentThreadId() const override; - -private: - DWORD id_; -}; - /** * Wrapper for a win32 thread. We don't use std::thread because it eats exceptions and leads to * unusable stack traces. @@ -49,7 +37,7 @@ class ThreadFactoryImplWin32 : public ThreadFactory { public: // Thread::ThreadFactory ThreadPtr createThread(std::function thread_routine) override; - ThreadIdPtr currentThreadId() override; + ThreadId currentThreadId() override; }; } // namespace Thread diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 4ffcf6181a82..996ce9960471 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -48,7 +48,7 @@ void DispatcherImpl::initializeStats(Stats::Scope& scope, const std::string& pre stats_ = std::make_unique( DispatcherStats{ALL_DISPATCHER_STATS(POOL_HISTOGRAM_PREFIX(scope, stats_prefix_ + "."))}); base_scheduler_.initializeStats(stats_.get()); - ENVOY_LOG(debug, "running {} on thread {}", stats_prefix_, run_tid_->debugString()); + ENVOY_LOG(debug, "running {} on thread {}", stats_prefix_, run_tid_.debugString()); }); } diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index b712f22d879e..ba95ee5684ea 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -72,12 +72,14 @@ class DispatcherImpl : Logger::Loggable, public Dispatcher { // Validate that an operation is thread safe, i.e. it's invoked on the same thread that the // dispatcher run loop is executing on. We allow run_tid_ == nullptr for tests where we don't // invoke run(). - bool isThreadSafe() const { return run_tid_ == nullptr || run_tid_->isCurrentThreadId(); } + bool isThreadSafe() const { + return run_tid_.isEmpty() || run_tid_ == api_.threadFactory().currentThreadId(); + } Api::Api& api_; std::string stats_prefix_; std::unique_ptr stats_; - Thread::ThreadIdPtr run_tid_; + Thread::ThreadId run_tid_; Buffer::WatermarkFactoryPtr buffer_factory_; LibeventScheduler base_scheduler_; SchedulerPtr scheduler_; diff --git a/source/common/singleton/BUILD b/source/common/singleton/BUILD index 3dad3f52a689..f3b1e651772d 100644 --- a/source/common/singleton/BUILD +++ b/source/common/singleton/BUILD @@ -21,6 +21,7 @@ envoy_cc_library( "//include/envoy/registry", "//include/envoy/singleton:manager_interface", "//source/common/common:assert_lib", + "//source/common/common:non_copyable", "//source/common/common:thread_lib", ], ) diff --git a/source/common/singleton/manager_impl.cc b/source/common/singleton/manager_impl.cc index 9023b2b744ca..e90d3b000c63 100644 --- a/source/common/singleton/manager_impl.cc +++ b/source/common/singleton/manager_impl.cc @@ -9,7 +9,8 @@ namespace Envoy { namespace Singleton { InstanceSharedPtr ManagerImpl::get(const std::string& name, SingletonFactoryCb cb) { - ASSERT(run_tid_->isCurrentThreadId()); + ASSERT(run_tid_ == thread_factory_.currentThreadId()); + if (nullptr == Registry::FactoryRegistry::getFactory(name)) { PANIC(fmt::format("invalid singleton name '{}'. Make sure it is registered.", name)); } diff --git a/source/common/singleton/manager_impl.h b/source/common/singleton/manager_impl.h index 9625087683d2..6f55ad3fadb2 100644 --- a/source/common/singleton/manager_impl.h +++ b/source/common/singleton/manager_impl.h @@ -5,6 +5,8 @@ #include "envoy/singleton/manager.h" #include "envoy/thread/thread.h" +#include "common/common/non_copyable.h" + namespace Envoy { namespace Singleton { @@ -13,16 +15,18 @@ namespace Singleton { * assumed the singleton manager is only used on the main thread so it is not thread safe. Asserts * verify that. */ -class ManagerImpl : public Manager { +class ManagerImpl : public Manager, NonCopyable { public: - ManagerImpl(Thread::ThreadIdPtr&& thread_id) : run_tid_(std::move(thread_id)) {} + explicit ManagerImpl(Thread::ThreadFactory& thread_factory) + : thread_factory_(thread_factory), run_tid_(thread_factory.currentThreadId()) {} // Singleton::Manager InstanceSharedPtr get(const std::string& name, SingletonFactoryCb cb) override; private: std::unordered_map> singletons_; - Thread::ThreadIdPtr run_tid_; + Thread::ThreadFactory& thread_factory_; + const Thread::ThreadId run_tid_; }; } // namespace Singleton diff --git a/source/server/config_validation/server.cc b/source/server/config_validation/server.cc index 96faf5f81bd6..6e66d550227d 100644 --- a/source/server/config_validation/server.cc +++ b/source/server/config_validation/server.cc @@ -44,7 +44,7 @@ ValidationInstance::ValidationInstance(const Options& options, Event::TimeSystem : options_(options), stats_store_(store), api_(new Api::ValidationImpl(thread_factory, store, time_system, file_system)), dispatcher_(api_->allocateDispatcher()), - singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory().currentThreadId())), + singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), access_log_manager_(options.fileFlushIntervalMsec(), *api_, *dispatcher_, access_log_lock, store), mutex_tracer_(nullptr), grpc_context_(stats_store_.symbolTable()), diff --git a/source/server/guarddog_impl.cc b/source/server/guarddog_impl.cc index 67b55ca00840..a9e3e583e318 100644 --- a/source/server/guarddog_impl.cc +++ b/source/server/guarddog_impl.cc @@ -103,7 +103,7 @@ void GuardDogImpl::step() { } } -WatchDogSharedPtr GuardDogImpl::createWatchDog(Thread::ThreadIdPtr&& thread_id) { +WatchDogSharedPtr GuardDogImpl::createWatchDog(Thread::ThreadId thread_id) { // Timer started by WatchDog will try to fire at 1/2 of the interval of the // minimum timeout specified. loop_interval_ is const so all shared state // accessed out of the locked section below is const (time_source_ has no diff --git a/source/server/guarddog_impl.h b/source/server/guarddog_impl.h index c1ba688e9ba0..5c0906ea55ad 100644 --- a/source/server/guarddog_impl.h +++ b/source/server/guarddog_impl.h @@ -87,7 +87,7 @@ class GuardDogImpl : public GuardDog { } // Server::GuardDog - WatchDogSharedPtr createWatchDog(Thread::ThreadIdPtr&& thread_id) override; + WatchDogSharedPtr createWatchDog(Thread::ThreadId thread_id) override; void stopWatching(WatchDogSharedPtr wd) override; private: diff --git a/source/server/server.cc b/source/server/server.cc index 8d2b5a58c8b6..dfd44d8f0f8c 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -61,7 +61,7 @@ InstanceImpl::InstanceImpl(const Options& options, Event::TimeSystem& time_syste start_time_(time(nullptr)), original_start_time_(start_time_), stats_store_(store), thread_local_(tls), api_(new Api::Impl(thread_factory, store, time_system, file_system)), dispatcher_(api_->allocateDispatcher()), - singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory().currentThreadId())), + singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), handler_(new ConnectionHandlerImpl(ENVOY_LOGGER(), *dispatcher_)), random_generator_(std::move(random_generator)), listener_component_factory_(*this), worker_factory_(thread_local_, *api_, hooks), diff --git a/source/server/watchdog_impl.h b/source/server/watchdog_impl.h index b4a4db1340af..ea4ea82e5c28 100644 --- a/source/server/watchdog_impl.h +++ b/source/server/watchdog_impl.h @@ -19,13 +19,12 @@ class WatchDogImpl : public WatchDog { /** * @param interval WatchDog timer interval (used after startWatchdog()) */ - WatchDogImpl(Thread::ThreadIdPtr&& thread_id, TimeSource& tsource, - std::chrono::milliseconds interval) - : thread_id_(std::move(thread_id)), time_source_(tsource), + WatchDogImpl(Thread::ThreadId thread_id, TimeSource& tsource, std::chrono::milliseconds interval) + : thread_id_(thread_id), time_source_(tsource), latest_touch_time_since_epoch_(tsource.monotonicTime().time_since_epoch()), timer_interval_(interval) {} - const Thread::ThreadId& threadId() const override { return *thread_id_; } + Thread::ThreadId threadId() const override { return thread_id_; } MonotonicTime lastTouchTime() const override { return MonotonicTime(latest_touch_time_since_epoch_.load()); } @@ -37,7 +36,7 @@ class WatchDogImpl : public WatchDog { } private: - Thread::ThreadIdPtr thread_id_; + const Thread::ThreadId thread_id_; TimeSource& time_source_; std::atomic latest_touch_time_since_epoch_; Event::TimerPtr timer_; diff --git a/test/common/common/BUILD b/test/common/common/BUILD index 9d2d93940b54..d72fea4b6521 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -204,6 +204,16 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "thread_id_test", + srcs = ["thread_id_test.cc"], + external_deps = ["abseil_hash_testing"], + deps = [ + "//source/common/common:thread_lib", + "//test/test_common:thread_factory_for_test_lib", + ], +) + envoy_cc_test( name = "stl_helpers_test", srcs = ["stl_helpers_test.cc"], diff --git a/test/common/common/thread_id_test.cc b/test/common/common/thread_id_test.cc new file mode 100644 index 000000000000..7288b932cfc4 --- /dev/null +++ b/test/common/common/thread_id_test.cc @@ -0,0 +1,50 @@ +#include "common/common/thread.h" + +#include "test/test_common/thread_factory_for_test.h" + +#include "absl/hash/hash_testing.h" +#include "gmock/gmock.h" + +namespace Envoy { +namespace { + +TEST(ThreadId, Equality) { + auto& thread_factory = Thread::threadFactoryForTest(); + + Thread::ThreadId main_thread = thread_factory.currentThreadId(); + Thread::ThreadId background_thread; + Thread::ThreadId null_thread; + + Thread::threadFactoryForTest() + .createThread([&]() { background_thread = thread_factory.currentThreadId(); }) + ->join(); + + EXPECT_EQ(main_thread, main_thread); + EXPECT_EQ(background_thread, background_thread); + EXPECT_EQ(null_thread, null_thread); + + EXPECT_NE(main_thread, background_thread); + EXPECT_NE(main_thread, null_thread); + EXPECT_NE(background_thread, null_thread); +} + +TEST(ThreadId, Hashability) { + auto& thread_factory = Thread::threadFactoryForTest(); + + Thread::ThreadId main_thread = thread_factory.currentThreadId(); + Thread::ThreadId background_thread; + Thread::ThreadId null_thread; + + Thread::threadFactoryForTest() + .createThread([&]() { background_thread = thread_factory.currentThreadId(); }) + ->join(); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly({ + null_thread, + main_thread, + background_thread, + })); +} + +} // namespace +} // namespace Envoy diff --git a/test/common/singleton/manager_impl_test.cc b/test/common/singleton/manager_impl_test.cc index 7fba6275cab5..14b3d9eacad2 100644 --- a/test/common/singleton/manager_impl_test.cc +++ b/test/common/singleton/manager_impl_test.cc @@ -12,7 +12,7 @@ namespace { // Must be a dedicated function so that TID is within the death test. static void deathTestWorker() { - ManagerImpl manager(Thread::threadFactoryForTest().currentThreadId()); + ManagerImpl manager(Thread::threadFactoryForTest()); manager.get("foo", [] { return nullptr; }); } @@ -35,7 +35,7 @@ class TestSingleton : public Instance { }; TEST(SingletonManagerImplTest, Basic) { - ManagerImpl manager(Thread::threadFactoryForTest().currentThreadId()); + ManagerImpl manager(Thread::threadFactoryForTest()); std::shared_ptr singleton = std::make_shared(); EXPECT_EQ(singleton, manager.get("test_singleton", [singleton] { return singleton; })); diff --git a/test/common/upstream/cluster_factory_impl_test.cc b/test/common/upstream/cluster_factory_impl_test.cc index 759faff9dd1f..85ed32e327c6 100644 --- a/test/common/upstream/cluster_factory_impl_test.cc +++ b/test/common/upstream/cluster_factory_impl_test.cc @@ -65,7 +65,7 @@ class ClusterFactoryTestBase { NiceMock runtime_; NiceMock random_; Stats::IsolatedStoreImpl stats_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock tls_; NiceMock validation_visitor_; Api::ApiPtr api_; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 69870436d991..75fa0ab06800 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -132,7 +132,7 @@ class TestClusterManagerFactory : public ClusterManagerFactory { NiceMock admin_; NiceMock secret_manager_; NiceMock log_manager_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock validation_visitor_; Api::ApiPtr api_; }; diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index 9a92063f073f..78854abad4f0 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -108,7 +108,7 @@ class EdsTest : public testing::Test { NiceMock runtime_; NiceMock local_info_; NiceMock admin_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock tls_; NiceMock validation_visitor_; Api::ApiPtr api_; diff --git a/test/common/upstream/hds_test.cc b/test/common/upstream/hds_test.cc index 330f7db3279a..c7fe3af8d73e 100644 --- a/test/common/upstream/hds_test.cc +++ b/test/common/upstream/hds_test.cc @@ -126,7 +126,7 @@ class HdsTest : public testing::Test { NiceMock cm_; NiceMock local_info_; NiceMock admin_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock tls_; }; diff --git a/test/common/upstream/logical_dns_cluster_test.cc b/test/common/upstream/logical_dns_cluster_test.cc index 367d7afa55aa..03170a23af5a 100644 --- a/test/common/upstream/logical_dns_cluster_test.cc +++ b/test/common/upstream/logical_dns_cluster_test.cc @@ -214,7 +214,7 @@ class LogicalDnsClusterTest : public testing::Test { NiceMock dispatcher_; NiceMock local_info_; NiceMock admin_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock validation_visitor_; Api::ApiPtr api_; }; diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index f51042c28294..e3c5638dfef2 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -98,7 +98,7 @@ class OriginalDstClusterTest : public testing::Test { NiceMock random_; NiceMock local_info_; NiceMock admin_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock tls_; NiceMock validation_visitor_; Api::ApiPtr api_; diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index cc94979718b3..9d10956937ab 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -58,7 +58,7 @@ class UpstreamImplTestBase { NiceMock runtime_; NiceMock random_; Stats::IsolatedStoreImpl stats_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock tls_; NiceMock validation_visitor_; Api::ApiPtr api_; @@ -1851,7 +1851,7 @@ class ClusterInfoImplTest : public testing::Test { NiceMock local_info_; NiceMock random_; NiceMock admin_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock tls_; ReadyWatcher initialized_; envoy::api::v2::Cluster cluster_config_; diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc index 568637c431a2..d15870581e24 100644 --- a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc +++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc @@ -94,7 +94,7 @@ class ClusterTest : public testing::Test, NiceMock dispatcher_; NiceMock local_info_; NiceMock admin_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock validation_visitor_; Api::ApiPtr api_{Api::createApiForTest(stats_store_)}; std::shared_ptr dns_cache_manager_{ diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index c035354cb101..09c4f16579e5 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -456,7 +456,7 @@ class RedisClusterTest : public testing::Test, NiceMock dispatcher_; NiceMock local_info_; NiceMock admin_; - Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager_{Thread::threadFactoryForTest()}; NiceMock validation_visitor_; Api::ApiPtr api_; std::shared_ptr hosts_; diff --git a/test/extensions/transport_sockets/alts/config_test.cc b/test/extensions/transport_sockets/alts/config_test.cc index 91252b991308..34fcd34ae645 100644 --- a/test/extensions/transport_sockets/alts/config_test.cc +++ b/test/extensions/transport_sockets/alts/config_test.cc @@ -24,7 +24,7 @@ namespace { TEST(UpstreamAltsConfigTest, CreateSocketFactory) { MockTransportSocketFactoryContext factory_context; - Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); UpstreamAltsTransportSocketConfigFactory factory; @@ -44,7 +44,7 @@ TEST(UpstreamAltsConfigTest, CreateSocketFactory) { TEST(DownstreamAltsConfigTest, CreateSocketFactory) { MockTransportSocketFactoryContext factory_context; - Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); DownstreamAltsTransportSocketConfigFactory factory; diff --git a/test/mocks/server/mocks.cc b/test/mocks/server/mocks.cc index 12e2ac5c3169..bfaa237f50d5 100644 --- a/test/mocks/server/mocks.cc +++ b/test/mocks/server/mocks.cc @@ -125,8 +125,8 @@ MockWorker::~MockWorker() = default; MockInstance::MockInstance() : secret_manager_(new Secret::SecretManagerImpl()), cluster_manager_(timeSource()), - ssl_context_manager_(timeSource()), singleton_manager_(new Singleton::ManagerImpl( - Thread::threadFactoryForTest().currentThreadId())), + ssl_context_manager_(timeSource()), + singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), grpc_context_(stats_store_.symbolTable()), http_context_(stats_store_.symbolTable()) { ON_CALL(*this, threadLocal()).WillByDefault(ReturnRef(thread_local_)); ON_CALL(*this, stats()).WillByDefault(ReturnRef(stats_store_)); @@ -170,8 +170,7 @@ MockMain::MockMain(int wd_miss, int wd_megamiss, int wd_kill, int wd_multikill) MockMain::~MockMain() = default; MockFactoryContext::MockFactoryContext() - : singleton_manager_( - new Singleton::ManagerImpl(Thread::threadFactoryForTest().currentThreadId())), + : singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), grpc_context_(scope_.symbolTable()), http_context_(scope_.symbolTable()) { ON_CALL(*this, accessLogManager()).WillByDefault(ReturnRef(access_log_manager_)); ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index fde44c23ac7c..6ab3baac302c 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -54,7 +54,7 @@ class MockOptions : public Options { public: MockOptions() : MockOptions(std::string()) {} MockOptions(const std::string& config_path); - ~MockOptions(); + ~MockOptions() override; MOCK_CONST_METHOD0(baseId, uint64_t()); MOCK_CONST_METHOD0(concurrency, uint32_t()); @@ -102,7 +102,7 @@ class MockOptions : public Options { class MockConfigTracker : public ConfigTracker { public: MockConfigTracker(); - ~MockConfigTracker(); + ~MockConfigTracker() override; struct MockEntryOwner : public EntryOwner {}; @@ -120,7 +120,7 @@ class MockConfigTracker : public ConfigTracker { class MockAdmin : public Admin { public: MockAdmin(); - ~MockAdmin(); + ~MockAdmin() override; // Server::Admin MOCK_METHOD5(addHandler, bool(const std::string& prefix, const std::string& help_text, @@ -142,7 +142,7 @@ class MockAdmin : public Admin { class MockAdminStream : public AdminStream { public: MockAdminStream(); - ~MockAdminStream(); + ~MockAdminStream() override; MOCK_METHOD1(setEndStreamOnComplete, void(bool)); MOCK_METHOD1(addOnDestroyCallback, void(std::function)); @@ -155,7 +155,7 @@ class MockAdminStream : public AdminStream { class MockDrainManager : public DrainManager { public: MockDrainManager(); - ~MockDrainManager(); + ~MockDrainManager() override; // Server::DrainManager MOCK_CONST_METHOD0(drainClose, bool()); @@ -168,22 +168,22 @@ class MockDrainManager : public DrainManager { class MockWatchDog : public WatchDog { public: MockWatchDog(); - ~MockWatchDog(); + ~MockWatchDog() override; // Server::WatchDog MOCK_METHOD1(startWatchdog, void(Event::Dispatcher& dispatcher)); MOCK_METHOD0(touch, void()); - MOCK_CONST_METHOD0(threadId, const Thread::ThreadId&()); + MOCK_CONST_METHOD0(threadId, Thread::ThreadId()); MOCK_CONST_METHOD0(lastTouchTime, MonotonicTime()); }; class MockGuardDog : public GuardDog { public: MockGuardDog(); - ~MockGuardDog(); + ~MockGuardDog() override; // Server::GuardDog - MOCK_METHOD1(createWatchDog, WatchDogSharedPtr(Thread::ThreadIdPtr&&)); + MOCK_METHOD1(createWatchDog, WatchDogSharedPtr(Thread::ThreadId)); MOCK_METHOD1(stopWatching, void(WatchDogSharedPtr wd)); std::shared_ptr watch_dog_; @@ -192,7 +192,7 @@ class MockGuardDog : public GuardDog { class MockHotRestart : public HotRestart { public: MockHotRestart(); - ~MockHotRestart(); + ~MockHotRestart() override; // Server::HotRestart MOCK_METHOD0(drainParentListeners, void()); @@ -218,7 +218,7 @@ class MockHotRestart : public HotRestart { class MockListenerComponentFactory : public ListenerComponentFactory { public: MockListenerComponentFactory(); - ~MockListenerComponentFactory(); + ~MockListenerComponentFactory() override; DrainManagerPtr createDrainManager(envoy::api::v2::Listener::DrainType drain_type) override { return DrainManagerPtr{createDrainManager_(drain_type)}; @@ -254,7 +254,7 @@ class MockListenerComponentFactory : public ListenerComponentFactory { class MockListenerManager : public ListenerManager { public: MockListenerManager(); - ~MockListenerManager(); + ~MockListenerManager() override; MOCK_METHOD3(addOrUpdateListener, bool(const envoy::api::v2::Listener& config, const std::string& version_info, bool modifiable)); @@ -270,7 +270,7 @@ class MockListenerManager : public ListenerManager { class MockServerLifecycleNotifier : public ServerLifecycleNotifier { public: MockServerLifecycleNotifier(); - ~MockServerLifecycleNotifier(); + ~MockServerLifecycleNotifier() override; MOCK_METHOD2(registerCallback, ServerLifecycleNotifier::HandlePtr(Stage, StageCallback)); MOCK_METHOD2(registerCallback, @@ -280,7 +280,7 @@ class MockServerLifecycleNotifier : public ServerLifecycleNotifier { class MockWorkerFactory : public WorkerFactory { public: MockWorkerFactory(); - ~MockWorkerFactory(); + ~MockWorkerFactory() override; // Server::WorkerFactory WorkerPtr createWorker(OverloadManager&) override { return WorkerPtr{createWorker_()}; } @@ -291,7 +291,7 @@ class MockWorkerFactory : public WorkerFactory { class MockWorker : public Worker { public: MockWorker(); - ~MockWorker(); + ~MockWorker() override; void callAddCompletion(bool success) { EXPECT_NE(nullptr, add_listener_completion_); @@ -324,7 +324,7 @@ class MockWorker : public Worker { class MockOverloadManager : public OverloadManager { public: MockOverloadManager(); - ~MockOverloadManager(); + ~MockOverloadManager() override; // OverloadManager MOCK_METHOD0(start, void()); @@ -338,7 +338,7 @@ class MockOverloadManager : public OverloadManager { class MockInstance : public Instance { public: MockInstance(); - ~MockInstance(); + ~MockInstance() override; Secret::SecretManager& secretManager() override { return *(secret_manager_.get()); } @@ -414,7 +414,7 @@ class MockMain : public Main { public: MockMain() : MockMain(0, 0, 0, 0) {} MockMain(int wd_miss, int wd_megamiss, int wd_kill, int wd_multikill); - ~MockMain(); + ~MockMain() override; MOCK_METHOD0(clusterManager, Upstream::ClusterManager*()); MOCK_METHOD0(httpTracer, Tracing::HttpTracer&()); @@ -434,7 +434,7 @@ class MockMain : public Main { class MockFactoryContext : public virtual FactoryContext { public: MockFactoryContext(); - ~MockFactoryContext(); + ~MockFactoryContext() override; MOCK_METHOD0(accessLogManager, AccessLog::AccessLogManager&()); MOCK_METHOD0(clusterManager, Upstream::ClusterManager&()); @@ -487,7 +487,7 @@ class MockFactoryContext : public virtual FactoryContext { class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { public: MockTransportSocketFactoryContext(); - ~MockTransportSocketFactoryContext(); + ~MockTransportSocketFactoryContext() override; Secret::SecretManager& secretManager() override { return *(secret_manager_.get()); } @@ -514,7 +514,7 @@ class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { class MockListenerFactoryContext : public MockFactoryContext, public ListenerFactoryContext { public: MockListenerFactoryContext(); - ~MockListenerFactoryContext(); + ~MockListenerFactoryContext() override; void addListenSocketOption(const Network::Socket::OptionConstSharedPtr& option) override { addListenSocketOption_(option); @@ -533,7 +533,7 @@ class MockListenerFactoryContext : public MockFactoryContext, public ListenerFac class MockHealthCheckerFactoryContext : public virtual HealthCheckerFactoryContext { public: MockHealthCheckerFactoryContext(); - ~MockHealthCheckerFactoryContext(); + ~MockHealthCheckerFactoryContext() override; MOCK_METHOD0(cluster, Upstream::Cluster&()); MOCK_METHOD0(dispatcher, Event::Dispatcher&()); diff --git a/test/server/config_validation/cluster_manager_test.cc b/test/server/config_validation/cluster_manager_test.cc index 2b2406725330..ce734f717301 100644 --- a/test/server/config_validation/cluster_manager_test.cc +++ b/test/server/config_validation/cluster_manager_test.cc @@ -43,7 +43,7 @@ TEST(ValidationClusterManagerTest, MockedMethods) { NiceMock admin; Http::ContextImpl http_context(stats_store.symbolTable()); AccessLog::MockAccessLogManager log_manager; - Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest().currentThreadId()}; + Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; ValidationClusterManagerFactory factory(admin, runtime, stats_store, tls, random, dns_resolver, ssl_context_manager, dispatcher, local_info, diff --git a/test/server/guarddog_impl_test.cc b/test/server/guarddog_impl_test.cc index c1e00ef03724..77b3397ffd59 100644 --- a/test/server/guarddog_impl_test.cc +++ b/test/server/guarddog_impl_test.cc @@ -316,7 +316,7 @@ TEST_P(GuardDogTestBase, WatchDogThreadIdTest) { initGuardDog(stats, config); auto watched_dog = guard_dog_->createWatchDog(api_->threadFactory().currentThreadId()); EXPECT_EQ(watched_dog->threadId().debugString(), - api_->threadFactory().currentThreadId()->debugString()); + api_->threadFactory().currentThreadId().debugString()); guard_dog_->stopWatching(watched_dog); } diff --git a/test/test_common/only_one_thread.cc b/test/test_common/only_one_thread.cc index a6a5869b7eda..06cc9855446d 100644 --- a/test/test_common/only_one_thread.cc +++ b/test/test_common/only_one_thread.cc @@ -13,10 +13,10 @@ OnlyOneThread::OnlyOneThread() : thread_factory_(threadFactoryForTest()) {} void OnlyOneThread::checkOneThread() { LockGuard lock(mutex_); - if (thread_advancing_time_ == nullptr) { + if (thread_advancing_time_.isEmpty()) { thread_advancing_time_ = thread_factory_.currentThreadId(); } else { - RELEASE_ASSERT(thread_advancing_time_->isCurrentThreadId(), + RELEASE_ASSERT(thread_advancing_time_ == thread_factory_.currentThreadId(), "time should only be advanced on one thread in the context of a test"); } } diff --git a/test/test_common/only_one_thread.h b/test/test_common/only_one_thread.h index 1051c632e78a..678e40b319a0 100644 --- a/test/test_common/only_one_thread.h +++ b/test/test_common/only_one_thread.h @@ -20,7 +20,7 @@ class OnlyOneThread { private: ThreadFactory& thread_factory_; - ThreadIdPtr thread_advancing_time_ GUARDED_BY(mutex_); + ThreadId thread_advancing_time_ GUARDED_BY(mutex_); mutable MutexBasicLockable mutex_; }; From 659805c55c5ca5cf738b9703de1d155be32a9a9f Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Mon, 1 Jul 2019 19:39:49 -0700 Subject: [PATCH 091/542] clang-tidy: enable most abseil checks as errors (#7442) Signed-off-by: Derek Argueta --- .clang-tidy | 31 +++++++++++++++++-- source/common/upstream/health_checker_impl.cc | 2 +- test/integration/utility.cc | 4 ++- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 76764919891d..8c0a761ab080 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,7 +1,34 @@ -Checks: 'clang-diagnostic-*,clang-analyzer-*,abseil-*,bugprone-*,modernize-*,performance-*,readability-redundant-*,readability-braces-around-statements,readability-container-size-empty' +Checks: 'abseil-*, + bugprone-*, + clang-analyzer-*, + clang-diagnostic-*, + modernize-*, + performance-*, + readability-braces-around-statements, + readability-container-size-empty' + readability-redundant-*, #TODO(lizan): grow this list, fix possible warnings and make more checks as error -WarningsAsErrors: 'bugprone-assert-side-effect,bugprone-use-after-move,modernize-make-shared,modernize-make-unique,modernize-use-using,performance-faster-string-find,performance-for-range-copy,performance-noexcept-move-constructor,performance-unnecessary-copy-initialization,readability-braces-around-statements,readability-container-size-empty,readability-redundant-smartptr-get,readability-redundant-string-cstr' +WarningsAsErrors: 'abseil-duration-*, + abseil-faster-strsplit-delimiter, + abseil-no-namespace, + abseil-redundant-strcat-calls, + abseil-str-cat-append, + abseil-string-find-startswith, + abseil-upgrade-duration-conversions, + bugprone-assert-side-effect, + bugprone-use-after-move, + modernize-make-shared, + modernize-make-unique, + modernize-use-using, + performance-faster-string-find, + performance-for-range-copy, + performance-noexcept-move-constructor, + performance-unnecessary-copy-initialization, + readability-braces-around-statements, + readability-container-size-empty, + readability-redundant-smartptr-get, + readability-redundant-string-cstr' CheckOptions: - key: bugprone-assert-side-effect.AssertMacros diff --git a/source/common/upstream/health_checker_impl.cc b/source/common/upstream/health_checker_impl.cc index a6f466e509ef..1490dced8594 100644 --- a/source/common/upstream/health_checker_impl.cc +++ b/source/common/upstream/health_checker_impl.cc @@ -249,7 +249,7 @@ HttpHealthCheckerImpl::HttpActiveHealthCheckSession::healthCheckResult() { response_headers_->EnvoyUpstreamHealthCheckedCluster()->value().getStringView()) : EMPTY_STRING; - if (service_cluster_healthchecked.find(parent_.service_name_.value()) == 0) { + if (absl::StartsWith(service_cluster_healthchecked, parent_.service_name_.value())) { return degraded ? HealthCheckResult::Degraded : HealthCheckResult::Succeeded; } else { return HealthCheckResult::Failed; diff --git a/test/integration/utility.cc b/test/integration/utility.cc index a5cbc2c6c83e..ecf5fd26ae7d 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -24,6 +24,8 @@ #include "test/test_common/printers.h" #include "test/test_common/utility.h" +#include "absl/strings/match.h" + namespace Envoy { void BufferingStreamDecoder::decodeHeaders(Http::HeaderMapPtr&& headers, bool end_stream) { ASSERT(!complete_); @@ -140,7 +142,7 @@ Network::FilterStatus WaitForPayloadReader::onData(Buffer::Instance& data, bool data_.append(data.toString()); data.drain(data.length()); read_end_stream_ = end_stream; - if ((!data_to_wait_for_.empty() && data_.find(data_to_wait_for_) == 0) || + if ((!data_to_wait_for_.empty() && absl::StartsWith(data_, data_to_wait_for_)) || (exact_match_ == false && data_.find(data_to_wait_for_) != std::string::npos) || end_stream) { data_to_wait_for_.clear(); dispatcher_.exit(); From ac21376c65c69d565e51f6b6570f20671d9e6416 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Mon, 1 Jul 2019 19:45:33 -0700 Subject: [PATCH 092/542] Update jwt_verify_lib to 2019-07-01 (#7439) Signed-off-by: Wayne Zhang --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index c0a699b12752..6c2b92352019 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -156,10 +156,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/msgpack/msgpack-c/archive/cpp-3.1.1.tar.gz"], ), com_github_google_jwt_verify = dict( - sha256 = "700be26170c1917e83d1319b88a2112dccd1179cd78c5672940483e7c45ca6ae", - strip_prefix = "jwt_verify_lib-85cf0edf1f1bc507ff7d96a8d6a9bc20307b0fcf", - # 2018-12-01 - urls = ["https://github.com/google/jwt_verify_lib/archive/85cf0edf1f1bc507ff7d96a8d6a9bc20307b0fcf.tar.gz"], + sha256 = "8ab9a0b3f8b7eab5f1cd059920e81fdc138cd4ee657c1412af891652929885c5", + strip_prefix = "jwt_verify_lib-6356535ae83a3f1820b6b06dae80cd6a0a03e7f2", + # 2019-07-01 + urls = ["https://github.com/google/jwt_verify_lib/archive/6356535ae83a3f1820b6b06dae80cd6a0a03e7f2.tar.gz"], ), com_github_nodejs_http_parser = dict( sha256 = "ef26268c54c8084d17654ba2ed5140bffeffd2a040a895ffb22a6cca3f6c613f", From b0bb785ff7d899df0fdc1d57b4392c01bd59d6f3 Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Tue, 2 Jul 2019 08:09:49 -0700 Subject: [PATCH 093/542] Fix a crashing bug in redis cluster when the dns resolution is empty. (#7392) Signed-off-by: Henry Yang --- .../clusters/redis/redis_cluster.cc | 28 +++++++++++---- .../extensions/clusters/redis/redis_cluster.h | 5 +-- .../clusters/redis/redis_cluster_test.cc | 34 +++++++++++++++---- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index 06c0ac9d6b0d..bf5160805944 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -54,7 +54,7 @@ RedisCluster::RedisCluster( void RedisCluster::startPreInit() { for (const DnsDiscoveryResolveTargetPtr& target : dns_discovery_resolve_targets_) { - target->startResolve(); + target->startResolveDns(); } } @@ -123,7 +123,7 @@ RedisCluster::DnsDiscoveryResolveTarget::~DnsDiscoveryResolveTarget() { } } -void RedisCluster::DnsDiscoveryResolveTarget::startResolve() { +void RedisCluster::DnsDiscoveryResolveTarget::startResolveDns() { ENVOY_LOG(trace, "starting async DNS resolution for {}", dns_address_); active_query_ = parent_.dns_resolver_->resolve( @@ -131,8 +131,24 @@ void RedisCluster::DnsDiscoveryResolveTarget::startResolve() { [this](std::list&& response) -> void { active_query_ = nullptr; ENVOY_LOG(trace, "async DNS resolution complete for {}", dns_address_); - parent_.redis_discovery_session_.registerDiscoveryAddress(std::move(response), port_); - parent_.redis_discovery_session_.startResolve(); + if (response.empty()) { + parent_.info_->stats().update_empty_.inc(); + if (!resolve_timer_) { + resolve_timer_ = + parent_.dispatcher_.createTimer([this]() -> void { startResolveDns(); }); + } + // if the initial dns resolved to empty, we'll skip the redis discovery phase and treat it + // as an empty cluster. + parent_.onPreInitComplete(); + resolve_timer_->enableTimer(parent_.cluster_refresh_rate_); + } else { + // Once the DNS resolve the initial set of addresses, call startResolveRedis on the + // RedisDiscoverySession. The RedisDiscoverySession will using the "cluster slots" command + // for service discovery and slot allocation. All subsequent discoveries are handled by + // RedisDiscoverySession and will not use DNS resolution again. + parent_.redis_discovery_session_.registerDiscoveryAddress(std::move(response), port_); + parent_.redis_discovery_session_.startResolveRedis(); + } }); } @@ -141,7 +157,7 @@ RedisCluster::RedisDiscoverySession::RedisDiscoverySession( Envoy::Extensions::Clusters::Redis::RedisCluster& parent, NetworkFilters::Common::Redis::Client::ClientFactory& client_factory) : parent_(parent), dispatcher_(parent.dispatcher_), - resolve_timer_(parent.dispatcher_.createTimer([this]() -> void { startResolve(); })), + resolve_timer_(parent.dispatcher_.createTimer([this]() -> void { startResolveRedis(); })), client_factory_(client_factory), buffer_timeout_(0) {} namespace { @@ -199,7 +215,7 @@ void RedisCluster::RedisDiscoverySession::registerDiscoveryAddress( } } -void RedisCluster::RedisDiscoverySession::startResolve() { +void RedisCluster::RedisDiscoverySession::startResolveRedis() { parent_.info_->stats().update_attempt_.inc(); // If a resolution is currently in progress, skip it. if (current_request_) { diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index d35dc0dfcd39..f3f0e141e830 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -160,12 +160,13 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { ~DnsDiscoveryResolveTarget(); - void startResolve(); + void startResolveDns(); RedisCluster& parent_; Network::ActiveDnsQuery* active_query_{}; const std::string dns_address_; const uint32_t port_; + Event::TimerPtr resolve_timer_; }; using DnsDiscoveryResolveTargetPtr = std::unique_ptr; @@ -198,7 +199,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { void registerDiscoveryAddress(std::list&& response, const uint32_t port); // Start discovery against a random host from existing hosts - void startResolve(); + void startResolveRedis(); // Extensions::NetworkFilters::Common::Redis::Client::Config bool disableOutlierEvents() const override { return true; } diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 09c4f16579e5..ca2d35893111 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -14,9 +14,7 @@ #include "test/common/upstream/utility.h" #include "test/extensions/clusters/redis/mocks.h" #include "test/extensions/filters/network/common/redis/mocks.h" -#include "test/mocks/common.h" #include "test/mocks/local_info/mocks.h" -#include "test/mocks/network/mocks.h" #include "test/mocks/protobuf/mocks.h" #include "test/mocks/server/mocks.h" #include "test/mocks/ssl/mocks.h" @@ -422,7 +420,7 @@ class RedisClusterTest : public testing::Test, return &active_dns_query_; })); ; - resolver_target.startResolve(); + resolver_target.startResolveDns(); EXPECT_CALL(active_dns_query_, cancel()); } @@ -434,10 +432,10 @@ class RedisClusterTest : public testing::Test, TestUtility::makeDnsResponse(std::list({"127.0.0.1", "127.0.0.2"})); discovery_session.registerDiscoveryAddress(std::move(dns_response), 22120); expectRedisResolve(true); - discovery_session.startResolve(); + discovery_session.startResolveRedis(); - // 2nd startResolve() call will be a no-opt until the first startResolve is done. - discovery_session.startResolve(); + // 2nd startResolveRedis() call will be a no-opt until the first startResolve is done. + discovery_session.startResolveRedis(); // Make sure cancel is called. EXPECT_CALL(pool_request_, cancel()); @@ -553,6 +551,30 @@ TEST_P(RedisDnsParamTest, ImmediateResolveDns) { expectHealthyHosts(std::get<3>(GetParam())); } +TEST_F(RedisClusterTest, EmptyDnsResponse) { + Event::MockTimer* dns_timer = new NiceMock(&dispatcher_); + setupFromV2Yaml(BasicConfig); + const std::list resolved_addresses{}; + EXPECT_CALL(*dns_timer, enableTimer(_)); + expectResolveDiscovery(Network::DnsLookupFamily::V4Only, "foo.bar.com", resolved_addresses); + + EXPECT_CALL(initialized_, ready()); + cluster_->initialize([&]() -> void { initialized_.ready(); }); + + EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); + EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); + EXPECT_EQ(1U, cluster_->info()->stats().update_empty_.value()); + + // Does not recreate the timer on subsequent DNS resolve calls. + EXPECT_CALL(*dns_timer, enableTimer(_)); + expectResolveDiscovery(Network::DnsLookupFamily::V4Only, "foo.bar.com", resolved_addresses); + dns_timer->invokeCallback(); + + EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); + EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); + EXPECT_EQ(2U, cluster_->info()->stats().update_empty_.value()); +} + TEST_F(RedisClusterTest, Basic) { // Using load assignment. const std::string basic_yaml_load_assignment = R"EOF( From c9a2c32add00968426bad80017d9ca556a1658a3 Mon Sep 17 00:00:00 2001 From: Sriduth Jayhari Date: Tue, 2 Jul 2019 20:40:57 +0530 Subject: [PATCH 094/542] docs: Add changelog for retry predicate added in #7230 under v1.11 version (#7444) history. Signed-off-by: Sriduth Jayhari --- docs/root/intro/version_history.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ecd9a14ee2bb..5ef39fb1d2a2 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -53,6 +53,7 @@ Version history :ref:`max_buffer_size_before_flush ` to batch commands together until the encoder buffer hits a certain size, and :ref:`buffer_flush_timeout ` to control how quickly the buffer is flushed if it is not full. * redis: added auth support :ref:`downstream_auth_password ` for downstream client authentication, and :ref:`auth_password ` to configure authentication passwords for upstream server clusters. +* retry: added a retry predicate that :ref:`rejects canary hosts. ` * router: add support for configuring a :ref:`grpc timeout offset ` on incoming requests. * router: added ability to control retry back-off intervals via :ref:`retry policy `. * router: added ability to issue a hedged retry in response to a per try timeout via a :ref:`hedge policy `. @@ -153,7 +154,6 @@ Version history * redis: added :ref:`success and error stats ` for commands. * redis: migrate hash function for host selection to `MurmurHash2 `_ from std::hash. MurmurHash2 is compatible with std::hash in GNU libstdc++ 3.4.20 or above. This is typically the case when compiled on Linux and not macOS. * redis: added :ref:`latency_in_micros ` to specify the redis commands stats time unit in microseconds. -* retry: added a retry predicate that :ref:`rejects canary hosts. ` * router: added ability to configure a :ref:`retry policy ` at the virtual host level. * router: added reset reason to response body when upstream reset happens. After this change, the response body will be of the form `upstream connect error or disconnect/reset before headers. reset reason:` From c0a1ded969095e2989bac70f0f4637ed4c42ffb1 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Tue, 2 Jul 2019 08:12:04 -0700 Subject: [PATCH 095/542] clang-tidy: modernize-return-braced-init-list (#7447) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + source/common/buffer/buffer_impl.h | 4 +- source/common/grpc/common.cc | 4 +- .../addr_family_aware_socket_option_impl.cc | 11 ++- source/common/router/config_utility.cc | 2 +- .../common/upstream/outlier_detection_impl.cc | 6 +- .../filters/network/dubbo_proxy/decoder.cc | 22 ++--- .../filters/network/thrift_proxy/decoder.cc | 82 +++++++++---------- .../tracers/zipkin/span_context_extractor.cc | 8 +- .../listener/original_src/config_test.cc | 2 +- 10 files changed, 70 insertions(+), 72 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 8c0a761ab080..13240046f954 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -20,6 +20,7 @@ WarningsAsErrors: 'abseil-duration-*, bugprone-use-after-move, modernize-make-shared, modernize-make-unique, + modernize-return-braced-init-list, modernize-use-using, performance-faster-string-find, performance-for-range-copy, diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 99ca54af12ff..9851686b4fbb 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -341,9 +341,9 @@ class SliceDeque { size_t index_; }; - ConstIterator begin() const noexcept { return ConstIterator(*this, 0); } + ConstIterator begin() const noexcept { return {*this, 0}; } - ConstIterator end() const noexcept { return ConstIterator(*this, size_); } + ConstIterator end() const noexcept { return {*this, size_}; } private: constexpr static size_t InlineRingCapacity = 8; diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index 43ff985a661f..d70c86ba5343 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -58,9 +58,9 @@ absl::optional Common::getGrpcStatus(const Http::HeaderMap& } if (!absl::SimpleAtoi(grpc_status_header->value().getStringView(), &grpc_status_code) || grpc_status_code > Status::GrpcStatus::MaximumValid) { - return absl::optional(Status::GrpcStatus::InvalidCode); + return {Status::GrpcStatus::InvalidCode}; } - return absl::optional(static_cast(grpc_status_code)); + return {static_cast(grpc_status_code)}; } std::string Common::getGrpcMessage(const Http::HeaderMap& trailers) { diff --git a/source/common/network/addr_family_aware_socket_option_impl.cc b/source/common/network/addr_family_aware_socket_option_impl.cc index c5c4acbdcaba..2e06974f5aa3 100644 --- a/source/common/network/addr_family_aware_socket_option_impl.cc +++ b/source/common/network/addr_family_aware_socket_option_impl.cc @@ -25,10 +25,9 @@ absl::optional getVersionFromSocket(const Socket& socket) { // TODO(htuch): Figure out a way to obtain a consistent interface for IP // version from socket. if (socket.localAddress()) { - return absl::optional(getVersionFromAddress(socket.localAddress())); + return {getVersionFromAddress(socket.localAddress())}; } else { - return absl::optional( - getVersionFromAddress(Address::addressFromFd(socket.ioHandle().fd()))); + return {getVersionFromAddress(Address::addressFromFd(socket.ioHandle().fd()))}; } } catch (const EnvoyException&) { // Ignore, we get here because we failed in getsockname(). @@ -48,15 +47,15 @@ getOptionForSocket(const Socket& socket, SocketOptionImpl& ipv4_option, // If the FD is v4, we can only try the IPv4 variant. if (*version == Network::Address::IpVersion::v4) { - return absl::optional>(ipv4_option); + return {ipv4_option}; } // If the FD is v6, we first try the IPv6 variant if the platform supports it and fallback to the // IPv4 variant otherwise. ASSERT(*version == Network::Address::IpVersion::v6); if (ipv6_option.isSupported()) { - return absl::optional>(ipv6_option); + return {ipv6_option}; } - return absl::optional>(ipv4_option); + return {ipv4_option}; } } // namespace diff --git a/source/common/router/config_utility.cc b/source/common/router/config_utility.cc index 2ca55c665e33..c48dd794d5f4 100644 --- a/source/common/router/config_utility.cc +++ b/source/common/router/config_utility.cc @@ -72,7 +72,7 @@ ConfigUtility::parseDirectResponseCode(const envoy::api::v2::route::Route& route } else if (route.has_direct_response()) { return static_cast(route.direct_response().status()); } - return absl::optional(); + return {}; } std::string ConfigUtility::parseDirectResponseBody(const envoy::api::v2::route::Route& route, diff --git a/source/common/upstream/outlier_detection_impl.cc b/source/common/upstream/outlier_detection_impl.cc index 812d72ab1575..948cbb1f361f 100644 --- a/source/common/upstream/outlier_detection_impl.cc +++ b/source/common/upstream/outlier_detection_impl.cc @@ -119,7 +119,7 @@ absl::optional DetectorHostMonitorImpl::resultToHttpCode(Result resu return absl::nullopt; } - return absl::optional(http_code); + return {http_code}; } // Method is called by putResult when external and local origin errors @@ -713,8 +713,8 @@ SuccessRateAccumulator::getSuccessRate(uint64_t success_rate_request_volume) { return absl::optional(); } - return absl::optional(backup_success_rate_bucket_->success_request_counter_ * 100.0 / - backup_success_rate_bucket_->total_request_counter_); + return {backup_success_rate_bucket_->success_request_counter_ * 100.0 / + backup_success_rate_bucket_->total_request_counter_}; } } // namespace Outlier diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.cc b/source/extensions/filters/network/dubbo_proxy/decoder.cc index c47433e373d2..6d7538c7ec6c 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.cc +++ b/source/extensions/filters/network/dubbo_proxy/decoder.cc @@ -13,45 +13,45 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::onTransportBegin(Buffer::Instance& buffer, Protocol::Context& context) { if (!protocol_.decode(buffer, &context, metadata_)) { ENVOY_LOG(debug, "dubbo decoder: need more data for {} protocol", protocol_.name()); - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } if (context.is_heartbeat_) { ENVOY_LOG(debug, "dubbo decoder: this is the {} heartbeat message", protocol_.name()); buffer.drain(context.header_size_); decoder_callbacks_.onHeartbeat(metadata_); - return DecoderStatus(ProtocolState::Done, Network::FilterStatus::Continue); + return {ProtocolState::Done, Network::FilterStatus::Continue}; } else { handler_ = decoder_callbacks_.newDecoderEventHandler(); } - return DecoderStatus(ProtocolState::OnTransferHeaderTo, handler_->transportBegin()); + return {ProtocolState::OnTransferHeaderTo, handler_->transportBegin()}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::onTransportEnd() { ENVOY_LOG(debug, "dubbo decoder: complete protocol processing"); - return DecoderStatus(ProtocolState::Done, handler_->transportEnd()); + return {ProtocolState::Done, handler_->transportEnd()}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::onTransferHeaderTo(Buffer::Instance& buffer, size_t length) { ENVOY_LOG(debug, "dubbo decoder: transfer protocol header, buffer size {}, header size {}", buffer.length(), length); - return DecoderStatus(ProtocolState::OnMessageBegin, handler_->transferHeaderTo(buffer, length)); + return {ProtocolState::OnMessageBegin, handler_->transferHeaderTo(buffer, length)}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::onTransferBodyTo(Buffer::Instance& buffer, int32_t length) { ENVOY_LOG(debug, "dubbo decoder: transfer protocol body, buffer size {}, body size {}", buffer.length(), length); - return DecoderStatus(ProtocolState::OnTransportEnd, handler_->transferBodyTo(buffer, length)); + return {ProtocolState::OnTransportEnd, handler_->transferBodyTo(buffer, length)}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::onMessageBegin() { ENVOY_LOG(debug, "dubbo decoder: start deserializing messages, deserializer name {}", deserializer_.name()); - return DecoderStatus(ProtocolState::OnMessageEnd, - handler_->messageBegin(metadata_->message_type(), metadata_->request_id(), - metadata_->serialization_type())); + return {ProtocolState::OnMessageEnd, + handler_->messageBegin(metadata_->message_type(), metadata_->request_id(), + metadata_->serialization_type())}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::onMessageEnd(Buffer::Instance& buffer, @@ -61,7 +61,7 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::onMessageEnd(Buffer::Ins if (buffer.length() < static_cast(message_size)) { ENVOY_LOG(debug, "dubbo decoder: need more data for {} deserialization, current size {}", deserializer_.name(), buffer.length()); - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } switch (metadata_->message_type()) { @@ -81,7 +81,7 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::onMessageEnd(Buffer::Ins } ENVOY_LOG(debug, "dubbo decoder: ends the deserialization of the message"); - return DecoderStatus(ProtocolState::OnTransferBodyTo, handler_->messageEnd(metadata_)); + return {ProtocolState::OnTransferBodyTo, handler_->messageEnd(metadata_)}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::handleState(Buffer::Instance& buffer) { diff --git a/source/extensions/filters/network/thrift_proxy/decoder.cc b/source/extensions/filters/network/thrift_proxy/decoder.cc index bc56bffea0b0..487e026232f6 100644 --- a/source/extensions/filters/network/thrift_proxy/decoder.cc +++ b/source/extensions/filters/network/thrift_proxy/decoder.cc @@ -17,42 +17,42 @@ namespace ThriftProxy { // MessageBegin -> StructBegin DecoderStateMachine::DecoderStatus DecoderStateMachine::messageBegin(Buffer::Instance& buffer) { if (!proto_.readMessageBegin(buffer, *metadata_)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } stack_.clear(); stack_.emplace_back(Frame(ProtocolState::MessageEnd)); - return DecoderStatus(ProtocolState::StructBegin, handler_.messageBegin(metadata_)); + return {ProtocolState::StructBegin, handler_.messageBegin(metadata_)}; } // MessageEnd -> Done DecoderStateMachine::DecoderStatus DecoderStateMachine::messageEnd(Buffer::Instance& buffer) { if (!proto_.readMessageEnd(buffer)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } - return DecoderStatus(ProtocolState::Done, handler_.messageEnd()); + return {ProtocolState::Done, handler_.messageEnd()}; } // StructBegin -> FieldBegin DecoderStateMachine::DecoderStatus DecoderStateMachine::structBegin(Buffer::Instance& buffer) { std::string name; if (!proto_.readStructBegin(buffer, name)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } - return DecoderStatus(ProtocolState::FieldBegin, handler_.structBegin(absl::string_view(name))); + return {ProtocolState::FieldBegin, handler_.structBegin(absl::string_view(name))}; } // StructEnd -> stack's return state DecoderStateMachine::DecoderStatus DecoderStateMachine::structEnd(Buffer::Instance& buffer) { if (!proto_.readStructEnd(buffer)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } ProtocolState next_state = popReturnState(); - return DecoderStatus(next_state, handler_.structEnd()); + return {next_state, handler_.structEnd()}; } // FieldBegin -> FieldValue, or @@ -62,17 +62,17 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::fieldBegin(Buffer::Insta FieldType field_type; int16_t field_id; if (!proto_.readFieldBegin(buffer, name, field_type, field_id)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } if (field_type == FieldType::Stop) { - return DecoderStatus(ProtocolState::StructEnd, FilterStatus::Continue); + return {ProtocolState::StructEnd, FilterStatus::Continue}; } stack_.emplace_back(Frame(ProtocolState::FieldEnd, field_type)); - return DecoderStatus(ProtocolState::FieldValue, - handler_.fieldBegin(absl::string_view(name), field_type, field_id)); + return {ProtocolState::FieldValue, + handler_.fieldBegin(absl::string_view(name), field_type, field_id)}; } // FieldValue -> FieldEnd (via stack return state) @@ -86,12 +86,12 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::fieldValue(Buffer::Insta // FieldEnd -> FieldBegin DecoderStateMachine::DecoderStatus DecoderStateMachine::fieldEnd(Buffer::Instance& buffer) { if (!proto_.readFieldEnd(buffer)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } popReturnState(); - return DecoderStatus(ProtocolState::FieldBegin, handler_.fieldEnd()); + return {ProtocolState::FieldBegin, handler_.fieldEnd()}; } // ListBegin -> ListValue @@ -99,12 +99,12 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::listBegin(Buffer::Instan FieldType elem_type; uint32_t size; if (!proto_.readListBegin(buffer, elem_type, size)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } stack_.emplace_back(Frame(ProtocolState::ListEnd, elem_type, size)); - return DecoderStatus(ProtocolState::ListValue, handler_.listBegin(elem_type, size)); + return {ProtocolState::ListValue, handler_.listBegin(elem_type, size)}; } // ListValue -> ListValue, ListBegin, MapBegin, SetBegin, StructBegin (depending on value type), or @@ -113,7 +113,7 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::listValue(Buffer::Instan ASSERT(!stack_.empty()); Frame& frame = stack_.back(); if (frame.remaining_ == 0) { - return DecoderStatus(popReturnState(), FilterStatus::Continue); + return {popReturnState(), FilterStatus::Continue}; } frame.remaining_--; @@ -123,11 +123,11 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::listValue(Buffer::Instan // ListEnd -> stack's return state DecoderStateMachine::DecoderStatus DecoderStateMachine::listEnd(Buffer::Instance& buffer) { if (!proto_.readListEnd(buffer)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } ProtocolState next_state = popReturnState(); - return DecoderStatus(next_state, handler_.listEnd()); + return {next_state, handler_.listEnd()}; } // MapBegin -> MapKey @@ -135,12 +135,12 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::mapBegin(Buffer::Instanc FieldType key_type, value_type; uint32_t size; if (!proto_.readMapBegin(buffer, key_type, value_type, size)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } stack_.emplace_back(Frame(ProtocolState::MapEnd, key_type, value_type, size)); - return DecoderStatus(ProtocolState::MapKey, handler_.mapBegin(key_type, value_type, size)); + return {ProtocolState::MapKey, handler_.mapBegin(key_type, value_type, size)}; } // MapKey -> MapValue, ListBegin, MapBegin, SetBegin, StructBegin (depending on key type), or @@ -149,7 +149,7 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::mapKey(Buffer::Instance& ASSERT(!stack_.empty()); Frame& frame = stack_.back(); if (frame.remaining_ == 0) { - return DecoderStatus(popReturnState(), FilterStatus::Continue); + return {popReturnState(), FilterStatus::Continue}; } return handleValue(buffer, frame.elem_type_, ProtocolState::MapValue); @@ -169,11 +169,11 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::mapValue(Buffer::Instanc // MapEnd -> stack's return state DecoderStateMachine::DecoderStatus DecoderStateMachine::mapEnd(Buffer::Instance& buffer) { if (!proto_.readMapEnd(buffer)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } ProtocolState next_state = popReturnState(); - return DecoderStatus(next_state, handler_.mapEnd()); + return {next_state, handler_.mapEnd()}; } // SetBegin -> SetValue @@ -181,12 +181,12 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::setBegin(Buffer::Instanc FieldType elem_type; uint32_t size; if (!proto_.readSetBegin(buffer, elem_type, size)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } stack_.emplace_back(Frame(ProtocolState::SetEnd, elem_type, size)); - return DecoderStatus(ProtocolState::SetValue, handler_.setBegin(elem_type, size)); + return {ProtocolState::SetValue, handler_.setBegin(elem_type, size)}; } // SetValue -> SetValue, ListBegin, MapBegin, SetBegin, StructBegin (depending on value type), or @@ -195,7 +195,7 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::setValue(Buffer::Instanc ASSERT(!stack_.empty()); Frame& frame = stack_.back(); if (frame.remaining_ == 0) { - return DecoderStatus(popReturnState(), FilterStatus::Continue); + return {popReturnState(), FilterStatus::Continue}; } frame.remaining_--; @@ -205,11 +205,11 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::setValue(Buffer::Instanc // SetEnd -> stack's return state DecoderStateMachine::DecoderStatus DecoderStateMachine::setEnd(Buffer::Instance& buffer) { if (!proto_.readSetEnd(buffer)) { - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } ProtocolState next_state = popReturnState(); - return DecoderStatus(next_state, handler_.setEnd()); + return {next_state, handler_.setEnd()}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::handleValue(Buffer::Instance& buffer, @@ -219,69 +219,69 @@ DecoderStateMachine::DecoderStatus DecoderStateMachine::handleValue(Buffer::Inst case FieldType::Bool: { bool value{}; if (proto_.readBool(buffer, value)) { - return DecoderStatus(return_state, handler_.boolValue(value)); + return {return_state, handler_.boolValue(value)}; } break; } case FieldType::Byte: { uint8_t value{}; if (proto_.readByte(buffer, value)) { - return DecoderStatus(return_state, handler_.byteValue(value)); + return {return_state, handler_.byteValue(value)}; } break; } case FieldType::I16: { int16_t value{}; if (proto_.readInt16(buffer, value)) { - return DecoderStatus(return_state, handler_.int16Value(value)); + return {return_state, handler_.int16Value(value)}; } break; } case FieldType::I32: { int32_t value{}; if (proto_.readInt32(buffer, value)) { - return DecoderStatus(return_state, handler_.int32Value(value)); + return {return_state, handler_.int32Value(value)}; } break; } case FieldType::I64: { int64_t value{}; if (proto_.readInt64(buffer, value)) { - return DecoderStatus(return_state, handler_.int64Value(value)); + return {return_state, handler_.int64Value(value)}; } break; } case FieldType::Double: { double value{}; if (proto_.readDouble(buffer, value)) { - return DecoderStatus(return_state, handler_.doubleValue(value)); + return {return_state, handler_.doubleValue(value)}; } break; } case FieldType::String: { std::string value; if (proto_.readString(buffer, value)) { - return DecoderStatus(return_state, handler_.stringValue(value)); + return {return_state, handler_.stringValue(value)}; } break; } case FieldType::Struct: stack_.emplace_back(Frame(return_state)); - return DecoderStatus(ProtocolState::StructBegin, FilterStatus::Continue); + return {ProtocolState::StructBegin, FilterStatus::Continue}; case FieldType::Map: stack_.emplace_back(Frame(return_state)); - return DecoderStatus(ProtocolState::MapBegin, FilterStatus::Continue); + return {ProtocolState::MapBegin, FilterStatus::Continue}; case FieldType::List: stack_.emplace_back(Frame(return_state)); - return DecoderStatus(ProtocolState::ListBegin, FilterStatus::Continue); + return {ProtocolState::ListBegin, FilterStatus::Continue}; case FieldType::Set: stack_.emplace_back(Frame(return_state)); - return DecoderStatus(ProtocolState::SetBegin, FilterStatus::Continue); + return {ProtocolState::SetBegin, FilterStatus::Continue}; default: throw EnvoyException(fmt::format("unknown field type {}", static_cast(elem_type))); } - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::handleState(Buffer::Instance& buffer) { diff --git a/source/extensions/tracers/zipkin/span_context_extractor.cc b/source/extensions/tracers/zipkin/span_context_extractor.cc index dc9413760367..50a057ae45af 100644 --- a/source/extensions/tracers/zipkin/span_context_extractor.cc +++ b/source/extensions/tracers/zipkin/span_context_extractor.cc @@ -112,11 +112,10 @@ std::pair SpanContextExtractor::extractSpanContext(bool is_sa } } } else { - return std::pair(SpanContext(), false); + return {SpanContext(), false}; } - return std::pair( - SpanContext(trace_id_high, trace_id, span_id, parent_id, is_sampled), true); + return {SpanContext(trace_id_high, trace_id, span_id, parent_id, is_sampled), true}; } std::pair @@ -219,8 +218,7 @@ SpanContextExtractor::extractSpanContextFromB3SingleFormat(bool is_sampled) { } } - return std::pair( - SpanContext(trace_id_high, trace_id, span_id, parent_id, is_sampled), true); + return {SpanContext(trace_id_high, trace_id, span_id, parent_id, is_sampled), true}; } } // namespace Zipkin diff --git a/test/extensions/filters/listener/original_src/config_test.cc b/test/extensions/filters/listener/original_src/config_test.cc index 6d38e14b918e..9ec92c7be9bb 100644 --- a/test/extensions/filters/listener/original_src/config_test.cc +++ b/test/extensions/filters/listener/original_src/config_test.cc @@ -19,7 +19,7 @@ class OriginalSrcConfigTest : public testing::Test { public: Config makeConfigFromProto( const envoy::config::filter::listener::original_src::v2alpha1::OriginalSrc& proto_config) { - return Config(proto_config); + return {proto_config}; } }; From f51fab0d4518e87ae93d71b3c42f8c69d68a9514 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Tue, 2 Jul 2019 08:13:34 -0700 Subject: [PATCH 096/542] clang-tidy: readability-redundant-member-init (#7448) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + source/common/config/metadata.h | 4 +--- source/common/filesystem/posix/directory_iterator_impl.h | 3 +-- source/common/upstream/subset_lb.cc | 5 ++--- source/extensions/filters/http/gzip/gzip_filter.cc | 2 +- source/extensions/filters/http/squash/squash_filter.cc | 5 ++--- source/extensions/filters/network/dubbo_proxy/decoder.h | 2 +- source/extensions/filters/network/thrift_proxy/decoder.h | 2 +- source/extensions/tracers/zipkin/zipkin_core_types.h | 8 ++++---- test/common/compressor/zlib_compressor_impl_test.cc | 2 +- test/common/config/delta_subscription_impl_test.cc | 2 +- test/common/upstream/eds_test.cc | 2 +- test/integration/sds_dynamic_integration_test.cc | 3 +-- 13 files changed, 18 insertions(+), 23 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 13240046f954..264254148fbe 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -28,6 +28,7 @@ WarningsAsErrors: 'abseil-duration-*, performance-unnecessary-copy-initialization, readability-braces-around-statements, readability-container-size-empty, + readability-redundant-member-init, readability-redundant-smartptr-get, readability-redundant-string-cstr' diff --git a/source/common/config/metadata.h b/source/common/config/metadata.h index 1b80acce0536..538d14c2356a 100644 --- a/source/common/config/metadata.h +++ b/source/common/config/metadata.h @@ -54,9 +54,7 @@ template class TypedMetadataImpl : public TypedMetadata public: static_assert(std::is_base_of::value, "Factory type must be inherited from Envoy::Config::TypedMetadataFactory."); - TypedMetadataImpl(const envoy::api::v2::core::Metadata& metadata) : data_() { - populateFrom(metadata); - } + TypedMetadataImpl(const envoy::api::v2::core::Metadata& metadata) { populateFrom(metadata); } const TypedMetadata::Object* getData(const std::string& key) const override { const auto& it = data_.find(key); diff --git a/source/common/filesystem/posix/directory_iterator_impl.h b/source/common/filesystem/posix/directory_iterator_impl.h index 4bd36f92722d..513b5f98406d 100644 --- a/source/common/filesystem/posix/directory_iterator_impl.h +++ b/source/common/filesystem/posix/directory_iterator_impl.h @@ -13,8 +13,7 @@ class DirectoryIteratorImpl : public DirectoryIterator { public: DirectoryIteratorImpl(const std::string& directory_path); DirectoryIteratorImpl() - : DirectoryIterator(), directory_path_(""), dir_(nullptr), - os_sys_calls_(Api::OsSysCallsSingleton::get()) {} + : directory_path_(""), dir_(nullptr), os_sys_calls_(Api::OsSysCallsSingleton::get()) {} ~DirectoryIteratorImpl(); diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index e867e4f39ced..0d2ae60db91c 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -615,9 +615,8 @@ SubsetLoadBalancer::PrioritySubsetImpl::PrioritySubsetImpl(const SubsetLoadBalan HostPredicate predicate, bool locality_weight_aware, bool scale_locality_weight) - : PrioritySetImpl(), original_priority_set_(subset_lb.original_priority_set_), - predicate_(predicate), locality_weight_aware_(locality_weight_aware), - scale_locality_weight_(scale_locality_weight) { + : original_priority_set_(subset_lb.original_priority_set_), predicate_(predicate), + locality_weight_aware_(locality_weight_aware), scale_locality_weight_(scale_locality_weight) { for (size_t i = 0; i < original_priority_set_.hostSetsPerPriority().size(); ++i) { empty_ &= getOrCreateHostSet(i).hosts().empty(); diff --git a/source/extensions/filters/http/gzip/gzip_filter.cc b/source/extensions/filters/http/gzip/gzip_filter.cc index 82d30d49f168..7a521f07c24c 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.cc +++ b/source/extensions/filters/http/gzip/gzip_filter.cc @@ -103,7 +103,7 @@ uint64_t GzipFilterConfig::windowBitsUint(Protobuf::uint32 window_bits) { } GzipFilter::GzipFilter(const GzipFilterConfigSharedPtr& config) - : skip_compression_{true}, compressed_data_(), compressor_(), config_(config) {} + : skip_compression_{true}, config_(config) {} Http::FilterHeadersStatus GzipFilter::decodeHeaders(Http::HeaderMap& headers, bool) { if (config_->runtime().snapshot().featureEnabled("gzip.filter_enabled", 100) && diff --git a/source/extensions/filters/http/squash/squash_filter.cc b/source/extensions/filters/http/squash/squash_filter.cc index 9e12c6c4d43f..be4d4e095c9b 100644 --- a/source/extensions/filters/http/squash/squash_filter.cc +++ b/source/extensions/filters/http/squash/squash_filter.cc @@ -121,9 +121,8 @@ std::string SquashFilterConfig::replaceEnv(const std::string& attachment_templat } SquashFilter::SquashFilter(SquashFilterConfigSharedPtr config, Upstream::ClusterManager& cm) - : config_(config), is_squashing_(false), debug_attachment_path_(), - attachment_poll_period_timer_(nullptr), attachment_timeout_timer_(nullptr), - in_flight_request_(nullptr), + : config_(config), is_squashing_(false), attachment_poll_period_timer_(nullptr), + attachment_timeout_timer_(nullptr), in_flight_request_(nullptr), create_attachment_callback_(std::bind(&SquashFilter::onCreateAttachmentSuccess, this, _1), std::bind(&SquashFilter::onCreateAttachmentFailure, this, _1)), check_attachment_callback_(std::bind(&SquashFilter::onGetAttachmentSuccess, this, _1), diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.h b/source/extensions/filters/network/dubbo_proxy/decoder.h index 6a320fb2a411..982b9dd31b9d 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.h +++ b/source/extensions/filters/network/dubbo_proxy/decoder.h @@ -77,7 +77,7 @@ class DecoderStateMachine : public Logger::Loggable { private: struct DecoderStatus { DecoderStatus() = default; - DecoderStatus(ProtocolState next_state) : next_state_(next_state), filter_status_{} {}; + DecoderStatus(ProtocolState next_state) : next_state_(next_state){}; DecoderStatus(ProtocolState next_state, Network::FilterStatus filter_status) : next_state_(next_state), filter_status_(filter_status){}; diff --git a/source/extensions/filters/network/thrift_proxy/decoder.h b/source/extensions/filters/network/thrift_proxy/decoder.h index 7e5678c556b2..1f7675b9f76f 100644 --- a/source/extensions/filters/network/thrift_proxy/decoder.h +++ b/source/extensions/filters/network/thrift_proxy/decoder.h @@ -119,7 +119,7 @@ class DecoderStateMachine : public Logger::Loggable { }; struct DecoderStatus { - DecoderStatus(ProtocolState next_state) : next_state_(next_state), filter_status_{} {}; + DecoderStatus(ProtocolState next_state) : next_state_(next_state){}; DecoderStatus(ProtocolState next_state, FilterStatus filter_status) : next_state_(next_state), filter_status_(filter_status){}; diff --git a/source/extensions/tracers/zipkin/zipkin_core_types.h b/source/extensions/tracers/zipkin/zipkin_core_types.h index 165ae642c840..d9b5f10fc2d8 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_types.h +++ b/source/extensions/tracers/zipkin/zipkin_core_types.h @@ -56,7 +56,7 @@ class Endpoint : public ZipkinBase { /** * Default constructor. Creates an empty Endpoint. */ - Endpoint() : service_name_(), address_(nullptr) {} + Endpoint() : address_(nullptr) {} /** * Constructor that initializes an endpoint with the given attributes. @@ -118,7 +118,7 @@ class Annotation : public ZipkinBase { /** * Default constructor. Creates an empty annotation. */ - Annotation() : timestamp_(0), value_() {} + Annotation() : timestamp_(0) {} /** * Constructor that creates an annotation based on the given parameters. @@ -217,7 +217,7 @@ class BinaryAnnotation : public ZipkinBase { /** * Default constructor. Creates an empty binary annotation. */ - BinaryAnnotation() : key_(), value_(), annotation_type_(STRING) {} + BinaryAnnotation() : annotation_type_(STRING) {} /** * Constructor that creates a binary annotation based on the given parameters. @@ -307,7 +307,7 @@ class Span : public ZipkinBase { * Default constructor. Creates an empty span. */ explicit Span(TimeSource& time_source) - : trace_id_(0), name_(), id_(0), debug_(false), sampled_(false), monotonic_start_time_(0), + : trace_id_(0), id_(0), debug_(false), sampled_(false), monotonic_start_time_(0), tracer_(nullptr), time_source_(time_source) {} /** diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 2410d78fe308..0d0c4457cb89 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -70,7 +70,7 @@ class ZlibCompressorImplTest : public testing::Test { class ZlibCompressorImplTester : public ZlibCompressorImpl { public: - ZlibCompressorImplTester() : ZlibCompressorImpl() {} + ZlibCompressorImplTester() {} ZlibCompressorImplTester(uint64_t chunk_size) : ZlibCompressorImpl(chunk_size) {} void compressThenFlush(Buffer::OwnedImpl& buffer) { compress(buffer, State::Flush); } void finish(Buffer::OwnedImpl& buffer) { compress(buffer, State::Finish); } diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index d2aa335a4fe7..55731c26a88a 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -8,7 +8,7 @@ namespace { class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public testing::Test { protected: - DeltaSubscriptionImplTest() : DeltaSubscriptionTestHarness() {} + DeltaSubscriptionImplTest() {} }; TEST_F(DeltaSubscriptionImplTest, UpdateResourcesCausesRequest) { diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index 78854abad4f0..864b7295558f 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -1658,7 +1658,7 @@ TEST_F(EdsTest, MalformedIP) { class EdsAssignmentTimeoutTest : public EdsTest { public: - EdsAssignmentTimeoutTest() : EdsTest(), interval_timer_(nullptr) { + EdsAssignmentTimeoutTest() : interval_timer_(nullptr) { EXPECT_CALL(dispatcher_, createTimer_(_)) .WillOnce(Invoke([this](Event::TimerCb cb) { timer_cb_ = cb; diff --git a/test/integration/sds_dynamic_integration_test.cc b/test/integration/sds_dynamic_integration_test.cc index 0ce8217ec06f..f91c7a5c383a 100644 --- a/test/integration/sds_dynamic_integration_test.cc +++ b/test/integration/sds_dynamic_integration_test.cc @@ -239,8 +239,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, WrongSecretFirst) { class SdsDynamicDownstreamCertValidationContextTest : public SdsDynamicDownstreamIntegrationTest { public: - SdsDynamicDownstreamCertValidationContextTest() - : SdsDynamicDownstreamIntegrationTest(), use_combined_validation_context_(false) {} + SdsDynamicDownstreamCertValidationContextTest() : use_combined_validation_context_(false) {} void initialize() override { config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { From ecd03a4eed07e1cfea9e9844e519b7fffada437a Mon Sep 17 00:00:00 2001 From: xyu-stripe <44764123+xyu-stripe@users.noreply.github.com> Date: Tue, 2 Jul 2019 09:31:48 -0700 Subject: [PATCH 097/542] router: add config to reject requests with invalid envoy headers (#7323) Description: Before this change, Envoy would silently ignore the `x-envoy-*` header if a client specifies an invalid value for this header (e.g. `x-envoy-max-retries: 3.0`). Introduce a `strict_check_headers` config option for `envoy.router` that adds optional support to reject requests with invalid values for the following headers: - x-envoy-upstream-rq-timeout-ms - x-envoy-upstream-rq-per-try-timeout-ms - x-envoy-max-retries - x-envoy-retry-on - x-envoy-retry-grpc-on On rejection, Envoy responds with HTTP status 400 and sets a new response flag `IH` to indicate the reason was due to an invalid header. Risk Level: Low/medium Testing: unit tests - unit test: `FilterUtility::StrictHeaderChecker` - test that router rejects request with HTTP status 400 + setting the `IH` response flag - test that config validation rejects unsupported values - manual end-to-end test `client -> envoy -> upstream server` to verify that Envoy returns a 400 and sets the response flag in the logs Docs Changes: - add inline docs to `router.proto` for `strict_check_headers` - add inline docs to `accesslog.proto` for `IH` response flag Release Notes: updated for router and accesslog Fixes #6482 Signed-off-by: Xiao Yu --- .../filter/accesslog/v2/accesslog.proto | 3 +- .../config/filter/http/router/v2/router.proto | 28 +++ api/envoy/data/accesslog/v2/accesslog.proto | 4 + docs/root/configuration/access_log.rst | 2 + docs/root/intro/version_history.rst | 3 + include/envoy/stream_info/stream_info.h | 6 +- source/common/http/async_client_impl.cc | 6 +- source/common/router/config_impl.cc | 4 +- source/common/router/retry_state_impl.cc | 19 +- source/common/router/retry_state_impl.h | 21 ++- source/common/router/router.cc | 37 ++++ source/common/router/router.h | 62 +++++- source/common/stream_info/utility.cc | 8 +- source/common/stream_info/utility.h | 1 + .../http_grpc/grpc_access_log_impl.cc | 6 +- .../common/access_log/access_log_impl_test.cc | 8 +- test/common/router/router_test.cc | 176 +++++++++++++++++- test/common/stream_info/utility_test.cc | 6 +- .../http_grpc/grpc_access_log_impl_test.cc | 1 + .../filters/http/router/config_test.cc | 24 +++ 20 files changed, 394 insertions(+), 31 deletions(-) diff --git a/api/envoy/config/filter/accesslog/v2/accesslog.proto b/api/envoy/config/filter/accesslog/v2/accesslog.proto index fffd2251eff3..4eec60d7442d 100644 --- a/api/envoy/config/filter/accesslog/v2/accesslog.proto +++ b/api/envoy/config/filter/accesslog/v2/accesslog.proto @@ -191,7 +191,8 @@ message ResponseFlagFilter { "RLSE", "DC", "URX", - "SI" + "SI", + "IH" ] }]; } diff --git a/api/envoy/config/filter/http/router/v2/router.proto b/api/envoy/config/filter/http/router/v2/router.proto index 02115cb81f73..e77675673357 100644 --- a/api/envoy/config/filter/http/router/v2/router.proto +++ b/api/envoy/config/filter/http/router/v2/router.proto @@ -11,6 +11,8 @@ import "envoy/config/filter/accesslog/v2/accesslog.proto"; import "google/protobuf/wrappers.proto"; +import "validate/validate.proto"; + // [#protodoc-title: Router] // Router :ref:`configuration overview `. @@ -36,4 +38,30 @@ message Router { // `, other Envoy filters and the HTTP // connection manager may continue to set *x-envoy-* headers. bool suppress_envoy_headers = 4; + + // Specifies a list of HTTP headers to strictly validate. Envoy will reject a + // request and respond with HTTP status 400 if the request contains an invalid + // value for any of the headers listed in this field. Strict header checking + // is only supported for the following headers: + // + // Value must be a ','-delimited list (i.e. no spaces) of supported retry + // policy values: + // + // * :ref:`config_http_filters_router_x-envoy-retry-grpc-on` + // * :ref:`config_http_filters_router_x-envoy-retry-on` + // + // Value must be an integer: + // + // * :ref:`config_http_filters_router_x-envoy-max-retries` + // * :ref:`config_http_filters_router_x-envoy-upstream-rq-timeout-ms` + // * :ref:`config_http_filters_router_x-envoy-upstream-rq-per-try-timeout-ms` + repeated string strict_check_headers = 5 [(validate.rules).repeated .items.string = { + in: [ + "x-envoy-upstream-rq-timeout-ms", + "x-envoy-upstream-rq-per-try-timeout-ms", + "x-envoy-max-retries", + "x-envoy-retry-grpc-on", + "x-envoy-retry-on" + ] + }]; } diff --git a/api/envoy/data/accesslog/v2/accesslog.proto b/api/envoy/data/accesslog/v2/accesslog.proto index 478d7b03b5a0..1396c729f88e 100644 --- a/api/envoy/data/accesslog/v2/accesslog.proto +++ b/api/envoy/data/accesslog/v2/accesslog.proto @@ -210,6 +210,10 @@ message ResponseFlags { // Indicates that the stream idle timeout was hit, resulting in a downstream 408. bool stream_idle_timeout = 17; + + // Indicates that the request was rejected because an envoy request header failed strict + // validation. + bool invalid_envoy_request_headers = 18; } // Properties of a negotiated TLS connection. diff --git a/docs/root/configuration/access_log.rst b/docs/root/configuration/access_log.rst index 367d27eaf64f..9350cc5fb522 100644 --- a/docs/root/configuration/access_log.rst +++ b/docs/root/configuration/access_log.rst @@ -218,6 +218,8 @@ The following command operators are supported: * **RL**: The request was ratelimited locally by the :ref:`HTTP rate limit filter ` in addition to 429 response code. * **UAEX**: The request was denied by the external authorization service. * **RLSE**: The request was rejected because there was an error in rate limit service. + * **IH**: The request was rejected because it set an invalid value for a + :ref:`strictly-checked header ` in addition to 400 response code. * **SI**: Stream idle timeout in addition to 408 response code. %RESPONSE_TX_DURATION% diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 5ef39fb1d2a2..b79c9c8abc22 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -7,6 +7,7 @@ Version history * access log: added a new field for route name to file and gRPC access logger. * access log: added a new field for response code details in :ref:`file access logger` and :ref:`gRPC access logger`. * access log: added several new variables for exposing information about the downstream TLS connection to :ref:`file access logger` and :ref:`gRPC access logger`. +* access log: added a new flag for request rejected due to failed strict header check. * admin: the administration interface now includes a :ref:`/ready endpoint ` for easier readiness checks. * admin: extend :ref:`/runtime_modify endpoint ` to support parameters within the request body. * admin: the :ref:`/listener endpoint ` now returns :ref:`listeners.proto` which includes listener names and ports. @@ -66,6 +67,8 @@ Version history * router: added :ref:`RouteAction's auto_host_rewrite_header ` to allow upstream host header substitution with some other header's value * router: added support for UPSTREAM_REMOTE_ADDRESS :ref:`header formatter `. +* router: add ability to reject a request that includes invalid values for + headers configured in :ref:`strict_check_headers ` * runtime: added support for :ref:`flexible layering configuration `. * runtime: added support for statically :ref:`specifying the runtime in the bootstrap configuration diff --git a/include/envoy/stream_info/stream_info.h b/include/envoy/stream_info/stream_info.h index fa2a0ffd92ed..1558503ccfe5 100644 --- a/include/envoy/stream_info/stream_info.h +++ b/include/envoy/stream_info/stream_info.h @@ -61,8 +61,10 @@ enum ResponseFlag { UpstreamRetryLimitExceeded = 0x8000, // Request hit the stream idle timeout, triggering a 408. StreamIdleTimeout = 0x10000, + // Request specified x-envoy-* header values that failed strict header checks. + InvalidEnvoyRequestHeaders = 0x20000, // ATTENTION: MAKE SURE THIS REMAINS EQUAL TO THE LAST FLAG. - LastFlag = StreamIdleTimeout + LastFlag = InvalidEnvoyRequestHeaders }; /** @@ -95,6 +97,8 @@ struct ResponseCodeDetailValues { const std::string MissingHost = "missing_host_header"; // The request was rejected due to the request headers being larger than the configured limit. const std::string RequestHeadersTooLarge = "request_headers_too_large"; + // The request was rejected due to x-envoy-* headers failing strict header validation. + const std::string InvalidEnvoyRequestHeaders = "request_headers_failed_strict_check"; // The request was rejected due to the Path or :path header field missing. const std::string MissingPath = "missing_path_rejected"; // The request was rejected due to using an absolute path on a route not supporting them. diff --git a/source/common/http/async_client_impl.cc b/source/common/http/async_client_impl.cc index f4364032a12c..a54b9eea5a56 100644 --- a/source/common/http/async_client_impl.cc +++ b/source/common/http/async_client_impl.cc @@ -36,9 +36,9 @@ AsyncClientImpl::AsyncClientImpl(Upstream::ClusterInfoConstSharedPtr cluster, Runtime::RandomGenerator& random, Router::ShadowWriterPtr&& shadow_writer, Http::Context& http_context) - : cluster_(cluster), - config_("http.async-client.", local_info, stats_store, cm, runtime, random, - std::move(shadow_writer), true, false, false, dispatcher.timeSource(), http_context), + : cluster_(cluster), config_("http.async-client.", local_info, stats_store, cm, runtime, random, + std::move(shadow_writer), true, false, false, {}, + dispatcher.timeSource(), http_context), dispatcher_(dispatcher) {} AsyncClientImpl::~AsyncClientImpl() { diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 8fc29afc398d..dc11c7bde57b 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -70,8 +70,8 @@ RetryPolicyImpl::RetryPolicyImpl(const envoy::api::v2::route::RetryPolicy& retry per_try_timeout_ = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(retry_policy, per_try_timeout, 0)); num_retries_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(retry_policy, num_retries, 1); - retry_on_ = RetryStateImpl::parseRetryOn(retry_policy.retry_on()); - retry_on_ |= RetryStateImpl::parseRetryGrpcOn(retry_policy.retry_on()); + retry_on_ = RetryStateImpl::parseRetryOn(retry_policy.retry_on()).first; + retry_on_ |= RetryStateImpl::parseRetryGrpcOn(retry_policy.retry_on()).first; for (const auto& host_predicate : retry_policy.retry_host_predicate()) { auto& factory = Envoy::Config::Utility::getAndCheckFactory( diff --git a/source/common/router/retry_state_impl.cc b/source/common/router/retry_state_impl.cc index 8badaa382cfa..c09f485a1c7a 100644 --- a/source/common/router/retry_state_impl.cc +++ b/source/common/router/retry_state_impl.cc @@ -79,10 +79,11 @@ RetryStateImpl::RetryStateImpl(const RetryPolicy& route_policy, Http::HeaderMap& // Merge in the headers. if (request_headers.EnvoyRetryOn()) { - retry_on_ |= parseRetryOn(request_headers.EnvoyRetryOn()->value().getStringView()); + retry_on_ |= parseRetryOn(request_headers.EnvoyRetryOn()->value().getStringView()).first; } if (request_headers.EnvoyRetryGrpcOn()) { - retry_on_ |= parseRetryGrpcOn(request_headers.EnvoyRetryGrpcOn()->value().getStringView()); + retry_on_ |= + parseRetryGrpcOn(request_headers.EnvoyRetryGrpcOn()->value().getStringView()).first; } if (retry_on_ != 0 && request_headers.EnvoyMaxRetries()) { uint64_t temp; @@ -113,8 +114,9 @@ void RetryStateImpl::enableBackoffTimer() { retry_timer_->enableTimer(std::chrono::milliseconds(backoff_strategy_->nextBackOffMs())); } -uint32_t RetryStateImpl::parseRetryOn(absl::string_view config) { +std::pair RetryStateImpl::parseRetryOn(absl::string_view config) { uint32_t ret = 0; + bool all_fields_valid = true; for (const auto retry_on : StringUtil::splitToken(config, ",")) { if (retry_on == Http::Headers::get().EnvoyRetryOnValues._5xx) { ret |= RetryPolicy::RETRY_ON_5XX; @@ -128,14 +130,17 @@ uint32_t RetryStateImpl::parseRetryOn(absl::string_view config) { ret |= RetryPolicy::RETRY_ON_REFUSED_STREAM; } else if (retry_on == Http::Headers::get().EnvoyRetryOnValues.RetriableStatusCodes) { ret |= RetryPolicy::RETRY_ON_RETRIABLE_STATUS_CODES; + } else { + all_fields_valid = false; } } - return ret; + return {ret, all_fields_valid}; } -uint32_t RetryStateImpl::parseRetryGrpcOn(absl::string_view retry_grpc_on_header) { +std::pair RetryStateImpl::parseRetryGrpcOn(absl::string_view retry_grpc_on_header) { uint32_t ret = 0; + bool all_fields_valid = true; for (const auto retry_on : StringUtil::splitToken(retry_grpc_on_header, ",")) { if (retry_on == Http::Headers::get().EnvoyRetryOnGrpcValues.Cancelled) { ret |= RetryPolicy::RETRY_ON_GRPC_CANCELLED; @@ -147,10 +152,12 @@ uint32_t RetryStateImpl::parseRetryGrpcOn(absl::string_view retry_grpc_on_header ret |= RetryPolicy::RETRY_ON_GRPC_UNAVAILABLE; } else if (retry_on == Http::Headers::get().EnvoyRetryOnGrpcValues.Internal) { ret |= RetryPolicy::RETRY_ON_GRPC_INTERNAL; + } else { + all_fields_valid = false; } } - return ret; + return {ret, all_fields_valid}; } void RetryStateImpl::resetRetry() { diff --git a/source/common/router/retry_state_impl.h b/source/common/router/retry_state_impl.h index 647680da4f2f..7177a3f2909f 100644 --- a/source/common/router/retry_state_impl.h +++ b/source/common/router/retry_state_impl.h @@ -29,10 +29,23 @@ class RetryStateImpl : public RetryState { Upstream::ResourcePriority priority); ~RetryStateImpl(); - static uint32_t parseRetryOn(absl::string_view config); - - // Returns the RetryPolicy extracted from the x-envoy-retry-grpc-on header. - static uint32_t parseRetryGrpcOn(absl::string_view retry_grpc_on_header); + /** + * Returns the RetryPolicy extracted from the x-envoy-retry-on header. + * @param config is the value of the header. + * @return std::pair the uint32_t is a bitset representing the + * valid retry policies in @param config. The bool is TRUE iff all the + * policies specified in @param config are valid. + */ + static std::pair parseRetryOn(absl::string_view config); + + /** + * Returns the RetryPolicy extracted from the x-envoy-retry-grpc-on header. + * @param config is the value of the header. + * @return std::pair the uint32_t is a bitset representing the + * valid retry policies in @param config. The bool is TRUE iff all the + * policies specified in @param config are valid. + */ + static std::pair parseRetryGrpcOn(absl::string_view retry_grpc_on_header); // Router::RetryState bool enabled() override { return retry_on_ != 0; } diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 7503d1e660a5..ca379197b838 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -222,6 +222,25 @@ Filter::~Filter() { ASSERT(!retry_state_); } +const FilterUtility::StrictHeaderChecker::HeaderCheckResult +FilterUtility::StrictHeaderChecker::checkHeader(Http::HeaderMap& headers, + const Http::LowerCaseString& target_header) { + if (target_header == Http::Headers::get().EnvoyUpstreamRequestTimeoutMs) { + return isInteger(headers.EnvoyUpstreamRequestTimeoutMs()); + } else if (target_header == Http::Headers::get().EnvoyUpstreamRequestPerTryTimeoutMs) { + return isInteger(headers.EnvoyUpstreamRequestPerTryTimeoutMs()); + } else if (target_header == Http::Headers::get().EnvoyMaxRetries) { + return isInteger(headers.EnvoyMaxRetries()); + } else if (target_header == Http::Headers::get().EnvoyRetryOn) { + return hasValidRetryFields(headers.EnvoyRetryOn(), &Router::RetryStateImpl::parseRetryOn); + } else if (target_header == Http::Headers::get().EnvoyRetryGrpcOn) { + return hasValidRetryFields(headers.EnvoyRetryGrpcOn(), + &Router::RetryStateImpl::parseRetryGrpcOn); + } + // Should only validate headers for which we have implemented a validator. + NOT_REACHED_GCOVR_EXCL_LINE +} + Stats::StatName Filter::upstreamZone(Upstream::HostDescriptionConstSharedPtr upstream_host) { return upstream_host ? upstream_host->localityZoneStatName() : config_.empty_stat_name_; } @@ -381,6 +400,24 @@ Http::FilterHeadersStatus Filter::decodeHeaders(Http::HeaderMap& headers, bool e ENVOY_STREAM_LOG(debug, "cluster '{}' match for URL '{}'", *callbacks_, route_entry_->clusterName(), headers.Path()->value().getStringView()); + if (config_.strict_check_headers_ != nullptr) { + for (const auto& header : *config_.strict_check_headers_) { + const auto res = FilterUtility::StrictHeaderChecker::checkHeader(headers, header); + if (!res.valid_) { + callbacks_->streamInfo().setResponseFlag( + StreamInfo::ResponseFlag::InvalidEnvoyRequestHeaders); + const std::string body = fmt::format("invalid header '{}' with value '{}'", + std::string(res.entry_->key().getStringView()), + std::string(res.entry_->value().getStringView())); + const std::string details = + absl::StrCat(StreamInfo::ResponseCodeDetails::get().InvalidEnvoyRequestHeaders, "{", + res.entry_->key().getStringView(), "}"); + callbacks_->sendLocalReply(Http::Code::BadRequest, body, nullptr, absl::nullopt, details); + return Http::FilterHeadersStatus::StopIteration; + } + } + } + const Http::HeaderEntry* request_alt_name = headers.EnvoyUpstreamAltStatName(); if (request_alt_name) { // TODO(#7003): converting this header value into a StatName requires diff --git a/source/common/router/router.h b/source/common/router/router.h index b50c94aeecb3..282d41997b37 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -67,6 +67,51 @@ class FilterUtility { bool hedge_on_per_try_timeout_; }; + class StrictHeaderChecker { + public: + struct HeaderCheckResult { + bool valid_ = true; + const Http::HeaderEntry* entry_; + }; + + /** + * Determine whether a given header's value passes the strict validation + * defined for that header. + * @param headers supplies the headers from which to get the target header. + * @param target_header is the header to be validated. + * @return HeaderCheckResult containing the entry for @param target_header + * and valid_ set to FALSE if @param target_header is set to an + * invalid value. If @param target_header doesn't appear in + * @param headers, return a result with valid_ set to TRUE. + */ + static const HeaderCheckResult checkHeader(Http::HeaderMap& headers, + const Http::LowerCaseString& target_header); + + using ParseRetryFlagsFunc = std::function(absl::string_view)>; + + private: + static HeaderCheckResult hasValidRetryFields(Http::HeaderEntry* header_entry, + const ParseRetryFlagsFunc& parseFn) { + HeaderCheckResult r; + if (header_entry) { + const auto flags_and_validity = parseFn(header_entry->value().getStringView()); + r.valid_ = flags_and_validity.second; + r.entry_ = header_entry; + } + return r; + } + + static HeaderCheckResult isInteger(Http::HeaderEntry* header_entry) { + HeaderCheckResult r; + if (header_entry) { + uint64_t out; + r.valid_ = absl::SimpleAtoi(header_entry->value().getStringView(), &out); + r.entry_ = header_entry; + } + return r; + } + }; + /** * Set the :scheme header based on the properties of the upstream cluster. */ @@ -115,6 +160,7 @@ class FilterConfig { Stats::Scope& scope, Upstream::ClusterManager& cm, Runtime::Loader& runtime, Runtime::RandomGenerator& random, ShadowWriterPtr&& shadow_writer, bool emit_dynamic_stats, bool start_child_span, bool suppress_envoy_headers, + const Protobuf::RepeatedPtrField& strict_check_headers, TimeSource& time_source, Http::Context& http_context) : scope_(scope), local_info_(local_info), cm_(cm), runtime_(runtime), random_(random), stats_{ALL_ROUTER_STATS(POOL_COUNTER_PREFIX(scope, stat_prefix))}, @@ -123,7 +169,14 @@ class FilterConfig { stat_name_pool_(scope_.symbolTable()), retry_(stat_name_pool_.add("retry")), zone_name_(stat_name_pool_.add(local_info_.zoneName())), empty_stat_name_(stat_name_pool_.add("")), shadow_writer_(std::move(shadow_writer)), - time_source_(time_source) {} + time_source_(time_source) { + if (!strict_check_headers.empty()) { + strict_check_headers_ = std::make_unique(); + for (const auto& header : strict_check_headers) { + strict_check_headers_->emplace_back(Http::LowerCaseString(header)); + } + } + } FilterConfig(const std::string& stat_prefix, Server::Configuration::FactoryContext& context, ShadowWriterPtr&& shadow_writer, @@ -132,11 +185,14 @@ class FilterConfig { context.runtime(), context.random(), std::move(shadow_writer), PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, dynamic_stats, true), config.start_child_span(), config.suppress_envoy_headers(), - context.api().timeSource(), context.httpContext()) { + config.strict_check_headers(), context.api().timeSource(), + context.httpContext()) { for (const auto& upstream_log : config.upstream_log()) { upstream_logs_.push_back(AccessLog::AccessLogFactory::fromProto(upstream_log, context)); } } + using HeaderVector = std::vector; + using HeaderVectorPtr = std::unique_ptr; ShadowWriter& shadowWriter() { return *shadow_writer_; } TimeSource& timeSource() { return time_source_; } @@ -150,6 +206,8 @@ class FilterConfig { const bool emit_dynamic_stats_; const bool start_child_span_; const bool suppress_envoy_headers_; + // TODO(xyu-stripe): Make this a bitset to keep cluster memory footprint down. + HeaderVectorPtr strict_check_headers_; std::list upstream_logs_; Http::Context& http_context_; Stats::StatNamePool stat_name_pool_; diff --git a/source/common/stream_info/utility.cc b/source/common/stream_info/utility.cc index 7821c8caab82..339094eacecc 100644 --- a/source/common/stream_info/utility.cc +++ b/source/common/stream_info/utility.cc @@ -23,6 +23,7 @@ const std::string ResponseFlagUtils::RATE_LIMITED = "RL"; const std::string ResponseFlagUtils::UNAUTHORIZED_EXTERNAL_SERVICE = "UAEX"; const std::string ResponseFlagUtils::RATELIMIT_SERVICE_ERROR = "RLSE"; const std::string ResponseFlagUtils::STREAM_IDLE_TIMEOUT = "SI"; +const std::string ResponseFlagUtils::INVALID_ENVOY_REQUEST_HEADERS = "IH"; void ResponseFlagUtils::appendString(std::string& result, const std::string& append) { if (result.empty()) { @@ -35,7 +36,7 @@ void ResponseFlagUtils::appendString(std::string& result, const std::string& app const std::string ResponseFlagUtils::toShortString(const StreamInfo& stream_info) { std::string result; - static_assert(ResponseFlag::LastFlag == 0x10000, "A flag has been added. Fix this code."); + static_assert(ResponseFlag::LastFlag == 0x20000, "A flag has been added. Fix this code."); if (stream_info.hasResponseFlag(ResponseFlag::FailedLocalHealthCheck)) { appendString(result, FAILED_LOCAL_HEALTH_CHECK); @@ -105,6 +106,10 @@ const std::string ResponseFlagUtils::toShortString(const StreamInfo& stream_info appendString(result, STREAM_IDLE_TIMEOUT); } + if (stream_info.hasResponseFlag(ResponseFlag::InvalidEnvoyRequestHeaders)) { + appendString(result, INVALID_ENVOY_REQUEST_HEADERS); + } + return result.empty() ? NONE : result; } @@ -129,6 +134,7 @@ absl::optional ResponseFlagUtils::toResponseFlag(const std::string ResponseFlag::DownstreamConnectionTermination}, {ResponseFlagUtils::UPSTREAM_RETRY_LIMIT_EXCEEDED, ResponseFlag::UpstreamRetryLimitExceeded}, {ResponseFlagUtils::STREAM_IDLE_TIMEOUT, ResponseFlag::StreamIdleTimeout}, + {ResponseFlagUtils::INVALID_ENVOY_REQUEST_HEADERS, ResponseFlag::InvalidEnvoyRequestHeaders}, }; const auto& it = map.find(flag); if (it != map.end()) { diff --git a/source/common/stream_info/utility.h b/source/common/stream_info/utility.h index bf588a7aa72d..c808d7e8aec1 100644 --- a/source/common/stream_info/utility.h +++ b/source/common/stream_info/utility.h @@ -38,6 +38,7 @@ class ResponseFlagUtils { const static std::string UNAUTHORIZED_EXTERNAL_SERVICE; const static std::string RATELIMIT_SERVICE_ERROR; const static std::string STREAM_IDLE_TIMEOUT; + const static std::string INVALID_ENVOY_REQUEST_HEADERS; }; /** diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc index 36c6584d2e09..c9c94146ef9c 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc @@ -108,7 +108,7 @@ void HttpGrpcAccessLog::responseFlagsToAccessLogResponseFlags( envoy::data::accesslog::v2::AccessLogCommon& common_access_log, const StreamInfo::StreamInfo& stream_info) { - static_assert(StreamInfo::ResponseFlag::LastFlag == 0x10000, + static_assert(StreamInfo::ResponseFlag::LastFlag == 0x20000, "A flag has been added. Fix this code."); if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::FailedLocalHealthCheck)) { @@ -180,6 +180,10 @@ void HttpGrpcAccessLog::responseFlagsToAccessLogResponseFlags( if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout)) { common_access_log.mutable_response_flags()->set_stream_idle_timeout(true); } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::InvalidEnvoyRequestHeaders)) { + common_access_log.mutable_response_flags()->set_invalid_envoy_request_headers(true); + } } void HttpGrpcAccessLog::log(const Http::HeaderMap* request_headers, diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index 468b0f9dbe1d..5e34f5c0e33d 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -869,11 +869,12 @@ name: envoy.file_access_log - DC - URX - SI + - IH config: path: /dev/null )EOF"; - static_assert(StreamInfo::ResponseFlag::LastFlag == 0x10000, + static_assert(StreamInfo::ResponseFlag::LastFlag == 0x20000, "A flag has been added. Fix this code."); std::vector all_response_flags = { @@ -894,6 +895,7 @@ name: envoy.file_access_log StreamInfo::ResponseFlag::DownstreamConnectionTermination, StreamInfo::ResponseFlag::UpstreamRetryLimitExceeded, StreamInfo::ResponseFlag::StreamIdleTimeout, + StreamInfo::ResponseFlag::InvalidEnvoyRequestHeaders, }; InstanceSharedPtr log = AccessLogFactory::fromProto(parseAccessLogFromV2Yaml(yaml), context_); @@ -924,7 +926,7 @@ name: envoy.file_access_log "[\"embedded message failed validation\"] | caused by " "ResponseFlagFilterValidationError.Flags[i]: [\"value must be in list \" [\"LH\" \"UH\" " "\"UT\" \"LR\" \"UR\" \"UF\" \"UC\" \"UO\" \"NR\" \"DI\" \"FI\" \"RL\" \"UAEX\" \"RLSE\" " - "\"DC\" \"URX\" \"SI\"]]): " + "\"DC\" \"URX\" \"SI\" \"IH\"]]): " "response_flag_filter {\n flags: \"UnsupportedFlag\"\n}\n"); } @@ -947,7 +949,7 @@ name: envoy.file_access_log "[\"embedded message failed validation\"] | caused by " "ResponseFlagFilterValidationError.Flags[i]: [\"value must be in list \" [\"LH\" \"UH\" " "\"UT\" \"LR\" \"UR\" \"UF\" \"UC\" \"UO\" \"NR\" \"DI\" \"FI\" \"RL\" \"UAEX\" \"RLSE\" " - "\"DC\" \"URX\" \"SI\"]]): " + "\"DC\" \"URX\" \"SI\" \"IH\"]]): " "response_flag_filter {\n flags: \"UnsupportedFlag\"\n}\n"); } diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index d5b6c75701ed..b9e2f9dc8172 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -48,6 +48,7 @@ using testing::Return; using testing::ReturnPointee; using testing::ReturnRef; using testing::SaveArg; +using testing::StartsWith; namespace Envoy { namespace Router { @@ -79,11 +80,12 @@ class TestFilter : public Filter { class RouterTestBase : public testing::Test { public: - RouterTestBase(bool start_child_span, bool suppress_envoy_headers) + RouterTestBase(bool start_child_span, bool suppress_envoy_headers, + Protobuf::RepeatedPtrField strict_headers_to_check) : http_context_(stats_store_.symbolTable()), shadow_writer_(new MockShadowWriter()), config_("test.", local_info_, stats_store_, cm_, runtime_, random_, ShadowWriterPtr{shadow_writer_}, true, start_child_span, suppress_envoy_headers, - test_time_.timeSystem(), http_context_), + std::move(strict_headers_to_check), test_time_.timeSystem(), http_context_), router_(config_) { router_.setDecoderFilterCallbacks(callbacks_); upstream_locality_.set_zone("to_az"); @@ -256,14 +258,15 @@ class RouterTestBase : public testing::Test { class RouterTest : public RouterTestBase { public: - RouterTest() : RouterTestBase(false, false) { + RouterTest() : RouterTestBase(false, false, Protobuf::RepeatedPtrField{}) { EXPECT_CALL(callbacks_, activeSpan()).WillRepeatedly(ReturnRef(span_)); }; }; class RouterTestSuppressEnvoyHeaders : public RouterTestBase { public: - RouterTestSuppressEnvoyHeaders() : RouterTestBase(false, true) {} + RouterTestSuppressEnvoyHeaders() + : RouterTestBase(false, true, Protobuf::RepeatedPtrField{}) {} }; TEST_F(RouterTest, RouteNotFound) { @@ -4099,7 +4102,7 @@ TEST_F(WatermarkTest, RetryRequestNotComplete) { class RouterTestChildSpan : public RouterTestBase { public: - RouterTestChildSpan() : RouterTestBase(true, false) {} + RouterTestChildSpan() : RouterTestBase(true, false, Protobuf::RepeatedPtrField{}) {} }; // Make sure child spans start/inject/finish with a normal flow. @@ -4168,5 +4171,168 @@ TEST_F(RouterTestChildSpan, ResetFlow) { encoder.stream_.resetStream(Http::StreamResetReason::RemoteReset); } +Protobuf::RepeatedPtrField protobufStrList(const std::vector& v) { + Protobuf::RepeatedPtrField res; + for (auto& field : v) { + *res.Add() = field; + } + + return res; +} + +class RouterTestStrictCheckOneHeader : public RouterTestBase, + public testing::WithParamInterface { +public: + RouterTestStrictCheckOneHeader() : RouterTestBase(false, false, protobufStrList({GetParam()})){}; +}; + +INSTANTIATE_TEST_SUITE_P(StrictHeaderCheck, RouterTestStrictCheckOneHeader, + testing::Values("x-envoy-upstream-rq-timeout-ms", + "x-envoy-upstream-rq-per-try-timeout-ms", + "x-envoy-max-retries", "x-envoy-retry-on", + "x-envoy-retry-grpc-on")); + +// Each test param instantiates a router that strict-checks one particular header. +// This test decodes a set of headers with invalid values and asserts that the +// strict header check only fails for the single header specified by the test param +TEST_P(RouterTestStrictCheckOneHeader, SingleInvalidHeader) { + Http::TestHeaderMapImpl req_headers{ + {"X-envoy-Upstream-rq-timeout-ms", "10.0"}, + {"x-envoy-upstream-rq-per-try-timeout-ms", "1.0"}, + {"x-envoy-max-retries", "2.0"}, + {"x-envoy-retry-on", "5xx,cancelled"}, // 'cancelled' is an invalid entry + {"x-envoy-retry-grpc-on", "cancelled, internal"}, // spaces are considered errors + }; + HttpTestUtility::addDefaultHeaders(req_headers); + auto checked_header = GetParam(); + + EXPECT_CALL(callbacks_.stream_info_, + setResponseFlag(StreamInfo::ResponseFlag::InvalidEnvoyRequestHeaders)); + + EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) + .WillOnce(Invoke([&](Http::HeaderMap& response_headers, bool end_stream) -> void { + EXPECT_EQ(enumToInt(Http::Code::BadRequest), + Envoy::Http::Utility::getResponseStatus(response_headers)); + EXPECT_FALSE(end_stream); + })); + + EXPECT_CALL(callbacks_, encodeData(_, _)) + .WillOnce(Invoke([&](Buffer::Instance& data, bool end_stream) -> void { + EXPECT_THAT(data.toString(), + StartsWith(fmt::format("invalid header '{}' with value ", checked_header))); + EXPECT_TRUE(end_stream); + })); + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, router_.decodeHeaders(req_headers, true)); + EXPECT_EQ(callbacks_.details_, + fmt::format("request_headers_failed_strict_check{{{}}}", checked_header)); +} + +class RouterTestStrictCheckSomeHeaders + : public RouterTestBase, + public testing::WithParamInterface> { +public: + RouterTestStrictCheckSomeHeaders() : RouterTestBase(false, false, protobufStrList(GetParam())){}; +}; + +INSTANTIATE_TEST_SUITE_P(StrictHeaderCheck, RouterTestStrictCheckSomeHeaders, + testing::Values(std::vector{"x-envoy-upstream-rq-timeout-ms", + "x-envoy-max-retries"}, + std::vector{})); + +// Request has headers with invalid values, but headers are *excluded* from the +// set to which strict-checks apply. Assert that these headers are not rejected. +TEST_P(RouterTestStrictCheckSomeHeaders, IgnoreOmittedHeaders) { + // Invalid, but excluded from the configured set of headers to strictly-check + Http::TestHeaderMapImpl headers{ + {"x-envoy-upstream-rq-per-try-timeout-ms", "1.0"}, + {"x-envoy-upstream-rq-timeout-ms", "5000"}, + {"x-envoy-retry-on", "5xx,cancelled"}, + }; + HttpTestUtility::addDefaultHeaders(headers); + + expectResponseTimerCreate(); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, router_.decodeHeaders(headers, true)); + router_.onDestroy(); +} + +const std::vector SUPPORTED_STRICT_CHECKED_HEADERS = { + "x-envoy-upstream-rq-timeout-ms", "x-envoy-upstream-rq-per-try-timeout-ms", "x-envoy-retry-on", + "x-envoy-retry-grpc-on", "x-envoy-max-retries"}; + +class RouterTestStrictCheckAllHeaders + : public RouterTestBase, + public testing::WithParamInterface> { +public: + RouterTestStrictCheckAllHeaders() + : RouterTestBase(false, false, protobufStrList(SUPPORTED_STRICT_CHECKED_HEADERS)){}; +}; + +INSTANTIATE_TEST_SUITE_P(StrictHeaderCheck, RouterTestStrictCheckAllHeaders, + testing::Combine(testing::ValuesIn(SUPPORTED_STRICT_CHECKED_HEADERS), + testing::ValuesIn(SUPPORTED_STRICT_CHECKED_HEADERS))); + +// Each instance of this test configures a router to strict-validate all +// supported headers and asserts that a request with invalid values set for some +// *pair* of headers is rejected. +TEST_P(RouterTestStrictCheckAllHeaders, MultipleInvalidHeaders) { + const auto& header1 = std::get<0>(GetParam()); + const auto& header2 = std::get<1>(GetParam()); + Http::TestHeaderMapImpl headers{{header1, "invalid"}, {header2, "invalid"}}; + HttpTestUtility::addDefaultHeaders(headers); + + EXPECT_CALL(callbacks_.stream_info_, + setResponseFlag(StreamInfo::ResponseFlag::InvalidEnvoyRequestHeaders)); + + EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) + .WillOnce(Invoke([&](Http::HeaderMap& response_headers, bool end_stream) -> void { + EXPECT_EQ(enumToInt(Http::Code::BadRequest), + Envoy::Http::Utility::getResponseStatus(response_headers)); + EXPECT_FALSE(end_stream); + })); + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, router_.decodeHeaders(headers, true)); + EXPECT_THAT(callbacks_.details_, + StartsWith(fmt::format("request_headers_failed_strict_check{{"))); + router_.onDestroy(); +} + +// Request has headers with invalid values, but headers are *excluded* from the +// set to which strict-checks apply. Assert that these headers are not rejected. +TEST(RouterFilterUtilityTest, StrictCheckValidHeaders) { + Http::TestHeaderMapImpl headers{ + {"X-envoy-Upstream-rq-timeout-ms", "100"}, + {"x-envoy-upstream-rq-per-try-timeout-ms", "100"}, + {"x-envoy-max-retries", "2"}, + {"not-checked", "always passes"}, + {"x-envoy-retry-on", + "5xx,gateway-error,retriable-4xx,refused-stream,connect-failure,retriable-status-codes"}, + {"x-envoy-retry-grpc-on", + "cancelled,internal,deadline-exceeded,resource-exhausted,unavailable"}, + }; + + for (const auto& target : SUPPORTED_STRICT_CHECKED_HEADERS) { + EXPECT_TRUE( + FilterUtility::StrictHeaderChecker::checkHeader(headers, Http::LowerCaseString(target)) + .valid_) + << fmt::format("'{}' should have passed strict validation", target); + } + + Http::TestHeaderMapImpl failing_headers{ + {"X-envoy-Upstream-rq-timeout-ms", "10.0"}, + {"x-envoy-upstream-rq-per-try-timeout-ms", "1.0"}, + {"x-envoy-max-retries", "2.0"}, + {"x-envoy-retry-on", "5xx,cancelled"}, // 'cancelled' is an invalid entry + {"x-envoy-retry-grpc-on", "cancelled, internal"}, // spaces are considered errors + }; + + for (const auto& target : SUPPORTED_STRICT_CHECKED_HEADERS) { + EXPECT_FALSE(FilterUtility::StrictHeaderChecker::checkHeader(failing_headers, + Http::LowerCaseString(target)) + .valid_) + << fmt::format("'{}' should have failed strict validation", target); + } +} + } // namespace Router } // namespace Envoy diff --git a/test/common/stream_info/utility_test.cc b/test/common/stream_info/utility_test.cc index dead80387417..abcf167e985d 100644 --- a/test/common/stream_info/utility_test.cc +++ b/test/common/stream_info/utility_test.cc @@ -15,7 +15,7 @@ namespace StreamInfo { namespace { TEST(ResponseFlagUtilsTest, toShortStringConversion) { - static_assert(ResponseFlag::LastFlag == 0x10000, "A flag has been added. Fix this code."); + static_assert(ResponseFlag::LastFlag == 0x20000, "A flag has been added. Fix this code."); std::vector> expected = { std::make_pair(ResponseFlag::FailedLocalHealthCheck, "LH"), @@ -35,6 +35,7 @@ TEST(ResponseFlagUtilsTest, toShortStringConversion) { std::make_pair(ResponseFlag::DownstreamConnectionTermination, "DC"), std::make_pair(ResponseFlag::UpstreamRetryLimitExceeded, "URX"), std::make_pair(ResponseFlag::StreamIdleTimeout, "SI"), + std::make_pair(ResponseFlag::InvalidEnvoyRequestHeaders, "IH"), }; for (const auto& test_case : expected) { @@ -63,7 +64,7 @@ TEST(ResponseFlagUtilsTest, toShortStringConversion) { } TEST(ResponseFlagsUtilsTest, toResponseFlagConversion) { - static_assert(ResponseFlag::LastFlag == 0x10000, "A flag has been added. Fix this code."); + static_assert(ResponseFlag::LastFlag == 0x20000, "A flag has been added. Fix this code."); std::vector> expected = { std::make_pair("LH", ResponseFlag::FailedLocalHealthCheck), @@ -83,6 +84,7 @@ TEST(ResponseFlagsUtilsTest, toResponseFlagConversion) { std::make_pair("DC", ResponseFlag::DownstreamConnectionTermination), std::make_pair("URX", ResponseFlag::UpstreamRetryLimitExceeded), std::make_pair("SI", ResponseFlag::StreamIdleTimeout), + std::make_pair("IH", ResponseFlag::InvalidEnvoyRequestHeaders), }; EXPECT_FALSE(ResponseFlagUtils::toResponseFlag("NonExistentFlag").has_value()); diff --git a/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc index abb92c9b0529..960bddb1b96f 100644 --- a/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc @@ -716,6 +716,7 @@ TEST(responseFlagsToAccessLogResponseFlagsTest, All) { common_access_log_expected.mutable_response_flags()->set_downstream_connection_termination(true); common_access_log_expected.mutable_response_flags()->set_upstream_retry_limit_exceeded(true); common_access_log_expected.mutable_response_flags()->set_stream_idle_timeout(true); + common_access_log_expected.mutable_response_flags()->set_invalid_envoy_request_headers(true); EXPECT_EQ(common_access_log_expected.DebugString(), common_access_log.DebugString()); } diff --git a/test/extensions/filters/http/router/config_test.cc b/test/extensions/filters/http/router/config_test.cc index a1d8e5a7dd03..70e0079558a0 100644 --- a/test/extensions/filters/http/router/config_test.cc +++ b/test/extensions/filters/http/router/config_test.cc @@ -47,6 +47,30 @@ TEST(RouterFilterConfigTest, BadRouterFilterConfig) { EXPECT_THROW(factory.createFilterFactory(*json_config, "stats", context), Json::Exception); } +TEST(RouterFilterConfigTest, RouterFilterWithUnsupportedStrictHeaderCheck) { + const std::string yaml = R"EOF( + strict_check_headers: + - unsupportedHeader + )EOF"; + + envoy::config::filter::http::router::v2::Router router_config; + TestUtility::loadFromYaml(yaml, router_config); + + NiceMock context; + RouterFilterConfig factory; + EXPECT_THROW_WITH_MESSAGE( + factory.createFilterFactoryFromProto(router_config, "stats", context), + ProtoValidationException, + "Proto constraint validation failed (RouterValidationError.StrictCheckHeaders[i]: " + "[\"value must be in list \" [" + "\"x-envoy-upstream-rq-timeout-ms\" " + "\"x-envoy-upstream-rq-per-try-timeout-ms\" " + "\"x-envoy-max-retries\" " + "\"x-envoy-retry-grpc-on\" " + "\"x-envoy-retry-on\"" + "]]): strict_check_headers: \"unsupportedHeader\"\n"); +} + TEST(RouterFilterConfigTest, RouterV2Filter) { envoy::config::filter::http::router::v2::Router router_config; router_config.mutable_dynamic_stats()->set_value(true); From a255cf0b1c3f7563385132c3b7e792120c7b4ef2 Mon Sep 17 00:00:00 2001 From: curiouserrandy Date: Tue, 2 Jul 2019 13:25:52 -0400 Subject: [PATCH 098/542] Upgrade version of googletest used by Envoy (#7426) Signed-off-by: Randy Smith --- bazel/repository_locations.bzl | 6 +++--- ci/build_setup.sh | 2 +- test/common/buffer/buffer_test.cc | 4 ++-- test/common/buffer/owned_impl_test.cc | 4 ++-- test/common/buffer/watermark_buffer_test.cc | 4 ++-- test/common/network/udp_listener_impl_test.cc | 6 +++--- test/common/stats/symbol_table_impl_test.cc | 4 ++-- test/common/stats/thread_local_store_test.cc | 4 ++-- .../filters/common/ext_authz/ext_authz_grpc_impl_test.cc | 2 +- .../filters/common/ratelimit/ratelimit_impl_test.cc | 2 +- .../quic_listeners/quiche/platform/quic_platform_test.cc | 6 +++--- test/integration/scoped_rds_integration_test.cc | 4 ++-- test/integration/vhds_integration_test.cc | 2 +- 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 6c2b92352019..f9abd071ef78 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -197,11 +197,11 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://files.pythonhosted.org/packages/f9/e7/4f80d582578f8489226370762d2cf6bc9381175d1929eba1754e03f70708/twitter.common.finagle-thrift-0.3.9.tar.gz"], ), com_google_googletest = dict( - sha256 = "a4cb4b0c3ebb191b798594aca674ad47eee255dcb4c26885cf7f49777703484f", - strip_prefix = "googletest-eb9225ce361affe561592e0912320b9db84985d0", + sha256 = "cbd251a40485fddd44cdf641af6df2953d45695853af6d68aeb11c7efcde6771", + strip_prefix = "googletest-d7003576dd133856432e2e07340f45926242cc3a", # TODO(akonradi): Switch this back to a released version later than 1.8.1 once there is # one available. - urls = ["https://github.com/google/googletest/archive/eb9225ce361affe561592e0912320b9db84985d0.tar.gz"], + urls = ["https://github.com/google/googletest/archive/d7003576dd133856432e2e07340f45926242cc3a.tar.gz"], ), com_google_protobuf = dict( sha256 = "c10ef8d8ad5a9e5f850483051b7f9ee2c8bb3ca2e0e16a4cf105bd1321afb2d6", diff --git a/ci/build_setup.sh b/ci/build_setup.sh index 951dd56ed2ff..a8069429be0d 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -85,7 +85,7 @@ if [ "$1" != "-nofetch" ]; then fi # This is the hash on https://github.com/envoyproxy/envoy-filter-example.git we pin to. - (cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" && git fetch origin && git checkout -f 6c0625cb4cc9a21df97cef2a1d065463f2ae81ae) + (cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" && git fetch origin && git checkout -f dcd3374baa9365ab7ab505018232994d6c8a8d81) cp -f "${ENVOY_SRCDIR}"/ci/WORKSPACE.filter.example "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/WORKSPACE fi diff --git a/test/common/buffer/buffer_test.cc b/test/common/buffer/buffer_test.cc index 72efc6191b59..aaa1c34ba02b 100644 --- a/test/common/buffer/buffer_test.cc +++ b/test/common/buffer/buffer_test.cc @@ -269,8 +269,8 @@ TEST(SliceDequeTest, CreateDelete) { class BufferHelperTest : public BufferImplementationParamTest {}; -INSTANTIATE_TEST_CASE_P(BufferHelperTest, BufferHelperTest, - testing::ValuesIn({BufferImplementation::Old, BufferImplementation::New})); +INSTANTIATE_TEST_SUITE_P(BufferHelperTest, BufferHelperTest, + testing::ValuesIn({BufferImplementation::Old, BufferImplementation::New})); TEST_P(BufferHelperTest, PeekI8) { { diff --git a/test/common/buffer/owned_impl_test.cc b/test/common/buffer/owned_impl_test.cc index b36ad169c3ca..d009db570f8d 100644 --- a/test/common/buffer/owned_impl_test.cc +++ b/test/common/buffer/owned_impl_test.cc @@ -36,8 +36,8 @@ class OwnedImplTest : public BufferImplementationParamTest { } }; -INSTANTIATE_TEST_CASE_P(OwnedImplTest, OwnedImplTest, - testing::ValuesIn({BufferImplementation::Old, BufferImplementation::New})); +INSTANTIATE_TEST_SUITE_P(OwnedImplTest, OwnedImplTest, + testing::ValuesIn({BufferImplementation::Old, BufferImplementation::New})); TEST_P(OwnedImplTest, AddBufferFragmentNoCleanup) { char input[] = "hello world"; diff --git a/test/common/buffer/watermark_buffer_test.cc b/test/common/buffer/watermark_buffer_test.cc index 571a72a85958..44ae41e1b317 100644 --- a/test/common/buffer/watermark_buffer_test.cc +++ b/test/common/buffer/watermark_buffer_test.cc @@ -27,8 +27,8 @@ class WatermarkBufferTest : public BufferImplementationParamTest { uint32_t times_high_watermark_called_{0}; }; -INSTANTIATE_TEST_CASE_P(WatermarkBufferTest, WatermarkBufferTest, - testing::ValuesIn({BufferImplementation::Old, BufferImplementation::New})); +INSTANTIATE_TEST_SUITE_P(WatermarkBufferTest, WatermarkBufferTest, + testing::ValuesIn({BufferImplementation::Old, BufferImplementation::New})); TEST_P(WatermarkBufferTest, TestWatermark) { ASSERT_EQ(10, buffer_.highWatermark()); } diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index 04e9860e09d2..d8f9f33b8a2b 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -88,9 +88,9 @@ class UdpListenerImplTest : public ListenerImplTestBase { std::unique_ptr listener_; }; -INSTANTIATE_TEST_CASE_P(IpVersions, UdpListenerImplTest, - testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), - TestUtility::ipTestParamsToString); +INSTANTIATE_TEST_SUITE_P(IpVersions, UdpListenerImplTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); // Test that socket options are set after the listener is setup. TEST_P(UdpListenerImplTest, UdpSetListeningSocketOptionsSuccess) { diff --git a/test/common/stats/symbol_table_impl_test.cc b/test/common/stats/symbol_table_impl_test.cc index 374ab5b80195..bfd856eabf61 100644 --- a/test/common/stats/symbol_table_impl_test.cc +++ b/test/common/stats/symbol_table_impl_test.cc @@ -74,8 +74,8 @@ class StatNameTest : public testing::TestWithParam { std::unique_ptr pool_; }; -INSTANTIATE_TEST_CASE_P(StatNameTest, StatNameTest, - testing::ValuesIn({SymbolTableType::Real, SymbolTableType::Fake})); +INSTANTIATE_TEST_SUITE_P(StatNameTest, StatNameTest, + testing::ValuesIn({SymbolTableType::Real, SymbolTableType::Fake})); TEST_P(StatNameTest, AllocFree) { encodeDecode("hello.world"); } diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 6b23b106c6aa..67a40c6e808b 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -763,8 +763,8 @@ class RememberStatsMatcherTest : public testing::TestWithParam { ScopePtr scope_; }; -INSTANTIATE_TEST_CASE_P(RememberStatsMatcherTest, RememberStatsMatcherTest, - testing::ValuesIn({false, true})); +INSTANTIATE_TEST_SUITE_P(RememberStatsMatcherTest, RememberStatsMatcherTest, + testing::ValuesIn({false, true})); // Tests that the logic for remembering rejected stats works properly, both // with and without threading. diff --git a/test/extensions/filters/common/ext_authz/ext_authz_grpc_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_grpc_impl_test.cc index c110a7d40627..c0a274f134ff 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_grpc_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_grpc_impl_test.cc @@ -47,7 +47,7 @@ class ExtAuthzGrpcClientTest : public testing::TestWithParam { .WillOnce(Invoke( [this]( absl::string_view service_full_name, absl::string_view method_name, - Buffer::InstancePtr&, Grpc::RawAsyncRequestCallbacks&, Tracing::Span&, + Buffer::InstancePtr&&, Grpc::RawAsyncRequestCallbacks&, Tracing::Span&, const absl::optional& timeout) -> Grpc::AsyncRequest* { EXPECT_EQ(use_alpha_ ? V2Alpha : V2, service_full_name); EXPECT_EQ("Check", method_name); diff --git a/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc b/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc index 39fd13de49d1..85c704dfa2de 100644 --- a/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc +++ b/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc @@ -67,7 +67,7 @@ TEST_F(RateLimitGrpcClientTest, Basic) { EXPECT_CALL(*async_client_, sendRaw(_, _, Grpc::ProtoBufferEq(request), Ref(client_), _, _)) .WillOnce( Invoke([this](absl::string_view service_full_name, absl::string_view method_name, - Buffer::InstancePtr&, Grpc::RawAsyncRequestCallbacks&, Tracing::Span&, + Buffer::InstancePtr&&, Grpc::RawAsyncRequestCallbacks&, Tracing::Span&, const absl::optional&) -> Grpc::AsyncRequest* { std::string service_name = "envoy.service.ratelimit.v2.RateLimitService"; EXPECT_EQ(service_name, service_full_name); diff --git a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc index 8e25b2dd1337..529ce889c7ae 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc @@ -729,9 +729,9 @@ class QuicMemSliceTest : public Envoy::Buffer::BufferImplementationParamTest { ~QuicMemSliceTest() override {} }; -INSTANTIATE_TEST_CASE_P(QuicMemSliceTests, QuicMemSliceTest, - testing::ValuesIn({Envoy::Buffer::BufferImplementation::Old, - Envoy::Buffer::BufferImplementation::New})); +INSTANTIATE_TEST_SUITE_P(QuicMemSliceTests, QuicMemSliceTest, + testing::ValuesIn({Envoy::Buffer::BufferImplementation::Old, + Envoy::Buffer::BufferImplementation::New})); TEST_P(QuicMemSliceTest, ConstructMemSliceFromBuffer) { std::string str(512, 'b'); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 7dc43494c232..32e916da6d9a 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -145,8 +145,8 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, FakeUpstreamInfo rds_upstream_info_; }; -INSTANTIATE_TEST_CASE_P(IpVersionsAndGrpcTypes, ScopedRdsIntegrationTest, - GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsAndGrpcTypes, ScopedRdsIntegrationTest, + GRPC_CLIENT_INTEGRATION_PARAMS); // Test that a SRDS DiscoveryResponse is successfully processed. TEST_P(ScopedRdsIntegrationTest, BasicSuccess) { diff --git a/test/integration/vhds_integration_test.cc b/test/integration/vhds_integration_test.cc index 1bdf251331b8..f31fbb9ef9b4 100644 --- a/test/integration/vhds_integration_test.cc +++ b/test/integration/vhds_integration_test.cc @@ -208,7 +208,7 @@ class VhdsIntegrationTest : public HttpIntegrationTest, bool use_rds_with_vhosts{false}; }; -INSTANTIATE_TEST_CASE_P(IpVersionsClientType, VhdsIntegrationTest, GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, VhdsIntegrationTest, GRPC_CLIENT_INTEGRATION_PARAMS); // tests a scenario when: // - a spontaneous VHDS DiscoveryResponse adds two virtual hosts From fd7b4e8828c5dff839c1d1a9c5623ac9c841ca5e Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Tue, 2 Jul 2019 13:13:55 -0700 Subject: [PATCH 099/542] fix check_format.py to report insufficient owners (#7438) CODEOWNERS lines for extensions with a single owner were ignored resulting in a misleading error about a new directory not having owners. Instead, report insufficient owners as a distinct error. Risk Level: low Testing: manually tested various owners lines Docs Changes: n/a Release Notes: n/a Signed-off-by: Stephan Zuercher --- CODEOWNERS | 2 +- tools/check_format.py | 30 +++++++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 18869803bd5e..e6d99af3b562 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -19,7 +19,7 @@ extensions/filters/common/original_src @snowp @klarose # jwt_authn http filter extension /*/extensions/filters/http/jwt_authn @qiwzhang @lizan # grpc_http1_reverse_bridge http filter extension -/*/extensions/filters/http/grpc_http1_reverse_bridge @snowp +/*/extensions/filters/http/grpc_http1_reverse_bridge @snowp @zuercher # header_to_metadata extension /*/extensions/filters/http/header_to_metadata @rgs1 @zuercher # alts transport socket extension diff --git a/tools/check_format.py b/tools/check_format.py index 57204eea5e06..ac8f4c2fd38c 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -84,7 +84,6 @@ "extensions/filters/http/ratelimit", "extensions/filters/http/buffer", "extensions/filters/http/grpc_http1_bridge", - "extensions/filters/http/grpc_http1_reverse_bridge", "extensions/filters/http/rbac", "extensions/filters/http/gzip", "extensions/filters/http/ip_tagging", @@ -829,35 +828,40 @@ def checkErrorMessages(error_messages): if args.add_excluded_prefixes: EXCLUDED_PREFIXES += tuple(args.add_excluded_prefixes) - # Returns the list of directories with owners listed in CODEOWNERS - def ownedDirectories(): + # Check whether all needed external tools are available. + ct_error_messages = checkTools() + if checkErrorMessages(ct_error_messages): + sys.exit(1) + + # Returns the list of directories with owners listed in CODEOWNERS. May append errors to + # error_messages. + def ownedDirectories(error_messages): owned = [] try: with open('./CODEOWNERS') as f: for line in f: # If this line is of the form "extensions/... @owner1 @owner2" capture the directory # name and store it in the list of directories with documented owners. - m = re.search(r'..*(extensions[^@]* )@.*@.*', line) - if m is not None: + m = re.search(r'.*(extensions[^@]*\s+)(@.*)', line) + if m is not None and not line.startswith('#'): owned.append(m.group(1).strip()) + owners = re.findall('@\S+', m.group(2).strip()) + if len(owners) < 2: + error_messages.append("Extensions require at least 2 owners in CODEOWNERS:\n" + " {}".format(line)) return owned except IOError: return [] # for the check format tests. # Calculate the list of owned directories once per run. - owned_directories = ownedDirectories() - - # Check whether all needed external tools are available. - ct_error_messages = checkTools() - if checkErrorMessages(ct_error_messages): - sys.exit(1) + error_messages = [] + owned_directories = ownedDirectories(error_messages) if os.path.isfile(target_path): - error_messages = checkFormat("./" + target_path) + error_messages += checkFormat("./" + target_path) else: pool = multiprocessing.Pool(processes=args.num_workers) results = [] - error_messages = [] # For each file in target_path, start a new task in the pool and collect the # results (results is passed by reference, and is used as an output). os.path.walk(target_path, checkFormatVisitor, From 1e39e303fe0bcf5695d89841df08d5cb223d3879 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Tue, 2 Jul 2019 13:35:05 -0700 Subject: [PATCH 100/542] [envoy] Remove max limit for endpoint/locality weight (#7437) Signed-off-by: Ruslan Nigmatullin --- api/envoy/api/v2/endpoint/endpoint.proto | 24 +++++----------------- docs/root/intro/version_history.rst | 1 + source/common/upstream/upstream_impl.cc | 2 +- test/common/upstream/upstream_impl_test.cc | 8 +++++--- 4 files changed, 12 insertions(+), 23 deletions(-) diff --git a/api/envoy/api/v2/endpoint/endpoint.proto b/api/envoy/api/v2/endpoint/endpoint.proto index 28136c2b867f..496bb8cdb278 100644 --- a/api/envoy/api/v2/endpoint/endpoint.proto +++ b/api/envoy/api/v2/endpoint/endpoint.proto @@ -74,22 +74,15 @@ message LbEndpoint { // to subset the endpoints considered in cluster load balancing. core.Metadata metadata = 3; - // The optional load balancing weight of the upstream host, in the range 1 - - // 128. Envoy uses the load balancing weight in some of the built in load + // The optional load balancing weight of the upstream host; at least 1. + // Envoy uses the load balancing weight in some of the built in load // balancers. The load balancing weight for an endpoint is divided by the sum // of the weights of all endpoints in the endpoint's locality to produce a // percentage of traffic for the endpoint. This percentage is then further // weighted by the endpoint's locality's load balancing weight from // LocalityLbEndpoints. If unspecified, each host is presumed to have equal // weight in a locality. - // - // .. attention:: - // - // The limit of 128 is somewhat arbitrary, but is applied due to performance - // concerns with the current implementation and can be removed when - // `this issue `_ is fixed. - google.protobuf.UInt32Value load_balancing_weight = 4 - [(validate.rules).uint32 = {gte: 1, lte: 128}]; + google.protobuf.UInt32Value load_balancing_weight = 4 [(validate.rules).uint32 = {gte: 1}]; } // A group of endpoints belonging to a Locality. @@ -103,7 +96,7 @@ message LocalityLbEndpoints { // The group of endpoints belonging to the locality specified. repeated LbEndpoint lb_endpoints = 2 [(gogoproto.nullable) = false]; - // Optional: Per priority/region/zone/sub_zone weight - range 1-128. The load + // Optional: Per priority/region/zone/sub_zone weight; at least 1. The load // balancing weight for a locality is divided by the sum of the weights of all // localities at the same priority level to produce the effective percentage // of traffic for the locality. @@ -113,14 +106,7 @@ message LocalityLbEndpoints { // configured. These weights are ignored otherwise. If no weights are // specified when locality weighted load balancing is enabled, the locality is // assigned no load. - // - // .. attention:: - // - // The limit of 128 is somewhat arbitrary, but is applied due to performance - // concerns with the current implementation and can be removed when - // `this issue `_ is fixed. - google.protobuf.UInt32Value load_balancing_weight = 3 - [(validate.rules).uint32 = {gte: 1, lte: 128}]; + google.protobuf.UInt32Value load_balancing_weight = 3 [(validate.rules).uint32 = {gte: 1}]; // Optional: the priority for this LocalityLbEndpoints. If unspecified this will // default to the highest priority (0). diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b79c9c8abc22..6a8b03c954e9 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -21,6 +21,7 @@ Version history * dns: added support for getting DNS record TTL which is used by STRICT_DNS/LOGICAL_DNS cluster as DNS refresh rate. * dubbo_proxy: support the :ref:`Dubbo proxy filter `. * eds: added support to specify max time for which endpoints can be used :ref:`gRPC filter `. +* eds: removed max limit for `load_balancing_weight`. * event: added :ref:`loop duration and poll delay statistics `. * ext_authz: added a `x-envoy-auth-partial-body` metadata header set to `false|true` indicating if there is a partial body sent in the authorization request message. * ext_authz: added configurable status code that allows customizing HTTP responses on filter check status errors. diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index a94ccd5d2094..03ca1318c195 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -275,7 +275,7 @@ HostImpl::createConnection(Event::Dispatcher& dispatcher, const ClusterInfo& clu return connection; } -void HostImpl::weight(uint32_t new_weight) { weight_ = std::max(1U, std::min(128U, new_weight)); } +void HostImpl::weight(uint32_t new_weight) { weight_ = std::max(1U, new_weight); } std::vector HostsPerLocalityImpl::filter( const std::vector>& predicates) const { diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 9d10956937ab..b0b8094c55b2 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -935,7 +935,9 @@ TEST(HostImplTest, Weight) { EXPECT_EQ(1U, makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", 0)->weight()); EXPECT_EQ(128U, makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", 128)->weight()); - EXPECT_EQ(128U, makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", 129)->weight()); + EXPECT_EQ(std::numeric_limits::max(), + makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", std::numeric_limits::max()) + ->weight()); HostSharedPtr host = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", 50); EXPECT_EQ(50U, host->weight()); @@ -943,8 +945,8 @@ TEST(HostImplTest, Weight) { EXPECT_EQ(51U, host->weight()); host->weight(0); EXPECT_EQ(1U, host->weight()); - host->weight(129); - EXPECT_EQ(128U, host->weight()); + host->weight(std::numeric_limits::max()); + EXPECT_EQ(std::numeric_limits::max(), host->weight()); } TEST(HostImplTest, HostnameCanaryAndLocality) { From 1ca1d13941a14b8fa09ea839d6e936ec32e914b6 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Tue, 2 Jul 2019 23:35:42 +0300 Subject: [PATCH 101/542] tls: make boringssl implementation of TLS optional (#7377) To achieve this instantiate the TLS implementation indirectly via Envoy::Ssl::ContextManagerFactory and introduce a new build option `--define boringssl=disable`. A user of this option is supposed to provide an alternative implementation of TLS and TLS inspector as a third party extension. Also disable the lua http filter when boringssl is disabled since it depends on boringssl's API. Signed-off-by: Dmitry Rozhkov --- bazel/BUILD | 5 ++ bazel/envoy_select.bzl | 3 +- bazel/repositories.bzl | 8 +++ include/envoy/ssl/BUILD | 1 + include/envoy/ssl/context_manager.h | 12 +++++ source/common/common/version.cc | 7 ++- source/exe/BUILD | 3 +- source/extensions/all_extensions.bzl | 1 - source/extensions/extensions_build_config.bzl | 1 + .../transport_sockets/tls/config.cc | 6 +++ .../extensions/transport_sockets/tls/config.h | 7 +++ source/server/BUILD | 13 ++++- source/server/config_validation/server.cc | 4 +- source/server/config_validation/server.h | 2 +- source/server/server.cc | 4 +- source/server/server.h | 4 +- source/server/ssl_context_manager.cc | 50 +++++++++++++++++++ source/server/ssl_context_manager.h | 13 +++++ test/integration/BUILD | 2 +- test/mocks/ssl/BUILD | 1 + test/mocks/ssl/mocks.cc | 6 +++ test/mocks/ssl/mocks.h | 43 ++++++++++++++++ test/server/BUILD | 12 +++++ test/server/ssl_context_manager_test.cc | 34 +++++++++++++ 24 files changed, 228 insertions(+), 14 deletions(-) create mode 100644 source/server/ssl_context_manager.cc create mode 100644 source/server/ssl_context_manager.h create mode 100644 test/server/ssl_context_manager_test.cc diff --git a/bazel/BUILD b/bazel/BUILD index c3ac13989d2a..c2fa24b88e93 100755 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -158,6 +158,11 @@ config_setting( values = {"define": "boringssl=fips"}, ) +config_setting( + name = "boringssl_disabled", + values = {"define": "boringssl=disabled"}, +) + config_setting( name = "enable_quiche", values = {"define": "quiche=enabled"}, diff --git a/bazel/envoy_select.bzl b/bazel/envoy_select.bzl index ceace0ce3e5f..f2167f29bec4 100644 --- a/bazel/envoy_select.bzl +++ b/bazel/envoy_select.bzl @@ -11,9 +11,10 @@ def envoy_cc_platform_dep(name): "//conditions:default": [name + "_posix"], }) -def envoy_select_boringssl(if_fips, default = None): +def envoy_select_boringssl(if_fips, default = None, if_disabled = None): return select({ "@envoy//bazel:boringssl_fips": if_fips, + "@envoy//bazel:boringssl_disabled": if_disabled or [], "//conditions:default": default or [], }) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index be0825a10619..c957a1b8cd5c 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -12,6 +12,14 @@ load("@envoy_api//bazel:repositories.bzl", "api_dependencies") # dict of {build recipe name: longform extension name,} PPC_SKIP_TARGETS = {"luajit": "envoy.filters.http.lua"} +NOBORINGSSL_SKIP_TARGETS = { + # The lua filter depends on BoringSSL + "lua": "envoy.filters.http.lua", + + # These two extensions are supposed to be replaced with alternative extensions. + "tls": "envoy.transport_sockets.tls", + "tls_inspector": "envoy.filters.listener.tls_inspector", +} # go version for rules_go GO_VERSION = "1.12.5" diff --git a/include/envoy/ssl/BUILD b/include/envoy/ssl/BUILD index d5537579a042..73373af9842c 100644 --- a/include/envoy/ssl/BUILD +++ b/include/envoy/ssl/BUILD @@ -40,6 +40,7 @@ envoy_cc_library( deps = [ ":context_config_interface", ":context_interface", + "//include/envoy/common:time_interface", "//include/envoy/stats:stats_interface", ], ) diff --git a/include/envoy/ssl/context_manager.h b/include/envoy/ssl/context_manager.h index 928e2bf9a54b..7358b7745b45 100644 --- a/include/envoy/ssl/context_manager.h +++ b/include/envoy/ssl/context_manager.h @@ -2,6 +2,7 @@ #include +#include "envoy/common/time.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" #include "envoy/stats/scope.h" @@ -40,5 +41,16 @@ class ContextManager { virtual void iterateContexts(std::function callback) PURE; }; +using ContextManagerPtr = std::unique_ptr; + +class ContextManagerFactory { +public: + virtual ~ContextManagerFactory() = default; + virtual ContextManagerPtr createContextManager(TimeSource& time_source) PURE; + + // There could be only one factory thus the name is static. + static std::string name() { return "ssl_context_manager"; } +}; + } // namespace Ssl } // namespace Envoy diff --git a/source/common/common/version.cc b/source/common/common/version.cc index ec11ffc489a3..d2bcc84e34c2 100644 --- a/source/common/common/version.cc +++ b/source/common/common/version.cc @@ -24,6 +24,11 @@ std::string VersionInfo::version() { #else "DEBUG", #endif - ENVOY_SSL_VERSION); +#ifdef ENVOY_SSL_VERSION + ENVOY_SSL_VERSION +#else + "no-ssl" +#endif + ); } } // namespace Envoy diff --git a/source/exe/BUILD b/source/exe/BUILD index 67cb796c249a..2a1873e019ca 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -15,7 +15,7 @@ load( "envoy_all_extensions", "envoy_windows_extensions", ) -load("//bazel:repositories.bzl", "PPC_SKIP_TARGETS") +load("//bazel:repositories.bzl", "NOBORINGSSL_SKIP_TARGETS", "PPC_SKIP_TARGETS") envoy_package() @@ -44,6 +44,7 @@ envoy_cc_library( ] + select({ "//bazel:windows_x86_64": envoy_windows_extensions(), "//bazel:linux_ppc": envoy_all_extensions(PPC_SKIP_TARGETS), + "//bazel:boringssl_disabled": envoy_all_extensions(NOBORINGSSL_SKIP_TARGETS), "//conditions:default": envoy_all_extensions(), }), ) diff --git a/source/extensions/all_extensions.bzl b/source/extensions/all_extensions.bzl index 4977c1e853b4..767f08883e6b 100644 --- a/source/extensions/all_extensions.bzl +++ b/source/extensions/all_extensions.bzl @@ -6,7 +6,6 @@ def envoy_all_extensions(blacklist = dict()): # Envoy build. all_extensions = [ "//source/extensions/transport_sockets/raw_buffer:config", - "//source/extensions/transport_sockets/tls:config", ] # These extensions can be removed on a site specific basis. diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 94d75a42835b..8e3970090ba2 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -130,6 +130,7 @@ EXTENSIONS = { "envoy.transport_sockets.alts": "//source/extensions/transport_sockets/alts:config", "envoy.transport_sockets.tap": "//source/extensions/transport_sockets/tap:config", + "envoy.transport_sockets.tls": "//source/extensions/transport_sockets/tls:config", # Retry host predicates "envoy.retry_host_predicates.previous_hosts": "//source/extensions/retry/host/previous_hosts:config", diff --git a/source/extensions/transport_sockets/tls/config.cc b/source/extensions/transport_sockets/tls/config.cc index 86801f2f3b42..15854a3698bb 100644 --- a/source/extensions/transport_sockets/tls/config.cc +++ b/source/extensions/transport_sockets/tls/config.cc @@ -47,6 +47,12 @@ ProtobufTypes::MessagePtr DownstreamSslSocketFactory::createEmptyConfigProto() { REGISTER_FACTORY(DownstreamSslSocketFactory, Server::Configuration::DownstreamTransportSocketConfigFactory); +Ssl::ContextManagerPtr SslContextManagerFactory::createContextManager(TimeSource& time_source) { + return std::make_unique(time_source); +} + +REGISTER_FACTORY(SslContextManagerFactory, Ssl::ContextManagerFactory); + } // namespace Tls } // namespace TransportSockets } // namespace Extensions diff --git a/source/extensions/transport_sockets/tls/config.h b/source/extensions/transport_sockets/tls/config.h index 2455f7448f93..a10d597e09db 100644 --- a/source/extensions/transport_sockets/tls/config.h +++ b/source/extensions/transport_sockets/tls/config.h @@ -44,6 +44,13 @@ class DownstreamSslSocketFactory DECLARE_FACTORY(DownstreamSslSocketFactory); +class SslContextManagerFactory : public Ssl::ContextManagerFactory { +public: + Ssl::ContextManagerPtr createContextManager(TimeSource& time_source) override; +}; + +DECLARE_FACTORY(SslContextManagerFactory); + } // namespace Tls } // namespace TransportSockets } // namespace Extensions diff --git a/source/server/BUILD b/source/server/BUILD index d9cd108fd7b0..9eb61a7fca69 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -270,8 +270,6 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", "//source/extensions/filters/listener:well_known_names", "//source/extensions/transport_sockets:well_known_names", - "//source/extensions/transport_sockets/tls:context_config_lib", - "//source/extensions/transport_sockets/tls:context_lib", "@envoy_api//envoy/admin/v2alpha:config_dump_cc", "@envoy_api//envoy/api/v2:lds_cc", ], @@ -340,6 +338,7 @@ envoy_cc_library( ":guarddog_lib", ":listener_hooks_lib", ":listener_manager_lib", + ":ssl_context_manager_lib", ":worker_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/event:signal_interface", @@ -384,6 +383,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "ssl_context_manager_lib", + srcs = ["ssl_context_manager.cc"], + hdrs = ["ssl_context_manager.h"], + deps = [ + "//include/envoy/registry", + "//include/envoy/ssl:context_manager_interface", + ], +) + envoy_cc_library( name = "listener_hooks_lib", hdrs = ["listener_hooks.h"], diff --git a/source/server/config_validation/server.cc b/source/server/config_validation/server.cc index 6e66d550227d..3b225808daff 100644 --- a/source/server/config_validation/server.cc +++ b/source/server/config_validation/server.cc @@ -13,6 +13,8 @@ #include "common/protobuf/utility.h" #include "common/singleton/manager_impl.h" +#include "server/ssl_context_manager.h" + namespace Envoy { namespace Server { @@ -92,7 +94,7 @@ void ValidationInstance::initialize(const Options& options, runtime_loader_ = component_factory.createRuntime(*this, initial_config); secret_manager_ = std::make_unique(); ssl_context_manager_ = - std::make_unique(api_->timeSource()); + createContextManager(Ssl::ContextManagerFactory::name(), api_->timeSource()); cluster_manager_factory_ = std::make_unique( admin(), runtime(), stats(), threadLocal(), random(), dnsResolver(), sslContextManager(), dispatcher(), localInfo(), *secret_manager_, messageValidationVisitor(), *api_, http_context_, diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index bc772e940574..2a710cd23675 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -181,7 +181,7 @@ class ValidationInstance : Logger::Loggable, Singleton::ManagerPtr singleton_manager_; Runtime::LoaderPtr runtime_loader_; Runtime::RandomGeneratorImpl random_generator_; - std::unique_ptr ssl_context_manager_; + std::unique_ptr ssl_context_manager_; Configuration::MainImpl config_; LocalInfo::LocalInfoPtr local_info_; AccessLog::AccessLogManagerImpl access_log_manager_; diff --git a/source/server/server.cc b/source/server/server.cc index dfd44d8f0f8c..74fcc9a8031f 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -43,6 +43,7 @@ #include "server/connection_handler_impl.h" #include "server/guarddog_impl.h" #include "server/listener_hooks.h" +#include "server/ssl_context_manager.h" namespace Envoy { namespace Server { @@ -354,8 +355,7 @@ void InstanceImpl::initialize(const Options& options, hooks.onRuntimeCreated(); // Once we have runtime we can initialize the SSL context manager. - ssl_context_manager_ = - std::make_unique(time_source_); + ssl_context_manager_ = createContextManager(Ssl::ContextManagerFactory::name(), time_source_); cluster_manager_factory_ = std::make_unique( *admin_, Runtime::LoaderSingleton::get(), stats_store_, thread_local_, *random_generator_, diff --git a/source/server/server.h b/source/server/server.h index 0c1614d42f22..21965812db25 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -38,8 +38,6 @@ #include "server/overload_manager_impl.h" #include "server/worker_impl.h" -#include "extensions/transport_sockets/tls/context_manager_impl.h" - #include "absl/container/node_hash_map.h" #include "absl/types/optional.h" @@ -250,7 +248,7 @@ class InstanceImpl : Logger::Loggable, Network::ConnectionHandlerPtr handler_; Runtime::RandomGeneratorPtr random_generator_; std::unique_ptr runtime_singleton_; - std::unique_ptr ssl_context_manager_; + std::unique_ptr ssl_context_manager_; ProdListenerComponentFactory listener_component_factory_; ProdWorkerFactory worker_factory_; std::unique_ptr listener_manager_; diff --git a/source/server/ssl_context_manager.cc b/source/server/ssl_context_manager.cc new file mode 100644 index 000000000000..3e422643ed43 --- /dev/null +++ b/source/server/ssl_context_manager.cc @@ -0,0 +1,50 @@ +#include "server/ssl_context_manager.h" + +#include "envoy/common/exception.h" +#include "envoy/registry/registry.h" + +namespace Envoy { +namespace Server { + +/** + * A stub that provides a SSL context manager capable of reporting on + * certificates' data in case there's no TLS implementation built + * into Envoy. + */ +class SslContextManagerNoTlsStub final : public Envoy::Ssl::ContextManager { + Ssl::ClientContextSharedPtr + createSslClientContext(Stats::Scope& /* scope */, + const Envoy::Ssl::ClientContextConfig& /* config */) override { + throwException(); + } + + Ssl::ServerContextSharedPtr + createSslServerContext(Stats::Scope& /* scope */, + const Envoy::Ssl::ServerContextConfig& /* config */, + const std::vector& /* server_names */) override { + throwException(); + } + + size_t daysUntilFirstCertExpires() const override { return std::numeric_limits::max(); } + + void iterateContexts(std::function /* callback */) override{}; + +private: + [[noreturn]] void throwException() { + throw EnvoyException("SSL is not supported in this configuration"); + } +}; + +Ssl::ContextManagerPtr createContextManager(const std::string& factory_name, + TimeSource& time_source) { + Ssl::ContextManagerFactory* factory = + Registry::FactoryRegistry::getFactory(factory_name); + if (factory != nullptr) { + return factory->createContextManager(time_source); + } + + return std::make_unique(); +} + +} // namespace Server +} // namespace Envoy diff --git a/source/server/ssl_context_manager.h b/source/server/ssl_context_manager.h new file mode 100644 index 000000000000..b337251309d4 --- /dev/null +++ b/source/server/ssl_context_manager.h @@ -0,0 +1,13 @@ +#pragma once + +#include "envoy/common/time.h" +#include "envoy/ssl/context_manager.h" + +namespace Envoy { +namespace Server { + +Ssl::ContextManagerPtr createContextManager(const std::string& factory_name, + TimeSource& time_source); + +} // namespace Server +} // namespace Envoy \ No newline at end of file diff --git a/test/integration/BUILD b/test/integration/BUILD index cc8752ebabc0..ebd2696a39ac 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -408,7 +408,7 @@ envoy_cc_test_library( "//source/extensions/access_loggers/file:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tap:config", - "//source/extensions/transport_sockets/tls:ssl_socket_lib", + "//source/extensions/transport_sockets/tls:config", "//source/server:connection_handler_lib", "//source/server:hot_restart_nop_lib", "//source/server:listener_hooks_lib", diff --git a/test/mocks/ssl/BUILD b/test/mocks/ssl/BUILD index 39330f6db353..28397ff0ed8b 100644 --- a/test/mocks/ssl/BUILD +++ b/test/mocks/ssl/BUILD @@ -13,6 +13,7 @@ envoy_cc_mock( srcs = ["mocks.cc"], hdrs = ["mocks.h"], deps = [ + "//include/envoy/ssl:certificate_validation_context_config_interface", "//include/envoy/ssl:connection_interface", "//include/envoy/ssl:context_config_interface", "//include/envoy/ssl:context_interface", diff --git a/test/mocks/ssl/mocks.cc b/test/mocks/ssl/mocks.cc index f19c75217d63..3318d4ec9804 100644 --- a/test/mocks/ssl/mocks.cc +++ b/test/mocks/ssl/mocks.cc @@ -12,5 +12,11 @@ MockConnectionInfo::~MockConnectionInfo() {} MockClientContext::MockClientContext() {} MockClientContext::~MockClientContext() {} +MockClientContextConfig::MockClientContextConfig() {} +MockClientContextConfig::~MockClientContextConfig() {} + +MockServerContextConfig::MockServerContextConfig() {} +MockServerContextConfig::~MockServerContextConfig() {} + } // namespace Ssl } // namespace Envoy diff --git a/test/mocks/ssl/mocks.h b/test/mocks/ssl/mocks.h index faf709bb6ef0..cd91a237327f 100644 --- a/test/mocks/ssl/mocks.h +++ b/test/mocks/ssl/mocks.h @@ -3,6 +3,7 @@ #include #include +#include "envoy/ssl/certificate_validation_context_config.h" #include "envoy/ssl/connection.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" @@ -65,5 +66,47 @@ class MockClientContext : public ClientContext { MOCK_CONST_METHOD0(getCertChainInformation, std::vector()); }; +class MockClientContextConfig : public ClientContextConfig { +public: + MockClientContextConfig(); + ~MockClientContextConfig(); + + MOCK_CONST_METHOD0(alpnProtocols, const std::string&()); + MOCK_CONST_METHOD0(cipherSuites, const std::string&()); + MOCK_CONST_METHOD0(ecdhCurves, const std::string&()); + MOCK_CONST_METHOD0(tlsCertificates, + std::vector>()); + MOCK_CONST_METHOD0(certificateValidationContext, const CertificateValidationContextConfig*()); + MOCK_CONST_METHOD0(minProtocolVersion, unsigned()); + MOCK_CONST_METHOD0(maxProtocolVersion, unsigned()); + MOCK_CONST_METHOD0(isReady, bool()); + MOCK_METHOD1(setSecretUpdateCallback, void(std::function callback)); + + MOCK_CONST_METHOD0(serverNameIndication, const std::string&()); + MOCK_CONST_METHOD0(allowRenegotiation, bool()); + MOCK_CONST_METHOD0(maxSessionKeys, size_t()); + MOCK_CONST_METHOD0(signingAlgorithmsForTest, const std::string&()); +}; + +class MockServerContextConfig : public ServerContextConfig { +public: + MockServerContextConfig(); + ~MockServerContextConfig(); + + MOCK_CONST_METHOD0(alpnProtocols, const std::string&()); + MOCK_CONST_METHOD0(cipherSuites, const std::string&()); + MOCK_CONST_METHOD0(ecdhCurves, const std::string&()); + MOCK_CONST_METHOD0(tlsCertificates, + std::vector>()); + MOCK_CONST_METHOD0(certificateValidationContext, const CertificateValidationContextConfig*()); + MOCK_CONST_METHOD0(minProtocolVersion, unsigned()); + MOCK_CONST_METHOD0(maxProtocolVersion, unsigned()); + MOCK_CONST_METHOD0(isReady, bool()); + MOCK_METHOD1(setSecretUpdateCallback, void(std::function callback)); + + MOCK_CONST_METHOD0(requireClientCertificate, bool()); + MOCK_CONST_METHOD0(sessionTicketKeys, const std::vector&()); +}; + } // namespace Ssl } // namespace Envoy diff --git a/test/server/BUILD b/test/server/BUILD index 233a1d794f91..b39f9cfcbd43 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -283,6 +283,18 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "ssl_context_manager_test", + srcs = ["ssl_context_manager_test.cc"], + deps = [ + "//source/server:ssl_context_manager_lib", + "//test/mocks/ssl:ssl_mocks", + "//test/mocks/stats:stats_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test_library( name = "utility_lib", hdrs = ["utility.h"], diff --git a/test/server/ssl_context_manager_test.cc b/test/server/ssl_context_manager_test.cc new file mode 100644 index 000000000000..31e57bb9732d --- /dev/null +++ b/test/server/ssl_context_manager_test.cc @@ -0,0 +1,34 @@ +#include "server/ssl_context_manager.h" + +#include "test/mocks/ssl/mocks.h" +#include "test/mocks/stats/mocks.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Server { +namespace { + +TEST(SslContextManager, createStub) { + Event::SimulatedTimeSystem time_system; + Stats::MockStore scope; + Ssl::MockClientContextConfig client_config; + Ssl::MockServerContextConfig server_config; + std::vector server_names; + + Ssl::ContextManagerPtr manager = createContextManager("fake_factory_name", time_system); + + // Check we've created a stub, not real manager. + EXPECT_EQ(manager->daysUntilFirstCertExpires(), std::numeric_limits::max()); + EXPECT_THROW_WITH_MESSAGE(manager->createSslClientContext(scope, client_config), EnvoyException, + "SSL is not supported in this configuration"); + EXPECT_THROW_WITH_MESSAGE(manager->createSslServerContext(scope, server_config, server_names), + EnvoyException, "SSL is not supported in this configuration"); + EXPECT_NO_THROW(manager->iterateContexts([](const Envoy::Ssl::Context&) -> void {})); +} + +} // namespace +} // namespace Server +} // namespace Envoy From b3abbb0f0014f3fc38e4c12477904fabbff0a541 Mon Sep 17 00:00:00 2001 From: Michael Rebello Date: Wed, 3 Jul 2019 09:07:58 -0700 Subject: [PATCH 102/542] libevent: update to include patch on master (#7453) Pulls in the change added by https://github.com/libevent/libevent/pull/849 to fix https://github.com/lyft/envoy-mobile/issues/215. Signed-off-by: Michael Rebello --- bazel/repository_locations.bzl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index f9abd071ef78..7bbc4589f5aa 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -127,15 +127,16 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/google/benchmark/archive/v1.5.0.tar.gz"], ), com_github_libevent_libevent = dict( - sha256 = "6f799dd920aab9487cb04cd40627a5d4104fbbd246ebb5c8fd5e520055af2ef5", + sha256 = "549d34065eb2485dfad6c8de638caaa6616ed130eec36dd978f73b6bdd5af113", # This SHA includes the new "prepare" and "check" watchers, used for event loop performance # stats (see https://github.com/libevent/libevent/pull/793) and the fix for a race condition # in the watchers (see https://github.com/libevent/libevent/pull/802). - # This also includes the fix for https://github.com/libevent/libevent/issues/806 + # This also includes the fixes for https://github.com/libevent/libevent/issues/806 + # and https://github.com/lyft/envoy-mobile/issues/215. # TODO(mergeconflict): Update to v2.2 when it is released. - strip_prefix = "libevent-3b1864b625ec37c3051512845982f347f4cc5621", - # 2019-05-16 - urls = ["https://github.com/libevent/libevent/archive/3b1864b625ec37c3051512845982f347f4cc5621.tar.gz"], + strip_prefix = "libevent-0d7d85c2083f7a4c9efe01c061486f332b576d28", + # 2019-07-02 + urls = ["https://github.com/libevent/libevent/archive/0d7d85c2083f7a4c9efe01c061486f332b576d28.tar.gz"], ), net_zlib = dict( # TODO(moderation): revert to com_github_madler_zlib name pending resolution of workaround From eac402e10cf282429d1a37ca8e3a2ab3e749d8c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Wed, 3 Jul 2019 12:44:58 -0400 Subject: [PATCH 103/542] Small fixes on utility.cc (#7455) * more constness * use refs when iterating over a std::vector Signed-off-by: Raul Gutierrez Segales --- source/common/network/utility.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 0290ff8fd169..b6b90c5a8acc 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -143,7 +143,7 @@ Address::InstanceConstSharedPtr Utility::parseInternetAddressAndPort(const std:: } if (ip_address[0] == '[') { // Appears to be an IPv6 address. Find the "]:" that separates the address from the port. - auto pos = ip_address.rfind("]:"); + const auto pos = ip_address.rfind("]:"); if (pos == std::string::npos) { throwWithMalformedIp(ip_address); } @@ -163,7 +163,7 @@ Address::InstanceConstSharedPtr Utility::parseInternetAddressAndPort(const std:: return std::make_shared(sa6, v6only); } // Treat it as an IPv4 address followed by a port. - auto pos = ip_address.rfind(':'); + const auto pos = ip_address.rfind(':'); if (pos == std::string::npos) { throwWithMalformedIp(ip_address); } @@ -259,7 +259,7 @@ bool Utility::isLocalConnection(const Network::ConnectionSocket& socket) { }); RELEASE_ASSERT(rc == 0, ""); - auto af_look_up = + const auto af_look_up = (remote_address->ip()->version() == Address::IpVersion::v4) ? AF_INET : AF_INET6; for (struct ifaddrs* ifa = ifaddr; ifa != nullptr; ifa = ifa->ifa_next) { @@ -269,7 +269,7 @@ bool Utility::isLocalConnection(const Network::ConnectionSocket& socket) { if (ifa->ifa_addr->sa_family == af_look_up) { const auto* addr = reinterpret_cast(ifa->ifa_addr); - auto local_address = Address::addressFromSockAddr( + const auto local_address = Address::addressFromSockAddr( *addr, (af_look_up == AF_INET) ? sizeof(sockaddr_in) : sizeof(sockaddr_in6)); if (remote_address == local_address) { @@ -416,7 +416,7 @@ Address::InstanceConstSharedPtr Utility::getOriginalDst(int fd) { void Utility::parsePortRangeList(absl::string_view string, std::list& list) { const auto ranges = StringUtil::splitToken(string, ","); - for (auto s : ranges) { + for (const auto& s : ranges) { const std::string s_string{s}; std::stringstream ss(s_string); uint32_t min = 0; @@ -508,7 +508,7 @@ Address::SocketType Utility::protobufAddressSocketType(const envoy::api::v2::core::Address& proto_address) { switch (proto_address.address_case()) { case envoy::api::v2::core::Address::kSocketAddress: { - auto protocol = proto_address.socket_address().protocol(); + const auto protocol = proto_address.socket_address().protocol(); switch (protocol) { case envoy::api::v2::core::SocketAddress::TCP: return Address::SocketType::Stream; From 80194c95b4e74d938eece40f2cbfcd35caf0391a Mon Sep 17 00:00:00 2001 From: Hallie Lomax Date: Wed, 3 Jul 2019 09:59:39 -0700 Subject: [PATCH 104/542] dynamo_request_parser: adding support for transactions (#7244) Signed-off-by: Hallie Lomax --- .../http/dynamo/dynamo_request_parser.cc | 43 +++++++++++- .../http/dynamo/dynamo_request_parser.h | 8 +++ .../http/dynamo/dynamo_request_parser_test.cc | 66 +++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/http/dynamo/dynamo_request_parser.cc b/source/extensions/filters/http/dynamo/dynamo_request_parser.cc index 3ddf79653ad2..3ec282c78428 100644 --- a/source/extensions/filters/http/dynamo/dynamo_request_parser.cc +++ b/source/extensions/filters/http/dynamo/dynamo_request_parser.cc @@ -38,6 +38,7 @@ const std::vector RequestParser::SUPPORTED_ERROR_TYPES{ // 4xx "AccessDeniedException", "ConditionalCheckFailedException", + "IdempotentParameterMismatchException", "IncompleteSignatureException", "ItemCollectionSizeLimitExceededException", "LimitExceededException", @@ -46,6 +47,8 @@ const std::vector RequestParser::SUPPORTED_ERROR_TYPES{ "ResourceInUseException", "ResourceNotFoundException", "ThrottlingException", + "TransactionCanceledException", + "TransactionInProgressException", "UnrecognizedClientException", "ValidationException"}; @@ -53,6 +56,11 @@ const std::vector RequestParser::SUPPORTED_ERROR_TYPES{ const std::vector RequestParser::BATCH_OPERATIONS{"BatchGetItem", "BatchWriteItem"}; +const std::vector RequestParser::TRANSACT_OPERATIONS{"TransactGetItems", + "TransactWriteItems"}; +const std::vector RequestParser::TRANSACT_ITEM_OPERATIONS{"ConditionCheck", "Delete", + "Get", "Put", "Update"}; + std::string RequestParser::parseOperation(const Http::HeaderMap& headerMap) { std::string operation; @@ -91,10 +99,42 @@ RequestParser::TableDescriptor RequestParser::parseTable(const std::string& oper } return true; }); + } else if (find(TRANSACT_OPERATIONS.begin(), TRANSACT_OPERATIONS.end(), operation) != + TRANSACT_OPERATIONS.end()) { + std::vector transact_items = + json_data.getObjectArray("TransactItems", true); + for (const Json::ObjectSharedPtr& transact_item : transact_items) { + const auto next_table_name = getTableNameFromTransactItem(*transact_item); + if (!next_table_name.has_value()) { + // if an operation is missing a table name, we want to throw the normal set of errors + table.table_name = ""; + table.is_single_table = true; + break; + } + if (table.table_name.empty()) { + table.table_name = next_table_name.value(); + } else if (table.table_name != next_table_name.value()) { + table.table_name = ""; + table.is_single_table = false; + break; + } + } } - return table; } + +absl::optional +RequestParser::getTableNameFromTransactItem(const Json::Object& transact_item) { + for (const std::string& operation : TRANSACT_ITEM_OPERATIONS) { + Json::ObjectSharedPtr item = transact_item.getObject(operation, true); + std::string table_name = item->getString("TableName", ""); + if (!table_name.empty()) { + return absl::make_optional(table_name); + } + } + return absl::nullopt; +} + std::vector RequestParser::parseBatchUnProcessedKeys(const Json::Object& json_data) { std::vector unprocessed_tables; Json::ObjectSharedPtr tables = json_data.getObject("UnprocessedKeys", true); @@ -105,6 +145,7 @@ std::vector RequestParser::parseBatchUnProcessedKeys(const Json::Ob return unprocessed_tables; } + std::string RequestParser::parseErrorType(const Json::Object& json_data) { std::string error_type = json_data.getString("__type", ""); if (error_type.empty()) { diff --git a/source/extensions/filters/http/dynamo/dynamo_request_parser.h b/source/extensions/filters/http/dynamo/dynamo_request_parser.h index 462f5dad9c7a..e913e740e2b5 100644 --- a/source/extensions/filters/http/dynamo/dynamo_request_parser.h +++ b/source/extensions/filters/http/dynamo/dynamo_request_parser.h @@ -58,6 +58,12 @@ class RequestParser { */ static TableDescriptor parseTable(const std::string& operation, const Json::Object& json_data); + /** + * @return string name of table in transaction object, or empty string if none + */ + static absl::optional + getTableNameFromTransactItem(const Json::Object& transact_item); + /** * Parse error details which might be provided for a given response code. * @return empty string if cannot get error details. @@ -95,6 +101,8 @@ class RequestParser { static const Http::LowerCaseString X_AMZ_TARGET; static const std::vector SINGLE_TABLE_OPERATIONS; static const std::vector BATCH_OPERATIONS; + static const std::vector TRANSACT_OPERATIONS; + static const std::vector TRANSACT_ITEM_OPERATIONS; // http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html static const std::vector SUPPORTED_ERROR_TYPES; diff --git a/test/extensions/filters/http/dynamo/dynamo_request_parser_test.cc b/test/extensions/filters/http/dynamo/dynamo_request_parser_test.cc index 8ebaeee2145a..7d5f26d05095 100644 --- a/test/extensions/filters/http/dynamo/dynamo_request_parser_test.cc +++ b/test/extensions/filters/http/dynamo/dynamo_request_parser_test.cc @@ -74,6 +74,72 @@ TEST(DynamoRequestParser, parseTableNameSingleOperation) { } } +TEST(DynamoRequestParser, parseTableNameTransactOperation) { + std::vector supported_transact_operations{"TransactGetItems", "TransactWriteItems"}; + // testing single table operation + { + std::string json_string = R"EOF( + { + "TransactItems": [ + { "Update": { "TableName": "Pets", "Key": { "Name": {"S": "Maxine"} }, "AnimalType": {"S": "Dog"} } }, + { "Put": { "TableName": "Pets", "Key": { "Name": {"S": "Max"} }, "AnimalType": {"S": "Puppy"} } }, + { "Put": { "TableName": "Pets", "Key": { "Name": {"S": "Oscar"} }, "AnimalType": {"S": "Puppy"} } }, + { "Put": { "TableName": "Pets", "Key": { "Name": {"S": "Chloe"} }, "AnimalType": {"S": "Puppy"} } } + ] + } + )EOF"; + Json::ObjectSharedPtr json_data = Json::Factory::loadFromString(json_string); + + for (const std::string& operation : supported_transact_operations) { + RequestParser::TableDescriptor table = RequestParser::parseTable(operation, *json_data); + EXPECT_EQ("Pets", table.table_name); + EXPECT_TRUE(table.is_single_table); + } + } + + // testing multi-table operation + { + std::string json_string = R"EOF( + { + "TransactItems": [ + { "Put": { "TableName": "Pets", "Key": { "AnimalType": {"S": "Dog"}, "Name": {"S": "Fido"} } } }, + { "Delete": { "TableName": "Strays", "Key": { "AnimalType": {"S": "Dog"}, "Name": {"S": "Fido"} } } }, + { "Put": { "TableName": "Pets", "Key": { "AnimalType": {"S": "Cat"}, "Name": {"S": "Max"} } } }, + { "Delete": { "TableName": "Strays", "Key": { "AnimalType": {"S": "Cat"}, "Name": {"S": "Max"} } } } + ] + } + )EOF"; + Json::ObjectSharedPtr json_data = Json::Factory::loadFromString(json_string); + + for (const std::string& operation : supported_transact_operations) { + RequestParser::TableDescriptor table = RequestParser::parseTable(operation, *json_data); + EXPECT_EQ("", table.table_name); + EXPECT_FALSE(table.is_single_table); + } + } + + // testing missing table + { + std::string json_string = R"EOF( + { + "TransactItems": [ + { "Put": { "TableName": "" } }, + { "Delete": { "TableName": "Strays", "Key": { "AnimalType": {"S": "Dog"}, "Name": {"S": "Fido"} } } }, + { "Put": { "TableName": "Pets", "Key": { "AnimalType": {"S": "Cat"}, "Name": {"S": "Max"} } } }, + { "Delete": { "TableName": "Strays", "Key": { "AnimalType": {"S": "Cat"}, "Name": {"S": "Max"} } } } + ] + } + )EOF"; + Json::ObjectSharedPtr json_data = Json::Factory::loadFromString(json_string); + + for (const std::string& operation : supported_transact_operations) { + RequestParser::TableDescriptor table = RequestParser::parseTable(operation, *json_data); + EXPECT_EQ("", table.table_name); + EXPECT_TRUE(table.is_single_table); + } + } +} + TEST(DynamoRequestParser, parseErrorType) { { EXPECT_EQ("ResourceNotFoundException", From 2c756f1b296980a993db18abe11c116fad7e882a Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Wed, 3 Jul 2019 10:00:57 -0700 Subject: [PATCH 105/542] [test] migrate Http2ProtocolOptions test stubs to v2 (#7449) Signed-off-by: Derek Argueta --- test/common/http/BUILD | 1 + test/common/http/utility_test.cc | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 72043800423b..4f5ac9cad9f9 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -333,6 +333,7 @@ envoy_cc_test( "//test/mocks/http:http_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2/core:protocol_cc", ], ) diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index bec525b6f753..daab4c044d18 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -1,6 +1,9 @@ #include #include +#include "envoy/api/v2/core/protocol.pb.h" +#include "envoy/api/v2/core/protocol.pb.validate.h" + #include "common/common/fmt.h" #include "common/config/protocol_json.h" #include "common/http/exception.h" @@ -244,11 +247,9 @@ TEST(HttpUtility, createSslRedirectPath) { namespace { -Http2Settings parseHttp2SettingsFromJson(const std::string& json_string) { +Http2Settings parseHttp2SettingsFromV2Yaml(const std::string& yaml) { envoy::api::v2::core::Http2ProtocolOptions http2_protocol_options; - auto json_object_ptr = Json::Factory::loadFromString(json_string); - Config::ProtocolJson::translateHttp2ProtocolOptions( - *json_object_ptr->getObject("http2_settings", true), http2_protocol_options); + TestUtility::loadFromYamlAndValidate(yaml, http2_protocol_options); return Utility::parseHttp2Settings(http2_protocol_options); } @@ -256,7 +257,7 @@ Http2Settings parseHttp2SettingsFromJson(const std::string& json_string) { TEST(HttpUtility, parseHttp2Settings) { { - auto http2_settings = parseHttp2SettingsFromJson("{}"); + auto http2_settings = parseHttp2SettingsFromV2Yaml("{}"); EXPECT_EQ(Http2Settings::DEFAULT_HPACK_TABLE_SIZE, http2_settings.hpack_table_size_); EXPECT_EQ(Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS, http2_settings.max_concurrent_streams_); @@ -267,18 +268,17 @@ TEST(HttpUtility, parseHttp2Settings) { } { - auto http2_settings = parseHttp2SettingsFromJson(R"raw({ - "http2_settings" : { - "hpack_table_size": 1, - "max_concurrent_streams": 2, - "initial_stream_window_size": 3, - "initial_connection_window_size": 4 - } - })raw"); + const std::string yaml = R"EOF( +hpack_table_size: 1 +max_concurrent_streams: 2 +initial_stream_window_size: 65535 +initial_connection_window_size: 65535 + )EOF"; + auto http2_settings = parseHttp2SettingsFromV2Yaml(yaml); EXPECT_EQ(1U, http2_settings.hpack_table_size_); EXPECT_EQ(2U, http2_settings.max_concurrent_streams_); - EXPECT_EQ(3U, http2_settings.initial_stream_window_size_); - EXPECT_EQ(4U, http2_settings.initial_connection_window_size_); + EXPECT_EQ(65535U, http2_settings.initial_stream_window_size_); + EXPECT_EQ(65535U, http2_settings.initial_connection_window_size_); } } From fa2b8194c47d492e720563c39ed80d4c85ce0230 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Wed, 3 Jul 2019 13:28:03 -0400 Subject: [PATCH 106/542] stats: remove virtual and multiple inheritance from stats, inline HeapStatData into counters and gauges (#7428) * stats: remove virtual and multiple inheritance from stats, inline HeapStatData into counters and gauges Signed-off-by: Joshua Marantz --- include/envoy/stats/histogram.h | 4 +- include/envoy/stats/stats.h | 24 +-- source/common/stats/heap_stat_data.cc | 160 +++++++++---------- source/common/stats/heap_stat_data.h | 26 --- source/common/stats/histogram_impl.h | 53 +++--- source/common/stats/metric_impl.cc | 64 +++++--- source/common/stats/metric_impl.h | 99 ++++++++---- source/common/stats/null_counter.h | 12 +- source/common/stats/null_gauge.h | 12 +- source/common/stats/thread_local_store.cc | 17 +- source/common/stats/thread_local_store.h | 18 ++- test/common/stats/BUILD | 4 +- test/common/stats/heap_allocator_test.cc | 78 +++++++++ test/common/stats/heap_stat_data_test.cc | 62 ------- test/common/stats/metric_impl_test.cc | 1 + test/common/stats/thread_local_store_test.cc | 8 +- test/integration/stats_integration_test.cc | 5 +- test/mocks/stats/mocks.cc | 51 ------ test/mocks/stats/mocks.h | 89 +++++++++-- 19 files changed, 426 insertions(+), 361 deletions(-) create mode 100644 test/common/stats/heap_allocator_test.cc delete mode 100644 test/common/stats/heap_stat_data_test.cc diff --git a/include/envoy/stats/histogram.h b/include/envoy/stats/histogram.h index 6ff5505fc6fa..43fb64044fa0 100644 --- a/include/envoy/stats/histogram.h +++ b/include/envoy/stats/histogram.h @@ -72,7 +72,7 @@ class HistogramStatistics { * class (Sinks, in particular) will need to explicitly differentiate between histograms * representing durations and histograms representing other types of data. */ -class Histogram : public virtual Metric, public RefcountHelper { +class Histogram : public Metric { public: ~Histogram() override = default; @@ -87,7 +87,7 @@ using HistogramSharedPtr = RefcountPtr; /** * A histogram that is stored in main thread and provides summary view of the histogram. */ -class ParentHistogram : public virtual Histogram { +class ParentHistogram : public Histogram { public: ~ParentHistogram() override = default; diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index 116fdd86028f..a61249ba6e50 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -21,7 +21,7 @@ struct Tag; /** * General interface for all stats objects. */ -class Metric { +class Metric : public RefcountInterface { public: virtual ~Metric() = default; /** @@ -45,6 +45,15 @@ class Metric { */ virtual std::vector tags() const PURE; + /** + * See a more detailed description in tagExtractedStatName(), which is the + * preferred API to use when feasible. This API needs to compose the + * std::string on the fly, and return it by value. + * + * @return The stat name with all tag values extracted, as a std::string. + */ + virtual std::string tagExtractedName() const PURE; + /** * Returns the name of the Metric with the portions designated as tags removed * as a string. For example, The stat name "vhost.foo.vcluster.bar.c1" would @@ -52,13 +61,8 @@ class Metric { * value of tag "vcluster". Thus the tagExtractedName is simply * "vhost.vcluster.c1". * - * @return The stat name with all tag values extracted. - */ - virtual std::string tagExtractedName() const PURE; - - /** - * Returns the name of the Metric with the portions designated as tags - * removed as a StatName + * @return the name of the Metric with the portions designated as tags + * removed. */ virtual StatName tagExtractedStatName() const PURE; @@ -110,7 +114,7 @@ class Metric { * global counter as well as periodic counter. Calling latch() returns the periodic counter and * clears it. */ -class Counter : public virtual Metric, public RefcountInterface { +class Counter : public Metric { public: ~Counter() override = default; @@ -126,7 +130,7 @@ using CounterSharedPtr = RefcountPtr; /** * A gauge that can both increment and decrement. */ -class Gauge : public virtual Metric, public RefcountInterface { +class Gauge : public Metric { public: enum class ImportMode { Uninitialized, // Gauge was discovered during hot-restart transfer. diff --git a/source/common/stats/heap_stat_data.cc b/source/common/stats/heap_stat_data.cc index 228d8331de9b..1ea8768e6f1b 100644 --- a/source/common/stats/heap_stat_data.cc +++ b/source/common/stats/heap_stat_data.cc @@ -25,17 +25,6 @@ HeapStatDataAllocator::~HeapStatDataAllocator() { ASSERT(gauges_.empty()); } -HeapStatData* HeapStatData::alloc(StatName stat_name, SymbolTable& symbol_table) { - symbol_table.incRefCount(stat_name); - return new (stat_name.size()) HeapStatData(stat_name); -} - -void HeapStatData::free(SymbolTable& symbol_table) { - ASSERT(ref_count_ == 0); - symbol_table.free(statName()); - delete this; -} - void HeapStatDataAllocator::removeCounterFromSet(Counter* counter) { Thread::LockGuard lock(mutex_); const size_t count = counters_.erase(counter->statName()); @@ -60,61 +49,85 @@ void HeapStatDataAllocator::debugPrint() { } #endif -class CounterImpl : public Counter, public MetricImpl { +// Counter and Gauge both inherit from from RefcountInterface and +// Metric. MetricImpl takes care of most of the Metric API, but we need to cover +// symbolTable() here, which we don't store directly, but get it via the alloc, +// which we need in order to clean up the counter and gauge maps in that class +// when they are destroyed. +// +// We implement the RefcountInterface API, using 16 bits that would otherwise be +// wasted in the alignment padding next to flags_. +template class StatsSharedImpl : public MetricImpl { public: - CounterImpl(HeapStatData& data, HeapStatDataAllocator& alloc, - absl::string_view tag_extracted_name, const std::vector& tags) - : MetricImpl(tag_extracted_name, tags, alloc.symbolTable()), data_(data), alloc_(alloc) {} + StatsSharedImpl(StatName name, HeapStatDataAllocator& alloc, absl::string_view tag_extracted_name, + const std::vector& tags) + : MetricImpl(name, tag_extracted_name, tags, alloc.symbolTable()), alloc_(alloc) {} - ~CounterImpl() override { + ~StatsSharedImpl() override { // MetricImpl must be explicitly cleared() before destruction, otherwise it // will not be able to access the SymbolTable& to free the symbols. An RAII // alternative would be to store the SymbolTable reference in the // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); - alloc_.removeCounterFromSet(this); - data_.free(symbolTable()); - } - - // Stats::Counter - void add(uint64_t amount) override { - data_.value_ += amount; - data_.pending_increment_ += amount; - data_.flags_ |= Flags::Used; + this->clear(symbolTable()); } - void inc() override { add(1); } - uint64_t latch() override { return data_.pending_increment_.exchange(0); } - void reset() override { data_.value_ = 0; } - bool used() const override { return data_.flags_ & Flags::Used; } - uint64_t value() const override { return data_.value_; } + // Metric SymbolTable& symbolTable() override { return alloc_.symbolTable(); } - StatName statName() const override { return data_.statName(); } + bool used() const override { return flags_ & Metric::Flags::Used; } // RefcountInterface - void incRefCount() override { ++data_.ref_count_; } + void incRefCount() override { ++ref_count_; } bool decRefCount() override { - ASSERT(data_.ref_count_ >= 1); - return --data_.ref_count_ == 0; + ASSERT(ref_count_ >= 1); + return --ref_count_ == 0; } - uint32_t use_count() const override { return data_.ref_count_; } + uint32_t use_count() const override { return ref_count_; } -private: - HeapStatData& data_; +protected: HeapStatDataAllocator& alloc_; + + // Holds backing store for both CounterImpl and GaugeImpl. This provides a level + // of indirection needed to enable stats created with the same name from + // different scopes to share the same value. + std::atomic value_{0}; + std::atomic pending_increment_{0}; + std::atomic flags_{0}; + std::atomic ref_count_{0}; +}; + +class CounterImpl : public StatsSharedImpl { +public: + CounterImpl(StatName name, HeapStatDataAllocator& alloc, absl::string_view tag_extracted_name, + const std::vector& tags) + : StatsSharedImpl(name, alloc, tag_extracted_name, tags) {} + ~CounterImpl() override { alloc_.removeCounterFromSet(this); } + + // Stats::Counter + void add(uint64_t amount) override { + // Note that a reader may see a new value but an old pending_increment_ or + // used(). From a system perspective this should be eventually consistent. + value_ += amount; + pending_increment_ += amount; + flags_ |= Flags::Used; + } + void inc() override { add(1); } + uint64_t latch() override { return pending_increment_.exchange(0); } + void reset() override { value_ = 0; } + bool used() const override { return flags_ & Flags::Used; } + uint64_t value() const override { return value_; } }; -class GaugeImpl : public Gauge, public MetricImpl { +class GaugeImpl : public StatsSharedImpl { public: - GaugeImpl(HeapStatData& data, HeapStatDataAllocator& alloc, absl::string_view tag_extracted_name, + GaugeImpl(StatName name, HeapStatDataAllocator& alloc, absl::string_view tag_extracted_name, const std::vector& tags, ImportMode import_mode) - : MetricImpl(tag_extracted_name, tags, alloc.symbolTable()), data_(data), alloc_(alloc) { + : StatsSharedImpl(name, alloc, tag_extracted_name, tags) { switch (import_mode) { case ImportMode::Accumulate: - data_.flags_ |= Flags::LogicAccumulate; + flags_ |= Flags::LogicAccumulate; break; case ImportMode::NeverImport: - data_.flags_ |= Flags::NeverImport; + flags_ |= Flags::NeverImport; break; case ImportMode::Uninitialized: // Note that we don't clear any flag bits for import_mode==Uninitialized, @@ -124,40 +137,30 @@ class GaugeImpl : public Gauge, public MetricImpl { break; } } - - ~GaugeImpl() override { - // MetricImpl must be explicitly cleared() before destruction, otherwise it - // will not be able to access the SymbolTable& to free the symbols. An RAII - // alternative would be to store the SymbolTable reference in the - // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); - alloc_.removeGaugeFromSet(this); - data_.free(symbolTable()); - } + ~GaugeImpl() override { alloc_.removeGaugeFromSet(this); } // Stats::Gauge void add(uint64_t amount) override { - data_.value_ += amount; - data_.flags_ |= Flags::Used; + value_ += amount; + flags_ |= Flags::Used; } void dec() override { sub(1); } void inc() override { add(1); } void set(uint64_t value) override { - data_.value_ = value; - data_.flags_ |= Flags::Used; + value_ = value; + flags_ |= Flags::Used; } void sub(uint64_t amount) override { - ASSERT(data_.value_ >= amount); + ASSERT(value_ >= amount); ASSERT(used() || amount == 0); - data_.value_ -= amount; + value_ -= amount; } - uint64_t value() const override { return data_.value_; } - bool used() const override { return data_.flags_ & Flags::Used; } + uint64_t value() const override { return value_; } ImportMode importMode() const override { - if (data_.flags_ & Flags::NeverImport) { + if (flags_ & Flags::NeverImport) { return ImportMode::NeverImport; - } else if (data_.flags_ & Flags::LogicAccumulate) { + } else if (flags_ & Flags::LogicAccumulate) { return ImportMode::Accumulate; } return ImportMode::Uninitialized; @@ -176,46 +179,31 @@ class GaugeImpl : public Gauge, public MetricImpl { break; case ImportMode::Accumulate: ASSERT(current == ImportMode::Uninitialized); - data_.flags_ |= Flags::LogicAccumulate; + flags_ |= Flags::LogicAccumulate; break; case ImportMode::NeverImport: ASSERT(current == ImportMode::Uninitialized); // A previous revision of Envoy may have transferred a gauge that it // thought was Accumulate. But the new version thinks it's NeverImport, so // we clear the accumulated value. - data_.value_ = 0; - data_.flags_ &= ~Flags::Used; - data_.flags_ |= Flags::NeverImport; + value_ = 0; + flags_ &= ~Flags::Used; + flags_ |= Flags::NeverImport; break; } } - - SymbolTable& symbolTable() override { return alloc_.symbolTable(); } - StatName statName() const override { return data_.statName(); } - - // RefcountInterface - void incRefCount() override { ++data_.ref_count_; } - bool decRefCount() override { - ASSERT(data_.ref_count_ >= 1); - return --data_.ref_count_ == 0; - } - uint32_t use_count() const override { return data_.ref_count_; } - -private: - HeapStatData& data_; - HeapStatDataAllocator& alloc_; }; CounterSharedPtr HeapStatDataAllocator::makeCounter(StatName name, absl::string_view tag_extracted_name, const std::vector& tags) { Thread::LockGuard lock(mutex_); + ASSERT(gauges_.find(name) == gauges_.end()); auto iter = counters_.find(name); if (iter != counters_.end()) { return CounterSharedPtr(*iter); } - auto counter = CounterSharedPtr( - new CounterImpl(*HeapStatData::alloc(name, symbolTable()), *this, tag_extracted_name, tags)); + auto counter = CounterSharedPtr(new CounterImpl(name, *this, tag_extracted_name, tags)); counters_.insert(counter.get()); return counter; } @@ -224,12 +212,12 @@ GaugeSharedPtr HeapStatDataAllocator::makeGauge(StatName name, absl::string_view const std::vector& tags, Gauge::ImportMode import_mode) { Thread::LockGuard lock(mutex_); + ASSERT(counters_.find(name) == counters_.end()); auto iter = gauges_.find(name); if (iter != gauges_.end()) { return GaugeSharedPtr(*iter); } - auto gauge = GaugeSharedPtr(new GaugeImpl(*HeapStatData::alloc(name, symbolTable()), *this, - tag_extracted_name, tags, import_mode)); + auto gauge = GaugeSharedPtr(new GaugeImpl(name, *this, tag_extracted_name, tags, import_mode)); gauges_.insert(gauge.get()); return gauge; } diff --git a/source/common/stats/heap_stat_data.h b/source/common/stats/heap_stat_data.h index 0be761b14095..c15f37a59958 100644 --- a/source/common/stats/heap_stat_data.h +++ b/source/common/stats/heap_stat_data.h @@ -16,37 +16,11 @@ namespace Envoy { namespace Stats { -/** - * Holds backing store for both CounterImpl and GaugeImpl. This provides a level - * of indirection needed to enable stats created with the same name from - * different scopes to share the same value. - */ -struct HeapStatData : public InlineStorage { -private: - explicit HeapStatData(StatName stat_name) { stat_name.copyToStorage(symbol_storage_); } - -public: - static HeapStatData* alloc(StatName stat_name, SymbolTable& symbol_table); - - void free(SymbolTable& symbol_table); - StatName statName() const { return StatName(symbol_storage_); } - - bool operator==(const HeapStatData& rhs) const { return statName() == rhs.statName(); } - uint64_t hash() const { return statName().hash(); } - - std::atomic value_{0}; - std::atomic pending_increment_{0}; - std::atomic flags_{0}; - std::atomic ref_count_{0}; - SymbolTable::Storage symbol_storage_; // This is a 'using' nickname for uint8_t[]. -}; - class HeapStatDataAllocator : public StatDataAllocator { public: HeapStatDataAllocator(SymbolTable& symbol_table) : symbol_table_(symbol_table) {} ~HeapStatDataAllocator() override; - HeapStatData& alloc(StatName name); void removeCounterFromSet(Counter* counter); void removeGaugeFromSet(Gauge* gauge); diff --git a/source/common/stats/histogram_impl.h b/source/common/stats/histogram_impl.h index 5af61def2fcd..926b3496d2b0 100644 --- a/source/common/stats/histogram_impl.h +++ b/source/common/stats/histogram_impl.h @@ -47,40 +47,46 @@ class HistogramStatisticsImpl : public HistogramStatistics, NonCopyable { double sample_sum_; }; +class HistogramImplHelper : public MetricImpl { +public: + HistogramImplHelper(StatName name, const std::string& tag_extracted_name, + const std::vector& tags, SymbolTable& symbol_table) + : MetricImpl(name, tag_extracted_name, tags, symbol_table) {} + HistogramImplHelper(SymbolTable& symbol_table) : MetricImpl(symbol_table) {} + + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + +private: + RefcountHelper refcount_helper_; +}; + /** * Histogram implementation for the heap. */ -class HistogramImpl : public Histogram, public MetricImpl { +class HistogramImpl : public HistogramImplHelper { public: HistogramImpl(StatName name, Store& parent, const std::string& tag_extracted_name, const std::vector& tags) - : MetricImpl(tag_extracted_name, tags, parent.symbolTable()), - name_(name, parent.symbolTable()), parent_(parent) {} + : HistogramImplHelper(name, tag_extracted_name, tags, parent.symbolTable()), parent_(parent) { + } ~HistogramImpl() { - // We must explicitly free the StatName here using the SymbolTable reference - // we access via parent_. A pure RAII alternative would be to use - // StatNameManagedStorage rather than StatNameStorage, which will cost a total - // of 16 bytes per stat, counting the extra SymbolTable& reference here, - // plus the extra SymbolTable& reference in MetricImpl. - name_.free(symbolTable()); - // We must explicitly free the StatName here in order to supply the // SymbolTable reference. An RAII alternative would be to store a // reference to the SymbolTable in MetricImpl, which would cost 8 bytes // per stat. - MetricImpl::clear(); + MetricImpl::clear(symbolTable()); } // Stats::Histogram void recordValue(uint64_t value) override { parent_.deliverHistogramToSinks(*this, value); } bool used() const override { return true; } - StatName statName() const override { return name_.statName(); } SymbolTable& symbolTable() override { return parent_.symbolTable(); } private: - StatNameStorage name_; - // This is used for delivering the histogram data to sinks. Store& parent_; }; @@ -89,18 +95,19 @@ class HistogramImpl : public Histogram, public MetricImpl { * Null histogram implementation. * No-ops on all calls and requires no underlying metric or data. */ -class NullHistogramImpl : public Histogram, NullMetricImpl { +class NullHistogramImpl : public HistogramImplHelper { public: - explicit NullHistogramImpl(SymbolTable& symbol_table) : NullMetricImpl(symbol_table) {} - ~NullHistogramImpl() override { - // MetricImpl must be explicitly cleared() before destruction, otherwise it - // will not be able to access the SymbolTable& to free the symbols. An RAII - // alternative would be to store the SymbolTable reference in the - // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); - } + explicit NullHistogramImpl(SymbolTable& symbol_table) + : HistogramImplHelper(symbol_table), symbol_table_(symbol_table) {} + ~NullHistogramImpl() { MetricImpl::clear(symbol_table_); } + + bool used() const override { return false; } + SymbolTable& symbolTable() override { return symbol_table_; } void recordValue(uint64_t) override {} + +private: + SymbolTable& symbol_table_; }; } // namespace Stats diff --git a/source/common/stats/metric_impl.cc b/source/common/stats/metric_impl.cc index b88607f4abd1..950679e6b7e0 100644 --- a/source/common/stats/metric_impl.cc +++ b/source/common/stats/metric_impl.cc @@ -7,23 +7,24 @@ namespace Envoy { namespace Stats { -MetricImpl::~MetricImpl() { - // The storage must be cleaned by a subclass of MetricImpl in its +MetricHelper::~MetricHelper() { + // The storage must be cleaned by a subclass of MetricHelper in its // destructor, because the symbol-table is owned by the subclass. - // Simply call MetricImpl::clear() in the subclass dtor. + // Simply call MetricHelper::clear() in the subclass dtor. ASSERT(!stat_names_.populated()); } -MetricImpl::MetricImpl(absl::string_view tag_extracted_name, const std::vector& tags, - SymbolTable& symbol_table) { +MetricHelper::MetricHelper(absl::string_view name, absl::string_view tag_extracted_name, + const std::vector& tags, SymbolTable& symbol_table) { // Encode all the names and tags into transient storage so we can count the - // required bytes. 1 is added to account for the tag_extracted_name, and we - // multiply the number of tags by 2 to account for the name and value of each - // tag. - const uint32_t num_names = 1 + 2 * tags.size(); + // required bytes. 2 is added to account for the name and tag_extracted_name, + // and we multiply the number of tags by 2 to account for the name and value + // of each tag. + const uint32_t num_names = 2 + 2 * tags.size(); STACK_ARRAY(names, absl::string_view, num_names); - names[0] = tag_extracted_name; - int index = 0; + names[0] = name; + names[1] = tag_extracted_name; + int index = 1; for (auto& tag : tags) { names[++index] = tag.name_; names[++index] = tag.value_; @@ -31,13 +32,7 @@ MetricImpl::MetricImpl(absl::string_view tag_extracted_name, const std::vector bool { stat_name = s; @@ -46,8 +41,27 @@ StatName MetricImpl::tagExtractedStatName() const { return stat_name; } -void MetricImpl::iterateTagStatNames(const TagStatNameIterFn& fn) const { - enum { TagExtractedName, TagName, TagValue } state = TagExtractedName; +StatName MetricHelper::tagExtractedStatName() const { + // The name is the first element in stat_names_. The second is the + // tag-extracted-name. We don't have random access in that format, + // so we iterate through them, skipping the first element (name), + // and terminating the iteration after capturing the tag-extracted + // name by returning false from the lambda. + StatName tag_extracted_stat_name; + bool skip = true; + stat_names_.iterate([&tag_extracted_stat_name, &skip](StatName s) -> bool { + if (skip) { + skip = false; + return true; + } + tag_extracted_stat_name = s; + return false; // Returning 'false' stops the iteration. + }); + return tag_extracted_stat_name; +} + +void MetricHelper::iterateTagStatNames(const Metric::TagStatNameIterFn& fn) const { + enum { Name, TagExtractedName, TagName, TagValue } state = Name; StatName tag_name; // StatNameList maintains a linear ordered collection of StatNames, and we @@ -56,6 +70,9 @@ void MetricImpl::iterateTagStatNames(const TagStatNameIterFn& fn) const { // as we iterate through the stat_names_. stat_names_.iterate([&state, &tag_name, &fn](StatName stat_name) -> bool { switch (state) { + case Name: + state = TagExtractedName; + break; case TagExtractedName: state = TagName; break; @@ -75,16 +92,15 @@ void MetricImpl::iterateTagStatNames(const TagStatNameIterFn& fn) const { ASSERT(state != TagValue); } -void MetricImpl::iterateTags(const TagIterFn& fn) const { - const SymbolTable& symbol_table = constSymbolTable(); +void MetricHelper::iterateTags(const SymbolTable& symbol_table, const Metric::TagIterFn& fn) const { iterateTagStatNames([&fn, &symbol_table](StatName name, StatName value) -> bool { return fn(Tag{symbol_table.toString(name), symbol_table.toString(value)}); }); } -std::vector MetricImpl::tags() const { +std::vector MetricHelper::tags(const SymbolTable& symbol_table) const { std::vector tags; - iterateTags([&tags](const Tag& tag) -> bool { + iterateTags(symbol_table, [&tags](const Tag& tag) -> bool { tags.emplace_back(tag); return true; }); diff --git a/source/common/stats/metric_impl.h b/source/common/stats/metric_impl.h index b1f5dad9de65..60e121976fad 100644 --- a/source/common/stats/metric_impl.h +++ b/source/common/stats/metric_impl.h @@ -14,27 +14,73 @@ namespace Envoy { namespace Stats { /** - * Implementation of the Metric interface. Virtual inheritance is used because the interfaces that - * will inherit from Metric will have other base classes that will also inherit from Metric. + * Helper class for implementing Metrics. This does not participate in any + * inheritance chains, but can be instantiated by classes that do. It just + * implements the mechanics of representing the name, tag-extracted-name, + * and all tags as a StatNameList. + */ +class MetricHelper { +public: + MetricHelper(absl::string_view name, absl::string_view tag_extracted_name, + const std::vector& tags, SymbolTable& symbol_table); + ~MetricHelper(); + + StatName statName() const; + std::string name(const SymbolTable& symbol_table) const; + std::vector tags(const SymbolTable& symbol_table) const; + StatName tagExtractedStatName() const; + void iterateTagStatNames(const Metric::TagStatNameIterFn& fn) const; + void iterateTags(const SymbolTable& symbol_table, const Metric::TagIterFn& fn) const; + void clear(SymbolTable& symbol_table) { stat_names_.clear(symbol_table); } + +private: + StatNameList stat_names_; +}; + +/** + * Partial implementation of the Metric interface on behalf of Counters, Gauges, + * and Histograms. It leaves symbolTable() unimplemented so that implementations + * of stats managed by an allocator, specifically Counters and Gauges, can keep + * a reference to the allocator instead, and derive the symbolTable() from that. * - * MetricImpl is not meant to be instantiated as-is. For performance reasons we keep name() virtual - * and expect child classes to implement it. + * We templatize on the base class (Counter, Gauge, or Histogram), rather than + * using multiple virtual inheritance, as this avoids the overhead of an extra + * vptr per instance. This is important for stats because there can be many + * stats in systems with large numbers of clusters and hosts, and a few 8-byte + * pointers per-stat here and there can add up to significant amounts of memory. + * + * Note the delegation of the implementation to a helper class, which is neither + * templatized nor virtual. This avoids having the compiler elaborate complete + * copies of the underlying implementation for each base class during template + * expansion. */ -class MetricImpl : public virtual Metric { +template class MetricImpl : public BaseClass { public: - MetricImpl(absl::string_view tag_extracted_name, const std::vector& tags, - SymbolTable& symbol_table); - ~MetricImpl(); + MetricImpl(absl::string_view name, absl::string_view tag_extracted_name, + const std::vector& tags, SymbolTable& symbol_table) + : helper_(name, tag_extracted_name, tags, symbol_table) {} + + // Alternate API to take the name as a StatName, which is needed at most call-sites. + // TODO(jmarantz): refactor impl to either be able to pass string_view at call-sites + // always, or to make it more efficient to populate a StatNameList with a mixture of + // StatName and string_view. + MetricImpl(StatName name, absl::string_view tag_extracted_name, const std::vector& tags, + SymbolTable& symbol_table) + : MetricImpl(symbol_table.toString(name), tag_extracted_name, tags, symbol_table) {} - std::string name() const override { return constSymbolTable().toString(statName()); } - std::string tagExtractedName() const override; - std::vector tags() const override; - StatName tagExtractedStatName() const override; - void iterateTagStatNames(const TagStatNameIterFn& fn) const override; - void iterateTags(const TagIterFn& fn) const override; + explicit MetricImpl(SymbolTable& symbol_table) + : MetricImpl("", "", std::vector(), symbol_table) {} + + std::vector tags() const override { return helper_.tags(constSymbolTable()); } + StatName statName() const override { return helper_.statName(); } + StatName tagExtractedStatName() const override { return helper_.tagExtractedStatName(); } + void iterateTagStatNames(const Metric::TagStatNameIterFn& fn) const override { + helper_.iterateTagStatNames(fn); + } + void iterateTags(const Metric::TagIterFn& fn) const override { + helper_.iterateTags(constSymbolTable(), fn); + } - // Metric implementations must each implement Metric::symbolTable(). However, - // they can inherit the const version of that accessor from MetricImpl. const SymbolTable& constSymbolTable() const override { // Cast our 'this', which is of type `const MetricImpl*` to a non-const // pointer, so we can use it to call the subclass implementation of @@ -45,25 +91,16 @@ class MetricImpl : public virtual Metric { // provide const and non-const variants of a method. return const_cast(this)->symbolTable(); } + std::string name() const override { return constSymbolTable().toString(this->statName()); } + std::string tagExtractedName() const override { + return constSymbolTable().toString(this->tagExtractedStatName()); + } protected: - void clear(); - -private: - StatNameList stat_names_; -}; - -class NullMetricImpl : public MetricImpl { -public: - explicit NullMetricImpl(SymbolTable& symbol_table) - : MetricImpl("", std::vector(), symbol_table), stat_name_storage_("", symbol_table) {} - - SymbolTable& symbolTable() override { return stat_name_storage_.symbolTable(); } - bool used() const override { return false; } - StatName statName() const override { return stat_name_storage_.statName(); } + void clear(SymbolTable& symbol_table) { helper_.clear(symbol_table); } private: - StatNameManagedStorage stat_name_storage_; + MetricHelper helper_; }; } // namespace Stats diff --git a/source/common/stats/null_counter.h b/source/common/stats/null_counter.h index 777e6a34c509..361ff739b954 100644 --- a/source/common/stats/null_counter.h +++ b/source/common/stats/null_counter.h @@ -11,15 +11,16 @@ namespace Stats { * Null counter implementation. * No-ops on all calls and requires no underlying metric or data. */ -class NullCounterImpl : public Counter, NullMetricImpl { +class NullCounterImpl : public MetricImpl { public: - explicit NullCounterImpl(SymbolTable& symbol_table) : NullMetricImpl(symbol_table) {} + explicit NullCounterImpl(SymbolTable& symbol_table) + : MetricImpl(symbol_table), symbol_table_(symbol_table) {} ~NullCounterImpl() override { // MetricImpl must be explicitly cleared() before destruction, otherwise it // will not be able to access the SymbolTable& to free the symbols. An RAII // alternative would be to store the SymbolTable reference in the // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); + MetricImpl::clear(symbolTable()); } void add(uint64_t) override {} @@ -28,6 +29,10 @@ class NullCounterImpl : public Counter, NullMetricImpl { void reset() override {} uint64_t value() const override { return 0; } + // Metric + bool used() const override { return false; } + SymbolTable& symbolTable() override { return symbol_table_; } + // RefcountInterface void incRefCount() override { refcount_helper_.incRefCount(); } bool decRefCount() override { return refcount_helper_.decRefCount(); } @@ -35,6 +40,7 @@ class NullCounterImpl : public Counter, NullMetricImpl { private: RefcountHelper refcount_helper_; + SymbolTable& symbol_table_; }; } // namespace Stats diff --git a/source/common/stats/null_gauge.h b/source/common/stats/null_gauge.h index f8d231dcf19e..c5491caf736f 100644 --- a/source/common/stats/null_gauge.h +++ b/source/common/stats/null_gauge.h @@ -11,15 +11,16 @@ namespace Stats { * Null gauge implementation. * No-ops on all calls and requires no underlying metric or data. */ -class NullGaugeImpl : public Gauge, NullMetricImpl { +class NullGaugeImpl : public MetricImpl { public: - explicit NullGaugeImpl(SymbolTable& symbol_table) : NullMetricImpl(symbol_table) {} + explicit NullGaugeImpl(SymbolTable& symbol_table) + : MetricImpl(symbol_table), symbol_table_(symbol_table) {} ~NullGaugeImpl() override { // MetricImpl must be explicitly cleared() before destruction, otherwise it // will not be able to access the SymbolTable& to free the symbols. An RAII // alternative would be to store the SymbolTable reference in the // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(); + MetricImpl::clear(symbolTable()); } void add(uint64_t) override {} @@ -31,6 +32,10 @@ class NullGaugeImpl : public Gauge, NullMetricImpl { ImportMode importMode() const override { return ImportMode::NeverImport; } void mergeImportMode(ImportMode /* import_mode */) override {} + // Metric + bool used() const override { return false; } + SymbolTable& symbolTable() override { return symbol_table_; } + // RefcountInterface void incRefCount() override { refcount_helper_.incRefCount(); } bool decRefCount() override { return refcount_helper_.decRefCount(); } @@ -38,6 +43,7 @@ class NullGaugeImpl : public Gauge, NullMetricImpl { private: RefcountHelper refcount_helper_; + SymbolTable& symbol_table_; }; } // namespace Stats diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 741a4caf006b..91603ad79083 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -567,19 +567,17 @@ Histogram& ThreadLocalStoreImpl::ScopeImpl::tlsHistogram(StatName name, } ThreadLocalHistogramImpl::ThreadLocalHistogramImpl(StatName name, - absl::string_view tag_extracted_name, + const std::string& tag_extracted_name, const std::vector& tags, SymbolTable& symbol_table) - : MetricImpl(tag_extracted_name, tags, symbol_table), current_active_(0), used_(false), - created_thread_id_(std::this_thread::get_id()), name_(name, symbol_table), - symbol_table_(symbol_table) { + : HistogramImplHelper(name, tag_extracted_name, tags, symbol_table), current_active_(0), + used_(false), created_thread_id_(std::this_thread::get_id()), symbol_table_(symbol_table) { histograms_[0] = hist_alloc(); histograms_[1] = hist_alloc(); } ThreadLocalHistogramImpl::~ThreadLocalHistogramImpl() { - MetricImpl::clear(); - name_.free(symbolTable()); + MetricImpl::clear(symbolTable()); hist_free(histograms_[0]); hist_free(histograms_[1]); } @@ -599,14 +597,13 @@ void ThreadLocalHistogramImpl::merge(histogram_t* target) { ParentHistogramImpl::ParentHistogramImpl(StatName name, Store& parent, TlsScope& tls_scope, absl::string_view tag_extracted_name, const std::vector& tags) - : MetricImpl(tag_extracted_name, tags, parent.symbolTable()), parent_(parent), + : MetricImpl(name, tag_extracted_name, tags, parent.symbolTable()), parent_(parent), tls_scope_(tls_scope), interval_histogram_(hist_alloc()), cumulative_histogram_(hist_alloc()), interval_statistics_(interval_histogram_), cumulative_statistics_(cumulative_histogram_), - merged_(false), name_(name, parent.symbolTable()) {} + merged_(false) {} ParentHistogramImpl::~ParentHistogramImpl() { - MetricImpl::clear(); - name_.free(symbolTable()); + MetricImpl::clear(symbolTable()); hist_free(interval_histogram_); hist_free(cumulative_histogram_); } diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 1e679646e50c..95babe5efafe 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -28,9 +28,9 @@ namespace Stats { * histograms, one to collect the values and other as backup that is used for merge process. The * swap happens during the merge process. */ -class ThreadLocalHistogramImpl : public Histogram, public MetricImpl { +class ThreadLocalHistogramImpl : public HistogramImplHelper { public: - ThreadLocalHistogramImpl(StatName name, absl::string_view tag_extracted_name, + ThreadLocalHistogramImpl(StatName name, const std::string& tag_extracted_name, const std::vector& tags, SymbolTable& symbol_table); ~ThreadLocalHistogramImpl() override; @@ -51,7 +51,6 @@ class ThreadLocalHistogramImpl : public Histogram, public MetricImpl { bool used() const override { return used_; } // Stats::Metric - StatName statName() const override { return name_.statName(); } SymbolTable& symbolTable() override { return symbol_table_; } private: @@ -60,7 +59,6 @@ class ThreadLocalHistogramImpl : public Histogram, public MetricImpl { histogram_t* histograms_[2]; std::atomic used_; std::thread::id created_thread_id_; - StatNameStorage name_; SymbolTable& symbol_table_; }; @@ -71,14 +69,13 @@ class TlsScope; /** * Log Linear Histogram implementation that is stored in the main thread. */ -class ParentHistogramImpl : public ParentHistogram, public MetricImpl { +class ParentHistogramImpl : public MetricImpl { public: ParentHistogramImpl(StatName name, Store& parent, TlsScope& tlsScope, absl::string_view tag_extracted_name, const std::vector& tags); ~ParentHistogramImpl() override; void addTlsHistogram(const TlsHistogramSharedPtr& hist_ptr); - bool used() const override; void recordValue(uint64_t value) override; /** @@ -97,8 +94,13 @@ class ParentHistogramImpl : public ParentHistogram, public MetricImpl { const std::string bucketSummary() const override; // Stats::Metric - StatName statName() const override { return name_.statName(); } SymbolTable& symbolTable() override { return parent_.symbolTable(); } + bool used() const override; + + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } private: bool usedLockHeld() const EXCLUSIVE_LOCKS_REQUIRED(merge_lock_); @@ -112,7 +114,7 @@ class ParentHistogramImpl : public ParentHistogram, public MetricImpl { mutable Thread::MutexBasicLockable merge_lock_; std::list tls_histograms_ GUARDED_BY(merge_lock_); bool merged_; - StatNameStorage name_; + RefcountHelper refcount_helper_; }; using ParentHistogramImplSharedPtr = RefcountPtr; diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 76e4e949a2a1..c6a724d90ca5 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -11,8 +11,8 @@ load( envoy_package() envoy_cc_test( - name = "heap_stat_data_test", - srcs = ["heap_stat_data_test.cc"], + name = "heap_allocator_test", + srcs = ["heap_allocator_test.cc"], deps = [ "//source/common/stats:fake_symbol_table_lib", "//source/common/stats:heap_stat_data_lib", diff --git a/test/common/stats/heap_allocator_test.cc b/test/common/stats/heap_allocator_test.cc new file mode 100644 index 000000000000..770257332d99 --- /dev/null +++ b/test/common/stats/heap_allocator_test.cc @@ -0,0 +1,78 @@ +#include + +#include "common/stats/fake_symbol_table_impl.h" +#include "common/stats/heap_stat_data.h" + +#include "test/test_common/logging.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Stats { +namespace { + +class HeapAllocatorTest : public testing::Test { +protected: + HeapAllocatorTest() : alloc_(symbol_table_), pool_(symbol_table_) {} + ~HeapAllocatorTest() { clearStorage(); } + + StatNameStorage makeStatStorage(absl::string_view name) { + return StatNameStorage(name, symbol_table_); + } + + StatName makeStat(absl::string_view name) { return pool_.add(name); } + + void clearStorage() { + pool_.clear(); + EXPECT_EQ(0, symbol_table_.numSymbols()); + } + + FakeSymbolTableImpl symbol_table_; + HeapStatDataAllocator alloc_; + StatNamePool pool_; +}; + +// Allocate 2 counters of the same name, and you'll get the same object. +TEST_F(HeapAllocatorTest, CountersWithSameName) { + StatName counter_name = makeStat("counter.name"); + CounterSharedPtr c1 = alloc_.makeCounter(counter_name, "", std::vector()); + EXPECT_EQ(1, c1->use_count()); + CounterSharedPtr c2 = alloc_.makeCounter(counter_name, "", std::vector()); + EXPECT_EQ(2, c1->use_count()); + EXPECT_EQ(2, c2->use_count()); + EXPECT_EQ(c1.get(), c2.get()); + EXPECT_FALSE(c1->used()); + EXPECT_FALSE(c2->used()); + c1->inc(); + EXPECT_TRUE(c1->used()); + EXPECT_TRUE(c2->used()); + c2->inc(); + EXPECT_EQ(2, c1->value()); + EXPECT_EQ(2, c2->value()); +} + +TEST_F(HeapAllocatorTest, GaugesWithSameName) { + StatName gauge_name = makeStat("gauges.name"); + GaugeSharedPtr g1 = + alloc_.makeGauge(gauge_name, "", std::vector(), Gauge::ImportMode::Accumulate); + EXPECT_EQ(1, g1->use_count()); + GaugeSharedPtr g2 = + alloc_.makeGauge(gauge_name, "", std::vector(), Gauge::ImportMode::Accumulate); + EXPECT_EQ(2, g1->use_count()); + EXPECT_EQ(2, g2->use_count()); + EXPECT_EQ(g1.get(), g2.get()); + EXPECT_FALSE(g1->used()); + EXPECT_FALSE(g2->used()); + g1->inc(); + EXPECT_TRUE(g1->used()); + EXPECT_TRUE(g2->used()); + EXPECT_EQ(1, g1->value()); + EXPECT_EQ(1, g2->value()); + g2->dec(); + EXPECT_EQ(0, g1->value()); + EXPECT_EQ(0, g2->value()); +} + +} // namespace +} // namespace Stats +} // namespace Envoy diff --git a/test/common/stats/heap_stat_data_test.cc b/test/common/stats/heap_stat_data_test.cc deleted file mode 100644 index 315c6b685c3f..000000000000 --- a/test/common/stats/heap_stat_data_test.cc +++ /dev/null @@ -1,62 +0,0 @@ -#include - -#include "common/stats/fake_symbol_table_impl.h" -#include "common/stats/heap_stat_data.h" - -#include "test/test_common/logging.h" - -#include "gtest/gtest.h" - -namespace Envoy { -namespace Stats { -namespace { - -class HeapStatDataTest : public testing::Test { -protected: - HeapStatDataTest() : alloc_(symbol_table_), pool_(symbol_table_) {} - ~HeapStatDataTest() { clearStorage(); } - - StatNameStorage makeStatStorage(absl::string_view name) { - return StatNameStorage(name, symbol_table_); - } - - StatName makeStat(absl::string_view name) { return pool_.add(name); } - - void clearStorage() { - pool_.clear(); - EXPECT_EQ(0, symbol_table_.numSymbols()); - } - - FakeSymbolTableImpl symbol_table_; - HeapStatDataAllocator alloc_; - StatNamePool pool_; -}; - -// No truncation occurs in the implementation of HeapStatData. -TEST_F(HeapStatDataTest, HeapNoTruncate) { - const std::string long_string(128, 'A'); - StatName stat_name = makeStat(long_string); - HeapStatData* stat{}; - EXPECT_NO_LOGS(stat = HeapStatData::alloc(stat_name, symbol_table_)); - EXPECT_EQ(stat->statName(), stat_name); - stat->free(symbol_table_); -}; - -TEST_F(HeapStatDataTest, HeapAlloc) { - HeapStatData* stat_1 = HeapStatData::alloc(makeStat("ref_name"), symbol_table_); - ASSERT_NE(stat_1, nullptr); - HeapStatData* stat_2 = HeapStatData::alloc(makeStat("ref_name"), symbol_table_); - ASSERT_NE(stat_2, nullptr); - HeapStatData* stat_3 = HeapStatData::alloc(makeStat("not_ref_name"), symbol_table_); - ASSERT_NE(stat_3, nullptr); - EXPECT_NE(stat_1, stat_2); - EXPECT_NE(stat_1, stat_3); - EXPECT_NE(stat_2, stat_3); - stat_1->free(symbol_table_); - stat_2->free(symbol_table_); - stat_3->free(symbol_table_); -} - -} // namespace -} // namespace Stats -} // namespace Envoy diff --git a/test/common/stats/metric_impl_test.cc b/test/common/stats/metric_impl_test.cc index 0500a94c7b54..493444fd1d0e 100644 --- a/test/common/stats/metric_impl_test.cc +++ b/test/common/stats/metric_impl_test.cc @@ -42,6 +42,7 @@ TEST_F(MetricImplTest, OneTag) { ASSERT_EQ(1, tags.size()); EXPECT_EQ("name", tags[0].name_); EXPECT_EQ("value", tags[0].value_); + EXPECT_EQ("counter.name.value", counter->name()); EXPECT_EQ("counter", counter->tagExtractedName()); EXPECT_EQ(makeStat("counter"), counter->tagExtractedStatName()); } diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 67a40c6e808b..edd3ba983591 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -861,8 +861,8 @@ TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTls) { TestUtil::forEachSampleStat( 1000, [&store](absl::string_view name) { store.counter(std::string(name)); }); const size_t million = 1000 * 1000; - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 17432336); // June 29, 2019 - EXPECT_MEMORY_LE(memory_test.consumedBytes(), 18 * million); + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 15268336); // June 30, 2019 + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 16 * million); } TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithTls) { @@ -881,8 +881,8 @@ TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithTls) { TestUtil::forEachSampleStat( 1000, [&store](absl::string_view name) { store.counter(std::string(name)); }); const size_t million = 1000 * 1000; - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 19660848); // June 29, 2019 - EXPECT_MEMORY_LE(memory_test.consumedBytes(), 20 * million); + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 17496848); // June 30, 2019 + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 18 * million); store.shutdownThreading(); tls.shutdownThread(); } diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 3818dc87d306..112475f923cf 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -208,6 +208,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // 2019/06/06 7208 49650 make memory targets approximate // 2019/06/17 7243 49412 49700 macros for exact/upper-bound memory checks // 2019/06/29 7364 45685 46000 combine 2 levels of stat ref-counting into 1 + // 2019/06/30 7428 42742 43000 remove stats multiple inheritance, inline HeapStatData // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -217,8 +218,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 45685); - EXPECT_MEMORY_LE(m_per_cluster, 46000); + EXPECT_MEMORY_EQ(m_per_cluster, 42742); + EXPECT_MEMORY_LE(m_per_cluster, 43000); } } // namespace diff --git a/test/mocks/stats/mocks.cc b/test/mocks/stats/mocks.cc index 33ea887eb1ab..56dfbaf84d40 100644 --- a/test/mocks/stats/mocks.cc +++ b/test/mocks/stats/mocks.cc @@ -17,57 +17,6 @@ using testing::ReturnRef; namespace Envoy { namespace Stats { -MockMetric::MockMetric() : name_(*this), tag_pool_(*symbol_table_) {} -MockMetric::~MockMetric() = default; - -MockMetric::MetricName::~MetricName() { - if (stat_name_storage_ != nullptr) { - stat_name_storage_->free(*mock_metric_.symbol_table_); - } -} - -void MockMetric::setTagExtractedName(absl::string_view name) { - tag_extracted_name_ = std::string(name); - tag_extracted_stat_name_ = - std::make_unique(tagExtractedName(), *symbol_table_); -} - -void MockMetric::setTags(const std::vector& tags) { - tag_pool_.clear(); - tags_ = tags; - for (const Tag& tag : tags) { - tag_names_and_values_.push_back(tag_pool_.add(tag.name_)); - tag_names_and_values_.push_back(tag_pool_.add(tag.value_)); - } -} -void MockMetric::addTag(const Tag& tag) { - tags_.emplace_back(tag); - tag_names_and_values_.push_back(tag_pool_.add(tag.name_)); - tag_names_and_values_.push_back(tag_pool_.add(tag.value_)); -} - -void MockMetric::iterateTags(const TagIterFn& fn) const { - for (const Tag& tag : tags_) { - if (!fn(tag)) { - return; - } - } -} - -void MockMetric::iterateTagStatNames(const TagStatNameIterFn& fn) const { - ASSERT((tag_names_and_values_.size() % 2) == 0); - for (size_t i = 0; i < tag_names_and_values_.size(); i += 2) { - if (!fn(tag_names_and_values_[i], tag_names_and_values_[i + 1])) { - return; - } - } -} - -void MockMetric::MetricName::MetricName::operator=(absl::string_view name) { - name_ = std::string(name); - stat_name_storage_ = std::make_unique(name, mock_metric_.symbolTable()); -} - MockCounter::MockCounter() { ON_CALL(*this, used()).WillByDefault(ReturnPointee(&used_)); ON_CALL(*this, value()).WillByDefault(ReturnPointee(&value_)); diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index dca7e8130482..987e86c43fa7 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -26,10 +26,10 @@ namespace Envoy { namespace Stats { -class MockMetric : public virtual Metric { +template class MockMetric : public BaseClass { public: - MockMetric(); - ~MockMetric() override; + MockMetric() : name_(*this), tag_pool_(*symbol_table_) {} + ~MockMetric() override = default; // This bit of C++ subterfuge allows us to support the wealth of tests that // do metric->name_ = "foo" even though names are more complex now. Note @@ -37,9 +37,16 @@ class MockMetric : public virtual Metric { class MetricName { public: explicit MetricName(MockMetric& mock_metric) : mock_metric_(mock_metric) {} - ~MetricName(); + ~MetricName() { + if (stat_name_storage_ != nullptr) { + stat_name_storage_->free(*mock_metric_.symbol_table_); + } + } - void operator=(absl::string_view str); + void operator=(absl::string_view str) { + name_ = std::string(str); + stat_name_storage_ = std::make_unique(str, mock_metric_.symbolTable()); + } std::string name() const { return name_; } StatName statName() const { return stat_name_storage_->statName(); } @@ -58,19 +65,47 @@ class MockMetric : public virtual Metric { std::string name() const override { return name_.name(); } StatName statName() const override { return name_.statName(); } std::vector tags() const override { return tags_; } - void setTagExtractedName(absl::string_view name); + void setTagExtractedName(absl::string_view name) { + tag_extracted_name_ = std::string(name); + tag_extracted_stat_name_ = + std::make_unique(tagExtractedName(), *symbol_table_); + } std::string tagExtractedName() const override { return tag_extracted_name_.empty() ? name() : tag_extracted_name_; } StatName tagExtractedStatName() const override { return tag_extracted_stat_name_->statName(); } - void iterateTagStatNames(const TagStatNameIterFn& fn) const override; - void iterateTags(const TagIterFn& fn) const override; + void iterateTagStatNames(const Metric::TagStatNameIterFn& fn) const override { + ASSERT((tag_names_and_values_.size() % 2) == 0); + for (size_t i = 0; i < tag_names_and_values_.size(); i += 2) { + if (!fn(tag_names_and_values_[i], tag_names_and_values_[i + 1])) { + return; + } + } + } + void iterateTags(const Metric::TagIterFn& fn) const override { + for (const Tag& tag : tags_) { + if (!fn(tag)) { + return; + } + } + } Test::Global symbol_table_; // Must outlive name_. MetricName name_; - void setTags(const std::vector& tags); - void addTag(const Tag& tag); + void setTags(const std::vector& tags) { + tag_pool_.clear(); + tags_ = tags; + for (const Tag& tag : tags) { + tag_names_and_values_.push_back(tag_pool_.add(tag.name_)); + tag_names_and_values_.push_back(tag_pool_.add(tag.value_)); + } + } + void addTag(const Tag& tag) { + tags_.emplace_back(tag); + tag_names_and_values_.push_back(tag_pool_.add(tag.name_)); + tag_names_and_values_.push_back(tag_pool_.add(tag.value_)); + } private: std::vector tags_; @@ -80,7 +115,17 @@ class MockMetric : public virtual Metric { std::unique_ptr tag_extracted_stat_name_; }; -class MockCounter : public Counter, public MockMetric { +template class MockStatWithRefcount : public MockMetric { +public: + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + + RefcountHelper refcount_helper_; +}; + +class MockCounter : public MockStatWithRefcount { public: MockCounter(); ~MockCounter() override; @@ -105,7 +150,7 @@ class MockCounter : public Counter, public MockMetric { RefcountHelper refcount_helper_; }; -class MockGauge : public Gauge, public MockMetric, public RefcountHelper { +class MockGauge : public MockStatWithRefcount { public: MockGauge(); ~MockGauge() override; @@ -134,7 +179,7 @@ class MockGauge : public Gauge, public MockMetric, public RefcountHelper { RefcountHelper refcount_helper_; }; -class MockHistogram : public Histogram, public MockMetric { +class MockHistogram : public MockMetric { public: MockHistogram(); ~MockHistogram() override; @@ -142,10 +187,18 @@ class MockHistogram : public Histogram, public MockMetric { MOCK_METHOD1(recordValue, void(uint64_t value)); MOCK_CONST_METHOD0(used, bool()); + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + Store* store_; + +private: + RefcountHelper refcount_helper_; }; -class MockParentHistogram : public ParentHistogram, public MockMetric { +class MockParentHistogram : public MockMetric { public: MockParentHistogram(); ~MockParentHistogram() override; @@ -159,10 +212,18 @@ class MockParentHistogram : public ParentHistogram, public MockMetric { MOCK_CONST_METHOD0(cumulativeStatistics, const HistogramStatistics&()); MOCK_CONST_METHOD0(intervalStatistics, const HistogramStatistics&()); + // RefcountInterface + void incRefCount() override { refcount_helper_.incRefCount(); } + bool decRefCount() override { return refcount_helper_.decRefCount(); } + uint32_t use_count() const override { return refcount_helper_.use_count(); } + bool used_; Store* store_; std::shared_ptr histogram_stats_ = std::make_shared(); + +private: + RefcountHelper refcount_helper_; }; class MockMetricSnapshot : public MetricSnapshot { From d5b3981d3b22e72b13ee45e38567eee01402868e Mon Sep 17 00:00:00 2001 From: Nikita Galaiko Date: Wed, 3 Jul 2019 19:50:13 +0200 Subject: [PATCH 107/542] docs: Add path prefix to grpc json transcoder example (#7450) Signed-off-by: Nikita Galaiko --- .../http_filters/grpc_json_transcoder_filter.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst b/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst index fed6cbef5e3a..ab6236d86ef7 100644 --- a/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst +++ b/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst @@ -116,7 +116,9 @@ gRPC or RESTful JSON requests to localhost:51051. - name: local_service domains: ["*"] routes: - - match: { prefix: "/" } + # NOTE: by default, matching happens based on the gRPC route, and not on the incoming request path. + # Reference: https://www.envoyproxy.io/docs/envoy/latest/configuration/http_filters/grpc_json_transcoder_filter#route-configs-for-transcoded-requests + - match: { prefix: "/helloworld.Greeter" } route: { cluster: grpc, timeout: { seconds: 60 } } http_filters: - name: envoy.grpc_json_transcoder From 0c005c484f6d78c2423c1e5a59c15261d4ecddfa Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Wed, 3 Jul 2019 10:58:06 -0700 Subject: [PATCH 108/542] python: use python3 vs. 3.5 for venvs (#7460) Signed-off-by: Matt Klein --- tools/shell_utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell_utils.sh b/tools/shell_utils.sh index a0c7c60d8def..4d0471e89aea 100644 --- a/tools/shell_utils.sh +++ b/tools/shell_utils.sh @@ -2,7 +2,7 @@ source_venv() { VENV_DIR=$1 if [[ "$VIRTUAL_ENV" == "" ]]; then if [[ ! -d "${VENV_DIR}"/venv ]]; then - virtualenv "${VENV_DIR}"/venv --no-site-packages --python=python3.5 + virtualenv "${VENV_DIR}"/venv --no-site-packages --python=python3 fi source "${VENV_DIR}"/venv/bin/activate else From c60df6448b90cccaacdd7069520303231f2d2de6 Mon Sep 17 00:00:00 2001 From: Adam Kotwasinski Date: Wed, 3 Jul 2019 16:15:04 -0700 Subject: [PATCH 109/542] Kafka codec: refactoring for request&response codecs (#7452) Signed-off-by: Adam Kotwasinski --- .../extensions/filters/network/kafka/codec.h | 119 ++++++++++++++++++ .../network/kafka/kafka_request_parser.cc | 12 -- .../network/kafka/kafka_request_parser.h | 29 +++-- .../network/kafka/kafka_response_parser.cc | 13 -- .../network/kafka/kafka_response_parser.h | 27 ++-- .../extensions/filters/network/kafka/parser.h | 25 ++++ .../filters/network/kafka/request_codec.cc | 60 +-------- .../filters/network/kafka/request_codec.h | 57 +++------ .../filters/network/kafka/response_codec.cc | 67 +--------- .../filters/network/kafka/response_codec.h | 50 ++------ .../kafka/protocol/requests_test_cc.j2 | 2 +- .../network/kafka/request_codec_unit_test.cc | 108 ++++++++-------- .../network/kafka/response_codec_unit_test.cc | 44 ++++++- 13 files changed, 301 insertions(+), 312 deletions(-) diff --git a/source/extensions/filters/network/kafka/codec.h b/source/extensions/filters/network/kafka/codec.h index a58c284a052a..6dfe638d31d0 100644 --- a/source/extensions/filters/network/kafka/codec.h +++ b/source/extensions/filters/network/kafka/codec.h @@ -3,6 +3,8 @@ #include "envoy/buffer/buffer.h" #include "envoy/common/pure.h" +#include "common/common/stack_array.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -22,6 +24,123 @@ class MessageDecoder { virtual void onData(Buffer::Instance& data) PURE; }; +template class MessageCallback { +public: + virtual ~MessageCallback() = default; + + /** + * Callback method invoked when message is successfully decoded. + * @param message message that has been decoded. + */ + virtual void onMessage(MessageType response) PURE; + + /** + * Callback method invoked when message could not be decoded. + * Invoked after all message's bytes have been consumed. + */ + virtual void onFailedParse(ParseFailureType failure_data) PURE; +}; + +/** + * Abstract message decoder, that resolves messages from Buffer instances provided. + * When the message has been parsed, notify the callbacks. + */ +template +class AbstractMessageDecoder : public MessageDecoder { +public: + virtual ~AbstractMessageDecoder() = default; + + /** + * Creates a decoder that will invoke given callbacks when a message has been parsed. + * @param callbacks callbacks to be invoked (in order). + */ + AbstractMessageDecoder(const std::vector callbacks) : callbacks_{callbacks} {}; + + /** + * Consumes all data present in a buffer. + * If a message can be successfully parsed, then callbacks get notified with parsed response. + * Updates decoder state. + * Can throw if codec's state does not permit usage, or there there were parse failures. + * Impl note: similar to redis codec, which also keeps state. + */ + void onData(Buffer::Instance& data) override { + // Convert buffer to slices and pass them to `doParse`. + uint64_t num_slices = data.getRawSlices(nullptr, 0); + STACK_ARRAY(slices, Buffer::RawSlice, num_slices); + data.getRawSlices(slices.begin(), num_slices); + for (const Buffer::RawSlice& slice : slices) { + doParse(slice); + } + } + + ParserType getCurrentParserForTest() const { return current_parser_; } + +protected: + /** + * Create a start parser for a new message. + */ + virtual ParserType createStartParser() PURE; + +private: + /** + * Main parse loop. + * + * If there is data to process, and the current parser is not present, + * create a new one with `createStartParser`. + * Feed data to a current parser until it returns a parse result. + * If the parse result is a parsed message, notify callbacks and reset current parser. + * If the parse result is another parser, update current parser, and keep feeding. + */ + void doParse(const Buffer::RawSlice& slice) { + const char* bytes = reinterpret_cast(slice.mem_); + absl::string_view data = {bytes, slice.len_}; + + while (!data.empty()) { + + // Re-initialize the parser. + if (!current_parser_) { + current_parser_ = createStartParser(); + } + + // Feed the data to the parser. + auto result = current_parser_->parse(data); + // This loop guarantees that parsers consuming 0 bytes also get processed in this invocation. + while (result.hasData()) { + if (!result.next_parser_) { + + // Next parser is not present, so we have finished parsing a message. + // Depending on whether the parse was successful, invoke the correct callback. + if (result.message_) { + for (auto& callback : callbacks_) { + callback->onMessage(result.message_); + } + } else { + for (auto& callback : callbacks_) { + callback->onFailedParse(result.failure_data_); + } + } + + // As we finished parsing this response, return to outer loop. + // If there is more data, the parser will be re-initialized. + current_parser_ = nullptr; + break; + } else { + + // The next parser that's supposed to consume the rest of payload was given. + current_parser_ = result.next_parser_; + } + + // Keep parsing the data. + result = current_parser_->parse(data); + } + } + } + + const std::vector callbacks_; + + ParserType current_parser_; +}; + /** * Kafka message encoder. * @param MessageType encoded message type (request or response). diff --git a/source/extensions/filters/network/kafka/kafka_request_parser.cc b/source/extensions/filters/network/kafka/kafka_request_parser.cc index b98f5b9b696d..e7328708334b 100644 --- a/source/extensions/filters/network/kafka/kafka_request_parser.cc +++ b/source/extensions/filters/network/kafka/kafka_request_parser.cc @@ -44,18 +44,6 @@ RequestParseResponse RequestHeaderParser::parse(absl::string_view& data) { } } -RequestParseResponse SentinelParser::parse(absl::string_view& data) { - const uint32_t min = std::min(context_->remaining_request_size_, data.size()); - data = {data.data() + min, data.size() - min}; - context_->remaining_request_size_ -= min; - if (0 == context_->remaining_request_size_) { - return RequestParseResponse::parseFailure( - std::make_shared(context_->request_header_)); - } else { - return RequestParseResponse::stillWaiting(); - } -} - } // namespace Kafka } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/kafka/kafka_request_parser.h b/source/extensions/filters/network/kafka/kafka_request_parser.h index 3171c5e46197..b588b9aff454 100644 --- a/source/extensions/filters/network/kafka/kafka_request_parser.h +++ b/source/extensions/filters/network/kafka/kafka_request_parser.h @@ -22,8 +22,18 @@ using RequestParserSharedPtr = std::shared_ptr; * Context that is shared between parsers that are handling the same single message. */ struct RequestContext { - int32_t remaining_request_size_{0}; + uint32_t remaining_request_size_{0}; RequestHeader request_header_{}; + + /** + * Bytes left to consume. + */ + uint32_t& remaining() { return remaining_request_size_; } + + /** + * Returns data needed for construction of parse failure message. + */ + const RequestHeader asFailureData() const { return request_header_; } }; using RequestContextSharedPtr = std::shared_ptr; @@ -126,19 +136,14 @@ class RequestHeaderParser : public RequestParser { * api_key & api_version. It does not attempt to capture any data, just throws it away until end of * message. */ -class SentinelParser : public RequestParser { +class SentinelParser : public AbstractSentinelParser, + public RequestParser { public: - SentinelParser(RequestContextSharedPtr context) : context_{context} {}; - - /** - * Returns failed parse data. Ignores (jumps over) the data provided. - */ - RequestParseResponse parse(absl::string_view& data) override; - - const RequestContextSharedPtr contextForTest() const { return context_; } + SentinelParser(RequestContextSharedPtr context) : AbstractSentinelParser{context} {}; -private: - const RequestContextSharedPtr context_; + RequestParseResponse parse(absl::string_view& data) override { + return AbstractSentinelParser::parse(data); + } }; /** diff --git a/source/extensions/filters/network/kafka/kafka_response_parser.cc b/source/extensions/filters/network/kafka/kafka_response_parser.cc index 88af19527cb7..85a5a27c1f46 100644 --- a/source/extensions/filters/network/kafka/kafka_response_parser.cc +++ b/source/extensions/filters/network/kafka/kafka_response_parser.cc @@ -28,19 +28,6 @@ ResponseParseResponse ResponseHeaderParser::parse(absl::string_view& data) { return ResponseParseResponse::nextParser(next_parser); } -ResponseParseResponse SentinelResponseParser::parse(absl::string_view& data) { - const uint32_t min = std::min(context_->remaining_response_size_, data.size()); - data = {data.data() + min, data.size() - min}; - context_->remaining_response_size_ -= min; - if (0 == context_->remaining_response_size_) { - const auto failure_data = std::make_shared( - context_->api_key_, context_->api_version_, context_->correlation_id_); - return ResponseParseResponse::parseFailure(failure_data); - } else { - return ResponseParseResponse::stillWaiting(); - } -} - } // namespace Kafka } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/kafka/kafka_response_parser.h b/source/extensions/filters/network/kafka/kafka_response_parser.h index a39a9487d521..0f1dd6125ea9 100644 --- a/source/extensions/filters/network/kafka/kafka_response_parser.h +++ b/source/extensions/filters/network/kafka/kafka_response_parser.h @@ -39,12 +39,22 @@ struct ResponseContext { /** * Bytes left to process. */ - int32_t remaining_response_size_; + uint32_t remaining_response_size_; /** * Response's correlation id. */ int32_t correlation_id_; + + /** + * Bytes left to consume. + */ + uint32_t& remaining() { return remaining_response_size_; } + + /** + * Returns data needed for construction of parse failure message. + */ + const ResponseMetadata asFailureData() const { return {api_key_, api_version_, correlation_id_}; } }; using ResponseContextSharedPtr = std::shared_ptr; @@ -107,16 +117,15 @@ class ResponseHeaderParser : public ResponseParser { * api_key & api_version. It does not attempt to capture any data, just throws it away until end of * message. */ -class SentinelResponseParser : public ResponseParser { +class SentinelResponseParser + : public AbstractSentinelParser, + public ResponseParser { public: - SentinelResponseParser(ResponseContextSharedPtr context) : context_{context} {}; - - ResponseParseResponse parse(absl::string_view& data) override; - - const ResponseContextSharedPtr contextForTest() const { return context_; } + SentinelResponseParser(ResponseContextSharedPtr context) : AbstractSentinelParser{context} {}; -private: - ResponseContextSharedPtr context_; + ResponseParseResponse parse(absl::string_view& data) override { + return AbstractSentinelParser::parse(data); + } }; /** diff --git a/source/extensions/filters/network/kafka/parser.h b/source/extensions/filters/network/kafka/parser.h index 031aaaef1dbd..f05a07bb3a84 100644 --- a/source/extensions/filters/network/kafka/parser.h +++ b/source/extensions/filters/network/kafka/parser.h @@ -43,6 +43,8 @@ using ParserSharedPtr = std::shared_ptr>; */ template class ParseResponse { public: + using failure_type = FailureDataType; + /** * Constructs a response that states that parser still needs data and should not be replaced. */ @@ -88,6 +90,29 @@ template class ParseResponse { FailureDataType failure_data_; }; +template class AbstractSentinelParser { +public: + AbstractSentinelParser(ContextType context) : context_{context} {}; + + ResponseType parse(absl::string_view& data) { + const uint32_t min = std::min(context_->remaining(), data.size()); + data = {data.data() + min, data.size() - min}; + context_->remaining() -= min; + if (0 == context_->remaining()) { + using failure_type = typename ResponseType::failure_type::element_type; + auto failure_data = std::make_shared(context_->asFailureData()); + return ResponseType::parseFailure(failure_data); + } else { + return ResponseType::stillWaiting(); + } + } + + const ContextType contextForTest() const { return context_; } + +private: + ContextType context_; +}; + } // namespace Kafka } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/kafka/request_codec.cc b/source/extensions/filters/network/kafka/request_codec.cc index bf74e81e79d0..398f0116131c 100644 --- a/source/extensions/filters/network/kafka/request_codec.cc +++ b/source/extensions/filters/network/kafka/request_codec.cc @@ -1,7 +1,6 @@ #include "extensions/filters/network/kafka/request_codec.h" #include "common/buffer/buffer_impl.h" -#include "common/common/stack_array.h" #include "absl/strings/string_view.h" @@ -20,63 +19,8 @@ const InitialParserFactory& InitialParserFactory::getDefaultInstance() { CONSTRUCT_ON_FIRST_USE(RequestStartParserFactory); } -void RequestDecoder::onData(Buffer::Instance& data) { - // Convert buffer to slices and pass them to `doParse`. - uint64_t num_slices = data.getRawSlices(nullptr, 0); - STACK_ARRAY(slices, Buffer::RawSlice, num_slices); - data.getRawSlices(slices.begin(), num_slices); - for (const Buffer::RawSlice& slice : slices) { - doParse(slice); - } -} - -/** - * Main parse loop: - * - forward data to current parser, - * - receive parser response: - * -- if still waiting, do nothing (we wait for more data), - * -- if a parser is given, replace current parser with the new one, and it the rest of the data - * -- if a message is given: - * --- notify callbacks, - * --- replace current parser with new start parser, as we are going to start parsing the next - * message. - */ -void RequestDecoder::doParse(const Buffer::RawSlice& slice) { - const char* bytes = reinterpret_cast(slice.mem_); - absl::string_view data = {bytes, slice.len_}; - - while (!data.empty()) { - - // Feed the data to the parser. - RequestParseResponse result = current_parser_->parse(data); - // This loop guarantees that parsers consuming 0 bytes also get processed in this invocation. - while (result.hasData()) { - if (!result.next_parser_) { - - // Next parser is not present, so we have finished parsing a message. - // Depending on whether the parse was successful, invoke the correct callback. - if (result.message_) { - for (auto& callback : callbacks_) { - callback->onMessage(result.message_); - } - } else { - for (auto& callback : callbacks_) { - callback->onFailedParse(result.failure_data_); - } - } - - // As we finished parsing this request, re-initialize the parser. - current_parser_ = factory_.create(parser_resolver_); - } else { - - // The next parser that's supposed to consume the rest of payload was given. - current_parser_ = result.next_parser_; - } - - // Keep parsing the data. - result = current_parser_->parse(data); - } - } +RequestParserSharedPtr RequestDecoder::createStartParser() { + return factory_.create(parser_resolver_); } void RequestEncoder::encode(const AbstractRequest& message) { diff --git a/source/extensions/filters/network/kafka/request_codec.h b/source/extensions/filters/network/kafka/request_codec.h index b33befb6ad9d..56fa10a8057d 100644 --- a/source/extensions/filters/network/kafka/request_codec.h +++ b/source/extensions/filters/network/kafka/request_codec.h @@ -13,25 +13,7 @@ namespace Extensions { namespace NetworkFilters { namespace Kafka { -/** - * Callback invoked when request is successfully decoded. - */ -class RequestCallback { -public: - virtual ~RequestCallback() = default; - - /** - * Callback method invoked when request is successfully decoded. - * @param request request that has been decoded. - */ - virtual void onMessage(AbstractRequestSharedPtr request) PURE; - - /** - * Callback method invoked when request could not be decoded. - * Invoked after all request's bytes have been consumed. - */ - virtual void onFailedParse(RequestParseFailureSharedPtr failure_data) PURE; -}; +using RequestCallback = MessageCallback; using RequestCallbackSharedPtr = std::shared_ptr; @@ -61,45 +43,34 @@ class InitialParserFactory { * Each parser along the line returns the fully parsed message or the next parser. * Stores parse state (as large message's payload can be provided through multiple `onData` calls). */ -class RequestDecoder : public MessageDecoder { +class RequestDecoder + : public AbstractMessageDecoder { public: /** - * Creates a decoder that can decode requests specified by RequestParserResolver, notifying - * callbacks on successful decoding. - * @param parserResolver supported parser resolver. + * Creates a decoder that will notify provided callbacks when a message is successfully parsed. * @param callbacks callbacks to be invoked (in order). */ - RequestDecoder(const RequestParserResolver& parserResolver, - const std::vector callbacks) - : RequestDecoder(InitialParserFactory::getDefaultInstance(), parserResolver, callbacks){}; + RequestDecoder(const std::vector callbacks) + : RequestDecoder(InitialParserFactory::getDefaultInstance(), + RequestParserResolver::getDefaultInstance(), callbacks){}; /** * Visible for testing. - * Allows injecting initial parser factory. + * Allows injecting initial parser factory and parser resolver. + * @param factory parser factory to be used when new message is to be processed. + * @param parserResolver supported parser resolver. + * @param callbacks callbacks to be invoked (in order). */ RequestDecoder(const InitialParserFactory& factory, const RequestParserResolver& parserResolver, const std::vector callbacks) - : factory_{factory}, parser_resolver_{parserResolver}, callbacks_{callbacks}, - current_parser_{factory_.create(parser_resolver_)} {}; + : AbstractMessageDecoder{callbacks}, factory_{factory}, parser_resolver_{parserResolver} {}; - /** - * Consumes all data present in a buffer. - * If a request can be successfully parsed, then callbacks get notified with parsed request. - * Updates decoder state. - * Impl note: similar to redis codec, which also keeps state. - */ - void onData(Buffer::Instance& data) override; +protected: + RequestParserSharedPtr createStartParser() override; private: - void doParse(const Buffer::RawSlice& slice); - const InitialParserFactory& factory_; - const RequestParserResolver& parser_resolver_; - - const std::vector callbacks_; - - RequestParserSharedPtr current_parser_; }; /** diff --git a/source/extensions/filters/network/kafka/response_codec.cc b/source/extensions/filters/network/kafka/response_codec.cc index 7b6dd20c97f6..3f434137e4e4 100644 --- a/source/extensions/filters/network/kafka/response_codec.cc +++ b/source/extensions/filters/network/kafka/response_codec.cc @@ -1,5 +1,6 @@ #include "extensions/filters/network/kafka/response_codec.h" +#include "common/buffer/buffer_impl.h" #include "common/common/stack_array.h" namespace Envoy { @@ -34,70 +35,8 @@ void ResponseDecoder::expectResponse(const int16_t api_key, const int16_t api_ve factory_->expectResponse(api_key, api_version); }; -void ResponseDecoder::onData(Buffer::Instance& data) { - // Convert buffer to slices and pass them to `doParse`. - uint64_t num_slices = data.getRawSlices(nullptr, 0); - STACK_ARRAY(slices, Buffer::RawSlice, num_slices); - data.getRawSlices(slices.begin(), num_slices); - for (const Buffer::RawSlice& slice : slices) { - doParse(slice); - } -} - -/** - * Main parse loop: - * - initialize parser, if it is not present (using information stored in `factory_`) - * - forward data to current parser, - * - receive parser response: - * -- if still waiting, do nothing (we wait for more data), - * -- if a parser is given, replace current parser with the new one, and it the rest of the data - * -- if a message is given: - * --- notify callbacks, - * --- clean current parser. - */ -void ResponseDecoder::doParse(const Buffer::RawSlice& slice) { - const char* bytes = reinterpret_cast(slice.mem_); - absl::string_view data = {bytes, slice.len_}; - - while (!data.empty()) { - - // Re-initialize the parser. - if (!current_parser_) { - current_parser_ = factory_->create(response_parser_resolver_); - } - - // Feed the data to the parser. - ResponseParseResponse result = current_parser_->parse(data); - // This loop guarantees that parsers consuming 0 bytes also get processed in this invocation. - while (result.hasData()) { - if (!result.next_parser_) { - - // Next parser is not present, so we have finished parsing a message. - // Depending on whether the parse was successful, invoke the correct callback. - if (result.message_) { - for (auto& callback : callbacks_) { - callback->onMessage(result.message_); - } - } else { - for (auto& callback : callbacks_) { - callback->onFailedParse(result.failure_data_); - } - } - - // As we finished parsing this response, return to outer loop. - // If there is more data, the parser will be re-initialized. - current_parser_ = nullptr; - break; - } else { - - // The next parser that's supposed to consume the rest of payload was given. - current_parser_ = result.next_parser_; - } - - // Keep parsing the data. - result = current_parser_->parse(data); - } - } +ResponseParserSharedPtr ResponseDecoder::createStartParser() { + return factory_->create(response_parser_resolver_); } void ResponseEncoder::encode(const AbstractResponse& message) { diff --git a/source/extensions/filters/network/kafka/response_codec.h b/source/extensions/filters/network/kafka/response_codec.h index 55d1e2ac58a5..30019e747c7a 100644 --- a/source/extensions/filters/network/kafka/response_codec.h +++ b/source/extensions/filters/network/kafka/response_codec.h @@ -10,25 +10,7 @@ namespace Extensions { namespace NetworkFilters { namespace Kafka { -/** - * Callback invoked when response is successfully decoded. - */ -class ResponseCallback { -public: - virtual ~ResponseCallback() = default; - - /** - * Callback method invoked when response is successfully decoded. - * @param response response that has been decoded. - */ - virtual void onMessage(AbstractResponseSharedPtr response) PURE; - - /** - * Callback method invoked when response could not be decoded. - * Invoked after all response's bytes have been consumed. - */ - virtual void onFailedParse(ResponseMetadataSharedPtr failure_data) PURE; -}; +using ResponseCallback = MessageCallback; using ResponseCallbackSharedPtr = std::shared_ptr; @@ -75,10 +57,12 @@ using ResponseInitialParserFactorySharedPtr = std::shared_ptr { +class ResponseDecoder + : public AbstractMessageDecoder, + public Logger::Loggable { public: /** - * Creates a decoder that will notify provided callbacks. + * Creates a decoder that will notify provided callbacks when a message is successfully parsed. * @param callbacks callbacks to be invoked (in order). */ ResponseDecoder(const std::vector callbacks) @@ -87,13 +71,16 @@ class ResponseDecoder : public MessageDecoder, public Logger::Loggable callbacks) - : factory_{factory}, response_parser_resolver_{response_parser_resolver}, callbacks_{ - callbacks} {}; + : AbstractMessageDecoder{callbacks}, factory_{factory}, response_parser_resolver_{ + response_parser_resolver} {}; /** * Registers an expected message. @@ -104,23 +91,12 @@ class ResponseDecoder : public MessageDecoder, public Logger::Loggable callbacks_; - - ResponseParserSharedPtr current_parser_; }; /** diff --git a/test/extensions/filters/network/kafka/protocol/requests_test_cc.j2 b/test/extensions/filters/network/kafka/protocol/requests_test_cc.j2 index 3232dd0b01cb..42ec616756ef 100644 --- a/test/extensions/filters/network/kafka/protocol/requests_test_cc.j2 +++ b/test/extensions/filters/network/kafka/protocol/requests_test_cc.j2 @@ -38,7 +38,7 @@ template std::shared_ptr RequestTest::serializeAndDeserialize(T putMessageIntoBuffer(message); std::shared_ptr mock_listener = std::make_shared(); - RequestDecoder testee{RequestParserResolver::getDefaultInstance(), {mock_listener}}; + RequestDecoder testee{ {mock_listener} }; AbstractRequestSharedPtr received_message; EXPECT_CALL(*mock_listener, onMessage(testing::_)) diff --git a/test/extensions/filters/network/kafka/request_codec_unit_test.cc b/test/extensions/filters/network/kafka/request_codec_unit_test.cc index ecd1eaf88e4f..9c7ff5218738 100644 --- a/test/extensions/filters/network/kafka/request_codec_unit_test.cc +++ b/test/extensions/filters/network/kafka/request_codec_unit_test.cc @@ -50,7 +50,7 @@ class RequestCodecUnitTest : public testing::Test, public BufferBasedTest { protected: MockParserFactory initial_parser_factory_{}; MockRequestParserResolver parser_resolver_{}; - MockRequestCallbackSharedPtr request_callback_{std::make_shared()}; + MockRequestCallbackSharedPtr callback_{std::make_shared()}; }; RequestParseResponse consumeOneByte(absl::string_view& data) { @@ -67,16 +67,16 @@ TEST_F(RequestCodecUnitTest, shouldDoNothingIfParserReturnsWaiting) { EXPECT_CALL(initial_parser_factory_, create(_)).WillOnce(Return(parser)); - EXPECT_CALL(*request_callback_, onMessage(_)).Times(0); - EXPECT_CALL(*request_callback_, onFailedParse(_)).Times(0); + EXPECT_CALL(*callback_, onMessage(_)).Times(0); + EXPECT_CALL(*callback_, onFailedParse(_)).Times(0); - RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; + RequestDecoder testee{initial_parser_factory_, parser_resolver_, {callback_}}; // when testee.onData(buffer_); // then - // There were no interactions with `request_callback_`. + // There were no interactions with `callback_`. } TEST_F(RequestCodecUnitTest, shouldUseNewParserAsResponse) { @@ -91,57 +91,61 @@ TEST_F(RequestCodecUnitTest, shouldUseNewParserAsResponse) { EXPECT_CALL(*parser3, parse(_)).Times(AnyNumber()).WillRepeatedly(Invoke(consumeOneByte)); EXPECT_CALL(initial_parser_factory_, create(_)).WillOnce(Return(parser1)); + EXPECT_CALL(parser_resolver_, createParser(_, _, _)).Times(0); - EXPECT_CALL(*request_callback_, onMessage(_)).Times(0); - EXPECT_CALL(*request_callback_, onFailedParse(_)).Times(0); + EXPECT_CALL(*callback_, onMessage(_)).Times(0); + EXPECT_CALL(*callback_, onFailedParse(_)).Times(0); - RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; + RequestDecoder testee{initial_parser_factory_, parser_resolver_, {callback_}}; // when testee.onData(buffer_); // then - // There were no interactions with `request_callback_`. + ASSERT_EQ(testee.getCurrentParserForTest(), parser3); + // Also, there were no interactions with `callback_`. } -TEST_F(RequestCodecUnitTest, shouldPassParsedMessageToCallbackAndReinitialize) { +TEST_F(RequestCodecUnitTest, shouldPassParsedMessageToCallback) { // given putGarbageIntoBuffer(); - AbstractRequestSharedPtr message = std::make_shared>(RequestHeader(), 0); + const AbstractRequestSharedPtr parsed_message = + std::make_shared>(RequestHeader{0, 0, 0, ""}, 0); - MockParserSharedPtr parser1 = std::make_shared(); - EXPECT_CALL(*parser1, parse(_)).WillOnce(Return(RequestParseResponse::parsedMessage(message))); - - MockParserSharedPtr parser2 = std::make_shared(); - EXPECT_CALL(*parser2, parse(_)).Times(AnyNumber()).WillRepeatedly(Invoke(consumeOneByte)); + MockParserSharedPtr all_consuming_parser = std::make_shared(); + auto consume_and_return = [&parsed_message](absl::string_view& data) -> RequestParseResponse { + data = {data.data() + data.size(), 0}; + return RequestParseResponse::parsedMessage(parsed_message); + }; + EXPECT_CALL(*all_consuming_parser, parse(_)).WillOnce(Invoke(consume_and_return)); - EXPECT_CALL(initial_parser_factory_, create(_)) - .WillOnce(Return(parser1)) - .WillOnce(Return(parser2)); + EXPECT_CALL(initial_parser_factory_, create(_)).WillOnce(Return(all_consuming_parser)); + EXPECT_CALL(parser_resolver_, createParser(_, _, _)).Times(0); - EXPECT_CALL(*request_callback_, onMessage(message)); - EXPECT_CALL(*request_callback_, onFailedParse(_)).Times(0); + EXPECT_CALL(*callback_, onMessage(parsed_message)); + EXPECT_CALL(*callback_, onFailedParse(_)).Times(0); - RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; + RequestDecoder testee{initial_parser_factory_, parser_resolver_, {callback_}}; // when testee.onData(buffer_); // then - // `request_callback_` had `onFailedParse` invoked once with matching argument. + ASSERT_EQ(testee.getCurrentParserForTest(), nullptr); + // Also, `callback_` had `onMessage` invoked once with matching argument. } -TEST_F(RequestCodecUnitTest, shouldPassParseFailureDataToCallbackAndReinitialize) { +TEST_F(RequestCodecUnitTest, shouldPassParsedMessageToCallbackAndInitializeNextParser) { // given putGarbageIntoBuffer(); - RequestParseFailureSharedPtr failure_data = - std::make_shared(RequestHeader()); + const AbstractRequestSharedPtr parsed_message = + std::make_shared>(RequestHeader(), 0); MockParserSharedPtr parser1 = std::make_shared(); EXPECT_CALL(*parser1, parse(_)) - .WillOnce(Return(RequestParseResponse::parseFailure(failure_data))); + .WillOnce(Return(RequestParseResponse::parsedMessage(parsed_message))); MockParserSharedPtr parser2 = std::make_shared(); EXPECT_CALL(*parser2, parse(_)).Times(AnyNumber()).WillRepeatedly(Invoke(consumeOneByte)); @@ -150,57 +154,47 @@ TEST_F(RequestCodecUnitTest, shouldPassParseFailureDataToCallbackAndReinitialize .WillOnce(Return(parser1)) .WillOnce(Return(parser2)); - EXPECT_CALL(*request_callback_, onMessage(_)).Times(0); - EXPECT_CALL(*request_callback_, onFailedParse(failure_data)); + EXPECT_CALL(*callback_, onMessage(parsed_message)); + EXPECT_CALL(*callback_, onFailedParse(_)).Times(0); - RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; + RequestDecoder testee{initial_parser_factory_, parser_resolver_, {callback_}}; // when testee.onData(buffer_); // then - // `request_callback_` had `onFailedParse` invoked once with matching argument. + ASSERT_EQ(testee.getCurrentParserForTest(), parser2); + // Also, `callback_` had `onMessage` invoked once with matching argument. } -TEST_F(RequestCodecUnitTest, shouldInvokeParsersEvenIfTheyDoNotConsumeZeroBytes) { +TEST_F(RequestCodecUnitTest, shouldPassParseFailureDataToCallback) { // given putGarbageIntoBuffer(); - MockParserSharedPtr parser1 = std::make_shared(); - MockParserSharedPtr parser2 = std::make_shared(); - MockParserSharedPtr parser3 = std::make_shared(); + const RequestParseFailureSharedPtr failure_data = + std::make_shared(RequestHeader()); - // parser1 consumes buffer_.length() bytes (== everything) and returns parser2 - auto consume_and_return = [this, &parser2](absl::string_view& data) -> RequestParseResponse { - data = {data.data() + buffer_.length(), data.size() - buffer_.length()}; - return RequestParseResponse::nextParser(parser2); + MockParserSharedPtr parser = std::make_shared(); + auto consume_and_return = [&failure_data](absl::string_view& data) -> RequestParseResponse { + data = {data.data() + data.size(), 0}; + return RequestParseResponse::parseFailure(failure_data); }; - EXPECT_CALL(*parser1, parse(_)).WillOnce(Invoke(consume_and_return)); - - // parser2 just returns parse result - RequestParseFailureSharedPtr failure_data = - std::make_shared(RequestHeader{}); - EXPECT_CALL(*parser2, parse(_)) - .WillOnce(Return(RequestParseResponse::parseFailure(failure_data))); + EXPECT_CALL(*parser, parse(_)).WillOnce(Invoke(consume_and_return)); - // parser3 just consumes everything - EXPECT_CALL(*parser3, parse(ResultOf([](absl::string_view arg) { return arg.size(); }, Eq(0)))) - .WillOnce(Return(RequestParseResponse::stillWaiting())); - - EXPECT_CALL(initial_parser_factory_, create(_)) - .WillOnce(Return(parser1)) - .WillOnce(Return(parser3)); + EXPECT_CALL(initial_parser_factory_, create(_)).WillOnce(Return(parser)); + EXPECT_CALL(parser_resolver_, createParser(_, _, _)).Times(0); - EXPECT_CALL(*request_callback_, onFailedParse(failure_data)); + EXPECT_CALL(*callback_, onMessage(_)).Times(0); + EXPECT_CALL(*callback_, onFailedParse(failure_data)); - RequestDecoder testee{initial_parser_factory_, parser_resolver_, {request_callback_}}; + RequestDecoder testee{initial_parser_factory_, parser_resolver_, {callback_}}; // when testee.onData(buffer_); // then - // `request_callback_` had `onFailedParse` invoked once with matching argument. - // After that, `parser3` was created and passed remaining data (that should have been empty). + ASSERT_EQ(testee.getCurrentParserForTest(), nullptr); + // Also, `callback_` had `onFailedParse` invoked once with matching argument. } } // namespace RequestCodecUnitTest diff --git a/test/extensions/filters/network/kafka/response_codec_unit_test.cc b/test/extensions/filters/network/kafka/response_codec_unit_test.cc index c8fcd016cce3..6d20d6ef6dc5 100644 --- a/test/extensions/filters/network/kafka/response_codec_unit_test.cc +++ b/test/extensions/filters/network/kafka/response_codec_unit_test.cc @@ -105,7 +105,8 @@ TEST_F(ResponseCodecUnitTest, shouldUseNewParserAsResponse) { testee.onData(buffer_); // then - // There were no interactions with `callback_`. + ASSERT_EQ(testee.getCurrentParserForTest(), parser3); + // Also, there were no interactions with `callback_`. } TEST_F(ResponseCodecUnitTest, shouldPassParsedMessageToCallback) { @@ -115,14 +116,14 @@ TEST_F(ResponseCodecUnitTest, shouldPassParsedMessageToCallback) { const AbstractResponseSharedPtr parsed_message = std::make_shared>(ResponseMetadata{0, 0, 0}, 0); - MockParserSharedPtr parser = std::make_shared(); + MockParserSharedPtr all_consuming_parser = std::make_shared(); auto consume_and_return = [&parsed_message](absl::string_view& data) -> ResponseParseResponse { data = {data.data() + data.size(), 0}; return ResponseParseResponse::parsedMessage(parsed_message); }; - EXPECT_CALL(*parser, parse(_)).WillOnce(Invoke(consume_and_return)); + EXPECT_CALL(*all_consuming_parser, parse(_)).WillOnce(Invoke(consume_and_return)); - EXPECT_CALL(*factory_, create(_)).WillOnce(Return(parser)); + EXPECT_CALL(*factory_, create(_)).WillOnce(Return(all_consuming_parser)); EXPECT_CALL(parser_resolver_, createParser(_)).Times(0); EXPECT_CALL(*callback_, onMessage(parsed_message)); @@ -134,7 +135,37 @@ TEST_F(ResponseCodecUnitTest, shouldPassParsedMessageToCallback) { testee.onData(buffer_); // then - // `callback_` had `onMessage` invoked once with matching argument. + ASSERT_EQ(testee.getCurrentParserForTest(), nullptr); + // Also, `callback_` had `onMessage` invoked once with matching argument. +} + +TEST_F(ResponseCodecUnitTest, shouldPassParsedMessageToCallbackAndInitializeNextParser) { + // given + putGarbageIntoBuffer(); + + const AbstractResponseSharedPtr parsed_message = + std::make_shared>(ResponseMetadata{0, 0, 0}, 0); + + MockParserSharedPtr parser1 = std::make_shared(); + EXPECT_CALL(*parser1, parse(_)) + .WillOnce(Return(ResponseParseResponse::parsedMessage(parsed_message))); + + MockParserSharedPtr parser2 = std::make_shared(); + EXPECT_CALL(*parser2, parse(_)).Times(AnyNumber()).WillRepeatedly(Invoke(consumeOneByte)); + + EXPECT_CALL(*factory_, create(_)).WillOnce(Return(parser1)).WillOnce(Return(parser2)); + + EXPECT_CALL(*callback_, onMessage(parsed_message)); + EXPECT_CALL(*callback_, onFailedParse(_)).Times(0); + + ResponseDecoder testee{factory_, parser_resolver_, {callback_}}; + + // when + testee.onData(buffer_); + + // then + ASSERT_EQ(testee.getCurrentParserForTest(), parser2); + // Also, `callback_` had `onMessage` invoked once with matching argument. } TEST_F(ResponseCodecUnitTest, shouldPassParseFailureDataToCallback) { @@ -162,7 +193,8 @@ TEST_F(ResponseCodecUnitTest, shouldPassParseFailureDataToCallback) { testee.onData(buffer_); // then - // `callback_` had `onFailedParse` invoked once with matching argument. + ASSERT_EQ(testee.getCurrentParserForTest(), nullptr); + // Also, `callback_` had `onFailedParse` invoked once with matching argument. } } // namespace ResponseCodecUnitTest From 733052a74d1e643f756a82ba3f80ef892f35dc63 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 3 Jul 2019 19:16:13 -0400 Subject: [PATCH 110/542] http: dumping session state on the decode path (#7390) Signed-off-by: Alyssa Wilk --- bazel/BUILD | 5 ++ bazel/README.md | 1 + bazel/envoy_internal.bzl | 3 + docs/root/intro/version_history.rst | 1 + include/envoy/common/BUILD | 5 ++ include/envoy/common/scope_tracker.h | 29 +++++++++ include/envoy/event/BUILD | 1 + include/envoy/event/dispatcher.h | 12 ++++ include/envoy/http/header_map.h | 18 +++--- source/common/common/BUILD | 16 +++++ source/common/common/dump_state_utils.h | 54 ++++++++++++++++ source/common/common/scope_tracker.h | 27 ++++++++ source/common/event/BUILD | 9 ++- source/common/event/dispatcher_impl.cc | 24 +++++-- source/common/event/dispatcher_impl.h | 24 ++++++- source/common/http/BUILD | 4 ++ source/common/http/conn_manager_impl.cc | 12 ++++ source/common/http/conn_manager_impl.h | 26 +++++++- source/common/http/header_map_impl.cc | 15 +++++ source/common/http/header_map_impl.h | 1 + source/common/signal/BUILD | 28 +++++++++ source/common/signal/fatal_error_handler.h | 18 ++++++ .../{exe => common/signal}/signal_action.cc | 51 ++++++++++++++- source/{exe => common/signal}/signal_action.h | 22 ++++++- source/common/stream_info/BUILD | 1 + source/common/stream_info/stream_info_impl.h | 8 +++ source/exe/BUILD | 14 +---- source/exe/main_common.h | 2 +- test/BUILD | 2 +- test/common/http/conn_manager_impl_test.cc | 63 +++++++++++++++++++ test/common/signal/BUILD | 19 ++++++ test/{exe => common/signal}/signals_test.cc | 22 ++++++- .../stream_info/stream_info_impl_test.cc | 14 +++++ test/exe/BUILD | 10 --- test/exe/main_common_test.cc | 2 +- test/main.cc | 2 +- test/mocks/event/mocks.h | 1 + 37 files changed, 518 insertions(+), 48 deletions(-) create mode 100644 include/envoy/common/scope_tracker.h create mode 100644 source/common/common/dump_state_utils.h create mode 100644 source/common/common/scope_tracker.h create mode 100644 source/common/signal/BUILD create mode 100644 source/common/signal/fatal_error_handler.h rename source/{exe => common/signal}/signal_action.cc (60%) rename source/{exe => common/signal}/signal_action.h (87%) create mode 100644 test/common/signal/BUILD rename test/{exe => common/signal}/signals_test.cc (84%) diff --git a/bazel/BUILD b/bazel/BUILD index c2fa24b88e93..c288f351054f 100755 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -100,6 +100,11 @@ config_setting( values = {"define": "signal_trace=disabled"}, ) +config_setting( + name = "disable_object_dump_on_signal_trace", + values = {"define": "object_dump_on_signal_trace=disabled"}, +) + config_setting( name = "disable_hot_restart", values = {"define": "hot_restart=disabled"}, diff --git a/bazel/README.md b/bazel/README.md index 357a1b802f2f..25743ffdc097 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -350,6 +350,7 @@ The following optional features can be disabled on the Bazel build command-line: * Hot restart with `--define hot_restart=disabled` * Google C++ gRPC client with `--define google_grpc=disabled` * Backtracing on signals with `--define signal_trace=disabled` +* Active stream state dump on signals with `--define signal_trace=disabled` or `--define disable_object_dump_on_signal_trace=disabled` * tcmalloc with `--define tcmalloc=disabled` ## Enabling optional features diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index f12d52b2570e..41b86a4d1b0a 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -49,6 +49,9 @@ def envoy_copts(repository, test = False): }) + select({ repository + "//bazel:disable_signal_trace": [], "//conditions:default": ["-DENVOY_HANDLE_SIGNALS"], + }) + select({ + repository + "//bazel:disable_object_dump_on_signal_trace": [], + "//conditions:default": ["-DENVOY_OBJECT_TRACE_ON_DUMP"], }) + select({ repository + "//bazel:enable_log_debug_assert_in_release": ["-DENVOY_LOG_DEBUG_ASSERT_IN_RELEASE"], "//conditions:default": [], diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 6a8b03c954e9..29b24fcd4db2 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -38,6 +38,7 @@ Version history * http: added support for :ref:`preserve_external_request_id` that represents whether the x-request-id should not be reset on edge entry inside mesh * http: changed `sendLocalReply` to send percent-encoded `GrpcMessage`. * http: added :ref:`dynamic forward proxy ` support. +* http: tracking the active stream and dumping state in Envoy crash handlers. This can be disabled by building with `--define disable_object_dump_on_signal_trace=disabled` * jwt_authn: make filter's parsing of JWT more flexible, allowing syntax like ``jwt=eyJhbGciOiJS...ZFnFIw,extra=7,realm=123`` * outlier_detector: added configuration :ref:`outlier_detection.split_external_local_origin_errors` to distinguish locally and externally generated errors. See :ref:`arch_overview_outlier_detection` for full details. * listener: added :ref:`source IP ` diff --git a/include/envoy/common/BUILD b/include/envoy/common/BUILD index 1929682fb13c..4e04008c25d0 100644 --- a/include/envoy/common/BUILD +++ b/include/envoy/common/BUILD @@ -51,3 +51,8 @@ envoy_cc_library( name = "backoff_strategy_interface", hdrs = ["backoff_strategy.h"], ) + +envoy_cc_library( + name = "scope_tracker_interface", + hdrs = ["scope_tracker.h"], +) diff --git a/include/envoy/common/scope_tracker.h b/include/envoy/common/scope_tracker.h new file mode 100644 index 000000000000..eb72edf3b7af --- /dev/null +++ b/include/envoy/common/scope_tracker.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include "envoy/common/pure.h" + +namespace Envoy { + +/* + * A class for tracking the scope of work. + * Currently this is only used for best-effort tracking the L7 stream doing + * work if a fatal error occurs. + */ +class ScopeTrackedObject { +public: + virtual ~ScopeTrackedObject() = default; + + /** + * Dump debug state of the object in question to the provided ostream + * + * This is called on Envoy fatal errors, so should do minimal memory allocation. + * + * @param os the ostream to output to. + * @param indent_level how far to indent, for pretty-printed classes and subclasses. + */ + virtual void dumpState(std::ostream& os, int indent_level = 0) const PURE; +}; + +} // namespace Envoy diff --git a/include/envoy/event/BUILD b/include/envoy/event/BUILD index 962e290e3808..05ea911bf8cc 100644 --- a/include/envoy/event/BUILD +++ b/include/envoy/event/BUILD @@ -20,6 +20,7 @@ envoy_cc_library( ":deferred_deletable", ":file_event_interface", ":signal_interface", + "//include/envoy/common:scope_tracker_interface", "//include/envoy/common:time_interface", "//include/envoy/event:timer_interface", "//include/envoy/filesystem:watcher_interface", diff --git a/include/envoy/event/dispatcher.h b/include/envoy/event/dispatcher.h index 50a6ddbadb86..4dbe2ab06e16 100644 --- a/include/envoy/event/dispatcher.h +++ b/include/envoy/event/dispatcher.h @@ -6,6 +6,7 @@ #include #include +#include "envoy/common/scope_tracker.h" #include "envoy/common/time.h" #include "envoy/event/file_event.h" #include "envoy/event/signal.h" @@ -199,6 +200,17 @@ class Dispatcher { * @return the watermark buffer factory for this dispatcher. */ virtual Buffer::WatermarkFactory& getWatermarkFactory() PURE; + + /** + * Sets a tracked object, which is currently operating in this Dispatcher. + * This should be cleared with another call to setTrackedObject() when the object is done doing + * work. Calling setTrackedObject(nullptr) results in no object being tracked. + * + * This is optimized for performance, to avoid allocation where we do scoped object tracking. + * + * @return The previously tracked object or nullptr if there was none. + */ + virtual const ScopeTrackedObject* setTrackedObject(const ScopeTrackedObject* object) PURE; }; using DispatcherPtr = std::unique_ptr; diff --git a/include/envoy/http/header_map.h b/include/envoy/http/header_map.h index 8af3b10e0b8d..050b3d0b2342 100644 --- a/include/envoy/http/header_map.h +++ b/include/envoy/http/header_map.h @@ -527,19 +527,23 @@ class HeaderMap { */ virtual bool empty() const PURE; + /** + * Dump the header map to the ostream specified + * + * @param os the stream to dump state to + * @param indent_level the depth, for pretty-printing. + * + * This function is called on Envoy fatal errors so should avoid memory allocation where possible. + */ + virtual void dumpState(std::ostream& os, int indent_level = 0) const PURE; + /** * Allow easy pretty-printing of the key/value pairs in HeaderMap * @param os supplies the ostream to print to. * @param headers the headers to print. */ friend std::ostream& operator<<(std::ostream& os, const HeaderMap& headers) { - headers.iterate( - [](const HeaderEntry& header, void* context) -> HeaderMap::Iterate { - *static_cast(context) << "'" << header.key().getStringView() << "', '" - << header.value().getStringView() << "'\n"; - return HeaderMap::Iterate::Continue; - }, - &os); + headers.dumpState(os); return os; } }; diff --git a/source/common/common/BUILD b/source/common/common/BUILD index bacfa4d4b06a..6ace52dbb343 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -109,6 +109,12 @@ envoy_cc_library( deps = [":assert_lib"], ) +# Contains macros and helpers for dumpState utilities +envoy_cc_library( + name = "dump_state_utils", + hdrs = ["dump_state_utils.h"], +) + # Contains minimal code for logging to stderr. envoy_cc_library( name = "minimal_logger_lib", @@ -140,6 +146,7 @@ envoy_cc_library( srcs = ["logger_delegates.cc"], hdrs = ["logger_delegates.h"], deps = [ + ":dump_state_utils", ":macros", ":minimal_logger_lib", "//include/envoy/access_log:access_log_interface", @@ -177,6 +184,15 @@ envoy_cc_library( hdrs = ["phantom.h"], ) +envoy_cc_library( + name = "scope_tracker", + hdrs = ["scope_tracker.h"], + deps = [ + "//include/envoy/common:scope_tracker_interface", + "//include/envoy/event:dispatcher_interface", + ], +) + envoy_cc_library( name = "stl_helpers", hdrs = ["stl_helpers.h"], diff --git a/source/common/common/dump_state_utils.h b/source/common/common/dump_state_utils.h new file mode 100644 index 000000000000..c17c7b6c2ec6 --- /dev/null +++ b/source/common/common/dump_state_utils.h @@ -0,0 +1,54 @@ +#pragma once + +#include + +namespace Envoy { + +// A collection of macros for pretty printing objects on fatal error. +// These are fairly ugly in an attempt to maximize the conditions where fatal error logging occurs, +// i.e. under the Envoy signal handler if encountering a crash due to OOM, where allocating more +// memory would likely lead to the crash handler itself causing a subsequent OOM. + +#define DUMP_MEMBER(member) ", " #member ": " << (member) + +#define DUMP_OPTIONAL_MEMBER(member) \ + ", " #member ": " << ((member).has_value() ? absl::StrCat((member).value()) : "null") + +// Macro assumes local member variables +// os (ostream) +// indent_level (int) +#define DUMP_DETAILS(member) \ + do { \ + os << spaces << #member ": "; \ + if ((member) != nullptr) { \ + os << "\n"; \ + (member)->dumpState(os, indent_level + 1); \ + } else { \ + os << spaces << "null\n"; \ + } \ + } while (false) + +// Return the const char* equivalent of string(level*2, ' '), without dealing +// with string creation overhead. Cap arbitrarily at 6 as we're (hopefully) +// not going to have nested objects deeper than that. +inline const char* spacesForLevel(int level) { + switch (level) { + case 0: + return ""; + case 1: + return " "; + case 2: + return " "; + case 3: + return " "; + case 4: + return " "; + case 5: + return " "; + default: + return " "; + } + return ""; +} + +} // namespace Envoy diff --git a/source/common/common/scope_tracker.h b/source/common/common/scope_tracker.h new file mode 100644 index 000000000000..bed58c3fa8c0 --- /dev/null +++ b/source/common/common/scope_tracker.h @@ -0,0 +1,27 @@ +#pragma once + +#include "envoy/common/scope_tracker.h" +#include "envoy/event/dispatcher.h" + +namespace Envoy { + +// A small class for tracking the scope of the object which is currently having +// work done in this thread. +// +// When created, it sets the tracked object in the dispatcher, and when destroyed it points the +// dispatcher at the previously tracked object. +class ScopeTrackerScopeState { +public: + ScopeTrackerScopeState(const ScopeTrackedObject* object, Event::Dispatcher& dispatcher) + : dispatcher_(dispatcher) { + latched_object_ = dispatcher_.setTrackedObject(object); + } + + ~ScopeTrackerScopeState() { dispatcher_.setTrackedObject(latched_object_); } + +private: + const ScopeTrackedObject* latched_object_; + Event::Dispatcher& dispatcher_; +}; + +} // namespace Envoy diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 478fc28eb4c8..78461dcd3181 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -22,6 +22,7 @@ envoy_cc_library( ":dispatcher_includes", ":libevent_scheduler_lib", ":real_time_system_lib", + "//include/envoy/common:scope_tracker_interface", "//include/envoy/common:time_interface", "//include/envoy/event:signal_interface", "//include/envoy/network:listen_socket_interface", @@ -74,7 +75,13 @@ envoy_cc_library( "//include/envoy/network:connection_handler_interface", "//source/common/common:minimal_logger_lib", "//source/common/common:thread_lib", - ], + "//source/common/signal:fatal_error_handler_lib", + ] + select({ + "//bazel:disable_signal_trace": [], + "//conditions:default": [ + "//source/common/signal:sigaction_lib", + ], + }), ) envoy_cc_library( diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 996ce9960471..7b718b8daf57 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -25,6 +25,10 @@ #include "event2/event.h" +#ifdef ENVOY_HANDLE_SIGNALS +#include "common/signal/signal_action.h" +#endif + namespace Envoy { namespace Event { @@ -35,11 +39,19 @@ DispatcherImpl::DispatcherImpl(Buffer::WatermarkFactoryPtr&& factory, Api::Api& Event::TimeSystem& time_system) : api_(api), buffer_factory_(std::move(factory)), scheduler_(time_system.createScheduler(base_scheduler_)), - deferred_delete_timer_(createTimer([this]() -> void { clearDeferredDeleteList(); })), - post_timer_(createTimer([this]() -> void { runPostCallbacks(); })), - current_to_delete_(&to_delete_1_) {} + deferred_delete_timer_(createTimerInternal([this]() -> void { clearDeferredDeleteList(); })), + post_timer_(createTimerInternal([this]() -> void { runPostCallbacks(); })), + current_to_delete_(&to_delete_1_) { +#ifdef ENVOY_HANDLE_SIGNALS + SignalAction::registerFatalErrorHandler(*this); +#endif +} -DispatcherImpl::~DispatcherImpl() {} +DispatcherImpl::~DispatcherImpl() { +#ifdef ENVOY_HANDLE_SIGNALS + SignalAction::removeFatalErrorHandler(*this); +#endif +} void DispatcherImpl::initializeStats(Stats::Scope& scope, const std::string& prefix) { // This needs to be run in the dispatcher's thread, so that we have a thread id to log. @@ -133,7 +145,9 @@ Network::ListenerPtr DispatcherImpl::createUdpListener(Network::Socket& socket, return Network::ListenerPtr{new Network::UdpListenerImpl(*this, socket, cb, timeSource())}; } -TimerPtr DispatcherImpl::createTimer(TimerCb cb) { +TimerPtr DispatcherImpl::createTimer(TimerCb cb) { return createTimerInternal(cb); } + +TimerPtr DispatcherImpl::createTimerInternal(TimerCb cb) { ASSERT(isThreadSafe()); return scheduler_->createTimer(cb); } diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index ba95ee5684ea..51f71e8ad807 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -7,6 +7,7 @@ #include #include "envoy/api/api.h" +#include "envoy/common/scope_tracker.h" #include "envoy/common/time.h" #include "envoy/event/deferred_deletable.h" #include "envoy/event/dispatcher.h" @@ -17,6 +18,7 @@ #include "common/common/thread.h" #include "common/event/libevent.h" #include "common/event/libevent_scheduler.h" +#include "common/signal/fatal_error_handler.h" namespace Envoy { namespace Event { @@ -24,7 +26,9 @@ namespace Event { /** * libevent implementation of Event::Dispatcher. */ -class DispatcherImpl : Logger::Loggable, public Dispatcher { +class DispatcherImpl : Logger::Loggable, + public Dispatcher, + public FatalErrorHandlerInterface { public: DispatcherImpl(Api::Api& api, Event::TimeSystem& time_system); DispatcherImpl(Buffer::WatermarkFactoryPtr&& factory, Api::Api& api, @@ -65,8 +69,25 @@ class DispatcherImpl : Logger::Loggable, public Dispatcher { void post(std::function callback) override; void run(RunType type) override; Buffer::WatermarkFactory& getWatermarkFactory() override { return *buffer_factory_; } + const ScopeTrackedObject* setTrackedObject(const ScopeTrackedObject* object) override { + const ScopeTrackedObject* return_object = current_object_; + current_object_ = object; + return return_object; + } + + // FatalErrorInterface + void onFatalError() const override { + // Dump the state of the tracked object if it is in the current thread. This generally results + // in dumping the active state only for the thread which caused the fatal error. + if (isThreadSafe()) { + if (current_object_) { + current_object_->dumpState(std::cerr); + } + } + } private: + TimerPtr createTimerInternal(TimerCb cb); void runPostCallbacks(); // Validate that an operation is thread safe, i.e. it's invoked on the same thread that the @@ -90,6 +111,7 @@ class DispatcherImpl : Logger::Loggable, public Dispatcher { std::vector* current_to_delete_; Thread::MutexBasicLockable post_lock_; std::list> post_callbacks_ GUARDED_BY(post_lock_); + const ScopeTrackedObject* current_object_{}; bool deferred_deleting_{}; }; diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 63ee261385d7..ce3737bef977 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -148,6 +148,7 @@ envoy_cc_library( ":utility_lib", "//include/envoy/access_log:access_log_interface", "//include/envoy/buffer:buffer_interface", + "//include/envoy/common:scope_tracker_interface", "//include/envoy/common:time_interface", "//include/envoy/event:deferred_deletable", "//include/envoy/event:dispatcher_interface", @@ -171,9 +172,11 @@ envoy_cc_library( "//source/common/access_log:access_log_formatter_lib", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", + "//source/common/common:dump_state_utils", "//source/common/common:empty_string", "//source/common/common:enum_to_int", "//source/common/common:linked_object", + "//source/common/common:scope_tracker", "//source/common/common:utility_lib", "//source/common/http/http1:codec_lib", "//source/common/http/http2:codec_lib", @@ -214,6 +217,7 @@ envoy_cc_library( ":headers_lib", "//include/envoy/http:header_map_interface", "//source/common/common:assert_lib", + "//source/common/common:dump_state_utils", "//source/common/common:empty_string", "//source/common/common:non_copyable", "//source/common/common:utility_lib", diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 0c5a221b7bf9..8e905ac5aba0 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -21,6 +21,7 @@ #include "common/common/empty_string.h" #include "common/common/enum_to_int.h" #include "common/common/fmt.h" +#include "common/common/scope_tracker.h" #include "common/common/utility.h" #include "common/http/codes.h" #include "common/http/conn_manager_utility.h" @@ -416,6 +417,9 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect connection_manager_.stats_.named_.downstream_rq_time_, connection_manager_.timeSource())), stream_info_(connection_manager_.codec_->protocol(), connection_manager_.timeSource()), upstream_options_(std::make_shared()) { + ScopeTrackerScopeState scope(this, + connection_manager_.read_callbacks_->connection().dispatcher()); + connection_manager_.stats_.named_.downstream_rq_total_.inc(); connection_manager_.stats_.named_.downstream_rq_active_.inc(); if (connection_manager_.codec_->protocol() == Protocol::Http2) { @@ -585,6 +589,8 @@ const Network::Connection* ConnectionManagerImpl::ActiveStream::connection() { // TODO(alyssawilk) all the calls here should be audited for order priority, // e.g. many early returns do not currently handle connection: close properly. void ConnectionManagerImpl::ActiveStream::decodeHeaders(HeaderMapPtr&& headers, bool end_stream) { + ScopeTrackerScopeState scope(this, + connection_manager_.read_callbacks_->connection().dispatcher()); request_headers_ = std::move(headers); if (Http::Headers::get().MethodValues.Head == request_headers_->Method()->value().getStringView()) { @@ -881,6 +887,8 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(ActiveStreamDecoderFilte } void ConnectionManagerImpl::ActiveStream::decodeData(Buffer::Instance& data, bool end_stream) { + ScopeTrackerScopeState scope(this, + connection_manager_.read_callbacks_->connection().dispatcher()); maybeEndDecode(end_stream); stream_info_.addBytesReceived(data.length()); @@ -890,6 +898,8 @@ void ConnectionManagerImpl::ActiveStream::decodeData(Buffer::Instance& data, boo void ConnectionManagerImpl::ActiveStream::decodeData( ActiveStreamDecoderFilter* filter, Buffer::Instance& data, bool end_stream, FilterIterationStartState filter_iteration_start_state) { + ScopeTrackerScopeState scope(this, + connection_manager_.read_callbacks_->connection().dispatcher()); resetIdleTimer(); // If we previously decided to decode only the headers, do nothing here. @@ -1029,6 +1039,8 @@ void ConnectionManagerImpl::ActiveStream::addDecodedData(ActiveStreamDecoderFilt } void ConnectionManagerImpl::ActiveStream::decodeTrailers(HeaderMapPtr&& trailers) { + ScopeTrackerScopeState scope(this, + connection_manager_.read_callbacks_->connection().dispatcher()); resetIdleTimer(); maybeEndDecode(true); request_trailers_ = std::move(trailers); diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 1e5b1901421e..a7ef3ef9eae8 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -9,6 +9,7 @@ #include #include "envoy/access_log/access_log.h" +#include "envoy/common/scope_tracker.h" #include "envoy/event/deferred_deletable.h" #include "envoy/http/codec.h" #include "envoy/http/codes.h" @@ -27,6 +28,7 @@ #include "envoy/upstream/upstream.h" #include "common/buffer/watermark_buffer.h" +#include "common/common/dump_state_utils.h" #include "common/common/linked_object.h" #include "common/grpc/common.h" #include "common/http/conn_manager_config.h" @@ -334,9 +336,10 @@ class ConnectionManagerImpl : Logger::Loggable, public StreamCallbacks, public StreamDecoder, public FilterChainFactoryCallbacks, - public Tracing::Config { + public Tracing::Config, + public ScopeTrackedObject { ActiveStream(ConnectionManagerImpl& connection_manager); - ~ActiveStream(); + ~ActiveStream() override; // Indicates which filter to start the iteration with. enum class FilterIterationStartState { AlwaysStartFromNext, CanStartFromCurrent }; @@ -417,6 +420,20 @@ class ConnectionManagerImpl : Logger::Loggable, const std::vector& requestHeadersForTags() const override; bool verbose() const override; + // ScopeTrackedObject + void dumpState(std::ostream& os, int indent_level = 0) const override { + const char* spaces = spacesForLevel(indent_level); + os << spaces << "ActiveStream " << this << DUMP_MEMBER(stream_id_) + << DUMP_MEMBER(has_continue_headers_) << DUMP_MEMBER(is_head_request_) + << DUMP_MEMBER(decoding_headers_only_) << DUMP_MEMBER(encoding_headers_only_) << "\n"; + + DUMP_DETAILS(request_headers_); + DUMP_DETAILS(request_trailers_); + DUMP_DETAILS(response_headers_); + DUMP_DETAILS(response_trailers_); + DUMP_DETAILS(&stream_info_); + } + void traceRequest(); void refreshCachedRoute(); @@ -494,6 +511,11 @@ class ConnectionManagerImpl : Logger::Loggable, bool hasCachedRoute() { return cached_route_.has_value() && cached_route_.value(); } + friend std::ostream& operator<<(std::ostream& os, const ActiveStream& s) { + s.dumpState(os); + return os; + } + ConnectionManagerImpl& connection_manager_; Router::ConfigConstSharedPtr snapped_route_config_; Router::ScopedConfigConstSharedPtr snapped_scoped_route_config_; diff --git a/source/common/http/header_map_impl.cc b/source/common/http/header_map_impl.cc index 2ceafa694186..ff018b77f9c6 100644 --- a/source/common/http/header_map_impl.cc +++ b/source/common/http/header_map_impl.cc @@ -6,6 +6,7 @@ #include #include "common/common/assert.h" +#include "common/common/dump_state_utils.h" #include "common/common/empty_string.h" #include "common/common/utility.h" #include "common/singleton/const_singleton.h" @@ -554,6 +555,20 @@ void HeaderMapImpl::removePrefix(const LowerCaseString& prefix) { }); } +void HeaderMapImpl::dumpState(std::ostream& os, int indent_level) const { + using IterateData = std::pair; + const char* spaces = spacesForLevel(indent_level); + IterateData iterate_data = std::make_pair(&os, spaces); + iterate( + [](const HeaderEntry& header, void* context) -> HeaderMap::Iterate { + auto* data = static_cast(context); + *data->first << data->second << "'" << header.key().getStringView() << "', '" + << header.value().getStringView() << "'\n"; + return HeaderMap::Iterate::Continue; + }, + &iterate_data); +} + HeaderMapImpl::HeaderEntryImpl& HeaderMapImpl::maybeCreateInline(HeaderEntryImpl** entry, const LowerCaseString& key) { if (*entry) { diff --git a/source/common/http/header_map_impl.h b/source/common/http/header_map_impl.h index 51b319499722..f7d3a66937f8 100644 --- a/source/common/http/header_map_impl.h +++ b/source/common/http/header_map_impl.h @@ -81,6 +81,7 @@ class HeaderMapImpl : public HeaderMap, NonCopyable { void removePrefix(const LowerCaseString& key) override; size_t size() const override { return headers_.size(); } bool empty() const override { return headers_.empty(); } + void dumpState(std::ostream& os, int indent_level = 0) const override; protected: // For tests only, unoptimized, they aren't intended for regular HeaderMapImpl users. diff --git a/source/common/signal/BUILD b/source/common/signal/BUILD new file mode 100644 index 000000000000..17dec6c9be55 --- /dev/null +++ b/source/common/signal/BUILD @@ -0,0 +1,28 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "fatal_error_handler_lib", + hdrs = ["fatal_error_handler.h"], +) + +envoy_cc_library( + name = "sigaction_lib", + srcs = ["signal_action.cc"], + hdrs = ["signal_action.h"], + tags = ["backtrace"], + deps = [ + ":fatal_error_handler_lib", + "//source/common/common:assert_lib", + "//source/common/common:non_copyable", + "//source/common/singleton:threadsafe_singleton", + "//source/server:backtrace_lib", + ], +) diff --git a/source/common/signal/fatal_error_handler.h b/source/common/signal/fatal_error_handler.h new file mode 100644 index 000000000000..95c185911d3e --- /dev/null +++ b/source/common/signal/fatal_error_handler.h @@ -0,0 +1,18 @@ +#pragma once + +#include "envoy/common/pure.h" + +namespace Envoy { + +// A simple class which allows registering functions to be called when Envoy +// receives one of the fatal signals, documented below. +// +// This is split out of signal_action.h because it is exempted from various +// builds. +class FatalErrorHandlerInterface { +public: + virtual ~FatalErrorHandlerInterface() = default; + virtual void onFatalError() const PURE; +}; + +} // namespace Envoy diff --git a/source/exe/signal_action.cc b/source/common/signal/signal_action.cc similarity index 60% rename from source/exe/signal_action.cc rename to source/common/signal/signal_action.cc index 00cfaf5d2c40..edc9dca40be9 100644 --- a/source/exe/signal_action.cc +++ b/source/common/signal/signal_action.cc @@ -1,4 +1,4 @@ -#include "exe/signal_action.h" +#include "common/signal/signal_action.h" #include @@ -7,6 +7,47 @@ #include "common/common/assert.h" namespace Envoy { + +ABSL_CONST_INIT static absl::Mutex failure_mutex(absl::kConstInit); +// Since we can't grab the failure mutex on fatal error (snagging locks under +// fatal crash causing potential deadlocks) access the handler list as an atomic +// operation, to minimize the chance that one thread is operating on the list +// while the crash handler is attempting to access it. +// This basically makes edits to the list thread-safe - if one thread is +// modifying the list rather than crashing in the crash handler due to accessing +// the list in a non-thread-safe manner, it simply won't log crash traces. +using FailureFunctionList = std::list; +ABSL_CONST_INIT std::atomic fatal_error_handlers{nullptr}; + +void SignalAction::registerFatalErrorHandler(const FatalErrorHandlerInterface& handler) { +#ifdef ENVOY_OBJECT_TRACE_ON_DUMP + absl::MutexLock l(&failure_mutex); + FailureFunctionList* list = fatal_error_handlers.exchange(nullptr, std::memory_order_relaxed); + if (list == nullptr) { + list = new FailureFunctionList; + } + list->push_back(&handler); + fatal_error_handlers.store(list, std::memory_order_release); +#else + UNREFERENCED_PARAMETER(handler); +#endif +} + +void SignalAction::removeFatalErrorHandler(const FatalErrorHandlerInterface& handler) { +#ifdef ENVOY_OBJECT_TRACE_ON_DUMP + absl::MutexLock l(&failure_mutex); + FailureFunctionList* list = fatal_error_handlers.exchange(nullptr, std::memory_order_relaxed); + list->remove(&handler); + if (list->empty()) { + delete list; + } else { + fatal_error_handlers.store(list, std::memory_order_release); + } +#else + UNREFERENCED_PARAMETER(handler); +#endif +} + constexpr int SignalAction::FATAL_SIGS[]; void SignalAction::sigHandler(int sig, siginfo_t* info, void* context) { @@ -20,6 +61,14 @@ void SignalAction::sigHandler(int sig, siginfo_t* info, void* context) { } tracer.logTrace(); + FailureFunctionList* list = fatal_error_handlers.exchange(nullptr, std::memory_order_relaxed); + if (list) { + // Finally after logging the stack trace, call any registered crash handlers. + for (const auto* handler : *list) { + handler->onFatalError(); + } + } + signal(sig, SIG_DFL); raise(sig); } diff --git a/source/exe/signal_action.h b/source/common/signal/signal_action.h similarity index 87% rename from source/exe/signal_action.h rename to source/common/signal/signal_action.h index 7af81167c5d0..12fcb1805a7b 100644 --- a/source/exe/signal_action.h +++ b/source/common/signal/signal_action.h @@ -4,12 +4,15 @@ #include #include +#include #include "common/common/non_copyable.h" +#include "common/signal/fatal_error_handler.h" #include "server/backtrace.h" namespace Envoy { + /** * This class installs signal handlers for fatal signal types. * @@ -50,8 +53,7 @@ class SignalAction : NonCopyable { public: SignalAction() : guard_size_(sysconf(_SC_PAGE_SIZE)), - altstack_size_(std::max(guard_size_ * 4, static_cast(MINSIGSTKSZ))), - altstack_(nullptr) { + altstack_size_(std::max(guard_size_ * 4, static_cast(MINSIGSTKSZ))), altstack_() { mapAndProtectStackMemory(); installSigHandlers(); } @@ -71,6 +73,18 @@ class SignalAction : NonCopyable { */ static void sigHandler(int sig, siginfo_t* info, void* context); + /** + * Add this handler to the list of functions which will be called if Envoy + * receives a fatal signal. + */ + static void registerFatalErrorHandler(const FatalErrorHandlerInterface& handler); + + /** + * Removes this handler from the list of functions which will be called if Envoy + * receives a fatal signal. + */ + static void removeFatalErrorHandler(const FatalErrorHandlerInterface& handler); + private: /** * Allocate this many bytes on each side of the area used for alt stack. @@ -125,8 +139,10 @@ class SignalAction : NonCopyable { * Unmap alternative stack memory. */ void unmapStackMemory(); - char* altstack_; + char* altstack_{}; std::array previous_handlers_; stack_t previous_altstack_; + std::list fatal_error_handlers_; }; + } // namespace Envoy diff --git a/source/common/stream_info/BUILD b/source/common/stream_info/BUILD index 639805f711f6..b80bdc6d4c06 100644 --- a/source/common/stream_info/BUILD +++ b/source/common/stream_info/BUILD @@ -15,6 +15,7 @@ envoy_cc_library( ":filter_state_lib", "//include/envoy/stream_info:stream_info_interface", "//source/common/common:assert_lib", + "//source/common/common:dump_state_utils", ], ) diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index d435a060e278..b434991af40b 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -7,6 +7,7 @@ #include "envoy/stream_info/stream_info.h" #include "common/common/assert.h" +#include "common/common/dump_state_utils.h" #include "common/stream_info/filter_state_impl.h" namespace Envoy { @@ -208,6 +209,13 @@ struct StreamInfoImpl : public StreamInfo { return upstream_transport_failure_reason_; } + void dumpState(std::ostream& os, int indent_level = 0) const { + const char* spaces = spacesForLevel(indent_level); + os << spaces << "StreamInfoImpl " << this << DUMP_OPTIONAL_MEMBER(protocol_) + << DUMP_OPTIONAL_MEMBER(response_code_) << DUMP_OPTIONAL_MEMBER(response_code_details_) + << DUMP_MEMBER(health_check_request_) << DUMP_MEMBER(route_name_) << "\n"; + } + TimeSource& time_source_; const SystemTime start_time_; const MonotonicTime start_time_monotonic_; diff --git a/source/exe/BUILD b/source/exe/BUILD index 2a1873e019ca..4117a4bb83ca 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -77,7 +77,7 @@ envoy_cc_library( ] + select({ "//bazel:disable_signal_trace": [], "//conditions:default": [ - ":sigaction_lib", + "//source/common/signal:sigaction_lib", ":terminate_handler_lib", ], }) + envoy_cc_platform_dep("platform_impl_lib"), @@ -117,18 +117,6 @@ envoy_cc_win32_library( ], ) -envoy_cc_library( - name = "sigaction_lib", - srcs = ["signal_action.cc"], - hdrs = ["signal_action.h"], - tags = ["backtrace"], - deps = [ - "//source/common/common:assert_lib", - "//source/common/common:non_copyable", - "//source/server:backtrace_lib", - ], -) - envoy_cc_library( name = "terminate_handler_lib", srcs = ["terminate_handler.cc"], diff --git a/source/exe/main_common.h b/source/exe/main_common.h index 3a49d8979099..a8944fd69b34 100644 --- a/source/exe/main_common.h +++ b/source/exe/main_common.h @@ -17,7 +17,7 @@ #include "server/server.h" #ifdef ENVOY_HANDLE_SIGNALS -#include "exe/signal_action.h" +#include "common/signal/signal_action.h" #include "exe/terminate_handler.h" #endif diff --git a/test/BUILD b/test/BUILD index fa80bf514d36..4c2e42990737 100644 --- a/test/BUILD +++ b/test/BUILD @@ -38,6 +38,6 @@ envoy_cc_test_library( "//test/test_common:printers_lib", ] + select({ "//bazel:disable_signal_trace": [], - "//conditions:default": ["//source/exe:sigaction_lib"], + "//conditions:default": ["//source/common/signal:sigaction_lib"], }), ) diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 4a407492b78f..ce4e8af46284 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -51,6 +51,7 @@ using testing::AnyNumber; using testing::AtLeast; using testing::DoAll; using testing::Eq; +using testing::HasSubstr; using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; @@ -4294,5 +4295,67 @@ TEST_F(HttpConnectionManagerImplTest, DisableKeepAliveWhenDraining) { Buffer::OwnedImpl fake_input; conn_manager_->onData(fake_input, false); } + +TEST_F(HttpConnectionManagerImplTest, TestSessionTrace) { + setup(false, ""); + + // Set up the codec. + EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance& data) -> void { + data.drain(4); + })); + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + + setupFilterChain(1, 1); + + // Create a new stream + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + + // Send headers to that stream, and verify we both set and clear the tracked object. + { + HeaderMapPtr headers{ + new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "POST"}}}; + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, setTrackedObject(_)) + .Times(2) + .WillOnce(Invoke([](const ScopeTrackedObject* object) -> const ScopeTrackedObject* { + ASSERT(object != nullptr); // On the first call, this should be the active stream. + std::stringstream out; + object->dumpState(out); + std::string state = out.str(); + EXPECT_THAT(state, testing::HasSubstr("request_headers_: null")); + EXPECT_THAT(state, testing::HasSubstr("protocol_: 1")); + return nullptr; + })) + .WillRepeatedly(Return(nullptr)); + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) + .WillOnce(Invoke([](HeaderMap&, bool) -> FilterHeadersStatus { + return FilterHeadersStatus::StopIteration; + })); + decoder->decodeHeaders(std::move(headers), false); + } + + // Send trailers to that stream, and verify by this point headers are in logged state. + { + HeaderMapPtr trailers{new TestHeaderMapImpl{{"foo", "bar"}}}; + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, setTrackedObject(_)) + .Times(2) + .WillOnce(Invoke([](const ScopeTrackedObject* object) -> const ScopeTrackedObject* { + ASSERT(object != nullptr); // On the first call, this should be the active stream. + std::stringstream out; + object->dumpState(out); + std::string state = out.str(); + EXPECT_THAT(state, testing::HasSubstr("request_headers_: \n")); + EXPECT_THAT(state, testing::HasSubstr("':authority', 'host'\n")); + EXPECT_THAT(state, testing::HasSubstr("protocol_: 1")); + return nullptr; + })) + .WillRepeatedly(Return(nullptr)); + EXPECT_CALL(*decoder_filters_[0], decodeComplete()); + EXPECT_CALL(*decoder_filters_[0], decodeTrailers(_)) + .WillOnce(Return(FilterTrailersStatus::StopIteration)); + decoder->decodeTrailers(std::move(trailers)); + } +} + } // namespace Http } // namespace Envoy diff --git a/test/common/signal/BUILD b/test/common/signal/BUILD new file mode 100644 index 000000000000..c29cfd2a56b3 --- /dev/null +++ b/test/common/signal/BUILD @@ -0,0 +1,19 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +envoy_package() + +envoy_cc_test( + name = "signals_test", + srcs = ["signals_test.cc"], + tags = ["backtrace"], + deps = [ + "//source/common/signal:sigaction_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/exe/signals_test.cc b/test/common/signal/signals_test.cc similarity index 84% rename from test/exe/signals_test.cc rename to test/common/signal/signals_test.cc index eb07fe61370a..72b28c7bbf8b 100644 --- a/test/exe/signals_test.cc +++ b/test/common/signal/signals_test.cc @@ -1,7 +1,7 @@ #include #include -#include "exe/signal_action.h" +#include "common/signal/signal_action.h" #include "test/test_common/utility.h" @@ -34,6 +34,25 @@ TEST(SignalsDeathTest, InvalidAddressDeathTest) { "backtrace.*Segmentation fault"); } +class TestFatalErrorHandler : public FatalErrorHandlerInterface { + virtual void onFatalError() const override { std::cerr << "HERE!"; } +}; + +TEST(SignalsDeathTest, RegisteredHandlerTest) { + TestFatalErrorHandler handler; + SignalAction::registerFatalErrorHandler(handler); + SignalAction actions; + // Make sure the fatal error log "HERE" registered above is logged on fatal error. + EXPECT_DEATH_LOG_TO_STDERR( + []() -> void { + // Oops! + volatile int* nasty_ptr = reinterpret_cast(0x0); + *(nasty_ptr) = 0; + }(), + "HERE"); + SignalAction::removeFatalErrorHandler(handler); +} + TEST(SignalsDeathTest, BusDeathTest) { SignalAction actions; EXPECT_DEATH_LOG_TO_STDERR( @@ -90,6 +109,7 @@ TEST(SignalsDeathTest, RestoredPreviousHandlerDeathTest) { // Outer SignalAction should be active again: EXPECT_DEATH_LOG_TO_STDERR([]() -> void { abort(); }(), "backtrace.*Abort(ed)?"); } + #endif TEST(SignalsDeathTest, IllegalStackAccessDeathTest) { diff --git a/test/common/stream_info/stream_info_impl_test.cc b/test/common/stream_info/stream_info_impl_test.cc index fec622ddc810..380f60ffcebb 100644 --- a/test/common/stream_info/stream_info_impl_test.cc +++ b/test/common/stream_info/stream_info_impl_test.cc @@ -201,6 +201,20 @@ TEST_F(StreamInfoImplTest, DynamicMetadataTest) { EXPECT_TRUE(json.find("\"another_key\":\"another_value\"") != std::string::npos); } +TEST_F(StreamInfoImplTest, DumpStateTest) { + StreamInfoImpl stream_info(Http::Protocol::Http2, test_time_.timeSystem()); + std::string prefix = ""; + + for (int i = 0; i < 7; ++i) { + std::stringstream out; + stream_info.dumpState(out, i); + std::string state = out.str(); + EXPECT_TRUE(absl::StartsWith(state, prefix)); + EXPECT_THAT(state, testing::HasSubstr("protocol_: 2")); + prefix = prefix + " "; + } +} + } // namespace } // namespace StreamInfo } // namespace Envoy diff --git a/test/exe/BUILD b/test/exe/BUILD index 641b1c9efc60..12235850ca35 100644 --- a/test/exe/BUILD +++ b/test/exe/BUILD @@ -52,16 +52,6 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "signals_test", - srcs = ["signals_test.cc"], - tags = ["backtrace"], - deps = [ - "//source/exe:sigaction_lib", - "//test/test_common:utility_lib", - ], -) - envoy_cc_test( name = "terminate_handler_test", srcs = ["terminate_handler_test.cc"], diff --git a/test/exe/main_common_test.cc b/test/exe/main_common_test.cc index 44c693a91f8e..baeeb243976a 100644 --- a/test/exe/main_common_test.cc +++ b/test/exe/main_common_test.cc @@ -16,7 +16,7 @@ #include "gtest/gtest.h" #ifdef ENVOY_HANDLE_SIGNALS -#include "exe/signal_action.h" +#include "common/signal/signal_action.h" #endif #include "absl/synchronization/notification.h" diff --git a/test/main.cc b/test/main.cc index 83a9d2b72c9f..b7295edd0b02 100644 --- a/test/main.cc +++ b/test/main.cc @@ -8,7 +8,7 @@ #include "absl/debugging/symbolize.h" #ifdef ENVOY_HANDLE_SIGNALS -#include "exe/signal_action.h" +#include "common/signal/signal_action.h" #endif // The main entry point (and the rest of this file) should have no logic in it, diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 4237a3b57ca0..5cbd2f076f5f 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -113,6 +113,7 @@ class MockDispatcher : public Dispatcher { MOCK_METHOD2(listenForSignal_, SignalEvent*(int signal_num, SignalCb cb)); MOCK_METHOD1(post, void(std::function callback)); MOCK_METHOD1(run, void(RunType type)); + MOCK_METHOD1(setTrackedObject, const ScopeTrackedObject*(const ScopeTrackedObject* object)); Buffer::WatermarkFactory& getWatermarkFactory() override { return buffer_factory_; } GlobalTimeSystem time_system_; From 4af0c54f33a68ac6b024ad22f111a21597de8e1d Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Fri, 5 Jul 2019 09:58:33 -0700 Subject: [PATCH 111/542] clang-tidy: performance-inefficient-vector-operation (#7471) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + source/server/http/admin.cc | 1 + test/integration/integration.cc | 1 + test/server/options_impl_test.cc | 6 +++--- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 264254148fbe..27e50fdd51d2 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -24,6 +24,7 @@ WarningsAsErrors: 'abseil-duration-*, modernize-use-using, performance-faster-string-find, performance-for-range-copy, + performance-inefficient-vector-operation, performance-noexcept-move-constructor, performance-unnecessary-copy-initialization, readability-braces-around-statements, diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 746b5c7c1e48..336c8694e5d2 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -825,6 +825,7 @@ std::string PrometheusStatsFormatter::sanitizeName(const std::string& name) { std::string PrometheusStatsFormatter::formattedTags(const std::vector& tags) { std::vector buf; + buf.reserve(tags.size()); for (const Stats::Tag& tag : tags) { buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), tag.value_)); } diff --git a/test/integration/integration.cc b/test/integration/integration.cc index 28e5eb47827c..a34d52b6a9ca 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -341,6 +341,7 @@ void BaseIntegrationTest::createEnvoy() { std::vector named_ports; const auto& static_resources = config_helper_.bootstrap().static_resources(); + named_ports.reserve(static_resources.listeners_size()); for (int i = 0; i < static_resources.listeners_size(); ++i) { named_ports.push_back(static_resources.listeners(i).name()); } diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index 0947d7c41cdd..e3dbe1dfd7af 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -37,9 +38,8 @@ class OptionsImplTest : public testing::Test { std::unique_ptr createOptionsImpl(const std::string& args) { std::vector words = TestUtility::split(args, ' '); std::vector argv; - for (const std::string& s : words) { - argv.push_back(s.c_str()); - } + std::transform(words.cbegin(), words.cend(), std::back_inserter(argv), + [](const std::string& arg) { return arg.c_str(); }); return std::make_unique( argv.size(), argv.data(), [](bool) { return "1"; }, spdlog::level::warn); } From 4de864e8070131fd76249ce0437fb411e51a34b1 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Fri, 5 Jul 2019 09:58:57 -0700 Subject: [PATCH 112/542] clang-tidy: readability-redundant-control-flow (#7472) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + source/extensions/filters/network/dubbo_proxy/hessian_utils.cc | 1 - test/common/network/dns_impl_test.cc | 1 - test/integration/filters/udp_listener_echo_filter.cc | 2 -- 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 27e50fdd51d2..d58e0a8be486 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -29,6 +29,7 @@ WarningsAsErrors: 'abseil-duration-*, performance-unnecessary-copy-initialization, readability-braces-around-statements, readability-container-size-empty, + readability-redundant-control-flow, readability-redundant-member-init, readability-redundant-smartptr-get, readability-redundant-string-cstr' diff --git a/source/extensions/filters/network/dubbo_proxy/hessian_utils.cc b/source/extensions/filters/network/dubbo_proxy/hessian_utils.cc index 7d37531d7149..c9559f1652be 100644 --- a/source/extensions/filters/network/dubbo_proxy/hessian_utils.cc +++ b/source/extensions/filters/network/dubbo_proxy/hessian_utils.cc @@ -458,7 +458,6 @@ void HessianUtils::readNull(Buffer::Instance& buffer) { size_t size; peekNull(buffer, &size); buffer.drain(size); - return; } std::chrono::milliseconds HessianUtils::peekDate(Buffer::Instance& buffer, size_t* size, diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index 78ad6b566524..2909bcb7f85f 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -237,7 +237,6 @@ class TestDnsServerQuery { buffer_.drain(size_); size_ = 0; } - return; } TestDnsServerQuery& parent_; diff --git a/test/integration/filters/udp_listener_echo_filter.cc b/test/integration/filters/udp_listener_echo_filter.cc index af3b9810d6ea..74f11a81a2a2 100644 --- a/test/integration/filters/udp_listener_echo_filter.cc +++ b/test/integration/filters/udp_listener_echo_filter.cc @@ -33,8 +33,6 @@ void UdpListenerEchoFilter::onData(Network::UdpRecvData& data) { auto send_result = read_callbacks_->udpListener().send(send_data); ASSERT(send_result.ok()); - - return; } /** From 7394ff1f77afd72866528fceb593d0f8b78d2dc6 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Fri, 5 Jul 2019 09:59:38 -0700 Subject: [PATCH 113/542] [test] add config test for zookeeper (#7470) Signed-off-by: Derek Argueta --- .../filters/network/zookeeper_proxy/BUILD | 16 ++++- .../network/zookeeper_proxy/config_test.cc | 65 +++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 test/extensions/filters/network/zookeeper_proxy/config_test.cc diff --git a/test/extensions/filters/network/zookeeper_proxy/BUILD b/test/extensions/filters/network/zookeeper_proxy/BUILD index bafa67b9d776..dd96daf08d29 100644 --- a/test/extensions/filters/network/zookeeper_proxy/BUILD +++ b/test/extensions/filters/network/zookeeper_proxy/BUILD @@ -2,14 +2,11 @@ licenses(["notice"]) # Apache 2 load( "//bazel:envoy_build_system.bzl", - "envoy_cc_test_library", "envoy_package", ) load( "//test/extensions:extensions_build_system.bzl", - "envoy_extension_cc_mock", "envoy_extension_cc_test", - "envoy_extension_cc_test_library", ) envoy_package() @@ -25,3 +22,16 @@ envoy_extension_cc_test( "//test/mocks/network:network_mocks", ], ) + +envoy_extension_cc_test( + name = "config_test", + srcs = [ + "config_test.cc", + ], + extension_name = "envoy.filters.network.zookeeper_proxy", + deps = [ + "//source/extensions/filters/network/zookeeper_proxy:config", + "//test/mocks/server:server_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/network/zookeeper_proxy/config_test.cc b/test/extensions/filters/network/zookeeper_proxy/config_test.cc new file mode 100644 index 000000000000..a05f63a77cd0 --- /dev/null +++ b/test/extensions/filters/network/zookeeper_proxy/config_test.cc @@ -0,0 +1,65 @@ +#include "envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.pb.validate.h" + +#include "extensions/filters/network/zookeeper_proxy/config.h" + +#include "test/mocks/server/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace ZooKeeperProxy { + +using ZooKeeperProxyProtoConfig = + envoy::config::filter::network::zookeeper_proxy::v1alpha1::ZooKeeperProxy; + +TEST(ZookeeperFilterConfigTest, ValidateFail) { + testing::NiceMock context; + EXPECT_THROW( + ZooKeeperConfigFactory().createFilterFactoryFromProto(ZooKeeperProxyProtoConfig(), context), + ProtoValidationException); +} + +TEST(ZookeeperFilterConfigTest, InvalidStatPrefix) { + const std::string yaml = R"EOF( +stat_prefix: "" + )EOF"; + + ZooKeeperProxyProtoConfig proto_config; + EXPECT_THROW(TestUtility::loadFromYamlAndValidate(yaml, proto_config), ProtoValidationException); +} + +TEST(ZookeeperFilterConfigTest, InvalidMaxPacketBytes) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix +max_packet_bytes: -1 + )EOF"; + + ZooKeeperProxyProtoConfig proto_config; + EXPECT_THROW(TestUtility::loadFromYamlAndValidate(yaml, proto_config), EnvoyException); +} + +TEST(ZookeeperFilterConfigTest, SimpleConfig) { + const std::string yaml = R"EOF( +stat_prefix: test_prefix + )EOF"; + + ZooKeeperProxyProtoConfig proto_config; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + + testing::NiceMock context; + ZooKeeperConfigFactory factory; + + Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); + Network::MockConnection connection; + EXPECT_CALL(connection, addFilter(_)); + cb(connection); +} + +} // namespace ZooKeeperProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file From d2bed97c607a64d3cf61d68b68d6fb9d1a2f76dc Mon Sep 17 00:00:00 2001 From: Alexey Baranov Date: Sat, 6 Jul 2019 02:03:55 +0300 Subject: [PATCH 114/542] Fix segfault in grpc upstream log reporting. (#7473) StreamInfoImpl::downstream_ssl_info_ was left uninitilaized in UpstreamRequest::stream_info_, and null check incorrectly passed for a pointer pointing nowhere. Risk Level: Low Testing: Surprisingly, all tests use NiceMock for StreamInfo which returns initialized null pointer :) Docs Changes: N/A Release Notes: N/A Signed-off-by: Alexey Baranov --- source/common/stream_info/stream_info_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index b434991af40b..7f7dd70ad3ef 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -243,7 +243,7 @@ struct StreamInfoImpl : public StreamInfo { Network::Address::InstanceConstSharedPtr downstream_local_address_; Network::Address::InstanceConstSharedPtr downstream_direct_remote_address_; Network::Address::InstanceConstSharedPtr downstream_remote_address_; - const Ssl::ConnectionInfo* downstream_ssl_info_; + const Ssl::ConnectionInfo* downstream_ssl_info_{}; std::string requested_server_name_; UpstreamTiming upstream_timing_; std::string upstream_transport_failure_reason_; From 7032710f1494e5da98c582545d11929009a5a832 Mon Sep 17 00:00:00 2001 From: Jyoti Mahapatra <49211422+jyotimahapatra@users.noreply.github.com> Date: Sun, 7 Jul 2019 20:47:00 -0700 Subject: [PATCH 115/542] Add support in route checker tool for runtimes in routes (#7418) Signed-off-by: Jyoti Mahapatra --- .../root/configuration/tools/router_check.rst | 113 +++++++----------- .../install/tools/route_table_check_tool.rst | 3 - test/tools/router_check/router.cc | 17 ++- test/tools/router_check/router.h | 6 +- .../test/config/Runtime.golden.proto.json | 47 ++++++++ .../router_check/test/config/Runtime.yaml | 18 +++ test/tools/router_check/test/route_tests.sh | 1 + test/tools/router_check/validation.proto | 5 + 8 files changed, 135 insertions(+), 75 deletions(-) create mode 100644 test/tools/router_check/test/config/Runtime.golden.proto.json create mode 100644 test/tools/router_check/test/config/Runtime.yaml diff --git a/docs/root/configuration/tools/router_check.rst b/docs/root/configuration/tools/router_check.rst index fbbf2e8dda0b..7698430b03db 100644 --- a/docs/root/configuration/tools/router_check.rst +++ b/docs/root/configuration/tools/router_check.rst @@ -32,66 +32,38 @@ Validate A simple tool configuration json has one test case and is written as follows. The test expects a cluster name match of "instant-server".:: - [ - { - "test_name: "Cluster_name_test", - "input": - { - ":authority":"api.lyft.com", - ":path": "/api/locations" - }, - "validate": - { - "cluster_name": "instant-server" - } - } - ] - -.. code-block:: json - - [ - { - "test_name": "...", - "input": - { - ":authority": "...", - ":path": "...", - ":method": "...", - "internal" : "...", - "random_value" : "...", - "ssl" : "...", - "additional_headers": [ - { - "field": "...", - "value": "..." - }, - { - "..." - } - ] - }, - "validate": { - "cluster_name": "...", - "virtual_cluster_name": "...", - "virtual_host_name": "...", - "host_rewrite": "...", - "path_rewrite": "...", - "path_redirect": "...", - "header_fields" : [ - { - "field": "...", - "value": "..." - }, - { - "..." - } - ] - } - }, - { - "..." - } - ] + tests + - test_name: Cluster_name_test, + input: + authority: api.lyft.com, + path: /api/locations + validate: + cluster_name: instant-server + +.. code-block:: yaml + + tests + - test_name: ..., + input: + authority: ..., + path": ..., + method": ..., + internal" : ..., + random_value" : ..., + ssl" : ..., + - additional_headers: + key: ..., + value: ... + validate: + cluster_name: ..., + virtual_cluster_name: ..., + virtual_host_name: ..., + host_rewrite: ..., + path_rewrite: ..., + path_redirect: ..., + - header_fields: + key: ..., + value: ... test_name *(required, string)* The name of a test object. @@ -99,15 +71,15 @@ test_name input *(required, object)* Input values sent to the router that determine the returned route. - :authority + authority *(required, string)* The url authority. This value along with the path parameter define the url to be matched. An example authority value is "api.lyft.com". - :path + path *(required, string)* The url path. An example path value is "/foo". - :method - *(optional, string)* The request method. If not specified, the default method is GET. The options + method + *(required, string)* The request method. If not specified, the default method is GET. The options are GET, PUT, or POST. internal @@ -115,7 +87,8 @@ input If not specified, or if internal is equal to false, x-envoy-internal is not set. random_value - *(optional, integer)* An integer used to identify the target for weighted cluster selection. + *(optional, integer)* An integer used to identify the target for weighted cluster selection + and as a factor for the routing engine to decide whether a runtime based route takes effect. The default value of random_value is 0. ssl @@ -125,11 +98,11 @@ input x-forwarded-proto set to http. additional_headers - *(optional, array)* Additional headers to be added as input for route determination. The ":authority", - ":path", ":method", "x-forwarded-proto", and "x-envoy-internal" fields are specified by the other config + *(optional, array)* Additional headers to be added as input for route determination. The "authority", + "path", "method", "x-forwarded-proto", and "x-envoy-internal" fields are specified by the other config options and should not be set here. - field + key *(required, string)* The name of the header field to add. value @@ -159,11 +132,11 @@ validate *(optional, string)* Match the returned redirect path. header_fields - *(optional, array)* Match the listed header fields. Examples header fields include the ":path", "cookie", + *(optional, array)* Match the listed header fields. Examples header fields include the "path", "cookie", and "date" fields. The header fields are checked after all other test cases. Thus, the header fields checked will be those of the redirected or rewritten routes when applicable. - field + key *(required, string)* The name of the header field to match. value diff --git a/docs/root/install/tools/route_table_check_tool.rst b/docs/root/install/tools/route_table_check_tool.rst index 9210aae416c0..ac0b523eec09 100644 --- a/docs/root/install/tools/route_table_check_tool.rst +++ b/docs/root/install/tools/route_table_check_tool.rst @@ -67,9 +67,6 @@ Output locations ats cluster_name Test_6 - Testing with valid :ref:`runtime values ` is not currently supported, - this may be added in future work. - Building The tool can be built locally using Bazel. :: diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index a8359d4a6cb4..74b1437fca59 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -82,7 +82,12 @@ RouterCheckTool::RouterCheckTool( std::unique_ptr config, std::unique_ptr stats, Api::ApiPtr api) : factory_context_(std::move(factory_context)), config_(std::move(config)), - stats_(std::move(stats)), api_(std::move(api)) {} + stats_(std::move(stats)), api_(std::move(api)) { + ON_CALL(factory_context_->runtime_loader_.snapshot_, + featureEnabled(_, testing::An(), + testing::An())) + .WillByDefault(testing::Invoke(this, &RouterCheckTool::runtimeMock)); +} // TODO(jyotima): Remove this code path once the json schema code path is deprecated. bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_json) { @@ -165,6 +170,7 @@ bool RouterCheckTool::compareEntries(const std::string& expected_routes) { bool no_failures = true; for (const envoy::RouterCheckToolSchema::ValidationItem& check_config : validation_config.tests()) { + active_runtime = check_config.input().runtime(); headers_finalized_ = false; ToolConfig tool_config = ToolConfig::create(check_config); tool_config.route_ = config_->route(*tool_config.headers_, tool_config.random_value_); @@ -396,6 +402,15 @@ bool RouterCheckTool::compareResults(const std::string& actual, const std::strin return false; } +// The Mock for runtime value checks. +// This is a simple implementation to mimic the actual runtime checks in Snapshot.featureEnabled +bool RouterCheckTool::runtimeMock(const std::string& key, + const envoy::type::FractionalPercent& default_value, + uint64_t random_value) { + return !active_runtime.empty() && active_runtime.compare(key) == 0 && + ProtobufPercentHelper::evaluateFractionalPercent(default_value, random_value); +} + Options::Options(int argc, char** argv) { TCLAP::CmdLine cmd("router_check_tool", ' ', "none", true); TCLAP::SwitchArg is_proto("p", "useproto", "Use Proto test file schema", cmd, false); diff --git a/test/tools/router_check/router.h b/test/tools/router_check/router.h index 5ff95f8c8efb..026bc1343a9a 100644 --- a/test/tools/router_check/router.h +++ b/test/tools/router_check/router.h @@ -119,7 +119,7 @@ class RouterCheckTool : Logger::Loggable { bool compareCustomHeaderField(ToolConfig& tool_config, const std::string& field, const std::string& expected); bool compareCustomHeaderField(ToolConfig& tool_config, - const envoy::RouterCheckToolSchema::ValidationAssert& expecte); + const envoy::RouterCheckToolSchema::ValidationAssert& expected); /** * Compare the expected and actual route parameter values. Print out match details if details_ * flag is set. @@ -130,6 +130,9 @@ class RouterCheckTool : Logger::Loggable { bool compareResults(const std::string& actual, const std::string& expected, const std::string& test_type); + bool runtimeMock(const std::string& key, const envoy::type::FractionalPercent& default_value, + uint64_t random_value); + bool headers_finalized_{false}; bool details_{false}; @@ -139,6 +142,7 @@ class RouterCheckTool : Logger::Loggable { std::unique_ptr config_; std::unique_ptr stats_; Api::ApiPtr api_; + std::string active_runtime; }; /** diff --git a/test/tools/router_check/test/config/Runtime.golden.proto.json b/test/tools/router_check/test/config/Runtime.golden.proto.json new file mode 100644 index 000000000000..e981f82c8a4f --- /dev/null +++ b/test/tools/router_check/test/config/Runtime.golden.proto.json @@ -0,0 +1,47 @@ +{ + "tests": [ + { + "test_name": "Test_1", + "input": { + "authority": "www.lyft.com", + "path": "/", + "method": "GET", + "ssl": true, + "internal": true + }, + "validate": { + "cluster_name": "www3" + } + }, + { + "test_name": "Test_2", + "input": { + "authority": "www.lyft.com", + "path": "/", + "method": "GET", + "ssl": true, + "internal": true, + "runtime": "runtime.key", + "random_value": 70 + }, + "validate": { + "cluster_name": "www3" + } + }, + { + "test_name": "Test_3", + "input": { + "authority": "www.lyft.com", + "path": "/", + "method": "GET", + "ssl": true, + "internal": true, + "runtime": "runtime.key", + "random_value": 20 + }, + "validate": { + "cluster_name": "www2" + } + } + ] + } diff --git a/test/tools/router_check/test/config/Runtime.yaml b/test/tools/router_check/test/config/Runtime.yaml new file mode 100644 index 000000000000..2e91810912e3 --- /dev/null +++ b/test/tools/router_check/test/config/Runtime.yaml @@ -0,0 +1,18 @@ +virtual_hosts: +- name: www2 + domains: + - www.lyft.com + routes: + - match: + prefix: / + runtime_fraction: + runtime_key: runtime.key + default_value: + numerator: 30 + denominator: HUNDRED + route: + cluster: www2 + - match: + prefix: / + route: + cluster: www3 diff --git a/test/tools/router_check/test/route_tests.sh b/test/tools/router_check/test/route_tests.sh index b0e910b65064..366c23a73333 100755 --- a/test/tools/router_check/test/route_tests.sh +++ b/test/tools/router_check/test/route_tests.sh @@ -18,6 +18,7 @@ done # Testing expected matches using --useproto # --useproto needs the test schema as a validation.proto message. +TESTS+=("Runtime") for t in "${TESTS[@]}" do TEST_OUTPUT=$("${PATH_BIN}" "-c" "${PATH_CONFIG}/${t}.yaml" "-t" "${PATH_CONFIG}/${t}.golden.proto.json" "--details" "--useproto") diff --git a/test/tools/router_check/validation.proto b/test/tools/router_check/validation.proto index b129e4ffa019..9c86153cd3e9 100644 --- a/test/tools/router_check/validation.proto +++ b/test/tools/router_check/validation.proto @@ -65,6 +65,11 @@ message ValidationInput { // The “:authority”, “:path”, “:method”, “x-forwarded-proto”, and “x-envoy-internal” fields are // specified by the other config options and should not be set here. repeated envoy.api.v2.core.HeaderValue additional_headers = 8; + + // Runtime setting key to enable for the test case. + // If a route depends on the runtime, the route will be enabled based on the random_value defined + // in the test. Only a random_value less than the fractional percentage will enable the route. + string runtime = 9; } // The validate object specifies the returned route parameters to match. From 18b8f196498b4dc26ed2ea9e48b9b637387746f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Mon, 8 Jul 2019 00:10:29 -0400 Subject: [PATCH 116/542] zookeeper: more constness (#7481) Spotted these while working on part 2 of the filter, but sending them out now to keep nits separate. Signed-off-by: Raul Gutierrez Segales --- source/extensions/filters/network/zookeeper_proxy/utils.cc | 6 +++--- source/extensions/filters/network/zookeeper_proxy/utils.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/utils.cc b/source/extensions/filters/network/zookeeper_proxy/utils.cc index ec2d524ee584..ed79ecba75ba 100644 --- a/source/extensions/filters/network/zookeeper_proxy/utils.cc +++ b/source/extensions/filters/network/zookeeper_proxy/utils.cc @@ -10,7 +10,7 @@ namespace ZooKeeperProxy { int32_t BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset) { ensureMaxLen(sizeof(int32_t)); - int32_t val = buffer.peekBEInt(offset); + const int32_t val = buffer.peekBEInt(offset); offset += sizeof(int32_t); return val; } @@ -18,7 +18,7 @@ int32_t BufferHelper::peekInt32(Buffer::Instance& buffer, uint64_t& offset) { int64_t BufferHelper::peekInt64(Buffer::Instance& buffer, uint64_t& offset) { ensureMaxLen(sizeof(int64_t)); - int64_t val = buffer.peekBEInt(offset); + const int64_t val = buffer.peekBEInt(offset); offset += sizeof(int64_t); return val; } @@ -34,7 +34,7 @@ bool BufferHelper::peekBool(Buffer::Instance& buffer, uint64_t& offset) { std::string BufferHelper::peekString(Buffer::Instance& buffer, uint64_t& offset) { std::string val; - uint32_t len = peekInt32(buffer, offset); + const uint32_t len = peekInt32(buffer, offset); if (len == 0) { return val; diff --git a/source/extensions/filters/network/zookeeper_proxy/utils.h b/source/extensions/filters/network/zookeeper_proxy/utils.h index ad210a8150f4..7ce749ed8641 100644 --- a/source/extensions/filters/network/zookeeper_proxy/utils.h +++ b/source/extensions/filters/network/zookeeper_proxy/utils.h @@ -37,7 +37,7 @@ class BufferHelper : public Logger::Loggable { private: void ensureMaxLen(uint32_t size); - uint32_t max_len_; + const uint32_t max_len_; uint32_t current_{}; }; From 91b4c9c013d4e57a107993d884027427acfbb82c Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Mon, 8 Jul 2019 09:31:26 -0700 Subject: [PATCH 117/542] [test] convert Outlier Detection test stubs to v2 YAML (#7485) Signed-off-by: Derek Argueta --- test/common/upstream/BUILD | 1 + .../upstream/outlier_detection_impl_test.cc | 26 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 8ced77fcc2c4..7bbf8314c71e 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -271,6 +271,7 @@ envoy_cc_test( "//test/mocks/runtime:runtime_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", ], ) diff --git a/test/common/upstream/outlier_detection_impl_test.cc b/test/common/upstream/outlier_detection_impl_test.cc index bccf463c5e91..c7a3cb08abf3 100644 --- a/test/common/upstream/outlier_detection_impl_test.cc +++ b/test/common/upstream/outlier_detection_impl_test.cc @@ -16,6 +16,7 @@ #include "test/mocks/runtime/mocks.h" #include "test/mocks/upstream/mocks.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" #include "absl/types/optional.h" #include "gmock/gmock.h" @@ -118,23 +119,20 @@ class OutlierDetectorImplTest : public testing::Test { }; TEST_F(OutlierDetectorImplTest, DetectorStaticConfig) { - const std::string json = R"EOF( - { - "interval_ms" : 100, - "base_ejection_time_ms" : 10000, - "consecutive_5xx" : 10, - "max_ejection_percent" : 50, - "enforcing_consecutive_5xx" : 10, - "enforcing_success_rate": 20, - "success_rate_minimum_hosts": 50, - "success_rate_request_volume": 200, - "success_rate_stdev_factor": 3000 - } + const std::string yaml = R"EOF( +interval: 0.1s +base_ejection_time: 10s +consecutive_5xx: 10 +max_ejection_percent: 50 +enforcing_consecutive_5xx: 10 +enforcing_success_rate: 20 +success_rate_minimum_hosts: 50 +success_rate_request_volume: 200 +success_rate_stdev_factor: 3000 )EOF"; envoy::api::v2::cluster::OutlierDetection outlier_detection; - Json::ObjectSharedPtr custom_config = Json::Factory::loadFromString(json); - Config::CdsJson::translateOutlierDetection(*custom_config, outlier_detection); + TestUtility::loadFromYaml(yaml, outlier_detection); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(100))); std::shared_ptr detector(DetectorImpl::create( cluster_, outlier_detection, dispatcher_, runtime_, time_system_, event_logger_)); From f683743ca6727d003a30c998ca617e31d9523e3d Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Mon, 8 Jul 2019 09:44:36 -0700 Subject: [PATCH 118/542] Fix clang-tidy (#7475) Signed-off-by: Derek Argueta --- .clang-tidy | 4 ++-- ci/run_clang_tidy.sh | 6 ++++++ source/common/grpc/common.cc | 2 +- source/common/stats/thread_local_store.cc | 2 +- source/common/upstream/outlier_detection_impl.cc | 2 +- test/common/protobuf/utility_test.cc | 2 +- .../quic_listeners/quiche/platform/quic_platform_test.cc | 2 +- 7 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index d58e0a8be486..0692d1c459ee 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -5,8 +5,8 @@ Checks: 'abseil-*, modernize-*, performance-*, readability-braces-around-statements, - readability-container-size-empty' - readability-redundant-*, + readability-container-size-empty, + readability-redundant-*' #TODO(lizan): grow this list, fix possible warnings and make more checks as error WarningsAsErrors: 'abseil-duration-*, diff --git a/ci/run_clang_tidy.sh b/ci/run_clang_tidy.sh index 8adbbd5089d8..dc0e8aead732 100755 --- a/ci/run_clang_tidy.sh +++ b/ci/run_clang_tidy.sh @@ -2,6 +2,12 @@ set -e +# Quick syntax check of .clang-tidy using PyYAML. +if ! python -c 'import yaml, sys; yaml.safe_load(sys.stdin)' < .clang-tidy > /dev/null; then + echo ".clang-tidy has a syntax error" + exit 1 +fi + echo "Generating compilation database..." cp -f .bazelrc .bazelrc.bak diff --git a/source/common/grpc/common.cc b/source/common/grpc/common.cc index d70c86ba5343..2683044b715c 100644 --- a/source/common/grpc/common.cc +++ b/source/common/grpc/common.cc @@ -54,7 +54,7 @@ absl::optional Common::getGrpcStatus(const Http::HeaderMap& uint64_t grpc_status_code; if (!grpc_status_header || grpc_status_header->value().empty()) { - return absl::optional(); + return {}; } if (!absl::SimpleAtoi(grpc_status_header->value().getStringView(), &grpc_status_code) || grpc_status_code > Status::GrpcStatus::MaximumValid) { diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 91603ad79083..6de61634cd10 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -368,7 +368,7 @@ ThreadLocalStoreImpl::ScopeImpl::findStatLockHeld( return absl::nullopt; } - return std::cref(*iter->second.get()); + return std::cref(*iter->second); } Counter& ThreadLocalStoreImpl::ScopeImpl::counterFromStatName(StatName name) { diff --git a/source/common/upstream/outlier_detection_impl.cc b/source/common/upstream/outlier_detection_impl.cc index 948cbb1f361f..0d0efa147366 100644 --- a/source/common/upstream/outlier_detection_impl.cc +++ b/source/common/upstream/outlier_detection_impl.cc @@ -710,7 +710,7 @@ SuccessRateAccumulatorBucket* SuccessRateAccumulator::updateCurrentWriter() { absl::optional SuccessRateAccumulator::getSuccessRate(uint64_t success_rate_request_volume) { if (backup_success_rate_bucket_->total_request_counter_ < success_rate_request_volume) { - return absl::optional(); + return {}; } return {backup_success_rate_bucket_->success_request_counter_ * 100.0 / diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 99a7dcb607fd..04fb200fc075 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -295,7 +295,7 @@ TEST_F(ProtobufUtilityTest, HashedValue) { EXPECT_EQ(hv1, hv2); EXPECT_NE(hv1, hv3); - HashedValue copy(hv1); + HashedValue copy(hv1); // NOLINT(performance-unnecessary-copy-initialization) EXPECT_EQ(hv1, copy); } diff --git a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc index 529ce889c7ae..8fcb0ff33365 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc @@ -755,7 +755,7 @@ TEST_P(QuicMemSliceTest, ConstructMemSliceFromBuffer) { quic::QuicMemSlice slice1{quic::QuicMemSliceImpl(buffer, str2.length())}; EXPECT_EQ(str.length(), buffer.length()); EXPECT_EQ(str2, std::string(slice1.data(), slice1.length())); - std::string str2_old = str2; + std::string str2_old = str2; // NOLINT(performance-unnecessary-copy-initialization) // slice1 is released, but str2 should not be affected. slice1.Reset(); EXPECT_TRUE(slice1.empty()); From 470aea0af22a86aeedc6f1e10c211f12b01f6f61 Mon Sep 17 00:00:00 2001 From: danzh Date: Mon, 8 Jul 2019 15:53:46 -0400 Subject: [PATCH 119/542] quiche: build quic http libraries (#7463) Add build targets for all files under quiche/quic/core/http/. These libraries brings in QUIC http interfaces. As Envoy doesn't have GURL which quiche/quic/core/http/spdy_server_push_utils.cc depends on, it's replaced by //source/extensions/quic_listeners/quiche/spdy_server_push_utils_for_envoy.cc as the definition of all the functions defined in spdy_server_push_utils.h. These envoy version function definitions does nothing but immediately fail if reached, because Envoy doesn't support server push. Update quiche tar ball: 2a930469533c3b541443488a629fe25cd8ff53d0 Rename quic_platform_mem_slice_storage_lib to quic_platform_mem_slice_storage to follow platform impl's naming convention. Testing: n/a Part of #2557 Signed-off-by: Dan Zhang --- bazel/external/quiche.BUILD | 1152 ++++++++++++++++- bazel/repository_locations.bzl | 6 +- source/extensions/quic_listeners/quiche/BUILD | 10 + .../quiche/platform/flags_list.h | 22 + .../quiche/platform/quic_logging_impl.h | 1 + .../quiche/platform/quic_pcc_sender_impl.h | 2 +- .../spdy_server_push_utils_for_envoy.cc | 37 + .../quic_listeners/quiche/platform/BUILD | 2 +- tools/spelling_dictionary.txt | 1 + 9 files changed, 1226 insertions(+), 7 deletions(-) create mode 100644 source/extensions/quic_listeners/quiche/spdy_server_push_utils_for_envoy.cc diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 250488e50400..d7b7ee37c9bb 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -100,6 +100,603 @@ envoy_cc_library( deps = ["@envoy//source/extensions/quic_listeners/quiche/platform:http2_platform_impl_lib"], ) +envoy_cc_library( + name = "http2_constants_lib", + srcs = ["quiche/http2/http2_constants.cc"], + hdrs = ["quiche/http2/http2_constants.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":http2_platform"], +) + +envoy_cc_library( + name = "http2_structures_lib", + srcs = ["quiche/http2/http2_structures.cc"], + hdrs = ["quiche/http2/http2_structures.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_decoder_decode_buffer_lib", + srcs = ["quiche/http2/decoder/decode_buffer.cc"], + hdrs = ["quiche/http2/decoder/decode_buffer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":http2_platform"], +) + +envoy_cc_library( + name = "http2_decoder_decode_http2_structures_lib", + srcs = ["quiche/http2/decoder/decode_http2_structures.cc"], + hdrs = ["quiche/http2/decoder/decode_http2_structures.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_decode_status_lib", + srcs = ["quiche/http2/decoder/decode_status.cc"], + hdrs = ["quiche/http2/decoder/decode_status.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":http2_platform"], +) + +envoy_cc_library( + name = "http2_decoder_frame_decoder_state_lib", + srcs = ["quiche/http2/decoder/frame_decoder_state.cc"], + hdrs = ["quiche/http2/decoder/frame_decoder_state.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_structure_decoder_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_frame_decoder_lib", + srcs = ["quiche/http2/decoder/http2_frame_decoder.cc"], + hdrs = [ + "quiche/http2/decoder/frame_decoder_state.h", + "quiche/http2/decoder/http2_frame_decoder.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_decoder_payload_decoders_altsvc_payload_decoder_lib", + ":http2_decoder_payload_decoders_continuation_payload_decoder_lib", + ":http2_decoder_payload_decoders_data_payload_decoder_lib", + ":http2_decoder_payload_decoders_goaway_payload_decoder_lib", + ":http2_decoder_payload_decoders_headers_payload_decoder_lib", + ":http2_decoder_payload_decoders_ping_payload_decoder_lib", + ":http2_decoder_payload_decoders_priority_payload_decoder_lib", + ":http2_decoder_payload_decoders_push_promise_payload_decoder_lib", + ":http2_decoder_payload_decoders_rst_stream_payload_decoder_lib", + ":http2_decoder_payload_decoders_settings_payload_decoder_lib", + ":http2_decoder_payload_decoders_unknown_payload_decoder_lib", + ":http2_decoder_payload_decoders_window_update_payload_decoder_lib", + ":http2_decoder_structure_decoder_lib", + ":http2_hpack_varint_hpack_varint_decoder_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_frame_decoder_listener_lib", + srcs = ["quiche/http2/decoder/http2_frame_decoder_listener.cc"], + hdrs = ["quiche/http2/decoder/http2_frame_decoder_listener.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_altsvc_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/altsvc_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/altsvc_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_continuation_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/continuation_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/continuation_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_data_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/data_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/data_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_goaway_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/goaway_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/goaway_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_headers_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/headers_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/headers_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_ping_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/ping_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/ping_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_priority_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/priority_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/priority_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_push_promise_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/push_promise_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/push_promise_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_rst_stream_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/rst_stream_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/rst_stream_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_settings_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/settings_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/settings_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_unknown_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/unknown_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/unknown_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_payload_decoders_window_update_payload_decoder_lib", + srcs = ["quiche/http2/decoder/payload_decoders/window_update_payload_decoder.cc"], + hdrs = ["quiche/http2/decoder/payload_decoders/window_update_payload_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_http2_structures_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_decoder_frame_decoder_state_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_decoder_structure_decoder_lib", + srcs = ["quiche/http2/decoder/http2_structure_decoder.cc"], + hdrs = ["quiche/http2/decoder/http2_structure_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_http2_structures_lib", + ":http2_decoder_decode_status_lib", + ":http2_platform", + ":http2_structures_lib", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_block_decoder_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_block_decoder.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_block_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_hpack_decoder_hpack_entry_decoder_lib", + ":http2_hpack_decoder_hpack_entry_decoder_listener_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_decoder_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_decoder.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_hpack_decoder_hpack_block_decoder_lib", + ":http2_hpack_decoder_hpack_decoder_listener_lib", + ":http2_hpack_decoder_hpack_decoder_state_lib", + ":http2_hpack_decoder_hpack_decoder_tables_lib", + ":http2_hpack_decoder_hpack_whole_entry_buffer_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_decoder_listener_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_decoder_listener.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_decoder_listener.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_hpack_hpack_constants_lib", + ":http2_hpack_hpack_string_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_decoder_state_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_decoder_state.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_decoder_state.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_hpack_decoder_hpack_decoder_listener_lib", + ":http2_hpack_decoder_hpack_decoder_string_buffer_lib", + ":http2_hpack_decoder_hpack_decoder_tables_lib", + ":http2_hpack_decoder_hpack_whole_entry_listener_lib", + ":http2_hpack_hpack_constants_lib", + ":http2_hpack_hpack_string_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_decoder_string_buffer_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_decoder_string_buffer.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_decoder_string_buffer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_hpack_huffman_hpack_huffman_decoder_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_decoder_tables_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_decoder_tables.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_decoder_tables.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_hpack_hpack_constants_lib", + ":http2_hpack_hpack_static_table_entries_lib", + ":http2_hpack_hpack_string_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_entry_decoder_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_entry_decoder.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_entry_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_hpack_decoder_hpack_entry_decoder_listener_lib", + ":http2_hpack_decoder_hpack_entry_type_decoder_lib", + ":http2_hpack_decoder_hpack_string_decoder_lib", + ":http2_hpack_hpack_constants_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_entry_decoder_listener_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_entry_decoder_listener.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_entry_decoder_listener.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_hpack_hpack_constants_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_entry_type_decoder_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_entry_type_decoder.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_entry_type_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_hpack_hpack_constants_lib", + ":http2_hpack_varint_hpack_varint_decoder_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_string_decoder_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_string_decoder.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_string_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_hpack_varint_hpack_varint_decoder_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_string_decoder_listener_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_string_decoder_listener.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_string_decoder_listener.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":http2_platform"], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_whole_entry_buffer_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_whole_entry_buffer.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_whole_entry_buffer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_hpack_decoder_hpack_decoder_string_buffer_lib", + ":http2_hpack_decoder_hpack_entry_decoder_listener_lib", + ":http2_hpack_decoder_hpack_whole_entry_listener_lib", + ":http2_hpack_hpack_constants_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_decoder_hpack_whole_entry_listener_lib", + srcs = ["quiche/http2/hpack/decoder/hpack_whole_entry_listener.cc"], + hdrs = ["quiche/http2/hpack/decoder/hpack_whole_entry_listener.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_hpack_decoder_hpack_decoder_string_buffer_lib", + ":http2_hpack_hpack_constants_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_huffman_hpack_huffman_decoder_lib", + srcs = ["quiche/http2/hpack/huffman/hpack_huffman_decoder.cc"], + hdrs = ["quiche/http2/hpack/huffman/hpack_huffman_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":http2_platform"], +) + +envoy_cc_library( + name = "http2_hpack_huffman_hpack_huffman_encoder_lib", + srcs = ["quiche/http2/hpack/huffman/hpack_huffman_encoder.cc"], + hdrs = ["quiche/http2/hpack/huffman/hpack_huffman_encoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_hpack_huffman_huffman_spec_tables_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_huffman_huffman_spec_tables_lib", + srcs = ["quiche/http2/hpack/huffman/huffman_spec_tables.cc"], + hdrs = ["quiche/http2/hpack/huffman/huffman_spec_tables.h"], + copts = quiche_copt, + repository = "@envoy", +) + +envoy_cc_library( + name = "http2_hpack_hpack_constants_lib", + srcs = ["quiche/http2/hpack/http2_hpack_constants.cc"], + hdrs = ["quiche/http2/hpack/http2_hpack_constants.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":http2_platform"], +) + +envoy_cc_library( + name = "http2_hpack_hpack_static_table_entries_lib", + hdrs = ["quiche/http2/hpack/hpack_static_table_entries.inc"], + repository = "@envoy", +) + +envoy_cc_library( + name = "http2_hpack_hpack_string_lib", + srcs = ["quiche/http2/hpack/hpack_string.cc"], + hdrs = ["quiche/http2/hpack/hpack_string.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":http2_platform"], +) + +envoy_cc_library( + name = "http2_hpack_varint_hpack_varint_decoder_lib", + srcs = ["quiche/http2/hpack/varint/hpack_varint_decoder.cc"], + hdrs = ["quiche/http2/hpack/varint/hpack_varint_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_platform", + ], +) + +envoy_cc_library( + name = "http2_hpack_varint_hpack_varint_encoder_lib", + srcs = ["quiche/http2/hpack/varint/hpack_varint_encoder.cc"], + hdrs = ["quiche/http2/hpack/varint/hpack_varint_encoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":http2_platform"], +) + envoy_cc_library( name = "spdy_platform", hdrs = [ @@ -167,6 +764,43 @@ envoy_cc_library( deps = [":spdy_platform"], ) +envoy_cc_library( + name = "spdy_core_framer_lib", + srcs = [ + "quiche/spdy/core/spdy_frame_builder.cc", + "quiche/spdy/core/spdy_framer.cc", + ], + hdrs = [ + "quiche/spdy/core/spdy_frame_builder.h", + "quiche/spdy/core/spdy_framer.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_platform", + ":spdy_core_alt_svc_wire_format_lib", + ":spdy_core_frame_reader_lib", + ":spdy_core_header_block_lib", + ":spdy_core_headers_handler_interface_lib", + ":spdy_core_hpack_hpack_lib", + ":spdy_core_protocol_lib", + ":spdy_core_zero_copy_output_buffer_lib", + ":spdy_platform", + ], +) + +envoy_cc_library( + name = "spdy_core_frame_reader_lib", + srcs = ["quiche/spdy/core/spdy_frame_reader.cc"], + hdrs = ["quiche/spdy/core/spdy_frame_reader.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":spdy_core_protocol_lib", + ":spdy_platform", + ], +) + envoy_cc_library( name = "spdy_core_header_block_lib", srcs = ["quiche/spdy/core/spdy_header_block.cc"], @@ -189,6 +823,79 @@ envoy_cc_library( deps = [":spdy_platform"], ) +envoy_cc_library( + name = "spdy_core_http2_deframer_lib", + srcs = ["quiche/spdy/core/http2_frame_decoder_adapter.cc"], + hdrs = ["quiche/spdy/core/http2_frame_decoder_adapter.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_constants_lib", + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_decoder_frame_decoder_lib", + ":http2_decoder_frame_decoder_listener_lib", + ":http2_platform", + ":http2_structures_lib", + ":spdy_core_alt_svc_wire_format_lib", + ":spdy_core_header_block_lib", + ":spdy_core_headers_handler_interface_lib", + ":spdy_core_hpack_hpack_decoder_adapter_lib", + ":spdy_core_hpack_hpack_lib", + ":spdy_core_protocol_lib", + ":spdy_platform", + ], +) + +envoy_cc_library( + name = "spdy_core_hpack_hpack_lib", + srcs = [ + "quiche/spdy/core/hpack/hpack_constants.cc", + "quiche/spdy/core/hpack/hpack_encoder.cc", + "quiche/spdy/core/hpack/hpack_entry.cc", + "quiche/spdy/core/hpack/hpack_header_table.cc", + "quiche/spdy/core/hpack/hpack_huffman_table.cc", + "quiche/spdy/core/hpack/hpack_output_stream.cc", + "quiche/spdy/core/hpack/hpack_static_table.cc", + ], + hdrs = [ + "quiche/spdy/core/hpack/hpack_constants.h", + "quiche/spdy/core/hpack/hpack_encoder.h", + "quiche/spdy/core/hpack/hpack_entry.h", + "quiche/spdy/core/hpack/hpack_header_table.h", + "quiche/spdy/core/hpack/hpack_huffman_table.h", + "quiche/spdy/core/hpack/hpack_output_stream.h", + "quiche/spdy/core/hpack/hpack_static_table.h", + ], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":spdy_core_protocol_lib", + ":spdy_platform", + ], +) + +envoy_cc_library( + name = "spdy_core_hpack_hpack_decoder_adapter_lib", + srcs = ["quiche/spdy/core/hpack/hpack_decoder_adapter.cc"], + hdrs = ["quiche/spdy/core/hpack/hpack_decoder_adapter.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":http2_hpack_decoder_hpack_decoder_lib", + ":http2_hpack_decoder_hpack_decoder_listener_lib", + ":http2_hpack_decoder_hpack_decoder_tables_lib", + ":http2_hpack_hpack_constants_lib", + ":http2_hpack_hpack_string_lib", + ":spdy_core_header_block_lib", + ":spdy_core_headers_handler_interface_lib", + ":spdy_core_hpack_hpack_lib", + ":spdy_platform", + ], +) + envoy_cc_library( name = "spdy_core_priority_write_scheduler_lib", srcs = ["quiche/spdy/core/priority_write_scheduler.h"], @@ -243,6 +950,13 @@ envoy_cc_test_library( ], ) +envoy_cc_library( + name = "spdy_core_zero_copy_output_buffer_lib", + hdrs = ["quiche/spdy/core/zero_copy_output_buffer.h"], + copts = quiche_copt, + repository = "@envoy", +) + envoy_cc_library( name = "quic_platform", srcs = [ @@ -1187,6 +1901,194 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quic_core_http_client_lib", + srcs = [ + "quiche/quic/core/http/quic_client_promised_info.cc", + "quiche/quic/core/http/quic_client_push_promise_index.cc", + "quiche/quic/core/http/quic_spdy_client_session.cc", + "quiche/quic/core/http/quic_spdy_client_session_base.cc", + "quiche/quic/core/http/quic_spdy_client_stream.cc", + ], + hdrs = [ + "quiche/quic/core/http/quic_client_promised_info.h", + "quiche/quic/core/http/quic_client_push_promise_index.h", + "quiche/quic/core/http/quic_spdy_client_session.h", + "quiche/quic/core/http/quic_spdy_client_session_base.h", + "quiche/quic/core/http/quic_spdy_client_stream.h", + ], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_alarm_interface_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_http_spdy_session_lib", + ":quic_core_packets_lib", + ":quic_core_server_id_lib", + ":quic_core_session_lib", + ":quic_core_types_lib", + ":quic_core_utils_lib", + ":quic_platform_base", + ":spdy_core_framer_lib", + ":spdy_core_protocol_lib", + "@envoy//source/extensions/quic_listeners/quiche:spdy_server_push_utils_for_envoy_lib", + ], +) + +envoy_cc_library( + name = "quic_core_http_header_list_lib", + srcs = ["quiche/quic/core/http/quic_header_list.cc"], + hdrs = ["quiche/quic/core/http/quic_header_list.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_packets_lib", + ":quic_platform_base", + ":spdy_core_header_block_lib", + ":spdy_core_headers_handler_interface_lib", + ":spdy_core_protocol_lib", + ], +) + +envoy_cc_library( + name = "quic_core_http_http_decoder_lib", + srcs = ["quiche/quic/core/http/http_decoder.cc"], + hdrs = ["quiche/quic/core/http/http_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_data_lib", + ":quic_core_error_codes_lib", + ":quic_core_http_http_frames_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_http_http_encoder_lib", + srcs = ["quiche/quic/core/http/http_encoder.cc"], + hdrs = ["quiche/quic/core/http/http_encoder.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_data_lib", + ":quic_core_error_codes_lib", + ":quic_core_http_http_frames_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_http_http_frames_lib", + hdrs = ["quiche/quic/core/http/http_frames.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_types_lib", + ":quic_platform_base", + ":spdy_core_framer_lib", + ], +) + +envoy_cc_library( + name = "quic_core_http_spdy_server_push_utils_header", + hdrs = ["quiche/quic/core/http/spdy_server_push_utils.h"], + copts = quiche_copt, + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":quic_platform_base", + ":spdy_core_header_block_lib", + ], +) + +envoy_cc_library( + name = "quic_core_http_spdy_session_lib", + srcs = [ + "quiche/quic/core/http/quic_headers_stream.cc", + "quiche/quic/core/http/quic_receive_control_stream.cc", + "quiche/quic/core/http/quic_send_control_stream.cc", + "quiche/quic/core/http/quic_server_session_base.cc", + "quiche/quic/core/http/quic_spdy_server_stream_base.cc", + "quiche/quic/core/http/quic_spdy_session.cc", + "quiche/quic/core/http/quic_spdy_stream.cc", + ], + hdrs = [ + "quiche/quic/core/http/quic_headers_stream.h", + "quiche/quic/core/http/quic_receive_control_stream.h", + "quiche/quic/core/http/quic_send_control_stream.h", + "quiche/quic/core/http/quic_server_session_base.h", + "quiche/quic/core/http/quic_spdy_server_stream_base.h", + "quiche/quic/core/http/quic_spdy_session.h", + "quiche/quic/core/http/quic_spdy_stream.h", + ], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_connection_lib", + ":quic_core_crypto_crypto_handshake_lib", + ":quic_core_error_codes_lib", + ":quic_core_http_header_list_lib", + ":quic_core_http_http_decoder_lib", + ":quic_core_http_http_encoder_lib", + ":quic_core_http_spdy_stream_body_buffer_lib", + ":quic_core_http_spdy_utils_lib", + ":quic_core_packets_lib", + ":quic_core_proto_cached_network_parameters_proto_header", + ":quic_core_qpack_qpack_decoded_headers_accumulator_lib", + ":quic_core_qpack_qpack_decoder_lib", + ":quic_core_qpack_qpack_decoder_stream_sender_lib", + ":quic_core_qpack_qpack_encoder_lib", + ":quic_core_qpack_qpack_encoder_stream_sender_lib", + ":quic_core_qpack_qpack_utils_lib", + ":quic_core_session_lib", + ":quic_core_utils_lib", + ":quic_core_versions_lib", + ":quic_platform_base", + ":quic_platform_mem_slice_storage", + ":spdy_core_framer_lib", + ":spdy_core_http2_deframer_lib", + ":spdy_core_protocol_lib", + ], +) + +envoy_cc_library( + name = "quic_core_http_spdy_stream_body_buffer_lib", + srcs = ["quiche/quic/core/http/quic_spdy_stream_body_buffer.cc"], + hdrs = ["quiche/quic/core/http/quic_spdy_stream_body_buffer.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_http_http_decoder_lib", + ":quic_core_session_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_http_spdy_utils_lib", + srcs = ["quiche/quic/core/http/spdy_utils.cc"], + hdrs = ["quiche/quic/core/http/spdy_utils.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_http_header_list_lib", + ":quic_core_packets_lib", + ":quic_platform_base", + ":spdy_core_framer_lib", + ":spdy_core_protocol_lib", + ], +) + envoy_cc_library( name = "quic_core_interval_lib", hdrs = ["quiche/quic/core/quic_interval.h"], @@ -1353,6 +2255,252 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quic_core_qpack_qpack_instruction_encoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_instruction_encoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_instruction_encoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_hpack_huffman_hpack_huffman_encoder_lib", + ":http2_hpack_varint_hpack_varint_encoder_lib", + ":quic_core_qpack_qpack_constants_lib", + ":quic_platform", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_instruction_decoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_instruction_decoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_instruction_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_hpack_huffman_hpack_huffman_decoder_lib", + ":http2_hpack_varint_hpack_varint_decoder_lib", + ":quic_core_qpack_qpack_constants_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_constants_lib", + srcs = ["quiche/quic/core/qpack/qpack_constants.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_constants.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_platform_base"], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_encoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_encoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_encoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_qpack_qpack_constants_lib", + ":quic_core_qpack_qpack_decoder_stream_receiver_lib", + ":quic_core_qpack_qpack_encoder_stream_sender_lib", + ":quic_core_qpack_qpack_header_table_lib", + ":quic_core_qpack_qpack_instruction_encoder_lib", + ":quic_core_qpack_value_splitting_header_list_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_progressive_decoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_progressive_decoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_progressive_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_qpack_qpack_constants_lib", + ":quic_core_qpack_qpack_decoder_stream_sender_lib", + ":quic_core_qpack_qpack_encoder_stream_receiver_lib", + ":quic_core_qpack_qpack_header_table_lib", + ":quic_core_qpack_qpack_instruction_decoder_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_decoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_decoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_qpack_qpack_decoder_stream_sender_lib", + ":quic_core_qpack_qpack_encoder_stream_receiver_lib", + ":quic_core_qpack_qpack_header_table_lib", + ":quic_core_qpack_qpack_progressive_decoder_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_header_table_lib", + srcs = ["quiche/quic/core/qpack/qpack_header_table.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_header_table.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_qpack_qpack_static_table_lib", + ":quic_platform_base", + ":spdy_core_hpack_hpack_lib", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_utils_lib", + hdrs = ["quiche/quic/core/qpack/qpack_utils.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_core_qpack_qpack_stream_sender_delegate_lib"], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_encoder_stream_sender_lib", + srcs = ["quiche/quic/core/qpack/qpack_encoder_stream_sender.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_encoder_stream_sender.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_qpack_qpack_constants_lib", + ":quic_core_qpack_qpack_instruction_encoder_lib", + ":quic_core_qpack_qpack_stream_sender_delegate_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_encoder_stream_receiver_lib", + srcs = ["quiche/quic/core/qpack/qpack_encoder_stream_receiver.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_encoder_stream_receiver.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":quic_core_qpack_qpack_constants_lib", + ":quic_core_qpack_qpack_instruction_decoder_lib", + ":quic_core_qpack_qpack_stream_receiver_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_decoder_stream_sender_lib", + srcs = ["quiche/quic/core/qpack/qpack_decoder_stream_sender.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_decoder_stream_sender.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_qpack_qpack_constants_lib", + ":quic_core_qpack_qpack_instruction_encoder_lib", + ":quic_core_qpack_qpack_stream_sender_delegate_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_decoder_stream_receiver_lib", + srcs = ["quiche/quic/core/qpack/qpack_decoder_stream_receiver.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_decoder_stream_receiver.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":http2_decoder_decode_buffer_lib", + ":http2_decoder_decode_status_lib", + ":quic_core_qpack_qpack_constants_lib", + ":quic_core_qpack_qpack_instruction_decoder_lib", + ":quic_core_qpack_qpack_stream_receiver_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_static_table_lib", + srcs = ["quiche/quic/core/qpack/qpack_static_table.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_static_table.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_platform_base", + ":spdy_core_hpack_hpack_lib", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_stream_receiver_lib", + hdrs = ["quiche/quic/core/qpack/qpack_stream_receiver.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_platform_base"], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_decoded_headers_accumulator_lib", + srcs = ["quiche/quic/core/qpack/qpack_decoded_headers_accumulator.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_decoded_headers_accumulator.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_http_header_list_lib", + ":quic_core_qpack_qpack_decoder_lib", + ":quic_core_qpack_qpack_progressive_decoder_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_value_splitting_header_list_lib", + srcs = ["quiche/quic/core/qpack/value_splitting_header_list.cc"], + hdrs = ["quiche/quic/core/qpack/value_splitting_header_list.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_platform_base", + ":spdy_core_header_block_lib", + ], +) + +# envoy_cc_library( +# name = "quic_core_qpack_qpack_streams_lib", +# srcs = [ +# "quiche/quic/core/qpack/qpack_receive_stream.cc", +# "quiche/quic/core/qpack/qpack_send_stream.cc", +# ], +# hdrs = [ +# "quiche/quic/core/qpack/qpack_receive_stream.h", +# "quiche/quic/core/qpack/qpack_send_stream.h", +# ], +# copts = quiche_copt, +# repository = "@envoy", +# deps = [ +# ":quic_core_http_spdy_session_lib", +# ":quic_core_qpack_qpack_stream_sender_delegate_lib", +# ":quic_core_session_lib", +# ":quic_platform_base", +# ], +# ) + +envoy_cc_library( + name = "quic_core_qpack_qpack_stream_sender_delegate_lib", + hdrs = ["quiche/quic/core/qpack/qpack_stream_sender_delegate.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_platform_base"], +) + envoy_cc_library( name = "quic_core_received_packet_manager_lib", srcs = ["quiche/quic/core/quic_received_packet_manager.cc"], @@ -1884,7 +3032,7 @@ envoy_cc_test_library( ) envoy_cc_library( - name = "quic_platform_mem_slice_storage_lib", + name = "quic_platform_mem_slice_storage", hdrs = ["quiche/quic/platform/api/quic_mem_slice_storage.h"], repository = "@envoy", visibility = ["//visibility:public"], @@ -1920,7 +3068,7 @@ envoy_cc_test( ":quic_core_buffer_allocator_lib", ":quic_platform", ":quic_platform_mem_slice_span", - ":quic_platform_mem_slice_storage_lib", + ":quic_platform_mem_slice_storage", ":quic_platform_test", ":quic_platform_test_mem_slice_vector_lib", ], diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 7bbc4589f5aa..58c12e35916f 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -247,8 +247,8 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/curl/curl/releases/download/curl-7_65_1/curl-7.65.1.tar.gz"], ), com_googlesource_quiche = dict( - # Static snapshot of https://quiche.googlesource.com/quiche/+archive/8f3a576515ada564d3e2b560220649f49a21dec1.tar.gz - sha256 = "ae2f3ecd7a87154ae4668f7b1876846c1cf4efa8abc987511809433f73914a68", - urls = ["https://storage.googleapis.com/quiche-envoy-integration/8f3a576515ada564d3e2b560220649f49a21dec1.tar.gz"], + # Static snapshot of https://quiche.googlesource.com/quiche/+archive/2a930469533c3b541443488a629fe25cd8ff53d0.tar.gz + sha256 = "fcdebf54c89d839ffa7eefae166c8e4b551c765559db13ff15bff98047f344fb", + urls = ["https://storage.googleapis.com/quiche-envoy-integration/2a930469533c3b541443488a629fe25cd8ff53d0.tar.gz"], ), ) diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 408364c76301..2e76136e0d8f 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -55,3 +55,13 @@ envoy_cc_library( "@com_googlesource_quiche//:quic_core_versions_lib", ], ) + +envoy_cc_library( + name = "spdy_server_push_utils_for_envoy_lib", + srcs = ["spdy_server_push_utils_for_envoy.cc"], + visibility = ["//visibility:public"], + deps = [ + "//source/common/common:assert_lib", + "@com_googlesource_quiche//:quic_core_http_spdy_server_push_utils_header", + ], +) diff --git a/source/extensions/quic_listeners/quiche/platform/flags_list.h b/source/extensions/quic_listeners/quiche/platform/flags_list.h index 6de7e5b36d6c..31ba6c3686f5 100644 --- a/source/extensions/quic_listeners/quiche/platform/flags_list.h +++ b/source/extensions/quic_listeners/quiche/platform/flags_list.h @@ -341,6 +341,28 @@ QUICHE_FLAG( QUICHE_FLAG(bool, quic_reloadable_flag_quic_version_negotiation_grease, false, "When true, QUIC Version Negotiation packets will randomly include fake versions.") +QUICHE_FLAG( + bool, quic_reloadable_flag_quic_fix_get_packet_header_size, false, + "Fixes quic::GetPacketHeaderSize and callsites when QuicVersionHasLongHeaderLengths is false.") + +QUICHE_FLAG( + bool, quic_reloadable_flag_quic_change_default_lumpy_pacing_size_to_two, false, + "If true and --quic_lumpy_pacing_size is 1, QUIC will use a lumpy size of two for pacing.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_window_update_on_read_only_stream, false, + "If true, QuicConnection will be closed if a WindowUpdate frame is received on a " + "READ_UNIDIRECTIONAL stream.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_clear_queued_packets_on_connection_close, false, + "Calls ClearQueuedPackets after sending a connection close packet") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_48, false, + "If true, enable QUIC version 48.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_avoid_empty_frame_after_empty_headers, false, + "If enabled, do not call OnStreamFrame() with empty frame after receiving empty or too " + "large headers with FIN.") + QUICHE_FLAG(bool, quic_restart_flag_quic_allow_loas_multipacket_chlo, false, "If true, inspects QUIC CHLOs for kLOAS and early creates sessions " "to allow multi-packet CHLOs") diff --git a/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h index 41cd2c1b4c01..13fa6dc81031 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h @@ -12,6 +12,7 @@ #include #include +#include "common/common/assert.h" #include "common/common/logger.h" #include "common/common/stl_helpers.h" diff --git a/source/extensions/quic_listeners/quiche/platform/quic_pcc_sender_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_pcc_sender_impl.h index 52d7836b9911..011120b1c298 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_pcc_sender_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_pcc_sender_impl.h @@ -13,7 +13,7 @@ namespace quic { class QuicClock; -class QuicConnectionStats; +struct QuicConnectionStats; class QuicRandom; class QuicUnackedPacketMap; class RttStats; diff --git a/source/extensions/quic_listeners/quiche/spdy_server_push_utils_for_envoy.cc b/source/extensions/quic_listeners/quiche/spdy_server_push_utils_for_envoy.cc new file mode 100644 index 000000000000..0ff435226efe --- /dev/null +++ b/source/extensions/quic_listeners/quiche/spdy_server_push_utils_for_envoy.cc @@ -0,0 +1,37 @@ +#include "quiche/quic/core/http/spdy_server_push_utils.h" + +// NOLINT(namespace-envoy) + +// This file has a substitute definition for +// quiche/quic/core/http/spdy_server_push_utils.cc which depends on GURL. +// Since Envoy doesn't support server push, these functions shouldn't be +// executed at all. + +using spdy::SpdyHeaderBlock; + +namespace quic { + +// static +std::string SpdyServerPushUtils::GetPromisedUrlFromHeaders(const SpdyHeaderBlock& /*headers*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +// static +std::string +SpdyServerPushUtils::GetPromisedHostNameFromHeaders(const SpdyHeaderBlock& /*headers*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +// static +bool SpdyServerPushUtils::PromisedUrlIsValid(const SpdyHeaderBlock& /*headers*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +// static +std::string SpdyServerPushUtils::GetPushPromiseUrl(QuicStringPiece /*scheme*/, + QuicStringPiece /*authority*/, + QuicStringPiece /*path*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +} // namespace quic diff --git a/test/extensions/quic_listeners/quiche/platform/BUILD b/test/extensions/quic_listeners/quiche/platform/BUILD index 654ddcd98160..911943c915ce 100644 --- a/test/extensions/quic_listeners/quiche/platform/BUILD +++ b/test/extensions/quic_listeners/quiche/platform/BUILD @@ -51,7 +51,7 @@ envoy_cc_test( "@com_googlesource_quiche//:quic_core_types_lib", "@com_googlesource_quiche//:quic_platform_expect_bug", "@com_googlesource_quiche//:quic_platform_mem_slice_span", - "@com_googlesource_quiche//:quic_platform_mem_slice_storage_lib", + "@com_googlesource_quiche//:quic_platform_mem_slice_storage", "@com_googlesource_quiche//:quic_platform_mock_log", "@com_googlesource_quiche//:quic_platform_port_utils", "@com_googlesource_quiche//:quic_platform_sleep", diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 21ef6280e234..d34f50ef8373 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -105,6 +105,7 @@ GOAWAY GRPC GTEST GiB +GURL HC HCM HDS From a12b9194d033b97e93a8784cf3ec463179fde856 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Tue, 9 Jul 2019 01:24:25 +0530 Subject: [PATCH 120/542] server: add initialization time stat (#7483) Description: Adds a server stat to capture the initialization time of Envoy. Risk Level: Low Testing: Added automated test Docs Changes: Updated Release Notes: Added Signed-off-by: Rama Chavali --- docs/root/configuration/statistics.rst | 1 + docs/root/intro/version_history.rst | 3 ++- source/server/server.cc | 7 +++++-- source/server/server.h | 11 ++++++++--- test/server/server_test.cc | 2 ++ 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/root/configuration/statistics.rst b/docs/root/configuration/statistics.rst index f9ddd2005a60..1e79f39a0bf9 100644 --- a/docs/root/configuration/statistics.rst +++ b/docs/root/configuration/statistics.rst @@ -23,6 +23,7 @@ Server related statistics are rooted at *server.* with following statistics: version, Gauge, Integer represented version number based on SCM revision days_until_first_cert_expiring, Gauge, Number of days until the next certificate being managed will expire hot_restart_epoch, Gauge, Current hot restart epoch + initialization_time_ms, Histogram, Total time taken for Envoy initialization in milliseconds. This is the time from server start-up until the worker threads are ready to accept new connections debug_assertion_failures, Counter, Number of debug assertion failures detected in a release build if compiled with `--define log_debug_assert_in_release=enabled` or zero otherwise File system diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 29b24fcd4db2..c7b1e05a8177 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -79,10 +79,11 @@ Version history * sandbox: added :ref:`CSRF sandbox `. * server: ``--define manual_stamp=manual_stamp`` was added to allow server stamping outside of binary rules. more info in the `bazel docs `_. +* server: added :ref:`Server State ` statistic. +* server: added :ref:`initialization_time_ms` statistic. * subset: added :ref:`list_as_any` option to the subset lb which allows matching metadata against any of the values in a list value on the endpoints. -* server: added :ref:`Server State ` statistic. * tool: added :repo:`proto ` support for :ref:`router check tool ` tests. * tracing: add trace sampling configuration to the route, to override the route level. * upstream: added :ref:`upstream_cx_pool_overflow ` for the connection pool circuit breaker. diff --git a/source/server/server.cc b/source/server/server.cc index 74fcc9a8031f..caed6af918d3 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -278,8 +278,11 @@ void InstanceImpl::initialize(const Options& options, const std::string server_stats_prefix = "server."; server_stats_ = std::make_unique( ServerStats{ALL_SERVER_STATS(POOL_COUNTER_PREFIX(stats_store_, server_stats_prefix), - POOL_GAUGE_PREFIX(stats_store_, server_stats_prefix))}); + POOL_GAUGE_PREFIX(stats_store_, server_stats_prefix), + POOL_HISTOGRAM_PREFIX(stats_store_, server_stats_prefix))}); + initialization_timer_ = + std::make_unique(server_stats_->initialization_time_ms_, timeSource()); server_stats_->concurrency_.set(options_.concurrency()); server_stats_->hot_restart_epoch_.set(options_.restartEpoch()); @@ -410,7 +413,7 @@ void InstanceImpl::initialize(const Options& options, void InstanceImpl::startWorkers() { listener_manager_->startWorkers(*guard_dog_); - + initialization_timer_->complete(); // At this point we are ready to take traffic and all listening ports are up. Notify our parent // if applicable that they can stop listening and drain. restarter_.drainParentListeners(); diff --git a/source/server/server.h b/source/server/server.h index 21965812db25..fba73e1b2e58 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -15,6 +15,7 @@ #include "envoy/server/tracer_config.h" #include "envoy/ssl/context_manager.h" #include "envoy/stats/stats_macros.h" +#include "envoy/stats/timespan.h" #include "envoy/tracing/http_tracer.h" #include "common/access_log/access_log_manager_impl.h" @@ -47,7 +48,7 @@ namespace Server { /** * All server wide stats. @see stats_macros.h */ -#define ALL_SERVER_STATS(COUNTER, GAUGE) \ +#define ALL_SERVER_STATS(COUNTER, GAUGE, HISTOGRAM) \ COUNTER(debug_assertion_failures) \ GAUGE(concurrency, NeverImport) \ GAUGE(days_until_first_cert_expiring, Accumulate) \ @@ -59,10 +60,11 @@ namespace Server { GAUGE(state, NeverImport) \ GAUGE(total_connections, Accumulate) \ GAUGE(uptime, Accumulate) \ - GAUGE(version, NeverImport) + GAUGE(version, NeverImport) \ + HISTOGRAM(initialization_time_ms) struct ServerStats { - ALL_SERVER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) + ALL_SERVER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT) }; /** @@ -275,6 +277,9 @@ class InstanceImpl : Logger::Loggable, std::unique_ptr process_context_; std::unique_ptr heap_shrinker_; const std::thread::id main_thread_id_; + // initialization_time is a histogram for tracking the initialization time across hot restarts + // whenever we have support for histogram merge across hot restarts. + Stats::TimespanPtr initialization_timer_; using LifecycleNotifierCallbacks = std::list; using LifecycleNotifierCompletionCallbacks = std::list; diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 48ce4bcc7d11..6839c8439fe4 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -314,6 +314,8 @@ TEST_P(ServerInstanceImplTest, EmptyShutdownLifecycleNotifications) { auto server_thread = startTestServer("test/server/node_bootstrap.yaml", false); server_->dispatcher().post([&] { server_->shutdown(); }); server_thread->join(); + // Validate that initialization_time histogram value has been set. + EXPECT_TRUE(stats_store_.histogram("server.initialization_time").used()); } TEST_P(ServerInstanceImplTest, LifecycleNotifications) { From 3b1983fe51ad4cf3be5b9be6e97760bd04029deb Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 8 Jul 2019 16:04:46 -0400 Subject: [PATCH 121/542] http: fixing a bug with HTTP1.0 HTTP1.1 pipelining (#7464) Fixing a bug where HTTP version was not set between requests. Risk Level: Medium (few people using HTTP/1.0) Testing: new unit test Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- source/common/http/http1/codec_impl.cc | 4 +++ test/common/http/http1/codec_impl_test.cc | 44 +++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index cdc166e9ac85..edab3707018e 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -477,6 +477,10 @@ void ConnectionImpl::onMessageCompleteBase() { void ConnectionImpl::onMessageBeginBase() { ENVOY_CONN_LOG(trace, "message begin", connection_); + // Make sure that if HTTP/1.0 and HTTP/1.1 requests share a connection Envoy correctly sets + // protocol for each request. Envoy defaults to 1.1 but sets the protocol to 1.0 where applicable + // in onHeadersCompleteBase + protocol_ = Protocol::Http11; ASSERT(!current_header_map_); current_header_map_ = std::make_unique(); header_parsing_state_ = HeaderParsingState::Field; diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 4d59984d6be7..c65d0ba18cb3 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -145,6 +145,50 @@ TEST_F(Http1ServerConnectionImplTest, Http10Absolute) { expectHeadersTest(Protocol::Http10, true, buffer, expected_headers); } +TEST_F(Http1ServerConnectionImplTest, Http10MultipleResponses) { + initialize(); + + Http::MockStreamDecoder decoder; + // Send a full HTTP/1.0 request and proxy a response. + { + Buffer::OwnedImpl buffer( + "GET /foobar HTTP/1.0\r\nHost: www.somewhere.com\r\nconnection: keep-alive\r\n\r\n"); + Http::StreamEncoder* response_encoder = nullptr; + EXPECT_CALL(callbacks_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamEncoder& encoder, bool) -> Http::StreamDecoder& { + response_encoder = &encoder; + return decoder; + })); + + EXPECT_CALL(decoder, decodeHeaders_(_, true)).Times(1); + codec_->dispatch(buffer); + + std::string output; + ON_CALL(connection_, write(_, _)).WillByDefault(AddBufferToString(&output)); + TestHeaderMapImpl headers{{":status", "200"}}; + response_encoder->encodeHeaders(headers, true); + EXPECT_EQ("HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n", output); + EXPECT_EQ(Protocol::Http10, codec_->protocol()); + } + + // Now send an HTTP/1.1 request and make sure the protocol is tracked correctly. + { + TestHeaderMapImpl expected_headers{ + {":authority", "www.somewhere.com"}, {":path", "/foobar"}, {":method", "GET"}}; + Buffer::OwnedImpl buffer("GET /foobar HTTP/1.1\r\nHost: www.somewhere.com\r\n\r\n"); + + Http::StreamEncoder* response_encoder = nullptr; + EXPECT_CALL(callbacks_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamEncoder& encoder, bool) -> Http::StreamDecoder& { + response_encoder = &encoder; + return decoder; + })); + EXPECT_CALL(decoder, decodeHeaders_(_, true)).Times(1); + codec_->dispatch(buffer); + EXPECT_EQ(Protocol::Http11, codec_->protocol()); + } +} + TEST_F(Http1ServerConnectionImplTest, Http11AbsolutePath1) { initialize(); From 3c2d0eaaa32285ddfc99f22902650b86b982ae39 Mon Sep 17 00:00:00 2001 From: ivitjuk <875244+ivitjuk@users.noreply.github.com> Date: Mon, 8 Jul 2019 13:26:12 -0700 Subject: [PATCH 122/542] Aws sigv4 metadata fetcher (#7303) Signed-off-by: Ivan Vitjuk --- .../extensions/filters/http/common/aws/BUILD | 1 + .../filters/http/common/aws/utility.cc | 51 ++++++ .../filters/http/common/aws/utility.h | 18 ++- test/integration/BUILD | 20 +++ .../aws_metadata_fetcher_integration_test.cc | 152 ++++++++++++++++++ tools/check_format.py | 1 + 6 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 test/integration/aws_metadata_fetcher_integration_test.cc diff --git a/source/extensions/filters/http/common/aws/BUILD b/source/extensions/filters/http/common/aws/BUILD index db1057ca56d0..caf07a7a5db6 100644 --- a/source/extensions/filters/http/common/aws/BUILD +++ b/source/extensions/filters/http/common/aws/BUILD @@ -59,6 +59,7 @@ envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], hdrs = ["utility.h"], + external_deps = ["curl"], deps = [ "//source/common/common:utility_lib", "//source/common/http:headers_lib", diff --git a/source/extensions/filters/http/common/aws/utility.cc b/source/extensions/filters/http/common/aws/utility.cc index 88836f7b7721..0410ef55df92 100644 --- a/source/extensions/filters/http/common/aws/utility.cc +++ b/source/extensions/filters/http/common/aws/utility.cc @@ -4,6 +4,7 @@ #include "common/common/utility.h" #include "absl/strings/str_join.h" +#include "curl/curl.h" namespace Envoy { namespace Extensions { @@ -87,6 +88,56 @@ Utility::joinCanonicalHeaderNames(const std::map& cano }); } +static size_t curlCallback(char* ptr, size_t, size_t nmemb, void* data) { + auto buf = static_cast(data); + buf->append(ptr, nmemb); + return nmemb; +} + +absl::optional Utility::metadataFetcher(const std::string& host, + const std::string& path, + const std::string& auth_token) { + static const size_t MAX_RETRIES = 4; + static const std::chrono::milliseconds RETRY_DELAY{1000}; + static const std::chrono::seconds TIMEOUT{5}; + + CURL* const curl = curl_easy_init(); + if (!curl) { + return absl::nullopt; + }; + + const std::string url = fmt::format("http://{}/{}", host, path); + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, TIMEOUT.count()); + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + + std::string buffer; + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlCallback); + + struct curl_slist* headers = nullptr; + if (!auth_token.empty()) { + const std::string auth = fmt::format("Authorization: {}", auth_token); + headers = curl_slist_append(headers, auth.c_str()); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + } + + for (size_t retry = 0; retry < MAX_RETRIES; retry++) { + const CURLcode res = curl_easy_perform(curl); + if (res == CURLE_OK) { + break; + } + ENVOY_LOG_MISC(debug, "Could not fetch AWS metadata: {}", curl_easy_strerror(res)); + buffer.clear(); + std::this_thread::sleep_for(RETRY_DELAY); + } + + curl_easy_cleanup(curl); + curl_slist_free_all(headers); + + return buffer.empty() ? absl::nullopt : absl::optional(buffer); +} + } // namespace Aws } // namespace Common } // namespace HttpFilters diff --git a/source/extensions/filters/http/common/aws/utility.h b/source/extensions/filters/http/common/aws/utility.h index ead9339fb826..c43236ceb242 100644 --- a/source/extensions/filters/http/common/aws/utility.h +++ b/source/extensions/filters/http/common/aws/utility.h @@ -39,10 +39,26 @@ class Utility { */ static std::string joinCanonicalHeaderNames(const std::map& canonical_headers); + + /** + * Fetch AWS instance or task metadata. + * + * @param host host or ip address of the metadata endpoint. + * @param path path of the metadata document. + * @auth_token authentication token to pass in the request, empty string indicates no auth. + * @return Metadata document or nullopt in case if unable to fetch it. + * + * @note In case of an error, function will log ENVOY_LOG_MISC(debug) message. + * + * @note This is not main loop safe method as it is blocking. It is intended to be used from the + * gRPC auth plugins that are able to schedule blocking plugins on a different thread. + */ + static absl::optional + metadataFetcher(const std::string& host, const std::string& path, const std::string& auth_token); }; } // namespace Aws } // namespace Common } // namespace HttpFilters } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/test/integration/BUILD b/test/integration/BUILD index ebd2696a39ac..cf1536d7910c 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -806,3 +806,23 @@ envoy_cc_test( "//test/test_common:utility_lib", ], ) + +envoy_cc_test( + name = "aws_metadata_fetcher_integration_test", + srcs = [ + "aws_metadata_fetcher_integration_test.cc", + ], + tags = ["exclusive"], + deps = [ + ":integration_lib", + "//source/common/common:fmt_lib", + "//source/extensions/filters/http/common/aws:utility_lib", + "//source/extensions/filters/http/fault:config", + "//source/extensions/filters/http/fault:fault_filter_lib", + "//source/extensions/filters/http/router:config", + "//source/extensions/filters/network/echo:config", + "//source/extensions/filters/network/http_connection_manager:config", + "//test/server:utility_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/integration/aws_metadata_fetcher_integration_test.cc b/test/integration/aws_metadata_fetcher_integration_test.cc new file mode 100644 index 000000000000..eb08642bc51a --- /dev/null +++ b/test/integration/aws_metadata_fetcher_integration_test.cc @@ -0,0 +1,152 @@ +#include "common/common/fmt.h" + +#include "extensions/filters/http/common/aws/utility.h" + +#include "test/integration/integration.h" +#include "test/integration/utility.h" +#include "test/server/utility.h" +#include "test/test_common/utility.h" + +namespace Envoy { + +using Envoy::Extensions::HttpFilters::Common::Aws::Utility; + +class AwsMetadataIntegrationTestBase : public ::testing::Test, public BaseIntegrationTest { +public: + AwsMetadataIntegrationTestBase(int status_code, int delay_s) + : BaseIntegrationTest(Network::Address::IpVersion::v4, renderConfig(status_code, delay_s)) {} + + static std::string renderConfig(int status_code, int delay_s) { + return fmt::format(ConfigHelper::BASE_CONFIG + R"EOF( + filter_chains: + filters: + name: envoy.http_connection_manager + config: + stat_prefix: metadata_test + http_filters: + - name: envoy.fault + config: + delay: + fixed_delay: + seconds: {} + nanos: {} + percentage: + numerator: 100 + denominator: HUNDRED + - name: envoy.router + codec_type: HTTP1 + route_config: + virtual_hosts: + name: metadata_endpoint + routes: + - name: auth_route + direct_response: + status: {} + body: + inline_string: METADATA_VALUE_WITH_AUTH + match: + prefix: "/" + headers: + - name: Authorization + exact_match: AUTH_TOKEN + - name: no_auth_route + direct_response: + status: {} + body: + inline_string: METADATA_VALUE + match: + prefix: "/" + domains: "*" + name: route_config_0 + )EOF", + delay_s, delay_s > 0 ? 0 : 1000, status_code, status_code); + } + + void SetUp() override { BaseIntegrationTest::initialize(); } + + void TearDown() override { + test_server_.reset(); + fake_upstreams_.clear(); + } +}; + +class AwsMetadataIntegrationTestSuccess : public AwsMetadataIntegrationTestBase { +public: + AwsMetadataIntegrationTestSuccess() : AwsMetadataIntegrationTestBase(200, 0) {} +}; + +TEST_F(AwsMetadataIntegrationTestSuccess, Success) { + const auto endpoint = fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(version_), + lookupPort("listener_0")); + const auto response = Utility::metadataFetcher(endpoint, "", ""); + + ASSERT_TRUE(response.has_value()); + EXPECT_EQ("METADATA_VALUE", *response); + + ASSERT_NE(nullptr, test_server_->counter("http.metadata_test.downstream_rq_completed")); + EXPECT_EQ(1, test_server_->counter("http.metadata_test.downstream_rq_completed")->value()); +} + +TEST_F(AwsMetadataIntegrationTestSuccess, AuthToken) { + const auto endpoint = fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(version_), + lookupPort("listener_0")); + const auto response = Utility::metadataFetcher(endpoint, "", "AUTH_TOKEN"); + + ASSERT_TRUE(response.has_value()); + EXPECT_EQ("METADATA_VALUE_WITH_AUTH", *response); + + ASSERT_NE(nullptr, test_server_->counter("http.metadata_test.downstream_rq_completed")); + EXPECT_EQ(1, test_server_->counter("http.metadata_test.downstream_rq_completed")->value()); +} + +class AwsMetadataIntegrationTestFailure : public AwsMetadataIntegrationTestBase { +public: + AwsMetadataIntegrationTestFailure() : AwsMetadataIntegrationTestBase(503, 0) {} +}; + +TEST_F(AwsMetadataIntegrationTestFailure, Failure) { + const auto endpoint = fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(version_), + lookupPort("listener_0")); + + const auto start_time = timeSystem().monotonicTime(); + const auto response = Utility::metadataFetcher(endpoint, "", ""); + const auto end_time = timeSystem().monotonicTime(); + + EXPECT_FALSE(response.has_value()); + + // Verify correct number of retries + ASSERT_NE(nullptr, test_server_->counter("http.metadata_test.downstream_rq_completed")); + EXPECT_EQ(4, test_server_->counter("http.metadata_test.downstream_rq_completed")->value()); + + // Verify correct sleep time between retries: 4 * 1000 = 4000 + EXPECT_LE(4000, + std::chrono::duration_cast(end_time - start_time).count()); +} + +class AwsMetadataIntegrationTestTimeout : public AwsMetadataIntegrationTestBase { +public: + AwsMetadataIntegrationTestTimeout() : AwsMetadataIntegrationTestBase(200, 10) {} +}; + +TEST_F(AwsMetadataIntegrationTestTimeout, Timeout) { + const auto endpoint = fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(version_), + lookupPort("listener_0")); + + const auto start_time = timeSystem().monotonicTime(); + const auto response = Utility::metadataFetcher(endpoint, "", ""); + const auto end_time = timeSystem().monotonicTime(); + + EXPECT_FALSE(response.has_value()); + + // We do now check http.metadata_test.downstream_rq_completed value here because it's + // behavior is different between Linux and Mac when Curl disconnects on timeout. On Mac it is + // incremented, while on Linux it is not. + + // Verify correct sleep time between retries: 4 * 5000 = 20000 + EXPECT_LE(20000, + std::chrono::duration_cast(end_time - start_time).count()); + EXPECT_GT(40000, + std::chrono::duration_cast(end_time - start_time).count()); +} + +} // namespace Envoy diff --git a/tools/check_format.py b/tools/check_format.py index ac8f4c2fd38c..b24a0514fcd0 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -30,6 +30,7 @@ # definitions for real-world time, the construction of them in main(), and perf annotation. # For now it includes the validation server but that really should be injected too. REAL_TIME_WHITELIST = ("./source/common/common/utility.h", + "./source/extensions/filters/http/common/aws/utility.cc", "./source/common/event/real_time_system.cc", "./source/common/event/real_time_system.h", "./source/exe/main_common.cc", "./source/exe/main_common.h", "./source/server/config_validation/server.cc", From 19b1a6fde0f677b80d1b4d0ea605beefe4c5ff6a Mon Sep 17 00:00:00 2001 From: Venil Noronha Date: Mon, 8 Jul 2019 13:29:12 -0700 Subject: [PATCH 123/542] Fix deprecation warning (#7490) This fixes the deprecation warning by adding a missing space character. Signed-off-by: Venil Noronha --- source/common/protobuf/utility.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index 70e5cb02d070..50a95b419974 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -201,7 +201,7 @@ void MessageUtil::checkForDeprecation(const Protobuf::Message& message, Runtime: const char fatal_error[] = " If continued use of this field is absolutely necessary, see " "https://www.envoyproxy.io/docs/envoy/latest/configuration/runtime" - "#using-runtime-overrides-for-deprecated-features for how to apply a temporary and" + "#using-runtime-overrides-for-deprecated-features for how to apply a temporary and " "highly discouraged override."; throw ProtoValidationException(err + fatal_error, message); } From 858095745866a8cc62bc6723a11005581ae00983 Mon Sep 17 00:00:00 2001 From: Jyoti Mahapatra <49211422+jyotimahapatra@users.noreply.github.com> Date: Mon, 8 Jul 2019 13:30:28 -0700 Subject: [PATCH 124/542] Adding missing description of the runtime field in router check config (#7491) Signed-off-by: Jyoti Mahapatra --- docs/root/configuration/tools/router_check.rst | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/root/configuration/tools/router_check.rst b/docs/root/configuration/tools/router_check.rst index 7698430b03db..fa6dd076eafb 100644 --- a/docs/root/configuration/tools/router_check.rst +++ b/docs/root/configuration/tools/router_check.rst @@ -46,11 +46,12 @@ expects a cluster name match of "instant-server".:: - test_name: ..., input: authority: ..., - path": ..., - method": ..., - internal" : ..., - random_value" : ..., - ssl" : ..., + path: ..., + method: ..., + internal: ..., + random_value: ..., + ssl: ..., + runtime: ..., - additional_headers: key: ..., value: ... @@ -97,6 +98,12 @@ input a client issuing a request via http or https. By default ssl is false which corresponds to x-forwarded-proto set to http. + runtime + *(optional, string)* A string representing the runtime setting to enable for the test. The runtime + setting along with the random_value is used by the router to decide if the route should be enabled. + Only a random_value lesser than the fractional percentage defined on the route entry enables the + route. + additional_headers *(optional, array)* Additional headers to be added as input for route determination. The "authority", "path", "method", "x-forwarded-proto", and "x-envoy-internal" fields are specified by the other config From 34dbd5c544ba0773d7911fc514ec6d7e22abf660 Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Mon, 8 Jul 2019 20:48:42 -0700 Subject: [PATCH 125/542] Fix redis_cluster_test for multiple DNS resolution scenario. (#7495) Signed-off-by: Henry Yang --- .../clusters/redis/redis_cluster_test.cc | 66 +++++++++++++------ 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index ca2d35893111..ef1603d87b76 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -425,22 +425,6 @@ class RedisClusterTest : public testing::Test, EXPECT_CALL(active_dns_query_, cancel()); } - void testRedisResolve() { - EXPECT_CALL(dispatcher_, createTimer_(_)); - RedisCluster::RedisDiscoverySession discovery_session(*cluster_, *this); - auto dns_response = - TestUtility::makeDnsResponse(std::list({"127.0.0.1", "127.0.0.2"})); - discovery_session.registerDiscoveryAddress(std::move(dns_response), 22120); - expectRedisResolve(true); - discovery_session.startResolveRedis(); - - // 2nd startResolveRedis() call will be a no-opt until the first startResolve is done. - discovery_session.startResolveRedis(); - - // Make sure cancel is called. - EXPECT_CALL(pool_request_, cancel()); - } - Stats::IsolatedStoreImpl stats_store_; Ssl::MockContextManager ssl_context_manager_; std::shared_ptr> dns_resolver_{ @@ -722,9 +706,53 @@ TEST_F(RedisClusterTest, DnsDiscoveryResolverBasic) { testDnsResolve("foo.bar.com", 22120); } -TEST_F(RedisClusterTest, RedisDiscoveryResolverBasic) { - setupFromV2Yaml(BasicConfig); - testRedisResolve(); +TEST_F(RedisClusterTest, MultipleDnsDiscovery) { + const std::string config = R"EOF( + name: name + connect_timeout: 0.25s + dns_lookup_family: V4_ONLY + hosts: + - socket_address: + address: foo.bar.com + port_value: 22120 + - socket_address: + address: foo1.bar.com + port_value: 22120 + cluster_type: + name: envoy.clusters.redis + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + cluster_refresh_rate: 4s + cluster_refresh_timeout: 0.25s + )EOF"; + + setupFromV2Yaml(config); + + // Only single in-flight "cluster slots" call. + expectRedisResolve(true); + + ReadyWatcher dns_resolve_1; + ReadyWatcher dns_resolve_2; + + EXPECT_CALL(*dns_resolver_, resolve("foo.bar.com", _, _)) + .WillOnce(Invoke([&](const std::string&, Network::DnsLookupFamily, + Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { + cb(TestUtility::makeDnsResponse(std::list({"127.0.0.1", "127.0.0.2"}))); + return nullptr; + })); + + EXPECT_CALL(*dns_resolver_, resolve("foo1.bar.com", _, _)) + .WillOnce(Invoke([&](const std::string&, Network::DnsLookupFamily, + Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { + cb(TestUtility::makeDnsResponse(std::list({"127.0.0.3", "127.0.0.4"}))); + return nullptr; + })); + + cluster_->initialize([&]() -> void { initialized_.ready(); }); + + // Pending RedisResolve will call cancel in the destructor. + EXPECT_CALL(pool_request_, cancel()); } } // namespace Redis From 3d6d6fa67530c2915a863ea775996f813b7a6e3b Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 8 Jul 2019 23:49:47 -0400 Subject: [PATCH 126/542] lds: fix listener removal crash before server start (#7461) Signed-off-by: Asra Ali --- source/server/listener_manager_impl.cc | 9 +++++++-- test/integration/ads_integration_test.cc | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 0b1f58741655..82f7b2261084 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -741,9 +741,14 @@ bool ListenerManagerImpl::removeListener(const std::string& name) { warming_listeners_.erase(existing_warming_listener); } - // If there is an active listener it needs to be moved to draining. + // If there is an active listener it needs to be moved to draining after workers have started, or + // destroyed directly. if (existing_active_listener != active_listeners_.end()) { - drainListener(std::move(*existing_active_listener)); + // Listeners in active_listeners_ are added to workers after workers start, so we drain + // listeners only after this occurs. + if (workers_started_) { + drainListener(std::move(*existing_active_listener)); + } active_listeners_.erase(existing_active_listener); } diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 8866d05da640..3ef9f4df589e 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -954,5 +954,26 @@ TEST_P(AdsIntegrationTest, XdsBatching) { initialize(); } +// Validates that listeners can be removed before server start. +TEST_P(AdsIntegrationTest, ListenerDrainBeforeServerStart) { + initialize(); + + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("cluster_0")}, + {buildCluster("cluster_0")}, {}, "1"); + sendDiscoveryResponse( + Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, + {buildClusterLoadAssignment("cluster_0")}, {}, "1"); + + sendDiscoveryResponse( + Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, + {buildListener("listener_0", "route_config_0")}, {}, "1"); + test_server_->waitForGaugeGe("listener_manager.total_listeners_active", 1); + + // Remove listener. + sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, {}, "1"); + test_server_->waitForGaugeEq("listener_manager.total_listeners_active", 0); +} + } // namespace } // namespace Envoy From 18e47ec1fa236b7ed7e645d5e7879f6b20c0fbcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Tue, 9 Jul 2019 09:40:52 -0400 Subject: [PATCH 127/542] Drop unused variables in utility.cc (#7494) The HMAC() and EVP_DigestFinal() functions accept nullptr insted of an int* when the caller doesn't care about the length. Signed-off-by: Raul Gutierrez Segales --- source/common/crypto/utility.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/common/crypto/utility.cc b/source/common/crypto/utility.cc index fbaff483eb3e..08e07c556360 100644 --- a/source/common/crypto/utility.cc +++ b/source/common/crypto/utility.cc @@ -25,8 +25,7 @@ std::vector Utility::getSha256Digest(const Buffer::Instance& buffer) { rc = EVP_DigestUpdate(ctx, slice.mem_, slice.len_); RELEASE_ASSERT(rc == 1, "Failed to update digest"); } - unsigned int len; - rc = EVP_DigestFinal(ctx, digest.data(), &len); + rc = EVP_DigestFinal(ctx, digest.data(), nullptr); RELEASE_ASSERT(rc == 1, "Failed to finalize digest"); EVP_MD_CTX_free(ctx); return digest; @@ -35,10 +34,9 @@ std::vector Utility::getSha256Digest(const Buffer::Instance& buffer) { std::vector Utility::getSha256Hmac(const std::vector& key, absl::string_view message) { std::vector hmac(SHA256_DIGEST_LENGTH); - unsigned int len; const auto ret = HMAC(EVP_sha256(), key.data(), key.size(), reinterpret_cast(message.data()), - message.size(), hmac.data(), &len); + message.size(), hmac.data(), nullptr); RELEASE_ASSERT(ret != nullptr, "Failed to create HMAC"); return hmac; } From 5e0130fc5ae50bcd7dff7cb20bfb0163b8711b7b Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 9 Jul 2019 10:28:56 -0400 Subject: [PATCH 128/542] stats: Remove field from GaugeImpl that was not used, by moving it to the counter-specific section. (#7477) * Remove field from GaugeImpl that was not used, by moving it to the counter-specific section. (#7477) Signed-off-by: Joshua Marantz --- source/common/stats/heap_stat_data.cc | 10 +++++----- test/integration/stats_integration_test.cc | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/common/stats/heap_stat_data.cc b/source/common/stats/heap_stat_data.cc index 1ea8768e6f1b..2bb3d7646db8 100644 --- a/source/common/stats/heap_stat_data.cc +++ b/source/common/stats/heap_stat_data.cc @@ -86,11 +86,9 @@ template class StatsSharedImpl : public MetricImpl protected: HeapStatDataAllocator& alloc_; - // Holds backing store for both CounterImpl and GaugeImpl. This provides a level - // of indirection needed to enable stats created with the same name from - // different scopes to share the same value. + // Holds backing store shared by both CounterImpl and GaugeImpl. CounterImpl + // adds another field, pending_increment_, that is not used in Gauge. std::atomic value_{0}; - std::atomic pending_increment_{0}; std::atomic flags_{0}; std::atomic ref_count_{0}; }; @@ -113,8 +111,10 @@ class CounterImpl : public StatsSharedImpl { void inc() override { add(1); } uint64_t latch() override { return pending_increment_.exchange(0); } void reset() override { value_ = 0; } - bool used() const override { return flags_ & Flags::Used; } uint64_t value() const override { return value_; } + +private: + std::atomic pending_increment_{0}; }; class GaugeImpl : public StatsSharedImpl { diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 112475f923cf..a404e8de89d5 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -209,6 +209,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // 2019/06/17 7243 49412 49700 macros for exact/upper-bound memory checks // 2019/06/29 7364 45685 46000 combine 2 levels of stat ref-counting into 1 // 2019/06/30 7428 42742 43000 remove stats multiple inheritance, inline HeapStatData + // 2019/07/06 7477 42742 43000 fork gauge representation to drop pending_increment_ // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -218,7 +219,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 42742); + EXPECT_MEMORY_EQ(m_per_cluster, 42742); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 43000); } From ba0b5a760acc0a445df47e7cc8bdbae47312776b Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 9 Jul 2019 12:00:39 -0700 Subject: [PATCH 129/542] Add _GNU_SOURCE to libevent build for Android builds (#7497) The Android NDK doesn't assume `_GNU_SOURCE`, clang only assumes it for C++ builds, not C builds. libevent uses `pipe2` which in the NDK is only exposed in the header when these variables are set. In bazel 0.27.0 these implicit use became an error. More details: - https://github.com/lyft/envoy-mobile/issues/116 - https://github.com/libevent/libevent/pull/850 Signed-off-by: Keith Smiley --- bazel/foreign_cc/BUILD | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 8b0b98ef0a43..00b870afa35a 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -99,6 +99,13 @@ envoy_cmake_external( # doesn't respect custom toolchains such as the Android NDK, # see https://github.com/bazelbuild/rules_foreign_cc/issues/252 "CMAKE_RANLIB": "", + # Force _GNU_SOURCE on for Android builds. This would be contained in + # a 'select' but the downstream macro uses a select on all of these + # options, and they cannot be nested. + # If https://github.com/bazelbuild/rules_foreign_cc/issues/289 is fixed + # this can be removed. + # More details https://github.com/lyft/envoy-mobile/issues/116 + "_GNU_SOURCE": "on", }, copy_pdb = True, lib_source = "@com_github_libevent_libevent//:all", From 655c9848642a4618b46fd30483d3f16156859415 Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Tue, 9 Jul 2019 13:52:15 -0700 Subject: [PATCH 130/542] docs: fix priority level algorithm in load balancing (#7489) Signed-off-by: Yan Xue --- .../arch_overview/upstream/load_balancing/panic_threshold.rst | 4 ++-- .../intro/arch_overview/upstream/load_balancing/priority.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst b/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst index 21f0f78ac4e3..80b062593a29 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst @@ -57,9 +57,9 @@ priority. +-------------+-------------+----------+--------------+----------+--------------+-------------+ | 71% | 71% | 99% | NO | 1% | NO | 100% | +-------------+-------------+----------+--------------+----------+--------------+-------------+ -| 50% | 60% | 50% | NO | 50% | NO | 100% | +| 50% | 60% | 70% | NO | 30% | NO | 100% | +-------------+-------------+----------+--------------+----------+--------------+-------------+ -| 25% | 100% | 25% | NO | 75% | NO | 100% | +| 25% | 100% | 35% | NO | 65% | NO | 100% | +-------------+-------------+----------+--------------+----------+--------------+-------------+ | 25% | 25% | 50% | YES | 50% | YES | 70% | +-------------+-------------+----------+--------------+----------+--------------+-------------+ diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/priority.rst b/docs/root/intro/arch_overview/upstream/load_balancing/priority.rst index 5cb256c0d8e3..23bc5fb6f22d 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/priority.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/priority.rst @@ -98,9 +98,9 @@ To sum this up in pseudo algorithms: health(P_X) = min(100, 1.4 * 100 * healthy_P_X_backends / total_P_X_backends) normalized_total_health = min(100, Σ(health(P_0)...health(P_X))) - priority_load(P_0) = min(100, health(P_0) / normalized_total_health) + priority_load(P_0) = min(100, health(P_0) * 100 / normalized_total_health) priority_load(P_X) = min(100 - Σ(priority_load(P_0)..priority_load(P_X-1)), - health(P_X) / normalized_total_health) + health(P_X) * 100 / normalized_total_health) Note: This sectioned talked about healthy priorities, but this also extends to :ref:`degraded priorities `. From 4675c788a32ff7669e1810980cff3c92b7e1a5fa Mon Sep 17 00:00:00 2001 From: htuch Date: Tue, 9 Jul 2019 21:06:47 -0400 Subject: [PATCH 131/542] ci: less invasive surgery in Bazel cache when running coverage. (#7356) Coverage runs are not sandboxed (due to the .gcno mucking), so to avoid strange behavior in runs beyond the initial one, we need to avoid writing to the runfiles without Bazel's knowledge. Today, we nuke the runfiles directory to cleanup on a previous run's .gcno files, which are unknown to Bazel but copied there for the gcovr run. This is fine in CI, which always does a clean build, but for local developer flow, it breaks coverage when we re-run tests a second time, since Bazel doesn't like you nuking random files in the cache that it owns. This PR limits the removed files to just .gcno. Also, some tests were modifying runtime files in rundir vs. tmpdir. Some general runtime related testing robustness improvements also added. Risk level: Low Testing: Local CI Docker rebuilds. Signed-off-by: Harvey Tuch --- test/common/runtime/filesystem_setup.sh | 1 + test/common/runtime/runtime_impl_test.cc | 4 ++-- test/config/integration/server.json | 2 +- test/config/integration/server.yaml | 2 +- test/integration/BUILD | 4 ---- test/integration/hotrestart_test.sh | 4 ++-- test/run_envoy_bazel_coverage.sh | 10 +++++++++- 7 files changed, 16 insertions(+), 11 deletions(-) diff --git a/test/common/runtime/filesystem_setup.sh b/test/common/runtime/filesystem_setup.sh index 21420c86819d..cf95b6550a6e 100755 --- a/test/common/runtime/filesystem_setup.sh +++ b/test/common/runtime/filesystem_setup.sh @@ -6,6 +6,7 @@ TEST_DATA=test/common/runtime/test_data # Regular runtime tests. cd "${TEST_RUNDIR}" +rm -rf "${TEST_TMPDIR}/${TEST_DATA}" mkdir -p "${TEST_TMPDIR}/${TEST_DATA}" cp -RfL "${TEST_DATA}"/* "${TEST_TMPDIR}/${TEST_DATA}" chmod -R u+rwX "${TEST_TMPDIR}/${TEST_DATA}" diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 3c18b387e289..cb73ae6e73bf 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -104,12 +104,12 @@ class LoaderImplTest : public testing::Test { class DiskLoaderImplTest : public LoaderImplTest { public: - static void SetUpTestSuite() { + void SetUp() override { TestEnvironment::exec( {TestEnvironment::runfilesPath("test/common/runtime/filesystem_setup.sh")}); } - static void TearDownTestSuite() { + void TearDown() override { TestEnvironment::removePath(TestEnvironment::temporaryPath("test/common/runtime/test_data")); } diff --git a/test/config/integration/server.json b/test/config/integration/server.json index a564e0905efe..6dd4cc49fd0d 100644 --- a/test/config/integration/server.json +++ b/test/config/integration/server.json @@ -451,7 +451,7 @@ }, "runtime": { - "symlink_root": "{{ test_rundir }}/test/common/runtime/test_data/current", + "symlink_root": "{{ test_tmpdir }}/test/common/runtime/test_data/current", "subdirectory": "envoy", "override_subdirectory": "envoy_override" }, diff --git a/test/config/integration/server.yaml b/test/config/integration/server.yaml index 11c5bf992d08..91fdf5da5dfd 100644 --- a/test/config/integration/server.yaml +++ b/test/config/integration/server.yaml @@ -416,7 +416,7 @@ stats_sinks: tcp_cluster_name: statsd watchdog: {} runtime: - symlink_root: "{{ test_rundir }}/test/common/runtime/test_data/current" + symlink_root: "{{ test_tmpdir }}/test/common/runtime/test_data/current" subdirectory: envoy override_subdirectory: envoy_override admin: diff --git a/test/integration/BUILD b/test/integration/BUILD index cf1536d7910c..b7679977ecdb 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -165,8 +165,6 @@ envoy_sh_test( data = [ "test_utility.sh", "//source/exe:envoy-static", - "//test/common/runtime:filesystem_setup.sh", - "//test/common/runtime:filesystem_test_data", "//test/config/integration:server_config_files", "//tools:socket_passing", ], @@ -178,8 +176,6 @@ envoy_sh_test( data = [ "test_utility.sh", "//source/exe:envoy-static", - "//test/common/runtime:filesystem_setup.sh", - "//test/common/runtime:filesystem_test_data", "//test/config/integration:server_config_files", ], ) diff --git a/test/integration/hotrestart_test.sh b/test/integration/hotrestart_test.sh index 25def55875e7..02dd157c15b1 100755 --- a/test/integration/hotrestart_test.sh +++ b/test/integration/hotrestart_test.sh @@ -8,8 +8,8 @@ source "$TEST_RUNDIR/test/integration/test_utility.sh" JSON_TEST_ARRAY=() # Ensure that the runtime watch root exist. -mkdir -p "${TEST_RUNDIR}"/test/common/runtime/test_data/current/envoy -mkdir -p "${TEST_RUNDIR}"/test/common/runtime/test_data/current/envoy_override +mkdir -p "${TEST_TMPDIR}"/test/common/runtime/test_data/current/envoy +mkdir -p "${TEST_TMPDIR}"/test/common/runtime/test_data/current/envoy_override # Parameterize IPv4 and IPv6 testing. if [[ -z "${ENVOY_IP_TEST_VERSIONS}" ]] || [[ "${ENVOY_IP_TEST_VERSIONS}" == "all" ]] \ diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 57a7e2e28120..12632254f1f4 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -34,7 +34,15 @@ fi # This is where we are going to copy the .gcno files into. GCNO_ROOT=bazel-out/k8-dbg/bin/test/coverage/coverage_tests.runfiles/"${WORKSPACE}" echo " GCNO_ROOT=${GCNO_ROOT}" -rm -rf ${GCNO_ROOT} + +echo "Cleaning .gcno from previous coverage runs..." +NUM_PREVIOUS_GCNO_FILES=0 +for f in $(find -L "${GCNO_ROOT}" -name "*.gcno") +do + rm -f "${f}" + let NUM_PREVIOUS_GCNO_FILES=NUM_PREVIOUS_GCNO_FILES+1 +done +echo "Cleanup completed. ${NUM_PREVIOUS_GCNO_FILES} files deleted." # Make sure //test/coverage:coverage_tests is up-to-date. SCRIPT_DIR="$(realpath "$(dirname "$0")")" From e30470c5d139ab381c4bd74126eab6ff613c0dc7 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 10 Jul 2019 06:20:19 +0300 Subject: [PATCH 132/542] decompress: reset zlib buffer upon updating output buffer (#7469) Reset zlib's buffer upon updating output buffer, otherwise the content of zlib's buffer is copied again to output upon flushing HTTP stream. Risk Level: low Testing: standard tests Docs Changes: N/A Release Notes: N/A Signed-off-by: Dmitry Rozhkov --- .../decompressor/zlib_decompressor_impl.cc | 23 +++++++++++-------- .../decompressor/zlib_decompressor_impl.h | 1 + .../zlib_decompressor_impl_test.cc | 7 ++++++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/source/common/decompressor/zlib_decompressor_impl.cc b/source/common/decompressor/zlib_decompressor_impl.cc index 87c704becd69..3dfcdc9b27a7 100644 --- a/source/common/decompressor/zlib_decompressor_impl.cc +++ b/source/common/decompressor/zlib_decompressor_impl.cc @@ -45,19 +45,14 @@ void ZlibDecompressorImpl::decompress(const Buffer::Instance& input_buffer, zstream_ptr_->next_in = static_cast(input_slice.mem_); while (inflateNext()) { if (zstream_ptr_->avail_out == 0) { - output_buffer.add(static_cast(chunk_char_ptr_.get()), - chunk_size_ - zstream_ptr_->avail_out); - chunk_char_ptr_ = std::make_unique(chunk_size_); - zstream_ptr_->avail_out = chunk_size_; - zstream_ptr_->next_out = chunk_char_ptr_.get(); + updateOutput(output_buffer); } } } - const uint64_t n_output{chunk_size_ - zstream_ptr_->avail_out}; - if (n_output > 0) { - output_buffer.add(static_cast(chunk_char_ptr_.get()), n_output); - } + // Flush z_stream and reset its buffer. Otherwise the stale content of the buffer + // will pollute output upon the next call to decompress(). + updateOutput(output_buffer); } bool ZlibDecompressorImpl::inflateNext() { @@ -77,5 +72,15 @@ bool ZlibDecompressorImpl::inflateNext() { return true; } +void ZlibDecompressorImpl::updateOutput(Buffer::Instance& output_buffer) { + const uint64_t n_output = chunk_size_ - zstream_ptr_->avail_out; + if (n_output > 0) { + output_buffer.add(static_cast(chunk_char_ptr_.get()), n_output); + } + chunk_char_ptr_ = std::make_unique(chunk_size_); + zstream_ptr_->avail_out = chunk_size_; + zstream_ptr_->next_out = chunk_char_ptr_.get(); +} + } // namespace Decompressor } // namespace Envoy diff --git a/source/common/decompressor/zlib_decompressor_impl.h b/source/common/decompressor/zlib_decompressor_impl.h index dd9fc3a62fe5..5bcd03372ccf 100644 --- a/source/common/decompressor/zlib_decompressor_impl.h +++ b/source/common/decompressor/zlib_decompressor_impl.h @@ -45,6 +45,7 @@ class ZlibDecompressorImpl : public Decompressor { private: bool inflateNext(); + void updateOutput(Buffer::Instance& output_buffer); const uint64_t chunk_size_; bool initialized_; diff --git a/test/common/decompressor/zlib_decompressor_impl_test.cc b/test/common/decompressor/zlib_decompressor_impl_test.cc index bb715643e123..6460d2a8b878 100644 --- a/test/common/decompressor/zlib_decompressor_impl_test.cc +++ b/test/common/decompressor/zlib_decompressor_impl_test.cc @@ -113,6 +113,7 @@ TEST_F(ZlibDecompressorImplTest, CallingChecksum) { TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { Buffer::OwnedImpl buffer; Buffer::OwnedImpl accumulation_buffer; + Buffer::OwnedImpl empty_buffer; Envoy::Compressor::ZlibCompressorImpl compressor; compressor.init(Envoy::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, @@ -144,6 +145,12 @@ TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) { decompressor.decompress(accumulation_buffer, buffer); std::string decompressed_text{buffer.toString()}; + // Check decompressor's internal state isn't broken. + drainBuffer(buffer); + ASSERT_EQ(0, buffer.length()); + decompressor.decompress(empty_buffer, buffer); + ASSERT_EQ(0, buffer.length()); + ASSERT_EQ(compressor.checksum(), decompressor.checksum()); ASSERT_EQ(original_text.length(), decompressed_text.length()); EXPECT_EQ(original_text, decompressed_text); From ca333500877135a5b25235edd5094df12192002d Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Tue, 9 Jul 2019 20:23:26 -0700 Subject: [PATCH 133/542] filterchainmanager: add benchmark (#7443) Signed-off-by: Yuchen Dai --- test/server/BUILD | 17 ++ test/server/filter_chain_benchmark_test.cc | 237 +++++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 test/server/filter_chain_benchmark_test.cc diff --git a/test/server/BUILD b/test/server/BUILD index b39f9cfcbd43..a6601e6ca5c4 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -4,6 +4,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_fuzz_test", "envoy_cc_test", + "envoy_cc_test_binary", "envoy_cc_test_library", "envoy_package", "envoy_select_hot_restart", @@ -318,3 +319,19 @@ envoy_cc_test( "//test/test_common:utility_lib", ], ) + +envoy_cc_test_binary( + name = "filter_chain_benchmark_test", + srcs = ["filter_chain_benchmark_test.cc"], + external_deps = [ + "benchmark", + "googletest", + ], + deps = [ + "//source/server:filter_chain_manager_lib", + "//test/test_common:environment_lib", + "//test/mocks/network:network_mocks", + # tranport socket config registration + "//source/extensions/transport_sockets/tls:config", + ], +) diff --git a/test/server/filter_chain_benchmark_test.cc b/test/server/filter_chain_benchmark_test.cc new file mode 100644 index 000000000000..5070cc1271ae --- /dev/null +++ b/test/server/filter_chain_benchmark_test.cc @@ -0,0 +1,237 @@ +#include +#include + +#include "envoy/network/connection.h" +#include "envoy/network/listen_socket.h" +#include "envoy/protobuf/message_validator.h" + +#include "server/filter_chain_manager_impl.h" + +#include "extensions/transport_sockets/well_known_names.h" + +#include "test/mocks/network/mocks.h" +#include "test/test_common/environment.h" +#include "test/test_common/utility.h" + +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "benchmark/benchmark.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Server { + +namespace { + +class MockFilterChainFactoryBuilder : public FilterChainFactoryBuilder { + std::unique_ptr + buildFilterChain(const ::envoy::api::v2::listener::FilterChain&) const override { + // A place holder to be found + return std::make_unique(); + } +}; + +class MockConnectionSocket : public Network::ConnectionSocket { +public: + MockConnectionSocket() = default; + static std::unique_ptr + createMockConnectionSocket(uint16_t destination_port, const std::string& destination_address, + const std::string& server_name, const std::string& transport_protocol, + const std::vector& application_protocols, + const std::string& source_address, uint16_t source_port) { + auto res = std::make_unique(); + + if (absl::StartsWith(destination_address, "/")) { + res->local_address_ = std::make_shared(destination_address); + } else { + res->local_address_ = + Network::Utility::parseInternetAddress(destination_address, destination_port); + } + if (absl::StartsWith(source_address, "/")) { + res->remote_address_ = std::make_shared(source_address); + } else { + res->remote_address_ = Network::Utility::parseInternetAddress(source_address, source_port); + } + res->server_name_ = server_name; + res->transport_protocol_ = transport_protocol; + res->application_protocols_ = application_protocols; + return res; + } + + const Network::Address::InstanceConstSharedPtr& remoteAddress() const override { + return remote_address_; + } + + const Network::Address::InstanceConstSharedPtr& localAddress() const override { + return local_address_; + } + + absl::string_view detectedTransportProtocol() const override { return transport_protocol_; } + + absl::string_view requestedServerName() const override { return server_name_; } + const std::vector& requestedApplicationProtocols() const override { + return application_protocols_; + } + + // Wont call + Network::IoHandle& ioHandle() override { return *io_handle_; } + const Network::IoHandle& ioHandle() const override { return *io_handle_; } + + // Dummy method + void close() override {} + Network::Address::SocketType socketType() const override { + return Network::Address::SocketType::Stream; + } + void setLocalAddress(const Network::Address::InstanceConstSharedPtr&) override {} + void restoreLocalAddress(const Network::Address::InstanceConstSharedPtr&) override { return; } + void setRemoteAddress(const Network::Address::InstanceConstSharedPtr&) override {} + bool localAddressRestored() const override { return true; } + void setDetectedTransportProtocol(absl::string_view) override {} + void setRequestedApplicationProtocols(const std::vector&) override {} + void addOption(const OptionConstSharedPtr&) override {} + void addOptions(const OptionsSharedPtr&) override {} + const OptionsSharedPtr& options() const override { return options_; } + void setRequestedServerName(absl::string_view) override {} + +private: + Network::IoHandlePtr io_handle_; + OptionsSharedPtr options_; + Network::Address::InstanceConstSharedPtr local_address_; + Network::Address::InstanceConstSharedPtr remote_address_; + std::string server_name_; + std::string transport_protocol_; + std::vector application_protocols_; +}; +const char YamlHeader[] = R"EOF( + address: + socket_address: { address: 127.0.0.1, port_value: 1234 } + listener_filters: + - name: "envoy.listener.tls_inspector" + config: {} + filter_chains: + - filter_chain_match: + # empty + tls_context: + common_tls_context: + tls_certificates: + - certificate_chain: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_cert.pem" } + private_key: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_key.pem" } + session_ticket_keys: + keys: + - filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ticket_key_a")EOF"; +const char YamlSingleServer[] = R"EOF( + - filter_chain_match: + server_names: "server1.example.com" + transport_protocol: "tls" + tls_context: + common_tls_context: + tls_certificates: + - certificate_chain: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns_cert.pem" } + private_key: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns_key.pem" } + session_ticket_keys: + keys: + - filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ticket_key_a")EOF"; +const char YamlSingleDstPortTop[] = R"EOF( + - filter_chain_match: + destination_port: )EOF"; +const char YamlSingleDstPortBottom[] = R"EOF( + tls_context: + common_tls_context: + tls_certificates: + - certificate_chain: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_multiple_dns_cert.pem" } + private_key: { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_multiple_dns_key.pem" } + session_ticket_keys: + keys: + - filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ticket_key_a")EOF"; +} // namespace + +class FilterChainBenchmarkFixture : public benchmark::Fixture { +public: + void SetUp(const ::benchmark::State& state) { + int64_t input_size = state.range(0); + std::vector port_chains; + for (int i = 0; i < input_size; i++) { + port_chains.push_back(absl::StrCat(YamlSingleDstPortTop, 10000 + i, YamlSingleDstPortBottom)); + } + listener_yaml_config_ = TestEnvironment::substitute( + absl::StrCat(YamlHeader, YamlSingleServer, absl::StrJoin(port_chains, "")), + Network::Address::IpVersion::v4); + TestUtility::loadFromYaml(listener_yaml_config_, listener_config_); + filter_chains_ = listener_config_.filter_chains(); + } + absl::Span filter_chains_; + std::string listener_yaml_config_; + envoy::api::v2::Listener listener_config_; + MockFilterChainFactoryBuilder dummy_builder_; + FilterChainManagerImpl filter_chain_manager_{ + std::make_shared("127.0.0.1", 1234)}; +}; + +BENCHMARK_DEFINE_F(FilterChainBenchmarkFixture, FilterChainManagerBuildTest) +(::benchmark::State& state) { + for (auto _ : state) { + FilterChainManagerImpl filter_chain_manager{ + std::make_shared("127.0.0.1", 1234)}; + filter_chain_manager.addFilterChain(filter_chains_, dummy_builder_); + } +} + +BENCHMARK_DEFINE_F(FilterChainBenchmarkFixture, FilterChainFindTest) +(::benchmark::State& state) { + std::vector sockets; + sockets.reserve(state.range(0)); + for (int i = 0; i < state.range(0); i++) { + sockets.push_back(std::move(*MockConnectionSocket::createMockConnectionSocket( + 10000 + i, "127.0.0.1", "", "tls", {}, "8.8.8.8", 111))); + } + FilterChainManagerImpl filter_chain_manager{ + std::make_shared("127.0.0.1", 1234)}; + filter_chain_manager.addFilterChain(filter_chains_, dummy_builder_); + for (auto _ : state) { + for (int i = 0; i < state.range(0); i++) { + filter_chain_manager.findFilterChain(sockets[i]); + } + } +} +BENCHMARK_REGISTER_F(FilterChainBenchmarkFixture, FilterChainManagerBuildTest) + ->Ranges({ + // scale of the chains + {1, 4096}, + }); +BENCHMARK_REGISTER_F(FilterChainBenchmarkFixture, FilterChainFindTest) + ->Ranges({ + // scale of the chains + {1, 4096}, + }); + +/* +clang-format off + +Run on (32 X 2200 MHz CPU s) +CPU Caches: + L1 Data 32K (x16) + L1 Instruction 32K (x16) + L2 Unified 256K (x16) + L3 Unified 56320K (x1) +Load Average: 19.05, 9.89, 3.92 +------------------------------------------------------------------------------------------------------- +Benchmark Time CPU Iterations +------------------------------------------------------------------------------------------------------- +FilterChainBenchmarkFixture/FilterChainManagerBuildTest/1 51002 ns 50998 ns 12033 +FilterChainBenchmarkFixture/FilterChainManagerBuildTest/8 205175 ns 205161 ns 3782 +FilterChainBenchmarkFixture/FilterChainManagerBuildTest/64 1400449 ns 1400328 ns 485 +FilterChainBenchmarkFixture/FilterChainManagerBuildTest/512 10488106 ns 10485949 ns 62 +FilterChainBenchmarkFixture/FilterChainManagerBuildTest/4096 118373326 ns 117786871 ns 7 +FilterChainBenchmarkFixture/FilterChainFindTest/1 209 ns 209 ns 3257004 +FilterChainBenchmarkFixture/FilterChainFindTest/8 1780 ns 1780 ns 391501 +FilterChainBenchmarkFixture/FilterChainFindTest/64 16707 ns 16705 ns 42110 +FilterChainBenchmarkFixture/FilterChainFindTest/512 150220 ns 150072 ns 4675 +FilterChainBenchmarkFixture/FilterChainFindTest/4096 2227852 ns 2227703 ns 320 + +clang-format on +*/ +} // namespace Server +} // namespace Envoy +BENCHMARK_MAIN(); From e3973ec38ba8b6c18d9e626d39a9ab59dfa75548 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Tue, 9 Jul 2019 20:24:12 -0700 Subject: [PATCH 134/542] update the build doc to show how to add customzied bazel options. (#7513) Signed-off-by: Jianfei Hu --- ci/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/README.md b/ci/README.md index 0316f30e776d..d7375cfc10c2 100644 --- a/ci/README.md +++ b/ci/README.md @@ -70,10 +70,10 @@ The build artifact can be found in `/tmp/envoy-docker-build/envoy/source/exe/env `$ENVOY_DOCKER_BUILD_DIR` points). To leverage a [bazel remote cache](https://github.com/envoyproxy/envoy/tree/master/bazel#advanced-caching-setup) add the http_remote_cache endpoint to -the BAZEL_BUILD_OPTIONS environment variable +the BAZEL_BUILD_EXTRA_OPTIONS environment variable ```bash -BAZEL_BUILD_OPTIONS='--remote_http_cache=http://127.0.0.1:28080' ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.release' +./ci/run_envoy_docker.sh "BAZEL_BUILD_EXTRA_OPTIONS='--remote_http_cache=http://127.0.0.1:28080' ./ci/do_ci.sh bazel.release" ``` The `./ci/run_envoy_docker.sh './ci/do_ci.sh '` targets are: From 79b071cc1b6bf877c7c97a6e79b5be8e9b6ace38 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Tue, 9 Jul 2019 22:26:13 -0700 Subject: [PATCH 135/542] test: add config test for HeaderToMetadata filter (#7506) Adds coverage for source/extensions/filters/http/header_to_metadata_config.cc Risk Level: Low Testing: Included Docs Changes: N/A Release Notes: N/A Signed-off-by: Derek Argueta --- .../filters/http/header_to_metadata/BUILD | 11 +++ .../http/header_to_metadata/config_test.cc | 74 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 test/extensions/filters/http/header_to_metadata/config_test.cc diff --git a/test/extensions/filters/http/header_to_metadata/BUILD b/test/extensions/filters/http/header_to_metadata/BUILD index b007e87eff8a..0c2b80bff21b 100644 --- a/test/extensions/filters/http/header_to_metadata/BUILD +++ b/test/extensions/filters/http/header_to_metadata/BUILD @@ -20,3 +20,14 @@ envoy_extension_cc_test( "//test/mocks/server:server_mocks", ], ) + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_name = "envoy.filters.http.header_to_metadata", + deps = [ + "//source/extensions/filters/http/header_to_metadata:config", + "//test/mocks/server:server_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/http/header_to_metadata/config_test.cc b/test/extensions/filters/http/header_to_metadata/config_test.cc new file mode 100644 index 000000000000..ecc375fbcf06 --- /dev/null +++ b/test/extensions/filters/http/header_to_metadata/config_test.cc @@ -0,0 +1,74 @@ +#include + +#include "envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.pb.validate.h" + +#include "extensions/filters/http/header_to_metadata/config.h" + +#include "test/mocks/server/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace HeaderToMetadataFilter { + +using HeaderToMetadataProtoConfig = envoy::config::filter::http::header_to_metadata::v2::Config; + +TEST(HeaderToMetadataFilterConfigTest, InvalidEmptyHeader) { + const std::string yaml = R"EOF( +request_rules: +- header: "" + )EOF"; + + HeaderToMetadataProtoConfig proto_config; + EXPECT_THROW(TestUtility::loadFromYamlAndValidate(yaml, proto_config), ProtoValidationException); +} + +TEST(HeaderToMetadataFilterConfigTest, InvalidEmptyKey) { + const std::string yaml = R"EOF( +request_rules: + - header: x-version + on_header_present: + metadata_namespace: envoy.lb + key: "" + type: STRING + )EOF"; + + HeaderToMetadataProtoConfig proto_config; + EXPECT_THROW(TestUtility::loadFromYamlAndValidate(yaml, proto_config), ProtoValidationException); +} + +TEST(HeaderToMetadataFilterConfigTest, SimpleConfig) { + const std::string yaml = R"EOF( +request_rules: + - header: x-version + on_header_present: + metadata_namespace: envoy.lb + key: version + type: STRING + on_header_missing: + metadata_namespace: envoy.lb + key: default + value: 'true' + type: STRING + )EOF"; + + HeaderToMetadataProtoConfig proto_config; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + + testing::NiceMock context; + HeaderToMetadataConfig factory; + + Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, "stats", context); + Http::MockFilterChainFactoryCallbacks filter_callbacks; + EXPECT_CALL(filter_callbacks, addStreamFilter(_)); + cb(filter_callbacks); +} + +} // namespace HeaderToMetadataFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy From 0d26d05fe353315113b193bd0c53a768fbdd1c4f Mon Sep 17 00:00:00 2001 From: James Peach Date: Thu, 11 Jul 2019 01:37:52 +1000 Subject: [PATCH 136/542] Add socket option names for debugging. (#7457) Capture a human-readable name for a socket option so that if setting it fails, the log message can indicate which option was problematic. Signed-off-by: James Peach --- include/envoy/network/listen_socket.h | 29 +++++++++- .../common/network/socket_option_factory.cc | 9 ++- source/common/network/socket_option_impl.cc | 17 ++++-- source/common/network/socket_option_impl.h | 45 +++++++-------- ...dr_family_aware_socket_option_impl_test.cc | 56 +++++++++---------- .../network/socket_option_factory_test.cc | 20 +++---- .../common/network/socket_option_impl_test.cc | 41 +++++++++++--- test/common/network/socket_option_test.h | 4 +- .../upstream/cluster_manager_impl_test.cc | 29 +++++----- test/server/listener_manager_impl_test.cc | 2 +- 10 files changed, 149 insertions(+), 103 deletions(-) diff --git a/include/envoy/network/listen_socket.h b/include/envoy/network/listen_socket.h index b2c3dc472cec..4b39a71eda69 100644 --- a/include/envoy/network/listen_socket.h +++ b/include/envoy/network/listen_socket.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include "envoy/api/v2/core/base.pb.h" @@ -15,9 +16,31 @@ namespace Envoy { namespace Network { -// Optional variant of setsockopt(2) optname. The idea here is that if the option is not supported -// on a platform, we can make this the empty value. This allows us to avoid proliferation of #ifdef. -using SocketOptionName = absl::optional>; +// SocketOptionName is an optional value that captures the setsockopt(2) +// arguments. The idea here is that if a socket option is not supported +// on a platform, we can make this the empty value, which allows us to +// avoid #ifdef proliferation. +struct SocketOptionName { + SocketOptionName() = default; + SocketOptionName(const SocketOptionName&) = default; + SocketOptionName(int level, int option, const std::string& name) + : value_(std::make_tuple(level, option, name)) {} + + int level() const { return std::get<0>(value_.value()); } + int option() const { return std::get<1>(value_.value()); } + const std::string& name() const { return std::get<2>(value_.value()); } + + bool has_value() const { return value_.has_value(); } + bool operator==(const SocketOptionName& rhs) const { return value_ == rhs.value_; } + +private: + absl::optional> value_; +}; + +// ENVOY_MAKE_SOCKET_OPTION_NAME is a helper macro to generate a +// SocketOptionName with a descriptive string name. +#define ENVOY_MAKE_SOCKET_OPTION_NAME(level, option) \ + Network::SocketOptionName(level, option, #level "/" #option) /** * Base class for Sockets diff --git a/source/common/network/socket_option_factory.cc b/source/common/network/socket_option_factory.cc index 451db09aae37..287170b56ea8 100644 --- a/source/common/network/socket_option_factory.cc +++ b/source/common/network/socket_option_factory.cc @@ -1,5 +1,6 @@ #include "common/network/socket_option_factory.h" +#include "common/common/fmt.h" #include "common/network/addr_family_aware_socket_option_impl.h" #include "common/network/socket_option_impl.h" @@ -73,13 +74,15 @@ std::unique_ptr SocketOptionFactory::buildLiteralOptions( buf.append(socket_option.buf_value()); break; default: - ENVOY_LOG(warn, "Socket option specified with no or uknown value: {}", + ENVOY_LOG(warn, "Socket option specified with no or unknown value: {}", socket_option.DebugString()); continue; } options->emplace_back(std::make_shared( socket_option.state(), - Network::SocketOptionName(std::make_pair(socket_option.level(), socket_option.name())), + Network::SocketOptionName( + socket_option.level(), socket_option.name(), + fmt::format("{}/{}", socket_option.level(), socket_option.name())), buf)); } return options; @@ -107,7 +110,7 @@ std::unique_ptr SocketOptionFactory::buildRxQueueOverFlowOption #ifdef SO_RXQ_OVFL options->push_back(std::make_shared( envoy::api::v2::core::SocketOption::STATE_BOUND, - Network::SocketOptionName(std::make_pair(SOL_SOCKET, SO_RXQ_OVFL)), 1)); + ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_RXQ_OVFL), 1)); #endif return options; } diff --git a/source/common/network/socket_option_impl.cc b/source/common/network/socket_option_impl.cc index 2065309ed7a5..65b364810f21 100644 --- a/source/common/network/socket_option_impl.cc +++ b/source/common/network/socket_option_impl.cc @@ -13,13 +13,20 @@ namespace Network { bool SocketOptionImpl::setOption(Socket& socket, envoy::api::v2::core::SocketOption::SocketState state) const { if (in_state_ == state) { + if (!optname_.has_value()) { + ENVOY_LOG(warn, "Failed to set unsupported option on socket"); + return false; + } + const Api::SysCallIntResult result = SocketOptionImpl::setSocketOption(socket, optname_, value_); if (result.rc_ != 0) { - ENVOY_LOG(warn, "Setting option on socket failed: {}", strerror(result.errno_)); + ENVOY_LOG(warn, "Setting {} option on socket failed: {}", optname_.name(), + strerror(result.errno_)); return false; } } + return true; } @@ -39,15 +46,15 @@ SocketOptionImpl::getOptionDetails(const Socket&, bool SocketOptionImpl::isSupported() const { return optname_.has_value(); } Api::SysCallIntResult SocketOptionImpl::setSocketOption(Socket& socket, - Network::SocketOptionName optname, + const Network::SocketOptionName& optname, const absl::string_view value) { - if (!optname.has_value()) { return {-1, ENOTSUP}; } + auto& os_syscalls = Api::OsSysCallsSingleton::get(); - return os_syscalls.setsockopt(socket.ioHandle().fd(), optname.value().first, - optname.value().second, value.data(), value.size()); + return os_syscalls.setsockopt(socket.ioHandle().fd(), optname.level(), optname.option(), + value.data(), value.size()); } } // namespace Network diff --git a/source/common/network/socket_option_impl.h b/source/common/network/socket_option_impl.h index a024cf6865ba..b75a139cb957 100644 --- a/source/common/network/socket_option_impl.h +++ b/source/common/network/socket_option_impl.h @@ -13,71 +13,63 @@ namespace Envoy { namespace Network { #ifdef IP_TRANSPARENT -#define ENVOY_SOCKET_IP_TRANSPARENT \ - Network::SocketOptionName(std::make_pair(IPPROTO_IP, IP_TRANSPARENT)) +#define ENVOY_SOCKET_IP_TRANSPARENT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_TRANSPARENT) #else #define ENVOY_SOCKET_IP_TRANSPARENT Network::SocketOptionName() #endif #ifdef IPV6_TRANSPARENT -#define ENVOY_SOCKET_IPV6_TRANSPARENT \ - Network::SocketOptionName(std::make_pair(IPPROTO_IPV6, IPV6_TRANSPARENT)) +#define ENVOY_SOCKET_IPV6_TRANSPARENT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_TRANSPARENT) #else #define ENVOY_SOCKET_IPV6_TRANSPARENT Network::SocketOptionName() #endif #ifdef IP_FREEBIND -#define ENVOY_SOCKET_IP_FREEBIND Network::SocketOptionName(std::make_pair(IPPROTO_IP, IP_FREEBIND)) +#define ENVOY_SOCKET_IP_FREEBIND ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_FREEBIND) #else #define ENVOY_SOCKET_IP_FREEBIND Network::SocketOptionName() #endif #ifdef IPV6_FREEBIND -#define ENVOY_SOCKET_IPV6_FREEBIND \ - Network::SocketOptionName(std::make_pair(IPPROTO_IPV6, IPV6_FREEBIND)) +#define ENVOY_SOCKET_IPV6_FREEBIND ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_FREEBIND) #else #define ENVOY_SOCKET_IPV6_FREEBIND Network::SocketOptionName() #endif #ifdef SO_KEEPALIVE -#define ENVOY_SOCKET_SO_KEEPALIVE \ - Network::SocketOptionName(std::make_pair(SOL_SOCKET, SO_KEEPALIVE)) +#define ENVOY_SOCKET_SO_KEEPALIVE ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_KEEPALIVE) #else #define ENVOY_SOCKET_SO_KEEPALIVE Network::SocketOptionName() #endif #ifdef SO_MARK -#define ENVOY_SOCKET_SO_MARK Network::SocketOptionName(std::make_pair(SOL_SOCKET, SO_MARK)) +#define ENVOY_SOCKET_SO_MARK ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_MARK) #else #define ENVOY_SOCKET_SO_MARK Network::SocketOptionName() #endif #ifdef TCP_KEEPCNT -#define ENVOY_SOCKET_TCP_KEEPCNT Network::SocketOptionName(std::make_pair(IPPROTO_TCP, TCP_KEEPCNT)) +#define ENVOY_SOCKET_TCP_KEEPCNT ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPCNT) #else #define ENVOY_SOCKET_TCP_KEEPCNT Network::SocketOptionName() #endif #ifdef TCP_KEEPIDLE -#define ENVOY_SOCKET_TCP_KEEPIDLE \ - Network::SocketOptionName(std::make_pair(IPPROTO_TCP, TCP_KEEPIDLE)) +#define ENVOY_SOCKET_TCP_KEEPIDLE ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPIDLE) #elif TCP_KEEPALIVE // macOS uses a different name from Linux for just this option. -#define ENVOY_SOCKET_TCP_KEEPIDLE \ - Network::SocketOptionName(std::make_pair(IPPROTO_TCP, TCP_KEEPALIVE)) +#define ENVOY_SOCKET_TCP_KEEPIDLE ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPALIVE) #else #define ENVOY_SOCKET_TCP_KEEPIDLE Network::SocketOptionName() #endif #ifdef TCP_KEEPINTVL -#define ENVOY_SOCKET_TCP_KEEPINTVL \ - Network::SocketOptionName(std::make_pair(IPPROTO_TCP, TCP_KEEPINTVL)) +#define ENVOY_SOCKET_TCP_KEEPINTVL ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_KEEPINTVL) #else #define ENVOY_SOCKET_TCP_KEEPINTVL Network::SocketOptionName() #endif #ifdef TCP_FASTOPEN -#define ENVOY_SOCKET_TCP_FASTOPEN \ - Network::SocketOptionName(std::make_pair(IPPROTO_TCP, TCP_FASTOPEN)) +#define ENVOY_SOCKET_TCP_FASTOPEN ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_TCP, TCP_FASTOPEN) #else #define ENVOY_SOCKET_TCP_FASTOPEN Network::SocketOptionName() #endif @@ -87,20 +79,22 @@ namespace Network { // FreeBSD uses IP_RECVDSTADDR for receiving destination address and IP_SENDSRCADDR for sending // source address. #ifdef IP_RECVDSTADDR -#define ENVOY_RECV_IP_PKT_INFO Network::SocketOptionName(std::make_pair(IPPROTO_IP, IP_RECVDSTADDR)) +#define ENVOY_RECV_IP_PKT_INFO ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_RECVDSTADDR) #elif IP_PKTINFO -#define ENVOY_RECV_IP_PKT_INFO Network::SocketOptionName(std::make_pair(IPPROTO_IP, IP_PKTINFO)) +#define ENVOY_RECV_IP_PKT_INFO ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_PKTINFO) +#else +#define ENVOY_RECV_IP_PKT_INFO Network::SocketOptionName() #endif // Both Linux and FreeBSD use IPV6_RECVPKTINFO for both sending source address and // receiving destination address. -#define ENVOY_RECV_IPV6_PKT_INFO \ - Network::SocketOptionName(std::make_pair(IPPROTO_IPV6, IPV6_RECVPKTINFO)) +#define ENVOY_RECV_IPV6_PKT_INFO ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_RECVPKTINFO) class SocketOptionImpl : public Socket::Option, Logger::Loggable { public: SocketOptionImpl(envoy::api::v2::core::SocketOption::SocketState in_state, - Network::SocketOptionName optname, int value) // Yup, int. See setsockopt(2). + Network::SocketOptionName optname, + int value) // Yup, int. See setsockopt(2). : SocketOptionImpl(in_state, optname, absl::string_view(reinterpret_cast(&value), sizeof(value))) {} @@ -129,7 +123,8 @@ class SocketOptionImpl : public Socket::Option, Logger::Loggable(); EXPECT_CALL(testing::Const(socket_), ioHandle()).WillOnce(testing::ReturnRef(*io_handle)); diff --git a/test/common/network/socket_option_factory_test.cc b/test/common/network/socket_option_factory_test.cc index d9934195d338..6ec4767ab372 100644 --- a/test/common/network/socket_option_factory_test.cc +++ b/test/common/network/socket_option_factory_test.cc @@ -55,8 +55,8 @@ TEST_F(SocketOptionFactoryTest, TestBuildSocketMarkOptions) { const auto expected_option = ENVOY_SOCKET_SO_MARK; CHECK_OPTION_SUPPORTED(expected_option); - const int type = expected_option.value().first; - const int option = expected_option.value().second; + const int type = expected_option.level(); + const int option = expected_option.option(); EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int))) .WillOnce(Invoke([type, option](int, int input_type, int input_option, const void* optval, socklen_t) -> int { @@ -79,8 +79,8 @@ TEST_F(SocketOptionFactoryTest, TestBuildIpv4TransparentOptions) { const auto expected_option = ENVOY_SOCKET_IP_TRANSPARENT; CHECK_OPTION_SUPPORTED(expected_option); - const int type = expected_option.value().first; - const int option = expected_option.value().second; + const int type = expected_option.level(); + const int option = expected_option.option(); EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int))) .Times(2) .WillRepeatedly(Invoke([type, option](int, int input_type, int input_option, @@ -106,8 +106,8 @@ TEST_F(SocketOptionFactoryTest, TestBuildIpv6TransparentOptions) { const auto expected_option = ENVOY_SOCKET_IPV6_TRANSPARENT; CHECK_OPTION_SUPPORTED(expected_option); - const int type = expected_option.value().first; - const int option = expected_option.value().second; + const int type = expected_option.level(); + const int option = expected_option.option(); EXPECT_CALL(os_sys_calls_mock_, setsockopt_(_, _, _, _, sizeof(int))) .Times(2) .WillRepeatedly(Invoke([type, option](int, int input_type, int input_option, @@ -152,8 +152,8 @@ TEST_F(SocketOptionFactoryTest, TestBuildLiteralOptions) { auto option_details = socket_options->at(0)->getOptionDetails( socket_mock_, envoy::api::v2::core::SocketOption::STATE_PREBIND); EXPECT_TRUE(option_details.has_value()); - EXPECT_EQ(SOL_SOCKET, option_details->name_->first); - EXPECT_EQ(SO_LINGER, option_details->name_->second); + EXPECT_EQ(SOL_SOCKET, option_details->name_.level()); + EXPECT_EQ(SO_LINGER, option_details->name_.option()); EXPECT_EQ(sizeof(struct linger), option_details->value_.size()); const struct linger* linger_ptr = reinterpret_cast(option_details->value_.data()); @@ -163,8 +163,8 @@ TEST_F(SocketOptionFactoryTest, TestBuildLiteralOptions) { option_details = socket_options->at(1)->getOptionDetails( socket_mock_, envoy::api::v2::core::SocketOption::STATE_PREBIND); EXPECT_TRUE(option_details.has_value()); - EXPECT_EQ(SOL_SOCKET, option_details->name_->first); - EXPECT_EQ(SO_KEEPALIVE, option_details->name_->second); + EXPECT_EQ(SOL_SOCKET, option_details->name_.level()); + EXPECT_EQ(SO_KEEPALIVE, option_details->name_.option()); EXPECT_EQ(sizeof(int), option_details->value_.size()); const int* flag_ptr = reinterpret_cast(option_details->value_.data()); EXPECT_EQ(1, *flag_ptr); diff --git a/test/common/network/socket_option_impl_test.cc b/test/common/network/socket_option_impl_test.cc index aa066522bde6..8d9e55ae2d19 100644 --- a/test/common/network/socket_option_impl_test.cc +++ b/test/common/network/socket_option_impl_test.cc @@ -13,9 +13,36 @@ TEST_F(SocketOptionImplTest, BadFd) { EXPECT_EQ(ENOTSUP, result.errno_); } +TEST_F(SocketOptionImplTest, HasName) { + auto optname = ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_SNDBUF); + + // Verify that the constructor macro sets all the fields correctly. + EXPECT_TRUE(optname.has_value()); + EXPECT_EQ(SOL_SOCKET, optname.level()); + EXPECT_EQ(SO_SNDBUF, optname.option()); + EXPECT_EQ("SOL_SOCKET/SO_SNDBUF", optname.name()); + + // The default constructor should not have a value, i.e. should + // be unsupported. + EXPECT_FALSE(SocketOptionName().has_value()); + + // If we fail to set an option, verify that the log message + // contains the option name so the operator can debug. + SocketOptionImpl socket_option{envoy::api::v2::core::SocketOption::STATE_PREBIND, optname, 1}; + EXPECT_CALL(os_sys_calls_, setsockopt_(_, _, _, _, _)) + .WillOnce(Invoke([](int, int, int, const void* optval, socklen_t) -> int { + EXPECT_EQ(1, *static_cast(optval)); + return -1; + })); + + EXPECT_LOG_CONTAINS( + "warning", "Setting SOL_SOCKET/SO_SNDBUF option on socket failed", + socket_option.setOption(socket_, envoy::api::v2::core::SocketOption::STATE_PREBIND)); +} + TEST_F(SocketOptionImplTest, SetOptionSuccessTrue) { SocketOptionImpl socket_option{envoy::api::v2::core::SocketOption::STATE_PREBIND, - Network::SocketOptionName(std::make_pair(5, 10)), 1}; + ENVOY_MAKE_SOCKET_OPTION_NAME(5, 10), 1}; EXPECT_CALL(os_sys_calls_, setsockopt_(_, 5, 10, _, sizeof(int))) .WillOnce(Invoke([](int, int, int, const void* optval, socklen_t) -> int { EXPECT_EQ(1, *static_cast(optval)); @@ -26,27 +53,27 @@ TEST_F(SocketOptionImplTest, SetOptionSuccessTrue) { TEST_F(SocketOptionImplTest, GetOptionDetailsCorrectState) { SocketOptionImpl socket_option{envoy::api::v2::core::SocketOption::STATE_PREBIND, - Network::SocketOptionName(std::make_pair(5, 10)), 1}; + ENVOY_MAKE_SOCKET_OPTION_NAME(5, 10), 1}; auto result = socket_option.getOptionDetails(socket_, envoy::api::v2::core::SocketOption::STATE_PREBIND); ASSERT_TRUE(result.has_value()); - EXPECT_EQ(*result, makeDetails(std::make_pair(5, 10), 1)); + EXPECT_EQ(*result, makeDetails(ENVOY_MAKE_SOCKET_OPTION_NAME(5, 10), 1)); } TEST_F(SocketOptionImplTest, GetMoreOptionDetailsCorrectState) { SocketOptionImpl socket_option{envoy::api::v2::core::SocketOption::STATE_LISTENING, - Network::SocketOptionName(std::make_pair(7, 9)), 5}; + ENVOY_MAKE_SOCKET_OPTION_NAME(7, 9), 5}; auto result = socket_option.getOptionDetails(socket_, envoy::api::v2::core::SocketOption::STATE_LISTENING); ASSERT_TRUE(result.has_value()); - EXPECT_EQ(*result, makeDetails(std::make_pair(7, 9), 5)); + EXPECT_EQ(*result, makeDetails(ENVOY_MAKE_SOCKET_OPTION_NAME(7, 9), 5)); } TEST_F(SocketOptionImplTest, GetOptionDetailsFailureWrongState) { SocketOptionImpl socket_option{envoy::api::v2::core::SocketOption::STATE_LISTENING, - Network::SocketOptionName(std::make_pair(7, 9)), 5}; + ENVOY_MAKE_SOCKET_OPTION_NAME(7, 9), 5}; auto result = socket_option.getOptionDetails(socket_, envoy::api::v2::core::SocketOption::STATE_BOUND); @@ -55,7 +82,7 @@ TEST_F(SocketOptionImplTest, GetOptionDetailsFailureWrongState) { TEST_F(SocketOptionImplTest, GetUnsupportedOptReturnsNullopt) { SocketOptionImpl socket_option{envoy::api::v2::core::SocketOption::STATE_LISTENING, - Network::SocketOptionName(absl::nullopt), 5}; + Network::SocketOptionName(), 5}; auto result = socket_option.getOptionDetails(socket_, envoy::api::v2::core::SocketOption::STATE_LISTENING); diff --git a/test/common/network/socket_option_test.h b/test/common/network/socket_option_test.h index fb218b015c3e..2fe7a59d2c7e 100644 --- a/test/common/network/socket_option_test.h +++ b/test/common/network/socket_option_test.h @@ -39,8 +39,8 @@ class SocketOptionTest : public testing::Test { const std::set& when) { for (auto state : when) { if (option_name.has_value()) { - EXPECT_CALL(os_sys_calls_, setsockopt_(_, option_name.value().first, - option_name.value().second, _, sizeof(int))) + EXPECT_CALL(os_sys_calls_, + setsockopt_(_, option_name.level(), option_name.option(), _, sizeof(int))) .WillOnce(Invoke([option_val](int, int, int, const void* optval, socklen_t) -> int { EXPECT_EQ(option_val, *static_cast(optval)); return 0; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 75fa0ab06800..badbc3df0394 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -2838,8 +2838,8 @@ class SockoptsTest : public ClusterManagerImplTest { expect_success = false; continue; } - EXPECT_CALL(os_sys_calls, setsockopt_(_, name_val.first.value().first, - name_val.first.value().second, _, sizeof(int))) + EXPECT_CALL(os_sys_calls, + setsockopt_(_, name_val.first.level(), name_val.first.option(), _, sizeof(int))) .WillOnce(Invoke([&name_val](int, int, int, const void* optval, socklen_t) -> int { EXPECT_EQ(name_val.second, *static_cast(optval)); return 0; @@ -3009,7 +3009,7 @@ TEST_F(SockoptsTest, SockoptsClusterOnly) { )EOF"; initialize(yaml); std::vector> names_vals{ - {Network::SocketOptionName({1, 2}), 3}, {Network::SocketOptionName({4, 5}), 6}}; + {ENVOY_MAKE_SOCKET_OPTION_NAME(1, 2), 3}, {ENVOY_MAKE_SOCKET_OPTION_NAME(4, 5), 6}}; expectSetsockopts(names_vals); } @@ -3037,7 +3037,7 @@ TEST_F(SockoptsTest, SockoptsClusterManagerOnly) { )EOF"; initialize(yaml); std::vector> names_vals{ - {Network::SocketOptionName({1, 2}), 3}, {Network::SocketOptionName({4, 5}), 6}}; + {ENVOY_MAKE_SOCKET_OPTION_NAME(1, 2), 3}, {ENVOY_MAKE_SOCKET_OPTION_NAME(4, 5), 6}}; expectSetsockopts(names_vals); } @@ -3067,7 +3067,7 @@ TEST_F(SockoptsTest, SockoptsClusterOverride) { )EOF"; initialize(yaml); std::vector> names_vals{ - {Network::SocketOptionName({1, 2}), 3}, {Network::SocketOptionName({4, 5}), 6}}; + {ENVOY_MAKE_SOCKET_OPTION_NAME(1, 2), 3}, {ENVOY_MAKE_SOCKET_OPTION_NAME(4, 5), 6}}; expectSetsockopts(names_vals); } @@ -3116,16 +3116,15 @@ class TcpKeepaliveTest : public ClusterManagerImplTest { options, socket, envoy::api::v2::core::SocketOption::STATE_PREBIND))); return connection_; })); - EXPECT_CALL(os_sys_calls, setsockopt_(_, ENVOY_SOCKET_SO_KEEPALIVE.value().first, - ENVOY_SOCKET_SO_KEEPALIVE.value().second, _, sizeof(int))) + EXPECT_CALL(os_sys_calls, setsockopt_(_, ENVOY_SOCKET_SO_KEEPALIVE.level(), + ENVOY_SOCKET_SO_KEEPALIVE.option(), _, sizeof(int))) .WillOnce(Invoke([](int, int, int, const void* optval, socklen_t) -> int { EXPECT_EQ(1, *static_cast(optval)); return 0; })); if (keepalive_probes.has_value()) { - EXPECT_CALL(os_sys_calls, - setsockopt_(_, ENVOY_SOCKET_TCP_KEEPCNT.value().first, - ENVOY_SOCKET_TCP_KEEPCNT.value().second, _, sizeof(int))) + EXPECT_CALL(os_sys_calls, setsockopt_(_, ENVOY_SOCKET_TCP_KEEPCNT.level(), + ENVOY_SOCKET_TCP_KEEPCNT.option(), _, sizeof(int))) .WillOnce( Invoke([&keepalive_probes](int, int, int, const void* optval, socklen_t) -> int { EXPECT_EQ(keepalive_probes.value(), *static_cast(optval)); @@ -3133,18 +3132,16 @@ class TcpKeepaliveTest : public ClusterManagerImplTest { })); } if (keepalive_time.has_value()) { - EXPECT_CALL(os_sys_calls, - setsockopt_(_, ENVOY_SOCKET_TCP_KEEPIDLE.value().first, - ENVOY_SOCKET_TCP_KEEPIDLE.value().second, _, sizeof(int))) + EXPECT_CALL(os_sys_calls, setsockopt_(_, ENVOY_SOCKET_TCP_KEEPIDLE.level(), + ENVOY_SOCKET_TCP_KEEPIDLE.option(), _, sizeof(int))) .WillOnce(Invoke([&keepalive_time](int, int, int, const void* optval, socklen_t) -> int { EXPECT_EQ(keepalive_time.value(), *static_cast(optval)); return 0; })); } if (keepalive_interval.has_value()) { - EXPECT_CALL(os_sys_calls, - setsockopt_(_, ENVOY_SOCKET_TCP_KEEPINTVL.value().first, - ENVOY_SOCKET_TCP_KEEPINTVL.value().second, _, sizeof(int))) + EXPECT_CALL(os_sys_calls, setsockopt_(_, ENVOY_SOCKET_TCP_KEEPINTVL.level(), + ENVOY_SOCKET_TCP_KEEPINTVL.option(), _, sizeof(int))) .WillOnce( Invoke([&keepalive_interval](int, int, int, const void* optval, socklen_t) -> int { EXPECT_EQ(keepalive_interval.value(), *static_cast(optval)); diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index cbdeb23e8c6b..44b5283aeafe 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -261,7 +261,7 @@ class ListenerManagerImplWithRealFiltersTest : public ListenerManagerImplTest { TestThreadsafeSingletonInjector os_calls(&os_sys_calls); if (expected_option.has_value()) { expectCreateListenSocket(expected_state, expected_num_options); - expectSetsockopt(os_sys_calls, expected_option.value().first, expected_option.value().second, + expectSetsockopt(os_sys_calls, expected_option.level(), expected_option.option(), expected_value, expected_num_options); manager_->addOrUpdateListener(listener, "", true); EXPECT_EQ(1U, manager_->listeners().size()); From 8017bf28b72d66aeed2cd4557b4604c9f1e4b8c8 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 10 Jul 2019 13:06:24 -0400 Subject: [PATCH 137/542] http: allowing altering the x-envoy header prefix (#7458) Adding bootstrap configuration so Envoy can trust and generate x-random-prefix headers instead fo x-envoy headers Risk Level: Medium Testing: UT, sorta integration test Docs Changes: n/a Release Notes: yes Fixes #5363 Signed-off-by: Alyssa Wilk --- api/envoy/config/bootstrap/v2/bootstrap.proto | 11 ++ docs/root/intro/version_history.rst | 1 + source/common/http/BUILD | 1 + source/common/http/headers.h | 109 ++++++++++++------ source/common/router/debug_config.h | 6 +- source/extensions/filters/common/fault/BUILD | 1 + .../filters/common/fault/fault_config.h | 8 +- source/server/server.cc | 7 ++ test/integration/BUILD | 9 ++ .../header_prefix_integration_test.cc | 59 ++++++++++ test/server/server_test.cc | 13 ++- tools/check_format.py | 4 + 12 files changed, 189 insertions(+), 40 deletions(-) create mode 100644 test/integration/header_prefix_integration_test.cc diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index f8083ca6bf8a..ab841058c0b1 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -133,6 +133,17 @@ message Bootstrap { // over the wire individually because the statsd protocol doesn't have any way to represent a // histogram summary. Be aware that this can be a very large volume of data. bool enable_dispatcher_stats = 16; + + // Optional string which will be used in lieu of x-envoy in prefixing headers. + // + // For example, if this string is present and set to X-Foo, then x-envoy-retry-on will be + // transformed into x-foo-retry-on etc. + // + // Note this applies to the headers Envoy will generate, the headers Envoy will sanitize, and the + // headers Envoy will trust for core code and core extensions only. Be VERY careful making + // changes to this string, especially in multi-layer Envoy deployments or deployments using + // extensions which are not upstream. + string header_prefix = 18; } // Administration interface :ref:`operations documentation diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index c7b1e05a8177..aaebdf46a4b0 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -37,6 +37,7 @@ Version history * http: mitigated a race condition with the :ref:`delayed_close_timeout` where it could trigger while actively flushing a pending write buffer for a downstream connection. * http: added support for :ref:`preserve_external_request_id` that represents whether the x-request-id should not be reset on edge entry inside mesh * http: changed `sendLocalReply` to send percent-encoded `GrpcMessage`. +* http: added a :ref:header_prefix` ` configuration option to allow Envoy to send and process x-custom- prefixed headers rather than x-envoy. * http: added :ref:`dynamic forward proxy ` support. * http: tracking the active stream and dumping state in Envoy crash handlers. This can be disabled by building with `--define disable_object_dump_on_signal_trace=disabled` * jwt_authn: make filter's parsing of JWT more flexible, allowing syntax like ``jwt=eyJhbGciOiJS...ZFnFIw,extra=7,realm=123`` diff --git a/source/common/http/BUILD b/source/common/http/BUILD index ce3737bef977..88ff9675e93c 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -231,6 +231,7 @@ envoy_cc_library( deps = [ "//include/envoy/http:header_map_interface", "//source/common/singleton:const_singleton", + "//source/common/singleton:threadsafe_singleton", ], ) diff --git a/source/common/http/headers.h b/source/common/http/headers.h index 2b9247747e56..f5a458ff04d6 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -5,15 +5,50 @@ #include "envoy/http/header_map.h" #include "common/singleton/const_singleton.h" +#include "common/singleton/threadsafe_singleton.h" namespace Envoy { namespace Http { +// This class allows early override of the x-envoy prefix from bootstrap config, +// so that servers can configure their own x-custom-string prefix. +// +// Once the HeaderValues const singleton has been created, changing the prefix +// is disallowed. Essentially this is write-once then read-only. +class PrefixValue { +public: + const char* prefix() { + absl::WriterMutexLock lock(&m_); + read_ = true; + return prefix_.c_str(); + } + + // The char* prefix is used directly, so must be available for the interval where prefix() may be + // called. + void setPrefix(const char* prefix) { + absl::WriterMutexLock lock(&m_); + // The check for unchanged string is purely for integration tests - this + // should not happen in production. + RELEASE_ASSERT(!read_ || prefix_ == std::string(prefix), + "Attempting to change the header prefix after it has been used!"); + if (!read_) { + prefix_ = prefix; + } + } + +private: + absl::Mutex m_; + bool read_ = false; + std::string prefix_ = "x-envoy"; +}; + /** * Constant HTTP headers and values. All lower case. */ class HeaderValues { public: + const char* prefix() { return ThreadSafeSingleton::get().prefix(); } + const LowerCaseString Accept{"accept"}; const LowerCaseString AcceptEncoding{"accept-encoding"}; const LowerCaseString AccessControlRequestHeaders{"access-control-request-headers"}; @@ -35,41 +70,49 @@ class HeaderValues { const LowerCaseString ContentType{"content-type"}; const LowerCaseString Cookie{"cookie"}; const LowerCaseString Date{"date"}; - const LowerCaseString EnvoyAttemptCount{"x-envoy-attempt-count"}; - const LowerCaseString EnvoyAuthPartialBody{"x-envoy-auth-partial-body"}; - const LowerCaseString EnvoyCluster{"x-envoy-cluster"}; - const LowerCaseString EnvoyDegraded{"x-envoy-degraded"}; - const LowerCaseString EnvoyDownstreamServiceCluster{"x-envoy-downstream-service-cluster"}; - const LowerCaseString EnvoyDownstreamServiceNode{"x-envoy-downstream-service-node"}; - const LowerCaseString EnvoyExternalAddress{"x-envoy-external-address"}; - const LowerCaseString EnvoyForceTrace{"x-envoy-force-trace"}; - const LowerCaseString EnvoyHedgeOnPerTryTimeout{"x-envoy-hedge-on-per-try-timeout"}; - const LowerCaseString EnvoyImmediateHealthCheckFail{"x-envoy-immediate-health-check-fail"}; - const LowerCaseString EnvoyOriginalUrl{"x-envoy-original-url"}; - const LowerCaseString EnvoyInternalRequest{"x-envoy-internal"}; - const LowerCaseString EnvoyIpTags{"x-envoy-ip-tags"}; - const LowerCaseString EnvoyMaxRetries{"x-envoy-max-retries"}; - const LowerCaseString EnvoyNotForwarded{"x-envoy-not-forwarded"}; - const LowerCaseString EnvoyOriginalDstHost{"x-envoy-original-dst-host"}; - const LowerCaseString EnvoyOriginalPath{"x-envoy-original-path"}; - const LowerCaseString EnvoyOverloaded{"x-envoy-overloaded"}; - const LowerCaseString EnvoyRateLimited{"x-envoy-ratelimited"}; - const LowerCaseString EnvoyRetryOn{"x-envoy-retry-on"}; - const LowerCaseString EnvoyRetryGrpcOn{"x-envoy-retry-grpc-on"}; - const LowerCaseString EnvoyRetriableStatusCodes{"x-envoy-retriable-status-codes"}; - const LowerCaseString EnvoyUpstreamAltStatName{"x-envoy-upstream-alt-stat-name"}; - const LowerCaseString EnvoyUpstreamCanary{"x-envoy-upstream-canary"}; - const LowerCaseString EnvoyUpstreamHostAddress{"x-envoy-upstream-host-address"}; - const LowerCaseString EnvoyUpstreamHostname{"x-envoy-upstream-hostname"}; + const LowerCaseString EnvoyAttemptCount{absl::StrCat(prefix(), "-attempt-count")}; + const LowerCaseString EnvoyAuthPartialBody{absl::StrCat(prefix(), "-auth-partial-body")}; + const LowerCaseString EnvoyCluster{absl::StrCat(prefix(), "-cluster")}; + const LowerCaseString EnvoyDegraded{absl::StrCat(prefix(), "-degraded")}; + const LowerCaseString EnvoyDownstreamServiceCluster{ + absl::StrCat(prefix(), "-downstream-service-cluster")}; + const LowerCaseString EnvoyDownstreamServiceNode{ + absl::StrCat(prefix(), "-downstream-service-node")}; + const LowerCaseString EnvoyExternalAddress{absl::StrCat(prefix(), "-external-address")}; + const LowerCaseString EnvoyForceTrace{absl::StrCat(prefix(), "-force-trace")}; + const LowerCaseString EnvoyHedgeOnPerTryTimeout{ + absl::StrCat(prefix(), "-hedge-on-per-try-timeout")}; + const LowerCaseString EnvoyImmediateHealthCheckFail{ + absl::StrCat(prefix(), "-immediate-health-check-fail")}; + const LowerCaseString EnvoyOriginalUrl{absl::StrCat(prefix(), "-original-url")}; + const LowerCaseString EnvoyInternalRequest{absl::StrCat(prefix(), "-internal")}; + const LowerCaseString EnvoyIpTags{absl::StrCat(prefix(), "-ip-tags")}; + const LowerCaseString EnvoyMaxRetries{absl::StrCat(prefix(), "-max-retries")}; + const LowerCaseString EnvoyNotForwarded{absl::StrCat(prefix(), "-not-forwarded")}; + const LowerCaseString EnvoyOriginalDstHost{absl::StrCat(prefix(), "-original-dst-host")}; + const LowerCaseString EnvoyOriginalPath{absl::StrCat(prefix(), "-original-path")}; + const LowerCaseString EnvoyOverloaded{absl::StrCat(prefix(), "-overloaded")}; + const LowerCaseString EnvoyRateLimited{absl::StrCat(prefix(), "-ratelimited")}; + const LowerCaseString EnvoyRetryOn{absl::StrCat(prefix(), "-retry-on")}; + const LowerCaseString EnvoyRetryGrpcOn{absl::StrCat(prefix(), "-retry-grpc-on")}; + const LowerCaseString EnvoyRetriableStatusCodes{ + absl::StrCat(prefix(), "-retriable-status-codes")}; + const LowerCaseString EnvoyUpstreamAltStatName{absl::StrCat(prefix(), "-upstream-alt-stat-name")}; + const LowerCaseString EnvoyUpstreamCanary{absl::StrCat(prefix(), "-upstream-canary")}; + const LowerCaseString EnvoyUpstreamHostAddress{absl::StrCat(prefix(), "-upstream-host-address")}; + const LowerCaseString EnvoyUpstreamHostname{absl::StrCat(prefix(), "-upstream-hostname")}; const LowerCaseString EnvoyUpstreamRequestTimeoutAltResponse{ - "x-envoy-upstream-rq-timeout-alt-response"}; - const LowerCaseString EnvoyUpstreamRequestTimeoutMs{"x-envoy-upstream-rq-timeout-ms"}; + absl::StrCat(prefix(), "-upstream-rq-timeout-alt-response")}; + const LowerCaseString EnvoyUpstreamRequestTimeoutMs{ + absl::StrCat(prefix(), "-upstream-rq-timeout-ms")}; const LowerCaseString EnvoyUpstreamRequestPerTryTimeoutMs{ - "x-envoy-upstream-rq-per-try-timeout-ms"}; - const LowerCaseString EnvoyExpectedRequestTimeoutMs{"x-envoy-expected-rq-timeout-ms"}; - const LowerCaseString EnvoyUpstreamServiceTime{"x-envoy-upstream-service-time"}; - const LowerCaseString EnvoyUpstreamHealthCheckedCluster{"x-envoy-upstream-healthchecked-cluster"}; - const LowerCaseString EnvoyDecoratorOperation{"x-envoy-decorator-operation"}; + absl::StrCat(prefix(), "-upstream-rq-per-try-timeout-ms")}; + const LowerCaseString EnvoyExpectedRequestTimeoutMs{ + absl::StrCat(prefix(), "-expected-rq-timeout-ms")}; + const LowerCaseString EnvoyUpstreamServiceTime{absl::StrCat(prefix(), "-upstream-service-time")}; + const LowerCaseString EnvoyUpstreamHealthCheckedCluster{ + absl::StrCat(prefix(), "-upstream-healthchecked-cluster")}; + const LowerCaseString EnvoyDecoratorOperation{absl::StrCat(prefix(), "-decorator-operation")}; const LowerCaseString Etag{"etag"}; const LowerCaseString Expect{"expect"}; const LowerCaseString ForwardedClientCert{"x-forwarded-client-cert"}; diff --git a/source/common/router/debug_config.h b/source/common/router/debug_config.h index 96f99080ab7c..6b698d07e760 100644 --- a/source/common/router/debug_config.h +++ b/source/common/router/debug_config.h @@ -39,7 +39,7 @@ struct DebugConfig : public StreamInfo::FilterState::Object { /** * Append cluster information as a response header if `append_cluster_` is true. The router will - * use `cluster_header_` as the header name, if specified, or "x-envoy-cluster" by default. + * use `cluster_header_` as the header name, if specified, or 'x-envoy-cluster' by default. */ bool append_cluster_{}; absl::optional cluster_header_; @@ -47,7 +47,7 @@ struct DebugConfig : public StreamInfo::FilterState::Object { /** * Append upstream host name and address as response headers, if `append_upstream_host_` is true. * The router will use `hostname_header_` and `host_address_header_` as the header names, if - * specified, or "x-envoy-upstream-hostname" and "x-envoy-upstream-host-address" by default. + * specified, or 'x-envoy-upstream-hostname' and 'x-envoy-upstream-host-address' by default. */ bool append_upstream_host_{}; absl::optional hostname_header_; @@ -57,7 +57,7 @@ struct DebugConfig : public StreamInfo::FilterState::Object { * Do not forward the associated request to the upstream cluster, if `do_not_forward_` is true. * If the router would have forwarded it (assuming all other preconditions are met), it will * instead respond with a 204 "no content." Append `not_forwarded_header_`, if specified, or - * "x-envoy-not-forwarded" by default. Any debug headers specified above (or others introduced by + * 'x-envoy-not-forwarded' by default. Any debug headers specified above (or others introduced by * other filters) will be appended to this empty response. */ bool do_not_forward_{}; diff --git a/source/extensions/filters/common/fault/BUILD b/source/extensions/filters/common/fault/BUILD index b8607b4f861b..a9d887673c5a 100644 --- a/source/extensions/filters/common/fault/BUILD +++ b/source/extensions/filters/common/fault/BUILD @@ -14,6 +14,7 @@ envoy_cc_library( hdrs = ["fault_config.h"], deps = [ "//include/envoy/http:header_map_interface", + "//source/common/http:headers_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/config/filter/fault/v2:fault_cc", ], diff --git a/source/extensions/filters/common/fault/fault_config.h b/source/extensions/filters/common/fault/fault_config.h index 64e966d87d79..57203ebf1ad9 100644 --- a/source/extensions/filters/common/fault/fault_config.h +++ b/source/extensions/filters/common/fault/fault_config.h @@ -3,6 +3,7 @@ #include "envoy/config/filter/fault/v2/fault.pb.h" #include "envoy/http/header_map.h" +#include "common/http/headers.h" #include "common/singleton/const_singleton.h" namespace Envoy { @@ -13,8 +14,11 @@ namespace Fault { class HeaderNameValues { public: - const Http::LowerCaseString DelayRequest{"x-envoy-fault-delay-request"}; - const Http::LowerCaseString ThroughputResponse{"x-envoy-fault-throughput-response"}; + const char* prefix() { return ThreadSafeSingleton::get().prefix(); } + + const Http::LowerCaseString DelayRequest{absl::StrCat(prefix(), "-fault-delay-request")}; + const Http::LowerCaseString ThroughputResponse{ + absl::StrCat(prefix(), "-fault-throughput-response")}; }; using HeaderNames = ConstSingleton; diff --git a/source/server/server.cc b/source/server/server.cc index caed6af918d3..072a7612ec2b 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -270,6 +270,13 @@ void InstanceImpl::initialize(const Options& options, InstanceUtil::loadBootstrapConfig(bootstrap_, options, messageValidationVisitor(), *api_); bootstrap_config_update_time_ = time_source_.systemTime(); + // Immediate after the bootstrap has been loaded, override the header prefix, if configured to + // do so. This must be set before any other code block references the HeaderValues ConstSingleton. + if (!bootstrap_.header_prefix().empty()) { + // setPrefix has a release assert verifying that setPrefix() is not called after prefix() + ThreadSafeSingleton::get().setPrefix(bootstrap_.header_prefix().c_str()); + } + // Needs to happen as early as possible in the instantiation to preempt the objects that require // stats. stats_store_.setTagProducer(Config::Utility::createTagProducer(bootstrap_)); diff --git a/test/integration/BUILD b/test/integration/BUILD index b7679977ecdb..fff182f022cb 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -556,6 +556,15 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "header_prefix_integration_test", + srcs = ["header_prefix_integration_test.cc"], + coverage = False, + deps = [ + ":http_protocol_integration_lib", + ], +) + envoy_cc_test( name = "overload_integration_test", srcs = ["overload_integration_test.cc"], diff --git a/test/integration/header_prefix_integration_test.cc b/test/integration/header_prefix_integration_test.cc new file mode 100644 index 000000000000..687080c0e2e4 --- /dev/null +++ b/test/integration/header_prefix_integration_test.cc @@ -0,0 +1,59 @@ +#include "test/integration/http_protocol_integration.h" +#include "test/integration/server.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +// Unfortunately in the Envoy test suite, the headers singleton is initialized +// well before server start-up, so by the time the server has parsed the +// bootstrap proto it's too late to set it. +// +// Instead, set the value early and regression test the bootstrap proto's validation of prefix +// injection. + +static const char* custom_prefix_ = "x-custom"; + +class HeaderPrefixIntegrationTest : public HttpProtocolIntegrationTest { +public: + static void SetUpTestSuite() { + ThreadSafeSingleton::get().setPrefix(custom_prefix_); + } +}; + +TEST_P(HeaderPrefixIntegrationTest, CustomHeaderPrefix) { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + bootstrap.set_header_prefix("x-custom"); + }); + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = + sendRequestAndWaitForResponse(default_request_headers_, 0, default_response_headers_, 0); + + EXPECT_TRUE(response->headers().get( + Envoy::Http::LowerCaseString{"x-custom-upstream-service-time"}) != nullptr); + EXPECT_EQ("x-custom-upstream-service-time", + response->headers().EnvoyUpstreamServiceTime()->key().getStringView()); + + EXPECT_TRUE(upstream_request_->headers().get( + Envoy::Http::LowerCaseString{"x-custom-expected-rq-timeout-ms"}) != nullptr); + EXPECT_EQ("x-custom-expected-rq-timeout-ms", + upstream_request_->headers().EnvoyExpectedRequestTimeoutMs()->key().getStringView()); +} + +// In this case, the header prefix set in the bootstrap will not match the +// singleton header prefix in SetUpTestSuite, and Envoy will RELEASE_ASSERT on +// start-up. +TEST_P(HeaderPrefixIntegrationTest, FailedCustomHeaderPrefix) { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + bootstrap.set_header_prefix("x-custom-but-not-set"); + }); + EXPECT_DEATH(initialize(), "Attempting to change the header prefix after it has been used!"); +} + +INSTANTIATE_TEST_SUITE_P(Protocols, HeaderPrefixIntegrationTest, + testing::ValuesIn(HttpProtocolIntegrationTest::getProtocolTestParams( + {Http::CodecClient::Type::HTTP1, Http::CodecClient::Type::HTTP2}, + {FakeHttpConnection::Type::HTTP1})), + HttpProtocolIntegrationTest::protocolTestParamsToString); +} // namespace Envoy diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 6839c8439fe4..8ddc2ff7fd4f 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -170,10 +170,10 @@ class ServerInstanceImplTest : public testing::TestWithParam Date: Wed, 10 Jul 2019 12:27:32 -0700 Subject: [PATCH 138/542] add JWT authentication to arch overview (#7515) Risk Level: Low Testing: N/A Docs Changes: Added JWT Authentication to arch overview Release Notes: N/A Fixes #7285 Signed-off-by: Yangmin Zhu --- .../security/jwt_authn_filter.rst | 28 +++++++++++++++++++ .../intro/arch_overview/security/security.rst | 1 + 2 files changed, 29 insertions(+) create mode 100644 docs/root/intro/arch_overview/security/jwt_authn_filter.rst diff --git a/docs/root/intro/arch_overview/security/jwt_authn_filter.rst b/docs/root/intro/arch_overview/security/jwt_authn_filter.rst new file mode 100644 index 000000000000..848d17298974 --- /dev/null +++ b/docs/root/intro/arch_overview/security/jwt_authn_filter.rst @@ -0,0 +1,28 @@ +.. _arch_overview_jwt_authn: + +JSON Web Token (JWT) Authentication +=================================== + +* :ref:`HTTP filter configuration `. + +The JSON Web Token (JWT) Authentication filter checks if the incoming request has a valid +`JSON Web Token (JWT) `_. It checks the validity of the JWT by +verifying the JWT signature, audiences and issuer based on the +:ref:`HTTP filter configuration `. The JWT Authentication filter +could be configured to either reject the request with invalid JWT immediately or defer the decision +to later filters by passing the JWT payload to other filters. + +The JWT Authentication filter supports to check the JWT under various conditions of the request, it +could be configured to check JWT only on specific paths so that you could whitelist some paths from +the JWT authentication, which is useful if a path is accessible publicly and doesn't require any JWT +authentication. + +The JWT Authentication filter supports to extract the JWT from various locations of the request and +could combine multiple JWT requirements for the same request. The +`JSON Web Key Set (JWKS) `_ needed for the JWT signature +verification could be either specified inline in the filter config or fetched from remote server +via HTTP/HTTPS. + +The JWT Authentication filter also supports to write the payloads of the successfully verified JWT +to :ref:`Dynamic State ` so that later filters could use +it to make their own decisions based on the JWT payloads. diff --git a/docs/root/intro/arch_overview/security/security.rst b/docs/root/intro/arch_overview/security/security.rst index 9a2be26f81f2..065935c6f342 100644 --- a/docs/root/intro/arch_overview/security/security.rst +++ b/docs/root/intro/arch_overview/security/security.rst @@ -5,5 +5,6 @@ Security :maxdepth: 2 ssl + jwt_authn_filter ext_authz_filter rbac_filter From 165290e19d4bf0e160f050e07f12947bb427b3ed Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 10 Jul 2019 23:29:06 +0300 Subject: [PATCH 139/542] redis: remove redundant move (#7520) Description: gcc9 reports the used std::move() as redundant Risk Level: low Testing: n/a Release Notes: N/A Documentation: N/A Signed-off-by: Dmitry Rozhkov --- source/extensions/filters/network/common/redis/client_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 462a7ca2157e..c0fee56b5eb3 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -31,7 +31,7 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); client->connection_->connect(); client->connection_->noDelay(true); - return std::move(client); + return client; } ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, From 08dc2442882665d6e96262eaec03099a9a8f535d Mon Sep 17 00:00:00 2001 From: asraa Date: Wed, 10 Jul 2019 17:19:15 -0400 Subject: [PATCH 140/542] fuzz: fix filesystem bug, use std::__fs::filesystem (#7507) Since OSS-Fuzz minijail prohibits mkdir calls, we use std::__fs::filesystem namespace which implements filesystem. The OSS-Fuzz image was taking the mkdir branch, since neither filesystem nor experimental/filesystem was found. However, we are able to use std::__fs::filesystem branch in the OSS-Fuzz image and the std::experimental::filesystem branch locally. Risk Level: Low Testing: Add RELEASE_ASSERT(1==0, "") under the #if defined(_LIBCPP_VERSION) && TEST_STD_VER < 17 branch to verify that branch is taken in oss-fuzz docker. run_fuzzer with oss-fuzz docker is successful. Fixes OSS-Fuzz issue: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=15414 Signed-off-by: Asra Ali --- test/test_common/environment.cc | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index 8afd806cd331..2291fb09a094 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -3,11 +3,12 @@ #include #include -#ifdef __has_include -#if __has_include() +// TODO(asraa): Remove and rely only on when Envoy requires +// Clang >= 9. +#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) #include -// TODO(asraa): Remove this when Envoy requires Clang >= 9. -#elif __has_include() +#elif defined __has_include +#if __has_include() #include #endif #endif @@ -41,8 +42,8 @@ std::string makeTempDir(char* name_template) { char* dirname = ::_mktemp(name_template); RELEASE_ASSERT(dirname != nullptr, fmt::format("failed to create tempdir from template: {} {}", name_template, strerror(errno))); -#ifdef __cpp_lib_filesystem - std::filesystem::create_directories(dirname); +#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) + std::__fs::filesystem::create_directories(dirname); #elif defined __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories(dirname); #endif @@ -84,10 +85,10 @@ char** argv_; } // namespace void TestEnvironment::createPath(const std::string& path) { -#ifdef __cpp_lib_filesystem +#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. - std::filesystem::create_directories(std::filesystem::path(path)); + std::__fs::filesystem::create_directories(std::__fs::filesystem::path(path)); #elif defined __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories(std::experimental::filesystem::path(path)); #else @@ -97,10 +98,10 @@ void TestEnvironment::createPath(const std::string& path) { } void TestEnvironment::createParentPath(const std::string& path) { -#ifdef __cpp_lib_filesystem +#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. - std::filesystem::create_directories(std::filesystem::path(path).parent_path()); + std::__fs::filesystem::create_directories(std::__fs::filesystem::path(path).parent_path()); #elif defined __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories( std::experimental::filesystem::path(path).parent_path()); @@ -112,13 +113,13 @@ void TestEnvironment::createParentPath(const std::string& path) { void TestEnvironment::removePath(const std::string& path) { RELEASE_ASSERT(absl::StartsWith(path, TestEnvironment::temporaryDirectory()), ""); -#ifdef __cpp_lib_filesystem +#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. - if (!std::filesystem::exists(path)) { + if (!std::__fs::filesystem::exists(path)) { return; } - std::filesystem::remove_all(std::filesystem::path(path)); + std::__fs::filesystem::remove_all(std::__fs::filesystem::path(path)); #elif defined __cpp_lib_experimental_filesystem if (!std::experimental::filesystem::exists(path)) { return; From 8246167b9d238797cbc6c03dccc9e3921c37617d Mon Sep 17 00:00:00 2001 From: asraa Date: Wed, 10 Jul 2019 17:21:27 -0400 Subject: [PATCH 141/542] dependencies: update protobuf to 3.8.0 (#7510) In addition to updating protobuf to 3.8.0, this PR also Removes old protobuf patch now included in 3.8.0 - Patches protocolbuffers/protobuf#6333 that fixes a UBSAN error in the protobuf library. - Patches protobuf's BUILD to depend on foreign_cc zlib Risk level: low/medium Testing: bazel test //test/... Signed-off-by: Asra Ali --- api/tools/data/tap2pcap_h2_ipv4.txt | 6 ++--- bazel/protobuf.patch | 41 ++++++++++++++--------------- bazel/repositories.bzl | 14 +++++----- bazel/repository_locations.bzl | 6 ++--- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/api/tools/data/tap2pcap_h2_ipv4.txt b/api/tools/data/tap2pcap_h2_ipv4.txt index f46bd469b8ca..ac8785ac1e49 100644 --- a/api/tools/data/tap2pcap_h2_ipv4.txt +++ b/api/tools/data/tap2pcap_h2_ipv4.txt @@ -1,6 +1,6 @@ 1 0.000000 127.0.0.1 → 127.0.0.1 HTTP2 157 Magic, SETTINGS[0], WINDOW_UPDATE[0], HEADERS[1]: GET / 2 0.013713 127.0.0.1 → 127.0.0.1 HTTP2 91 SETTINGS[0], SETTINGS[0], WINDOW_UPDATE[0] - 3 0.013820 127.0.0.1 → 127.0.0.1 HTTP2 63 SETTINGS[0] + 3 0.013821 127.0.0.1 → 127.0.0.1 HTTP2 63 SETTINGS[0] 4 0.128649 127.0.0.1 → 127.0.0.1 HTTP2 5586 HEADERS[1]: 200 OK - 5 0.130006 127.0.0.1 → 127.0.0.1 HTTP2 7573 DATA[1] - 6 0.131044 127.0.0.1 → 127.0.0.1 HTTP2 3152 DATA[1], DATA[1] (text/html) + 5 0.130007 127.0.0.1 → 127.0.0.1 HTTP2 7573 DATA[1] + 6 0.131045 127.0.0.1 → 127.0.0.1 HTTP2 3152 DATA[1], DATA[1] (text/html) diff --git a/bazel/protobuf.patch b/bazel/protobuf.patch index 69c7cc28e0ba..d51b67e92457 100644 --- a/bazel/protobuf.patch +++ b/bazel/protobuf.patch @@ -1,13 +1,13 @@ diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc -index 1d34870deb..3844fa6b8b 100644 +index 3844fa6b8b..5486887295 100644 --- a/src/google/protobuf/stubs/strutil.cc +++ b/src/google/protobuf/stubs/strutil.cc -@@ -1116,10 +1116,12 @@ char* FastUInt64ToBufferLeft(uint64 u64, char* buffer) { +@@ -1065,10 +1065,12 @@ char* FastUInt32ToBufferLeft(uint32 u, char* buffer) { } - char* FastInt64ToBufferLeft(int64 i, char* buffer) { -- uint64 u = i; -+ uint64 u = 0; + char* FastInt32ToBufferLeft(int32 i, char* buffer) { +- uint32 u = i; ++ uint32 u = 0; if (i < 0) { *buffer++ = '-'; - u = -i; @@ -15,20 +15,19 @@ index 1d34870deb..3844fa6b8b 100644 + } else { + u = i; } - return FastUInt64ToBufferLeft(u, buffer); + return FastUInt32ToBufferLeft(u, buffer); } -diff --git a/src/google/protobuf/text_format.cc b/src/google/protobuf/text_format.cc -index ba0c3028ee..801a8e3786 100644 ---- a/src/google/protobuf/text_format.cc -+++ b/src/google/protobuf/text_format.cc -@@ -1315,7 +1315,9 @@ class TextFormat::Printer::TextGenerator - while (size > buffer_size_) { - // Data exceeds space in the buffer. Write what we can and request a new - // buffer. -- memset(buffer_, ' ', buffer_size_); -+ if (buffer_size_ > 0) { -+ memset(buffer_, ' ', buffer_size_); -+ } - size -= buffer_size_; - void* void_buffer; - failed_ = !output_->Next(&void_buffer, &buffer_size_); + +diff --git a/BUILD b/BUILD +index 6665de94..55f28582 100644 +--- a/BUILD ++++ b/BUILD +@@ -19,6 +19,6 @@ config_setting( + # ZLIB configuration + ################################################################################ + +-ZLIB_DEPS = ["@zlib//:zlib"] ++ZLIB_DEPS = ["//external:zlib"] + + ################################################################################ + # Protobuf Runtime Library \ No newline at end of file diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index c957a1b8cd5c..ed14486d3a5e 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -486,9 +486,10 @@ def _com_google_absl(): def _com_google_protobuf(): _repository_impl( "com_google_protobuf", - # The patch is only needed until - # https://github.com/protocolbuffers/protobuf/pull/5901 is available. - # TODO(htuch): remove this when > protobuf 3.7.1 is released. + # The patch includes + # https://github.com/protocolbuffers/protobuf/pull/6333 and also uses + # foreign_cc build for zlib as its dependency. + # TODO(asraa): remove this when > protobuf 3.8.0 is released. patch_args = ["-p1"], patches = ["@envoy//bazel:protobuf.patch"], ) @@ -499,9 +500,10 @@ def _com_google_protobuf(): _repository_impl( "com_google_protobuf_cc", repository_key = "com_google_protobuf", - # The patch is only needed until - # https://github.com/protocolbuffers/protobuf/pull/5901 is available. - # TODO(htuch): remove this when > protobuf 3.7.1 is released. + # The patch includes + # https://github.com/protocolbuffers/protobuf/pull/6333 and also uses + # foreign_cc build for zlib as its dependency. + # TODO(asraa): remove this when > protobuf 3.8.0 is released. patch_args = ["-p1"], patches = ["@envoy//bazel:protobuf.patch"], ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 58c12e35916f..6e25d2fd589e 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -205,9 +205,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/google/googletest/archive/d7003576dd133856432e2e07340f45926242cc3a.tar.gz"], ), com_google_protobuf = dict( - sha256 = "c10ef8d8ad5a9e5f850483051b7f9ee2c8bb3ca2e0e16a4cf105bd1321afb2d6", - strip_prefix = "protobuf-3.7.1", - urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-all-3.7.1.tar.gz"], + sha256 = "b7220b41481011305bf9100847cf294393973e869973a9661046601959b2960b", + strip_prefix = "protobuf-3.8.0", + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v3.8.0/protobuf-all-3.8.0.tar.gz"], ), grpc_httpjson_transcoding = dict( sha256 = "dedd76b0169eb8c72e479529301a1d9b914a4ccb4d2b5ddb4ebe92d63a7b2152", From 70ae47eba01ec6b4993c587752ddfcf85b08a201 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 11 Jul 2019 05:03:59 -0700 Subject: [PATCH 142/542] build: remove -lc++fs from linkopts (#7534) Description: Cherry-picking latest commit of #7329 before release. To unbreak fuzz build. Address #7507 that bring real c++fs dependency. Risk Level: Low Testing: local fuzz, CI Docs Changes: Release Notes: Signed-off-by: Lizan Zhou --- bazel/envoy_test.bzl | 2 +- test/test_common/environment.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index 5e93e1585871..ea564d619fa8 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -60,7 +60,7 @@ def _envoy_test_linkopts(): # TODO(mattklein123): It's not great that we universally link against the following libs. # In particular, -latomic and -lrt are not needed on all platforms. Make this more granular. "//conditions:default": ["-pthread", "-lrt", "-ldl"], - }) + envoy_select_force_libcpp(["-lc++fs"], ["-lstdc++fs", "-latomic"]) + }) + envoy_select_force_libcpp([], ["-lstdc++fs", "-latomic"]) # Envoy C++ fuzz test targets. These are not included in coverage runs. def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs): diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index 2291fb09a094..ee770090818d 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -42,7 +42,7 @@ std::string makeTempDir(char* name_template) { char* dirname = ::_mktemp(name_template); RELEASE_ASSERT(dirname != nullptr, fmt::format("failed to create tempdir from template: {} {}", name_template, strerror(errno))); -#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION >= 9000 && !defined(__APPLE__) std::__fs::filesystem::create_directories(dirname); #elif defined __cpp_lib_experimental_filesystem std::experimental::filesystem::create_directories(dirname); @@ -85,7 +85,7 @@ char** argv_; } // namespace void TestEnvironment::createPath(const std::string& path) { -#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION >= 9000 && !defined(__APPLE__) // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. std::__fs::filesystem::create_directories(std::__fs::filesystem::path(path)); @@ -98,7 +98,7 @@ void TestEnvironment::createPath(const std::string& path) { } void TestEnvironment::createParentPath(const std::string& path) { -#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION >= 9000 && !defined(__APPLE__) // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. std::__fs::filesystem::create_directories(std::__fs::filesystem::path(path).parent_path()); @@ -113,7 +113,7 @@ void TestEnvironment::createParentPath(const std::string& path) { void TestEnvironment::removePath(const std::string& path) { RELEASE_ASSERT(absl::StartsWith(path, TestEnvironment::temporaryDirectory()), ""); -#if defined(_LIBCPP_VERSION) && !defined(__APPLE__) +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION >= 9000 && !defined(__APPLE__) // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. if (!std::__fs::filesystem::exists(path)) { From 9e8abe02c518223d6c9151df9cea8e6755e67644 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 Jul 2019 09:06:40 -0400 Subject: [PATCH 143/542] docs: cleanup for 1.11 release (#7522) Risk Level: n/a Testing: n/a Docs Changes: relnote cleanup Release Notes: n/a Signed-off-by: Alyssa Wilk --- docs/root/intro/version_history.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index aaebdf46a4b0..f3df12fa7bab 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -11,7 +11,7 @@ Version history * admin: the administration interface now includes a :ref:`/ready endpoint ` for easier readiness checks. * admin: extend :ref:`/runtime_modify endpoint ` to support parameters within the request body. * admin: the :ref:`/listener endpoint ` now returns :ref:`listeners.proto` which includes listener names and ports. -* admin: added host priority to :http:get:`/clusters` and :http:get:`/clusters?format=json` end point response +* admin: added host priority to :http:get:`/clusters` and :http:get:`/clusters?format=json` endpoint response * admin: the :ref:`/clusters endpoint ` now shows hostname for each host, useful for DNS based clusters. * api: track and report requests issued since last load report. @@ -19,7 +19,8 @@ Version history * control-plane: management servers can respond with HTTP 304 to indicate that config is up to date for Envoy proxies polling a :ref:`REST API Config Type ` * csrf: added support for whitelisting additional source origins. * dns: added support for getting DNS record TTL which is used by STRICT_DNS/LOGICAL_DNS cluster as DNS refresh rate. -* dubbo_proxy: support the :ref:`Dubbo proxy filter `. +* dubbo_proxy: support the :ref:`dubbo proxy filter `. +* dynamo_request_parser: adding support for transactions. Adds check for new types of dynamodb operations (TransactWriteItems, TransactGetItems) and awareness for new types of dynamodb errors (IdempotentParameterMismatchException, TransactionCanceledException, TransactionInProgressException). * eds: added support to specify max time for which endpoints can be used :ref:`gRPC filter `. * eds: removed max limit for `load_balancing_weight`. * event: added :ref:`loop duration and poll delay statistics `. @@ -41,12 +42,12 @@ Version history * http: added :ref:`dynamic forward proxy ` support. * http: tracking the active stream and dumping state in Envoy crash handlers. This can be disabled by building with `--define disable_object_dump_on_signal_trace=disabled` * jwt_authn: make filter's parsing of JWT more flexible, allowing syntax like ``jwt=eyJhbGciOiJS...ZFnFIw,extra=7,realm=123`` -* outlier_detector: added configuration :ref:`outlier_detection.split_external_local_origin_errors` to distinguish locally and externally generated errors. See :ref:`arch_overview_outlier_detection` for full details. * listener: added :ref:`source IP ` and :ref:`source port ` filter chain matching. * lua: exposed functions to Lua to verify digital signature. * original_src filter: added the :ref:`filter`. +* outlier_detector: added configuration :ref:`outlier_detection.split_external_local_origin_errors` to distinguish locally and externally generated errors. See :ref:`arch_overview_outlier_detection` for full details. * rbac: migrated from v2alpha to v2. * redis: add support for Redis cluster custom cluster type. * redis: automatically route commands using cluster slots for Redis cluster. @@ -58,7 +59,7 @@ Version history :ref:`buffer_flush_timeout ` to control how quickly the buffer is flushed if it is not full. * redis: added auth support :ref:`downstream_auth_password ` for downstream client authentication, and :ref:`auth_password ` to configure authentication passwords for upstream server clusters. * retry: added a retry predicate that :ref:`rejects canary hosts. ` -* router: add support for configuring a :ref:`grpc timeout offset ` on incoming requests. +* router: add support for configuring a :ref:`gRPC timeout offset ` on incoming requests. * router: added ability to control retry back-off intervals via :ref:`retry policy `. * router: added ability to issue a hedged retry in response to a per try timeout via a :ref:`hedge policy `. * router: added a route name field to each http route in route.Route list @@ -80,12 +81,12 @@ Version history * sandbox: added :ref:`CSRF sandbox `. * server: ``--define manual_stamp=manual_stamp`` was added to allow server stamping outside of binary rules. more info in the `bazel docs `_. -* server: added :ref:`Server State ` statistic. +* server: added :ref:`server state ` statistic. * server: added :ref:`initialization_time_ms` statistic. * subset: added :ref:`list_as_any` option to the subset lb which allows matching metadata against any of the values in a list value on the endpoints. -* tool: added :repo:`proto ` support for :ref:`router check tool ` tests. +* tools: added :repo:`proto ` support for :ref:`router check tool ` tests. * tracing: add trace sampling configuration to the route, to override the route level. * upstream: added :ref:`upstream_cx_pool_overflow ` for the connection pool circuit breaker. * upstream: an EDS management server can now force removal of a host that is still passing active @@ -167,7 +168,7 @@ Version history * router: added :ref:`rq_reset_after_downstream_response_started ` counter stat to router stats. * router: added per-route configuration of :ref:`internal redirects `. * router: removed deprecated route-action level headers_to_add/remove. -* router: made :ref: `max retries header ` take precedence over the number of retries in route and virtual host retry policies. +* router: made :ref:`max retries header ` take precedence over the number of retries in route and virtual host retry policies. * router: added support for prefix wildcards in :ref:`virtual host domains` * stats: added support for histograms in prometheus * stats: added usedonly flag to prometheus stats to only output metrics which have been From 47f2be32ce74d6c9e5fb2f8930d38c091125a7f2 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 11 Jul 2019 16:30:41 +0300 Subject: [PATCH 144/542] tests: remove redundant std::move() (#7535) Description: gcc9 reports the uses of std::move() redundant. Risk Level: low Testing: unit tests Release Notes: N/A Documentation: N/A Signed-off-by: Dmitry Rozhkov --- .../filters/common/ext_authz/check_request_utils_test.cc | 2 +- test/server/overload_manager_impl_test.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc index 791e019e6aa9..7567a9a95023 100644 --- a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc +++ b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc @@ -48,7 +48,7 @@ class CheckRequestUtilsTest : public testing::Test { Buffer::OwnedImpl("Lorem ipsum dolor sit amet, consectetuer adipiscing elit."); buffer->add(new_buffer); } - return std::move(buffer); + return buffer; } Network::Address::InstanceConstSharedPtr addr_; diff --git a/test/server/overload_manager_impl_test.cc b/test/server/overload_manager_impl_test.cc index 38a82df7f888..7ee741245d5d 100644 --- a/test/server/overload_manager_impl_test.cc +++ b/test/server/overload_manager_impl_test.cc @@ -64,7 +64,7 @@ class FakeResourceMonitorFactory Server::Configuration::ResourceMonitorFactoryContext& context) override { auto monitor = std::make_unique(context.dispatcher()); monitor_ = monitor.get(); - return std::move(monitor); + return monitor; } FakeResourceMonitor* monitor_; // not owned From ea41ce1fa02830f099760e857df219f27714f0a9 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Thu, 11 Jul 2019 08:09:16 -0700 Subject: [PATCH 145/542] bazel: Update to 0.28.0 (#7533) * bazel: Update to 0.28.0 Signed-off-by: Keith Smiley --- .azure-pipelines/linux.yml | 4 ++-- .circleci/config.yml | 2 +- bazel/external/boringssl_fips.genrule_cmd | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index 4bb64ea59faa..725e299b19cb 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -1,7 +1,7 @@ resources: containers: - container: envoy-build - image: envoyproxy/envoy-build:d0cefa7f071dbd4ef24399c2db8656c3a5d8c3ef + image: envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d jobs: - job: EnvoyFullTest @@ -10,7 +10,7 @@ jobs: vmImage: 'Ubuntu 16.04' container: envoy-build steps: - # bazel.dev isn't for CI purpose but we use it here to experiment with Azure Pipeline + # bazel.dev isn't for CI purpose but we use it here to experiment with Azure Pipeline # as it builds faster, before Azure can provision larger VM or we can use RBE. - script: ci/do_ci.sh bazel.dev env: diff --git a/.circleci/config.yml b/.circleci/config.yml index d3458d108886..69c8a7ed7437 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ executors: ubuntu-build: description: "A regular build executor based on ubuntu image" docker: - - image: envoyproxy/envoy-build:d0cefa7f071dbd4ef24399c2db8656c3a5d8c3ef + - image: envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d resource_class: xlarge working_directory: /source diff --git a/bazel/external/boringssl_fips.genrule_cmd b/bazel/external/boringssl_fips.genrule_cmd index d498d2ffd3db..cff25f0f084e 100644 --- a/bazel/external/boringssl_fips.genrule_cmd +++ b/bazel/external/boringssl_fips.genrule_cmd @@ -65,7 +65,7 @@ PLATFORM="linux" curl -sLO https://github.com/ninja-build/ninja/releases/download/v"$$VERSION"/ninja-"$$PLATFORM".zip \ && echo "$$SHA256" ninja-"$$PLATFORM".zip | sha256sum --check -unzip ninja-"$$PLATFORM".zip +unzip -o ninja-"$$PLATFORM".zip export PATH="$$PWD:$$PATH" From 3094b9e1ef2c377c277fe4e3c07a50d2d58f5794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dan=20No=C3=A9?= Date: Thu, 11 Jul 2019 11:53:53 -0400 Subject: [PATCH 146/542] Bump c-ARES version to include important fixes. (#7539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In #7395, the c-ARES dependency version number was increased to gain access to a new getaddrinfo API by importing a specific nonrelease SHA from c-ARES. This specific SHA did not include c-ares/c-ares@b949cc3 , made subsequently, which contains important security relevant fixes for the getaddrinfo API. This PR bumps the c-ARES version number to include them. Signed-off-by: Dan Noé dpn@google.com Description: Risk Level: Low - external dependency version update Testing: bazel test //test/... Signed-off-by: Dan Noé --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 6e25d2fd589e..383f40e90c60 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -27,13 +27,13 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://files.pythonhosted.org/packages/c6/b4/510617906f8e0c5660e7d96fbc5585113f83ad547a3989b80297ac72a74c/thrift-0.11.0.tar.gz"], ), com_github_c_ares_c_ares = dict( - sha256 = "96edccdb19d79f6bc48c2c0e5916346c8f0507efa72e76bd146a1b9d05f93c2a", - strip_prefix = "c-ares-5dd3629bc93449840c36dd635ea6cce606b8c366", + sha256 = "bbaab13d6ad399a278d476f533e4d88a7ec7d729507348bb9c2e3b207ba4c606", + strip_prefix = "c-ares-d7e070e7283f822b1d2787903cce3615536c5610", # 2019-06-19 - # 21 new commits from release-1.15.0. Upgrade for commit 7d3591ee8a1a63e7748e68e6d880bd1763a32885 "getaddrinfo enhancements" + # 27 new commits from release-1.15.0. Upgrade for commit 7d3591ee8a1a63e7748e68e6d880bd1763a32885 "getaddrinfo enhancements" and follow up fixes. # Use getaddrinfo to query DNS record and TTL. # TODO(crazyxy): Update to release-1.16.0 when it is released. - urls = ["https://github.com/c-ares/c-ares/archive/5dd3629bc93449840c36dd635ea6cce606b8c366.tar.gz"], + urls = ["https://github.com/c-ares/c-ares/archive/d7e070e7283f822b1d2787903cce3615536c5610.tar.gz"], ), com_github_circonus_labs_libcircllhist = dict( sha256 = "8165aa25e529d7d4b9ae849d3bf30371255a99d6db0421516abcff23214cdc2c", From bf169f9d3c8f4c682650c5390c088a4898940913 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 Jul 2019 11:56:34 -0400 Subject: [PATCH 147/542] release: bump to 1.11.0 (#7538) Risk Level: n/a Testing: n/a Docs Changes: 1.11! Release Notes: 1.11! Signed-off-by: Alyssa Wilk --- VERSION | 2 +- docs/root/intro/deprecated.rst | 4 ++-- docs/root/intro/version_history.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 1f724bf455d7..1cac385c6cb8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0-dev +1.11.0 diff --git a/docs/root/intro/deprecated.rst b/docs/root/intro/deprecated.rst index 31b147dcf4fe..63b624e3f126 100644 --- a/docs/root/intro/deprecated.rst +++ b/docs/root/intro/deprecated.rst @@ -10,8 +10,8 @@ The following features have been DEPRECATED and will be removed in the specified A logged warning is expected for each deprecated item that is in deprecation window. Deprecated items below are listed in chronological order. -Version 1.11.0 (Pending) -======================== +Version 1.11.0 (July 11, 2019) +============================== * The --max-stats and --max-obj-name-len flags no longer has any effect. * Use of :ref:`cluster ` in :ref:`redis_proxy.proto ` is deprecated. Set a :ref:`catch_all_route ` instead. * Use of :ref:`catch_all_cluster ` in :ref:`redis_proxy.proto ` is deprecated. Set a :ref:`catch_all_route ` instead. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index f3df12fa7bab..bcd2848341e1 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -1,8 +1,8 @@ Version history --------------- -1.11.0 (Pending) -================ +1.11.0 (July 11, 2019) +====================== * access log: added a new field for downstream TLS session ID to file and gRPC access logger. * access log: added a new field for route name to file and gRPC access logger. * access log: added a new field for response code details in :ref:`file access logger` and :ref:`gRPC access logger`. From 2c2411e6790894f57aa3aa4b5ff6f7e8317e02b3 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 Jul 2019 15:15:21 -0400 Subject: [PATCH 148/542] release: kicking off 1.12 (#7544) Signed-off-by: Alyssa Wilk --- VERSION | 2 +- docs/root/intro/deprecated.rst | 4 ++++ docs/root/intro/version_history.rst | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1cac385c6cb8..381cf02417c4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0 +1.12.0-dev diff --git a/docs/root/intro/deprecated.rst b/docs/root/intro/deprecated.rst index 63b624e3f126..6fdd161ae631 100644 --- a/docs/root/intro/deprecated.rst +++ b/docs/root/intro/deprecated.rst @@ -10,6 +10,10 @@ The following features have been DEPRECATED and will be removed in the specified A logged warning is expected for each deprecated item that is in deprecation window. Deprecated items below are listed in chronological order. +Version 1.12.0 (pending) +======================== + + Version 1.11.0 (July 11, 2019) ============================== * The --max-stats and --max-obj-name-len flags no longer has any effect. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index bcd2848341e1..70e8fe3e8c87 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -1,6 +1,9 @@ Version history --------------- +1.12.0 (pending) +================ + 1.11.0 (July 11, 2019) ====================== * access log: added a new field for downstream TLS session ID to file and gRPC access logger. From 2f8b6a26a0e63051d50194bd2500a41aa63b293c Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 Jul 2019 15:28:28 -0400 Subject: [PATCH 149/542] test: hopefully deflaking lua test (#7542) As discussed on slack I think this is hitting timeout/OOM due to the large loop factor. Toning it down for tsan to get CI to stop flaking while we figure out why it wasn't logging Risk Level: n/a (test only) Testing: n/a Docs Changes: n/a Release Notes: n/a mitigating #7374 Signed-off-by: Alyssa Wilk --- test/extensions/filters/http/lua/lua_filter_test.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/lua/lua_filter_test.cc b/test/extensions/filters/http/lua/lua_filter_test.cc index e24b267b7823..c626b104346d 100644 --- a/test/extensions/filters/http/lua/lua_filter_test.cc +++ b/test/extensions/filters/http/lua/lua_filter_test.cc @@ -1257,7 +1257,14 @@ TEST_F(LuaHttpFilterTest, ImmediateResponse) { config_->runtimeGC(); const uint64_t mem_use_at_start = config_->runtimeBytesUsed(); - for (uint64_t i = 0; i < 2000; i++) { + uint64_t num_loops = 2000; +#if defined(__has_feature) && (__has_feature(thread_sanitizer)) + // per https://github.com/envoyproxy/envoy/issues/7374 this test is causing + // problems on tsan + num_loops = 200; +#endif + + for (uint64_t i = 0; i < num_loops; i++) { Http::TestHeaderMapImpl request_headers{{":path", "/"}}; Http::TestHeaderMapImpl expected_headers{{":status", "503"}, {"content-length", "4"}}; EXPECT_CALL(decoder_callbacks_, encodeHeaders_(HeaderMapEqualRef(&expected_headers), false)); From e75ff19facdeed4a4085ac0ddfe33d2297052b87 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 11 Jul 2019 15:31:27 -0400 Subject: [PATCH 150/542] tools: update our release scripts (#7493) Functionally this removes the script which filed bugs to remove old config (per plan of record in #6271) adds a script which files bugs to remove old code (cleanup of runtime guarded features) updates our release process In practice, this just hot swaps what our deprecation script does and updates the docs. Risk Level: n/a (tooling + comment) Testing: manual testing. Will do one more manual test with actual bug filing just before submitting. Docs Changes: updated Release Notes: n/a Fixes #6472 Signed-off-by: Alyssa Wilk --- GOVERNANCE.md | 2 +- source/common/runtime/runtime_features.cc | 5 - tools/deprecate_version/deprecate_version.py | 114 ++++++++++++------- 3 files changed, 76 insertions(+), 45 deletions(-) diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 6639dbffca17..aa5cb0da5972 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -101,7 +101,7 @@ or you can subscribe to the iCal feed [here](https://app.opsgenie.com/webcal/get the same time, also add a new empty "pending" section to the [release notes](docs/root/intro/version_history.rst) and to [deprecated log](docs/root/intro/deprecated.rst) for the following version. E.g., "1.7.0 (pending)". -* Run the deprecate_versions.py script (e.g. `sh tools/deprecate_version/deprecate_version.sh 1.8.0 1.10.0`) +* Run the deprecate_versions.py script (e.g. `sh tools/deprecate_version/deprecate_version.sh`) to file tracking issues for code which can be removed. * Run the deprecate_features.py script (e.g. `sh tools/deprecate_version/deprecate_features.sh`) to make the last release's deprecated features fatal-by-default. Submit the resultant PR and send diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index e9c20ff38199..bb1800a1b198 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -38,11 +38,6 @@ constexpr const char* runtime_features[] = { // // The release cycle after a feature has been marked disallowed, it is officially removable, and // the maintainer team will run a script creating a tracking issue for proto and code clean up. -// -// TODO(alyssawilk) handle deprecation of reloadable_features and update the above comment. Ideally -// runtime override of a deprecated feature will log(warn) on runtime-load if not deprecated -// and hard-fail once it has been deprecated. - constexpr const char* disallowed_features[] = { // Acts as both a test entry for deprecated.proto and a marker for the Envoy // deprecation scripts. diff --git a/tools/deprecate_version/deprecate_version.py b/tools/deprecate_version/deprecate_version.py index b73c152d6d56..caeecd0d25a3 100644 --- a/tools/deprecate_version/deprecate_version.py +++ b/tools/deprecate_version/deprecate_version.py @@ -1,15 +1,14 @@ -# Script for automating cleanup PR creation for deprecated features at a given -# version. This script is generally run via +# Script for automating cleanup PR creation for deprecated runtime features # # sh tools/deprecate_version/deprecate_version.sh # # Direct usage (not recommended): # -# python tools/deprecate_version/deprecate_version.py <2 releases ago> +# python tools/deprecate_version/deprecate_version.py # # e.g # -# python tools/deprecate_version/deprecate_version.py 1.5.0 1.7.0 +# python tools/deprecate_version/deprecate_version.py # # A GitHub access token must be set in GH_ACCESS_TOKEN. To create one, go to # Settings -> Developer settings -> Personal access tokens in GitHub and create @@ -20,14 +19,13 @@ # Known issues: # - Minor fixup PRs (e.g. fixing a typo) will result in the creation of spurious # issues. -# - Later PRs can clobber earlier changed to DEPRECATED.md, meaning we miss -# issues. from __future__ import print_function from collections import defaultdict import os import re +import subprocess import sys import github @@ -72,25 +70,15 @@ def GetConfirmation(): return input('Creates issues? [yN] ').strip().lower() in ('y', 'yes') -def CreateIssues(deprecate_for_version, deprecate_by_version, access_token, commits): - """Create issues in GitHub corresponding to a set of commits. +def CreateIssues(access_token, runtime_and_pr): + """Create issues in GitHub for code to clean up old runtime guarded features. Args: - deprecate_for_version: string providing version to deprecate for, e.g. - 1.6.0. - deprecate_by_version: string providing version to deprecate by, e.g. 1.7.0. access_token: GitHub access token (see comment at top of file). - commits: set of git commit objects. + runtime_and_pr: a list of runtime guards and the PRs they were added. """ repo = github.Github(access_token).get_repo('envoyproxy/envoy') - # Find GitHub milestone object for deprecation target. - milestone = None - for m in repo.get_milestones(): - if m.title == deprecate_by_version: - milestone = m - break - if not milestone: - raise DeprecateVersionError('Unknown milestone %s' % deprecate_by_version) + # Find GitHub label objects for LABELS. labels = [] for label in repo.get_labels(): @@ -98,19 +86,19 @@ def CreateIssues(deprecate_for_version, deprecate_by_version, access_token, comm labels.append(label) if len(labels) != len(LABELS): raise DeprecateVersionError('Unknown labels (expected %s, got %s)' % (LABELS, labels)) - # What are the PRs corresponding to the commits? - prs = (int(re.search('\(#(\d+)\)', c.message).group(1)) for c in commits) + issues = [] - for pr in sorted(prs): + for runtime_guard, pr in runtime_and_pr: # Who is the author? pr_info = repo.get_pull(pr) - title = '[v%s deprecation] Remove features marked deprecated in #%d' % (deprecate_for_version, - pr) - body = ('#%d (%s) introduced a deprecation notice for v%s. This issue ' - 'tracks source code cleanup.') % (pr, pr_info.title, deprecate_for_version) + + title = '%s deprecation' % (runtime_guard) + body = ('#%d (%s) introduced a runtime guarded feature. This issue ' + 'tracks source code cleanup.') % (pr, pr_info.title) print(title) print(body) print(' >> Assigning to %s' % pr_info.user.login) + # TODO(htuch): Figure out how to do this without legacy and faster. exists = repo.legacy_search_issues('open', '"%s"' % title) or repo.legacy_search_issues( 'closed', '"%s"' % title) @@ -118,12 +106,16 @@ def CreateIssues(deprecate_for_version, deprecate_by_version, access_token, comm print(' >> Issue already exists, not posting!') else: issues.append((title, body, pr_info.user)) + + if not issues: + print('No features to deprecate in this release') + return + if GetConfirmation(): print('Creating issues...') for title, body, user in issues: try: - repo.create_issue( - title, body=body, assignees=[user.login], milestone=milestone, labels=labels) + repo.create_issue(title, body=body, assignees=[user.login], labels=labels) except github.GithubException as e: print(('GithubException while creating issue. This is typically because' ' a user is not a member of envoyproxy org. Check that %s is in ' @@ -131,18 +123,62 @@ def CreateIssues(deprecate_for_version, deprecate_by_version, access_token, comm raise +def GetRuntimeAlreadyTrue(): + """Returns a list of runtime flags already defaulted to true + """ + runtime_already_true = [] + runtime_features = re.compile(r'.*"(envoy.reloadable_features..*)",.*') + with open('source/common/runtime/runtime_features.cc', 'r') as features: + for line in features.readlines(): + match = runtime_features.match(line) + if match and 'test_feature_true' not in match.group(1): + print("Found existing flag " + match.group(1)) + runtime_already_true.append(match.group(1)) + + return runtime_already_true + + +def GetRuntimeAndPr(): + """Returns a list of tuples of [runtime features to deprecate, PR the feature was added] + """ + repo = Repo(os.getcwd()) + + runtime_already_true = GetRuntimeAlreadyTrue() + + # grep source code looking for reloadable features which are true to find the + # PR they were added. + grep_output = subprocess.check_output('grep -r "envoy.reloadable_features\." source/', shell=True) + features_to_flip = [] + runtime_feature_regex = re.compile(r'.*(source.*cc).*"(envoy.reloadable_features\.[^"]+)".*') + for line in grep_output.splitlines(): + match = runtime_feature_regex.match(str(line)) + if match: + filename = (match.group(1)) + runtime_guard = match.group(2) + # If this runtime guard isn't true, ignore it for this release. + if not runtime_guard in runtime_already_true: + continue + # For true runtime guards, walk the blame of the file they were added to, + # to find the pr the feature was added. + for commit, lines in repo.blame('HEAD', filename): + for line in lines: + if runtime_guard in line: + pr = (int(re.search('\(#(\d+)\)', commit.message).group(1))) + # Add the runtime guard and PR to the list to file issues about. + features_to_flip.append((runtime_guard, pr)) + + else: + print('no match in ' + str(line) + ' please address manually!') + + return features_to_flip + + if __name__ == '__main__': - if len(sys.argv) != 3: - print('Usage: %s ' % sys.argv[0]) - sys.exit(1) + runtime_and_pr = GetRuntimeAndPr() + access_token = os.getenv('GH_ACCESS_TOKEN') if not access_token: print('Missing GH_ACCESS_TOKEN') sys.exit(1) - deprecate_for_version = sys.argv[1] - deprecate_by_version = sys.argv[2] - history = GetHistory() - if deprecate_for_version not in history: - print('Unknown version: %s (valid versions: %s)' % (deprecate_for_version, history.keys())) - CreateIssues(deprecate_for_version, deprecate_by_version, access_token, - history[deprecate_for_version]) \ No newline at end of file + + CreateIssues(access_token, runtime_and_pr) From 00a6251eae775b8eb88092eb51d60dfa4e1009b4 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 11 Jul 2019 12:45:29 -0700 Subject: [PATCH 151/542] build: simplify cc_configure and static link for libc++ (#7329) Description: Take a similar approach of https://github.com/bazelbuild/bazel/commit/ab9c1f5f98a993b4da532224a9fed1b1b9a90148, which use `-l:libstdc++.a` to statically link libstdc++. This makes us closer to remove our own `cc_wrapper` and `cc_configure` in the future. Also it will allow us do static link with libc++. Risk Level: Med Testing: local, CI Docs Changes: N/A Release Notes: N/A Signed-off-by: Lizan Zhou --- .bazelrc | 2 ++ bazel/cc_configure.bzl | 52 +---------------------------------- bazel/cc_wrapper.py | 27 ++++++++++-------- bazel/envoy_internal.bzl | 4 +-- bazel/foreign_cc/BUILD | 2 ++ test/exe/envoy_static_test.sh | 4 +-- 6 files changed, 22 insertions(+), 69 deletions(-) diff --git a/.bazelrc b/.bazelrc index 786af870ed82..a365b1d064bc 100644 --- a/.bazelrc +++ b/.bazelrc @@ -41,6 +41,7 @@ build:clang-tsan --define ENVOY_CONFIG_TSAN=1 build:clang-tsan --copt -fsanitize=thread build:clang-tsan --linkopt -fsanitize=thread build:clang-tsan --linkopt -fuse-ld=lld +build:clang-tsan --linkopt -static-libsan build:clang-tsan --define tcmalloc=disabled # Needed due to https://github.com/libevent/libevent/issues/777 build:clang-tsan --copt -DEVENT__DISABLE_DEBUG_MODE @@ -60,6 +61,7 @@ build:libc++ --action_env=CC build:libc++ --action_env=CXX build:libc++ --action_env=CXXFLAGS=-stdlib=libc++ build:libc++ --action_env=PATH +build:libc++ --host_linkopt=-fuse-ld=lld build:libc++ --define force_libcpp=enabled # Optimize build for binary size reduction. diff --git a/bazel/cc_configure.bzl b/bazel/cc_configure.bzl index 9122735d9a2d..30275140b917 100644 --- a/bazel/cc_configure.bzl +++ b/bazel/cc_configure.bzl @@ -2,57 +2,8 @@ load("@bazel_tools//tools/cpp:cc_configure.bzl", _upstream_cc_autoconf_impl = "c load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") load("@bazel_tools//tools/cpp:unix_cc_configure.bzl", "find_cc") -# Stub for `repository_ctx.which()` that always succeeds. See comments in -# `_find_cxx` for details. -def _quiet_fake_which(program): - return struct(_envoy_fake_which = program) - -# Stub for `repository_ctx.which()` that always fails. See comments in -# `_find_cxx` for details. -def _noisy_fake_which(program): - return None - -# Find a good path for the C++ compiler, by hooking into Bazel's C compiler -# detection. Uses `$CXX` if found, otherwise defaults to `g++` because Bazel -# defaults to `gcc`. -def _find_cxx(repository_ctx): - # Bazel's `find_cc` helper uses the repository context to inspect `$CC`. - # Replace this value with `$CXX` if set. - environ_cxx = repository_ctx.os.environ.get("CXX", "g++") - fake_os = struct( - environ = {"CC": environ_cxx}, - ) - - # We can't directly assign `repository_ctx.which` to a struct attribute - # because Skylark doesn't support bound method references. Instead, stub - # out `which()` using a two-pass approach: - # - # * The first pass uses a stub that always succeeds, passing back a special - # value containing the original parameter. - # * If we detect the special value, we know that `find_cc` found a compiler - # name but don't know if that name could be resolved to an executable path. - # So do the `which()` call ourselves. - # * If our `which()` failed, call `find_cc` again with a dummy which that - # always fails. The error raised by `find_cc` will be identical to what Bazel - # would generate for a missing C compiler. - # - # See https://github.com/bazelbuild/bazel/issues/4644 for more context. - real_cxx = find_cc(struct( - which = _quiet_fake_which, - os = fake_os, - ), {}) - if hasattr(real_cxx, "_envoy_fake_which"): - real_cxx = repository_ctx.which(real_cxx._envoy_fake_which) - if real_cxx == None: - find_cc(struct( - which = _noisy_fake_which, - os = fake_os, - ), {}) - return real_cxx - def _build_envoy_cc_wrapper(repository_ctx): real_cc = find_cc(repository_ctx, {}) - real_cxx = _find_cxx(repository_ctx) # Copy our CC wrapper script into @local_config_cc, with the true paths # to the C and C++ compiler injected in. The wrapper will use these paths @@ -64,7 +15,6 @@ def _build_envoy_cc_wrapper(repository_ctx): repository_ctx.template("extra_tools/envoy_cc_wrapper", repository_ctx.attr._envoy_cc_wrapper, { "{ENVOY_REAL_CC}": repr(str(real_cc)), "{ENVOY_CFLAGS}": repr(str(repository_ctx.os.environ.get("CFLAGS", ""))), - "{ENVOY_REAL_CXX}": repr(str(real_cxx)), "{ENVOY_CXXFLAGS}": repr(str(repository_ctx.os.environ.get("CXXFLAGS", ""))), }) return repository_ctx.path("extra_tools/envoy_cc_wrapper") @@ -93,6 +43,7 @@ cc_autoconf = repository_rule( "ABI_VERSION", "BAZEL_COMPILER", "BAZEL_HOST_SYSTEM", + "BAZEL_CXXOPTS", "BAZEL_LINKOPTS", "BAZEL_PYTHON", "BAZEL_SH", @@ -108,7 +59,6 @@ cc_autoconf = repository_rule( "USE_CLANG_CL", "CC", "CFLAGS", - "CXX", "CXXFLAGS", "CC_CONFIGURE_DEBUG", "CC_TOOLCHAIN_NAME", diff --git a/bazel/cc_wrapper.py b/bazel/cc_wrapper.py index c4883edd065d..53037e67bd2e 100755 --- a/bazel/cc_wrapper.py +++ b/bazel/cc_wrapper.py @@ -5,8 +5,7 @@ import sys import tempfile -envoy_real_cc = {ENVOY_REAL_CC} -envoy_real_cxx = {ENVOY_REAL_CXX} +compiler = {ENVOY_REAL_CC} envoy_cflags = {ENVOY_CFLAGS} envoy_cxxflags = {ENVOY_CXXFLAGS} @@ -25,7 +24,7 @@ def sanitize_flagfile(in_path, out_fd): if line != "-lstdc++\n": os.write(out_fd, line) elif "-stdlib=libc++" in envoy_cxxflags: - os.write(out_fd, "-lc++\n") + os.write(out_fd, "-l:libc++.a\n-l:libc++abi.a\n") # Is the arg a flag indicating that we're building for C++ (rather than C)? @@ -37,11 +36,9 @@ def is_cpp_flag(arg): def modify_driver_args(input_driver_flags): # Detect if we're building for C++ or vanilla C. if any(map(is_cpp_flag, input_driver_flags)): - compiler = envoy_real_cxx # Append CXXFLAGS to all C++ targets (this is mostly for dependencies). argv = shlex.split(envoy_cxxflags) else: - compiler = envoy_real_cc # Append CFLAGS to all C targets (this is mostly for dependencies). argv = shlex.split(envoy_cflags) @@ -50,9 +47,8 @@ def modify_driver_args(input_driver_flags): # b) replace all occurrences of -lstdc++ with -lc++ (when linking against libc++). if "-static-libstdc++" in input_driver_flags or "-stdlib=libc++" in envoy_cxxflags: for arg in input_driver_flags: - if arg == "-lstdc++": - if "-stdlib=libc++" in envoy_cxxflags: - argv.append("-lc++") + if arg in ("-lstdc++", "-static-libstdc++"): + pass elif arg.startswith("-Wl,@"): # tempfile.mkstemp will write to the out-of-sandbox tempdir # unless the user has explicitly set environment variables @@ -67,6 +63,13 @@ def modify_driver_args(input_driver_flags): else: argv += input_driver_flags + # This flags should after all libraries + if "-static-libstdc++" in input_driver_flags: + argv.append("-l:libstdc++.a") + if "-lstdc++" in input_driver_flags and "-stdlib=libc++" in envoy_cxxflags: + argv.append("-l:libc++.a") + argv.append("-l:libc++abi.a") + # Bazel will add -fuse-ld=gold in some cases, gcc/clang will take the last -fuse-ld argument, # so whenever we see lld once, add it to the end. if "-fuse-ld=lld" in argv: @@ -87,13 +90,13 @@ def modify_driver_args(input_driver_flags): # See https://github.com/envoyproxy/envoy/issues/2987 argv.append("-Wno-maybe-uninitialized") - return compiler, argv + return argv def main(): # Append CXXFLAGS to correctly detect include paths for either libstdc++ or libc++. if sys.argv[1:5] == ["-E", "-xc++", "-", "-v"]: - os.execv(envoy_real_cxx, [envoy_real_cxx] + sys.argv[1:] + shlex.split(envoy_cxxflags)) + os.execv(compiler, [compiler] + sys.argv[1:] + shlex.split(envoy_cxxflags)) if sys.argv[1].startswith("@"): # Read flags from file @@ -102,7 +105,7 @@ def main(): input_driver_flags = fd.read().splitlines() # Compute new args - compiler, new_driver_args = modify_driver_args(input_driver_flags) + new_driver_args = modify_driver_args(input_driver_flags) # Write args to temp file (new_flagfile_fd, new_flagfile_path) = tempfile.mkstemp(dir="./", suffix=".linker-params") @@ -116,7 +119,7 @@ def main(): else: # TODO(https://github.com/bazelbuild/bazel/issues/7687): Remove this branch # when Bazel 0.27 is released. - compiler, new_args = modify_driver_args(sys.argv[1:]) + new_args = modify_driver_args(sys.argv[1:]) os.execv(compiler, [compiler] + new_args) diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 41b86a4d1b0a..4ea71659d295 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -84,9 +84,7 @@ def envoy_select_force_libcpp(if_libcpp, default = None): def envoy_static_link_libstdcpp_linkopts(): return envoy_select_force_libcpp( - # TODO(PiotrSikora): statically link libc++ once that's possible. - # See: https://reviews.llvm.org/D53238 - ["-stdlib=libc++"], + ["-stdlib=libc++", "-l:libc++.a", "-l:libc++abi.a", "-static-libgcc"], ["-static-libstdc++", "-static-libgcc"], ) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 00b870afa35a..f6049369a9ad 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -125,6 +125,7 @@ envoy_cmake_external( "ENABLE_LIB_ONLY": "on", "CMAKE_BUILD_TYPE": "RelWithDebInfo", "CMAKE_INSTALL_LIBDIR": "lib", + "CMAKE_CXX_COMPILER_FORCED": "on", # Disable ranlib because it is not handled by bazel, and therefore # doesn't respect custom toolchains such as the Android NDK, # see https://github.com/bazelbuild/rules_foreign_cc/issues/252 @@ -147,6 +148,7 @@ envoy_cmake_external( "YAML_CPP_BUILD_TESTS": "off", "YAML_CPP_BUILD_TOOLS": "off", "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "CMAKE_CXX_COMPILER_FORCED": "on", # Disable ranlib because it is not handled by bazel, and therefore # doesn't respect custom toolchains such as the Android NDK, # see https://github.com/bazelbuild/rules_foreign_cc/issues/252 diff --git a/test/exe/envoy_static_test.sh b/test/exe/envoy_static_test.sh index 8324aeca736e..17832c183efd 100755 --- a/test/exe/envoy_static_test.sh +++ b/test/exe/envoy_static_test.sh @@ -17,9 +17,7 @@ fi if [[ ${DYNLIBS} =~ "libc++" ]]; then echo "libc++ is dynamically linked:" echo "${DYNLIBS}" - # TODO(PiotrSikora): enforce this once there is a way to statically link libc++. - # See: https://reviews.llvm.org/D53238 - exit 0 + exit 1 fi if [[ ${DYNLIBS} =~ "libstdc++" || ${DYNLIBS} =~ "libgcc" ]]; then From feaada367d84107632b01d1c3d91ba7657dcaa8a Mon Sep 17 00:00:00 2001 From: Michael Puncel Date: Thu, 11 Jul 2019 16:23:38 -0400 Subject: [PATCH 152/542] Fix crash in request hedging handling. (#7530) This commit fixes a crash in which an upstream request would not be properly reset after the router was done with it and the callbacks destroyed which would cause a segfault when data was received on the upstream request. This happened in particular when "bad" headers are received for an upstream request but there are still requests in flight. When hedging is disabled (i.e. default behavior) this is not an issue because the filter will be destroyed synchronously which will reset the stream, however with hedging it needs to be done more proactively. Signed-off-by: Michael Puncel --- source/common/router/router.cc | 5 +- test/common/router/router_test.cc | 83 +++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/source/common/router/router.cc b/source/common/router/router.cc index ca379197b838..1a7dcd831cd4 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -1052,7 +1052,10 @@ void Filter::onUpstreamHeaders(uint64_t response_code, Http::HeaderMapPtr&& head // chance to return before returning a response downstream. if (could_not_retry && (numRequestsAwaitingHeaders() > 0 || pending_retries_ > 0)) { upstream_request.upstream_host_->stats().rq_error_.inc(); - upstream_request.removeFromList(upstream_requests_); + + // Reset the stream because there are other in-flight requests that we'll + // wait around for and we're not interested in consuming any body/trailers. + upstream_request.removeFromList(upstream_requests_)->resetStream(); return; } diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index b9e2f9dc8172..23aa2817d88f 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -1592,6 +1592,89 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { // TODO: Verify hedge stats here once they are implemented. } +// Tests that an upstream request is reset even if it can't be retried as long as there is +// another in-flight request we're waiting on. +// Sequence: +// 1) first upstream request per try timeout +// 2) second upstream request sent +// 3) second upstream request gets 5xx, retries exhausted, assert it's reset +// 4) first upstream request gets 2xx +TEST_F(RouterTest, HedgedPerTryTimeoutResetsOnBadHeaders) { + enableHedgeOnPerTryTimeout(); + + NiceMock encoder1; + Http::StreamDecoder* response_decoder1 = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder1 = &decoder; + EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + return nullptr; + })); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, + absl::optional(absl::nullopt))) + .Times(2); + expectPerTryTimerCreate(); + expectResponseTimerCreate(); + + Http::TestHeaderMapImpl headers{{"x-envoy-upstream-rq-per-try-timeout-ms", "5"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + EXPECT_CALL( + cm_.conn_pool_.host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); + EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); + NiceMock encoder2; + Http::StreamDecoder* response_decoder2 = nullptr; + router_.retry_state_->expectHedgedPerTryTimeoutRetry(); + per_try_timeout_->callback_(); + + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder2 = &decoder; + EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + return nullptr; + })); + expectPerTryTimerCreate(); + router_.retry_state_->callback_(); + + // We should not have updated any stats yet because no requests have been + // canceled + EXPECT_TRUE(verifyHostUpstreamStats(0, 0)); + + // Now write a 5xx back on the 2nd request with no retries remaining. The 2nd request + // should be reset immediately. + Http::HeaderMapPtr bad_response_headers(new Http::TestHeaderMapImpl{{":status", "500"}}); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(500)); + EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); + EXPECT_CALL(encoder2.stream_, resetStream(_)); + EXPECT_CALL(*router_.retry_state_, shouldRetryHeaders(_, _)) + .WillOnce(Return(RetryStatus::NoOverflow)); + // Not end_stream, otherwise we wouldn't need to reset. + response_decoder2->decodeHeaders(std::move(bad_response_headers), false); + + // Now write a 200 back. We expect the 2nd stream to be reset and stats to be + // incremented properly. + Http::HeaderMapPtr response_headers(new Http::TestHeaderMapImpl{{":status", "200"}}); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(200)); + EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); + + EXPECT_CALL(callbacks_, encodeHeaders_(_, _)) + .WillOnce(Invoke([&](Http::HeaderMap& headers, bool end_stream) -> void { + EXPECT_EQ(headers.Status()->value(), "200"); + EXPECT_TRUE(end_stream); + })); + response_decoder1->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 1)); + + // TODO: Verify hedge stats here once they are implemented. +} + // Three requests sent: 1) 5xx error, 2) per try timeout, 3) gets good response // headers. TEST_F(RouterTest, HedgedPerTryTimeoutThirdRequestSucceeds) { From 832d39c9e6623c06d95c214fdc78fa401e2d9809 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Thu, 11 Jul 2019 13:23:59 -0700 Subject: [PATCH 153/542] config dump, listeners are added to warming list if workers are not started yet (#7511) Signed-off-by: Jianfei Hu --- source/server/listener_manager_impl.cc | 19 ++++++++++++++----- test/integration/ads_integration_test.cc | 3 +++ test/server/listener_manager_impl_test.cc | 4 ++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 82f7b2261084..00978ccbb933 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -441,13 +441,22 @@ ProtobufTypes::MessagePtr ListenerManagerImpl::dumpListenerConfigs() { static_listener.mutable_listener()->MergeFrom(listener->config()); TimestampUtil::systemClockToTimestamp(listener->last_updated_, *(static_listener.mutable_last_updated())); + continue; + } + envoy::admin::v2alpha::ListenersConfigDump_DynamicListener* dump_listener; + // Listeners are always added to active_listeners_ list before workers are started. + // This applies even when the listeners are still waiting for initialization. + // To avoid confusion in config dump, in that case, we add these listeners to warming + // listeners config dump rather than active ones. + if (workers_started_) { + dump_listener = config_dump->mutable_dynamic_active_listeners()->Add(); } else { - auto& dynamic_listener = *config_dump->mutable_dynamic_active_listeners()->Add(); - dynamic_listener.set_version_info(listener->versionInfo()); - dynamic_listener.mutable_listener()->MergeFrom(listener->config()); - TimestampUtil::systemClockToTimestamp(listener->last_updated_, - *(dynamic_listener.mutable_last_updated())); + dump_listener = config_dump->mutable_dynamic_warming_listeners()->Add(); } + dump_listener->set_version_info(listener->versionInfo()); + dump_listener->mutable_listener()->MergeFrom(listener->config()); + TimestampUtil::systemClockToTimestamp(listener->last_updated_, + *(dump_listener->mutable_last_updated())); } for (const auto& listener : warming_listeners_) { diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 3ef9f4df589e..3745513ee431 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -969,6 +969,9 @@ TEST_P(AdsIntegrationTest, ListenerDrainBeforeServerStart) { Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, {buildListener("listener_0", "route_config_0")}, {}, "1"); test_server_->waitForGaugeGe("listener_manager.total_listeners_active", 1); + // Before server is started, even though listeners are added to active list + // we mark them as "warming" in config dump since they're not initialized yet. + EXPECT_EQ(getListenersConfigDump().dynamic_warming_listeners().size(), 1); // Remove listener. sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, {}, "1"); diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 44b5283aeafe..46c733415434 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -766,6 +766,7 @@ filter_chains: {} version_info: version1 static_listeners: dynamic_active_listeners: +dynamic_warming_listeners: version_info: "version1" listener: name: "foo" @@ -777,7 +778,6 @@ version_info: version1 last_updated: seconds: 1001001001 nanos: 1000000 -dynamic_warming_listeners: dynamic_draining_listeners: )EOF"); @@ -808,6 +808,7 @@ per_connection_buffer_limit_bytes: 10 version_info: version2 static_listeners: dynamic_active_listeners: +dynamic_warming_listeners: version_info: "version2" listener: name: "foo" @@ -820,7 +821,6 @@ version_info: version2 last_updated: seconds: 2002002002 nanos: 2000000 -dynamic_warming_listeners: dynamic_draining_listeners: )EOF"); From 37c5ce2a4df171f49148809b18afaa4adb813676 Mon Sep 17 00:00:00 2001 From: danzh Date: Thu, 11 Jul 2019 16:24:22 -0400 Subject: [PATCH 154/542] quiche: add EnvoyQuicPacketWriter with sendmsg to set source address (#7416) Signed-off-by: Dan Zhang --- bazel/external/quiche.BUILD | 3 + include/envoy/api/io_error.h | 6 + include/envoy/network/io_handle.h | 8 +- include/envoy/network/listener.h | 4 +- source/common/network/io_socket_error_impl.cc | 9 +- .../common/network/io_socket_handle_impl.cc | 131 +++++++++-------- source/common/network/io_socket_handle_impl.h | 3 +- .../common/network/socket_option_factory.cc | 4 +- source/common/network/socket_option_impl.h | 15 +- source/common/network/udp_listener_impl.cc | 19 ++- source/extensions/quic_listeners/quiche/BUILD | 18 +++ .../quiche/envoy_quic_packet_writer.cc | 51 +++++++ .../quiche/envoy_quic_packet_writer.h | 51 +++++++ .../quic_listeners/quiche/envoy_quic_utils.h | 28 ++++ .../quic_listeners/quiche/platform/BUILD | 1 + .../platform/quic_error_code_wrappers_impl.h | 6 +- test/common/network/BUILD | 1 + test/common/network/udp_listener_impl_test.cc | 97 +++++++++---- test/config/utility.cc | 8 +- test/extensions/quic_listeners/quiche/BUILD | 11 ++ .../quiche/envoy_quic_writer_test.cc | 134 ++++++++++++++++++ .../filters/udp_listener_echo_filter.cc | 6 +- test/integration/udp_echo_integration_test.cc | 114 ++++++++++----- tools/spelling_dictionary.txt | 2 + 24 files changed, 579 insertions(+), 151 deletions(-) create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_utils.h create mode 100644 test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index d7b7ee37c9bb..23276db96e7c 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -55,6 +55,7 @@ genrule( quiche_copt = [ # Remove these after upstream fix. "-Wno-unused-parameter", + "-Wno-type-limits", # quic_inlined_frame.h uses offsetof() to optimize memory usage in frames. "-Wno-invalid-offsetof", ] @@ -2194,6 +2195,7 @@ envoy_cc_library( copts = quiche_copt, repository = "@envoy", tags = ["nofips"], + visibility = ["//visibility:public"], deps = [ ":quic_core_packets_lib", ":quic_core_types_lib", @@ -2682,6 +2684,7 @@ envoy_cc_library( deps = [ ":quic_platform_base", ":quic_platform_socket_address", + ":spdy_core_priority_write_scheduler_lib", ], ) diff --git a/include/envoy/api/io_error.h b/include/envoy/api/io_error.h index dd80d36d58f5..70699e7443dc 100644 --- a/include/envoy/api/io_error.h +++ b/include/envoy/api/io_error.h @@ -25,6 +25,12 @@ class IoError { InProgress, // Permission denied. Permission, + // Message too big to send. + MessageTooBig, + // Kernel interrupt. + Interrupt, + // Requested a nonexistent interface or a non-local source address. + AddressNotAvailable, // Other error codes cannot be mapped to any one above in getErrorCode(). UnknownError }; diff --git a/include/envoy/network/io_handle.h b/include/envoy/network/io_handle.h index 8bc0634f2ddc..5062fae0c317 100644 --- a/include/envoy/network/io_handle.h +++ b/include/envoy/network/io_handle.h @@ -11,6 +11,7 @@ struct RawSlice; namespace Network { namespace Address { class Instance; +class Ip; } // namespace Address /** @@ -71,12 +72,15 @@ class IoHandle { * Send a message to the address. * @param slices points to the location of data to be sent. * @param num_slice indicates number of slices |slices| contains. - * @param address is the destination address. + * @param self_ip is the source address whose port should be ignored. Nullptr + * if caller wants kernel to select source address. + * @param peer_address is the destination address. * @return a Api::IoCallUint64Result with err_ = an Api::IoError instance or * err_ = nullptr and rc_ = the bytes written for success. */ virtual Api::IoCallUint64Result sendmsg(const Buffer::RawSlice* slices, uint64_t num_slice, - int flags, const Address::Instance& address) PURE; + int flags, const Address::Ip* self_ip, + const Address::Instance& peer_address) PURE; struct RecvMsgOutput { /* diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 47f1f8a48d85..8fd34c72795f 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -134,7 +134,9 @@ struct UdpRecvData { * Encapsulates the information needed to send a udp packet to a target */ struct UdpSendData { - Address::InstanceConstSharedPtr send_address_; + const Address::Ip* local_ip_; + const Address::Instance& peer_address_; + // The buffer is a reference so that it can be reused by the sender to send different // messages Buffer::Instance& buffer_; diff --git a/source/common/network/io_socket_error_impl.cc b/source/common/network/io_socket_error_impl.cc index 8f625e3c02a5..9f69361092b6 100644 --- a/source/common/network/io_socket_error_impl.cc +++ b/source/common/network/io_socket_error_impl.cc @@ -19,7 +19,14 @@ Api::IoError::IoErrorCode IoSocketError::getErrorCode() const { return IoErrorCode::InProgress; case EPERM: return IoErrorCode::Permission; + case EMSGSIZE: + return IoErrorCode::MessageTooBig; + case EINTR: + return IoErrorCode::Interrupt; + case EADDRNOTAVAIL: + return IoErrorCode::AddressNotAvailable; default: + ENVOY_LOG_MISC(error, "Unknown error code {} details {}", errno_, ::strerror(errno_)); return IoErrorCode::UnknownError; } } @@ -33,7 +40,7 @@ IoSocketError* IoSocketError::getIoSocketEagainInstance() { void IoSocketError::deleteIoError(Api::IoError* err) { ASSERT(err != nullptr); - if (err->getErrorCode() != Api::IoError::IoErrorCode::Again) { + if (err != getIoSocketEagainInstance()) { delete err; } } diff --git a/source/common/network/io_socket_handle_impl.cc b/source/common/network/io_socket_handle_impl.cc index e17698d2a3af..f2442973bd11 100644 --- a/source/common/network/io_socket_handle_impl.cc +++ b/source/common/network/io_socket_handle_impl.cc @@ -85,8 +85,9 @@ Api::IoCallUint64Result IoSocketHandleImpl::sendto(const Buffer::RawSlice& slice Api::IoCallUint64Result IoSocketHandleImpl::sendmsg(const Buffer::RawSlice* slices, uint64_t num_slice, int flags, - const Address::Instance& address) { - const auto* address_base = dynamic_cast(&address); + const Address::Ip* self_ip, + const Address::Instance& peer_address) { + const auto* address_base = dynamic_cast(&peer_address); sockaddr* sock_addr = const_cast(address_base->sockAddr()); STACK_ARRAY(iov, iovec, num_slice); @@ -107,14 +108,51 @@ Api::IoCallUint64Result IoSocketHandleImpl::sendmsg(const Buffer::RawSlice* slic message.msg_namelen = address_base->sockAddrLen(); message.msg_iov = iov.begin(); message.msg_iovlen = num_slices_to_write; - message.msg_control = nullptr; - message.msg_controllen = 0; message.msg_flags = 0; - auto& os_syscalls = Api::OsSysCallsSingleton::get(); - const Api::SysCallSizeResult result = os_syscalls.sendmsg(fd_, &message, flags); - - return sysCallResultToIoCallResult(result); + if (self_ip == nullptr) { + message.msg_control = nullptr; + message.msg_controllen = 0; + const Api::SysCallSizeResult result = os_syscalls.sendmsg(fd_, &message, flags); + return sysCallResultToIoCallResult(result); + } else { + const size_t space_v6 = CMSG_SPACE(sizeof(in6_pktinfo)); + // FreeBSD only needs in_addr size, but allocates more to unify code in two platforms. + const size_t space_v4 = CMSG_SPACE(sizeof(in_pktinfo)); + const size_t cmsg_space = (space_v4 < space_v6) ? space_v6 : space_v4; + // kSpaceForIp should be big enough to hold both IPv4 and IPv6 packet info. + STACK_ARRAY(cbuf, char, cmsg_space); + memset(cbuf.begin(), 0, cmsg_space); + + message.msg_control = cbuf.begin(); + message.msg_controllen = cmsg_space * sizeof(char); + cmsghdr* const cmsg = CMSG_FIRSTHDR(&message); + RELEASE_ASSERT(cmsg != nullptr, fmt::format("cbuf with size {} is not enough, cmsghdr size {}", + sizeof(cbuf), sizeof(cmsghdr))); + if (self_ip->version() == Address::IpVersion::v4) { + cmsg->cmsg_level = IPPROTO_IP; +#ifndef IP_SENDSRCADDR + cmsg->cmsg_len = CMSG_LEN(sizeof(in_pktinfo)); + cmsg->cmsg_type = IP_PKTINFO; + auto pktinfo = reinterpret_cast(CMSG_DATA(cmsg)); + pktinfo->ipi_ifindex = 0; + pktinfo->ipi_spec_dst.s_addr = self_ip->ipv4()->address(); +#else + cmsg->cmsg_type = IP_SENDSRCADDR; + cmsg->cmsg_len = CMSG_LEN(sizeof(in_addr)); + *(reinterpret_cast(CMSG_DATA(cmsg))).s_addr = self_ip->ipv4()->address(); +#endif + } else if (self_ip->version() == Address::IpVersion::v6) { + cmsg->cmsg_len = CMSG_LEN(sizeof(in6_pktinfo)); + cmsg->cmsg_level = IPPROTO_IPV6; + cmsg->cmsg_type = IPV6_PKTINFO; + auto pktinfo = reinterpret_cast(CMSG_DATA(cmsg)); + pktinfo->ipi6_ifindex = 0; + *(reinterpret_cast(pktinfo->ipi6_addr.s6_addr)) = self_ip->ipv6()->address(); + } + const Api::SysCallSizeResult result = os_syscalls.sendmsg(fd_, &message, flags); + return sysCallResultToIoCallResult(result); + } } Api::IoCallUint64Result @@ -124,6 +162,7 @@ IoSocketHandleImpl::sysCallResultToIoCallResult(const Api::SysCallSizeResult& re return Api::IoCallUint64Result(result.rc_, Api::IoErrorPtr(nullptr, IoSocketError::deleteIoError)); } + RELEASE_ASSERT(result.errno_ != EINVAL, "Invalid argument passed in."); return Api::IoCallUint64Result( /*rc=*/0, (result.errno_ == EAGAIN @@ -136,34 +175,34 @@ IoSocketHandleImpl::sysCallResultToIoCallResult(const Api::SysCallSizeResult& re Address::InstanceConstSharedPtr maybeGetDstAddressFromHeader(const struct cmsghdr& cmsg, uint32_t self_port) { if (cmsg.cmsg_type == IPV6_PKTINFO) { - const struct in6_pktinfo* info = reinterpret_cast(CMSG_DATA(&cmsg)); - sockaddr_in6 ipv6_addr; - memset(&ipv6_addr, 0, sizeof(sockaddr_in6)); - ipv6_addr.sin6_family = AF_INET6; - ipv6_addr.sin6_addr = info->ipi6_addr; - ipv6_addr.sin6_port = htons(self_port); - return Address::addressFromSockAddr(reinterpret_cast(ipv6_addr), - sizeof(sockaddr_in6), /*v6only=*/false); + auto info = reinterpret_cast(CMSG_DATA(&cmsg)); + sockaddr_storage ss; + auto ipv6_addr = reinterpret_cast(&ss); + memset(ipv6_addr, 0, sizeof(sockaddr_in6)); + ipv6_addr->sin6_family = AF_INET6; + ipv6_addr->sin6_addr = info->ipi6_addr; + ipv6_addr->sin6_port = htons(self_port); + return Address::addressFromSockAddr(ss, sizeof(sockaddr_in6), /*v6only=*/false); } #ifndef IP_RECVDSTADDR if (cmsg.cmsg_type == IP_PKTINFO) { - const struct in_pktinfo* info = reinterpret_cast(CMSG_DATA(&cmsg)); + auto info = reinterpret_cast(CMSG_DATA(&cmsg)); #else if (cmsg.cmsg_type == IP_RECVDSTADDR) { - const struct in_addr* addr = reinterpret_cast(CMSG_DATA(&cmsg)); + auto addr = reinterpret_cast(CMSG_DATA(&cmsg)); #endif - sockaddr_in ipv4_addr; - memset(&ipv4_addr, 0, sizeof(sockaddr_in)); - ipv4_addr.sin_family = AF_INET; - ipv4_addr.sin_addr = + sockaddr_storage ss; + auto ipv4_addr = reinterpret_cast(&ss); + memset(ipv4_addr, 0, sizeof(sockaddr_in)); + ipv4_addr->sin_family = AF_INET; + ipv4_addr->sin_addr = #ifndef IP_RECVDSTADDR info->ipi_addr; #else *addr; #endif - ipv4_addr.sin_port = htons(self_port); - return Address::addressFromSockAddr(reinterpret_cast(ipv4_addr), - sizeof(sockaddr_in), /*v6only=*/false); + ipv4_addr->sin_port = htons(self_port); + return Address::addressFromSockAddr(ss, sizeof(sockaddr_in), /*v6only=*/false); } return nullptr; } @@ -184,45 +223,13 @@ Api::IoCallUint64Result IoSocketHandleImpl::recvmsg(Buffer::RawSlice* slices, const uint64_t num_slice, uint32_t self_port, RecvMsgOutput& output) { -#ifndef __APPLE__ // The minimum cmsg buffer size to filled in destination address and packets dropped when // receiving a packet. It is possible for a received packet to contain both IPv4 and IPv6 // addresses. - constexpr int cmsg_space = CMSG_SPACE(sizeof(int)) + CMSG_SPACE(sizeof(struct in_pktinfo)) + - CMSG_SPACE(sizeof(struct in6_pktinfo)); - char cbuf[cmsg_space]; -#else - // CMSG_SPACE() is supposed to be a constant expression, and most - // BSD-ish code uses it to determine the size of buffers used to - // send/receive descriptors in the control message for - // sendmsg/recvmsg. Such buffers are often automatic variables. - - // In Darwin, CMSG_SPACE uses __DARWIN_ALIGN. And __DARWIN_ALIGN(p) - // expands to - - // ((__darwin_size_t)((char *)(__darwin_size_t)(p) + __DARNWIN_ALIGNBYTES) - // &~ __DARWIN_ALIGNBYTES) - - // which Clang (to which many Apple employees contribute!) complains - // about when invoked with -pedantic -- and with our -Werror, causes - // Clang-based builds to fail. Clang says this is not a constant - // expression: - - // error: variable length array folded to constant array as an - // extension [-Werror,-pedantic] - - // Possibly true per standard definition, since that cast to (char *) - // is ugly, but actually Clang was able to convert to a compile-time - // constant... it just had to work harder. - - // As a ugly workaround, we define the following constant for use in - // automatic array declarations, and in all cases have an assert to - // verify that the value is of sufficient size. (The assert should - // constantly propagate and be dead-code eliminated in normal compiles - // and should cause a quick death if ever violated.) - constexpr int cmsg_space = 128; - char cbuf[cmsg_space]; -#endif + const size_t cmsg_space = CMSG_SPACE(sizeof(int)) + CMSG_SPACE(sizeof(struct in_pktinfo)) + + CMSG_SPACE(sizeof(struct in6_pktinfo)); + STACK_ARRAY(cbuf, char, cmsg_space); + memset(cbuf.begin(), 0, cmsg_space); STACK_ARRAY(iov, iovec, num_slice); uint64_t num_slices_for_read = 0; @@ -242,7 +249,7 @@ Api::IoCallUint64Result IoSocketHandleImpl::recvmsg(Buffer::RawSlice* slices, hdr.msg_iovlen = num_slices_for_read; hdr.msg_flags = 0; - struct cmsghdr* cmsg = reinterpret_cast(cbuf); + auto cmsg = reinterpret_cast(cbuf.begin()); cmsg->cmsg_len = cmsg_space; hdr.msg_control = cmsg; hdr.msg_controllen = cmsg_space; diff --git a/source/common/network/io_socket_handle_impl.h b/source/common/network/io_socket_handle_impl.h index 580ce942f233..78e6211d35da 100644 --- a/source/common/network/io_socket_handle_impl.h +++ b/source/common/network/io_socket_handle_impl.h @@ -35,7 +35,8 @@ class IoSocketHandleImpl : public IoHandle, protected Logger::Loggable SocketOptionFactory::buildIpPacketInfoOptions() { std::unique_ptr options = std::make_unique(); options->push_back(std::make_shared( - envoy::api::v2::core::SocketOption::STATE_BOUND, ENVOY_RECV_IP_PKT_INFO, - ENVOY_RECV_IPV6_PKT_INFO, 1)); + envoy::api::v2::core::SocketOption::STATE_BOUND, ENVOY_SELF_IP_ADDR, ENVOY_SELF_IPV6_ADDR, + 1)); return options; } diff --git a/source/common/network/socket_option_impl.h b/source/common/network/socket_option_impl.h index b75a139cb957..59931506323c 100644 --- a/source/common/network/socket_option_impl.h +++ b/source/common/network/socket_option_impl.h @@ -77,18 +77,21 @@ namespace Network { // Linux uses IP_PKTINFO for both sending source address and receiving destination // address. // FreeBSD uses IP_RECVDSTADDR for receiving destination address and IP_SENDSRCADDR for sending -// source address. +// source address. And these two have same value for convenience purpose. #ifdef IP_RECVDSTADDR -#define ENVOY_RECV_IP_PKT_INFO ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_RECVDSTADDR) +#ifdef IP_SENDSRCADDR +static_assert(IP_RECVDSTADDR == IP_SENDSRCADDR); +#endif +#define ENVOY_IP_PKTINFO IP_RECVDSTADDR #elif IP_PKTINFO -#define ENVOY_RECV_IP_PKT_INFO ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, IP_PKTINFO) -#else -#define ENVOY_RECV_IP_PKT_INFO Network::SocketOptionName() +#define ENVOY_IP_PKTINFO IP_PKTINFO #endif +#define ENVOY_SELF_IP_ADDR ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IP, ENVOY_IP_PKTINFO) + // Both Linux and FreeBSD use IPV6_RECVPKTINFO for both sending source address and // receiving destination address. -#define ENVOY_RECV_IPV6_PKT_INFO ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_RECVPKTINFO) +#define ENVOY_SELF_IPV6_ADDR ENVOY_MAKE_SOCKET_OPTION_NAME(IPPROTO_IPV6, IPV6_RECVPKTINFO) class SocketOptionImpl : public Socket::Option, Logger::Loggable { public: diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index 475ed94b72df..290511b29843 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -16,6 +16,7 @@ #include "common/common/stack_array.h" #include "common/event/dispatcher_impl.h" #include "common/network/address_impl.h" +#include "common/network/io_socket_error_impl.h" #include "event2/listener.h" @@ -88,8 +89,7 @@ void UdpListenerImpl::handleReadCallback() { if (!result.ok()) { // No more to read or encountered a system error. if (result.err_->getErrorCode() != Api::IoError::IoErrorCode::Again) { - ENVOY_UDP_LOG(error, "recvfrom result {}: {}", - static_cast(result.err_->getErrorCode()), + ENVOY_UDP_LOG(error, "recvmsg result {}: {}", static_cast(result.err_->getErrorCode()), result.err_->getErrorDetails()); cb_.onReceiveError(UdpListenerCallbacks::ErrorCode::SyscallError, result.err_->getErrorCode()); @@ -159,15 +159,22 @@ Api::IoCallUint64Result UdpListenerImpl::send(const UdpSendData& send_data) { uint64_t num_slices = buffer.getRawSlices(nullptr, 0); STACK_ARRAY(slices, Buffer::RawSlice, num_slices); buffer.getRawSlices(slices.begin(), num_slices); - Api::IoCallUint64Result send_result = - socket_.ioHandle().sendmsg(slices.begin(), num_slices, 0, *send_data.send_address_); + Api::IoCallUint64Result send_result( + /*rc=*/0, /*err=*/Api::IoErrorPtr(nullptr, IoSocketError::deleteIoError)); + do { + send_result = socket_.ioHandle().sendmsg(slices.begin(), num_slices, 0, send_data.local_ip_, + send_data.peer_address_); + } while (!send_result.ok() && + // Send again if interrupted. + send_result.err_->getErrorCode() == Api::IoError::IoErrorCode::Interrupt); if (send_result.ok()) { ASSERT(send_result.rc_ == buffer.length()); ENVOY_UDP_LOG(trace, "sendmsg sent:{} bytes", send_result.rc_); } else { - ENVOY_UDP_LOG(debug, "sendmsg failed with error {}. Ret {}", - static_cast(send_result.err_->getErrorCode()), send_result.rc_); + ENVOY_UDP_LOG(debug, "sendmsg failed with error code {}: {}", + static_cast(send_result.err_->getErrorCode()), + send_result.err_->getErrorDetails()); } // The send_result normalizes the rc_ value to 0 in error conditions. diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 2e76136e0d8f..93b9f31ff664 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -34,6 +34,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "envoy_quic_packet_writer_lib", + srcs = ["envoy_quic_packet_writer.cc"], + hdrs = ["envoy_quic_packet_writer.h"], + external_deps = ["quiche_quic_platform"], + deps = [ + ":envoy_quic_utils_lib", + "@com_googlesource_quiche//:quic_core_packet_writer_interface_lib", + ], +) + envoy_cc_library( name = "envoy_quic_proof_source_lib", hdrs = ["envoy_quic_fake_proof_source.h"], @@ -65,3 +76,10 @@ envoy_cc_library( "@com_googlesource_quiche//:quic_core_http_spdy_server_push_utils_header", ], ) + +envoy_cc_library( + name = "envoy_quic_utils_lib", + hdrs = ["envoy_quic_utils.h"], + external_deps = ["quiche_quic_platform"], + deps = ["//source/common/network:address_lib"], +) diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc new file mode 100644 index 000000000000..b173fa00c7b0 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc @@ -0,0 +1,51 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" + +#pragma GCC diagnostic push + +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/core/quic_types.h" + +#pragma GCC diagnostic pop + +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "common/buffer/buffer_impl.h" + +namespace Envoy { +namespace Quic { + +quic::WriteResult EnvoyQuicPacketWriter::WritePacket(const char* buffer, size_t buf_len, + const quic::QuicIpAddress& self_ip, + const quic::QuicSocketAddress& peer_address, + quic::PerPacketOptions* options) { + ASSERT(options == nullptr, "Per packet option is not supported yet."); + ASSERT(!write_blocked_, "Cannot write while IO handle is blocked."); + + Buffer::BufferFragmentImpl fragment(buffer, buf_len, nullptr); + Buffer::OwnedImpl buffer_wrapper; + buffer_wrapper.addBufferFragment(fragment); + quic::QuicSocketAddress self_address(self_ip, /*port=*/0); + Network::Address::InstanceConstSharedPtr local_addr = + quicAddressToEnvoyAddressInstance(self_address); + Network::Address::InstanceConstSharedPtr remote_addr = + quicAddressToEnvoyAddressInstance(peer_address); + Network::UdpSendData send_data{local_addr == nullptr ? nullptr : local_addr->ip(), *remote_addr, + buffer_wrapper}; + Api::IoCallUint64Result result = listener_.send(send_data); + if (result.ok()) { + return {quic::WRITE_STATUS_OK, static_cast(result.rc_)}; + } + quic::WriteStatus status = result.err_->getErrorCode() == Api::IoError::IoErrorCode::Again + ? quic::WRITE_STATUS_BLOCKED + : quic::WRITE_STATUS_ERROR; + if (quic::IsWriteBlockedStatus(status)) { + write_blocked_ = true; + } + return {status, static_cast(result.err_->getErrorCode())}; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h new file mode 100644 index 000000000000..d82ac08ca7a7 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h @@ -0,0 +1,51 @@ +#pragma once + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/core/quic_packet_writer.h" + +#pragma GCC diagnostic pop + +#include "envoy/network/listener.h" + +namespace Envoy { +namespace Quic { + +class EnvoyQuicPacketWriter : public quic::QuicPacketWriter { +public: + EnvoyQuicPacketWriter(Network::UdpListener& listener) + : write_blocked_(false), listener_(listener) {} + + quic::WriteResult WritePacket(const char* buffer, size_t buf_len, + const quic::QuicIpAddress& self_address, + const quic::QuicSocketAddress& peer_address, + quic::PerPacketOptions* options) override; + + // quic::QuicPacketWriter + bool IsWriteBlocked() const override { return write_blocked_; } + void SetWritable() override { write_blocked_ = false; } + quic::QuicByteCount + GetMaxPacketSize(const quic::QuicSocketAddress& /*peer_address*/) const override { + return quic::kMaxOutgoingPacketSize; + } + // Currently this writer doesn't support pacing offload or batch writing. + bool SupportsReleaseTime() const override { return false; } + bool IsBatchMode() const override { return false; } + char* GetNextWriteLocation(const quic::QuicIpAddress& /*self_address*/, + const quic::QuicSocketAddress& /*peer_address*/) override { + return nullptr; + } + quic::WriteResult Flush() override { return {quic::WRITE_STATUS_OK, 0}; } + +private: + // Modified by WritePacket() to indicate underlying IoHandle status. + bool write_blocked_; + Network::UdpListener& listener_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_utils.h b/source/extensions/quic_listeners/quiche/envoy_quic_utils.h new file mode 100644 index 000000000000..2d411aa8dc98 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_utils.h @@ -0,0 +1,28 @@ +#include + +#include "common/common/assert.h" +#include "common/network/address_impl.h" + +#include "quiche/quic/platform/api/quic_ip_address.h" +#include "quiche/quic/platform/api/quic_socket_address.h" + +namespace Envoy { +namespace Quic { + +// TODO(danzh): this is called on each write. Consider to return an address instance on the stack if +// the heap allocation is too expensive. +inline Network::Address::InstanceConstSharedPtr +quicAddressToEnvoyAddressInstance(const quic::QuicSocketAddress& quic_address) { + ASSERT(quic_address.host().address_family() != quic::IpAddressFamily::IP_UNSPEC); + return quic_address.IsInitialized() + ? Network::Address::addressFromSockAddr(quic_address.generic_address(), + quic_address.host().address_family() == + quic::IpAddressFamily::IP_V4 + ? sizeof(sockaddr_in) + : sizeof(sockaddr_in6), + false) + : nullptr; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD index 15880743587e..e30ad336b9e8 100644 --- a/source/extensions/quic_listeners/quiche/platform/BUILD +++ b/source/extensions/quic_listeners/quiche/platform/BUILD @@ -157,6 +157,7 @@ envoy_cc_library( deps = [ ":flags_impl_lib", ":string_utils_lib", + "//include/envoy/api:io_error_interface", "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:byte_order_lib", diff --git a/source/extensions/quic_listeners/quiche/platform/quic_error_code_wrappers_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_error_code_wrappers_impl.h index 8d6901c04700..a5935812538a 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_error_code_wrappers_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_error_code_wrappers_impl.h @@ -6,6 +6,8 @@ // consumed or referenced directly by other Envoy code. It serves purely as a // porting layer for QUICHE. -#include +#include -#define QUIC_EMSGSIZE_IMPL EMSGSIZE +#include "envoy/api/io_error.h" + +#define QUIC_EMSGSIZE_IMPL static_cast(Envoy::Api::IoError::IoErrorCode::MessageTooBig) diff --git a/test/common/network/BUILD b/test/common/network/BUILD index 0b62e822f71f..4beccfa5da83 100644 --- a/test/common/network/BUILD +++ b/test/common/network/BUILD @@ -185,6 +185,7 @@ envoy_cc_test( "//source/common/event:dispatcher_lib", "//source/common/network:address_lib", "//source/common/network:listener_lib", + "//source/common/network:socket_option_lib", "//source/common/network:utility_lib", "//source/common/stats:stats_lib", "//test/common/network:listener_impl_test_base_lib", diff --git a/test/common/network/udp_listener_impl_test.cc b/test/common/network/udp_listener_impl_test.cc index d8f9f33b8a2b..4296414a5a86 100644 --- a/test/common/network/udp_listener_impl_test.cc +++ b/test/common/network/udp_listener_impl_test.cc @@ -4,6 +4,7 @@ #include "common/network/address_impl.h" #include "common/network/socket_option_factory.h" +#include "common/network/socket_option_impl.h" #include "common/network/udp_listener_impl.h" #include "common/network/utility.h" @@ -54,8 +55,15 @@ class UdpListenerImplTest : public ListenerImplTestBase { } SocketPtr createServerSocket(bool bind) { + // Set IP_FREEBIND to allow sendmsg to send with nonlocal IPv6 source address. return std::make_unique>>( - Network::Test::getAnyAddress(version_), nullptr, bind); + Network::Test::getAnyAddress(version_), +#ifdef IP_FREEBIND + SocketOptionFactory::buildIpFreebindOptions(), +#else + nullptr, +#endif + bind); } SocketPtr createClientSocket(bool bind) { @@ -342,7 +350,8 @@ TEST_P(UdpListenerImplTest, UdpListenerRecvMsgError) { /** * Tests UDP listener for sending datagrams to destination. * 1. Setup a udp listener and client socket - * 2. Send the data from the udp listener to the client socket and validate the contents + * 2. Send the data from the udp listener to the client socket and validate the contents and source + * address. */ TEST_P(UdpListenerImplTest, SendData) { // Setup client socket. @@ -352,68 +361,98 @@ TEST_P(UdpListenerImplTest, SendData) { const std::string payload("hello world"); Buffer::InstancePtr buffer(new Buffer::OwnedImpl()); buffer->add(payload); - UdpSendData send_data{client_socket_->localAddress(), *buffer}; - - auto send_result = listener_->send(send_data); + // Use a self address that is unlikely to be picked by source address discovery + // algorithm if not specified in recvmsg. Port is not taken into + // consideration. + Address::InstanceConstSharedPtr send_from_addr; + if (version_ == Address::IpVersion::v4) { + // Linux kernel regards any 127.x.x.x as local address. But Mac OS doesn't. + send_from_addr.reset(new Address::Ipv4Instance( +#ifndef __APPLE__ + "127.1.2.3", +#else + "127.0.0.1", +#endif + server_socket_->localAddress()->ip()->port())); + } else { + // Only use non-local v6 address if IP_FREEBIND is supported. Otherwise use + // ::1 to avoid EINVAL error. Unfortunately this can't verify that sendmsg with + // customized source address is doing the work because kernel also picks ::1 + // if it's not specified in cmsghdr. + send_from_addr.reset(new Address::Ipv6Instance( +#ifdef IP_FREEBIND + "::9", +#else + "::1", +#endif + server_socket_->localAddress()->ip()->port())); + } - EXPECT_EQ(send_result.ok(), true); + UdpSendData send_data{send_from_addr->ip(), *client_socket_->localAddress(), *buffer}; - // This is trigerred on opening the listener on registering the event - EXPECT_CALL(listener_callbacks_, onWriteReady_(_)).WillOnce(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); - })); + auto send_result = listener_->send(send_data); - dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + EXPECT_TRUE(send_result.ok()) << "send() failed : " << send_result.err_->getErrorDetails(); - Buffer::InstancePtr result_buffer(new Buffer::OwnedImpl()); - const uint64_t bytes_to_read = 11; + const uint64_t bytes_to_read = payload.length(); + // Make receive buffer 1 byte larger for trailing '\0'. + auto recv_buf = std::make_unique(bytes_to_read + 1); uint64_t bytes_read = 0; int retry = 0; + auto& os_sys_calls = Api::OsSysCallsSingleton::get(); + sockaddr_storage peer_addr; + socklen_t addr_len = sizeof(sockaddr_storage); do { - Api::IoCallUint64Result result = - result_buffer->read(client_socket_->ioHandle(), bytes_to_read - bytes_read); - - if (result.ok()) { - bytes_read += result.rc_; - } else if (retry == 10 || result.err_->getErrorCode() != Api::IoError::IoErrorCode::Again) { + Api::SysCallSizeResult result = + os_sys_calls.recvfrom(client_socket_->ioHandle().fd(), recv_buf.get(), bytes_to_read, 0, + reinterpret_cast(&peer_addr), &addr_len); + if (result.rc_ >= 0) { + bytes_read = result.rc_; + Address::InstanceConstSharedPtr peer_address = + Address::addressFromSockAddr(peer_addr, addr_len, false); + EXPECT_EQ(send_from_addr->asString(), peer_address->asString()); + } else if (retry == 10 || result.errno_ != EAGAIN) { break; } - if (bytes_read == bytes_to_read) { + if (bytes_read >= bytes_to_read) { break; } retry++; ::usleep(10000); + ASSERT(bytes_read == 0); } while (true); - - EXPECT_EQ(result_buffer->toString(), payload); + EXPECT_EQ(bytes_to_read, bytes_read); + recv_buf[bytes_to_read] = '\0'; + EXPECT_EQ(recv_buf.get(), payload); } /** * The send fails because the server_socket is created with bind=false. */ TEST_P(UdpListenerImplTest, SendDataError) { + Logger::StderrSinkDelegate stderr_sink(Logger::Registry::getSink()); // For coverage build. const std::string payload("hello world"); Buffer::InstancePtr buffer(new Buffer::OwnedImpl()); buffer->add(payload); // send data to itself - UdpSendData send_data{server_socket_->localAddress(), *buffer}; + UdpSendData send_data{send_to_addr_->ip(), *server_socket_->localAddress(), *buffer}; - // This is trigerred on opening the listener on registering the event - EXPECT_CALL(listener_callbacks_, onWriteReady_(_)).WillOnce(Invoke([&](const Socket& socket) { - EXPECT_EQ(socket.ioHandle().fd(), server_socket_->ioHandle().fd()); - })); // Inject mocked OsSysCalls implementation to mock a write failure. Api::MockOsSysCalls os_sys_calls; TestThreadsafeSingletonInjector os_calls(&os_sys_calls); EXPECT_CALL(os_sys_calls, sendmsg(_, _, _)).WillOnce(Return(Api::SysCallSizeResult{-1, ENOTSUP})); auto send_result = listener_->send(send_data); - dispatcher_->run(Event::Dispatcher::RunType::NonBlock); EXPECT_FALSE(send_result.ok()); EXPECT_EQ(send_result.err_->getErrorCode(), Api::IoError::IoErrorCode::NoSupport); - dispatcher_->exit(); + // Failed write shouldn't drain the data. + EXPECT_EQ(payload.length(), buffer->length()); + + ON_CALL(os_sys_calls, sendmsg(_, _, _)).WillByDefault(Return(Api::SysCallSizeResult{-1, EINVAL})); + // EINVAL should cause RELEASE_ASSERT. + EXPECT_DEATH(listener_->send(send_data), "Invalid argument passed in"); } } // namespace diff --git a/test/config/utility.cc b/test/config/utility.cc index 8a90d921b1ec..b49638a6ca01 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -63,7 +63,7 @@ const std::string ConfigHelper::BASE_UDP_LISTENER_CONFIG = R"EOF( name: listener_0 address: socket_address: - address: 127.0.0.1 + address: 0.0.0.0 port_value: 0 protocol: udp )EOF"; @@ -240,7 +240,11 @@ ConfigHelper::ConfigHelper(const Network::Address::IpVersion version, Api::Api& for (int i = 0; i < static_resources->listeners_size(); ++i) { auto* listener = static_resources->mutable_listeners(i); auto* listener_socket_addr = listener->mutable_address()->mutable_socket_address(); - listener_socket_addr->set_address(Network::Test::getLoopbackAddressString(version)); + if (listener_socket_addr->address() == "0.0.0.0" || listener_socket_addr->address() == "::") { + listener_socket_addr->set_address(Network::Test::getAnyAddressString(version)); + } else { + listener_socket_addr->set_address(Network::Test::getLoopbackAddressString(version)); + } } for (int i = 0; i < static_resources->clusters_size(); ++i) { diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index 2e3bd787c694..a51b6ea79b34 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -25,6 +25,17 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "envoy_quic_writer_test", + srcs = ["envoy_quic_writer_test.cc"], + external_deps = ["quiche_quic_platform"], + deps = [ + "//source/common/network:io_socket_error_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_packet_writer_lib", + "//test/mocks/network:network_mocks", + ], +) + envoy_cc_test( name = "envoy_quic_proof_source_test", srcs = ["envoy_quic_proof_source_test.cc"], diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc new file mode 100644 index 000000000000..f8ef123bd736 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc @@ -0,0 +1,134 @@ +#include +#include + +#include "common/network/io_socket_error_impl.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" + +#include "test/mocks/network/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Quic { + +class EnvoyQuicWriterTest : public ::testing::Test { +public: + EnvoyQuicWriterTest() : envoy_quic_writer_(udp_listener_) { + self_address_.FromString("0.0.0.0"); + quic::QuicIpAddress peer_ip; + peer_ip.FromString("127.0.0.1"); + peer_address_ = quic::QuicSocketAddress(peer_ip, /*port=*/123); + EXPECT_CALL(udp_listener_, onDestroy()); + ON_CALL(udp_listener_, send(_)) + .WillByDefault(testing::Invoke([](const Network::UdpSendData& send_data) { + return Api::IoCallUint64Result( + send_data.buffer_.length(), + Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)); + })); + } + + void verifySendData(const std::string& content, const Network::UdpSendData send_data) { + EXPECT_EQ(peer_address_.ToString(), send_data.peer_address_.asString()); + EXPECT_EQ(self_address_.ToString(), send_data.local_ip_->addressAsString()); + EXPECT_EQ(content, send_data.buffer_.toString()); + } + +protected: + testing::NiceMock udp_listener_; + quic::QuicIpAddress self_address_; + quic::QuicSocketAddress peer_address_; + EnvoyQuicPacketWriter envoy_quic_writer_; +}; + +TEST_F(EnvoyQuicWriterTest, AssertOnNonNullPacketOption) { + std::string str("Hello World!"); + EXPECT_DEBUG_DEATH(envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, + peer_address_, + reinterpret_cast(0x1)), + "Per packet option is not supported yet."); +} + +TEST_F(EnvoyQuicWriterTest, SendSuccessfully) { + std::string str("Hello World!"); + EXPECT_CALL(udp_listener_, send(_)) + .WillOnce(testing::Invoke([this, str](const Network::UdpSendData& send_data) { + verifySendData(str, send_data); + return Api::IoCallUint64Result( + str.length(), Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)); + })); + quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, + peer_address_, nullptr); + EXPECT_EQ(quic::WRITE_STATUS_OK, result.status); + EXPECT_EQ(str.length(), result.bytes_written); + EXPECT_FALSE(envoy_quic_writer_.IsWriteBlocked()); +} + +TEST_F(EnvoyQuicWriterTest, SendBlocked) { + std::string str("Hello World!"); + EXPECT_CALL(udp_listener_, send(_)) + .WillOnce(testing::Invoke([this, str](const Network::UdpSendData& send_data) { + verifySendData(str, send_data); + return Api::IoCallUint64Result( + 0u, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), + Network::IoSocketError::deleteIoError)); + })); + quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, + peer_address_, nullptr); + EXPECT_EQ(quic::WRITE_STATUS_BLOCKED, result.status); + EXPECT_EQ(static_cast(Api::IoError::IoErrorCode::Again), result.error_code); + EXPECT_TRUE(envoy_quic_writer_.IsWriteBlocked()); + // Writing while blocked is not allowed. +#ifdef NDEBUG + EXPECT_CALL(udp_listener_, send(_)) + .WillOnce(testing::Invoke([this, str](const Network::UdpSendData& send_data) { + verifySendData(str, send_data); + return Api::IoCallUint64Result( + 0u, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), + Network::IoSocketError::deleteIoError)); + })); +#endif + EXPECT_DEBUG_DEATH(envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, + peer_address_, nullptr), + "Cannot write while IO handle is blocked."); + envoy_quic_writer_.SetWritable(); + EXPECT_FALSE(envoy_quic_writer_.IsWriteBlocked()); +} + +TEST_F(EnvoyQuicWriterTest, SendFailure) { + std::string str("Hello World!"); + EXPECT_CALL(udp_listener_, send(_)) + .WillOnce(testing::Invoke( + [this, str](const Network::UdpSendData& send_data) -> Api::IoCallUint64Result { + verifySendData(str, send_data); + return Api::IoCallUint64Result(0u, + Api::IoErrorPtr(new Network::IoSocketError(ENOTSUP), + Network::IoSocketError::deleteIoError)); + })); + quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, + peer_address_, nullptr); + EXPECT_EQ(quic::WRITE_STATUS_ERROR, result.status); + EXPECT_EQ(static_cast(Api::IoError::IoErrorCode::NoSupport), result.error_code); + EXPECT_FALSE(envoy_quic_writer_.IsWriteBlocked()); +} + +TEST_F(EnvoyQuicWriterTest, SendFailureMessageTooBig) { + std::string str("Hello World!"); + EXPECT_CALL(udp_listener_, send(_)) + .WillOnce(testing::Invoke([this, str](const Network::UdpSendData& send_data) { + verifySendData(str, send_data); + return Api::IoCallUint64Result(0u, Api::IoErrorPtr(new Network::IoSocketError(EMSGSIZE), + Network::IoSocketError::deleteIoError)); + })); + quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, + peer_address_, nullptr); + // Currently MessageSize should be propagated through error_code. This test + // would fail if QUICHE changes to propagate through status in the future. + EXPECT_EQ(quic::WRITE_STATUS_ERROR, result.status); + EXPECT_EQ(static_cast(Api::IoError::IoErrorCode::MessageTooBig), result.error_code); + EXPECT_FALSE(envoy_quic_writer_.IsWriteBlocked()); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/integration/filters/udp_listener_echo_filter.cc b/test/integration/filters/udp_listener_echo_filter.cc index 74f11a81a2a2..d8a46c3f6117 100644 --- a/test/integration/filters/udp_listener_echo_filter.cc +++ b/test/integration/filters/udp_listener_echo_filter.cc @@ -25,11 +25,11 @@ UdpListenerEchoFilter::UdpListenerEchoFilter(Network::UdpReadFilterCallbacks& ca : Network::UdpListenerReadFilter(callbacks) {} void UdpListenerEchoFilter::onData(Network::UdpRecvData& data) { - ENVOY_LOG(trace, "UdpEchoFilter: Got {} bytes from {}", data.buffer_->length(), - data.peer_address_->asString()); + ENVOY_LOG(trace, "UdpEchoFilter: Got {} bytes from {} on {}", data.buffer_->length(), + data.peer_address_->asString(), data.local_address_->asString()); // send back the received data - Network::UdpSendData send_data{data.peer_address_, *data.buffer_}; + Network::UdpSendData send_data{data.local_address_->ip(), *data.peer_address_, *data.buffer_}; auto send_result = read_callbacks_->udpListener().send(send_data); ASSERT(send_result.ok()); diff --git a/test/integration/udp_echo_integration_test.cc b/test/integration/udp_echo_integration_test.cc index e976c9163571..d579e238d8c2 100644 --- a/test/integration/udp_echo_integration_test.cc +++ b/test/integration/udp_echo_integration_test.cc @@ -1,3 +1,4 @@ +#include "common/network/address_impl.h" #include "common/network/listen_socket_impl.h" #include "test/integration/integration.h" @@ -12,7 +13,7 @@ class UdpEchoIntegrationTest : public testing::TestWithParam; + + // Setup client socket. + Network::SocketPtr client_socket = + std::make_unique>( + Network::Test::getCanonicalLoopbackAddress(version_), nullptr, true); + ASSERT_NE(client_socket, nullptr); + + const std::string request("hello world"); + const void* void_pointer = static_cast(request.c_str()); + Buffer::RawSlice slice{const_cast(void_pointer), request.length()}; + auto send_rc = client_socket->ioHandle().sendto(slice, 0, *listener_address); + ASSERT_EQ(send_rc.rc_, request.length()); + + auto& os_sys_calls = Api::OsSysCallsSingleton::get(); + sockaddr_storage peer_addr; + socklen_t addr_len = sizeof(sockaddr_storage); + + const uint64_t bytes_to_read = request.length(); + auto recv_buf = std::make_unique(bytes_to_read + 1); + uint64_t bytes_read = 0; + + int retry = 0; + do { + Api::SysCallSizeResult result = + os_sys_calls.recvfrom(client_socket->ioHandle().fd(), recv_buf.get(), bytes_to_read, 0, + reinterpret_cast(&peer_addr), &addr_len); + if (result.rc_ >= 0) { + bytes_read = result.rc_; + Network::Address::InstanceConstSharedPtr peer_address = + Network::Address::addressFromSockAddr(peer_addr, addr_len, false); + // Expect to receive from the same peer address as it sent to. + EXPECT_EQ(listener_address->asString(), peer_address->asString()); + } else if (retry == 10 || result.errno_ != EAGAIN) { + break; + } + + if (bytes_read >= bytes_to_read) { + break; + } + ASSERT(bytes_read == 0); + // Retry after 10ms + timeSystem().sleep(std::chrono::milliseconds(10)); + retry++; + } while (true); + + recv_buf[bytes_to_read] = '\0'; + EXPECT_EQ(recv_buf.get(), request); + } }; INSTANTIATE_TEST_SUITE_P(IpVersions, UdpEchoIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); -TEST_P(UdpEchoIntegrationTest, HelloWorld) { +TEST_P(UdpEchoIntegrationTest, HelloWorldOnLoopback) { uint32_t port = lookupPort("listener_0"); auto listener_address = Network::Utility::resolveUrl( fmt::format("tcp://{}:{}", Network::Test::getLoopbackAddressUrlString(version_), port)); + requestResponseWithListenerAddress(listener_address); +} + +TEST_P(UdpEchoIntegrationTest, HelloWorldOnNonLocalAddress) { + uint32_t port = lookupPort("listener_0"); + Network::Address::InstanceConstSharedPtr listener_address; + if (version_ == Network::Address::IpVersion::v4) { + // Kernel regards any 127.x.x.x as local address. + listener_address.reset(new Network::Address::Ipv4Instance( +#ifndef __APPLE__ + "127.0.0.3", +#else + "127.0.0.1", +#endif + port)); + } else { + // IPv6 doesn't allow any nonlocal source address for sendmsg. And the only + // local address guaranteed in tests in loopback. Unfortunately, even if it's not + // specified, kernel will pick this address as source address. So this test + // only checks if IoSocketHandle::sendmsg() sets up CMSG_DATA correctly, + // i.e. cmsg_len is big enough when that code path is executed. + listener_address.reset(new Network::Address::Ipv6Instance("::1", port)); + } - using NetworkSocketTraitType = - Network::NetworkSocketTrait; - - // Setup client socket. - Network::SocketPtr client_socket = - std::make_unique>( - Network::Test::getCanonicalLoopbackAddress(version_), nullptr, true); - ASSERT_NE(client_socket, nullptr); - - const std::string request("hello world"); - const void* void_pointer = static_cast(request.c_str()); - Buffer::RawSlice slice{const_cast(void_pointer), request.length()}; - auto send_rc = client_socket->ioHandle().sendto(slice, 0, *listener_address); - ASSERT_EQ(send_rc.rc_, request.length()); - - Buffer::InstancePtr response_buffer(new Buffer::OwnedImpl()); - int retry = 0; - do { - Api::IoCallUint64Result result = - response_buffer->read(client_socket->ioHandle(), request.size()); - - if (result.ok() || retry == 10 || - result.err_->getErrorCode() != Api::IoError::IoErrorCode::Again) { - break; - } - - // Retry after 10ms - timeSystem().sleep(std::chrono::milliseconds(10)); - retry++; - } while (true); - - EXPECT_EQ(response_buffer->toString(), request); + requestResponseWithListenerAddress(listener_address); } } // namespace Envoy diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index d34f50ef8373..1218eefbb186 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -70,6 +70,7 @@ ECONNREFUSED EDESTRUCTION EDF EDS +EINVAL EIP ELB ENOTSUP @@ -609,6 +610,7 @@ optimizations ostream outlier outliers +offsetof overridable param parameterizable From c41bfbdc2ff174d1c35cc0b7c307a8bd5b7cb743 Mon Sep 17 00:00:00 2001 From: soya3129 <43042079+soya3129@users.noreply.github.com> Date: Thu, 11 Jul 2019 16:26:08 -0400 Subject: [PATCH 155/542] Common: Consume, proxy and insert metadata from downstream to upstream (#5656) Signed-off-by: Yang Song --- include/envoy/http/filter.h | 25 ++ source/common/http/async_client_impl.h | 1 + source/common/http/conn_manager_impl.cc | 97 +++++- source/common/http/conn_manager_impl.h | 71 ++++- source/common/router/router.cc | 43 ++- source/common/router/router.h | 4 + source/docs/h2_metadata.md | 56 +++- .../filters/http/common/pass_through_filter.h | 1 - .../filters/http/csrf/csrf_filter.h | 6 +- .../filters/http/buffer/buffer_filter_test.cc | 5 + .../filters/http/cors/cors_filter_test.cc | 2 + .../filters/http/csrf/csrf_filter_test.cc | 2 + .../filters/http/dynamo/dynamo_filter_test.cc | 6 +- .../filters/http/ext_authz/ext_authz_test.cc | 2 + .../filters/http/fault/fault_filter_test.cc | 2 + .../http1_bridge_filter_test.cc | 2 + .../json_transcoder_filter_test.cc | 2 + .../http/grpc_web/grpc_web_filter_test.cc | 2 + .../filters/http/gzip/gzip_filter_test.cc | 2 + .../header_to_metadata_filter_test.cc | 2 + .../http/health_check/health_check_test.cc | 2 + .../filters/http/jwt_authn/filter_test.cc | 2 + .../filters/http/lua/lua_filter_test.cc | 2 + .../filters/http/ratelimit/ratelimit_test.cc | 2 + .../filters/http/rbac/rbac_filter_test.cc | 2 + .../filters/http/squash/squash_filter_test.cc | 2 + test/integration/BUILD | 4 +- test/integration/fake_upstream.cc | 8 + test/integration/fake_upstream.h | 11 +- test/integration/filters/BUILD | 30 ++ .../filters/metadata_stop_all_filter.cc | 74 +++++ .../filters/request_metadata_filter.cc | 65 ++++ test/integration/http2_integration_test.cc | 291 +++++++++++++++++- test/integration/http2_integration_test.h | 14 + test/integration/http_integration.cc | 16 +- test/integration/http_integration.h | 6 +- test/mocks/http/mocks.h | 3 + test/server/http/admin_test.cc | 2 + 38 files changed, 831 insertions(+), 38 deletions(-) create mode 100644 test/integration/filters/metadata_stop_all_filter.cc create mode 100644 test/integration/filters/request_metadata_filter.cc diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index 1e1bacaab3fc..e34aeec39a94 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -302,6 +302,15 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { const absl::optional grpc_status, absl::string_view details) PURE; + /** + * Adds decoded metadata. This function can only be called in + * StreamDecoderFilter::decodeHeaders/Data/Trailers(). Do not call in + * StreamDecoderFilter::decodeMetadata(). + * + * @return a reference to metadata map vector, where new metadata map can be added. + */ + virtual MetadataMapVector& addDecodedMetadata() PURE; + /** * Called with 100-Continue headers to be encoded. * @@ -468,6 +477,22 @@ class StreamDecoderFilter : public StreamFilterBase { */ virtual FilterTrailersStatus decodeTrailers(HeaderMap& trailers) PURE; + /** + * Called with decoded metadata. Add new metadata to metadata_map directly. Do not call + * StreamDecoderFilterCallbacks::addDecodedMetadata() to add new metadata. + * + * Note: decodeMetadata() currently cannot stop the filter iteration, and always returns Continue. + * That means metadata will go through the complete filter chain at once, even if the other frame + * types return StopIteration. If metadata should not pass through all filters at once, users + * should consider using StopAllIterationAndBuffer or StopAllIterationAndWatermark in + * decodeHeaders() to prevent metadata passing to the following filters. + * + * @param metadata supplies the decoded metadata. + */ + virtual FilterMetadataStatus decodeMetadata(MetadataMap& /* metadata_map */) { + return Http::FilterMetadataStatus::Continue; + } + /** * Called by the filter manager once to initialize the filter decoder callbacks that the * filter should use. Callbacks will not be invoked by the filter after onDestroy() is called. diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index 9830b98627cc..e0a079dddf73 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -302,6 +302,7 @@ class AsyncStreamImpl : public AsyncClient::Stream, // filter which uses this function for buffering. ASSERT(buffered_body_ != nullptr); } + MetadataMapVector& addDecodedMetadata() override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void injectDecodedDataToFilterChain(Buffer::Instance&, bool) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 8e905ac5aba0..cf3e4f90ded6 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -856,6 +856,21 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(ActiveStreamDecoderFilte ENVOY_STREAM_LOG(trace, "decode headers called: filter={} status={}", *this, static_cast((*entry).get()), static_cast(status)); + const bool new_metadata_added = processNewlyAddedMetadata(); + + // If end_stream is set in headers, and a filter adds new metadata, we need to delay end_stream + // in headers by inserting an empty data frame with end_stream set. The empty data frame is sent + // after the new metadata. + if ((*entry)->end_stream_ && new_metadata_added && !buffered_request_data_) { + Buffer::OwnedImpl empty_data(""); + ENVOY_STREAM_LOG( + trace, "inserting an empty data frame for end_stream due metadata being added.", *this); + // Metadata frame doesn't carry end of stream bit. We need an empty data frame to end the + // stream. + addDecodedData(*((*entry).get()), empty_data, true); + } + + (*entry)->decode_headers_called_ = true; if (!(*entry)->commonHandleAfterHeadersCallback(status, decoding_headers_only_) && std::next(entry) != decoder_filters_.end()) { // Stop iteration IFF this is not the last filter. If it is the last filter, continue with @@ -981,6 +996,8 @@ void ConnectionManagerImpl::ActiveStream::decodeData( ENVOY_STREAM_LOG(trace, "decode data called: filter={} status={}", *this, static_cast((*entry).get()), static_cast(status)); + processNewlyAddedMetadata(); + if (!trailers_exists_at_start && request_trailers_ && trailers_added_entry == decoder_filters_.end()) { trailers_added_entry = entry; @@ -1038,6 +1055,10 @@ void ConnectionManagerImpl::ActiveStream::addDecodedData(ActiveStreamDecoderFilt } } +MetadataMapVector& ConnectionManagerImpl::ActiveStream::addDecodedMetadata() { + return *getRequestMetadataMapVector(); +} + void ConnectionManagerImpl::ActiveStream::decodeTrailers(HeaderMapPtr&& trailers) { ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); @@ -1077,6 +1098,9 @@ void ConnectionManagerImpl::ActiveStream::decodeTrailers(ActiveStreamDecoderFilt state_.filter_call_state_ &= ~FilterCallState::DecodeTrailers; ENVOY_STREAM_LOG(trace, "decode trailers called: filter={} status={}", *this, static_cast((*entry).get()), static_cast(status)); + + processNewlyAddedMetadata(); + if (!(*entry)->commonHandleAfterTrailersCallback(status)) { return; } @@ -1084,6 +1108,38 @@ void ConnectionManagerImpl::ActiveStream::decodeTrailers(ActiveStreamDecoderFilt disarmRequestTimeout(); } +void ConnectionManagerImpl::ActiveStream::decodeMetadata(MetadataMapPtr&& metadata_map) { + resetIdleTimer(); + // After going through filters, the ownership of metadata_map will be passed to terminal filter. + // The terminal filter may encode metadata_map to the next hop immediately or store metadata_map + // and encode later when connection pool is ready. + decodeMetadata(nullptr, *metadata_map); +} + +void ConnectionManagerImpl::ActiveStream::decodeMetadata(ActiveStreamDecoderFilter* filter, + MetadataMap& metadata_map) { + // Filter iteration may start at the current filter. + std::list::iterator entry = + commonDecodePrefix(filter, FilterIterationStartState::CanStartFromCurrent); + + for (; entry != decoder_filters_.end(); entry++) { + // If the filter pointed by entry has stopped for all frame type, stores metadata and returns. + // If the filter pointed by entry hasn't returned from decodeHeaders, stores newly added + // metadata in case decodeHeaders returns StopAllIteration. The latter can happen when headers + // callbacks generate new metadata. + if (!(*entry)->decode_headers_called_ || (*entry)->stoppedAll()) { + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + (*entry)->getSavedRequestMetadata()->emplace_back(std::move(metadata_map_ptr)); + return; + } + + FilterMetadataStatus status = (*entry)->handle_->decodeMetadata(metadata_map); + ENVOY_STREAM_LOG(trace, "decode metadata called: filter={} status={}, metadata: {}", *this, + static_cast((*entry).get()), static_cast(status), + metadata_map); + } +} + void ConnectionManagerImpl::ActiveStream::maybeEndDecode(bool end_stream) { ASSERT(!state_.remote_complete_); state_.remote_complete_ = end_stream; @@ -1555,6 +1611,17 @@ void ConnectionManagerImpl::ActiveStream::maybeEndEncode(bool end_stream) { } } +bool ConnectionManagerImpl::ActiveStream::processNewlyAddedMetadata() { + if (request_metadata_map_vector_ == nullptr) { + return false; + } + for (const auto& metadata_map : *getRequestMetadataMapVector()) { + decodeMetadata(nullptr, *metadata_map); + } + getRequestMetadataMapVector()->clear(); + return true; +} + bool ConnectionManagerImpl::ActiveStream::handleDataIfStopAll(ActiveStreamFilterBase& filter, Buffer::Instance& data, bool& filter_streaming) { @@ -1698,6 +1765,8 @@ void ConnectionManagerImpl::ActiveStreamFilterBase::commonContinue() { doHeaders(complete() && !bufferedData() && !trailers()); } + doMetadata(); + // Make sure we handle filters returning StopIterationNoBuffer and then commonContinue by flushing // the terminal fin. const bool end_stream_with_data = complete() && !trailers(); @@ -1740,22 +1809,25 @@ bool ConnectionManagerImpl::ActiveStreamFilterBase::commonHandleAfterHeadersCall if (status == FilterHeadersStatus::StopIteration) { iteration_state_ = IterationState::StopSingleIteration; - return false; } else if (status == FilterHeadersStatus::StopAllIterationAndBuffer) { iteration_state_ = IterationState::StopAllBuffer; - return false; } else if (status == FilterHeadersStatus::StopAllIterationAndWatermark) { iteration_state_ = IterationState::StopAllWatermark; - return false; } else if (status == FilterHeadersStatus::ContinueAndEndStream) { // Set headers_only to true so we know to end early if necessary, // but continue filter iteration so we actually write the headers/run the cleanup code. headers_only = true; ENVOY_STREAM_LOG(debug, "converting to headers only", parent_); - return true; } else { ASSERT(status == FilterHeadersStatus::Continue); headers_continued_ = true; + } + + handleMetadataAfterHeadersCallback(); + + if (stoppedAll() || status == FilterHeadersStatus::StopIteration) { + return false; + } else { return true; } } @@ -1871,6 +1943,19 @@ Buffer::WatermarkBufferPtr ConnectionManagerImpl::ActiveStreamDecoderFilter::cre return buffer; } +void ConnectionManagerImpl::ActiveStreamDecoderFilter::handleMetadataAfterHeadersCallback() { + // If we drain accumulated metadata, the iteration must start with the current filter. + const bool saved_state = iterate_from_current_filter_; + iterate_from_current_filter_ = true; + // If decodeHeaders() returns StopAllIteration, we should skip draining metadata, and wait + // for doMetadata() to drain the metadata after iteration continues. + if (!stoppedAll() && saved_request_metadata_ != nullptr && !getSavedRequestMetadata()->empty()) { + drainSavedRequestMetadata(); + } + // Restores the original value of iterate_from_current_filter_. + iterate_from_current_filter_ = saved_state; +} + HeaderMap& ConnectionManagerImpl::ActiveStreamDecoderFilter::addDecodedTrailers() { return parent_.addDecodedTrailers(); } @@ -1880,6 +1965,10 @@ void ConnectionManagerImpl::ActiveStreamDecoderFilter::addDecodedData(Buffer::In parent_.addDecodedData(*this, data, streaming); } +MetadataMapVector& ConnectionManagerImpl::ActiveStreamDecoderFilter::addDecodedMetadata() { + return parent_.addDecodedMetadata(); +} + void ConnectionManagerImpl::ActiveStreamDecoderFilter::injectDecodedDataToFilterChain( Buffer::Instance& data, bool end_stream) { parent_.decodeData(this, data, end_stream, diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index a7ef3ef9eae8..0a7139a40f12 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -100,7 +100,8 @@ class ConnectionManagerImpl : Logger::Loggable, ActiveStreamFilterBase(ActiveStream& parent, bool dual_filter) : parent_(parent), iteration_state_(IterationState::Continue), iterate_from_current_filter_(false), headers_continued_(false), - continue_headers_continued_(false), end_stream_(false), dual_filter_(dual_filter) {} + continue_headers_continued_(false), end_stream_(false), dual_filter_(dual_filter), + decode_headers_called_(false) {} // Functions in the following block are called after the filter finishes processing // corresponding data. Those functions handle state updates and data storage (if needed) @@ -128,6 +129,9 @@ class ConnectionManagerImpl : Logger::Loggable, virtual void doData(bool end_stream) PURE; virtual void doTrailers() PURE; virtual const HeaderMapPtr& trailers() PURE; + virtual void doMetadata() PURE; + // TODO(soya3129): make this pure when adding impl to encodefilter. + virtual void handleMetadataAfterHeadersCallback() {} // Http::StreamFilterCallbacks const Network::Connection* connection() override; @@ -151,7 +155,25 @@ class ConnectionManagerImpl : Logger::Loggable, ASSERT(iteration_state_ != IterationState::Continue); iteration_state_ = IterationState::Continue; } + MetadataMapVector* getSavedRequestMetadata() { + if (saved_request_metadata_ == nullptr) { + saved_request_metadata_ = std::make_unique(); + } + return saved_request_metadata_.get(); + } + MetadataMapVector* getSavedResponseMetadata() { + if (saved_response_metadata_ == nullptr) { + saved_response_metadata_ = std::make_unique(); + } + return saved_response_metadata_.get(); + } + // A vector to save metadata when the current filter's [de|en]codeMetadata() can not be called, + // either because [de|en]codeHeaders() of the current filter returns StopAllIteration or because + // [de|en]codeHeaders() adds new metadata to [de|en]code, but we don't know + // [de|en]codeHeaders()'s return value yet. The storage is created on demand. + std::unique_ptr saved_request_metadata_; + std::unique_ptr saved_response_metadata_; // The state of iteration. enum class IterationState { Continue, // Iteration has not stopped for any frame type. @@ -173,6 +195,7 @@ class ConnectionManagerImpl : Logger::Loggable, // If true, end_stream is called for this filter. bool end_stream_ : 1; const bool dual_filter_ : 1; + bool decode_headers_called_ : 1; }; /** @@ -205,13 +228,29 @@ class ConnectionManagerImpl : Logger::Loggable, parent_.decodeData(this, *parent_.buffered_request_data_, end_stream, ActiveStream::FilterIterationStartState::CanStartFromCurrent); } + void doMetadata() override { + if (saved_request_metadata_ != nullptr) { + drainSavedRequestMetadata(); + } + } void doTrailers() override { parent_.decodeTrailers(this, *parent_.request_trailers_); } const HeaderMapPtr& trailers() override { return parent_.request_trailers_; } + void drainSavedRequestMetadata() { + ASSERT(saved_request_metadata_ != nullptr); + for (auto& metadata_map : *getSavedRequestMetadata()) { + parent_.decodeMetadata(this, *metadata_map); + } + getSavedRequestMetadata()->clear(); + } + // This function is called after the filter calls decodeHeaders() to drain accumulated metadata. + void handleMetadataAfterHeadersCallback() override; + // Http::StreamDecoderFilterCallbacks void addDecodedData(Buffer::Instance& data, bool streaming) override; void injectDecodedDataToFilterChain(Buffer::Instance& data, bool end_stream) override; HeaderMap& addDecodedTrailers() override; + MetadataMapVector& addDecodedMetadata() override; void continueDecoding() override; const Buffer::Instance* decodingBuffer() override { return parent_.buffered_request_data_.get(); @@ -299,6 +338,19 @@ class ConnectionManagerImpl : Logger::Loggable, parent_.encodeData(this, *parent_.buffered_response_data_, end_stream, ActiveStream::FilterIterationStartState::CanStartFromCurrent); } + void drainSavedResponseMetadata() { + ASSERT(saved_response_metadata_ != nullptr); + for (auto& metadata_map : *getSavedResponseMetadata()) { + parent_.encodeMetadata(this, std::move(metadata_map)); + } + getSavedResponseMetadata()->clear(); + } + + void doMetadata() override { + if (saved_response_metadata_ != nullptr) { + drainSavedResponseMetadata(); + } + } void doTrailers() override { parent_.encodeTrailers(this, *parent_.response_trailers_); } const HeaderMapPtr& trailers() override { return parent_.response_trailers_; } @@ -358,12 +410,14 @@ class ConnectionManagerImpl : Logger::Loggable, const Network::Connection* connection(); void addDecodedData(ActiveStreamDecoderFilter& filter, Buffer::Instance& data, bool streaming); HeaderMap& addDecodedTrailers(); + MetadataMapVector& addDecodedMetadata(); void decodeHeaders(ActiveStreamDecoderFilter* filter, HeaderMap& headers, bool end_stream); // Sends data through decoding filter chains. filter_iteration_start_state indicates which // filter to start the iteration with. void decodeData(ActiveStreamDecoderFilter* filter, Buffer::Instance& data, bool end_stream, FilterIterationStartState filter_iteration_start_state); void decodeTrailers(ActiveStreamDecoderFilter* filter, HeaderMap& trailers); + void decodeMetadata(ActiveStreamDecoderFilter* filter, MetadataMap& metadata_map); void disarmRequestTimeout(); void maybeEndDecode(bool end_stream); void addEncodedData(ActiveStreamEncoderFilter& filter, Buffer::Instance& data, bool streaming); @@ -382,6 +436,8 @@ class ConnectionManagerImpl : Logger::Loggable, void encodeTrailers(ActiveStreamEncoderFilter* filter, HeaderMap& trailers); void encodeMetadata(ActiveStreamEncoderFilter* filter, MetadataMapPtr&& metadata_map_ptr); void maybeEndEncode(bool end_stream); + // Returns true if new metadata is decoded. Otherwise, returns false. + bool processNewlyAddedMetadata(); uint64_t streamId() { return stream_id_; } // Returns true if filter has stopped iteration for all frame types. Otherwise, returns false. // filter_streaming is the variable to indicate if stream is streaming, and its value may be @@ -400,7 +456,7 @@ class ConnectionManagerImpl : Logger::Loggable, void decodeHeaders(HeaderMapPtr&& headers, bool end_stream) override; void decodeData(Buffer::Instance& data, bool end_stream) override; void decodeTrailers(HeaderMapPtr&& trailers) override; - void decodeMetadata(MetadataMapPtr&&) override { NOT_REACHED_GCOVR_EXCL_LINE; } + void decodeMetadata(MetadataMapPtr&&) override; // Http::FilterChainFactoryCallbacks void addStreamDecoderFilter(StreamDecoderFilterSharedPtr filter) override { @@ -516,6 +572,13 @@ class ConnectionManagerImpl : Logger::Loggable, return os; } + MetadataMapVector* getRequestMetadataMapVector() { + if (request_metadata_map_vector_ == nullptr) { + request_metadata_map_vector_ = std::make_unique(); + } + return request_metadata_map_vector_.get(); + } + ConnectionManagerImpl& connection_manager_; Router::ConfigConstSharedPtr snapped_route_config_; Router::ScopedConfigConstSharedPtr snapped_scoped_route_config_; @@ -543,6 +606,10 @@ class ConnectionManagerImpl : Logger::Loggable, absl::optional cached_route_; absl::optional cached_cluster_info_; std::list watermark_callbacks_{}; + // Stores metadata added in the decoding filter that is being processed. Will be cleared before + // processing the next filter. The storage is created on demand. We need to store metadata + // temporarily in the filter in case the filter has stopped all while processing headers. + std::unique_ptr request_metadata_map_vector_{nullptr}; uint32_t buffer_limit_{0}; uint32_t high_watermark_count_{0}; const std::string* decorated_operation_{nullptr}; diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 1a7dcd831cd4..d01fae5bf5a7 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -606,6 +606,13 @@ Http::FilterTrailersStatus Filter::decodeTrailers(Http::HeaderMap& trailers) { return Http::FilterTrailersStatus::StopIteration; } +Http::FilterMetadataStatus Filter::decodeMetadata(Http::MetadataMap& metadata_map) { + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + ASSERT(upstream_requests_.size() == 1); + upstream_requests_.front()->encodeMetadata(std::move(metadata_map_ptr)); + return Http::FilterMetadataStatus::Continue; +} + void Filter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) { callbacks_ = &callbacks; // As the decoder filter only pushes back via watermarks once data has reached @@ -1403,6 +1410,8 @@ void Filter::UpstreamRequest::encodeData(Buffer::Instance& data, bool end_stream buffered_request_body_->move(data); } else { + ASSERT(downstream_metadata_map_vector_.empty()); + ENVOY_STREAM_LOG(trace, "proxying {} bytes", *parent_.callbacks_, data.length()); stream_info_.addBytesSent(data.length()); request_encoder_->encodeData(data, end_stream); @@ -1420,12 +1429,27 @@ void Filter::UpstreamRequest::encodeTrailers(const Http::HeaderMap& trailers) { if (!request_encoder_) { ENVOY_STREAM_LOG(trace, "buffering trailers", *parent_.callbacks_); } else { + ASSERT(downstream_metadata_map_vector_.empty()); + ENVOY_STREAM_LOG(trace, "proxying trailers", *parent_.callbacks_); request_encoder_->encodeTrailers(trailers); upstream_timing_.onLastUpstreamTxByteSent(parent_.callbacks_->dispatcher().timeSource()); } } +void Filter::UpstreamRequest::encodeMetadata(Http::MetadataMapPtr&& metadata_map_ptr) { + if (!request_encoder_) { + ENVOY_STREAM_LOG(trace, "request_encoder_ not ready. Store metadata_map to encode later: {}", + *parent_.callbacks_, *metadata_map_ptr); + downstream_metadata_map_vector_.emplace_back(std::move(metadata_map_ptr)); + } else { + ENVOY_STREAM_LOG(trace, "Encode metadata: {}", *parent_.callbacks_, *metadata_map_ptr); + Http::MetadataMapVector metadata_map_vector; + metadata_map_vector.emplace_back(std::move(metadata_map_ptr)); + request_encoder_->encodeMetadata(metadata_map_vector); + } +} + void Filter::UpstreamRequest::onResetStream(Http::StreamResetReason reason, absl::string_view transport_failure_reason) { clearRequestEncoder(); @@ -1529,8 +1553,13 @@ void Filter::UpstreamRequest::onPoolReady(Http::StreamEncoder& request_encoder, } upstream_timing_.onFirstUpstreamTxByteSent(parent_.callbacks_->dispatcher().timeSource()); + + const bool end_stream = !buffered_request_body_ && encode_complete_ && !encode_trailers_; + // If end_stream is set in headers, and there are metadata to send, delays end_stream. The case + // only happens when decoding headers filters return ContinueAndEndStream. + const bool delay_headers_end_stream = end_stream && !downstream_metadata_map_vector_.empty(); request_encoder.encodeHeaders(*parent_.downstream_headers_, - !buffered_request_body_ && encode_complete_ && !encode_trailers_); + end_stream && !delay_headers_end_stream); calling_encode_headers_ = false; // It is possible to get reset in the middle of an encodeHeaders() call. This happens for example @@ -1541,6 +1570,18 @@ void Filter::UpstreamRequest::onPoolReady(Http::StreamEncoder& request_encoder, if (deferred_reset_reason_) { onResetStream(deferred_reset_reason_.value(), absl::string_view()); } else { + // Encode metadata after headers and before any other frame type. + if (!downstream_metadata_map_vector_.empty()) { + ENVOY_STREAM_LOG(debug, "Send metadata onPoolReady. {}", *parent_.callbacks_, + downstream_metadata_map_vector_); + request_encoder.encodeMetadata(downstream_metadata_map_vector_); + downstream_metadata_map_vector_.clear(); + if (delay_headers_end_stream) { + Buffer::OwnedImpl empty_data(""); + request_encoder.encodeData(empty_data, true); + } + } + if (buffered_request_body_) { stream_info_.addBytesSent(buffered_request_body_->length()); request_encoder.encodeData(*buffered_request_body_, encode_complete_ && !encode_trailers_); diff --git a/source/common/router/router.h b/source/common/router/router.h index 282d41997b37..2767db0a2921 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -243,6 +243,7 @@ class Filter : Logger::Loggable, Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap& headers, bool end_stream) override; Http::FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override; Http::FilterTrailersStatus decodeTrailers(Http::HeaderMap& trailers) override; + Http::FilterMetadataStatus decodeMetadata(Http::MetadataMap& metadata_map) override; void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override; // Upstream::LoadBalancerContext @@ -361,6 +362,7 @@ class Filter : Logger::Loggable, void encodeHeaders(bool end_stream); void encodeData(Buffer::Instance& data, bool end_stream); void encodeTrailers(const Http::HeaderMap& trailers); + void encodeMetadata(Http::MetadataMapPtr&& metadata_map_ptr); void resetStream(); void setupPerTryTimeout(); @@ -458,6 +460,7 @@ class Filter : Logger::Loggable, // access logging is configured. Http::HeaderMapPtr upstream_headers_; Http::HeaderMapPtr upstream_trailers_; + Http::MetadataMapVector downstream_metadata_map_vector_; bool calling_encode_headers_ : 1; bool upstream_canary_ : 1; @@ -523,6 +526,7 @@ class Filter : Logger::Loggable, // for the remaining upstream requests to return. void resetOtherUpstreams(UpstreamRequest& upstream_request); void sendNoHealthyUpstreamResponse(); + // TODO(soya3129): Save metadata for retry, redirect and shadowing case. bool setupRetry(); bool setupRedirect(const Http::HeaderMap& headers, UpstreamRequest& upstream_request); void updateOutlierDetection(Upstream::Outlier::Result result, UpstreamRequest& upstream_request, diff --git a/source/docs/h2_metadata.md b/source/docs/h2_metadata.md index e986ff27a211..81fce3daac7a 100644 --- a/source/docs/h2_metadata.md +++ b/source/docs/h2_metadata.md @@ -22,10 +22,9 @@ stream flag. Because metadata frames must be associated with an existing frame, ensure metadata frames to be received before the end of stream is received by the peer. -Metadata associated with a response can be sent before response headers, after response headers, -between response data or after response data. If metadata frames have to be sent last, +Metadata associated with a stream can be sent before headers, after headers, +between data or after data. If metadata frames have to be sent last, users must put the end of stream in an empty data frame and send the empty data frame after metadata frames. -TODO(soya3129): add more conditions for metadata in requests. Envoy only allows up to 1M metadata to be sent per stream. If the accumulated metadata size exceeds the limit, the stream will be reset. @@ -36,8 +35,6 @@ Envoy provides the functionality to proxy, process and add metadata. ## Proxying metadata -(To be implemented) - If not specified, all the metadata received by Envoy is proxied to the next hop unmodified. Note that, we do not guarantee the same frame order will be preserved from hop by hop. That is, metadata from upstream at the beginning of a stream can be @@ -45,8 +42,6 @@ received by the downstream at the end of the stream. ## Consuming metadata -(To be implemented) - If Envoy needs to take actions when a metadata frame is received, users should create a new filter. @@ -74,17 +69,20 @@ If the metadata is left in the map, it will be passed to the next hop. ## Inserting metadata -(To be implemented) - Envoy filters can be used to add new metadata to a stream. If users need to add new metadata for a request from downstream to upstream, a StreamDecoderFilter should be created. The StreamDecoderFilterCallbacks object that Envoy passes to the StreamDecoderFilter has an interface MetadataMapVector& StreamDecoderFilterCallbacks::addDecodedMetadata(). By calling the interface, -users get a reference to the metadata map associated with the request stream. Users can -insert new metadata to the metadata map, and Envoy will proxy the new metadata -map to the upstream. +users get a reference to a vector of metadata map associated with the request stream. Users can +insert new metadata map to the metadata map vector, and Envoy will proxy the new metadata +map to the upstream. StreamDecoderFilterCallbacks::addDecodedMetadata() can be called in +StreamDecoderFilter::decodeHeaders(), StreamDecoderFilter::decodeData() and +StreamDecoderFilter::decodeTrailers(). Do not call +StreamDecoderFilterCallbacks::addDecodedMetadata() in +StreamDecoderFilter::decodeMetadata(MetadataMap metadata\_map). New metadata can +be added directly to metadata\_map. If users need to add new metadata for a response to downstream, a StreamFilter should be created. Users pass the metadata to be added to @@ -171,3 +169,37 @@ ConnectionManagerImpl::ActiveStream::encodeMetadata(ActiveStreamEncoderFilter\* to go through all the encoding filters. Or new metadata can be added to metadata\_map in StreamFilter::encodeMetadata(MetadataMap& metadata\_map) directly. + +## Request metadata handling + +We first explain how request metadata get consumed or proxied. +In function EnvoyConnectionManagerImpl::ActiveStream::decodeMetadata(ActiveStreamDecoderFilter\* filter, +MetadataMap& metadata\_map), Envoy passes request metadata received from downstream to filters by +calling the following filter interface: + +FilterMetadatasStatus StreamDecoderFilter::decodeMetadata(MetadataMap& metadata\_map). + +Filters, by implementing the interface, can consume or modify request metadata. If no filter +touches the metadata, it is proxied to upstream unchanged. + +The last filter in the filter chain is router filter. The router filter calls +Filter::request\_encoder\_-\>encodeMetadata(const MetadataMapVector& metadata\_map\_vector) to pass +the metadata to codec, and codec encodes and forwards the metadata to the upstream. If the connection +to the upstream has not been established when metadata is received, the metadata is temporarily stored in +Filter::downstream\_metadata\_map\_vector\_. When the connection is ready +(Filter::UpstreamRequest::onPoolReady()), the metadata is then passed to codec, and forwarded to +the upstream. + +Envoy can also add new request metadata through filters's decoding interfaces (See section +[Inserting metadata](#inserting-metadata) for detailed interfaces). Filters can add new +metadata to ActiveStream::request\_metadata\_map\_vector\_ by calling +StreamDecoderFilterCallbacks::addDecodedMetadata(). After calling each filter's decoding function, +Envoy checks if new metadata is added to ActiveStream::request\_metadata\_map\_vector\_. If so, +then Envoy calls ConnectionManagerImpl::ActiveStream::decodeMetadata(ActiveStreamEncoderFilter\* filter, +MetadataMapPtr&& metadata\_map) to go through all the filters. + +Note that, because metadata frames do not carry end\_stream, if new metadata is added to a headers +only request, Envoy moves end\_stream from headers to an empty data frame which is sent after the new +metadata. In addition, Envoy drains metadata in router filter before any other types of +frames except headers to make sure end\_stream is handled correctly. + diff --git a/source/extensions/filters/http/common/pass_through_filter.h b/source/extensions/filters/http/common/pass_through_filter.h index 2527e05bde35..30d3e4e44d82 100644 --- a/source/extensions/filters/http/common/pass_through_filter.h +++ b/source/extensions/filters/http/common/pass_through_filter.h @@ -18,7 +18,6 @@ class PassThroughDecoderFilter : public virtual StreamDecoderFilter { Http::FilterDataStatus decodeData(Buffer::Instance&, bool) override { return Http::FilterDataStatus::Continue; } - Http::FilterTrailersStatus decodeTrailers(Http::HeaderMap&) override { return Http::FilterTrailersStatus::Continue; } diff --git a/source/extensions/filters/http/csrf/csrf_filter.h b/source/extensions/filters/http/csrf/csrf_filter.h index da62a7800873..57def213db6e 100644 --- a/source/extensions/filters/http/csrf/csrf_filter.h +++ b/source/extensions/filters/http/csrf/csrf_filter.h @@ -96,13 +96,13 @@ class CsrfFilter : public Http::StreamDecoderFilter { Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap& headers, bool end_stream) override; Http::FilterDataStatus decodeData(Buffer::Instance&, bool) override { return Http::FilterDataStatus::Continue; - }; + } Http::FilterTrailersStatus decodeTrailers(Http::HeaderMap&) override { return Http::FilterTrailersStatus::Continue; - }; + } void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) override { callbacks_ = &callbacks; - }; + } private: void determinePolicy(); diff --git a/test/extensions/filters/http/buffer/buffer_filter_test.cc b/test/extensions/filters/http/buffer/buffer_filter_test.cc index 34c1bd05298b..4af97078d5c7 100644 --- a/test/extensions/filters/http/buffer/buffer_filter_test.cc +++ b/test/extensions/filters/http/buffer/buffer_filter_test.cc @@ -59,6 +59,11 @@ TEST_F(BufferFilterTest, HeaderOnlyRequest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(headers, true)); } +TEST_F(BufferFilterTest, TestMetadata) { + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_.decodeMetadata(metadata_map)); +} + TEST_F(BufferFilterTest, RequestWithData) { InSequence s; diff --git a/test/extensions/filters/http/cors/cors_filter_test.cc b/test/extensions/filters/http/cors/cors_filter_test.cc index 1558eb3462ab..d56cb8ac6a4a 100644 --- a/test/extensions/filters/http/cors/cors_filter_test.cc +++ b/test/extensions/filters/http/cors/cors_filter_test.cc @@ -70,6 +70,8 @@ TEST_F(CorsFilterTest, RequestWithoutOrigin) { EXPECT_EQ(0, stats_.counter("test.cors.origin_valid").value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(request_headers_)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_.decodeMetadata(metadata_map)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.encodeHeaders(request_headers_, false)); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.encodeData(data_, false)); diff --git a/test/extensions/filters/http/csrf/csrf_filter_test.cc b/test/extensions/filters/http/csrf/csrf_filter_test.cc index 55def661cb19..99835d935c1a 100644 --- a/test/extensions/filters/http/csrf/csrf_filter_test.cc +++ b/test/extensions/filters/http/csrf/csrf_filter_test.cc @@ -102,6 +102,8 @@ TEST_F(CsrfFilterTest, RequestWithNonMutableMethod) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_.decodeMetadata(metadata_map)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(request_headers_)); EXPECT_EQ(0U, config_->stats().missing_source_origin_.value()); diff --git a/test/extensions/filters/http/dynamo/dynamo_filter_test.cc b/test/extensions/filters/http/dynamo/dynamo_filter_test.cc index df5c19b318cb..f7021a9b05e9 100644 --- a/test/extensions/filters/http/dynamo/dynamo_filter_test.cc +++ b/test/extensions/filters/http/dynamo/dynamo_filter_test.cc @@ -58,14 +58,14 @@ TEST_F(DynamoFilterTest, operatorPresent) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers, true)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata_map)); Http::TestHeaderMapImpl continue_headers{{":status", "100"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encode100ContinueHeaders(continue_headers)); - Http::MetadataMap metadata_map{{"metadata", "metadata"}}; - EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata_map)); - Http::TestHeaderMapImpl response_headers{{":status", "200"}}; EXPECT_CALL(stats_, counter("prefix.dynamodb.operation_missing")).Times(0); EXPECT_CALL(stats_, counter("prefix.dynamodb.table_missing")); diff --git a/test/extensions/filters/http/ext_authz/ext_authz_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_test.cc index f2f07af5f8ff..76c3ca3759a9 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_test.cc @@ -806,6 +806,8 @@ TEST_F(HttpFilterTestParam, ContextExtensions) { // Engage the filter so that check is called. filter_->decodeHeaders(request_headers_, false); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); // Make sure that the extensions appear in the check request issued by the filter. EXPECT_EQ("value_vhost", check_request.attributes().context_extensions().at("key_vhost")); diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index a22a9819a7f7..f6181e467675 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -299,6 +299,8 @@ TEST_F(FaultFilterTest, AbortWithHttpStatus) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); EXPECT_EQ(1UL, config_->stats().active_faults_.value()); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data_, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_headers_)); diff --git a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc index d7c40ab86991..aedebdfa6f0b 100644 --- a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc +++ b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc @@ -52,6 +52,8 @@ TEST_F(GrpcHttp1BridgeFilterTest, NoRoute) { {":path", "/lyft.users.BadCompanions/GetBadCompanions"}}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, true)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_.decodeMetadata(metadata_map)); Http::TestHeaderMapImpl response_headers{{":status", "404"}}; } diff --git a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc index 3fbb89ac8bd2..437bc9d373b4 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc @@ -335,6 +335,8 @@ TEST_F(GrpcJsonTranscoderFilterTest, NoTranscoding) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); EXPECT_EQ(expected_request_headers, request_headers); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_.decodeMetadata(metadata_map)); Buffer::OwnedImpl request_data{"{}"}; EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(request_data, false)); diff --git a/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc b/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc index 1500708fe3c6..e0a501f48d69 100644 --- a/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc +++ b/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc @@ -123,6 +123,8 @@ TEST_F(GrpcWebFilterTest, SupportedContentTypes) { Http::TestHeaderMapImpl request_headers; request_headers.addCopy(Http::Headers::get().ContentType, content_type); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_.decodeMetadata(metadata_map)); EXPECT_EQ(Http::Headers::get().ContentTypeValues.Grpc, request_headers.ContentType()->value().getStringView()); } diff --git a/test/extensions/filters/http/gzip/gzip_filter_test.cc b/test/extensions/filters/http/gzip/gzip_filter_test.cc index 6eb74308e67d..a45e7760a222 100644 --- a/test/extensions/filters/http/gzip/gzip_filter_test.cc +++ b/test/extensions/filters/http/gzip/gzip_filter_test.cc @@ -226,6 +226,8 @@ TEST_F(GzipFilterTest, AvailableCombinationCompressionStrategyAndLevelConfig) { // Acceptance Testing with default configuration. TEST_F(GzipFilterTest, AcceptanceGzipEncoding) { doRequest({{":method", "get"}, {"accept-encoding", "deflate, gzip"}}, false); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); Buffer::OwnedImpl data("hello"); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); Http::TestHeaderMapImpl trailers; diff --git a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc index 5908af6e6361..1cfa6acfe73f 100644 --- a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc +++ b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc @@ -79,6 +79,8 @@ TEST_F(HeaderToMetadataTest, BasicRequestTest) { EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); EXPECT_CALL(req_info_, setDynamicMetadata("envoy.lb", MapEq(expected))); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(incoming_headers, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); Buffer::OwnedImpl data("data"); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(incoming_headers)); diff --git a/test/extensions/filters/http/health_check/health_check_test.cc b/test/extensions/filters/http/health_check/health_check_test.cc index 03ead8af99cd..6a2d4b5de94b 100644 --- a/test/extensions/filters/http/health_check/health_check_test.cc +++ b/test/extensions/filters/http/health_check/health_check_test.cc @@ -99,6 +99,8 @@ TEST_F(HealthCheckFilterNoPassThroughTest, OkOrFailed) { EXPECT_CALL(callbacks_.active_span_, setSampled(false)); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); } TEST_F(HealthCheckFilterNoPassThroughTest, NotHcRequest) { diff --git a/test/extensions/filters/http/jwt_authn/filter_test.cc b/test/extensions/filters/http/jwt_authn/filter_test.cc index d408599540ee..3a65d74c3bcc 100644 --- a/test/extensions/filters/http/jwt_authn/filter_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_test.cc @@ -70,6 +70,8 @@ TEST_F(FilterTest, InlineOK) { auto headers = Http::TestHeaderMapImpl{}; EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(headers, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); EXPECT_EQ(1U, mock_config_->stats().allowed_.value()); Buffer::OwnedImpl data(""); diff --git a/test/extensions/filters/http/lua/lua_filter_test.cc b/test/extensions/filters/http/lua/lua_filter_test.cc index c626b104346d..d9be20d3f301 100644 --- a/test/extensions/filters/http/lua/lua_filter_test.cc +++ b/test/extensions/filters/http/lua/lua_filter_test.cc @@ -250,6 +250,8 @@ TEST_F(LuaHttpFilterTest, ScriptBodyChunksRequestBody) { Http::TestHeaderMapImpl request_headers{{":path", "/"}}; EXPECT_CALL(*filter_, scriptLog(spdlog::level::trace, StrEq("/"))); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); Buffer::OwnedImpl data("hello"); EXPECT_CALL(*filter_, scriptLog(spdlog::level::trace, StrEq("5"))); diff --git a/test/extensions/filters/http/ratelimit/ratelimit_test.cc b/test/extensions/filters/http/ratelimit/ratelimit_test.cc index ed1b4afc135d..c956709a2a54 100644 --- a/test/extensions/filters/http/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/http/ratelimit/ratelimit_test.cc @@ -198,6 +198,8 @@ TEST_F(HttpRateLimitFilterTest, OkResponse) { request_headers_.addCopy(Http::Headers::get().RequestId, "requestid"); EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(request_headers_, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(data_, false)); EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(request_headers_)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, diff --git a/test/extensions/filters/http/rbac/rbac_filter_test.cc b/test/extensions/filters/http/rbac/rbac_filter_test.cc index c7ab01f93109..62967602dfc6 100644 --- a/test/extensions/filters/http/rbac/rbac_filter_test.cc +++ b/test/extensions/filters/http/rbac/rbac_filter_test.cc @@ -88,6 +88,8 @@ TEST_F(RoleBasedAccessControlFilterTest, Allowed) { setDestinationPort(123); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(headers_, false)); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_.decodeMetadata(metadata_map)); EXPECT_EQ(1U, config_->stats().allowed_.value()); EXPECT_EQ(1U, config_->stats().shadow_denied_.value()); diff --git a/test/extensions/filters/http/squash/squash_filter_test.cc b/test/extensions/filters/http/squash/squash_filter_test.cc index 03c19f42d95b..534b59b87a74 100644 --- a/test/extensions/filters/http/squash/squash_filter_test.cc +++ b/test/extensions/filters/http/squash/squash_filter_test.cc @@ -209,6 +209,8 @@ class SquashFilterTest : public testing::Test { void doDownstreamRequest() { startDownstreamRequest(); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); Envoy::Http::TestHeaderMapImpl trailers{}; // Complete a full request cycle Envoy::Buffer::OwnedImpl buffer("nothing here"); diff --git a/test/integration/BUILD b/test/integration/BUILD index fff182f022cb..2bf9113fa201 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -207,6 +207,9 @@ envoy_cc_test( "//source/extensions/filters/http/buffer:config", "//source/extensions/filters/http/dynamo:config", "//source/extensions/filters/http/health_check:config", + "//test/integration/filters:metadata_stop_all_filter_config_lib", + "//test/integration/filters:request_metadata_filter_config_lib", + "//test/integration/filters:response_metadata_filter_config_lib", "//test/integration/filters:stop_iteration_and_continue", "//test/mocks/http:http_mocks", "//test/mocks/upstream:upstream_mocks", @@ -313,7 +316,6 @@ envoy_cc_test_library( "//test/integration/filters:modify_buffer_filter_config_lib", "//test/integration/filters:passthrough_filter_config_lib", "//test/integration/filters:pause_filter_lib", - "//test/integration/filters:response_metadata_filter_config_lib", "//test/test_common:registry_lib", ], ) diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 546f159fa895..8cca9265525e 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -51,6 +51,7 @@ void FakeStream::decodeHeaders(Http::HeaderMapPtr&& headers, bool end_stream) { } void FakeStream::decodeData(Buffer::Instance& data, bool end_stream) { + received_data_ = true; Thread::LockGuard lock(lock_); body_.add(data); setEndStream(end_stream); @@ -64,6 +65,13 @@ void FakeStream::decodeTrailers(Http::HeaderMapPtr&& trailers) { decoder_event_.notifyOne(); } +void FakeStream::decodeMetadata(Http::MetadataMapPtr&& metadata_map_ptr) { + for (const auto& metadata : *metadata_map_ptr) { + duplicated_metadata_key_count_[metadata.first]++; + metadata_map_.insert(metadata); + } +} + void FakeStream::encode100ContinueHeaders(const Http::HeaderMapImpl& headers) { std::shared_ptr headers_copy( new Http::HeaderMapImpl(static_cast(headers))); diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 38d66690e41e..7ea255edf4f9 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -62,6 +62,7 @@ class FakeStream : public Http::StreamDecoder, const Http::HeaderMap& headers() { return *headers_; } void setAddServedByHeader(bool add_header) { add_served_by_header_ = add_header; } const Http::HeaderMapPtr& trailers() { return trailers_; } + bool receivedData() { return received_data_; } ABSL_MUST_USE_RESULT testing::AssertionResult @@ -150,7 +151,7 @@ class FakeStream : public Http::StreamDecoder, void decodeHeaders(Http::HeaderMapPtr&& headers, bool end_stream) override; void decodeData(Buffer::Instance& data, bool end_stream) override; void decodeTrailers(Http::HeaderMapPtr&& trailers) override; - void decodeMetadata(Http::MetadataMapPtr&&) override {} + void decodeMetadata(Http::MetadataMapPtr&& metadata_map_ptr) override; // Http::StreamCallbacks void onResetStream(Http::StreamResetReason reason, @@ -162,6 +163,11 @@ class FakeStream : public Http::StreamDecoder, Event::TestTimeSystem& timeSystem() { return time_system_; } + Http::MetadataMap& metadata_map() { return metadata_map_; } + std::unordered_map& duplicated_metadata_key_count() { + return duplicated_metadata_key_count_; + } + protected: Http::HeaderMapPtr headers_; @@ -178,6 +184,9 @@ class FakeStream : public Http::StreamDecoder, std::vector decoded_grpc_frames_; bool add_served_by_header_{}; Event::TestTimeSystem& time_system_; + Http::MetadataMap metadata_map_; + std::unordered_map duplicated_metadata_key_count_; + bool received_data_{false}; }; using FakeStreamPtr = std::unique_ptr; diff --git a/test/integration/filters/BUILD b/test/integration/filters/BUILD index 7e2647880eb7..b06b673294d6 100644 --- a/test/integration/filters/BUILD +++ b/test/integration/filters/BUILD @@ -155,6 +155,20 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "request_metadata_filter_config_lib", + srcs = [ + "request_metadata_filter.cc", + ], + deps = [ + "//include/envoy/http:filter_interface", + "//include/envoy/registry", + "//include/envoy/server:filter_config_interface", + "//source/extensions/filters/http/common:empty_http_filter_config_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + ], +) + envoy_cc_test_library( name = "random_pause_filter_lib", srcs = [ @@ -247,3 +261,19 @@ envoy_cc_test_library( "//source/common/network:connection_lib", ], ) + +envoy_cc_test_library( + name = "metadata_stop_all_filter_config_lib", + srcs = [ + "metadata_stop_all_filter.cc", + ], + deps = [ + ":common_lib", + "//include/envoy/event:timer_interface", + "//include/envoy/http:filter_interface", + "//include/envoy/registry", + "//include/envoy/server:filter_config_interface", + "//source/extensions/filters/http/common:empty_http_filter_config_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + ], +) diff --git a/test/integration/filters/metadata_stop_all_filter.cc b/test/integration/filters/metadata_stop_all_filter.cc new file mode 100644 index 000000000000..3ff6b7983d01 --- /dev/null +++ b/test/integration/filters/metadata_stop_all_filter.cc @@ -0,0 +1,74 @@ +#include +#include + +#include "envoy/event/timer.h" +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "common/buffer/buffer_impl.h" + +#include "extensions/filters/http/common/empty_http_filter_config.h" +#include "extensions/filters/http/common/pass_through_filter.h" + +#include "test/integration/filters/common.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +class MetadataStopAllFilter : public Http::PassThroughFilter { +public: + constexpr static char name[] = "metadata-stop-all-filter"; + + Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap& header_map, bool) override { + Http::HeaderEntry* entry_content = header_map.get(Envoy::Http::LowerCaseString("content_size")); + ASSERT(entry_content != nullptr); + content_size_ = std::stoul(std::string(entry_content->value().getStringView())); + + createTimerForContinue(); + + return Http::FilterHeadersStatus::StopAllIterationAndBuffer; + } + + Http::FilterDataStatus decodeData(Buffer::Instance&, bool) override { + ASSERT(timer_triggered_); + return Http::FilterDataStatus::Continue; + } + + Http::FilterTrailersStatus decodeTrailers(Http::HeaderMap&) override { + ASSERT(timer_triggered_); + return Http::FilterTrailersStatus::Continue; + } + + Http::FilterMetadataStatus decodeMetadata(Http::MetadataMap&) override { + ASSERT(timer_triggered_); + return Http::FilterMetadataStatus::Continue; + } + +private: + // Creates a timer to continue iteration after conditions meet. + void createTimerForContinue() { + delay_timer_ = decoder_callbacks_->dispatcher().createTimer([this]() -> void { + if (content_size_ > 0 && decoder_callbacks_->streamInfo().bytesReceived() >= content_size_) { + timer_triggered_ = true; + decoder_callbacks_->continueDecoding(); + } else { + // Creates a new timer to try again later. + createTimerForContinue(); + } + }); + delay_timer_->enableTimer(std::chrono::milliseconds(50)); + } + + Event::TimerPtr delay_timer_; + bool timer_triggered_ = false; + size_t content_size_ = 0; +}; + +constexpr char MetadataStopAllFilter::name[]; +static Registry::RegisterFactory, + Server::Configuration::NamedHttpFilterConfigFactory> + register_; + +} // namespace Envoy diff --git a/test/integration/filters/request_metadata_filter.cc b/test/integration/filters/request_metadata_filter.cc new file mode 100644 index 000000000000..19c6425bdc3a --- /dev/null +++ b/test/integration/filters/request_metadata_filter.cc @@ -0,0 +1,65 @@ +#include + +#include "envoy/http/filter.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "extensions/filters/http/common/empty_http_filter_config.h" +#include "extensions/filters/http/common/pass_through_filter.h" + +namespace Envoy { +// A filter tests request metadata consuming and inserting. The filter inserts new +// metadata when decodeHeaders/Data/Trailers() are called. If the received metadata with key +// "consume", the metadata will be consumed and not forwarded to the next hop. +class RequestMetadataStreamFilter : public Http::PassThroughFilter { +public: + Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap&, bool) override { + Http::MetadataMap metadata_map = {{"headers", "headers"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + decoder_callbacks_->addDecodedMetadata().emplace_back(std::move(metadata_map_ptr)); + return Http::FilterHeadersStatus::Continue; + } + + Http::FilterDataStatus decodeData(Buffer::Instance&, bool) override { + Http::MetadataMap metadata_map = {{"data", "data"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + decoder_callbacks_->addDecodedMetadata().emplace_back(std::move(metadata_map_ptr)); + return Http::FilterDataStatus::Continue; + } + + Http::FilterTrailersStatus decodeTrailers(Http::HeaderMap&) override { + Http::MetadataMap metadata_map = {{"trailers", "trailers"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + decoder_callbacks_->addDecodedMetadata().emplace_back(std::move(metadata_map_ptr)); + return Http::FilterTrailersStatus::Continue; + } + + // If metadata_map contains key "consume", consumes the metadata, and replace it with a new one. + // The function also adds a new metadata using addDecodedMetadata(). + Http::FilterMetadataStatus decodeMetadata(Http::MetadataMap& metadata_map) override { + auto it = metadata_map.find("consume"); + if (it != metadata_map.end()) { + metadata_map.erase("consume"); + metadata_map.emplace("replace", "replace"); + } + metadata_map["metadata"] = "metadata"; + return Http::FilterMetadataStatus::Continue; + } +}; + +class AddRequestMetadataStreamFilterConfig + : public Extensions::HttpFilters::Common::EmptyHttpFilterConfig { +public: + AddRequestMetadataStreamFilterConfig() : EmptyHttpFilterConfig("request-metadata-filter") {} + Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(std::make_shared<::Envoy::RequestMetadataStreamFilter>()); + }; + } +}; + +// perform static registration +static Registry::RegisterFactory + register_; +} // namespace Envoy diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index cddc0a62530c..22929816f97f 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -265,7 +265,6 @@ void verifyExpectedMetadata(Http::MetadataMap metadata_map, std::setcomplete()); } +// Verifies small metadata can be sent at different locations of a request. +TEST_P(Http2MetadataIntegrationTest, ProxySmallMetadataInRequest) { + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto encoder_decoder = codec_client_->startRequest(default_request_headers_); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + Http::MetadataMap metadata_map = {{"key", "value"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 1, false); + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 1, false); + codec_client_->sendMetadata(*request_encoder_, metadata_map); + Http::TestHeaderMapImpl request_trailers{{"request", "trailer"}}; + codec_client_->sendTrailers(*request_encoder_, request_trailers); + + waitForNextUpstreamRequest(); + + // Verifies metadata is received by upstream. + upstream_request_->encodeHeaders(default_response_headers_, true); + EXPECT_EQ(upstream_request_->metadata_map().find("key")->second, "value"); + EXPECT_EQ(upstream_request_->metadata_map().size(), 1); + EXPECT_EQ(upstream_request_->duplicated_metadata_key_count().find("key")->second, 3); + + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); +} + +// Verifies large metadata can be sent at different locations of a request. +TEST_P(Http2MetadataIntegrationTest, ProxyLargeMetadataInRequest) { + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto encoder_decoder = codec_client_->startRequest(default_request_headers_); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + std::string value = std::string(80 * 1024, '1'); + Http::MetadataMap metadata_map = {{"key", value}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 1, false); + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 1, false); + codec_client_->sendMetadata(*request_encoder_, metadata_map); + Http::TestHeaderMapImpl request_trailers{{"request", "trailer"}}; + codec_client_->sendTrailers(*request_encoder_, request_trailers); + + waitForNextUpstreamRequest(); + + // Verifies metadata is received upstream. + upstream_request_->encodeHeaders(default_response_headers_, true); + EXPECT_EQ(upstream_request_->metadata_map().find("key")->second, value); + EXPECT_EQ(upstream_request_->metadata_map().size(), 1); + EXPECT_EQ(upstream_request_->duplicated_metadata_key_count().find("key")->second, 3); + + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); +} + +static std::string request_metadata_filter = R"EOF( +name: request-metadata-filter +config: {} +)EOF"; + +TEST_P(Http2MetadataIntegrationTest, ConsumeAndInsertRequestMetadata) { + addFilters({request_metadata_filter}); + config_helper_.addConfigModifier( + [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { hcm.set_proxy_100_continue(true); }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + // Sends a headers only request. + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + // Verifies a headers metadata added. + std::set expected_metadata_keys = {"headers"}; + expected_metadata_keys.insert("metadata"); + verifyExpectedMetadata(upstream_request_->metadata_map(), expected_metadata_keys); + + // Sends a headers only request with metadata. An empty data frame carries end_stream. + auto encoder_decoder = codec_client_->startRequest(default_request_headers_); + request_encoder_ = &encoder_decoder.first; + response = std::move(encoder_decoder.second); + Http::MetadataMap metadata_map = {{"consume", "consume"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 0, true); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + expected_metadata_keys.insert("data"); + expected_metadata_keys.insert("metadata"); + expected_metadata_keys.insert("replace"); + verifyExpectedMetadata(upstream_request_->metadata_map(), expected_metadata_keys); + EXPECT_EQ(upstream_request_->duplicated_metadata_key_count().find("metadata")->second, 3); + // Verifies zero length data received, and end_stream is true. + EXPECT_EQ(true, upstream_request_->receivedData()); + EXPECT_EQ(0, upstream_request_->bodyLength()); + EXPECT_EQ(true, upstream_request_->complete()); + + // Sends headers, data, metadata and trailer. + auto encoder_decoder_2 = codec_client_->startRequest(default_request_headers_); + request_encoder_ = &encoder_decoder_2.first; + response = std::move(encoder_decoder_2.second); + codec_client_->sendData(*request_encoder_, 10, false); + metadata_map = {{"consume", "consume"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + Http::TestHeaderMapImpl request_trailers{{"trailer", "trailer"}}; + codec_client_->sendTrailers(*request_encoder_, request_trailers); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + expected_metadata_keys.insert("trailers"); + verifyExpectedMetadata(upstream_request_->metadata_map(), expected_metadata_keys); + EXPECT_EQ(upstream_request_->duplicated_metadata_key_count().find("metadata")->second, 4); + + // Sends headers, large data, metadata. Large data triggers decodeData() multiple times, and each + // time, a "data" metadata is added. + auto encoder_decoder_3 = codec_client_->startRequest(default_request_headers_); + request_encoder_ = &encoder_decoder_3.first; + response = std::move(encoder_decoder_3.second); + codec_client_->sendData(*request_encoder_, 100000, false); + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 100000, true); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + + expected_metadata_keys.erase("trailers"); + verifyExpectedMetadata(upstream_request_->metadata_map(), expected_metadata_keys); + EXPECT_GE(upstream_request_->duplicated_metadata_key_count().find("data")->second, 2); + EXPECT_GE(upstream_request_->duplicated_metadata_key_count().find("metadata")->second, 3); + + // Sends multiple metadata. + auto encoder_decoder_4 = codec_client_->startRequest(default_request_headers_); + request_encoder_ = &encoder_decoder_4.first; + response = std::move(encoder_decoder_4.second); + metadata_map = {{"metadata1", "metadata1"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 10, false); + metadata_map = {{"metadata2", "metadata2"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + metadata_map = {{"consume", "consume"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendTrailers(*request_encoder_, request_trailers); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + expected_metadata_keys.insert("metadata1"); + expected_metadata_keys.insert("metadata2"); + expected_metadata_keys.insert("trailers"); + verifyExpectedMetadata(upstream_request_->metadata_map(), expected_metadata_keys); + EXPECT_EQ(upstream_request_->duplicated_metadata_key_count().find("metadata")->second, 6); +} + +static std::string decode_headers_only = R"EOF( +name: decode-headers-only +config: {} +)EOF"; + +void Http2MetadataIntegrationTest::runHeaderOnlyTest(bool send_request_body, size_t body_size) { + config_helper_.addConfigModifier( + [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { hcm.set_proxy_100_continue(true); }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + // Sends a request with body. Only headers will pass through filters. + IntegrationStreamDecoderPtr response; + if (send_request_body) { + response = + codec_client_->makeRequestWithBody(Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}}, + body_size); + } else { + response = + codec_client_->makeHeaderOnlyRequest(Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", "host"}}); + } + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); +} + +void Http2MetadataIntegrationTest::verifyHeadersOnlyTest() { + // Verifies a headers metadata added. + std::set expected_metadata_keys = {"headers"}; + expected_metadata_keys.insert("metadata"); + verifyExpectedMetadata(upstream_request_->metadata_map(), expected_metadata_keys); + + // Verifies zero length data received, and end_stream is true. + EXPECT_EQ(true, upstream_request_->receivedData()); + EXPECT_EQ(0, upstream_request_->bodyLength()); + EXPECT_EQ(true, upstream_request_->complete()); +} + +TEST_P(Http2MetadataIntegrationTest, DecodingHeadersOnlyRequestWithRequestMetadataEmptyData) { + addFilters({request_metadata_filter, decode_headers_only}); + + // Send a request with body, and body size is 0. + runHeaderOnlyTest(true, 0); + verifyHeadersOnlyTest(); +} + +TEST_P(Http2MetadataIntegrationTest, DecodingHeadersOnlyRequestWithRequestMetadataNoneEmptyData) { + addFilters({request_metadata_filter, decode_headers_only}); + // Send a request with body, and body size is 128. + runHeaderOnlyTest(true, 128); + verifyHeadersOnlyTest(); +} + +TEST_P(Http2MetadataIntegrationTest, DecodingHeadersOnlyRequestWithRequestMetadataDiffFilterOrder) { + addFilters({decode_headers_only, request_metadata_filter}); + // Send a request with body, and body size is 128. + runHeaderOnlyTest(true, 128); + verifyHeadersOnlyTest(); +} + +TEST_P(Http2MetadataIntegrationTest, HeadersOnlyRequestWithRequestMetadata) { + addFilters({request_metadata_filter}); + // Send a headers only request. + runHeaderOnlyTest(false, 0); + verifyHeadersOnlyTest(); +} + +void Http2MetadataIntegrationTest::testRequestMetadataWithStopAllFilter() { + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + // Sends multiple metadata. + const size_t size = 10; + default_request_headers_.addCopy("content_size", std::to_string(size)); + auto encoder_decoder = codec_client_->startRequest(default_request_headers_); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + Http::MetadataMap metadata_map = {{"metadata1", "metadata1"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, size, false); + metadata_map = {{"metadata2", "metadata2"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + metadata_map = {{"consume", "consume"}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + Http::TestHeaderMapImpl request_trailers{{"trailer", "trailer"}}; + codec_client_->sendTrailers(*request_encoder_, request_trailers); + waitForNextUpstreamRequest(); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + std::set expected_metadata_keys = {"headers", "data", "metadata", "metadata1", + "metadata2", "replace", "trailers"}; + verifyExpectedMetadata(upstream_request_->metadata_map(), expected_metadata_keys); + EXPECT_EQ(upstream_request_->duplicated_metadata_key_count().find("metadata")->second, 6); +} + +static std::string metadata_stop_all_filter = R"EOF( +name: metadata-stop-all-filter +config: {} +)EOF"; + +TEST_P(Http2MetadataIntegrationTest, RequestMetadataWithStopAllFilterBeforeMetadataFilter) { + addFilters({request_metadata_filter, metadata_stop_all_filter}); + testRequestMetadataWithStopAllFilter(); +} + +TEST_P(Http2MetadataIntegrationTest, RequestMetadataWithStopAllFilterAfterMetadataFilter) { + addFilters({metadata_stop_all_filter, request_metadata_filter}); + testRequestMetadataWithStopAllFilter(); +} + TEST_P(Http2IntegrationTest, GrpcRouterNotFound) { config_helper_.setDefaultHostAndRoute("foo.com", "/found"); initialize(); diff --git a/test/integration/http2_integration_test.h b/test/integration/http2_integration_test.h index c910d6e78cbd..efa53788f2bd 100644 --- a/test/integration/http2_integration_test.h +++ b/test/integration/http2_integration_test.h @@ -13,6 +13,14 @@ class Http2IntegrationTest : public testing::TestWithParam filters) { + for (const auto& filter : filters) { + config_helper_.addFilter(filter); + } + } }; class Http2RingHashIntegrationTest : public Http2IntegrationTest { @@ -45,5 +53,11 @@ class Http2MetadataIntegrationTest : public Http2IntegrationTest { setDownstreamProtocol(Http::CodecClient::Type::HTTP2); setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); } + + void testRequestMetadataWithStopAllFilter(); + + void verifyHeadersOnlyTest(); + + void runHeaderOnlyTest(bool send_request_body, size_t body_size); }; } // namespace Envoy diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index e4f16603652e..fbd7308fd4a6 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -132,6 +132,15 @@ void IntegrationCodecClient::sendReset(Http::StreamEncoder& encoder) { flushWrite(); } +void IntegrationCodecClient::sendMetadata(Http::StreamEncoder& encoder, + Http::MetadataMap metadata_map) { + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + Http::MetadataMapVector metadata_map_vector; + metadata_map_vector.push_back(std::move(metadata_map_ptr)); + encoder.encodeMetadata(metadata_map_vector); + flushWrite(); +} + std::pair IntegrationCodecClient::startRequest(const Http::HeaderMap& headers) { auto response = std::make_unique(dispatcher_); @@ -300,6 +309,7 @@ HttpIntegrationTest::waitForNextUpstreamRequest(const std::vector& ups uint64_t upstream_with_request; // If there is no upstream connection, wait for it to be established. if (!fake_upstream_connection_) { + AssertionResult result = AssertionFailure(); for (auto upstream_index : upstream_indices) { result = fake_upstreams_[upstream_index]->waitForHttpConnection( @@ -327,12 +337,6 @@ void HttpIntegrationTest::waitForNextUpstreamRequest(uint64_t upstream_index) { waitForNextUpstreamRequest(std::vector({upstream_index})); } -void HttpIntegrationTest::addFilters(std::vector filters) { - for (const auto& filter : filters) { - config_helper_.addFilter(filter); - } -} - void HttpIntegrationTest::checkSimpleRequestSuccess(uint64_t expected_request_size, uint64_t expected_response_size, IntegrationStreamDecoder* response) { diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index 5db6086cf572..2df23c0462d0 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -34,12 +34,15 @@ class IntegrationCodecClient : public Http::CodecClientProd { void sendData(Http::StreamEncoder& encoder, uint64_t size, bool end_stream); void sendTrailers(Http::StreamEncoder& encoder, const Http::HeaderMap& trailers); void sendReset(Http::StreamEncoder& encoder); + // Intentionally makes a copy of metadata_map. + void sendMetadata(Http::StreamEncoder& encoder, Http::MetadataMap metadata_map); std::pair startRequest(const Http::HeaderMap& headers); bool waitForDisconnect(std::chrono::milliseconds time_to_wait = std::chrono::milliseconds(0)); Network::ClientConnection* connection() const { return connection_.get(); } Network::ConnectionEvent last_connection_event() const { return last_connection_event_; } Network::Connection& rawConnection() { return *connection_; } + bool disconnected() { return disconnected_; } private: struct ConnectionCallbacks : public Network::ConnectionCallbacks { @@ -135,9 +138,6 @@ class HttpIntegrationTest : public BaseIntegrationTest { // Close |codec_client_| and |fake_upstream_connection_| cleanly. void cleanupUpstreamAndDownstream(); - // Utility function to add filters. - void addFilters(std::vector filters); - // Check for completion of upstream_request_, and a simple "200" response. void checkSimpleRequestSuccess(uint64_t expected_request_size, uint64_t expected_response_size, IntegrationStreamDecoder* response); diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 64ebef385c5f..a3a3f53768c9 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -172,6 +172,7 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, MOCK_METHOD2(addDecodedData, void(Buffer::Instance& data, bool streaming)); MOCK_METHOD2(injectDecodedDataToFilterChain, void(Buffer::Instance& data, bool end_stream)); MOCK_METHOD0(addDecodedTrailers, HeaderMap&()); + MOCK_METHOD0(addDecodedMetadata, MetadataMapVector&()); MOCK_METHOD0(decodingBuffer, const Buffer::Instance*()); MOCK_METHOD1(modifyDecodingBuffer, void(std::function)); MOCK_METHOD1(encode100ContinueHeaders_, void(HeaderMap& headers)); @@ -241,6 +242,7 @@ class MockStreamDecoderFilter : public StreamDecoderFilter { MOCK_METHOD2(decodeHeaders, FilterHeadersStatus(HeaderMap& headers, bool end_stream)); MOCK_METHOD2(decodeData, FilterDataStatus(Buffer::Instance& data, bool end_stream)); MOCK_METHOD1(decodeTrailers, FilterTrailersStatus(HeaderMap& trailers)); + MOCK_METHOD1(decodeMetadata, FilterMetadataStatus(Http::MetadataMap& metadata_map)); MOCK_METHOD1(setDecoderFilterCallbacks, void(StreamDecoderFilterCallbacks& callbacks)); MOCK_METHOD0(decodeComplete, void()); @@ -279,6 +281,7 @@ class MockStreamFilter : public StreamFilter { MOCK_METHOD2(decodeHeaders, FilterHeadersStatus(HeaderMap& headers, bool end_stream)); MOCK_METHOD2(decodeData, FilterDataStatus(Buffer::Instance& data, bool end_stream)); MOCK_METHOD1(decodeTrailers, FilterTrailersStatus(HeaderMap& trailers)); + MOCK_METHOD1(decodeMetadata, FilterMetadataStatus(Http::MetadataMap& metadata_map)); MOCK_METHOD1(setDecoderFilterCallbacks, void(StreamDecoderFilterCallbacks& callbacks)); // Http::MockStreamEncoderFilter diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index c72496089e7e..274f4dfcd524 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -557,6 +557,8 @@ TEST_P(AdminFilterTest, Body) { EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(request_headers_, false)); Buffer::OwnedImpl data("hello"); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_.decodeMetadata(metadata_map)); EXPECT_CALL(callbacks_, addDecodedData(_, false)); EXPECT_CALL(callbacks_, encodeHeaders_(_, false)); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_.decodeData(data, true)); From 964ee9111845d6de45b8bb6081dce9e0a4b4eb24 Mon Sep 17 00:00:00 2001 From: Scott LaVigne <1406859+lavignes@users.noreply.github.com> Date: Thu, 11 Jul 2019 13:26:39 -0700 Subject: [PATCH 156/542] Remove exclusive tag from aws_metadata_fetcher_integration_test (#7545) Resolves feedback from #7303 Signed-off-by: LaVigne, Scott --- test/integration/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/BUILD b/test/integration/BUILD index 2bf9113fa201..708030bc66fb 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -819,7 +819,6 @@ envoy_cc_test( srcs = [ "aws_metadata_fetcher_integration_test.cc", ], - tags = ["exclusive"], deps = [ ":integration_lib", "//source/common/common:fmt_lib", From 74e348720af8d157bbdb00b629ac7c6022606fc8 Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Thu, 11 Jul 2019 23:27:19 +0300 Subject: [PATCH 157/542] bazel: adapt cc_wraper.py to python3 (#7519) In Arch and Clear linux /usr/bin/python points to python3 causing build failures due to type mismatch in os.write(): string is used where bytestring is expected. Explicitly convert string to bytestring. Risk Level: low Testing: unit tests Release Notes: N/A Documentation: N/A Signed-off-by: Dmitry Rozhkov --- bazel/cc_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bazel/cc_wrapper.py b/bazel/cc_wrapper.py index 53037e67bd2e..41029a05eb4b 100755 --- a/bazel/cc_wrapper.py +++ b/bazel/cc_wrapper.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import contextlib import os import shlex @@ -112,7 +112,7 @@ def main(): with closing_fd(new_flagfile_fd): for arg in new_driver_args: - os.write(new_flagfile_fd, arg + "\n") + os.write(new_flagfile_fd, bytes(str(arg + "\n").encode("utf-8"))) # Provide new arguments using the temp file containing the args new_args = ["@" + new_flagfile_path] From fcec70fc3a798013a293ed080bd7e836f7edbb7b Mon Sep 17 00:00:00 2001 From: Paul Banks Date: Fri, 12 Jul 2019 14:34:02 +0200 Subject: [PATCH 158/542] Allow /dev/fd/ paths for config files. Fixes #7258 (#7279) As noted in #7528 the newly added sanity checking of possibly sensitive file paths prevents legitimate usage of passing bootstrap via a non-CLOEXEC file descriptor from a generator helper that execs Envoy. This PR relaxes the validation such that any path resolving to a canonical path with the prefix /dev/fd/ is considered valid. Risk Level: Low Testing: Unit test case is added that was failing before the change and passes afterwards. In addition I've manually verified that the old behaviour of allowing /dev/fd/ paths works with my dev binary. Fixes #7258 Signed-off-by: Paul Banks --- source/common/filesystem/posix/filesystem_impl.cc | 11 +++++++++++ test/common/filesystem/filesystem_impl_test.cc | 2 ++ tools/spelling_dictionary.txt | 2 ++ 3 files changed, 15 insertions(+) diff --git a/source/common/filesystem/posix/filesystem_impl.cc b/source/common/filesystem/posix/filesystem_impl.cc index 214d97aed324..5d1b3b03eb8d 100644 --- a/source/common/filesystem/posix/filesystem_impl.cc +++ b/source/common/filesystem/posix/filesystem_impl.cc @@ -87,6 +87,17 @@ std::string InstanceImplPosix::fileReadToEnd(const std::string& path) { } bool InstanceImplPosix::illegalPath(const std::string& path) { + // Special case, allow /dev/fd/* access here so that config can be passed in a + // file descriptor from an execing bootstrap script. The reason we do this + // _before_ canonicalizing the path is that different unix flavors implement + // /dev/fd/* differently, for example on linux they are symlinks to /dev/pts/* + // which are symlinks to /proc/self/fds/. On BSD (and darwin) they are not + // symlinks at all. To avoid lots of platform, specifics, we whitelist + // /dev/fd/* _before_ resolving the canonical path. + if (absl::StartsWith(path, "/dev/fd/")) { + return false; + } + const Api::SysCallStringResult canonical_path = canonicalPath(path); if (canonical_path.rc_.empty()) { ENVOY_LOG_MISC(debug, "Unable to determine canonical path for {}: {}", path, diff --git a/test/common/filesystem/filesystem_impl_test.cc b/test/common/filesystem/filesystem_impl_test.cc index 47a657d220a8..f5576e822b65 100644 --- a/test/common/filesystem/filesystem_impl_test.cc +++ b/test/common/filesystem/filesystem_impl_test.cc @@ -131,6 +131,8 @@ TEST_F(FileSystemImplTest, IllegalPath) { #else EXPECT_TRUE(file_system_.illegalPath("/dev")); EXPECT_TRUE(file_system_.illegalPath("/dev/")); + // Exception to allow opening from file descriptors. See #7258. + EXPECT_FALSE(file_system_.illegalPath("/dev/fd/0")); EXPECT_TRUE(file_system_.illegalPath("/proc")); EXPECT_TRUE(file_system_.illegalPath("/proc/")); EXPECT_TRUE(file_system_.illegalPath("/sys")); diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 1218eefbb186..f7e6dfa0bbe6 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -649,6 +649,7 @@ pthreads pton ptr ptrs +pts pwd py qps @@ -736,6 +737,7 @@ structs subexpr subdirs symlink +symlinks symlinked sync sys From 23d82b9d14a6cf9f49ebcd3ae584fe3079f597d1 Mon Sep 17 00:00:00 2001 From: "Tejasvi (Teju) Nareddy" Date: Fri, 12 Jul 2019 06:11:37 -0700 Subject: [PATCH 159/542] api filter http: add build rules for go protobufs (#7526) All http filters have build rules to generate cc protobufs, but not go protobufs. Added build rules (to a few filters) to generate go protobuf files. Emulates the rules in the health_check http filter. Risk Level: Low Testing: These rules were copied to google3 and tested internally. Unfortunately, I am having a bit of trouble with bazel build directly on these targets ("Package is considered deleted due to --deleted_packages"). Please let me know if there is a better way to test this change. Signed-off-by: Teju Nareddy --- api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD | 12 +++++++++++- api/envoy/config/filter/http/router/v2/BUILD | 8 +++++++- api/envoy/config/filter/http/transcoder/v2/BUILD | 7 ++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD b/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD index d637732d32cb..e48aa582676c 100644 --- a/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD +++ b/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD @@ -1,6 +1,6 @@ licenses(["notice"]) # Apache 2 -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") api_proto_library_internal( name = "jwt_authn", @@ -11,3 +11,13 @@ api_proto_library_internal( "//envoy/api/v2/route", ], ) + +api_go_proto_library( + name = "jwt_authn", + proto = ":jwt_authn", + deps = [ + "//envoy/api/v2/core:base_go_proto", + "//envoy/api/v2/core:http_uri_go_proto", + "//envoy/api/v2/route:route_go_proto", + ], +) diff --git a/api/envoy/config/filter/http/router/v2/BUILD b/api/envoy/config/filter/http/router/v2/BUILD index 990d8154afad..7a80299a2cf7 100644 --- a/api/envoy/config/filter/http/router/v2/BUILD +++ b/api/envoy/config/filter/http/router/v2/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") licenses(["notice"]) # Apache 2 @@ -7,3 +7,9 @@ api_proto_library_internal( srcs = ["router.proto"], deps = ["//envoy/config/filter/accesslog/v2:accesslog"], ) + +api_go_proto_library( + name = "router", + proto = ":router", + deps = ["//envoy/config/filter/accesslog/v2:accesslog_go_proto"], +) diff --git a/api/envoy/config/filter/http/transcoder/v2/BUILD b/api/envoy/config/filter/http/transcoder/v2/BUILD index 8ecd7759a5ca..c1a845bcd96e 100644 --- a/api/envoy/config/filter/http/transcoder/v2/BUILD +++ b/api/envoy/config/filter/http/transcoder/v2/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") licenses(["notice"]) # Apache 2 @@ -6,3 +6,8 @@ api_proto_library_internal( name = "transcoder", srcs = ["transcoder.proto"], ) + +api_go_proto_library( + name = "transcoder", + proto = ":transcoder", +) From 7ef20d7609fb6f570a058fcf4b4e000922d7eeba Mon Sep 17 00:00:00 2001 From: htuch Date: Fri, 12 Jul 2019 09:22:06 -0400 Subject: [PATCH 160/542] api: clarify non-empty LDS/CDS resource hints. (#7512) In order to better support clients such as gRPC-LB that want to access only a single listener/cluster, provide the scope in the xDS specification to specify explicit resource hints. Signed-off-by: Harvey Tuch --- api/envoy/api/v2/discovery.proto | 8 ++++---- api/xds_protocol.rst | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index fb95252a9aec..2bbe53cfe24f 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -35,10 +35,10 @@ message DiscoveryRequest { // List of resources to subscribe to, e.g. list of cluster names or a route // configuration name. If this is empty, all resources for the API are - // returned. LDS/CDS expect empty resource_names, since this is global - // discovery for the Envoy instance. The LDS and CDS responses will then imply - // a number of resources that need to be fetched via EDS/RDS, which will be - // explicitly enumerated in resource_names. + // returned. LDS/CDS may have empty resource_names, which will cause all + // resources for the Envoy instance to be returned. The LDS and CDS responses + // will then imply a number of resources that need to be fetched via EDS/RDS, + // which will be explicitly enumerated in resource_names. repeated string resource_names = 3; // Type of the resource that is being requested, e.g. diff --git a/api/xds_protocol.rst b/api/xds_protocol.rst index 515c86343836..4cbda0a72663 100644 --- a/api/xds_protocol.rst +++ b/api/xds_protocol.rst @@ -160,8 +160,8 @@ Resource hints ^^^^^^^^^^^^^^ The :ref:`resource_names ` specified in the :ref:`DiscoveryRequest ` are a hint. -Some resource types, e.g. `Clusters` and `Listeners` will -specify an empty :ref:`resource_names ` list, since Envoy is interested in +Some resource types, e.g. `Clusters` and `Listeners` may +specify an empty :ref:`resource_names ` list, since a client such as Envoy is interested in learning about all the :ref:`Clusters (CDS) ` and :ref:`Listeners (LDS) ` that the management server(s) know about corresponding to its node identification. Other resource types, e.g. :ref:`RouteConfiguration (RDS) ` @@ -169,10 +169,11 @@ and :ref:`ClusterLoadAssignment (EDS) `, fo CDS/LDS updates and Envoy is able to explicitly enumerate these resources. -LDS/CDS resource hints will always be empty and it is expected that the -management server will provide the complete state of the LDS/CDS -resources in each response. An absent `Listener` or `Cluster` will -be deleted. +Envoy will always set the LDS/CDS resource hints to empty and it is expected that the management +server will provide the complete state of the LDS/CDS resources in each response. An absent +`Listener` or `Cluster` will be deleted. Other xDS clients may specify explicit LDS/CDS resources as +resource hints, for example if they only have a singleton listener and already know its name from +some out-of-band configuration. For EDS/RDS, the management server does not need to supply every requested resource and may also supply additional, unrequested From e591a13ef5dfd15c60fb20514db80383c4e627f0 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Fri, 12 Jul 2019 12:49:20 -0700 Subject: [PATCH 161/542] filterchain: use span to avoid construct vector (#7523) Signed-off-by: Yuchen Dai --- source/server/filter_chain_manager_impl.cc | 59 ++++++++++------------ source/server/filter_chain_manager_impl.h | 31 +++++++----- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/source/server/filter_chain_manager_impl.cc b/source/server/filter_chain_manager_impl.cc index a412c21c8ccf..44306d43f21c 100644 --- a/source/server/filter_chain_manager_impl.cc +++ b/source/server/filter_chain_manager_impl.cc @@ -72,19 +72,12 @@ void FilterChainManagerImpl::addFilterChain( } } - std::vector server_names(filter_chain_match.server_names().begin(), - filter_chain_match.server_names().end()); - - std::vector application_protocols( - filter_chain_match.application_protocols().begin(), - filter_chain_match.application_protocols().end()); - - // TODO(silentdai): use absl::Span to avoid vector construction at server_names and alpn addFilterChainForDestinationPorts( destination_ports_map_, PROTOBUF_GET_WRAPPED_OR_DEFAULT(filter_chain_match, destination_port, 0), destination_ips, - server_names, filter_chain_match.transport_protocol(), application_protocols, - filter_chain_match.source_type(), source_ips, filter_chain_match.source_ports(), + filter_chain_match.server_names(), filter_chain_match.transport_protocol(), + filter_chain_match.application_protocols(), filter_chain_match.source_type(), source_ips, + filter_chain_match.source_ports(), std::shared_ptr( filter_chain_factory_builder.buildFilterChain(*filter_chain))); } @@ -93,11 +86,12 @@ void FilterChainManagerImpl::addFilterChain( void FilterChainManagerImpl::addFilterChainForDestinationPorts( DestinationPortsMap& destination_ports_map, uint16_t destination_port, - const std::vector& destination_ips, const std::vector& server_names, - const std::string& transport_protocol, const std::vector& application_protocols, + const std::vector& destination_ips, + const absl::Span server_names, const std::string& transport_protocol, + const absl::Span application_protocols, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain) { if (destination_ports_map.find(destination_port) == destination_ports_map.end()) { destination_ports_map[destination_port] = @@ -110,11 +104,11 @@ void FilterChainManagerImpl::addFilterChainForDestinationPorts( void FilterChainManagerImpl::addFilterChainForDestinationIPs( DestinationIPsMap& destination_ips_map, const std::vector& destination_ips, - const std::vector& server_names, const std::string& transport_protocol, - const std::vector& application_protocols, + const absl::Span server_names, const std::string& transport_protocol, + const absl::Span application_protocols, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain) { if (destination_ips.empty()) { addFilterChainForServerNames(destination_ips_map[EMPTY_STRING], server_names, @@ -130,11 +124,12 @@ void FilterChainManagerImpl::addFilterChainForDestinationIPs( } void FilterChainManagerImpl::addFilterChainForServerNames( - ServerNamesMapSharedPtr& server_names_map_ptr, const std::vector& server_names, - const std::string& transport_protocol, const std::vector& application_protocols, + ServerNamesMapSharedPtr& server_names_map_ptr, + const absl::Span server_names, const std::string& transport_protocol, + const absl::Span application_protocols, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain) { if (server_names_map_ptr == nullptr) { server_names_map_ptr = std::make_shared(); @@ -146,16 +141,16 @@ void FilterChainManagerImpl::addFilterChainForServerNames( application_protocols, source_type, source_ips, source_ports, filter_chain); } else { - for (const auto& server_name : server_names) { - if (isWildcardServerName(server_name)) { + for (const auto& server_name_ptr : server_names) { + if (isWildcardServerName(*server_name_ptr)) { // Add mapping for the wildcard domain, i.e. ".example.com" for "*.example.com". addFilterChainForApplicationProtocols( - server_names_map[server_name.substr(1)][transport_protocol], application_protocols, + server_names_map[server_name_ptr->substr(1)][transport_protocol], application_protocols, source_type, source_ips, source_ports, filter_chain); } else { - addFilterChainForApplicationProtocols(server_names_map[server_name][transport_protocol], - application_protocols, source_type, source_ips, - source_ports, filter_chain); + addFilterChainForApplicationProtocols( + server_names_map[*server_name_ptr][transport_protocol], application_protocols, + source_type, source_ips, source_ports, filter_chain); } } } @@ -163,18 +158,18 @@ void FilterChainManagerImpl::addFilterChainForServerNames( void FilterChainManagerImpl::addFilterChainForApplicationProtocols( ApplicationProtocolsMap& application_protocols_map, - const std::vector& application_protocols, + const absl::Span application_protocols, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain) { if (application_protocols.empty()) { addFilterChainForSourceTypes(application_protocols_map[EMPTY_STRING], source_type, source_ips, source_ports, filter_chain); } else { - for (const auto& application_protocol : application_protocols) { - addFilterChainForSourceTypes(application_protocols_map[application_protocol], source_type, - source_ips, source_ports, filter_chain); + for (const auto& application_protocol_ptr : application_protocols) { + addFilterChainForSourceTypes(application_protocols_map[*application_protocol_ptr], + source_type, source_ips, source_ports, filter_chain); } } } @@ -183,7 +178,7 @@ void FilterChainManagerImpl::addFilterChainForSourceTypes( SourceTypesArray& source_types_array, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain) { if (source_ips.empty()) { addFilterChainForSourceIPs(source_types_array[source_type].first, EMPTY_STRING, source_ports, @@ -198,7 +193,7 @@ void FilterChainManagerImpl::addFilterChainForSourceTypes( void FilterChainManagerImpl::addFilterChainForSourceIPs( SourceIPsMap& source_ips_map, const std::string& source_ip, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain) { if (source_ports.empty()) { addFilterChainForSourcePorts(source_ips_map[source_ip], 0, filter_chain); diff --git a/source/server/filter_chain_manager_impl.h b/source/server/filter_chain_manager_impl.h index 34175170aa76..6a06bcc7c6a0 100644 --- a/source/server/filter_chain_manager_impl.h +++ b/source/server/filter_chain_manager_impl.h @@ -62,42 +62,47 @@ class FilterChainManagerImpl : public Network::FilterChainManager, void addFilterChainForDestinationPorts( DestinationPortsMap& destination_ports_map, uint16_t destination_port, - const std::vector& destination_ips, const std::vector& server_names, - const std::string& transport_protocol, const std::vector& application_protocols, + const std::vector& destination_ips, + const absl::Span server_names, + const std::string& transport_protocol, + const absl::Span application_protocols, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain); void addFilterChainForDestinationIPs( DestinationIPsMap& destination_ips_map, const std::vector& destination_ips, - const std::vector& server_names, const std::string& transport_protocol, - const std::vector& application_protocols, + const absl::Span server_names, + const std::string& transport_protocol, + const absl::Span application_protocols, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain); void addFilterChainForServerNames( - ServerNamesMapSharedPtr& server_names_map_ptr, const std::vector& server_names, - const std::string& transport_protocol, const std::vector& application_protocols, + ServerNamesMapSharedPtr& server_names_map_ptr, + const absl::Span server_names, + const std::string& transport_protocol, + const absl::Span application_protocols, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain); void addFilterChainForApplicationProtocols( ApplicationProtocolsMap& application_protocol_map, - const std::vector& application_protocols, + const absl::Span application_protocols, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain); void addFilterChainForSourceTypes( SourceTypesArray& source_types_array, const envoy::api::v2::listener::FilterChainMatch_ConnectionSourceType source_type, const std::vector& source_ips, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain); void addFilterChainForSourceIPs(SourceIPsMap& source_ips_map, const std::string& source_ip, - const Protobuf::RepeatedField& source_ports, + const absl::Span source_ports, const Network::FilterChainSharedPtr& filter_chain); void addFilterChainForSourcePorts(SourcePortsMapSharedPtr& source_ports_map_ptr, uint32_t source_port, From 1a2a2359251c3cb2ebcdce09b542286ebd6ec5a5 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 12 Jul 2019 14:43:34 -0700 Subject: [PATCH 162/542] coverage: ignore generated file precisely (#7551) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Description: Regression in #6866. k8-dbg/bin is excluded to ignore generated files but virtual_includes are there too. Risk Level: Low Testing: CI 🙏 Docs Changes: Release Notes: Fixes #7474 Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 12632254f1f4..6c1d22de56b6 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -77,7 +77,7 @@ BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} -c dbg --copt=-DNDEBUG" # stats. The #foo# pattern is because gcov produces files such as # bazel-out#local-fastbuild#bin#external#spdlog_git#_virtual_includes#spdlog#spdlog#details#pattern_formatter_impl.h.gcov. # To find these while modifying this regex, perform a gcov run with -k set. -[[ -z "${GCOVR_EXCLUDE_REGEX}" ]] && GCOVR_EXCLUDE_REGEX=".*pb.h.gcov|.*#k8-dbg#bin#.*|test#.*|external#.*|.*#external#.*|.*#prebuilt#.*|.*#config_validation#.*|.*#chromium_url#.*" +[[ -z "${GCOVR_EXCLUDE_REGEX}" ]] && GCOVR_EXCLUDE_REGEX=".*pb\\..*|.*#test#.*|external#.*|.*#external#.*|.*#prebuilt#.*|.*#config_validation#.*|.*#chromium_url#.*" [[ -z "${GCOVR_EXCLUDE_DIR}" ]] && GCOVR_EXCLUDE_DIR=".*/external/.*" COVERAGE_DIR="${SRCDIR}"/generated/coverage From 258adf10b70f4af5b4d5bd68b9003eace4f1eff9 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 12 Jul 2019 15:01:34 -0700 Subject: [PATCH 163/542] remove unnecessary Heap::forceLink (#7553) Description: Heap::forceLink is no longer needed as admin endpoint is added. Improves coverage a bit. Risk Level: Low Testing: CI Docs Changes: N/A Release Notes: N/A Signed-off-by: Lizan Zhou --- bazel/PPROF.md | 2 +- source/common/profiler/profiler.cc | 7 ------- source/common/profiler/profiler.h | 3 --- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/bazel/PPROF.md b/bazel/PPROF.md index 8345eedb9fce..443665785ae2 100644 --- a/bazel/PPROF.md +++ b/bazel/PPROF.md @@ -10,7 +10,7 @@ specific place yourself. Static linking is already available (because of a `HeapProfilerDump()` call inside -[`Envoy::Profiler::Heap::forceLink()`](https://github.com/envoyproxy/envoy/blob/master/source/common/profiler/profiler.cc#L21-L26)). +[`Envoy::Profiler::Heap::stopProfiler())`](https://github.com/envoyproxy/envoy/blob/master/source/common/profiler/profiler.cc#L32-L39)). ### Compiling a statically-linked Envoy diff --git a/source/common/profiler/profiler.cc b/source/common/profiler/profiler.cc index 74fb9478cba3..b3ef32ca832b 100644 --- a/source/common/profiler/profiler.cc +++ b/source/common/profiler/profiler.cc @@ -38,13 +38,6 @@ bool Heap::stopProfiler() { return true; } -void Heap::forceLink() { - // Currently this is here to force the inclusion of the heap profiler during static linking. - // Without this call the heap profiler will not be included and cannot be started via env - // variable. In the future we can add admin support. - HeapProfilerDump(""); -} - } // namespace Profiler } // namespace Envoy diff --git a/source/common/profiler/profiler.h b/source/common/profiler/profiler.h index d61ff851058c..fdf4b20ee8f9 100644 --- a/source/common/profiler/profiler.h +++ b/source/common/profiler/profiler.h @@ -60,9 +60,6 @@ class Heap { * @return bool whether the file is dumped */ static bool stopProfiler(); - -private: - static void forceLink(); }; } // namespace Profiler From 9370f5efa53ca7024296d559eced7ce4c8f45fdb Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 12 Jul 2019 15:15:06 -0700 Subject: [PATCH 164/542] forward proxy: per host SNI and SAN verification (#7466) 1) Implement auto SAN/SNI setting on a per-host basis. 2) Cleanup some v1 config translation. Part of https://github.com/envoyproxy/envoy/issues/1606 Signed-off-by: Matt Klein --- .../dynamic_forward_proxy_filter.rst | 10 +- .../intro/arch_overview/http/http_proxy.rst | 2 + include/envoy/network/transport_socket.h | 9 +- source/common/config/tls_context_json.cc | 14 --- source/common/config/tls_context_json.h | 9 -- .../network/transport_socket_options_impl.h | 10 +- source/common/upstream/logical_dns_cluster.cc | 4 +- source/common/upstream/logical_host.cc | 4 +- source/common/upstream/logical_host.h | 7 +- .../clusters/dynamic_forward_proxy/BUILD | 1 + .../clusters/dynamic_forward_proxy/cluster.cc | 34 ++++--- .../common/dynamic_forward_proxy/dns_cache.h | 6 ++ .../dynamic_forward_proxy/dns_cache_impl.cc | 22 ++--- .../dynamic_forward_proxy/dns_cache_impl.h | 11 ++- .../tls/context_config_impl.cc | 11 --- .../tls/context_config_impl.h | 3 - .../transport_sockets/tls/context_impl.cc | 30 ++++-- .../transport_sockets/tls/context_impl.h | 7 +- .../transport_sockets/tls/ssl_socket.cc | 9 +- .../transport_sockets/tls/ssl_socket.h | 3 +- test/config/integration/certs/README.md | 4 + test/config/integration/certs/certs.sh | 2 + .../integration/certs/upstreamcacert.pem | 43 ++++----- .../integration/certs/upstreamcakey.pem | 50 +++++----- .../config/integration/certs/upstreamcert.pem | 46 +++++----- .../integration/certs/upstreamcert_hash.h | 4 +- test/config/integration/certs/upstreamkey.pem | 50 +++++----- .../certs/upstreamlocalhostcert.cfg | 38 ++++++++ .../certs/upstreamlocalhostcert.pem | 25 +++++ .../certs/upstreamlocalhostcert_hash.h | 4 + .../certs/upstreamlocalhostkey.pem | 27 ++++++ .../dynamic_forward_proxy/cluster_test.cc | 1 + .../common/dynamic_forward_proxy/mocks.cc | 2 + .../common/dynamic_forward_proxy/mocks.h | 2 + .../filters/http/dynamic_forward_proxy/BUILD | 3 + .../proxy_filter_integration_test.cc | 91 ++++++++++++++++++- test/integration/xfcc_integration_test.cc | 16 ++-- 37 files changed, 412 insertions(+), 202 deletions(-) create mode 100644 test/config/integration/certs/upstreamlocalhostcert.cfg create mode 100644 test/config/integration/certs/upstreamlocalhostcert.pem create mode 100644 test/config/integration/certs/upstreamlocalhostcert_hash.h create mode 100644 test/config/integration/certs/upstreamlocalhostkey.pem diff --git a/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst b/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst index 80c569ae5a3e..a627ce3d2120 100644 --- a/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst +++ b/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst @@ -25,14 +25,12 @@ HTTP dynamic forward proxy. ` parameter has been set to true to allow Envoy to proxy absolute HTTP URLs. -.. attention:: +.. note:: - While configuring a :ref:`tls_context ` on the cluster with + Configuring a :ref:`tls_context ` on the cluster with *trusted_ca* certificates instructs Envoy to use TLS when connecting to upstream hosts and verify - the certificate chain, currently it is not possible to configure per-host TLS configuration - parameters including SNI, subject alt name verification, etc. This will be added in a future - change. **This means that the following configuration will not fully validate TLS certificates**. - Use with care until full support for per-host validation is implemented. + the certificate chain. Additionally, Envoy will automatically perform SAN verification for the + resolved host name as well as specify the host name via SNI. .. code-block:: yaml diff --git a/docs/root/intro/arch_overview/http/http_proxy.rst b/docs/root/intro/arch_overview/http/http_proxy.rst index 496feab8c570..2ed691203abb 100644 --- a/docs/root/intro/arch_overview/http/http_proxy.rst +++ b/docs/root/intro/arch_overview/http/http_proxy.rst @@ -27,6 +27,8 @@ follows: * A special load balancer will select the right host to use based on the HTTP host/authority header during forwarding. * Hosts that have not been used for a period of time are subject to a TTL that will purge them. +* When the upstream cluster has been configured with a TLS context, Envoy will automatically perform + SAN verification for the resolved host name as well as specify the host name via SNI. The above implementation details mean that at steady state Envoy can forward a large volume of HTTP proxy traffic while all DNS resolution happens asynchronously in the background. Additionally, diff --git a/include/envoy/network/transport_socket.h b/include/envoy/network/transport_socket.h index dab4a2054c31..1167f46e19d3 100644 --- a/include/envoy/network/transport_socket.h +++ b/include/envoy/network/transport_socket.h @@ -165,6 +165,12 @@ class TransportSocketOptions { */ virtual const absl::optional& serverNameOverride() const PURE; + /** + * @return the optional overridden SAN names to verify, if the transport socket supports SAN + * verification. + */ + virtual const std::vector& verifySubjectAltNameListOverride() const PURE; + /** * @param vector of bytes to which the option should append hash key data that will be used * to separate connections based on the option. Any data already in the key vector must @@ -173,7 +179,8 @@ class TransportSocketOptions { virtual void hashKey(std::vector& key) const PURE; }; -using TransportSocketOptionsSharedPtr = std::shared_ptr; +// TODO(mattklein123): Rename to TransportSocketOptionsConstSharedPtr in a dedicated follow up. +using TransportSocketOptionsSharedPtr = std::shared_ptr; /** * A factory for creating transport socket. It will be associated to filter chains and clusters. diff --git a/source/common/config/tls_context_json.cc b/source/common/config/tls_context_json.cc index 6072c55b2a77..fece59b47676 100644 --- a/source/common/config/tls_context_json.cc +++ b/source/common/config/tls_context_json.cc @@ -9,20 +9,6 @@ namespace Envoy { namespace Config { -void TlsContextJson::translateDownstreamTlsContext( - const Json::Object& json_tls_context, - envoy::api::v2::auth::DownstreamTlsContext& downstream_tls_context) { - translateCommonTlsContext(json_tls_context, *downstream_tls_context.mutable_common_tls_context()); - JSON_UTIL_SET_BOOL(json_tls_context, downstream_tls_context, require_client_certificate); - - const std::vector paths = - json_tls_context.getStringArray("session_ticket_key_paths", true); - for (const std::string& path : paths) { - downstream_tls_context.mutable_session_ticket_keys()->mutable_keys()->Add()->set_filename(path); - } - MessageUtil::validate(downstream_tls_context); -} - void TlsContextJson::translateUpstreamTlsContext( const Json::Object& json_tls_context, envoy::api::v2::auth::UpstreamTlsContext& upstream_tls_context) { diff --git a/source/common/config/tls_context_json.h b/source/common/config/tls_context_json.h index 53a1a14afe75..1eb9cbc1139d 100644 --- a/source/common/config/tls_context_json.h +++ b/source/common/config/tls_context_json.h @@ -8,15 +8,6 @@ namespace Config { class TlsContextJson { public: - /** - * Translate a v1 JSON TLS context to v2 envoy::api::v2::auth::DownstreamTlsContext. - * @param json_tls_context source v1 JSON TLS context object. - * @param downstream_tls_context destination v2 envoy::api::v2::Cluster. - */ - static void - translateDownstreamTlsContext(const Json::Object& json_tls_context, - envoy::api::v2::auth::DownstreamTlsContext& downstream_tls_context); - /** * Translate a v1 JSON TLS context to v2 envoy::api::v2::auth::UpstreamTlsContext. * @param json_tls_context source v1 JSON TLS context object. diff --git a/source/common/network/transport_socket_options_impl.h b/source/common/network/transport_socket_options_impl.h index ba544a67d765..0010a46d763a 100644 --- a/source/common/network/transport_socket_options_impl.h +++ b/source/common/network/transport_socket_options_impl.h @@ -7,19 +7,25 @@ namespace Network { class TransportSocketOptionsImpl : public TransportSocketOptions { public: - TransportSocketOptionsImpl(absl::string_view override_server_name = "") + TransportSocketOptionsImpl(absl::string_view override_server_name = "", + std::vector&& override_verify_san_list = {}) : override_server_name_(override_server_name.empty() ? absl::nullopt - : absl::optional(override_server_name)) {} + : absl::optional(override_server_name)), + override_verify_san_list_{std::move(override_verify_san_list)} {} // Network::TransportSocketOptions const absl::optional& serverNameOverride() const override { return override_server_name_; } + const std::vector& verifySubjectAltNameListOverride() const override { + return override_verify_san_list_; + } void hashKey(std::vector& key) const override; private: const absl::optional override_server_name_; + const std::vector override_verify_san_list_; }; } // namespace Network diff --git a/source/common/upstream/logical_dns_cluster.cc b/source/common/upstream/logical_dns_cluster.cc index 88ce5e02da5b..06c1e415b845 100644 --- a/source/common/upstream/logical_dns_cluster.cc +++ b/source/common/upstream/logical_dns_cluster.cc @@ -89,8 +89,8 @@ void LogicalDnsCluster::startResolve() { } if (!logical_host_) { - logical_host_.reset( - new LogicalHost(info_, hostname_, new_address, localityLbEndpoint(), lbEndpoint())); + logical_host_.reset(new LogicalHost(info_, hostname_, new_address, localityLbEndpoint(), + lbEndpoint(), nullptr)); const auto& locality_lb_endpoint = localityLbEndpoint(); PriorityStateManager priority_state_manager(*this, local_info_, nullptr); diff --git a/source/common/upstream/logical_host.cc b/source/common/upstream/logical_host.cc index a6c9c20ede4c..a196e8fbd234 100644 --- a/source/common/upstream/logical_host.cc +++ b/source/common/upstream/logical_host.cc @@ -8,7 +8,9 @@ Upstream::Host::CreateConnectionData LogicalHost::createConnection( Network::TransportSocketOptionsSharedPtr transport_socket_options) const { const auto current_address = address(); return {HostImpl::createConnection(dispatcher, cluster(), current_address, options, - transport_socket_options), + override_transport_socket_options_ != nullptr + ? override_transport_socket_options_ + : transport_socket_options), std::make_shared(current_address, shared_from_this())}; } diff --git a/source/common/upstream/logical_host.h b/source/common/upstream/logical_host.h index d22a415a2fcf..a1ed7eee2f9e 100644 --- a/source/common/upstream/logical_host.h +++ b/source/common/upstream/logical_host.h @@ -14,11 +14,13 @@ class LogicalHost : public HostImpl { LogicalHost(const ClusterInfoConstSharedPtr& cluster, const std::string& hostname, const Network::Address::InstanceConstSharedPtr& address, const envoy::api::v2::endpoint::LocalityLbEndpoints& locality_lb_endpoint, - const envoy::api::v2::endpoint::LbEndpoint& lb_endpoint) + const envoy::api::v2::endpoint::LbEndpoint& lb_endpoint, + const Network::TransportSocketOptionsSharedPtr& override_transport_socket_options) : HostImpl(cluster, hostname, address, lb_endpoint.metadata(), lb_endpoint.load_balancing_weight().value(), locality_lb_endpoint.locality(), lb_endpoint.endpoint().health_check_config(), locality_lb_endpoint.priority(), - lb_endpoint.health_status()) {} + lb_endpoint.health_status()), + override_transport_socket_options_(override_transport_socket_options) {} // Set the new address. Updates are typically rare so a R/W lock is used for address updates. // Note that the health check address update requires no lock to be held since it is only @@ -51,6 +53,7 @@ class LogicalHost : public HostImpl { } private: + const Network::TransportSocketOptionsSharedPtr override_transport_socket_options_; mutable absl::Mutex address_lock_; }; diff --git a/source/extensions/clusters/dynamic_forward_proxy/BUILD b/source/extensions/clusters/dynamic_forward_proxy/BUILD index 861a2da0a10b..51ba531f70b7 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/BUILD +++ b/source/extensions/clusters/dynamic_forward_proxy/BUILD @@ -13,6 +13,7 @@ envoy_cc_library( srcs = ["cluster.cc"], hdrs = ["cluster.h"], deps = [ + "//source/common/network:transport_socket_options_lib", "//source/common/upstream:cluster_factory_lib", "//source/common/upstream:logical_host_lib", "//source/extensions/clusters:well_known_names", diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index e67066f9a444..37bfa8f6f5b5 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -1,5 +1,7 @@ #include "extensions/clusters/dynamic_forward_proxy/cluster.h" +#include "common/network/transport_socket_options_impl.h" + #include "extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h" namespace Envoy { @@ -7,10 +9,6 @@ namespace Extensions { namespace Clusters { namespace DynamicForwardProxy { -// TODO(mattklein123): Make sure that the cluster's hosts display their host name in admin output. -// TODO(mattklein123): Allow customizing TLS on a per-host basis. For example, setting SNI and -// doing certificate validation. - Cluster::Cluster( const envoy::api::v2::Cluster& cluster, const envoy::config::cluster::dynamic_forward_proxy::v2alpha::ClusterConfig& config, @@ -58,14 +56,15 @@ void Cluster::onDnsHostAddOrUpdate( HostInfoMapSharedPtr current_map = getCurrentHostMap(); const auto host_map_it = current_map->find(host); if (host_map_it != current_map->end()) { - // If we only have an address change, we can do that swap inline without any other updates. The - // appropriate R/W locking is in place to allow this. The details of this locking are: + // If we only have an address change, we can do that swap inline without any other updates. + // The appropriate R/W locking is in place to allow this. The details of this locking are: // - Hosts are not thread local, they are global. // - We take a read lock when reading the address and a write lock when changing it. // - Address updates are very rare. - // - Address reads are only done when a connection is being made and a "real" host description - // is created or the host is queries via the admin endpoint. Both of these operations are - // relatively rare and the read lock is held for a short period of time. + // - Address reads are only done when a connection is being made and a "real" host + // description is created or the host is queried via the admin endpoint. Both of + // these operations are relatively rare and the read lock is held for a short period + // of time. // // TODO(mattklein123): Right now the dynamic forward proxy / DNS cache works similar to how // logical DNS works, meaning that we only store a single address per @@ -82,11 +81,20 @@ void Cluster::onDnsHostAddOrUpdate( } ENVOY_LOG(debug, "adding new dfproxy cluster host '{}'", host); + + // Create an override transport socket options that automatically provides both SNI as well as + // SAN verification for the resolved host if the cluster has been configured with TLS. + // TODO(mattklein123): If the host is an IP address we should not set SNI. + Network::TransportSocketOptionsSharedPtr transport_socket_options = + std::make_shared( + host_info->resolvedHost(), std::vector{host_info->resolvedHost()}); + const auto new_host_map = std::make_shared(*current_map); - const auto emplaced = new_host_map->try_emplace( - host, host_info, - std::make_shared(info(), host, host_info->address(), - dummy_locality_lb_endpoint_, dummy_lb_endpoint_)); + const auto emplaced = + new_host_map->try_emplace(host, host_info, + std::make_shared( + info(), host, host_info->address(), dummy_locality_lb_endpoint_, + dummy_lb_endpoint_, transport_socket_options)); Upstream::HostVector hosts_added; hosts_added.emplace_back(emplaced.first->second.logical_host_); diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache.h b/source/extensions/common/dynamic_forward_proxy/dns_cache.h index 96ee2d5a7473..3a0fb4580ef3 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache.h @@ -23,6 +23,12 @@ class DnsHostInfo { */ virtual Network::Address::InstanceConstSharedPtr address() PURE; + /** + * Returns the host that was actually resolved via DNS. If port was originally specified it will + * be stripped from this return value. + */ + virtual const std::string& resolvedHost() PURE; + /** * Indicates that the host has been used and should not be purged depending on any configured * TTL policy diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index b8173e2dfd00..0e9a978ff40f 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -132,12 +132,12 @@ void DnsCacheImpl::onReResolve(const std::string& host) { void DnsCacheImpl::startResolve(const std::string& host, PrimaryHostInfo& host_info) { ENVOY_LOG(debug, "starting main thread resolve for host='{}' dns='{}' port='{}'", host, - host_info.host_to_resolve_, host_info.port_); + host_info.host_info_->resolved_host_, host_info.port_); ASSERT(host_info.active_query_ == nullptr); stats_.dns_query_attempt_.inc(); host_info.active_query_ = - resolver_->resolve(host_info.host_to_resolve_, dns_lookup_family_, + resolver_->resolve(host_info.host_info_->resolved_host_, dns_lookup_family_, [this, host](std::list&& response) { finishResolve(host, std::move(response)); }); @@ -151,11 +151,8 @@ void DnsCacheImpl::finishResolve(const std::string& host, auto& primary_host_info = *primary_host_it->second; primary_host_info.active_query_ = nullptr; - const bool first_resolve = primary_host_info.host_info_ == nullptr; - if (primary_host_info.host_info_ == nullptr) { - primary_host_info.host_info_ = - std::make_shared(main_thread_dispatcher_.timeSource()); - } + const bool first_resolve = !primary_host_info.host_info_->first_resolve_complete_; + primary_host_info.host_info_->first_resolve_complete_ = true; const auto new_address = !response.empty() ? Network::Utility::getAddressWithPort(*(response.front().address_), @@ -211,9 +208,8 @@ void DnsCacheImpl::runRemoveCallbacks(const std::string& host) { void DnsCacheImpl::updateTlsHostsMap() { TlsHostMapSharedPtr new_host_map = std::make_shared(); for (const auto& primary_host : primary_hosts_) { - // Do not include hosts without host info. This only happens before we get the first - // resolution. - if (primary_host.second->host_info_ != nullptr) { + // Do not include hosts that have not resolved at least once. + if (primary_host.second->host_info_->first_resolve_complete_) { new_host_map->emplace(primary_host.first, primary_host.second->host_info_); } } @@ -249,8 +245,10 @@ void DnsCacheImpl::ThreadLocalHostInfo::updateHostMap(const TlsHostMapSharedPtr& DnsCacheImpl::PrimaryHostInfo::PrimaryHostInfo(DnsCacheImpl& parent, absl::string_view host_to_resolve, uint16_t port, const Event::TimerCb& timer_cb) - : parent_(parent), host_to_resolve_(host_to_resolve), port_(port), - refresh_timer_(parent.main_thread_dispatcher_.createTimer(timer_cb)) { + : parent_(parent), port_(port), + refresh_timer_(parent.main_thread_dispatcher_.createTimer(timer_cb)), + host_info_(std::make_shared(parent.main_thread_dispatcher_.timeSource(), + host_to_resolve)) { parent_.stats_.host_added_.inc(); parent_.stats_.num_hosts_.inc(); } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index 7ecccdf3a8d2..440617b61c70 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -71,13 +71,19 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable ContextImpl::parseAlpnProtocols(const std::string& alpn_pro return out; } -bssl::UniquePtr ContextImpl::newSsl(absl::optional) { +bssl::UniquePtr ContextImpl::newSsl(const Network::TransportSocketOptions*) { // We use the first certificate for a new SSL object, later in the // SSL_CTX_set_select_certificate_cb() callback following ClientHello, we replace with the // selected certificate via SSL_set_SSL_CTX(). @@ -419,12 +419,18 @@ int ContextImpl::verifyCallback(X509_STORE_CTX* store_ctx, void* arg) { SSL* ssl = reinterpret_cast( X509_STORE_CTX_get_ex_data(store_ctx, SSL_get_ex_data_X509_STORE_CTX_idx())); bssl::UniquePtr cert(SSL_get_peer_certificate(ssl)); - return impl->verifyCertificate(cert.get()); + + const Network::TransportSocketOptions* transport_socket_options = + static_cast(SSL_get_app_data(ssl)); + return impl->verifyCertificate( + cert.get(), transport_socket_options && + !transport_socket_options->verifySubjectAltNameListOverride().empty() + ? transport_socket_options->verifySubjectAltNameListOverride() + : impl->verify_subject_alt_name_list_); } -int ContextImpl::verifyCertificate(X509* cert) { - if (!verify_subject_alt_name_list_.empty() && - !verifySubjectAltName(cert, verify_subject_alt_name_list_)) { +int ContextImpl::verifyCertificate(X509* cert, const std::vector& verify_san_list) { + if (!verify_san_list.empty() && !verifySubjectAltName(cert, verify_san_list)) { stats_.fail_verify_san_.inc(); return 0; } @@ -664,17 +670,23 @@ ClientContextImpl::ClientContextImpl(Stats::Scope& scope, } } -bssl::UniquePtr ClientContextImpl::newSsl(absl::optional override_server_name) { - bssl::UniquePtr ssl_con(ContextImpl::newSsl(absl::nullopt)); +bssl::UniquePtr ClientContextImpl::newSsl(const Network::TransportSocketOptions* options) { + bssl::UniquePtr ssl_con(ContextImpl::newSsl(options)); - std::string server_name_indication = - override_server_name.has_value() ? override_server_name.value() : server_name_indication_; + const std::string server_name_indication = options && options->serverNameOverride().has_value() + ? options->serverNameOverride().value() + : server_name_indication_; if (!server_name_indication.empty()) { int rc = SSL_set_tlsext_host_name(ssl_con.get(), server_name_indication.c_str()); RELEASE_ASSERT(rc, ""); } + if (options && !options->verifySubjectAltNameListOverride().empty()) { + SSL_set_app_data(ssl_con.get(), options); + SSL_set_verify(ssl_con.get(), SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr); + } + if (allow_renegotiation_) { SSL_set_renegotiate_mode(ssl_con.get(), ssl_renegotiate_freely); } diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index 57c8ffbdec35..c4cf67d6cfa8 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -5,6 +5,7 @@ #include #include +#include "envoy/network/transport_socket.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" #include "envoy/stats/scope.h" @@ -46,7 +47,7 @@ struct SslStats { class ContextImpl : public virtual Envoy::Ssl::Context { public: - virtual bssl::UniquePtr newSsl(absl::optional override_server_name); + virtual bssl::UniquePtr newSsl(const Network::TransportSocketOptions* options); /** * Logs successful TLS handshake and updates stats. @@ -94,7 +95,7 @@ class ContextImpl : public virtual Envoy::Ssl::Context { // A SSL_CTX_set_cert_verify_callback for custom cert validation. static int verifyCallback(X509_STORE_CTX* store_ctx, void* arg); - int verifyCertificate(X509* cert); + int verifyCertificate(X509* cert, const std::vector& verify_san_list); /** * Verifies certificate hash for pinning. The hash is a hex-encoded SHA-256 of the DER-encoded @@ -168,7 +169,7 @@ class ClientContextImpl : public ContextImpl, public Envoy::Ssl::ClientContext { ClientContextImpl(Stats::Scope& scope, const Envoy::Ssl::ClientContextConfig& config, TimeSource& time_source); - bssl::UniquePtr newSsl(absl::optional override_server_name) override; + bssl::UniquePtr newSsl(const Network::TransportSocketOptions* options) override; private: int newSessionKey(SSL_SESSION* session); diff --git a/source/extensions/transport_sockets/tls/ssl_socket.cc b/source/extensions/transport_sockets/tls/ssl_socket.cc index 519495a1b76b..d6b63be7d091 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.cc +++ b/source/extensions/transport_sockets/tls/ssl_socket.cc @@ -43,11 +43,10 @@ class NotReadySslSocket : public Network::TransportSocket { } // namespace SslSocket::SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, - Network::TransportSocketOptionsSharedPtr transport_socket_options) - : ctx_(std::dynamic_pointer_cast(ctx)), - ssl_(ctx_->newSsl(transport_socket_options != nullptr - ? transport_socket_options->serverNameOverride() - : absl::nullopt)) { + const Network::TransportSocketOptionsSharedPtr& transport_socket_options) + : transport_socket_options_(transport_socket_options), + ctx_(std::dynamic_pointer_cast(ctx)), + ssl_(ctx_->newSsl(transport_socket_options_.get())) { if (state == InitialState::Client) { SSL_set_connect_state(ssl_.get()); } else { diff --git a/source/extensions/transport_sockets/tls/ssl_socket.h b/source/extensions/transport_sockets/tls/ssl_socket.h index 7b00ba406387..549ffbf83696 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.h +++ b/source/extensions/transport_sockets/tls/ssl_socket.h @@ -44,7 +44,7 @@ class SslSocket : public Network::TransportSocket, protected Logger::Loggable { public: SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, - Network::TransportSocketOptionsSharedPtr transport_socket_options); + const Network::TransportSocketOptionsSharedPtr& transport_socket_options); // Ssl::ConnectionInfo bool peerCertificatePresented() const override; @@ -90,6 +90,7 @@ class SslSocket : public Network::TransportSocket, void drainErrorQueue(); void shutdownSsl(); + const Network::TransportSocketOptionsSharedPtr transport_socket_options_; Network::TransportSocketCallbacks* callbacks_{}; ContextImplSharedPtr ctx_; bssl::UniquePtr ssl_; diff --git a/test/config/integration/certs/README.md b/test/config/integration/certs/README.md index 78eb2a677c2d..2f6dfabaff11 100644 --- a/test/config/integration/certs/README.md +++ b/test/config/integration/certs/README.md @@ -11,6 +11,10 @@ There are 5 identities: - **Upstream**: It has the certificate *upstreamcert.pem*, which is signed by the **Upstream CA** using the config *upstreamcert.cfg*. *upstreamkey.pem* is its private key. +- **Upstream localhost**: It has the certificate *upstreamlocalhostcert.pem*, which is signed by + the **Upstream CA** using the config *upstreamlocalhostcert.cfg*. *upstreamlocalhostkey.pem* is + its private key. The different between this certificate and **Upstream** is that this certifcate + has a SAN for "localhost". # How to update certificates **certs.sh** has the commands to generate all files. Running certs.sh directly diff --git a/test/config/integration/certs/certs.sh b/test/config/integration/certs/certs.sh index 8798fe9a5013..d67da84352da 100755 --- a/test/config/integration/certs/certs.sh +++ b/test/config/integration/certs/certs.sh @@ -52,6 +52,8 @@ generate_ca upstreamca # Generate cert for the upstream node. generate_rsa_key upstream upstreamca generate_x509_cert upstream upstreamca +generate_rsa_key upstreamlocalhost upstreamca +generate_x509_cert upstreamlocalhost upstreamca rm *.csr rm *.srl diff --git a/test/config/integration/certs/upstreamcacert.pem b/test/config/integration/certs/upstreamcacert.pem index b812dee4e3a2..c3d5692354bd 100644 --- a/test/config/integration/certs/upstreamcacert.pem +++ b/test/config/integration/certs/upstreamcacert.pem @@ -1,23 +1,24 @@ -----BEGIN CERTIFICATE----- -MIID5DCCAsygAwIBAgIJAK6F1p2EwsE8MA0GCSqGSIb3DQEBCwUAMH8xCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2luZWVyaW5nMRkw -FwYDVQQDDBBUZXN0IFVwc3RyZWFtIENBMB4XDTE4MTIxNzIwMTgwMFoXDTIwMTIx -NjIwMTgwMFowfzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAU -BgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5 -ZnQgRW5naW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDppJMdCm+G30EI7kqkk637fjk0IkIg -Q9ELZ1yAmJHq4HPXjhUmuKRlKlw0iZjAMAExBaUcZbDw/Wyr2MnpjvqxuQpKczAw -7oBrXYSRr/23XlvvRjbVKgg6DsftwiUDtyfX83FpKdB+w67rLAhDmtrqUUbkek+o -SKGEOBznY/0M0Q+XG1mToxOha64VxdSU6uucUeCr7AbECcK91raeCPVk27np2Pwg -VXL7SSAaLL27E7iIj3BZ6mI28QMjEIZ438fmUr4spmL07nUewfa5gckoubxWINVj -1jza5FvVZIJLiexZ1TkHiktZvSxEtoAjW0hKOQNrNonJlZHEhLk8lb1BAgMBAAGj -YzBhMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBSp -uxWfLuvEfPkiCROch7WbHSxYcDAfBgNVHSMEGDAWgBSpuxWfLuvEfPkiCROch7Wb -HSxYcDANBgkqhkiG9w0BAQsFAAOCAQEAubeccqQYQBoAdBqZpgoHbKehT+LwsrFC -BCj/c+j0+2Vv/MW5Oxycdu+UXXTC/8KiQBtNbr8O0yJRZsPuVjSoFi4GVvm0jKWP -qezQCpNrt8qutKPYUqimwqh2/OiJ+FWfB2RQZRxAf7eO5qECJcM2doG/4FCmT3HB -Qkd8fB22zF+bItLCYrWEVazdhLJiwtpIADbxOz6+gDcMuHiesS7FvZlmkihmIC2F -jaAp6NzmW11E50i9YG9i0mFZL6oQX9sOE4sEDtW+oTtWzsyXMwrdPZFYANaWBY+P -WItPPVu2jj/bntnbLuaI+vRifgJ9JPKIVpDlt3fzxCZxqEWCKOPb9w== +MIID7zCCAtegAwIBAgIUQygBeIE4nv9JGaDKixnhwkK5viEwDQYJKoZIhvcNAQEL +BQAwfzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n +aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMTkwNzA4MjE0 +NTU2WhcNMjEwNzA3MjE0NTU2WjB/MQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2Fs +aWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzENMAsGA1UECgwETHlmdDEZ +MBcGA1UECwwQTHlmdCBFbmdpbmVlcmluZzEZMBcGA1UEAwwQVGVzdCBVcHN0cmVh +bSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMJ7AetbhOCUxB/A +yYt+4rxyMVUFX9izqbOU9nuUxsB/avGhYpVjj5cNaLPdGX+c7g65Vz0yGDSskDGD +ukcSFqRSZ2E4/S4gKSIMEslBr2OX+Dqh0XmoAwl4IrtZefCE3inivJdzm0JwI7Yr +k2qQqsTpJnsWkMSxXUQJYTJ56UFXTkKqF3jSReIQtFMV65T/2x2NLRJ8KuMS7Mbo +BTBATRsUfbJJWCnzcp2LrKV5sZ/HsJLK/F74jdcvfJQMW49Lq1TZaB5NYSVyFEf6 +tiT43JOcvVkRPBgHDtaiDhWF2WTmPSEB6cHaRwGgBFwjQ1SvZR6f6xexocn44GZE +oSqWJN8CAwEAAaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYw +HQYDVR0OBBYEFOLTMLryzNAcuxe3cKEhClkL7IduMB8GA1UdIwQYMBaAFOLTMLry +zNAcuxe3cKEhClkL7IduMA0GCSqGSIb3DQEBCwUAA4IBAQBT88sT8RsoAk6PnnVs +KWBoC75BnIZr8o1nxBK0zog6Ez4J32aVzEXPicgBg6hf6v77eqbbQ+O7Ayf+YQWj +l9w9IiXRW1x94tKBrX44O85qTr/xtkfpmQWGKq5fBpdJnZp7lSfGfaG9gasPUNpG +gfvF/vlYrrJoyvUOG6HQjZ7n7m6f8GEUymCtC68oJcLVL0xkvx/jcvGeJfI5U6yr +z9nc1W7FcOhrFEetOIH2BwlIN5To3vPbN4zEzt9VPUHZ3m2899hUiMZJaanEexp7 +TZJJ12rHSIJ4MKwQQ5fEmioeluM0uY7EIR72VEsudA8bkXSkbDGs6Q49K9OX+nRB +4P3c -----END CERTIFICATE----- diff --git a/test/config/integration/certs/upstreamcakey.pem b/test/config/integration/certs/upstreamcakey.pem index c9f1c73c7cb8..2fe99b9c2c10 100644 --- a/test/config/integration/certs/upstreamcakey.pem +++ b/test/config/integration/certs/upstreamcakey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEA6aSTHQpvht9BCO5KpJOt+345NCJCIEPRC2dcgJiR6uBz144V -JrikZSpcNImYwDABMQWlHGWw8P1sq9jJ6Y76sbkKSnMwMO6Aa12Eka/9t15b70Y2 -1SoIOg7H7cIlA7cn1/NxaSnQfsOu6ywIQ5ra6lFG5HpPqEihhDgc52P9DNEPlxtZ -k6MToWuuFcXUlOrrnFHgq+wGxAnCvda2ngj1ZNu56dj8IFVy+0kgGiy9uxO4iI9w -WepiNvEDIxCGeN/H5lK+LKZi9O51HsH2uYHJKLm8ViDVY9Y82uRb1WSCS4nsWdU5 -B4pLWb0sRLaAI1tISjkDazaJyZWRxIS5PJW9QQIDAQABAoIBAExI9denjp6EymE2 -HJz7svTIU7kX7mtGeTy19Nfv+MStoGUi+Pj5lIOLfyuQOZWWlu0AoNZSxaEJva+m -Sta9XlEkz51bWsK/PKLRl/VRdw+l+XJ4hHK5FJKQPOr+VsONy66Qx9jEVFTvY29Z -oyEfsJaNDw6OeO/DNylKgPV0Ci1ifXQqlS3BAXGzWZ6qRCaSTnX6z7RDBQyojh5Q -eqIPLP8dtcs2sai93IWHGUCOata3SaGReOW/cfGgLW7x/1vLS/ywNSzkRI582Prt -w5+yQ39EAINvQlzUOvS+dWzDgpKnRkkh7BL+6v9zEH5uqzeuLLt68vQKLW5tUPwY -J6BdbjkCgYEA9u8Mbk0NLeLc6dO75/W0hxsgFsaLB4VELQLOo3u8c66XAWxT5mS8 -8K4c8eHXA+i1T/q4ngKBx2hYBWyArrARQVbswXuBebXqp8oOngtb5IOrVDtiSUtH -bXm5ZYX0xt3cv5F42e8XC8LiuA1CE9sMWQ6Z9kFNeKHZulPyBY7AkAsCgYEA8jia -iy3Rgqb6yIpx1OA3yr5KPw5VhSWtN6lLTLJwskHgdCR0u2OxI6C1Ok68JIZTZcrL -CzPTCaySQTkX0a91r4AO/QmolzCX/cNCsHPiz2cdMkxfY99vlllHXK5q1b8mvTpZ -S1goPGDrifLjf5yWkYNcEsA7seSJVnQsJu5yu2MCgYAmE/K8x5DytHsQa6AcQt1V -wC8QlAk4XaqHrlkjCJ+kzxVmGMhPTNV937uC6Sp45defv6/cXdKZZ1O7cmHdjjT6 -+GaF53+tvwmyWgwq/uFquYsf8BBV8Q/Qp+aY6zE1wVybBdm28ZGCNMk1TIYV/b9H -tGK1gJhrs7mZa/x0MvEqxQKBgFRDisfmTZ9lFZNUTltfESmv30ZmZyvluofFllN9 -NCVfM4VT9WQHP2WEj+dT4rHWJQchcFdaVQ1lgo+8G+QvZQKDyzMN/B90oTt/hSC7 -f+jlF0wbM4gb/8bPEjtU1ge78u8bcFr8tSqkEOyxmaEYSW0fxJUlWN7/ASQZUA7P -Hwy/AoGATg6bpVXwk1c19vkAGZupxGzIhphyWLGgv8ei4kwT/X+yHTjkeRfRR3Nv -nsX7y8UwN9Qc7segRjzxb6QdKhTtCJ0O0Su33iJk4d3niRlDbr4ZDn/XhSK/eH0x -g1KfnxxeyEBqzhnKSK57jCuRRTf7IMqUA9/kG+W1PbZ+hDpLfGY= +MIIEowIBAAKCAQEAwnsB61uE4JTEH8DJi37ivHIxVQVf2LOps5T2e5TGwH9q8aFi +lWOPlw1os90Zf5zuDrlXPTIYNKyQMYO6RxIWpFJnYTj9LiApIgwSyUGvY5f4OqHR +eagDCXgiu1l58ITeKeK8l3ObQnAjtiuTapCqxOkmexaQxLFdRAlhMnnpQVdOQqoX +eNJF4hC0UxXrlP/bHY0tEnwq4xLsxugFMEBNGxR9sklYKfNynYuspXmxn8ewksr8 +XviN1y98lAxbj0urVNloHk1hJXIUR/q2JPjck5y9WRE8GAcO1qIOFYXZZOY9IQHp +wdpHAaAEXCNDVK9lHp/rF7GhyfjgZkShKpYk3wIDAQABAoIBAD+CoBvWJUyaCHo+ +IRNW+oCD4ixbtvMzqOWmbd/ptAZFFg2WoHUcsFWp4VlriNoty2gvipfHdjQtbmFd +HUX8WDyNVIlhbPzVL9mYi8IBm18wz7WGBrxt65/6BY2dKL8tBMg07VWgQUGvEVp6 +XIfeeoYXhaOIuPoi2cxQK9eqDExzvb5AA1AS+FbYcKF1ma2Kb/mO52OQAsPmPnul +yyosInO2PFdNqlvYd5qOfJdPF1747nn4taigH1CKdDZ86GNufShWvcdiR/uYL/Ln +vu4Um7Ha05AFl9p+7TPqyuE1+nH1nKOqP8++C5TkDqLhPyzDUxENU50eCpQHhNiK +Jrvt1VECgYEA+k5E+pyg+Ji4XQnwNMbP2P8jgnFDSL7HfHuE3NfdomvoDit78LFw +/FzosBpv79lSXh8wplBIs5UwrAqaWoV0GQWoySM27hRDM5/c0xhqWS2c6gMGeUup +Tn2YvuXTmi1OsIPayTzQ92GeT3Xg+ojLZdqtLmkEJzAtUndww1HlLIUCgYEAxuef +fXXMfEYCrdEA1cvFGilVxnJXHjzHnky5RUrglwV6wkgfwE18dr4cmOgBte69shvc +8TS6I+KFffelKjSzm7OAEWEL+pGHKK0ALTBBXUJ3qa0PYSrgbCD1/nI3Q08JLSVV ++Xo5kmIGyLJMsLGkH/CSZCNUj450WDmfdcGrqxMCgYACDwC8OuuL/92MTleeZ4Aw +HbESEpJmF8OWP4HROylEe7S14R+s1BjEypLTV/RRuazWv1TsGT7v0ytKTvAEDJLu +3cAMn3CFNr9yvj7XsZy2TQy8U/gKqVekIJ5P+53o57R8+SikfQ6O6kueBa8rAFMD +7G9+MTjqhZfp1Lels5e57QKBgD4+q9WaMKTPT/VPC6DcRNE8EECq9YJb6OgsAGqj +1QbNyy3TXkRSu1l5gv+C00446Ro8x/af1oR2VeomvoQnu/FEyhYmNZZzRkW/Zee+ +SyZBL6tkogR5Y4PTCMhYu9yPdkKvhWkuC6g4jwDtczx0SvVH1rgJqmPGY7hcR/+U +3QELAoGBANxKIoJhDSecjGmjdHBGGmtXHsBZID033Qq6LEPStcHb7aMNdSYCIjZA +FpfqNYPywrqPOjlUVzM2Erz3gmdd5o3OxgbTkSjJPhndSvw9fCU29Oy0PP7qTXgE +Ksfuj92ATYeT+wwWZJ5kfhMjmvhPKhOAdi9au27y5tiO5upDeReh -----END RSA PRIVATE KEY----- diff --git a/test/config/integration/certs/upstreamcert.pem b/test/config/integration/certs/upstreamcert.pem index 19d9ced64a22..509397d6caa8 100644 --- a/test/config/integration/certs/upstreamcert.pem +++ b/test/config/integration/certs/upstreamcert.pem @@ -1,25 +1,25 @@ -----BEGIN CERTIFICATE----- -MIIEMzCCAxugAwIBAgIJAOOWZZgU2DxcMA0GCSqGSIb3DQEBCwUAMH8xCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp -c2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBMeWZ0IEVuZ2luZWVyaW5nMRkw -FwYDVQQDDBBUZXN0IFVwc3RyZWFtIENBMB4XDTE4MTIxNzIwMTgwMFoXDTIwMTIx -NjIwMTgwMFowgYMxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYw -FAYDVQQHDA1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKDARMeWZ0MRkwFwYDVQQLDBBM -eWZ0IEVuZ2luZWVyaW5nMR0wGwYDVQQDDBRUZXN0IFVwc3RyZWFtIFNlcnZlcjCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALxFMTgM8XOMFueOZ1SFVuEf -aa/WZs2Tm5OQJBCnQUlSq8QafvT7jg6YqB8ap6AA/LwYn9G6Q/dN+7SlVxEzIZlj -UspSd5iCuhYVVXpR1PrmYIyl5pDY5nX7eRrMhhgxUSnWf9bt4azu2FH+K89wazV9 -4uw1gpxkw3Ocj9HWiseNz+IE1iwMlSr/1n0jRZFKqnnfveuGByIANRj7DpnNJgZm -dMenBr4KeJEl/alvESRb0Zf0vWgDfwwUHs4rR7ZQl48Spjd7xjIoAQZ5DY8UKxB0 -BTTzeSkuvuxVSrAYefIIpq15rU4laKBcRkj9f8tvd53DJHWmvsVkCLlA6BkLqnUC -AwEAAaOBrDCBqTAMBgNVHRMBAf8EAjAAMAsGA1UdDwQEAwIF4DAdBgNVHSUEFjAU -BggrBgEFBQcDAgYIKwYBBQUHAwEwLQYDVR0RBCYwJIIKKi5seWZ0LmNvbYcEAAAA -AIcQAAAAAAAAAAAAAAAAAAAAADAdBgNVHQ4EFgQU/HB8QxqmxcgL4WPoyewp4ooz -UhcwHwYDVR0jBBgwFoAUqbsVny7rxHz5IgkTnIe1mx0sWHAwDQYJKoZIhvcNAQEL -BQADggEBAI1JScIpfxjLV5BrgflVKCjeABBuMGFj3qZOCBuiBbFTt+KOpc9dmfzL -bDSg9cLRe9MhFqaLFrzqAOZfqwCxB8jsAEMaki4anwm3fefC8D5H8uBB5KckZExk -/j/y0UYpnH21xvj7Z4M4bOMACTneel2IJ8rTzJD5QbSersbxB7FF/cOEZNt2B94l -6hkPaZzvzxjwizK6XDnpLPAQnytMPAcrPSAS9wmRPfjGgG4xFcc3EoqJKpCuymTt -hSLzAsLBP7NOITlgA0iRCU5Ly//TLnmLcA4iAVYQaog21cJYxEbRJTIm9z73/88D -bf/hoN53Jhgwt+xFK0CKZyahhZGsuXs= +MIIEPjCCAyagAwIBAgIUS0ht/ypqxlVqt86GiCya6cw/jJwwDQYJKoZIhvcNAQEL +BQAwfzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n +aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMTkwNzA4MjE0 +NTU3WhcNMjEwNzA3MjE0NTU3WjCBgzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh +bGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQx +GTAXBgNVBAsMEEx5ZnQgRW5naW5lZXJpbmcxHTAbBgNVBAMMFFRlc3QgVXBzdHJl +YW0gU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA+evHY2ld +y4iKlGEHtenqIK26QeFpU9t2iqiRjUz0+lZ+92haR+I+x1nL41SO71i2SaHp8L2Q +5cWkOS0zuYivsZSz/l9dHinAS+N50QsERo01moWOMyxfqSEbnZHMQS4OI/mf1dja +Rykp7zhCXie2BlUtmiBMW+YnvLmBm1z6icwg7ZBJ8mt2ChpeH6qBggzwQQms9wvK +/mcHR5HYHalLQdjhou3wwa6MB9bbeEoDd8I0tueRgnrq55mVJrm3yg1TSgUwkWCB +J3VUrvdk3olgGwHv4njAB+uNfUn3od7MuipyHL8GJQJHOcus63M/Ax/UVxu0BiDy +LfWW4MVO/5OX5QIDAQABo4GsMIGpMAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgXg +MB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAtBgNVHREEJjAkggoqLmx5 +ZnQuY29thwQAAAAAhxAAAAAAAAAAAAAAAAAAAAAAMB0GA1UdDgQWBBR3BvXiz3zi +p+/5cojIhCEz3nn39jAfBgNVHSMEGDAWgBTi0zC68szQHLsXt3ChIQpZC+yHbjAN +BgkqhkiG9w0BAQsFAAOCAQEAEB9RWuGIcRhZMM2AqXyOr4FOG63yfVg4fi3/WIcu +p7iVPhtdByefx4FQxg7913rdJyeQrI+hab0uPl/CjylwMVwWtBqRx4oKo8im59/4 +N7MRYZKJ44/fBSIGoM0pibSpDzfd7y6Drusp1mqi3CXGPXsVFIDQ66d7yoFt+t7h +nB2A565e/C1eXaS80XTHeJzfS5dJ6ssjgyszTGM5PdN9C335pDGfQV0CqGNAMZqo +tbBI1B0NgQ2KJJ787Wi3pexxi3haliMNrSKEAkLVDZ6R0a1PgpN/hBth3Nf2Oj+O ++pBNtkiA0fnkoKS6ps9Vgj+NB08OLeYNpfGFHa9xxFdPoA== -----END CERTIFICATE----- diff --git a/test/config/integration/certs/upstreamcert_hash.h b/test/config/integration/certs/upstreamcert_hash.h index 9b872dc40cb5..4342078b9380 100644 --- a/test/config/integration/certs/upstreamcert_hash.h +++ b/test/config/integration/certs/upstreamcert_hash.h @@ -1,3 +1,3 @@ // NOLINT(namespace-envoy) -constexpr char TEST_UPSTREAM_CERT_HASH[] = "89:92:C7:6D:B7:92:42:B3:77:77:6F:77:1C:51:04:1B:D7:27:" - "25:A7:7E:75:A9:6A:0A:F7:23:F9:F6:50:A5:34"; +constexpr char TEST_UPSTREAM_CERT_HASH[] = "57:0E:EF:74:60:9C:8E:3D:AA:EA:3F:3E:02:69:89:40:E6:00:" + "AD:CA:86:69:73:BA:9E:0B:01:A2:E2:F3:75:7F"; diff --git a/test/config/integration/certs/upstreamkey.pem b/test/config/integration/certs/upstreamkey.pem index d5e94728c7f5..58945f2125ad 100644 --- a/test/config/integration/certs/upstreamkey.pem +++ b/test/config/integration/certs/upstreamkey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAvEUxOAzxc4wW545nVIVW4R9pr9ZmzZObk5AkEKdBSVKrxBp+ -9PuODpioHxqnoAD8vBif0bpD9037tKVXETMhmWNSylJ3mIK6FhVVelHU+uZgjKXm -kNjmdft5GsyGGDFRKdZ/1u3hrO7YUf4rz3BrNX3i7DWCnGTDc5yP0daKx43P4gTW -LAyVKv/WfSNFkUqqed+964YHIgA1GPsOmc0mBmZ0x6cGvgp4kSX9qW8RJFvRl/S9 -aAN/DBQezitHtlCXjxKmN3vGMigBBnkNjxQrEHQFNPN5KS6+7FVKsBh58gimrXmt -TiVooFxGSP1/y293ncMkdaa+xWQIuUDoGQuqdQIDAQABAoIBAAoYQ61XtFKXvlqo -Hg5AIApuHsKY4mY/deYRon1qGmwODLu1F/2Wx2Us9kbErRw9MU/8mgUq0Z4fBlIH -U4XOkgyhcLz8cwEwcT3h4vVuEddqJt8jvhsiJilJVJMFSGNfsZRmtfAWTTVykRLE -aCD1TCpQF6zGqbwtAvXd/TApKsPMVRjjkTRB6lIIW+SFTihpFtLrdfV1ddfkg+br -pS7OqPzL7bsqPmwzL1ig/aE/aVb9xAsOT4AcrdZ8fUg1jixxiO64jFeGCOs4aT7B -+NizRRbIWfNR+NNqOLGKncLmVtvrRSdazGTQVU419+G+m8dJey4ZoSpqDtyBU7xj -zZSZooECgYEA9hLAgm6X7udnypO/1nuqSaLrq2Xd/eVMnirryp5AZ4wxmY6v8F4f -qJaYxt++qRZTQq+Ijcq4Y88prXPx+s7VWPtugBwR1xawB1qSnb5hPrLAt4DWLiI0 -bzvu4W43AdBjwg0Ayr34awpyAdSQgK72V2HOddn7k6ROltdfchZ7WBECgYEAw91/ -VHs5mQJFAFZJcwoGbRP7cG74E+uqIkNULO24LbAyDK6Mm85jfMvBpfEZRQ6JI3cK -O61OwKvq5ZKMwqHbTuB2503MsDfIqSiIfLva3HsHB3+qwoxdhu1/9cANV+mKAW41 -TBSRFGvcIKH+u/Mlv/SAnD8O4cy1Q2x6TjQo8CUCgYEAxX6hYU3PxR+WjuDsbAFO -19DZovOcKuV5C8zY+ALxH+pF+L+rd5ijghR0Q9FZ3a2cX34wc9TLDtg61AqloK2W -T9dkhY+BxgZge1Z3LAGbXM3snJrby6UKPmh0vhtOLLeLCTiUdSPpGEgG3m8zFwTV -k6ZdJPsxzfpmVOxAn3lpv3ECgYEAwJDXbAbOpQlfL6qmAe1cTge0UGE5k9RB6/fI -HXgGeRzeyCsgYNq0Y3CsTerRjlxxJiYWMH/+il07z0ObEowxYsY7AMQztxjRNsZ8 -Ei5bSiPG0G+LQkTgexSrlsCgHcuk/C0PR2J9FNfKj2bVXJH8jlHj1DoG9qbdm5Fe -Wd7cVOUCgYA77jRopwj89S68Qaz3YbejQmWF8Y4deTnHfPGoj2OQr4XJ5GWC0j8N -eTMTJMNQFx/Ge3Un9A9wjLP2KRmCHXl16pYBLdej+VPg7WGsOEleV7HrdKjq9daY -aCz5vX92lnvOhzAJkadAznSMW97nBlxQaCBnAIC+AIQU9Z4ZHzMmaQ== +MIIEpAIBAAKCAQEA+evHY2ldy4iKlGEHtenqIK26QeFpU9t2iqiRjUz0+lZ+92ha +R+I+x1nL41SO71i2SaHp8L2Q5cWkOS0zuYivsZSz/l9dHinAS+N50QsERo01moWO +MyxfqSEbnZHMQS4OI/mf1djaRykp7zhCXie2BlUtmiBMW+YnvLmBm1z6icwg7ZBJ +8mt2ChpeH6qBggzwQQms9wvK/mcHR5HYHalLQdjhou3wwa6MB9bbeEoDd8I0tueR +gnrq55mVJrm3yg1TSgUwkWCBJ3VUrvdk3olgGwHv4njAB+uNfUn3od7MuipyHL8G +JQJHOcus63M/Ax/UVxu0BiDyLfWW4MVO/5OX5QIDAQABAoIBAFqMy9w/8+TnntYt +5b5KdzLJ3x85jZD9hhCtDLd2d5gwOKZpX7SFy5ss9Mtz+qnLqZg6GunHtTUbC+pP +b1s8o/OiXii+4p0oIW0diShtZmothYtr8l6mKC6+OSQ5DBldl2//ZKL1g/ieeHwd +FSbKGpBm0jPymdf+Js2hJM1mvbuoy8ZxkdAtuYA/7tqQVG3/yFfB9Hm9JmGjU5iH +2m0qrZsch0pusKvw7zwoPshLcNvDeIt/i1gkoCNi5ZSxQ/Ow4dHaxSuQyggbzhn4 +j0SHpGtRhrOkccxKmc8EDxZynWYb5nCPAOSsb5SOtXMDrJdZUi0c9a2ZYLpcjz9m +RJH7a8ECgYEA/a0hrbUq43wNUuFeGs7W42u+R4L+bQ8aGF7MRkcF1CFfP9aYkebH +7DIt5Tz4ESCUKdM8SZLm1L+JsSQDq3T/8S+UrQpAvH/0FDbChGXe7mPTveMa7mTo +/l4gQ0BtqSknj68I0NGM30yuk1OdIRRFMFWEJZvRvb43JCDS2NRPnHUCgYEA/DXX +WNk9aABvW1IlCf0iufwHOWINNvaELHX1LhRhpgF/zinOjN+QBxibLyEQwD+x3zcH +SiN6xOt+KUuM+b7yeoPJhfMyCTENrMezOun89tTnpI6SYiPn6ugHeR8hQHPKe2X3 +T6sCY6KnzN29j8LICZJGRKeqKZUm006Lcoh1X7ECgYEAnRzztPB2Bbq5TdHDRPtC +YExE52mcRtOJp/perlAirgWVRqaUjBjRTdquTkJ6qbDx0w2/Uxom2TFgCFRz6Wdn +dWuwu5OUEKt28mYQB4xIjIFLjVnxPiFFpPWLKdvnj1Or6vPPk/WVOF/358trkCdL +yunMFLbzKn97C2dA74ZfYFkCgYEAvslb5fIv6YSquEIjkrLSmi50qIvrwzAoPBnf +JsR0OcfYjnRBs39KzJNokPZKXaPRQjG2afb84Anknghw1FwFwXf/8jxOFXXuCk3m +3yIyIeZcdLcFNQhEYAa14IIT/VWaTk6MDtAmNojMtsTmqOGHwPXOAhFzP5F8lUxN +YI6pe4ECgYA/Q3cX3R2P0WZq19+0IASRzuxSeIuT/Pw51/1qnESkFw5HvQ9HFSmT +J2lvWI0N4oyCBEuynVPFneR2lK2FN7ZOIVlQMFNQ6nJDFwvTQr4cXHn0eURTKKf1 +frP8QXXeP9rdsoo9veCciJpHZz22vpVE7FZlC0WDTOTl1kltO2z78A== -----END RSA PRIVATE KEY----- diff --git a/test/config/integration/certs/upstreamlocalhostcert.cfg b/test/config/integration/certs/upstreamlocalhostcert.cfg new file mode 100644 index 000000000000..fbea513733e9 --- /dev/null +++ b/test/config/integration/certs/upstreamlocalhostcert.cfg @@ -0,0 +1,38 @@ +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = US +countryName_default = US +stateOrProvinceName = California +stateOrProvinceName_default = California +localityName = San Francisco +localityName_default = San Francisco +organizationName = Lyft +organizationName_default = Lyft +organizationalUnitName = Lyft Engineering +organizationalUnitName_default = Lyft Engineering +commonName = Test Upstream Server +commonName_default = Test Upstream Server +commonName_max = 64 + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = @alt_names +subjectKeyIdentifier = hash + +[v3_ca] +basicConstraints = critical, CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = @alt_names +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always + +[alt_names] +DNS.2 = localhost +IP.1 = 0.0.0.0 +IP.2 = :: diff --git a/test/config/integration/certs/upstreamlocalhostcert.pem b/test/config/integration/certs/upstreamlocalhostcert.pem new file mode 100644 index 000000000000..e34ab833d86c --- /dev/null +++ b/test/config/integration/certs/upstreamlocalhostcert.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEPTCCAyWgAwIBAgIUS0ht/ypqxlVqt86GiCya6cw/jJ0wDQYJKoZIhvcNAQEL +BQAwfzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM +DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n +aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMTkwNzA4MjE0 +NTU3WhcNMjEwNzA3MjE0NTU3WjCBgzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh +bGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQx +GTAXBgNVBAsMEEx5ZnQgRW5naW5lZXJpbmcxHTAbBgNVBAMMFFRlc3QgVXBzdHJl +YW0gU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAujOqV+UB +T8oKxxnIlgKMPn10hIZxOOEzA96CDMQtQ2+180HLfSTErLWzQFNeP6jRDcbXTN0w +tYJlIUmVJtPaj7Dh4VvpORhRwAPZt9bkHcKKFCIaGYj61YCv3YpNyBSfJ0vwgATD +Yn6I2R8nobMKau/hMk4SpPZ6Z3pwSEt0GHd9/cE7t1WvE4BhqIjznexeFO+YrgvF +2ea4j7u4hJxezZhzAqOUyqtlbfkHQwXXzg/93PxBY5Y1mUPszjY+doGhW3DfTI1O +qgU2OfAoFZ6SKtUphUG/gt5DKHvKeARCWMEaUXC9UkyzhSNIl7s8qnRzweZwyOIk +KClryNQCtTjHOwIDAQABo4GrMIGoMAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgXg +MB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAsBgNVHREEJTAjgglsb2Nh +bGhvc3SHBAAAAACHEAAAAAAAAAAAAAAAAAAAAAAwHQYDVR0OBBYEFPlBuX/WSFDb ++nUGMLTu7svrrtolMB8GA1UdIwQYMBaAFOLTMLryzNAcuxe3cKEhClkL7IduMA0G +CSqGSIb3DQEBCwUAA4IBAQCgknWXc/Iz/Av6inDxGOncsNlYehKU+UBoR69HlcUE +AEOW7nFaPey5zLL3dgTJd1nOe0u97yT5Hoy9b7O9z5cBWHkNYqFh2oEZKXeDtS81 +z/N4ZQuPxxAlS4d7krAsQNB2vjMFp81eGude680twbto6LKRg9iJMv+AeEJD9p0j +ubeZA20j8YV8Aijm/kNe82d+TQYULxQeLo5QM6VU0pK2VcCunHywcYFc/t99Ync6 +NaqGrxOu6Jfduzg0TsZsIX7GveYC4dmx3CK1qOSB7SE2SVQjAITZL7gIvbLQPEKu +XJrmpIIhgUw+AgPq7D8JEaLoYERCRQLWt4v/yVus2Tgd +-----END CERTIFICATE----- diff --git a/test/config/integration/certs/upstreamlocalhostcert_hash.h b/test/config/integration/certs/upstreamlocalhostcert_hash.h new file mode 100644 index 000000000000..409ad0af1cd3 --- /dev/null +++ b/test/config/integration/certs/upstreamlocalhostcert_hash.h @@ -0,0 +1,4 @@ +// NOLINT(namespace-envoy) +constexpr char TEST_UPSTREAMLOCALHOST_CERT_HASH[] = + "5B:5C:02:47:DE:17:B7:1B:98:05:0A:DB:41:2C:F6:8F:65:E3:86:E6:03:B3:9A:EC:67:33:2E:39:1F:05:88:" + "B0"; diff --git a/test/config/integration/certs/upstreamlocalhostkey.pem b/test/config/integration/certs/upstreamlocalhostkey.pem new file mode 100644 index 000000000000..7bf369f08b6d --- /dev/null +++ b/test/config/integration/certs/upstreamlocalhostkey.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAujOqV+UBT8oKxxnIlgKMPn10hIZxOOEzA96CDMQtQ2+180HL +fSTErLWzQFNeP6jRDcbXTN0wtYJlIUmVJtPaj7Dh4VvpORhRwAPZt9bkHcKKFCIa +GYj61YCv3YpNyBSfJ0vwgATDYn6I2R8nobMKau/hMk4SpPZ6Z3pwSEt0GHd9/cE7 +t1WvE4BhqIjznexeFO+YrgvF2ea4j7u4hJxezZhzAqOUyqtlbfkHQwXXzg/93PxB +Y5Y1mUPszjY+doGhW3DfTI1OqgU2OfAoFZ6SKtUphUG/gt5DKHvKeARCWMEaUXC9 +UkyzhSNIl7s8qnRzweZwyOIkKClryNQCtTjHOwIDAQABAoIBABS0H/Gr9exgQ7iF +pmb/m4ZrPqRpqncvmxOIDx/KRFomNq34l96vUur9PRQe8PDVHYGRpWjXg037VLFR +1DLABaJKgaMkLBd8G8Lk6rVlQHIKqn24mPxT3cgVifhxI1rm6BdfeztQzETMWv0B +WM/C75qaV4jXY31SJqQQ2iE/uoXpuqHtBqxVE9TnBi0NvN2ZXlcxGgwnv7SYKrkE +4c9M5q0F5mJAAkHsrgyyQY18/op/vtQGbKvsdkuT9ihzruaeXxB6M04nevxdDn3r +dC5GUz05c+GCqCmMppU2gRPKYH0t/mvXfZhoGiOujfhTUbiIjQBle1kaxXmZl7Zr +Kz+lO+kCgYEA8IOstfsxJ6Zdf4Xvqi6MLr3MKZT3MmSpKT1dOixzYtD1IS1qc0hF +t4+SNhlP1ny5TlFF12356AC4TMM+kjlhCu9uAsvjqI5M/XSQ00SvybADbYoQc7jy +v9d69AEPOslmfQ7GYDAlFKT5NTK6tXEVpX1MIs6+LSl2pi4emtW7EzUCgYEAxjDG +Db/yEzanLyPC2Yfjk8KotnmprVZmXvmSWl0frOjWxbmYq2auVMNgBysY0+k8OHWk +MQjtjbVqA5b4Ze5Rm0Z7XlWieioxLt1naG3Gb0NAtaivmouaovlrIlZf7GtT4haC +Q4/GN9gGUxdjLLJRJ/WcdOG4X4gp7Opj19eazq8CgYAXVF5rZIs3El8dYIuH0W4N +lqF4Ixf7TmJOOsKRQwCKRESSzEn4FrmUfZusHbZt0rlSzHVe2S8VfwRhhcrK+j/c +hK8CHG7fybXUG/t0UsROZwFeHbdM0lLRowAtLPEiPajwVn+Nkv31y67UpzAPK4Hz +BH1fHvi5fr0gj3auhC7aRQKBgEEAAhTEXSp8BDzrp54ceUEe2KJwKHwXGCASDjPg +0uCsxLO4eR/N32MhaL8xHUVy+zMxMhZ67R5K32gp/XHAxbb9WLzJrS4P5G2QY7fW +OPyIvBJYLq+rFZ5Z2w858N/jG3HNHA/4eXQbP4fE5dvk58UJQrT6yrNaPxXakcBa +kAU1AoGBAJSSjgV9kR2tGzhBFqL0K11E5GP9uzO3sib3Zd7gtOJTO1cvJaLES3BV +Q6sxR9gPJ5cSpTXCNbaJwZ+jAsTfhJj8PE15am/JG7d7xZkflZIdfSf9/23g26w1 +dagOuDPRC2mcbjzprdPRLbNk3NfI/Llw+CboMP6R/smOYf/HRpS9 +-----END RSA PRIVATE KEY----- diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc index d15870581e24..6d8db1e762b7 100644 --- a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc +++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc @@ -67,6 +67,7 @@ class ClusterTest : public testing::Test, // Allow touch() to still be strict. EXPECT_CALL(*host_map_[host], address()).Times(AtLeast(0)); + EXPECT_CALL(*host_map_[host], resolvedHost()).Times(AtLeast(0)); } void updateTestHostAddress(const std::string& host, const std::string& address) { diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.cc b/test/extensions/common/dynamic_forward_proxy/mocks.cc index f2d6c207c009..9fc213794334 100644 --- a/test/extensions/common/dynamic_forward_proxy/mocks.cc +++ b/test/extensions/common/dynamic_forward_proxy/mocks.cc @@ -3,6 +3,7 @@ using testing::_; using testing::Return; using testing::ReturnPointee; +using testing::ReturnRef; namespace Envoy { namespace Extensions { @@ -22,6 +23,7 @@ MockDnsCacheManager::~MockDnsCacheManager() = default; MockDnsHostInfo::MockDnsHostInfo() { ON_CALL(*this, address()).WillByDefault(ReturnPointee(&address_)); + ON_CALL(*this, resolvedHost()).WillByDefault(ReturnRef(resolved_host_)); } MockDnsHostInfo::~MockDnsHostInfo() = default; diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.h b/test/extensions/common/dynamic_forward_proxy/mocks.h index 0d2bfad3f1c7..0c65895c1f7d 100644 --- a/test/extensions/common/dynamic_forward_proxy/mocks.h +++ b/test/extensions/common/dynamic_forward_proxy/mocks.h @@ -62,9 +62,11 @@ class MockDnsHostInfo : public DnsHostInfo { ~MockDnsHostInfo(); MOCK_METHOD0(address, Network::Address::InstanceConstSharedPtr()); + MOCK_METHOD0(resolvedHost, const std::string&()); MOCK_METHOD0(touch, void()); Network::Address::InstanceConstSharedPtr address_; + std::string resolved_host_; }; class MockUpdateCallbacks : public DnsCache::UpdateCallbacks { diff --git a/test/extensions/filters/http/dynamic_forward_proxy/BUILD b/test/extensions/filters/http/dynamic_forward_proxy/BUILD index f3b9f0a3edff..881a1be457d7 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/BUILD +++ b/test/extensions/filters/http/dynamic_forward_proxy/BUILD @@ -26,6 +26,9 @@ envoy_extension_cc_test( envoy_extension_cc_test( name = "proxy_filter_integration_test", srcs = ["proxy_filter_integration_test.cc"], + data = [ + "//test/config/integration/certs", + ], extension_name = "envoy.filters.http.dynamic_forward_proxy", deps = [ "//source/extensions/clusters/dynamic_forward_proxy:cluster", diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index ae7d16374682..b2750046922b 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -1,3 +1,6 @@ +#include "extensions/transport_sockets/tls/context_config_impl.h" +#include "extensions/transport_sockets/tls/ssl_socket.h" + #include "test/integration/http_integration.h" namespace Envoy { @@ -36,11 +39,19 @@ name: envoy.filters.http.dynamic_forward_proxy config_helper_.addFilter(filter); config_helper_.addConfigModifier( - [max_hosts](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + [this, max_hosts](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters(0); cluster_0->clear_hosts(); cluster_0->set_lb_policy(envoy::api::v2::Cluster::CLUSTER_PROVIDED); + if (upstream_tls_) { + auto context = cluster_0->mutable_tls_context(); + auto* validation_context = + context->mutable_common_tls_context()->mutable_validation_context(); + validation_context->mutable_trusted_ca()->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); + } + const std::string cluster_type_config = fmt::format(R"EOF( name: envoy.clusters.dynamic_forward_proxy @@ -58,6 +69,36 @@ name: envoy.clusters.dynamic_forward_proxy HttpIntegrationTest::initialize(); } + + void createUpstreams() override { + if (upstream_tls_) { + fake_upstreams_.emplace_back(new FakeUpstream( + createUpstreamSslContext(), 0, FakeHttpConnection::Type::HTTP1, version_, timeSystem())); + } else { + HttpIntegrationTest::createUpstreams(); + } + } + + // TODO(mattklein123): This logic is duplicated in various places. Cleanup in a follow up. + Network::TransportSocketFactoryPtr createUpstreamSslContext() { + envoy::api::v2::auth::DownstreamTlsContext tls_context; + auto* common_tls_context = tls_context.mutable_common_tls_context(); + auto* tls_cert = common_tls_context->add_tls_certificates(); + tls_cert->mutable_certificate_chain()->set_filename(TestEnvironment::runfilesPath( + fmt::format("test/config/integration/certs/{}cert.pem", upstream_cert_name_))); + tls_cert->mutable_private_key()->set_filename(TestEnvironment::runfilesPath( + fmt::format("test/config/integration/certs/{}key.pem", upstream_cert_name_))); + + auto cfg = std::make_unique( + tls_context, factory_context_); + + static Stats::Scope* upstream_stats_store = new Stats::IsolatedStoreImpl(); + return std::make_unique( + std::move(cfg), context_manager_, *upstream_stats_store, std::vector{}); + } + + bool upstream_tls_{}; + std::string upstream_cert_name_{"upstreamlocalhost"}; }; INSTANTIATE_TEST_SUITE_P(IpVersions, ProxyFilterIntegrationTest, @@ -142,5 +183,53 @@ TEST_P(ProxyFilterIntegrationTest, DNSCacheHostOverflow) { EXPECT_EQ(1, test_server_->counter("dns_cache.foo.host_overflow")->value()); } +// Verify that upstream TLS works with auto verification for SAN as well as auto setting SNI. +TEST_P(ProxyFilterIntegrationTest, UpstreamTls) { + upstream_tls_ = true; + setup(); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + waitForNextUpstreamRequest(); + + const Extensions::TransportSockets::Tls::SslSocket* ssl_socket = + dynamic_cast( + fake_upstream_connection_->connection().ssl()); + EXPECT_STREQ("localhost", + SSL_get_servername(ssl_socket->rawSslForTest(), TLSEXT_NAMETYPE_host_name)); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + checkSimpleRequestSuccess(0, 0, response.get()); +} + +// Verify that auto-SAN verification fails with an incorrect certificate. +TEST_P(ProxyFilterIntegrationTest, UpstreamTlsInvalidSAN) { + upstream_tls_ = true; + upstream_cert_name_ = "upstream"; + setup(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", + fmt::format("localhost:{}", fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + response->waitForEndStream(); + EXPECT_EQ("503", response->headers().Status()->value().getStringView()); + + EXPECT_EQ(1, test_server_->counter("cluster.cluster_0.ssl.fail_verify_san")->value()); +} + } // namespace } // namespace Envoy diff --git a/test/integration/xfcc_integration_test.cc b/test/integration/xfcc_integration_test.cc index b7e850486499..2e9a5f12d8a6 100644 --- a/test/integration/xfcc_integration_test.cc +++ b/test/integration/xfcc_integration_test.cc @@ -70,16 +70,16 @@ Network::TransportSocketFactoryPtr XfccIntegrationTest::createClientSslContext(b } Network::TransportSocketFactoryPtr XfccIntegrationTest::createUpstreamSslContext() { - std::string json = R"EOF( -{ - "cert_chain_file": "{{ test_rundir }}/test/config/integration/certs/upstreamcert.pem", - "private_key_file": "{{ test_rundir }}/test/config/integration/certs/upstreamkey.pem" -} -)EOF"; + envoy::api::v2::auth::DownstreamTlsContext tls_context; + auto* common_tls_context = tls_context.mutable_common_tls_context(); + auto* tls_cert = common_tls_context->add_tls_certificates(); + tls_cert->mutable_certificate_chain()->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcert.pem")); + tls_cert->mutable_private_key()->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamkey.pem")); - Json::ObjectSharedPtr loader = TestEnvironment::jsonLoadFromString(json); auto cfg = std::make_unique( - *loader, factory_context_); + tls_context, factory_context_); static Stats::Scope* upstream_stats_store = new Stats::TestIsolatedStoreImpl(); return std::make_unique( std::move(cfg), *context_manager_, *upstream_stats_store, std::vector{}); From bfc7602ec77e68429d9bad9af80e8cb648c6dd77 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 12 Jul 2019 22:16:15 -0700 Subject: [PATCH 165/542] coverage: ignore test (#7566) Description: Fix regression of #7551 Risk Level: Testing: Docs Changes: Release Notes: Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 6c1d22de56b6..04499887105d 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -77,7 +77,7 @@ BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} -c dbg --copt=-DNDEBUG" # stats. The #foo# pattern is because gcov produces files such as # bazel-out#local-fastbuild#bin#external#spdlog_git#_virtual_includes#spdlog#spdlog#details#pattern_formatter_impl.h.gcov. # To find these while modifying this regex, perform a gcov run with -k set. -[[ -z "${GCOVR_EXCLUDE_REGEX}" ]] && GCOVR_EXCLUDE_REGEX=".*pb\\..*|.*#test#.*|external#.*|.*#external#.*|.*#prebuilt#.*|.*#config_validation#.*|.*#chromium_url#.*" +[[ -z "${GCOVR_EXCLUDE_REGEX}" ]] && GCOVR_EXCLUDE_REGEX=".*pb\\..*|test#.*|.*#test#.*|external#.*|.*#external#.*|.*#prebuilt#.*|.*#config_validation#.*|.*#chromium_url#.*" [[ -z "${GCOVR_EXCLUDE_DIR}" ]] && GCOVR_EXCLUDE_DIR=".*/external/.*" COVERAGE_DIR="${SRCDIR}"/generated/coverage From 12ef480a67cf5881bf820b56e422856796c40755 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sat, 13 Jul 2019 13:53:55 -0700 Subject: [PATCH 166/542] admin: remove unused code (#7568) Signed-off-by: Lizan Zhou --- source/server/http/admin.cc | 29 ----------------------------- source/server/http/admin.h | 4 ---- 2 files changed, 33 deletions(-) diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 336c8694e5d2..6537b978123b 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -1104,35 +1104,6 @@ Http::Code AdminImpl::handlerRuntime(absl::string_view url, Http::HeaderMap& res return Http::Code::OK; } -std::string AdminImpl::runtimeAsJson( - const std::vector>& entries) { - rapidjson::Document document; - document.SetObject(); - rapidjson::Value entries_array(rapidjson::kArrayType); - rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); - for (const auto& entry : entries) { - Value entry_obj; - entry_obj.SetObject(); - - entry_obj.AddMember("name", {entry.first.c_str(), allocator}, allocator); - - Value entry_value; - if (entry.second.uint_value_) { - entry_value.SetUint64(entry.second.uint_value_.value()); - } else { - entry_value.SetString(entry.second.raw_string_value_.c_str(), allocator); - } - entry_obj.AddMember("value", entry_value, allocator); - - entries_array.PushBack(entry_obj, allocator); - } - document.AddMember("runtime", entries_array, allocator); - rapidjson::StringBuffer strbuf; - rapidjson::PrettyWriter writer(strbuf); - document.Accept(writer); - return strbuf.GetString(); -} - bool AdminImpl::isFormUrlEncoded(const Http::HeaderEntry* content_type) const { if (content_type == nullptr) { return false; diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 2991515cfef3..6fc5d3a6e254 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -235,11 +235,7 @@ class AdminImpl : public Admin, const absl::optional regex = absl::nullopt, bool pretty_print = false); - static std::string - runtimeAsJson(const std::vector>& entries); std::vector sortedHandlers() const; - static const std::vector> - sortedRuntime(const std::unordered_map& entries); envoy::admin::v2alpha::ServerInfo::State serverState(); /** * URL handlers. From 80a4ed73982936fe3a0ff08f697f18023c849ea9 Mon Sep 17 00:00:00 2001 From: Mitch Sukalski Date: Sat, 13 Jul 2019 14:09:33 -0700 Subject: [PATCH 167/542] =?UTF-8?q?redis:=20upstream=20client=20draining?= =?UTF-8?q?=20and=20temp=20host=20connection=20limit=20and=20cl=E2=80=A6?= =?UTF-8?q?=20(#7104)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mitch Sukalski --- .../network/redis_proxy/v2/redis_proxy.proto | 8 + .../arch_overview/other_protocols/redis.rst | 9 + .../extensions/clusters/redis/redis_cluster.h | 1 + .../filters/network/common/redis/BUILD | 1 + .../filters/network/common/redis/client.h | 23 + .../network/common/redis/client_impl.cc | 15 +- .../network/common/redis/client_impl.h | 6 + .../filters/network/redis_proxy/BUILD | 1 + .../filters/network/redis_proxy/config.cc | 4 +- .../network/redis_proxy/conn_pool_impl.cc | 124 ++++-- .../network/redis_proxy/conn_pool_impl.h | 28 +- .../extensions/health_checkers/redis/redis.h | 2 + .../clusters/redis/redis_cluster_test.cc | 1 + .../network/common/redis/client_impl_test.cc | 4 + .../filters/network/common/redis/mocks.h | 1 + .../filters/network/common/redis/test_utils.h | 3 +- .../filters/network/redis_proxy/BUILD | 1 - .../redis_proxy/conn_pool_impl_test.cc | 392 +++++++++++++++++- .../redis_proxy_integration_test.cc | 3 +- .../health_checkers/redis/redis_test.cc | 1 + 20 files changed, 570 insertions(+), 58 deletions(-) diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index 00e339f6c25c..e2fdd24522e9 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -10,6 +10,7 @@ option go_package = "v2"; import "envoy/api/v2/core/base.proto"; import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; import "gogoproto/gogo.proto"; @@ -82,6 +83,13 @@ message RedisProxy { // If `max_buffer_size_before_flush` is set, but `buffer_flush_timeout` is not, the latter // defaults to 3ms. google.protobuf.Duration buffer_flush_timeout = 5 [(gogoproto.stdduration) = true]; + + // `max_upstream_unknown_connections` controls how many upstream connections to unknown hosts + // can be created at any given time by any given worker thread (see `enable_redirection` for + // more details). If the host is unknown and a connection cannot be created due to enforcing + // this limit, then redirection will fail and the original redirection error will be passed + // downstream unchanged. This limit defaults to 100. + google.protobuf.UInt32Value max_upstream_unknown_connections = 6; } // Network settings for the connection pool to the upstream clusters. diff --git a/docs/root/intro/arch_overview/other_protocols/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst index 42c3b288cabe..22650a4264ad 100644 --- a/docs/root/intro/arch_overview/other_protocols/redis.rst +++ b/docs/root/intro/arch_overview/other_protocols/redis.rst @@ -81,6 +81,15 @@ following information: For topology configuration details, see the Redis Cluster :ref:`v2 API reference `. +Every Redis cluster has its own extra statistics tree rooted at *cluster..redis_cluster.* with the following statistics: + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + max_upstream_unknown_connections_reached, Counter, Total number of times that an upstream connection to an unknown host is not created after redirection having reached the connection pool's max_upstream_unknown_connections limit + upstream_cx_drained, Counter, Total number of upstream connections drained of active requests before being closed + Supported commands ------------------ diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index f3f0e141e830..d7b776353671 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -211,6 +211,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { bool enableRedirection() const override { return false; } uint32_t maxBufferSizeBeforeFlush() const override { return 0; } std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } + uint32_t maxUpstreamUnknownConnections() const override { return 0; } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index f74e283743c6..1757c2a7b9b4 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -65,6 +65,7 @@ envoy_cc_library( "//source/common/network:filter_lib", "//source/common/protobuf:utility_lib", "//source/common/upstream:load_balancer_lib", + "//source/common/upstream:upstream_lib", "@envoy_api//envoy/config/filter/network/redis_proxy/v2:redis_proxy_cc", ], ) diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index e6db95d75c25..4e041ad33552 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -76,6 +76,12 @@ class Client : public Event::DeferredDeletable { */ virtual void addConnectionCallbacks(Network::ConnectionCallbacks& callbacks) PURE; + /** + * Called to determine if the client has pending requests. + * @return bool true if the client is processing requests or false if it is currently idle. + */ + virtual bool active() PURE; + /** * Closes the underlying network connection. */ @@ -134,6 +140,23 @@ class Config { * @return timeout for batching commands for a single upstream host. */ virtual std::chrono::milliseconds bufferFlushTimeoutInMs() const PURE; + + /** + * @return the maximum number of upstream connections to unknown hosts when enableRedirection() is + * true. + * + * This value acts as an upper bound on the number of servers in a cluster if only a subset + * of the cluster's servers are known via configuration (cluster size - number of servers in + * cluster known to cluster manager <= maxUpstreamUnknownConnections() for proper operation). + * Redirection errors are processed if enableRedirection() is true, and a new upstream connection + * to a previously unknown server will be made as a result of redirection if the number of unknown + * server connections is currently less than maxUpstreamUnknownConnections(). If a connection + * cannot be made, then the original redirection error will be passed though unchanged to the + * downstream client. If a cluster is using the Redis cluster protocol (RedisCluster), then the + * cluster logic will periodically discover all of the servers in the cluster; this should + * minimize the need for a large maxUpstreamUnknownConnections() value. + */ + virtual uint32_t maxUpstreamUnknownConnections() const PURE; }; /** diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index c0fee56b5eb3..20bc5f02079a 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -16,9 +16,10 @@ ConfigImpl::ConfigImpl( config.max_buffer_size_before_flush()), // This is a scalar, so default is zero. buffer_flush_timeout_(PROTOBUF_GET_MS_OR_DEFAULT( config, buffer_flush_timeout, - 3)) // Default timeout is 3ms. If max_buffer_size_before_flush is zero, this is not used - // as the buffer is flushed on each request immediately. -{} + 3)), // Default timeout is 3ms. If max_buffer_size_before_flush is zero, this is not used + // as the buffer is flushed on each request immediately. + max_upstream_unknown_connections_( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)) {} ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, @@ -124,14 +125,12 @@ void ClientImpl::putOutlierEvent(Upstream::Outlier::Result result) { void ClientImpl::onEvent(Network::ConnectionEvent event) { if (event == Network::ConnectionEvent::RemoteClose || event == Network::ConnectionEvent::LocalClose) { + + Upstream::reportUpstreamCxDestroy(host_, event); if (!pending_requests_.empty()) { - host_->cluster().stats().upstream_cx_destroy_with_active_rq_.inc(); + Upstream::reportUpstreamCxDestroyActiveRequest(host_, event); if (event == Network::ConnectionEvent::RemoteClose) { putOutlierEvent(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED); - host_->cluster().stats().upstream_cx_destroy_remote_with_active_rq_.inc(); - } - if (event == Network::ConnectionEvent::LocalClose) { - host_->cluster().stats().upstream_cx_destroy_local_with_active_rq_.inc(); } } diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 20a7adcd0cc5..d88fc4c93e23 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -12,6 +12,7 @@ #include "common/protobuf/utility.h" #include "common/singleton/const_singleton.h" #include "common/upstream/load_balancer_impl.h" +#include "common/upstream/upstream_impl.h" #include "extensions/filters/network/common/redis/client.h" @@ -45,6 +46,9 @@ class ConfigImpl : public Config { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_flush_timeout_; } + uint32_t maxUpstreamUnknownConnections() const override { + return max_upstream_unknown_connections_; + } private: const std::chrono::milliseconds op_timeout_; @@ -52,6 +56,7 @@ class ConfigImpl : public Config { const bool enable_redirection_; const uint32_t max_buffer_size_before_flush_; const std::chrono::milliseconds buffer_flush_timeout_; + const uint32_t max_upstream_unknown_connections_; }; class ClientImpl : public Client, public DecoderCallbacks, public Network::ConnectionCallbacks { @@ -68,6 +73,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne } void close() override; PoolRequest* makeRequest(const RespValue& request, PoolCallbacks& callbacks) override; + bool active() override { return !pending_requests_.empty(); } void flushBufferAndResetTimer(); private: diff --git a/source/extensions/filters/network/redis_proxy/BUILD b/source/extensions/filters/network/redis_proxy/BUILD index b67729cedfde..bbc18dff95d2 100644 --- a/source/extensions/filters/network/redis_proxy/BUILD +++ b/source/extensions/filters/network/redis_proxy/BUILD @@ -75,6 +75,7 @@ envoy_cc_library( deps = [ ":config_interface", ":conn_pool_interface", + "//include/envoy/stats:stats_macros", "//include/envoy/thread_local:thread_local_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/buffer:buffer_lib", diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index 0fb6a602e5ff..5e3e4018260d 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -59,11 +59,13 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP Upstreams upstreams; for (auto& cluster : unique_clusters) { + Stats::ScopePtr stats_scope = + context.scope().createScope(fmt::format("cluster.{}.redis_cluster", cluster)); upstreams.emplace(cluster, std::make_shared( cluster, context.clusterManager(), Common::Redis::Client::ClientFactoryImpl::instance_, context.threadLocal(), proto_config.settings(), context.api(), - context.scope().symbolTable())); + std::move(stats_scope))); } auto router = diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index c9da9ef18806..3f998b191efb 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -6,6 +6,7 @@ #include #include "common/common/assert.h" +#include "common/stats/utility.h" #include "extensions/filters/network/redis_proxy/config.h" #include "extensions/filters/network/well_known_names.h" @@ -15,7 +16,6 @@ namespace Extensions { namespace NetworkFilters { namespace RedisProxy { namespace ConnPool { - namespace { Common::Redis::Client::DoNothingPoolCallbacks null_pool_callbacks; } // namespace @@ -24,9 +24,10 @@ InstanceImpl::InstanceImpl( const std::string& cluster_name, Upstream::ClusterManager& cm, Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, - Api::Api& api, Stats::SymbolTable& symbol_table) + Api::Api& api, Stats::ScopePtr&& stats_scope) : cm_(cm), client_factory_(client_factory), tls_(tls.allocateSlot()), config_(config), - api_(api), symbol_table_(symbol_table) { + api_(api), stats_scope_(std::move(stats_scope)), redis_cluster_stats_{REDIS_CLUSTER_STATS( + POOL_COUNTER(*stats_scope_))} { tls_->set([this, cluster_name]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(*this, dispatcher, cluster_name); @@ -48,8 +49,9 @@ InstanceImpl::makeRequestToHost(const std::string& host_address, InstanceImpl::ThreadLocalPool::ThreadLocalPool(InstanceImpl& parent, Event::Dispatcher& dispatcher, std::string cluster_name) - : parent_(parent), dispatcher_(dispatcher), cluster_name_(std::move(cluster_name)) { - + : parent_(parent), dispatcher_(dispatcher), cluster_name_(std::move(cluster_name)), + drain_timer_(dispatcher.createTimer([this]() -> void { drainClients(); })), + is_redis_cluster_(false) { cluster_update_handle_ = parent_.cm_.addThreadLocalClusterUpdateCallbacks(*this); Upstream::ThreadLocalCluster* cluster = parent_.cm_.get(cluster_name_); if (cluster != nullptr) { @@ -69,6 +71,9 @@ InstanceImpl::ThreadLocalPool::~ThreadLocalPool() { while (!client_map_.empty()) { client_map_.begin()->second->redis_client_->close(); } + while (!clients_to_drain_.empty()) { + (*clients_to_drain_.begin())->redis_client_->close(); + } } void InstanceImpl::ThreadLocalPool::onClusterAddOrUpdateNonVirtual( @@ -86,8 +91,9 @@ void InstanceImpl::ThreadLocalPool::onClusterAddOrUpdateNonVirtual( cluster_ = &cluster; ASSERT(host_set_member_update_cb_handle_ == nullptr); host_set_member_update_cb_handle_ = cluster_->prioritySet().addMemberUpdateCb( - [this](const std::vector&, + [this](const std::vector& hosts_added, const std::vector& hosts_removed) -> void { + onHostsAdded(hosts_added); onHostsRemoved(hosts_removed); }); @@ -97,6 +103,16 @@ void InstanceImpl::ThreadLocalPool::onClusterAddOrUpdateNonVirtual( host_address_map_[host->address()->asString()] = host; } } + + // Figure out if the cluster associated with this ConnPool is a Redis cluster + // with its own hash slot sharding scheme and ability to dynamically discover + // its members. This is done once to minimize overhead in the data path, makeRequest() in + // particular. + Upstream::ClusterInfoConstSharedPtr info = cluster_->info(); + const auto& cluster_type = info->clusterType(); + is_redis_cluster_ = info->lbType() == Upstream::LoadBalancerType::ClusterProvided && + cluster_type.has_value() && + cluster_type->name() == Extensions::Clusters::ClusterTypes::get().Redis; } void InstanceImpl::ThreadLocalPool::onClusterRemoval(const std::string& cluster_name) { @@ -106,25 +122,71 @@ void InstanceImpl::ThreadLocalPool::onClusterRemoval(const std::string& cluster_ // Treat cluster removal as a removal of all hosts. Close all connections and fail all pending // requests. + if (host_set_member_update_cb_handle_ != nullptr) { + host_set_member_update_cb_handle_->remove(); + host_set_member_update_cb_handle_ = nullptr; + } while (!client_map_.empty()) { client_map_.begin()->second->redis_client_->close(); } + while (!clients_to_drain_.empty()) { + (*clients_to_drain_.begin())->redis_client_->close(); + } cluster_ = nullptr; - host_set_member_update_cb_handle_ = nullptr; host_address_map_.clear(); } +void InstanceImpl::ThreadLocalPool::onHostsAdded( + const std::vector& hosts_added) { + for (const auto& host : hosts_added) { + std::string host_address = host->address()->asString(); + // Insert new host into address map, possibly overwriting a previous host's entry. + host_address_map_[host_address] = host; + for (const auto& created_host : created_via_redirect_hosts_) { + if (created_host->address()->asString() == host_address) { + // Remove our "temporary" host created in makeRequestToHost(). + onHostsRemoved({created_host}); + created_via_redirect_hosts_.remove(created_host); + break; + } + } + } +} + void InstanceImpl::ThreadLocalPool::onHostsRemoved( const std::vector& hosts_removed) { for (const auto& host : hosts_removed) { auto it = client_map_.find(host); if (it != client_map_.end()) { - // We don't currently support any type of draining for redis connections. If a host is gone, - // we just close the connection. This will fail any pending requests. - it->second->redis_client_->close(); + if (it->second->redis_client_->active()) { + // Put the ThreadLocalActiveClient to the side to drain. + clients_to_drain_.push_back(std::move(it->second)); + client_map_.erase(it); + if (!drain_timer_->enabled()) { + drain_timer_->enableTimer(std::chrono::seconds(1)); + } + } else { + // There are no pending requests so close the connection. + it->second->redis_client_->close(); + } + } + // There is the possibility that multiple hosts with the same address + // are registered in host_address_map_ given that hosts may be created + // upon redirection or supplied as part of the cluster's definition. + auto it2 = host_address_map_.find(host->address()->asString()); + if ((it2 != host_address_map_.end()) && (it2->second == host)) { + host_address_map_.erase(it2); } - host_address_map_.erase(host->address()->asString()); + } +} + +void InstanceImpl::ThreadLocalPool::drainClients() { + while (!clients_to_drain_.empty() && !(*clients_to_drain_.begin())->redis_client_->active()) { + (*clients_to_drain_.begin())->redis_client_->close(); + } + if (!clients_to_drain_.empty()) { + drain_timer_->enableTimer(std::chrono::seconds(1)); } } @@ -155,11 +217,7 @@ InstanceImpl::ThreadLocalPool::makeRequest(const std::string& key, return nullptr; } - Upstream::ClusterInfoConstSharedPtr info = cluster_->info(); - const auto& cluster_type = info->clusterType(); - const bool use_crc16 = info->lbType() == Upstream::LoadBalancerType::ClusterProvided && - cluster_type.has_value() && - cluster_type->name() == Extensions::Clusters::ClusterTypes::get().Redis; + const bool use_crc16 = is_redis_cluster_; Clusters::Redis::RedisLoadBalancerContext lb_context(key, parent_.config_.enableHashtagging(), use_crc16); Upstream::HostConstSharedPtr host = cluster_->loadBalancer().chooseHost(&lb_context); @@ -169,12 +227,6 @@ InstanceImpl::ThreadLocalPool::makeRequest(const std::string& key, ThreadLocalActiveClientPtr& client = threadLocalActiveClient(host); - // Keep host_address_map_ in sync with client_map_. - auto host_cached_by_address = host_address_map_.find(host->address()->asString()); - if (host_cached_by_address == host_address_map_.end()) { - host_address_map_[host->address()->asString()] = host; - } - return client->redis_client_->makeRequest(request, callbacks); } @@ -217,9 +269,11 @@ InstanceImpl::ThreadLocalPool::makeRequestToHost(const std::string& host_address auto it = host_address_map_.find(host_address_map_key); if (it == host_address_map_.end()) { // This host is not known to the cluster manager. Create a new host and insert it into the map. - // TODO(msukalski): Add logic to track the number of these "unknown" host connections, - // cap the number of these connections, and implement time-out and cleaning logic, etc. - + if (created_via_redirect_hosts_.size() == parent_.config_.maxUpstreamUnknownConnections()) { + // Too many upstream connections to unknown hosts have been created. + parent_.redis_cluster_stats_.max_upstream_unknown_connections_reached_.inc(); + return nullptr; + } if (!ipv6) { // Only create an IPv4 address instance if we need a new Upstream::HostImpl. const auto ip_port = absl::string_view(host_address).substr(colon_pos + 1); @@ -239,6 +293,7 @@ InstanceImpl::ThreadLocalPool::makeRequestToHost(const std::string& host_address envoy::api::v2::endpoint::Endpoint::HealthCheckConfig::default_instance(), 0, envoy::api::v2::core::HealthStatus::UNKNOWN)}; host_address_map_[host_address_map_key] = new_host; + created_via_redirect_hosts_.push_back(new_host); it = host_address_map_.find(host_address_map_key); } @@ -251,9 +306,22 @@ void InstanceImpl::ThreadLocalActiveClient::onEvent(Network::ConnectionEvent eve if (event == Network::ConnectionEvent::RemoteClose || event == Network::ConnectionEvent::LocalClose) { auto client_to_delete = parent_.client_map_.find(host_); - ASSERT(client_to_delete != parent_.client_map_.end()); - parent_.dispatcher_.deferredDelete(std::move(client_to_delete->second->redis_client_)); - parent_.client_map_.erase(client_to_delete); + if (client_to_delete != parent_.client_map_.end()) { + parent_.dispatcher_.deferredDelete(std::move(redis_client_)); + parent_.client_map_.erase(client_to_delete); + } else { + for (auto it = parent_.clients_to_drain_.begin(); it != parent_.clients_to_drain_.end(); + it++) { + if ((*it).get() == this) { + if (!redis_client_->active()) { + parent_.parent_.redis_cluster_stats_.upstream_cx_drained_.inc(); + } + parent_.dispatcher_.deferredDelete(std::move(redis_client_)); + parent_.clients_to_drain_.erase(it); + break; + } + } + } } } diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index 34f86c5bb78e..ff5b52ed5067 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -9,6 +9,7 @@ #include #include "envoy/config/filter/network/redis_proxy/v2/redis_proxy.pb.h" +#include "envoy/stats/stats_macros.h" #include "envoy/thread_local/thread_local.h" #include "envoy/upstream/cluster_manager.h" @@ -36,13 +37,21 @@ namespace ConnPool { // TODO(mattklein123): Circuit breaking // TODO(rshriram): Fault injection +#define REDIS_CLUSTER_STATS(COUNTER) \ + COUNTER(upstream_cx_drained) \ + COUNTER(max_upstream_unknown_connections_reached) + +struct RedisClusterStats { + REDIS_CLUSTER_STATS(GENERATE_COUNTER_STRUCT) +}; + class InstanceImpl : public Instance { public: InstanceImpl( const std::string& cluster_name, Upstream::ClusterManager& cm, Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, - Api::Api& api, Stats::SymbolTable& symbol_table); + Api::Api& api, Stats::ScopePtr&& stats_scope); // RedisProxy::ConnPool::Instance Common::Redis::Client::PoolRequest* makeRequest(const std::string& key, const Common::Redis::RespValue& request, @@ -50,7 +59,6 @@ class InstanceImpl : public Instance { Common::Redis::Client::PoolRequest* makeRequestToHost(const std::string& host_address, const Common::Redis::RespValue& request, Common::Redis::Client::PoolCallbacks& callbacks) override; - Stats::SymbolTable& symbolTable() { return symbol_table_; } // Allow the unit test to have access to private members. friend class RedisConnPoolImplTest; @@ -85,7 +93,9 @@ class InstanceImpl : public Instance { makeRequestToHost(const std::string& host_address, const Common::Redis::RespValue& request, Common::Redis::Client::PoolCallbacks& callbacks); void onClusterAddOrUpdateNonVirtual(Upstream::ThreadLocalCluster& cluster); + void onHostsAdded(const std::vector& hosts_added); void onHostsRemoved(const std::vector& hosts_removed); + void drainClients(); // Upstream::ClusterUpdateCallbacks void onClusterAddOrUpdate(Upstream::ThreadLocalCluster& cluster) override { @@ -102,6 +112,17 @@ class InstanceImpl : public Instance { Envoy::Common::CallbackHandle* host_set_member_update_cb_handle_{}; std::unordered_map host_address_map_; std::string auth_password_; + std::list created_via_redirect_hosts_; + std::list clients_to_drain_; + + /* This timer is used to poll the active clients in clients_to_drain_ to determine whether they + * have been drained (have no active requests) or not. It is only enabled after a client has + * been added to clients_to_drain_, and is only re-enabled as long as that list is not empty. A + * timer is being used as opposed to using a callback to avoid adding a check of + * clients_to_drain_ to the main data code path as this should only rarely be not empty. + */ + Event::TimerPtr drain_timer_; + bool is_redis_cluster_; }; Upstream::ClusterManager& cm_; @@ -109,7 +130,8 @@ class InstanceImpl : public Instance { ThreadLocal::SlotPtr tls_; Common::Redis::Client::ConfigImpl config_; Api::Api& api_; - Stats::SymbolTable& symbol_table_; + Stats::ScopePtr stats_scope_; + RedisClusterStats redis_cluster_stats_; }; } // namespace ConnPool diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index b046da958931..63f9dd0de0d5 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -77,6 +77,8 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { return std::chrono::milliseconds(1); } + uint32_t maxUpstreamUnknownConnections() const override { return 0; } + // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; void onFailure() override; diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index ef1603d87b76..4d60412503ff 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -401,6 +401,7 @@ class RedisClusterTest : public testing::Test, RedisCluster::RedisDiscoverySession discovery_session(*cluster_, *this); EXPECT_FALSE(discovery_session.enableHashtagging()); EXPECT_EQ(discovery_session.bufferFlushTimeoutInMs(), std::chrono::milliseconds(0)); + EXPECT_EQ(discovery_session.maxUpstreamUnknownConnections(), 0); NetworkFilters::Common::Redis::RespValue dummy_value; dummy_value.type(NetworkFilters::Common::Redis::RespType::Error); diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 444ef060931f..ecea5c887c22 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -80,6 +80,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF *config_); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_cx_total_.value()); EXPECT_EQ(1UL, host_->stats_.cx_total_.value()); + EXPECT_EQ(false, client_->active()); // NOP currently. upstream_connection_->runHighWatermarkCallbacks(); @@ -95,6 +96,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF Common::Redis::RespValuePtr response1{new Common::Redis::RespValue()}; response1->type(Common::Redis::RespType::SimpleString); response1->asString() = "OK"; + EXPECT_EQ(true, client_->active()); ClientImpl* client_impl = dynamic_cast(client_.get()); EXPECT_NE(client_impl, nullptr); client_impl->onRespValue(std::move(response1)); @@ -155,6 +157,7 @@ class ConfigBufferSizeGTSingleRequest : public Config { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return std::chrono::milliseconds(1); } + uint32_t maxUpstreamUnknownConnections() const override { return 0; } }; TEST_F(RedisClientImplTest, BatchWithTimerFiring) { @@ -466,6 +469,7 @@ class ConfigOutlierDisabled : public Config { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return std::chrono::milliseconds(0); } + uint32_t maxUpstreamUnknownConnections() const override { return 0; } }; TEST_F(RedisClientImplTest, OutlierDisabled) { diff --git a/test/extensions/filters/network/common/redis/mocks.h b/test/extensions/filters/network/common/redis/mocks.h index 37b90626c4e4..829983abd53d 100644 --- a/test/extensions/filters/network/common/redis/mocks.h +++ b/test/extensions/filters/network/common/redis/mocks.h @@ -69,6 +69,7 @@ class MockClient : public Client { } MOCK_METHOD1(addConnectionCallbacks, void(Network::ConnectionCallbacks& callbacks)); + MOCK_METHOD0(active, bool()); MOCK_METHOD0(close, void()); MOCK_METHOD2(makeRequest, PoolRequest*(const Common::Redis::RespValue& request, PoolCallbacks& callbacks)); diff --git a/test/extensions/filters/network/common/redis/test_utils.h b/test/extensions/filters/network/common/redis/test_utils.h index c81cb2647f95..e720394b83f2 100644 --- a/test/extensions/filters/network/common/redis/test_utils.h +++ b/test/extensions/filters/network/common/redis/test_utils.h @@ -17,11 +17,12 @@ namespace Client { inline envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings createConnPoolSettings(int64_t millis = 20, bool hashtagging = true, - bool redirection_support = true) { + bool redirection_support = true, uint32_t max_unknown_conns = 100) { envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings setting{}; setting.mutable_op_timeout()->CopyFrom(Protobuf::util::TimeUtil::MillisecondsToDuration(millis)); setting.set_enable_hashtagging(hashtagging); setting.set_enable_redirection(redirection_support); + setting.mutable_max_upstream_unknown_connections()->set_value(max_unknown_conns); return setting; } diff --git a/test/extensions/filters/network/redis_proxy/BUILD b/test/extensions/filters/network/redis_proxy/BUILD index d7b4bc5dacb1..1f9a52b6a029 100644 --- a/test/extensions/filters/network/redis_proxy/BUILD +++ b/test/extensions/filters/network/redis_proxy/BUILD @@ -127,7 +127,6 @@ envoy_extension_cc_test( envoy_extension_cc_test( name = "redis_proxy_integration_test", - size = "small", srcs = ["redis_proxy_integration_test.cc"], extension_name = "envoy.filters.network.redis_proxy", deps = [ diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 5ecb9e9aed53..ee16cfaa5015 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -1,7 +1,6 @@ #include #include "common/network/utility.h" -#include "common/stats/fake_symbol_table_impl.h" #include "common/upstream/upstream_impl.h" #include "extensions/filters/network/common/redis/utility.h" @@ -40,7 +39,8 @@ namespace ConnPool { class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client::ClientFactory { public: - void setup(bool cluster_exists = true, bool hashtagging = true) { + void setup(bool cluster_exists = true, bool hashtagging = true, + uint32_t max_unknown_conns = 100) { EXPECT_CALL(cm_, addThreadLocalClusterUpdateCallbacks_(_)) .WillOnce(DoAll(SaveArgAddress(&update_callbacks_), ReturnNew())); @@ -48,9 +48,33 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client EXPECT_CALL(cm_, get(Eq("fake_cluster"))).WillOnce(Return(nullptr)); } + std::unique_ptr> store = + std::make_unique>(); + + upstream_cx_drained_.value_ = 0; + ON_CALL(*store, counter(Eq("upstream_cx_drained"))) + .WillByDefault(ReturnRef(upstream_cx_drained_)); + ON_CALL(upstream_cx_drained_, value()).WillByDefault(Invoke([&]() -> uint64_t { + return upstream_cx_drained_.value_; + })); + ON_CALL(upstream_cx_drained_, inc()).WillByDefault(Invoke([&]() { + upstream_cx_drained_.value_++; + })); + + max_upstream_unknown_connections_reached_.value_ = 0; + ON_CALL(*store, counter(Eq("max_upstream_unknown_connections_reached"))) + .WillByDefault(ReturnRef(max_upstream_unknown_connections_reached_)); + ON_CALL(max_upstream_unknown_connections_reached_, value()) + .WillByDefault( + Invoke([&]() -> uint64_t { return max_upstream_unknown_connections_reached_.value_; })); + ON_CALL(max_upstream_unknown_connections_reached_, inc()).WillByDefault(Invoke([&]() { + max_upstream_unknown_connections_reached_.value_++; + })); + std::unique_ptr conn_pool_impl = std::make_unique( cluster_name_, cm_, *this, tls_, - Common::Redis::Client::createConnPoolSettings(20, hashtagging, true), api_, *symbol_table_); + Common::Redis::Client::createConnPoolSettings(20, hashtagging, true, max_unknown_conns), + api_, std::move(store)); // Set the authentication password for this connection pool. conn_pool_impl->tls_->getTyped().auth_password_ = auth_password_; conn_pool_ = std::move(conn_pool_impl); @@ -86,6 +110,53 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client EXPECT_EQ(&active_request, request); } + std::unordered_map& + clientMap() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->tls_->getTyped().client_map_; + } + + InstanceImpl::ThreadLocalActiveClient* clientMap(Upstream::HostConstSharedPtr host) { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->tls_->getTyped().client_map_[host].get(); + } + + std::unordered_map& hostAddressMap() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->tls_->getTyped().host_address_map_; + } + + std::list& createdViaRedirectHosts() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->tls_->getTyped() + .created_via_redirect_hosts_; + } + + std::list& clientsToDrain() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->tls_->getTyped().clients_to_drain_; + } + + Event::TimerPtr& drainTimer() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->tls_->getTyped().drain_timer_; + } + + void drainClients() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + conn_pool_impl->tls_->getTyped().drainClients(); + } + + Stats::Counter& upstreamCxDrained() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->redis_cluster_stats_.upstream_cx_drained_; + } + + Stats::Counter& maxUpstreamUnknownConnectionsReached() { + InstanceImpl* conn_pool_impl = dynamic_cast(conn_pool_.get()); + return conn_pool_impl->redis_cluster_stats_.max_upstream_unknown_connections_reached_; + } + // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Common::Redis::Client::Config&) override { @@ -95,7 +166,6 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client MOCK_METHOD1(create_, Common::Redis::Client::Client*(Upstream::HostConstSharedPtr host)); const std::string cluster_name_{"fake_cluster"}; - Envoy::Test::Global symbol_table_; NiceMock cm_; NiceMock tls_; InstanceSharedPtr conn_pool_; @@ -104,6 +174,8 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client Network::Address::InstanceConstSharedPtr test_address_; std::string auth_password_; NiceMock api_; + NiceMock upstream_cx_drained_; + NiceMock max_upstream_unknown_connections_reached_; }; TEST_F(RedisConnPoolImplTest, Basic) { @@ -234,7 +306,7 @@ TEST_F(RedisConnPoolImplTest, HashtaggingNotEnabled) { tls_.shutdownThread(); }; -// Conn pool created when no cluster exists at creation time. Dynamic cluster creation and removal +// ConnPool created when no cluster exists at creation time. Dynamic cluster creation and removal // work correctly. TEST_F(RedisConnPoolImplTest, NoClusterAtConstruction) { InSequence s; @@ -285,6 +357,8 @@ TEST_F(RedisConnPoolImplTest, NoClusterAtConstruction) { update_callbacks_->onClusterRemoval("fake_cluster"); } +// This test removes a single host from the ConnPool after learning about 2 hosts from the +// associated load balancer. TEST_F(RedisConnPoolImplTest, HostRemove) { InSequence s; @@ -329,6 +403,23 @@ TEST_F(RedisConnPoolImplTest, HostRemove) { testing::Mock::AllowLeak(host2.get()); } +// This test removes a host from a ConnPool that was never added in the first place. No errors +// should be encountered. +TEST_F(RedisConnPoolImplTest, HostRemovedNeverAdded) { + InSequence s; + + setup(); + + std::shared_ptr host1(new Upstream::MockHost()); + auto host1_test_address = Network::Utility::resolveUrl("tcp://10.0.0.1:3000"); + EXPECT_CALL(*host1, address()).WillOnce(Return(host1_test_address)); + EXPECT_NO_THROW(cm_.thread_local_cluster_.cluster_.prioritySet().getMockHostSet(0)->runCallbacks( + {}, {host1})); + EXPECT_EQ(hostAddressMap().size(), 0); + + tls_.shutdownThread(); +} + TEST_F(RedisConnPoolImplTest, DeleteFollowedByClusterUpdateCallback) { setup(); conn_pool_.reset(); @@ -377,7 +468,7 @@ TEST_F(RedisConnPoolImplTest, RemoteClose) { tls_.shutdownThread(); } -TEST_F(RedisConnPoolImplTest, makeRequestToHost) { +TEST_F(RedisConnPoolImplTest, MakeRequestToHost) { InSequence s; setup(false); @@ -437,7 +528,22 @@ TEST_F(RedisConnPoolImplTest, makeRequestToHost) { tls_.shutdownThread(); } -TEST_F(RedisConnPoolImplTest, makeRequestToHostWithAuthPassword) { +TEST_F(RedisConnPoolImplTest, MakeRequestToHostWithZeroMaxUnknownUpstreamConnectionLimit) { + InSequence s; + + // Create a ConnPool with a max_upstream_unknown_connections setting of 0. + setup(true, true, 0); + + Common::Redis::RespValue value; + Common::Redis::Client::MockPoolCallbacks callbacks1; + + // The max_unknown_upstream_connections is set to 0. Request should fail. + EXPECT_EQ(nullptr, conn_pool_->makeRequestToHost("10.0.0.1:3000", value, callbacks1)); + EXPECT_EQ(maxUpstreamUnknownConnectionsReached().value(), 1); + tls_.shutdownThread(); +} + +TEST_F(RedisConnPoolImplTest, MakeRequestToHostWithAuthPassword) { InSequence s; auth_password_ = "superduperpassword"; @@ -490,17 +596,275 @@ TEST_F(RedisConnPoolImplTest, makeRequestToHostWithAuthPassword) { tls_.shutdownThread(); } +// This test forces the creation of 2 hosts (one with an IPv4 address, and the other with an IPv6 +// address) and pending requests using makeRequestToHost(). After their creation, "new" hosts are +// discovered, and the original hosts are put aside to drain. The test then verifies the drain +// logic. +TEST_F(RedisConnPoolImplTest, HostsAddedAndRemovedWithDraining) { + setup(); + + Common::Redis::RespValue value; + Common::Redis::Client::MockPoolRequest auth_request1, active_request1; + Common::Redis::Client::MockPoolRequest auth_request2, active_request2; + Common::Redis::Client::MockPoolCallbacks callbacks1; + Common::Redis::Client::MockPoolCallbacks callbacks2; + Common::Redis::Client::MockClient* client1 = new NiceMock(); + Common::Redis::Client::MockClient* client2 = new NiceMock(); + Upstream::HostConstSharedPtr host1; + Upstream::HostConstSharedPtr host2; + + EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host1), Return(client1))); + EXPECT_CALL(*client1, makeRequest(Ref(value), Ref(callbacks1))) + .WillOnce(Return(&active_request1)); + Common::Redis::Client::PoolRequest* request1 = + conn_pool_->makeRequestToHost("10.0.0.1:3000", value, callbacks1); + EXPECT_EQ(&active_request1, request1); + EXPECT_EQ(host1->address()->asString(), "10.0.0.1:3000"); + + // IPv6 address returned from Redis server will not have square brackets + // around it, while Envoy represents Address::Ipv6Instance addresses with square brackets around + // the address. + EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host2), Return(client2))); + EXPECT_CALL(*client2, makeRequest(Ref(value), Ref(callbacks2))) + .WillOnce(Return(&active_request2)); + Common::Redis::Client::PoolRequest* request2 = + conn_pool_->makeRequestToHost("2001:470:813B:0:0:0:0:1:3333", value, callbacks2); + EXPECT_EQ(&active_request2, request2); + EXPECT_EQ(host2->address()->asString(), "[2001:470:813b::1]:3333"); + + std::unordered_map& host_address_map = + hostAddressMap(); + EXPECT_EQ(host_address_map.size(), 2); // host1 and host2 have been created. + EXPECT_EQ(host_address_map[host1->address()->asString()], host1); + EXPECT_EQ(host_address_map[host2->address()->asString()], host2); + EXPECT_EQ(clientMap().size(), 2); + EXPECT_NE(clientMap().find(host1), clientMap().end()); + EXPECT_NE(clientMap().find(host2), clientMap().end()); + void* host1_active_client = clientMap(host1); + EXPECT_EQ(createdViaRedirectHosts().size(), 2); + EXPECT_EQ(clientsToDrain().size(), 0); + EXPECT_EQ(drainTimer()->enabled(), false); + + std::shared_ptr new_host1(new Upstream::MockHost()); + std::shared_ptr new_host2(new Upstream::MockHost()); + auto new_host1_test_address = Network::Utility::resolveUrl("tcp://10.0.0.1:3000"); + auto new_host2_test_address = Network::Utility::resolveUrl("tcp://[2001:470:813b::1]:3333"); + EXPECT_CALL(*new_host1, address()).WillRepeatedly(Return(new_host1_test_address)); + EXPECT_CALL(*new_host2, address()).WillRepeatedly(Return(new_host2_test_address)); + EXPECT_CALL(*client1, active()).WillOnce(Return(true)); + EXPECT_CALL(*client2, active()).WillOnce(Return(false)); + EXPECT_CALL(*client2, close()); + + cm_.thread_local_cluster_.cluster_.prioritySet().getMockHostSet(0)->runCallbacks( + {new_host1, new_host2}, {}); + + host_address_map = hostAddressMap(); + EXPECT_EQ(host_address_map.size(), 2); // new_host1 and new_host2 have been added. + EXPECT_EQ(host_address_map[new_host1_test_address->asString()], new_host1); + EXPECT_EQ(host_address_map[new_host2_test_address->asString()], new_host2); + EXPECT_EQ(clientMap().size(), 0); + EXPECT_EQ(createdViaRedirectHosts().size(), 0); + EXPECT_EQ(clientsToDrain().size(), 1); // client2 has already been drained. + EXPECT_EQ(clientsToDrain().front().get(), host1_active_client); // client1 is still active. + EXPECT_EQ(drainTimer()->enabled(), true); + + cm_.thread_local_cluster_.cluster_.prioritySet().getMockHostSet(0)->runCallbacks( + {}, {new_host1, new_host2}); + + EXPECT_EQ(host_address_map.size(), 0); // new_host1 and new_host2 have been removed. + EXPECT_EQ(clientMap().size(), 0); + EXPECT_EQ(createdViaRedirectHosts().size(), 0); + EXPECT_EQ(clientsToDrain().size(), 1); + EXPECT_EQ(clientsToDrain().front().get(), host1_active_client); + EXPECT_EQ(drainTimer()->enabled(), true); + + EXPECT_CALL(*client1, active()).WillOnce(Return(true)); + drainTimer()->disableTimer(); + drainClients(); + EXPECT_EQ(clientsToDrain().size(), 1); // Nothing happened. client1 is still active. + EXPECT_EQ(drainTimer()->enabled(), true); + + EXPECT_CALL(*client1, active()).Times(2).WillRepeatedly(Return(false)); + EXPECT_CALL(*client1, close()); + drainTimer()->disableTimer(); + drainClients(); + EXPECT_EQ(clientsToDrain().size(), 0); // client1 has been drained and closed. + EXPECT_EQ(drainTimer()->enabled(), false); + EXPECT_EQ(upstreamCxDrained().value(), 1); + + tls_.shutdownThread(); +} + +// This test creates 2 hosts (one with an IPv4 address, and the other with an IPv6 +// address) and pending requests using makeRequestToHost(). After their creation, "new" hosts are +// discovered (added), and the original hosts are put aside to drain. Destructors are then +// called on these not yet drained clients, and the underlying connections should be closed. +TEST_F(RedisConnPoolImplTest, HostsAddedAndEndWithNoDraining) { + setup(); + + Common::Redis::RespValue value; + Common::Redis::Client::MockPoolRequest auth_request1, active_request1; + Common::Redis::Client::MockPoolRequest auth_request2, active_request2; + Common::Redis::Client::MockPoolCallbacks callbacks1; + Common::Redis::Client::MockPoolCallbacks callbacks2; + Common::Redis::Client::MockClient* client1 = new NiceMock(); + Common::Redis::Client::MockClient* client2 = new NiceMock(); + Upstream::HostConstSharedPtr host1; + Upstream::HostConstSharedPtr host2; + + EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host1), Return(client1))); + EXPECT_CALL(*client1, makeRequest(Ref(value), Ref(callbacks1))) + .WillOnce(Return(&active_request1)); + Common::Redis::Client::PoolRequest* request1 = + conn_pool_->makeRequestToHost("10.0.0.1:3000", value, callbacks1); + EXPECT_EQ(&active_request1, request1); + EXPECT_EQ(host1->address()->asString(), "10.0.0.1:3000"); + + // IPv6 address returned from Redis server will not have square brackets + // around it, while Envoy represents Address::Ipv6Instance addresses with square brackets around + // the address. + EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host2), Return(client2))); + EXPECT_CALL(*client2, makeRequest(Ref(value), Ref(callbacks2))) + .WillOnce(Return(&active_request2)); + Common::Redis::Client::PoolRequest* request2 = + conn_pool_->makeRequestToHost("2001:470:813B:0:0:0:0:1:3333", value, callbacks2); + EXPECT_EQ(&active_request2, request2); + EXPECT_EQ(host2->address()->asString(), "[2001:470:813b::1]:3333"); + + std::unordered_map& host_address_map = + hostAddressMap(); + EXPECT_EQ(host_address_map.size(), 2); // host1 and host2 have been created. + EXPECT_EQ(host_address_map[host1->address()->asString()], host1); + EXPECT_EQ(host_address_map[host2->address()->asString()], host2); + EXPECT_EQ(clientMap().size(), 2); + EXPECT_NE(clientMap().find(host1), clientMap().end()); + EXPECT_NE(clientMap().find(host2), clientMap().end()); + EXPECT_EQ(createdViaRedirectHosts().size(), 2); + EXPECT_EQ(clientsToDrain().size(), 0); + EXPECT_EQ(drainTimer()->enabled(), false); + + std::shared_ptr new_host1(new Upstream::MockHost()); + std::shared_ptr new_host2(new Upstream::MockHost()); + auto new_host1_test_address = Network::Utility::resolveUrl("tcp://10.0.0.1:3000"); + auto new_host2_test_address = Network::Utility::resolveUrl("tcp://[2001:470:813b::1]:3333"); + EXPECT_CALL(*new_host1, address()).WillRepeatedly(Return(new_host1_test_address)); + EXPECT_CALL(*new_host2, address()).WillRepeatedly(Return(new_host2_test_address)); + EXPECT_CALL(*client1, active()).WillOnce(Return(true)); + EXPECT_CALL(*client2, active()).WillOnce(Return(true)); + + cm_.thread_local_cluster_.cluster_.prioritySet().getMockHostSet(0)->runCallbacks( + {new_host1, new_host2}, {}); + + host_address_map = hostAddressMap(); + EXPECT_EQ(host_address_map.size(), 2); // new_host1 and new_host2 have been added. + EXPECT_EQ(host_address_map[new_host1_test_address->asString()], new_host1); + EXPECT_EQ(host_address_map[new_host2_test_address->asString()], new_host2); + EXPECT_EQ(clientMap().size(), 0); + EXPECT_EQ(createdViaRedirectHosts().size(), 0); + EXPECT_EQ(clientsToDrain().size(), 2); // host1 and host2 have been put aside to drain. + EXPECT_EQ(drainTimer()->enabled(), true); + + EXPECT_CALL(*client1, close()); + EXPECT_CALL(*client2, close()); + EXPECT_CALL(*client1, active()).WillOnce(Return(true)); + EXPECT_CALL(*client2, active()).WillOnce(Return(true)); + EXPECT_EQ(upstreamCxDrained().value(), 0); + + tls_.shutdownThread(); +} + +// This test creates 2 hosts (one with an IPv4 address, and the other with an IPv6 +// address) and pending requests using makeRequestToHost(). After their creation, "new" hosts are +// discovered (added), and the original hosts are put aside to drain. The cluster is removed and the +// underlying connections should be closed. +TEST_F(RedisConnPoolImplTest, HostsAddedAndEndWithClusterRemoval) { + setup(); + + Common::Redis::RespValue value; + Common::Redis::Client::MockPoolRequest auth_request1, active_request1; + Common::Redis::Client::MockPoolRequest auth_request2, active_request2; + Common::Redis::Client::MockPoolCallbacks callbacks1; + Common::Redis::Client::MockPoolCallbacks callbacks2; + Common::Redis::Client::MockClient* client1 = new NiceMock(); + Common::Redis::Client::MockClient* client2 = new NiceMock(); + Upstream::HostConstSharedPtr host1; + Upstream::HostConstSharedPtr host2; + + EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host1), Return(client1))); + EXPECT_CALL(*client1, makeRequest(Ref(value), Ref(callbacks1))) + .WillOnce(Return(&active_request1)); + Common::Redis::Client::PoolRequest* request1 = + conn_pool_->makeRequestToHost("10.0.0.1:3000", value, callbacks1); + EXPECT_EQ(&active_request1, request1); + EXPECT_EQ(host1->address()->asString(), "10.0.0.1:3000"); + + // IPv6 address returned from Redis server will not have square brackets + // around it, while Envoy represents Address::Ipv6Instance addresses with square brackets around + // the address. + EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host2), Return(client2))); + EXPECT_CALL(*client2, makeRequest(Ref(value), Ref(callbacks2))) + .WillOnce(Return(&active_request2)); + Common::Redis::Client::PoolRequest* request2 = + conn_pool_->makeRequestToHost("2001:470:813B:0:0:0:0:1:3333", value, callbacks2); + EXPECT_EQ(&active_request2, request2); + EXPECT_EQ(host2->address()->asString(), "[2001:470:813b::1]:3333"); + + std::unordered_map& host_address_map = + hostAddressMap(); + EXPECT_EQ(host_address_map.size(), 2); // host1 and host2 have been created. + EXPECT_EQ(host_address_map[host1->address()->asString()], host1); + EXPECT_EQ(host_address_map[host2->address()->asString()], host2); + EXPECT_EQ(clientMap().size(), 2); + EXPECT_NE(clientMap().find(host1), clientMap().end()); + EXPECT_NE(clientMap().find(host2), clientMap().end()); + EXPECT_EQ(createdViaRedirectHosts().size(), 2); + EXPECT_EQ(clientsToDrain().size(), 0); + EXPECT_EQ(drainTimer()->enabled(), false); + + std::shared_ptr new_host1(new Upstream::MockHost()); + std::shared_ptr new_host2(new Upstream::MockHost()); + auto new_host1_test_address = Network::Utility::resolveUrl("tcp://10.0.0.1:3000"); + auto new_host2_test_address = Network::Utility::resolveUrl("tcp://[2001:470:813b::1]:3333"); + EXPECT_CALL(*new_host1, address()).WillRepeatedly(Return(new_host1_test_address)); + EXPECT_CALL(*new_host2, address()).WillRepeatedly(Return(new_host2_test_address)); + EXPECT_CALL(*client1, active()).WillOnce(Return(true)); + EXPECT_CALL(*client2, active()).WillOnce(Return(true)); + + cm_.thread_local_cluster_.cluster_.prioritySet().getMockHostSet(0)->runCallbacks( + {new_host1, new_host2}, {}); + + host_address_map = hostAddressMap(); + EXPECT_EQ(host_address_map.size(), 2); // new_host1 and new_host2 have been added. + EXPECT_EQ(host_address_map[new_host1_test_address->asString()], new_host1); + EXPECT_EQ(host_address_map[new_host2_test_address->asString()], new_host2); + EXPECT_EQ(clientMap().size(), 0); + EXPECT_EQ(createdViaRedirectHosts().size(), 0); + EXPECT_EQ(clientsToDrain().size(), 2); // host1 and host2 have been put aside to drain. + EXPECT_EQ(drainTimer()->enabled(), true); + + EXPECT_CALL(*client1, close()); + EXPECT_CALL(*client2, close()); + EXPECT_CALL(*client1, active()).WillOnce(Return(true)); + EXPECT_CALL(*client2, active()).WillOnce(Return(true)); + update_callbacks_->onClusterRemoval("fake_cluster"); + + EXPECT_EQ(hostAddressMap().size(), 0); + EXPECT_EQ(clientMap().size(), 0); + EXPECT_EQ(clientsToDrain().size(), 0); + EXPECT_EQ(upstreamCxDrained().value(), 0); + + tls_.shutdownThread(); +} + TEST_F(RedisConnPoolImplTest, MakeRequestToRedisCluster) { absl::optional cluster_type; cluster_type.emplace(); cluster_type->set_name("envoy.clusters.redis"); EXPECT_CALL(*cm_.thread_local_cluster_.cluster_.info_, clusterType()) - .Times(2) - .WillRepeatedly(ReturnRef(cluster_type)); + .WillOnce(ReturnRef(cluster_type)); EXPECT_CALL(*cm_.thread_local_cluster_.cluster_.info_, lbType()) - .Times(2) - .WillRepeatedly(Return(Upstream::LoadBalancerType::ClusterProvided)); + .WillOnce(Return(Upstream::LoadBalancerType::ClusterProvided)); setup(); @@ -518,11 +882,9 @@ TEST_F(RedisConnPoolImplTest, MakeRequestToRedisClusterHashtag) { cluster_type.emplace(); cluster_type->set_name("envoy.clusters.redis"); EXPECT_CALL(*cm_.thread_local_cluster_.cluster_.info_, clusterType()) - .Times(2) - .WillRepeatedly(ReturnRef(cluster_type)); + .WillOnce(ReturnRef(cluster_type)); EXPECT_CALL(*cm_.thread_local_cluster_.cluster_.info_, lbType()) - .Times(2) - .WillRepeatedly(Return(Upstream::LoadBalancerType::ClusterProvided)); + .WillOnce(Return(Upstream::LoadBalancerType::ClusterProvided)); setup(); diff --git a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc index c16599e6e274..444bbc4c1e47 100644 --- a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc +++ b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc @@ -976,7 +976,8 @@ TEST_P(RedisProxyWithMirrorsIntegrationTest, ExcludeReadCommands) { // command is not mirrored to cluster 1 FakeRawConnectionPtr cluster_1_connection; - EXPECT_FALSE(fake_upstreams_[2]->waitForRawConnection(cluster_1_connection)); + EXPECT_FALSE(fake_upstreams_[2]->waitForRawConnection(cluster_1_connection, + std::chrono::milliseconds(500))); redis_client->waitForData(get_response); EXPECT_EQ(get_response, redis_client->data()); diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index aa171538ee0f..d5741a158eb4 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -167,6 +167,7 @@ class RedisHealthCheckerTest EXPECT_TRUE(session->enableRedirection()); EXPECT_EQ(session->maxBufferSizeBeforeFlush(), 0); EXPECT_EQ(session->bufferFlushTimeoutInMs(), std::chrono::milliseconds(1)); + EXPECT_EQ(session->maxUpstreamUnknownConnections(), 0); session->onDeferredDeleteBase(); // This must be called to pass assertions in the destructor. } From 32bfd01da2a95c51e4ae18d134449ceb0123efb8 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Sun, 14 Jul 2019 15:49:33 -0700 Subject: [PATCH 168/542] ci: Enable bazel limited download flags (#7563) * ci: Enable bazel limited download flags Signed-off-by: Keith Smiley --- ci/setup_cache.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/setup_cache.sh b/ci/setup_cache.sh index 3e10133e717d..f97aa9b34963 100755 --- a/ci/setup_cache.sh +++ b/ci/setup_cache.sh @@ -17,6 +17,13 @@ if [[ ! -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_FILE}" fi +if [[ -n "${BAZEL_REMOTE_CACHE}" ]]; then + export BAZEL_BUILD_EXTRA_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS} \ + --experimental_inmemory_dotd_files \ + --experimental_inmemory_jdeps_files \ + --experimental_remote_download_outputs=toplevel" +fi + if [[ "${BAZEL_REMOTE_CACHE}" =~ ^http ]]; then if [[ ! -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then export BAZEL_BUILD_EXTRA_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS} \ From 9e3871ade779664ffb0a8f09b11688878253a082 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Sun, 14 Jul 2019 20:39:03 -0700 Subject: [PATCH 169/542] forward proxy: fix test flake (#7575) Fixes https://github.com/envoyproxy/envoy/issues/7570 Risk Level: Low Testing: 1000 runs of the flaking test Docs Changes: N/A Release Notes: N/A Signed-off-by: Matt Klein --- .../dynamic_forward_proxy/proxy_filter_integration_test.cc | 3 +++ test/integration/fake_upstream.cc | 7 +++++-- test/integration/fake_upstream.h | 4 +++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index b2750046922b..ddef72bc47fa 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -214,7 +214,10 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTlsInvalidSAN) { upstream_tls_ = true; upstream_cert_name_ = "upstream"; setup(); + // The upstream connection is going to fail handshake so make sure it can read and we expect + // it to disconnect. fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + fake_upstreams_[0]->setReadDisableOnNewConnection(false); codec_client_ = makeHttpConnection(lookupPort("http")); const Http::TestHeaderMapImpl request_headers{ diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 8cca9265525e..e6f9ad9d4b7b 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -397,7 +397,8 @@ FakeUpstream::FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket api_(Api::createApiForTest(stats_store_)), time_system_(time_system), dispatcher_(api_->allocateDispatcher()), handler_(new Server::ConnectionHandlerImpl(ENVOY_LOGGER(), *dispatcher_)), - allow_unexpected_disconnects_(false), enable_half_close_(enable_half_close), listener_(*this), + allow_unexpected_disconnects_(false), read_disable_on_new_connection_(true), + enable_half_close_(enable_half_close), listener_(*this), filter_chain_(Network::Test::createEmptyFilterChain(std::move(transport_socket_factory))) { thread_ = api_->threadFactory().createThread([this]() -> void { threadRoutine(); }); server_initialized_.waitReady(); @@ -416,7 +417,9 @@ void FakeUpstream::cleanUp() { bool FakeUpstream::createNetworkFilterChain(Network::Connection& connection, const std::vector&) { Thread::LockGuard lock(lock_); - connection.readDisable(true); + if (read_disable_on_new_connection_) { + connection.readDisable(true); + } auto connection_wrapper = std::make_unique(connection, allow_unexpected_disconnects_); connection_wrapper->moveIntoListBack(std::move(connection_wrapper), new_connections_); diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 7ea255edf4f9..72b831ad4664 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -568,8 +568,9 @@ class FakeUpstream : Logger::Loggable, bool createListenerFilterChain(Network::ListenerFilterManager& listener) override; bool createUdpListenerFilterChain(Network::UdpListenerFilterManager& udp_listener, Network::UdpReadFilterCallbacks& callbacks) override; - void set_allow_unexpected_disconnects(bool value) { allow_unexpected_disconnects_ = value; } + void set_allow_unexpected_disconnects(bool value) { allow_unexpected_disconnects_ = value; } + void setReadDisableOnNewConnection(bool value) { read_disable_on_new_connection_ = value; } Event::TestTimeSystem& timeSystem() { return time_system_; } // Stops the dispatcher loop and joins the listening thread. @@ -628,6 +629,7 @@ class FakeUpstream : Logger::Loggable, // deleted) on the same thread that allocated the connection. std::list consumed_connections_ GUARDED_BY(lock_); bool allow_unexpected_disconnects_; + bool read_disable_on_new_connection_; const bool enable_half_close_; FakeListener listener_; const Network::FilterChainSharedPtr filter_chain_; From c5dddce0766d17f69010aeb3c45d222a0f86a343 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 15 Jul 2019 09:16:48 -0700 Subject: [PATCH 170/542] remove unused code (#7577) Risk Level: Low Testing: CI Docs Changes: n/a Release Notes: n/a Signed-off-by: Lizan Zhou --- source/common/http/http2/metadata_decoder.h | 7 ------- source/common/json/BUILD | 6 ------ source/common/json/json_validator.h | 21 ------------------- source/extensions/filters/http/gzip/BUILD | 2 -- .../filters/http/gzip/gzip_filter.h | 2 -- 5 files changed, 38 deletions(-) delete mode 100644 source/common/json/json_validator.h diff --git a/source/common/http/http2/metadata_decoder.h b/source/common/http/http2/metadata_decoder.h index 23331b67700b..16510776e563 100644 --- a/source/common/http/http2/metadata_decoder.h +++ b/source/common/http/http2/metadata_decoder.h @@ -44,13 +44,6 @@ class MetadataDecoder : Logger::Loggable { */ bool onMetadataFrameComplete(bool end_metadata); - /** - * @return payload_. - */ - Buffer::OwnedImpl& payload() { return payload_; } - - MetadataMap& getMetadataMap() { return *metadata_map_; } - private: friend class MetadataEncoderDecoderTest_VerifyEncoderDecoderOnMultipleMetadataMaps_Test; friend class MetadataEncoderDecoderTest_VerifyEncoderDecoderMultipleMetadataReachSizeLimit_Test; diff --git a/source/common/json/BUILD b/source/common/json/BUILD index 2fdeb38e270f..b21eab655757 100644 --- a/source/common/json/BUILD +++ b/source/common/json/BUILD @@ -30,9 +30,3 @@ envoy_cc_library( "//source/common/common:utility_lib", ], ) - -envoy_cc_library( - name = "json_validator_lib", - hdrs = ["json_validator.h"], - deps = ["//include/envoy/json:json_object_interface"], -) diff --git a/source/common/json/json_validator.h b/source/common/json/json_validator.h deleted file mode 100644 index b906eb23b3cd..000000000000 --- a/source/common/json/json_validator.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include - -#include "envoy/json/json_object.h" - -namespace Envoy { -namespace Json { - -/** - * Base class to inherit from to validate config schema before initializing member variables. - */ -class Validator { -public: - Validator(const Json::Object& config, const std::string& schema) { - config.validateSchema(schema); - } -}; - -} // namespace Json -} // namespace Envoy diff --git a/source/extensions/filters/http/gzip/BUILD b/source/extensions/filters/http/gzip/BUILD index 91efd5965891..6a1df0f1dd12 100644 --- a/source/extensions/filters/http/gzip/BUILD +++ b/source/extensions/filters/http/gzip/BUILD @@ -23,8 +23,6 @@ envoy_cc_library( "//source/common/compressor:compressor_lib", "//source/common/http:header_map_lib", "//source/common/http:headers_lib", - "//source/common/json:config_schemas_lib", - "//source/common/json:json_validator_lib", "//source/common/protobuf", "@envoy_api//envoy/config/filter/http/gzip/v2:gzip_cc", ], diff --git a/source/extensions/filters/http/gzip/gzip_filter.h b/source/extensions/filters/http/gzip/gzip_filter.h index 2346157070bb..99b3eaf3336e 100644 --- a/source/extensions/filters/http/gzip/gzip_filter.h +++ b/source/extensions/filters/http/gzip/gzip_filter.h @@ -11,8 +11,6 @@ #include "common/buffer/buffer_impl.h" #include "common/compressor/zlib_compressor_impl.h" #include "common/http/header_map_impl.h" -#include "common/json/config_schemas.h" -#include "common/json/json_validator.h" #include "common/protobuf/protobuf.h" namespace Envoy { From afb2370a11ce9d15ddbb1add68792dc215c0120d Mon Sep 17 00:00:00 2001 From: Matt Hoey Date: Mon, 15 Jul 2019 09:23:34 -0700 Subject: [PATCH 171/542] router: Add RETRY_ON_RESET retry policy (#7505) Signed-off-by: Matt Hoey --- .../configuration/http_filters/router_filter.rst | 3 +++ docs/root/faq/transient_failures.rst | 2 +- include/envoy/router/router.h | 1 + source/common/http/headers.h | 1 + source/common/router/retry_state_impl.cc | 10 ++++++++-- test/common/router/config_impl_test.cc | 10 +++++----- test/common/router/retry_state_impl_test.cc | 13 +++++++++++++ test/common/router/router_test.cc | 4 ++-- 8 files changed, 34 insertions(+), 10 deletions(-) diff --git a/docs/root/configuration/http_filters/router_filter.rst b/docs/root/configuration/http_filters/router_filter.rst index 0d416ac86502..c235c9faecae 100644 --- a/docs/root/configuration/http_filters/router_filter.rst +++ b/docs/root/configuration/http_filters/router_filter.rst @@ -79,6 +79,9 @@ gateway-error This policy is similar to the *5xx* policy but will only retry requests that result in a 502, 503, or 504. +reset + Envoy will attempt a retry if the upstream server does not respond at all (disconnect/reset/read timeout.) + connect-failure Envoy will attempt a retry if a request is failed because of a connection failure to the upstream server (connect timeout, etc.). (Included in *5xx*) diff --git a/docs/root/faq/transient_failures.rst b/docs/root/faq/transient_failures.rst index aadfa0986518..2e31976b5eb4 100644 --- a/docs/root/faq/transient_failures.rst +++ b/docs/root/faq/transient_failures.rst @@ -108,7 +108,7 @@ of times the host has been ejected). .. code-block:: json { - "retry_on": "cancelled,connect-failure,gateway-error,refused-stream,resource-exhausted,unavailable", + "retry_on": "cancelled,connect-failure,gateway-error,refused-stream,reset,resource-exhausted,unavailable", "num_retries": 1, "retry_host_predicate": [ { diff --git a/include/envoy/router/router.h b/include/envoy/router/router.h index c705a6499300..a14864b4b287 100644 --- a/include/envoy/router/router.h +++ b/include/envoy/router/router.h @@ -163,6 +163,7 @@ class RetryPolicy { static const uint32_t RETRY_ON_GRPC_UNAVAILABLE = 0x100; static const uint32_t RETRY_ON_GRPC_INTERNAL = 0x200; static const uint32_t RETRY_ON_RETRIABLE_STATUS_CODES = 0x400; + static const uint32_t RETRY_ON_RESET = 0x800; // clang-format on virtual ~RetryPolicy() = default; diff --git a/source/common/http/headers.h b/source/common/http/headers.h index f5a458ff04d6..b711fb4b142a 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -204,6 +204,7 @@ class HeaderValues { const std::string RefusedStream{"refused-stream"}; const std::string Retriable4xx{"retriable-4xx"}; const std::string RetriableStatusCodes{"retriable-status-codes"}; + const std::string Reset{"reset"}; } EnvoyRetryOnValues; struct { diff --git a/source/common/router/retry_state_impl.cc b/source/common/router/retry_state_impl.cc index c09f485a1c7a..e7674bd9ba93 100644 --- a/source/common/router/retry_state_impl.cc +++ b/source/common/router/retry_state_impl.cc @@ -22,6 +22,7 @@ const uint32_t RetryPolicy::RETRY_ON_GATEWAY_ERROR; const uint32_t RetryPolicy::RETRY_ON_CONNECT_FAILURE; const uint32_t RetryPolicy::RETRY_ON_RETRIABLE_4XX; const uint32_t RetryPolicy::RETRY_ON_RETRIABLE_STATUS_CODES; +const uint32_t RetryPolicy::RETRY_ON_RESET; const uint32_t RetryPolicy::RETRY_ON_GRPC_CANCELLED; const uint32_t RetryPolicy::RETRY_ON_GRPC_DEADLINE_EXCEEDED; const uint32_t RetryPolicy::RETRY_ON_GRPC_RESOURCE_EXHAUSTED; @@ -130,6 +131,8 @@ std::pair RetryStateImpl::parseRetryOn(absl::string_view config) ret |= RetryPolicy::RETRY_ON_REFUSED_STREAM; } else if (retry_on == Http::Headers::get().EnvoyRetryOnValues.RetriableStatusCodes) { ret |= RetryPolicy::RETRY_ON_RETRIABLE_STATUS_CODES; + } else if (retry_on == Http::Headers::get().EnvoyRetryOnValues.Reset) { + ret |= RetryPolicy::RETRY_ON_RESET; } else { all_fields_valid = false; } @@ -293,10 +296,13 @@ bool RetryStateImpl::wouldRetryFromReset(const Http::StreamResetReason reset_rea return false; } + if (retry_on_ & RetryPolicy::RETRY_ON_RESET) { + return true; + } + if (retry_on_ & (RetryPolicy::RETRY_ON_5XX | RetryPolicy::RETRY_ON_GATEWAY_ERROR)) { // Currently we count an upstream reset as a "5xx" (since it will result in - // one). We may eventually split this out into its own type. I.e., - // RETRY_ON_RESET. + // one). With RETRY_ON_RESET we may eventually remove these policies. return true; } diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 20dba970a5c7..755ef68d98d9 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -2319,7 +2319,7 @@ TEST_F(RouteMatcherTest, Retry) { retry_policy: per_try_timeout: 1s num_retries: 3 - retry_on: 5xx,gateway-error,connect-failure + retry_on: 5xx,gateway-error,connect-failure,reset )EOF"; TestConfigImpl config(parseRouteConfigurationFromV2Yaml(yaml), factory_context_, true); @@ -2363,7 +2363,7 @@ TEST_F(RouteMatcherTest, Retry) { ->retryPolicy() .numRetries()); EXPECT_EQ(RetryPolicy::RETRY_ON_CONNECT_FAILURE | RetryPolicy::RETRY_ON_5XX | - RetryPolicy::RETRY_ON_GATEWAY_ERROR, + RetryPolicy::RETRY_ON_GATEWAY_ERROR | RetryPolicy::RETRY_ON_RESET, config.route(genHeaders("www.lyft.com", "/", "GET"), 0) ->routeEntry() ->retryPolicy() @@ -2376,7 +2376,7 @@ name: RetryVirtualHostLevel virtual_hosts: - domains: [www.lyft.com] name: www - retry_policy: {num_retries: 3, per_try_timeout: 1s, retry_on: '5xx,gateway-error,connect-failure'} + retry_policy: {num_retries: 3, per_try_timeout: 1s, retry_on: '5xx,gateway-error,connect-failure,reset'} routes: - match: {prefix: /foo} route: @@ -2417,7 +2417,7 @@ name: RetryVirtualHostLevel ->retryPolicy() .numRetries()); EXPECT_EQ(RetryPolicy::RETRY_ON_CONNECT_FAILURE | RetryPolicy::RETRY_ON_5XX | - RetryPolicy::RETRY_ON_GATEWAY_ERROR, + RetryPolicy::RETRY_ON_GATEWAY_ERROR | RetryPolicy::RETRY_ON_RESET, config.route(genHeaders("www.lyft.com", "/bar", "GET"), 0) ->routeEntry() ->retryPolicy() @@ -2432,7 +2432,7 @@ name: RetryVirtualHostLevel ->retryPolicy() .numRetries()); EXPECT_EQ(RetryPolicy::RETRY_ON_CONNECT_FAILURE | RetryPolicy::RETRY_ON_5XX | - RetryPolicy::RETRY_ON_GATEWAY_ERROR, + RetryPolicy::RETRY_ON_GATEWAY_ERROR | RetryPolicy::RETRY_ON_RESET, config.route(genHeaders("www.lyft.com", "/", "GET"), 0) ->routeEntry() ->retryPolicy() diff --git a/test/common/router/retry_state_impl_test.cc b/test/common/router/retry_state_impl_test.cc index 415fa5b32ec2..7650fd5a3e54 100644 --- a/test/common/router/retry_state_impl_test.cc +++ b/test/common/router/retry_state_impl_test.cc @@ -412,6 +412,19 @@ TEST_F(RouterRetryStateImplTest, RetriableStatusCodesHeader) { } } +TEST_F(RouterRetryStateImplTest, PolicyResetRemoteReset) { + Http::TestHeaderMapImpl request_headers{{"x-envoy-retry-on", "reset"}}; + setup(request_headers); + EXPECT_TRUE(state_->enabled()); + + expectTimerCreateAndEnable(); + EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(remote_reset_, callback_)); + EXPECT_CALL(callback_ready_, ready()); + retry_timer_->callback_(); + + EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryReset(remote_reset_, callback_)); +} + TEST_F(RouterRetryStateImplTest, RouteConfigNoHeaderConfig) { policy_.num_retries_ = 1; policy_.retry_on_ = RetryPolicy::RETRY_ON_CONNECT_FAILURE; diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 23aa2817d88f..aa29d83fd57e 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -4388,8 +4388,8 @@ TEST(RouterFilterUtilityTest, StrictCheckValidHeaders) { {"x-envoy-upstream-rq-per-try-timeout-ms", "100"}, {"x-envoy-max-retries", "2"}, {"not-checked", "always passes"}, - {"x-envoy-retry-on", - "5xx,gateway-error,retriable-4xx,refused-stream,connect-failure,retriable-status-codes"}, + {"x-envoy-retry-on", "5xx,gateway-error,retriable-4xx,refused-stream,connect-failure," + "retriable-status-codes,reset"}, {"x-envoy-retry-grpc-on", "cancelled,internal,deadline-exceeded,resource-exhausted,unavailable"}, }; From e442ba565c948fb2d6c9d037ce12eaecbb75b252 Mon Sep 17 00:00:00 2001 From: "leilei.gll" Date: Tue, 16 Jul 2019 01:05:19 +0800 Subject: [PATCH 172/542] dubbo_proxy: Redefine DecoderFilter interface and add EncoderFilter support (#7118) Description: Redefine DecoderFilter interface and add EncoderFilter support Redefining DecoderFilter is mainly for 2 reasons: The original interface is not clear and complicated for the user, for example, the user needs to understand the meaning of transportBegin, transportEnd, messageBegin, messageEnd, transferHeaderTo, transferBodyTo, and we want to minimize this cost, so we decided to provide only one after internal discussion: onMessageDecoded interface. We used to have an internal implementation version and submitted a version to the community, but this obviously wastes a lot of development resources, so we decided to merge the two versions, mainly based on the open source version. As mentioned above, we merged the internal and open source versions. EncoderFilter is currently needed internally, such as the retry mechanism. we will consider adding more features to Dubbo later, similar to HTTP, so EncoderFilter is also necessary. we want to bring more features to users around Dubbo. Risk Level: low Testing: unit test Docs Changes: N/A Release Notes: N/A Signed-off-by: leilei.gll --- .../filters/network/dubbo_proxy/BUILD | 36 +- .../network/dubbo_proxy/active_message.cc | 364 ++++++++++------- .../network/dubbo_proxy/active_message.h | 152 ++++--- .../network/dubbo_proxy/app_exception.cc | 29 -- .../network/dubbo_proxy/app_exception.h | 38 +- .../filters/network/dubbo_proxy/config.cc | 36 +- .../filters/network/dubbo_proxy/config.h | 3 +- .../network/dubbo_proxy/conn_manager.cc | 36 +- .../network/dubbo_proxy/conn_manager.h | 15 +- .../filters/network/dubbo_proxy/decoder.cc | 136 +++---- .../filters/network/dubbo_proxy/decoder.h | 127 ++++-- .../dubbo_proxy/decoder_event_handler.h | 113 +++--- .../network/dubbo_proxy/deserializer_impl.cc | 9 - .../network/dubbo_proxy/deserializer_impl.h | 23 -- .../dubbo_hessian2_serializer_impl.cc | 118 ++++++ .../dubbo_hessian2_serializer_impl.h | 30 ++ .../dubbo_proxy/dubbo_protocol_impl.cc | 105 ++++- .../network/dubbo_proxy/dubbo_protocol_impl.h | 14 +- .../filters/network/dubbo_proxy/filters/BUILD | 2 +- .../network/dubbo_proxy/filters/filter.h | 124 ++++-- .../network/dubbo_proxy/heartbeat_response.cc | 10 +- .../network/dubbo_proxy/heartbeat_response.h | 4 +- .../dubbo_proxy/hessian_deserializer_impl.cc | 111 ----- .../dubbo_proxy/hessian_deserializer_impl.h | 27 -- .../filters/network/dubbo_proxy/message.h | 97 ++++- .../network/dubbo_proxy/message_impl.h | 65 +++ .../filters/network/dubbo_proxy/metadata.h | 105 ++--- .../filters/network/dubbo_proxy/protocol.h | 103 +++-- .../network/dubbo_proxy/protocol_constants.h | 98 +++++ .../filters/network/dubbo_proxy/router/BUILD | 17 +- .../network/dubbo_proxy/router/config.cc | 3 +- .../network/dubbo_proxy/router/route.h | 121 ++++++ .../dubbo_proxy/router/route_matcher.cc | 55 ++- .../dubbo_proxy/router/route_matcher.h | 17 +- .../network/dubbo_proxy/router/router.h | 3 + .../network/dubbo_proxy/router/router_impl.cc | 69 +--- .../network/dubbo_proxy/router/router_impl.h | 13 +- .../filters/network/dubbo_proxy/serializer.h | 128 ++++++ .../network/dubbo_proxy/serializer_impl.cc | 48 +++ .../network/dubbo_proxy/serializer_impl.h | 64 +++ .../filters/network/dubbo_proxy/BUILD | 16 +- .../network/dubbo_proxy/app_exception_test.cc | 33 +- .../network/dubbo_proxy/conn_manager_test.cc | 381 +++++++++--------- .../network/dubbo_proxy/decoder_test.cc | 260 ++++++------ ...=> dubbo_hessian2_serializer_impl_test.cc} | 90 +++-- .../dubbo_proxy/dubbo_protocol_impl_test.cc | 94 +++-- .../network/dubbo_proxy/metadata_test.cc | 62 +-- .../filters/network/dubbo_proxy/mocks.cc | 62 +-- .../filters/network/dubbo_proxy/mocks.h | 204 +++++++--- .../network/dubbo_proxy/route_matcher_test.cc | 196 +++++---- .../network/dubbo_proxy/router_test.cc | 113 +++--- 51 files changed, 2571 insertions(+), 1608 deletions(-) delete mode 100644 source/extensions/filters/network/dubbo_proxy/deserializer_impl.cc delete mode 100644 source/extensions/filters/network/dubbo_proxy/deserializer_impl.h create mode 100644 source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.cc create mode 100644 source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h delete mode 100644 source/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.cc delete mode 100644 source/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h create mode 100644 source/extensions/filters/network/dubbo_proxy/message_impl.h create mode 100644 source/extensions/filters/network/dubbo_proxy/protocol_constants.h create mode 100644 source/extensions/filters/network/dubbo_proxy/router/route.h create mode 100644 source/extensions/filters/network/dubbo_proxy/serializer.h create mode 100644 source/extensions/filters/network/dubbo_proxy/serializer_impl.cc create mode 100644 source/extensions/filters/network/dubbo_proxy/serializer_impl.h rename test/extensions/filters/network/dubbo_proxy/{hessian_deserializer_impl_test.cc => dubbo_hessian2_serializer_impl_test.cc} (56%) diff --git a/source/extensions/filters/network/dubbo_proxy/BUILD b/source/extensions/filters/network/dubbo_proxy/BUILD index 88c33334bca7..9ac6ff1e40d7 100644 --- a/source/extensions/filters/network/dubbo_proxy/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/BUILD @@ -36,6 +36,7 @@ envoy_cc_library( ":buffer_helper_lib", ":message_lib", ":metadata_lib", + ":serializer_interface", "//source/common/common:assert_lib", "//source/common/config:utility_lib", "//source/common/singleton:const_singleton", @@ -54,11 +55,12 @@ envoy_cc_library( ) envoy_cc_library( - name = "deserializer_interface", - srcs = ["deserializer_impl.cc"], + name = "serializer_interface", + srcs = ["serializer_impl.cc"], hdrs = [ - "deserializer.h", - "deserializer_impl.h", + "protocol_constants.h", + "serializer.h", + "serializer_impl.h", ], deps = [ ":message_lib", @@ -71,15 +73,15 @@ envoy_cc_library( ) envoy_cc_library( - name = "hessian_deserializer_impl_lib", - srcs = ["hessian_deserializer_impl.cc"], + name = "dubbo_hessian2_serializer_impl_lib", + srcs = ["dubbo_hessian2_serializer_impl.cc"], hdrs = [ - "hessian_deserializer_impl.h", + "dubbo_hessian2_serializer_impl.h", ], deps = [ ":buffer_helper_lib", - ":deserializer_interface", ":hessian_utils_lib", + ":serializer_interface", "//source/common/singleton:const_singleton", ], ) @@ -90,9 +92,8 @@ envoy_cc_library( hdrs = ["decoder.h"], deps = [ ":decoder_events_lib", + ":dubbo_hessian2_serializer_impl_lib", ":dubbo_protocol_impl_lib", - ":heartbeat_response_lib", - ":hessian_deserializer_impl_lib", "//source/common/buffer:buffer_lib", "//source/common/common:logger_lib", ], @@ -127,13 +128,20 @@ envoy_cc_library( external_deps = ["abseil_optional"], deps = [ ":message_lib", + "//source/common/buffer:buffer_lib", "//source/common/http:header_map_lib", ], ) envoy_cc_library( name = "message_lib", - hdrs = ["message.h"], + hdrs = [ + "message.h", + "message_impl.h", + ], + deps = [ + "//source/common/buffer:buffer_lib", + ], ) envoy_cc_library( @@ -161,10 +169,10 @@ envoy_cc_library( srcs = ["app_exception.cc"], hdrs = ["app_exception.h"], deps = [ - ":deserializer_interface", ":message_lib", ":metadata_lib", ":protocol_interface", + ":serializer_interface", "//include/envoy/buffer:buffer_interface", "//source/common/buffer:buffer_lib", "//source/extensions/filters/network/dubbo_proxy/filters:filter_interface", @@ -176,9 +184,9 @@ envoy_cc_library( srcs = ["heartbeat_response.cc"], hdrs = ["heartbeat_response.h"], deps = [ - ":deserializer_interface", ":metadata_lib", ":protocol_interface", + ":serializer_interface", "//include/envoy/buffer:buffer_interface", "//source/extensions/filters/network/dubbo_proxy/filters:filter_interface", ], @@ -198,9 +206,9 @@ envoy_cc_library( ":app_exception_lib", ":decoder_events_lib", ":decoder_lib", + ":dubbo_hessian2_serializer_impl_lib", ":dubbo_protocol_impl_lib", ":heartbeat_response_lib", - ":hessian_deserializer_impl_lib", ":stats_lib", "//include/envoy/event:deferred_deletable", "//include/envoy/event:dispatcher_interface", diff --git a/source/extensions/filters/network/dubbo_proxy/active_message.cc b/source/extensions/filters/network/dubbo_proxy/active_message.cc index 179c6852b370..4f44e3a9f638 100644 --- a/source/extensions/filters/network/dubbo_proxy/active_message.cc +++ b/source/extensions/filters/network/dubbo_proxy/active_message.cc @@ -8,94 +8,134 @@ namespace Extensions { namespace NetworkFilters { namespace DubboProxy { -// class ResponseDecoder -ResponseDecoder::ResponseDecoder(Buffer::Instance& buffer, DubboFilterStats& stats, - Network::Connection& connection, Deserializer& deserializer, - Protocol& protocol) - : response_buffer_(buffer), stats_(stats), response_connection_(connection), - decoder_(std::make_unique(protocol, deserializer, *this)), complete_(false) {} - -bool ResponseDecoder::onData(Buffer::Instance& data) { +// class ActiveResponseDecoder +ActiveResponseDecoder::ActiveResponseDecoder(ActiveMessage& parent, DubboFilterStats& stats, + Network::Connection& connection, + ProtocolPtr&& protocol) + : parent_(parent), stats_(stats), response_connection_(connection), + protocol_(std::move(protocol)), + decoder_(std::make_unique(*protocol_, *this)), complete_(false), + response_status_(DubboFilters::UpstreamResponseStatus::MoreData) {} + +DubboFilters::UpstreamResponseStatus ActiveResponseDecoder::onData(Buffer::Instance& data) { ENVOY_LOG(debug, "dubbo response: the received reply data length is {}", data.length()); bool underflow = false; decoder_->onData(data, underflow); ASSERT(complete_ || underflow); - return complete_; + + return response_status_; } -Network::FilterStatus ResponseDecoder::transportBegin() { - stats_.response_.inc(); - response_buffer_.drain(response_buffer_.length()); - ProtocolDataPassthroughConverter::initProtocolConverter(response_buffer_); +void ActiveResponseDecoder::onStreamDecoded(MessageMetadataSharedPtr metadata, + ContextSharedPtr ctx) { + ASSERT(metadata->message_type() == MessageType::Response || + metadata->message_type() == MessageType::Exception); + ASSERT(metadata->hasResponseStatus()); - return Network::FilterStatus::Continue; -} + if (applyMessageEncodedFilters(metadata, ctx) != FilterStatus::Continue) { + return; + } -Network::FilterStatus ResponseDecoder::transportEnd() { if (response_connection_.state() != Network::Connection::State::Open) { throw DownstreamConnectionCloseException("Downstream has closed or closing"); } - response_connection_.write(response_buffer_, false); + response_connection_.write(ctx->message_origin_data(), false); ENVOY_LOG(debug, "dubbo response: the upstream response message has been forwarded to the downstream"); - return Network::FilterStatus::Continue; -} - -Network::FilterStatus ResponseDecoder::messageBegin(MessageType, int64_t, SerializationType) { - return Network::FilterStatus::Continue; -} - -Network::FilterStatus ResponseDecoder::messageEnd(MessageMetadataSharedPtr metadata) { - ASSERT(metadata->message_type() == MessageType::Response || - metadata->message_type() == MessageType::Exception); - ASSERT(metadata->response_status().has_value()); + stats_.response_.inc(); stats_.response_decoding_success_.inc(); if (metadata->message_type() == MessageType::Exception) { stats_.response_business_exception_.inc(); } metadata_ = metadata; - switch (metadata->response_status().value()) { + switch (metadata->response_status()) { case ResponseStatus::Ok: stats_.response_success_.inc(); break; default: stats_.response_error_.inc(); ENVOY_LOG(error, "dubbo response status: {}", - static_cast(metadata->response_status().value())); + static_cast(metadata->response_status())); break; } complete_ = true; + response_status_ = DubboFilters::UpstreamResponseStatus::Complete; + ENVOY_LOG(debug, "dubbo response: complete processing of upstream response messages, id is {}", metadata->request_id()); - - return Network::FilterStatus::Continue; } -DecoderEventHandler* ResponseDecoder::newDecoderEventHandler() { return this; } +FilterStatus ActiveResponseDecoder::applyMessageEncodedFilters(MessageMetadataSharedPtr metadata, + ContextSharedPtr ctx) { + parent_.encoder_filter_action_ = [metadata, + ctx](DubboFilters::EncoderFilter* filter) -> FilterStatus { + return filter->onMessageEncoded(metadata, ctx); + }; -// class ActiveMessageDecoderFilter -ActiveMessageDecoderFilter::ActiveMessageDecoderFilter(ActiveMessage& parent, - DubboFilters::DecoderFilterSharedPtr filter) - : parent_(parent), handle_(filter) {} + auto status = parent_.applyEncoderFilters( + nullptr, ActiveMessage::FilterIterationStartState::CanStartFromCurrent); + switch (status) { + case FilterStatus::StopIteration: + break; + case FilterStatus::Retry: + response_status_ = DubboFilters::UpstreamResponseStatus::Retry; + decoder_->reset(); + break; + default: + ASSERT(FilterStatus::Continue == status); + break; + } -uint64_t ActiveMessageDecoderFilter::requestId() const { return parent_.requestId(); } + return status; +} + +// class ActiveMessageFilterBase +uint64_t ActiveMessageFilterBase::requestId() const { return parent_.requestId(); } -uint64_t ActiveMessageDecoderFilter::streamId() const { return parent_.streamId(); } +uint64_t ActiveMessageFilterBase::streamId() const { return parent_.streamId(); } -const Network::Connection* ActiveMessageDecoderFilter::connection() const { +const Network::Connection* ActiveMessageFilterBase::connection() const { return parent_.connection(); } +Router::RouteConstSharedPtr ActiveMessageFilterBase::route() { return parent_.route(); } + +SerializationType ActiveMessageFilterBase::serializationType() const { + return parent_.serializationType(); +} + +ProtocolType ActiveMessageFilterBase::protocolType() const { return parent_.protocolType(); } + +Event::Dispatcher& ActiveMessageFilterBase::dispatcher() { return parent_.dispatcher(); } + +void ActiveMessageFilterBase::resetStream() { parent_.resetStream(); } + +StreamInfo::StreamInfo& ActiveMessageFilterBase::streamInfo() { return parent_.streamInfo(); } + +// class ActiveMessageDecoderFilter +ActiveMessageDecoderFilter::ActiveMessageDecoderFilter(ActiveMessage& parent, + DubboFilters::DecoderFilterSharedPtr filter, + bool dual_filter) + : ActiveMessageFilterBase(parent, dual_filter), handle_(filter) {} + void ActiveMessageDecoderFilter::continueDecoding() { - const Network::FilterStatus status = parent_.applyDecoderFilters(this); - if (status == Network::FilterStatus::Continue) { + ASSERT(parent_.context()); + auto state = ActiveMessage::FilterIterationStartState::AlwaysStartFromNext; + if (0 != parent_.context()->message_origin_data().length()) { + state = ActiveMessage::FilterIterationStartState::CanStartFromCurrent; + ENVOY_LOG(warn, "The original message data is not consumed, triggering the decoder filter from " + "the current location"); + } + const FilterStatus status = parent_.applyDecoderFilters(this, state); + if (status == FilterStatus::Continue) { + ENVOY_LOG(debug, "dubbo response: start upstream"); // All filters have been executed for the current decoder state. - if (parent_.pending_transport_end()) { + if (parent_.pending_stream_decoded()) { // If the filter stack was paused during messageEnd, handle end-of-request details. parent_.finalizeRequest(); } @@ -103,25 +143,12 @@ void ActiveMessageDecoderFilter::continueDecoding() { } } -Router::RouteConstSharedPtr ActiveMessageDecoderFilter::route() { return parent_.route(); } - -SerializationType ActiveMessageDecoderFilter::downstreamSerializationType() const { - return parent_.downstreamSerializationType(); -} - -ProtocolType ActiveMessageDecoderFilter::downstreamProtocolType() const { - return parent_.downstreamProtocolType(); -} - void ActiveMessageDecoderFilter::sendLocalReply(const DubboFilters::DirectResponse& response, bool end_stream) { parent_.sendLocalReply(response, end_stream); } -void ActiveMessageDecoderFilter::startUpstreamResponse(Deserializer& deserializer, - Protocol& protocol) { - parent_.startUpstreamResponse(deserializer, protocol); -} +void ActiveMessageDecoderFilter::startUpstreamResponse() { parent_.startUpstreamResponse(); } DubboFilters::UpstreamResponseStatus ActiveMessageDecoderFilter::upstreamData(Buffer::Instance& buffer) { @@ -132,16 +159,32 @@ void ActiveMessageDecoderFilter::resetDownstreamConnection() { parent_.resetDownstreamConnection(); } -void ActiveMessageDecoderFilter::resetStream() { parent_.resetStream(); } +// class ActiveMessageEncoderFilter +ActiveMessageEncoderFilter::ActiveMessageEncoderFilter(ActiveMessage& parent, + DubboFilters::EncoderFilterSharedPtr filter, + bool dual_filter) + : ActiveMessageFilterBase(parent, dual_filter), handle_(filter) {} -StreamInfo::StreamInfo& ActiveMessageDecoderFilter::streamInfo() { return parent_.streamInfo(); } +void ActiveMessageEncoderFilter::continueEncoding() { + ASSERT(parent_.context()); + auto state = ActiveMessage::FilterIterationStartState::AlwaysStartFromNext; + if (0 != parent_.context()->message_origin_data().length()) { + state = ActiveMessage::FilterIterationStartState::CanStartFromCurrent; + ENVOY_LOG(warn, "The original message data is not consumed, triggering the encoder filter from " + "the current location"); + } + const FilterStatus status = parent_.applyEncoderFilters(this, state); + if (FilterStatus::Continue == status) { + ENVOY_LOG(debug, "All encoding filters have been executed"); + } +} // class ActiveMessage ActiveMessage::ActiveMessage(ConnectionManager& parent) : parent_(parent), request_timer_(std::make_unique( parent_.stats().request_time_ms_, parent.time_system())), request_id_(-1), stream_id_(parent.random_generator().random()), - stream_info_(parent.time_system()), pending_transport_end_(false), + stream_info_(parent.time_system()), pending_stream_decoded_(false), local_response_sent_(false) { parent_.stats().request_active_.inc(); stream_info_.setDownstreamLocalAddress(parent_.connection().localAddress()); @@ -157,95 +200,64 @@ ActiveMessage::~ActiveMessage() { ENVOY_LOG(debug, "ActiveMessage::~ActiveMessage()"); } -Network::FilterStatus ActiveMessage::transportBegin() { - filter_action_ = [](DubboFilters::DecoderFilter* filter) -> Network::FilterStatus { - return filter->transportBegin(); - }; - - return this->applyDecoderFilters(nullptr); -} - -Network::FilterStatus ActiveMessage::transportEnd() { - filter_action_ = [](DubboFilters::DecoderFilter* filter) -> Network::FilterStatus { - return filter->transportEnd(); - }; - - Network::FilterStatus status = applyDecoderFilters(nullptr); - if (status == Network::FilterStatus::StopIteration) { - pending_transport_end_ = true; - return status; +std::list::iterator +ActiveMessage::commonEncodePrefix(ActiveMessageEncoderFilter* filter, + FilterIterationStartState state) { + // Only do base state setting on the initial call. Subsequent calls for filtering do not touch + // the base state. + if (filter == nullptr) { + // ASSERT(!state_.local_complete_); + // state_.local_complete_ = end_stream; + return encoder_filters_.begin(); } - finalizeRequest(); - - ENVOY_LOG(debug, "dubbo request: complete processing of downstream request messages, id is {}", - request_id_); - - return status; + if (state == FilterIterationStartState::CanStartFromCurrent) { + // The filter iteration has been stopped for all frame types, and now the iteration continues. + // The current filter's encoding callback has not be called. Call it now. + return filter->entry(); + } + return std::next(filter->entry()); } -Network::FilterStatus ActiveMessage::messageBegin(MessageType type, int64_t message_id, - SerializationType serialization_type) { - request_id_ = message_id; - filter_action_ = [type, message_id, serialization_type]( - DubboFilters::DecoderFilter* filter) -> Network::FilterStatus { - return filter->messageBegin(type, message_id, serialization_type); - }; - - return applyDecoderFilters(nullptr); +std::list::iterator +ActiveMessage::commonDecodePrefix(ActiveMessageDecoderFilter* filter, + FilterIterationStartState state) { + if (!filter) { + return decoder_filters_.begin(); + } + if (state == FilterIterationStartState::CanStartFromCurrent) { + // The filter iteration has been stopped for all frame types, and now the iteration continues. + // The current filter's callback function has not been called. Call it now. + return filter->entry(); + } + return std::next(filter->entry()); } -Network::FilterStatus ActiveMessage::messageEnd(MessageMetadataSharedPtr metadata) { - ASSERT(metadata->message_type() == MessageType::Request || - metadata->message_type() == MessageType::Oneway); - - // Currently only hessian serialization is implemented. - ASSERT(metadata->serialization_type() == SerializationType::Hessian); - - ENVOY_LOG(debug, "dubbo request: start processing downstream request messages, id is {}", - metadata->request_id()); - +void ActiveMessage::onStreamDecoded(MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) { parent_.stats().request_decoding_success_.inc(); metadata_ = metadata; - filter_action_ = [metadata](DubboFilters::DecoderFilter* filter) -> Network::FilterStatus { - return filter->messageEnd(metadata); + context_ = ctx; + filter_action_ = [metadata, ctx](DubboFilters::DecoderFilter* filter) -> FilterStatus { + return filter->onMessageDecoded(metadata, ctx); }; - return applyDecoderFilters(nullptr); -} - -Network::FilterStatus ActiveMessage::transferHeaderTo(Buffer::Instance& header_buf, size_t size) { - filter_action_ = [&header_buf, - size](DubboFilters::DecoderFilter* filter) -> Network::FilterStatus { - return filter->transferHeaderTo(header_buf, size); - }; - - // If a local reply is generated, the filter callback is skipped and - // the buffer data needs to be actively released. - if (local_response_sent_) { - header_buf.drain(size); + auto status = applyDecoderFilters(nullptr, FilterIterationStartState::CanStartFromCurrent); + if (status == FilterStatus::StopIteration) { + ENVOY_LOG(debug, "dubbo request: stop calling decoder filter, id is {}", + metadata->request_id()); + pending_stream_decoded_ = true; + return; } - return applyDecoderFilters(nullptr); -} - -Network::FilterStatus ActiveMessage::transferBodyTo(Buffer::Instance& body_buf, size_t size) { - filter_action_ = [&body_buf, size](DubboFilters::DecoderFilter* filter) -> Network::FilterStatus { - return filter->transferBodyTo(body_buf, size); - }; - - // If a local reply is generated, the filter callback is skipped and - // the buffer data needs to be actively released. - if (local_response_sent_) { - body_buf.drain(size); - } + finalizeRequest(); - return applyDecoderFilters(nullptr); + ENVOY_LOG(debug, "dubbo request: complete processing of downstream request messages, id is {}", + metadata->request_id()); } void ActiveMessage::finalizeRequest() { - pending_transport_end_ = false; + pending_stream_decoded_ = false; parent_.stats().request_.inc(); bool is_one_way = false; switch (metadata_->message_type()) { @@ -284,32 +296,47 @@ DubboProxy::Router::RouteConstSharedPtr ActiveMessage::route() { return nullptr; } -Network::FilterStatus ActiveMessage::applyDecoderFilters(ActiveMessageDecoderFilter* filter) { +FilterStatus ActiveMessage::applyDecoderFilters(ActiveMessageDecoderFilter* filter, + FilterIterationStartState state) { ASSERT(filter_action_ != nullptr); - if (!local_response_sent_) { - std::list::iterator entry; - if (!filter) { - entry = decoder_filters_.begin(); - } else { - entry = std::next(filter->entry()); + for (auto entry = commonDecodePrefix(filter, state); entry != decoder_filters_.end(); entry++) { + const FilterStatus status = filter_action_((*entry)->handler().get()); + if (local_response_sent_) { + break; + } + + if (status != FilterStatus::Continue) { + return status; + } } + } + + filter_action_ = nullptr; - for (; entry != decoder_filters_.end(); entry++) { - const Network::FilterStatus status = filter_action_((*entry)->handler().get()); + return FilterStatus::Continue; +} + +FilterStatus ActiveMessage::applyEncoderFilters(ActiveMessageEncoderFilter* filter, + FilterIterationStartState state) { + ASSERT(encoder_filter_action_ != nullptr); + + if (!local_response_sent_) { + for (auto entry = commonEncodePrefix(filter, state); entry != encoder_filters_.end(); entry++) { + const FilterStatus status = encoder_filter_action_((*entry)->handler().get()); if (local_response_sent_) { break; } - if (status != Network::FilterStatus::Continue) { + if (status != FilterStatus::Continue) { return status; } } } - filter_action_ = nullptr; + encoder_filter_action_ = nullptr; - return Network::FilterStatus::Continue; + return FilterStatus::Continue; } void ActiveMessage::sendLocalReply(const DubboFilters::DirectResponse& response, bool end_stream) { @@ -328,21 +355,25 @@ void ActiveMessage::sendLocalReply(const DubboFilters::DirectResponse& response, local_response_sent_ = true; } -void ActiveMessage::startUpstreamResponse(Deserializer& deserializer, Protocol& protocol) { +void ActiveMessage::startUpstreamResponse() { ENVOY_LOG(debug, "dubbo response: start upstream"); ASSERT(response_decoder_ == nullptr); + auto protocol = + NamedProtocolConfigFactory::getFactory(protocolType()).createProtocol(serializationType()); + // Create a response message decoder. - response_decoder_ = std::make_unique( - response_buffer_, parent_.stats(), parent_.connection(), deserializer, protocol); + response_decoder_ = std::make_unique( + *this, parent_.stats(), parent_.connection(), std::move(protocol)); } DubboFilters::UpstreamResponseStatus ActiveMessage::upstreamData(Buffer::Instance& buffer) { ASSERT(response_decoder_ != nullptr); try { - if (response_decoder_->onData(buffer)) { + auto status = response_decoder_->onData(buffer); + if (status == DubboFilters::UpstreamResponseStatus::Complete) { if (requestId() != response_decoder_->requestId()) { throw EnvoyException(fmt::format("dubbo response: request ID is not equal, {}:{}", requestId(), response_decoder_->requestId())); @@ -350,9 +381,11 @@ DubboFilters::UpstreamResponseStatus ActiveMessage::upstreamData(Buffer::Instanc // Completed upstream response. parent_.deferredMessage(*this); - return DubboFilters::UpstreamResponseStatus::Complete; + } else if (status == DubboFilters::UpstreamResponseStatus::Retry) { + response_decoder_.reset(); } - return DubboFilters::UpstreamResponseStatus::MoreData; + + return status; } catch (const DownstreamConnectionCloseException& ex) { ENVOY_CONN_LOG(error, "dubbo response: exception ({})", parent_.connection(), ex.what()); onReset(); @@ -381,24 +414,45 @@ uint64_t ActiveMessage::streamId() const { return stream_id_; } void ActiveMessage::continueDecoding() { parent_.continueDecoding(); } -SerializationType ActiveMessage::downstreamSerializationType() const { +SerializationType ActiveMessage::serializationType() const { return parent_.downstreamSerializationType(); } -ProtocolType ActiveMessage::downstreamProtocolType() const { - return parent_.downstreamProtocolType(); -} +ProtocolType ActiveMessage::protocolType() const { return parent_.downstreamProtocolType(); } StreamInfo::StreamInfo& ActiveMessage::streamInfo() { return stream_info_; } +Event::Dispatcher& ActiveMessage::dispatcher() { return parent_.connection().dispatcher(); } + const Network::Connection* ActiveMessage::connection() const { return &parent_.connection(); } void ActiveMessage::addDecoderFilter(DubboFilters::DecoderFilterSharedPtr filter) { + addDecoderFilterWorker(filter, false); +} + +void ActiveMessage::addEncoderFilter(DubboFilters::EncoderFilterSharedPtr filter) { + addEncoderFilterWorker(filter, false); +} + +void ActiveMessage::addFilter(DubboFilters::CodecFilterSharedPtr filter) { + addDecoderFilterWorker(filter, true); + addEncoderFilterWorker(filter, true); +} + +void ActiveMessage::addDecoderFilterWorker(DubboFilters::DecoderFilterSharedPtr filter, + bool dual_filter) { ActiveMessageDecoderFilterPtr wrapper = - std::make_unique(*this, filter); + std::make_unique(*this, filter, dual_filter); filter->setDecoderFilterCallbacks(*wrapper); wrapper->moveIntoListBack(std::move(wrapper), decoder_filters_); } +void ActiveMessage::addEncoderFilterWorker(DubboFilters::EncoderFilterSharedPtr filter, + bool dual_filter) { + ActiveMessageEncoderFilterPtr wrapper = + std::make_unique(*this, filter, dual_filter); + filter->setEncoderFilterCallbacks(*wrapper); + wrapper->moveIntoListBack(std::move(wrapper), encoder_filters_); +} void ActiveMessage::onReset() { parent_.deferredMessage(*this); } diff --git a/source/extensions/filters/network/dubbo_proxy/active_message.h b/source/extensions/filters/network/dubbo_proxy/active_message.h index 712651ce9a77..6dca717c577d 100644 --- a/source/extensions/filters/network/dubbo_proxy/active_message.h +++ b/source/extensions/filters/network/dubbo_proxy/active_message.h @@ -27,75 +27,111 @@ namespace DubboProxy { class ConnectionManager; class ActiveMessage; -class ResponseDecoder : public DecoderCallbacks, - public DecoderEventHandler, - Logger::Loggable { +class ActiveResponseDecoder : public ResponseDecoderCallbacks, + public StreamHandler, + Logger::Loggable { public: - ResponseDecoder(Buffer::Instance& buffer, DubboFilterStats& stats, - Network::Connection& connection, Deserializer& deserializer, Protocol& protocol); - ~ResponseDecoder() override = default; + ActiveResponseDecoder(ActiveMessage& parent, DubboFilterStats& stats, + Network::Connection& connection, ProtocolPtr&& protocol); + ~ActiveResponseDecoder() override = default; - bool onData(Buffer::Instance& data); + DubboFilters::UpstreamResponseStatus onData(Buffer::Instance& data); - // DecoderEventHandler - Network::FilterStatus transportBegin() override; - Network::FilterStatus transportEnd() override; - Network::FilterStatus messageBegin(MessageType type, int64_t message_id, - SerializationType serialization_type) override; - Network::FilterStatus messageEnd(MessageMetadataSharedPtr metadata) override; + // StreamHandler + void onStreamDecoded(MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) override; - // DecoderCallbacks - DecoderEventHandler* newDecoderEventHandler() override; + // ResponseDecoderCallbacks + StreamHandler& newStream() override { return *this; } + void onHeartbeat(MessageMetadataSharedPtr) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } uint64_t requestId() const { return metadata_ ? metadata_->request_id() : 0; } private: - Buffer::Instance& response_buffer_; + FilterStatus applyMessageEncodedFilters(MessageMetadataSharedPtr metadata, ContextSharedPtr ctx); + + ActiveMessage& parent_; DubboFilterStats& stats_; Network::Connection& response_connection_; - DecoderPtr decoder_; + ProtocolPtr protocol_; + ResponseDecoderPtr decoder_; MessageMetadataSharedPtr metadata_; bool complete_ : 1; + DubboFilters::UpstreamResponseStatus response_status_; }; -using ResponseDecoderPtr = std::unique_ptr; +using ActiveResponseDecoderPtr = std::unique_ptr; + +class ActiveMessageFilterBase : public virtual DubboFilters::FilterCallbacksBase { +public: + ActiveMessageFilterBase(ActiveMessage& parent, bool dual_filter) + : parent_(parent), dual_filter_(dual_filter) {} + ~ActiveMessageFilterBase() override = default; + + // DubboFilters::FilterCallbacksBase + uint64_t requestId() const override; + uint64_t streamId() const override; + const Network::Connection* connection() const override; + DubboProxy::Router::RouteConstSharedPtr route() override; + SerializationType serializationType() const override; + ProtocolType protocolType() const override; + StreamInfo::StreamInfo& streamInfo() override; + Event::Dispatcher& dispatcher() override; + void resetStream() override; + +protected: + ActiveMessage& parent_; + const bool dual_filter_ : 1; +}; // Wraps a DecoderFilter and acts as the DecoderFilterCallbacks for the filter, enabling filter // chain continuation. class ActiveMessageDecoderFilter : public DubboFilters::DecoderFilterCallbacks, - public LinkedObject { + public ActiveMessageFilterBase, + public LinkedObject, + Logger::Loggable { public: - ActiveMessageDecoderFilter(ActiveMessage& parent, DubboFilters::DecoderFilterSharedPtr filter); + ActiveMessageDecoderFilter(ActiveMessage& parent, DubboFilters::DecoderFilterSharedPtr filter, + bool dual_filter); ~ActiveMessageDecoderFilter() override = default; - // DubboFilters::DecoderFilterCallbacks - uint64_t requestId() const override; - uint64_t streamId() const override; - const Network::Connection* connection() const override; void continueDecoding() override; - DubboProxy::Router::RouteConstSharedPtr route() override; - SerializationType downstreamSerializationType() const override; - ProtocolType downstreamProtocolType() const override; void sendLocalReply(const DubboFilters::DirectResponse& response, bool end_stream) override; - void startUpstreamResponse(Deserializer& deserializer, Protocol& protocol) override; + void startUpstreamResponse() override; DubboFilters::UpstreamResponseStatus upstreamData(Buffer::Instance& buffer) override; void resetDownstreamConnection() override; - StreamInfo::StreamInfo& streamInfo() override; - void resetStream() override; DubboFilters::DecoderFilterSharedPtr handler() { return handle_; } private: - ActiveMessage& parent_; DubboFilters::DecoderFilterSharedPtr handle_; }; using ActiveMessageDecoderFilterPtr = std::unique_ptr; +// Wraps a EncoderFilter and acts as the EncoderFilterCallbacks for the filter, enabling filter +// chain continuation. +class ActiveMessageEncoderFilter : public ActiveMessageFilterBase, + public DubboFilters::EncoderFilterCallbacks, + public LinkedObject, + Logger::Loggable { +public: + ActiveMessageEncoderFilter(ActiveMessage& parent, DubboFilters::EncoderFilterSharedPtr filter, + bool dual_filter); + ~ActiveMessageEncoderFilter() override = default; + + void continueEncoding() override; + DubboFilters::EncoderFilterSharedPtr handler() { return handle_; } + +private: + DubboFilters::EncoderFilterSharedPtr handle_; +}; + +using ActiveMessageEncoderFilterPtr = std::unique_ptr; + // ActiveMessage tracks downstream requests for which no response has been received. class ActiveMessage : public LinkedObject, public Event::DeferredDeletable, - public DecoderEventHandler, + public StreamHandler, public DubboFilters::DecoderFilterCallbacks, public DubboFilters::FilterChainFactoryCallbacks, Logger::Loggable { @@ -103,52 +139,70 @@ class ActiveMessage : public LinkedObject, ActiveMessage(ConnectionManager& parent); ~ActiveMessage() override; + // Indicates which filter to start the iteration with. + enum class FilterIterationStartState { AlwaysStartFromNext, CanStartFromCurrent }; + + // Returns the encoder filter to start iteration with. + std::list::iterator + commonEncodePrefix(ActiveMessageEncoderFilter* filter, FilterIterationStartState state); + // Returns the decoder filter to start iteration with. + std::list::iterator + commonDecodePrefix(ActiveMessageDecoderFilter* filter, FilterIterationStartState state); + // Dubbo::FilterChainFactoryCallbacks void addDecoderFilter(DubboFilters::DecoderFilterSharedPtr filter) override; + void addEncoderFilter(DubboFilters::EncoderFilterSharedPtr filter) override; + void addFilter(DubboFilters::CodecFilterSharedPtr filter) override; - // DecoderEventHandler - Network::FilterStatus transportBegin() override; - Network::FilterStatus transportEnd() override; - Network::FilterStatus messageBegin(MessageType type, int64_t message_id, - SerializationType serialization_type) override; - Network::FilterStatus messageEnd(MessageMetadataSharedPtr metadata) override; - Network::FilterStatus transferHeaderTo(Buffer::Instance& header_buf, size_t size) override; - Network::FilterStatus transferBodyTo(Buffer::Instance& body_buf, size_t size) override; + // StreamHandler + void onStreamDecoded(MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) override; // DubboFilters::DecoderFilterCallbacks uint64_t requestId() const override; uint64_t streamId() const override; const Network::Connection* connection() const override; void continueDecoding() override; - SerializationType downstreamSerializationType() const override; - ProtocolType downstreamProtocolType() const override; + SerializationType serializationType() const override; + ProtocolType protocolType() const override; StreamInfo::StreamInfo& streamInfo() override; Router::RouteConstSharedPtr route() override; void sendLocalReply(const DubboFilters::DirectResponse& response, bool end_stream) override; - void startUpstreamResponse(Deserializer& deserializer, Protocol& protocol) override; + void startUpstreamResponse() override; DubboFilters::UpstreamResponseStatus upstreamData(Buffer::Instance& buffer) override; void resetDownstreamConnection() override; + Event::Dispatcher& dispatcher() override; void resetStream() override; void createFilterChain(); - Network::FilterStatus applyDecoderFilters(ActiveMessageDecoderFilter* filter); + FilterStatus applyDecoderFilters(ActiveMessageDecoderFilter* filter, + FilterIterationStartState state); + FilterStatus applyEncoderFilters(ActiveMessageEncoderFilter* filter, + FilterIterationStartState state); void finalizeRequest(); void onReset(); void onError(const std::string& what); MessageMetadataSharedPtr metadata() const { return metadata_; } - bool pending_transport_end() const { return pending_transport_end_; } + ContextSharedPtr context() const { return context_; } + bool pending_stream_decoded() const { return pending_stream_decoded_; } private: + void addDecoderFilterWorker(DubboFilters::DecoderFilterSharedPtr filter, bool dual_filter); + void addEncoderFilterWorker(DubboFilters::EncoderFilterSharedPtr, bool dual_filter); + ConnectionManager& parent_; + ContextSharedPtr context_; MessageMetadataSharedPtr metadata_; Stats::TimespanPtr request_timer_; - ResponseDecoderPtr response_decoder_; + ActiveResponseDecoderPtr response_decoder_; absl::optional cached_route_; std::list decoder_filters_; - std::function filter_action_; + std::function filter_action_; + + std::list encoder_filters_; + std::function encoder_filter_action_; int32_t request_id_; @@ -158,8 +212,10 @@ class ActiveMessage : public LinkedObject, Buffer::OwnedImpl response_buffer_; - bool pending_transport_end_ : 1; + bool pending_stream_decoded_ : 1; bool local_response_sent_ : 1; + + friend class ActiveResponseDecoder; }; using ActiveMessagePtr = std::unique_ptr; diff --git a/source/extensions/filters/network/dubbo_proxy/app_exception.cc b/source/extensions/filters/network/dubbo_proxy/app_exception.cc index 8c35ce60c492..dc5a8a083012 100644 --- a/source/extensions/filters/network/dubbo_proxy/app_exception.cc +++ b/source/extensions/filters/network/dubbo_proxy/app_exception.cc @@ -9,35 +9,6 @@ namespace Extensions { namespace NetworkFilters { namespace DubboProxy { -AppException::AppException(ResponseStatus status, const std::string& what) - : EnvoyException(what), status_(status), - response_type_(RpcResponseType::ResponseWithException) {} - -AppException::ResponseType AppException::encode(MessageMetadata& metadata, - DubboProxy::Protocol& protocol, - Deserializer& deserializer, - Buffer::Instance& buffer) const { - ASSERT(buffer.length() == 0); - - ENVOY_LOG(debug, "err {}", what()); - - // Serialize the response content to get the serialized response length. - const std::string& response = what(); - size_t serialized_body_size = deserializer.serializeRpcResult(buffer, response, response_type_); - - metadata.setResponseStatus(status_); - metadata.setMessageType(MessageType::Response); - - Buffer::OwnedImpl protocol_buffer; - if (!protocol.encode(protocol_buffer, serialized_body_size, metadata)) { - throw EnvoyException("failed to encode local reply message"); - } - - buffer.prepend(protocol_buffer); - - return DirectResponse::ResponseType::Exception; -} - DownstreamConnectionCloseException::DownstreamConnectionCloseException(const std::string& what) : EnvoyException(what) {} diff --git a/source/extensions/filters/network/dubbo_proxy/app_exception.h b/source/extensions/filters/network/dubbo_proxy/app_exception.h index ae68fb47d593..f7415018d654 100644 --- a/source/extensions/filters/network/dubbo_proxy/app_exception.h +++ b/source/extensions/filters/network/dubbo_proxy/app_exception.h @@ -2,30 +2,48 @@ #include "envoy/common/exception.h" -#include "extensions/filters/network/dubbo_proxy/deserializer.h" #include "extensions/filters/network/dubbo_proxy/filters/filter.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" #include "extensions/filters/network/dubbo_proxy/protocol.h" +#include "extensions/filters/network/dubbo_proxy/serializer.h" namespace Envoy { namespace Extensions { namespace NetworkFilters { namespace DubboProxy { -struct AppException : public EnvoyException, - public DubboFilters::DirectResponse, - Logger::Loggable { - AppException(ResponseStatus status, const std::string& what); - AppException(const AppException& ex) = default; +using ResponseType = DubboFilters::DirectResponse::ResponseType; - using ResponseType = DubboFilters::DirectResponse::ResponseType; - ResponseType encode(MessageMetadata& metadata, Protocol& protocol, Deserializer& deserializer, - Buffer::Instance& buffer) const override; +template +struct AppExceptionBase : public EnvoyException, + public DubboFilters::DirectResponse, + Logger::Loggable { + AppExceptionBase(const AppExceptionBase& ex) = default; + AppExceptionBase(T status, const std::string& what) + : EnvoyException(what), status_(status), + response_type_(RpcResponseType::ResponseWithException) {} - const ResponseStatus status_; + ResponseType encode(MessageMetadata& metadata, DubboProxy::Protocol& protocol, + Buffer::Instance& buffer) const override { + ASSERT(buffer.length() == 0); + + ENVOY_LOG(debug, "Exception information: {}", what()); + + metadata.setResponseStatus(status_); + metadata.setMessageType(MessageType::Response); + if (!protocol.encode(buffer, metadata, what(), response_type_)) { + throw EnvoyException("Failed to encode local reply message"); + } + + return ResponseType::Exception; + } + + const T status_; const RpcResponseType response_type_; }; +using AppException = AppExceptionBase<>; + struct DownstreamConnectionCloseException : public EnvoyException { DownstreamConnectionCloseException(const std::string& what); }; diff --git a/source/extensions/filters/network/dubbo_proxy/config.cc b/source/extensions/filters/network/dubbo_proxy/config.cc index 1c145d590acd..92ea0c534aac 100644 --- a/source/extensions/filters/network/dubbo_proxy/config.cc +++ b/source/extensions/filters/network/dubbo_proxy/config.cc @@ -9,6 +9,8 @@ #include "extensions/filters/network/dubbo_proxy/filters/well_known_names.h" #include "extensions/filters/network/dubbo_proxy/stats.h" +#include "absl/container/flat_hash_map.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -66,7 +68,27 @@ class SerializationTypeMapper { static const SerializationTypeMap& serializationTypeMap() { CONSTRUCT_ON_FIRST_USE(SerializationTypeMap, { - {ConfigSerializationType::Hessian2, SerializationType::Hessian}, + {ConfigSerializationType::Hessian2, SerializationType::Hessian2}, + }); + } +}; + +class RouteMatcherTypeMapper { +public: + using ConfigProtocolType = envoy::config::filter::network::dubbo_proxy::v2alpha1::ProtocolType; + using RouteMatcherTypeMap = absl::flat_hash_map; + + static Router::RouteMatcherType lookupRouteMatcherType(ConfigProtocolType type) { + const auto& iter = routeMatcherTypeMap().find(type); + ASSERT(iter != routeMatcherTypeMap().end()); + return iter->second; + } + +private: + static const RouteMatcherTypeMap& routeMatcherTypeMap() { + CONSTRUCT_ON_FIRST_USE(RouteMatcherTypeMap, + { + {ConfigProtocolType::Dubbo, Router::RouteMatcherType::Default}, }); } }; @@ -78,8 +100,10 @@ ConfigImpl::ConfigImpl(const DubboProxyConfig& config, stats_(DubboFilterStats::generateStats(stats_prefix_, context_.scope())), serialization_type_( SerializationTypeMapper::lookupSerializationType(config.serialization_type())), - protocol_type_(ProtocolTypeMapper::lookupProtocolType(config.protocol_type())), - route_matcher_(std::make_unique(config.route_config())) { + protocol_type_(ProtocolTypeMapper::lookupProtocolType(config.protocol_type())) { + auto type = RouteMatcherTypeMapper::lookupRouteMatcherType(config.protocol_type()); + route_matcher_ = Router::NamedRouteMatcherConfigFactory::getFactory(type).createRouteMatcher( + config.route_config(), context); if (config.dubbo_filters().empty()) { ENVOY_LOG(debug, "using default router filter"); @@ -105,11 +129,7 @@ Router::RouteConstSharedPtr ConfigImpl::route(const MessageMetadata& metadata, } ProtocolPtr ConfigImpl::createProtocol() { - return NamedProtocolConfigFactory::getFactory(protocol_type_).createProtocol(); -} - -DeserializerPtr ConfigImpl::createDeserializer() { - return NamedDeserializerConfigFactory::getFactory(serialization_type_).createDeserializer(); + return NamedProtocolConfigFactory::getFactory(protocol_type_).createProtocol(serialization_type_); } void ConfigImpl::registerFilter(const DubboFilterConfig& proto_config) { diff --git a/source/extensions/filters/network/dubbo_proxy/config.h b/source/extensions/filters/network/dubbo_proxy/config.h index 076298217bf8..c0c09967b95a 100644 --- a/source/extensions/filters/network/dubbo_proxy/config.h +++ b/source/extensions/filters/network/dubbo_proxy/config.h @@ -55,7 +55,6 @@ class ConfigImpl : public Config, DubboFilters::FilterChainFactory& filterFactory() override { return *this; } Router::Config& routerConfig() override { return *this; } ProtocolPtr createProtocol() override; - DeserializerPtr createDeserializer() override; private: void registerFilter(const DubboFilterConfig& proto_config); @@ -65,7 +64,7 @@ class ConfigImpl : public Config, DubboFilterStats stats_; const SerializationType serialization_type_; const ProtocolType protocol_type_; - std::unique_ptr route_matcher_; + Router::RouteMatcherPtr route_matcher_; std::list filter_factories_; }; diff --git a/source/extensions/filters/network/dubbo_proxy/conn_manager.cc b/source/extensions/filters/network/dubbo_proxy/conn_manager.cc index 94f935df3ca6..534b3d292964 100644 --- a/source/extensions/filters/network/dubbo_proxy/conn_manager.cc +++ b/source/extensions/filters/network/dubbo_proxy/conn_manager.cc @@ -7,9 +7,9 @@ #include "common/common/fmt.h" #include "extensions/filters/network/dubbo_proxy/app_exception.h" +#include "extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h" #include "extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.h" #include "extensions/filters/network/dubbo_proxy/heartbeat_response.h" -#include "extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h" namespace Envoy { namespace Extensions { @@ -21,9 +21,8 @@ constexpr uint32_t BufferLimit = UINT32_MAX; ConnectionManager::ConnectionManager(Config& config, Runtime::RandomGenerator& random_generator, TimeSource& time_system) : config_(config), time_system_(time_system), stats_(config_.stats()), - random_generator_(random_generator), deserializer_(config.createDeserializer()), - protocol_(config.createProtocol()), - decoder_(std::make_unique(*protocol_.get(), *deserializer_.get(), *this)) {} + random_generator_(random_generator), protocol_(config.createProtocol()), + decoder_(std::make_unique(*protocol_, *this)) {} Network::FilterStatus ConnectionManager::onData(Buffer::Instance& data, bool end_stream) { ENVOY_LOG(trace, "dubbo: read {} bytes", data.length()); @@ -79,13 +78,13 @@ void ConnectionManager::onBelowWriteBufferLowWatermark() { read_callbacks_->connection().readDisable(false); } -DecoderEventHandler* ConnectionManager::newDecoderEventHandler() { +StreamHandler& ConnectionManager::newStream() { ENVOY_LOG(debug, "dubbo: create the new docoder event handler"); ActiveMessagePtr new_message(std::make_unique(*this)); new_message->createFilterChain(); new_message->moveIntoList(std::move(new_message), active_message_list_); - return (*active_message_list_.begin()).get(); + return **active_message_list_.begin(); } void ConnectionManager::onHeartbeat(MessageMetadataSharedPtr metadata) { @@ -97,12 +96,11 @@ void ConnectionManager::onHeartbeat(MessageMetadataSharedPtr metadata) { } metadata->setResponseStatus(ResponseStatus::Ok); - metadata->setMessageType(MessageType::Response); - metadata->setEventFlag(true); + metadata->setMessageType(MessageType::HeartbeatResponse); HeartbeatResponse heartbeat; Buffer::OwnedImpl response_buffer; - heartbeat.encode(*metadata, *protocol_, *deserializer_, response_buffer); + heartbeat.encode(*metadata, *protocol_, response_buffer); read_callbacks_->connection().write(response_buffer, false); } @@ -121,11 +119,7 @@ void ConnectionManager::dispatch() { try { bool underflow = false; while (!underflow) { - Network::FilterStatus status = decoder_->onData(request_buffer_, underflow); - if (status == Network::FilterStatus::StopIteration) { - stopped_ = true; - break; - } + decoder_->onData(request_buffer_, underflow); } return; } catch (const EnvoyException& ex) { @@ -143,10 +137,16 @@ void ConnectionManager::sendLocalReply(MessageMetadata& metadata, return; } - Buffer::OwnedImpl buffer; - const DubboFilters::DirectResponse::ResponseType result = - response.encode(metadata, *protocol_, *deserializer_, buffer); - read_callbacks_->connection().write(buffer, end_stream); + DubboFilters::DirectResponse::ResponseType result = + DubboFilters::DirectResponse::ResponseType::ErrorReply; + + try { + Buffer::OwnedImpl buffer; + result = response.encode(metadata, *protocol_, buffer); + read_callbacks_->connection().write(buffer, end_stream); + } catch (const EnvoyException& ex) { + ENVOY_CONN_LOG(error, "dubbo error: {}", read_callbacks_->connection(), ex.what()); + } if (end_stream) { read_callbacks_->connection().close(Network::ConnectionCloseType::FlushWrite); diff --git a/source/extensions/filters/network/dubbo_proxy/conn_manager.h b/source/extensions/filters/network/dubbo_proxy/conn_manager.h index c46417862b8c..b274c61a867a 100644 --- a/source/extensions/filters/network/dubbo_proxy/conn_manager.h +++ b/source/extensions/filters/network/dubbo_proxy/conn_manager.h @@ -15,9 +15,9 @@ #include "extensions/filters/network/dubbo_proxy/active_message.h" #include "extensions/filters/network/dubbo_proxy/decoder.h" #include "extensions/filters/network/dubbo_proxy/decoder_event_handler.h" -#include "extensions/filters/network/dubbo_proxy/deserializer.h" #include "extensions/filters/network/dubbo_proxy/filters/filter.h" #include "extensions/filters/network/dubbo_proxy/protocol.h" +#include "extensions/filters/network/dubbo_proxy/serializer.h" #include "extensions/filters/network/dubbo_proxy/stats.h" namespace Envoy { @@ -35,14 +35,13 @@ class Config { virtual DubboFilters::FilterChainFactory& filterFactory() PURE; virtual DubboFilterStats& stats() PURE; virtual ProtocolPtr createProtocol() PURE; - virtual DeserializerPtr createDeserializer() PURE; virtual Router::Config& routerConfig() PURE; }; // class ActiveMessagePtr; class ConnectionManager : public Network::ReadFilter, public Network::ConnectionCallbacks, - public DecoderCallbacks, + public RequestDecoderCallbacks, Logger::Loggable { public: using ConfigProtocolType = envoy::config::filter::network::dubbo_proxy::v2alpha1::ProtocolType; @@ -63,8 +62,8 @@ class ConnectionManager : public Network::ReadFilter, void onAboveWriteBufferHighWatermark() override; void onBelowWriteBufferLowWatermark() override; - // DecoderCallbacks - DecoderEventHandler* newDecoderEventHandler() override; + // RequestDecoderCallbacks + StreamHandler& newStream() override; void onHeartbeat(MessageMetadataSharedPtr metadata) override; DubboFilterStats& stats() const { return stats_; } @@ -72,7 +71,7 @@ class ConnectionManager : public Network::ReadFilter, TimeSource& time_system() const { return time_system_; } Runtime::RandomGenerator& random_generator() const { return random_generator_; } Config& config() const { return config_; } - SerializationType downstreamSerializationType() const { return deserializer_->type(); } + SerializationType downstreamSerializationType() const { return protocol_->serializer()->type(); } ProtocolType downstreamProtocolType() const { return protocol_->type(); } void continueDecoding(); @@ -95,9 +94,9 @@ class ConnectionManager : public Network::ReadFilter, DubboFilterStats& stats_; Runtime::RandomGenerator& random_generator_; - DeserializerPtr deserializer_; + SerializerPtr serializer_; ProtocolPtr protocol_; - DecoderPtr decoder_; + RequestDecoderPtr decoder_; Network::ReadFilterCallbacks* read_callbacks_{}; }; diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.cc b/source/extensions/filters/network/dubbo_proxy/decoder.cc index 6d7538c7ec6c..a88ade7d9ed0 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.cc +++ b/source/extensions/filters/network/dubbo_proxy/decoder.cc @@ -2,102 +2,64 @@ #include "common/common/macros.h" -#include "extensions/filters/network/dubbo_proxy/heartbeat_response.h" - namespace Envoy { namespace Extensions { namespace NetworkFilters { namespace DubboProxy { DecoderStateMachine::DecoderStatus -DecoderStateMachine::onTransportBegin(Buffer::Instance& buffer, Protocol::Context& context) { - if (!protocol_.decode(buffer, &context, metadata_)) { +DecoderStateMachine::onDecodeStreamHeader(Buffer::Instance& buffer) { + ASSERT(!active_stream_); + + auto metadata = std::make_shared(); + auto ret = protocol_.decodeHeader(buffer, metadata); + if (!ret.second) { ENVOY_LOG(debug, "dubbo decoder: need more data for {} protocol", protocol_.name()); return {ProtocolState::WaitForData}; } - if (context.is_heartbeat_) { + // The heartbeat message has no body. + auto context = ret.first; + if (metadata->message_type() == MessageType::HeartbeatRequest || + metadata->message_type() == MessageType::HeartbeatResponse) { ENVOY_LOG(debug, "dubbo decoder: this is the {} heartbeat message", protocol_.name()); - buffer.drain(context.header_size_); - decoder_callbacks_.onHeartbeat(metadata_); - return {ProtocolState::Done, Network::FilterStatus::Continue}; - } else { - handler_ = decoder_callbacks_.newDecoderEventHandler(); + buffer.drain(context->header_size()); + delegate_.onHeartbeat(metadata); + return DecoderStatus(ProtocolState::Done); } - return {ProtocolState::OnTransferHeaderTo, handler_->transportBegin()}; -} -DecoderStateMachine::DecoderStatus DecoderStateMachine::onTransportEnd() { - ENVOY_LOG(debug, "dubbo decoder: complete protocol processing"); - return {ProtocolState::Done, handler_->transportEnd()}; -} - -DecoderStateMachine::DecoderStatus DecoderStateMachine::onTransferHeaderTo(Buffer::Instance& buffer, - size_t length) { - ENVOY_LOG(debug, "dubbo decoder: transfer protocol header, buffer size {}, header size {}", - buffer.length(), length); - return {ProtocolState::OnMessageBegin, handler_->transferHeaderTo(buffer, length)}; -} + active_stream_ = delegate_.newStream(metadata, context); + ASSERT(active_stream_); + context->message_origin_data().move(buffer, context->header_size()); -DecoderStateMachine::DecoderStatus DecoderStateMachine::onTransferBodyTo(Buffer::Instance& buffer, - int32_t length) { - ENVOY_LOG(debug, "dubbo decoder: transfer protocol body, buffer size {}, body size {}", - buffer.length(), length); - return {ProtocolState::OnTransportEnd, handler_->transferBodyTo(buffer, length)}; + return DecoderStatus(ProtocolState::OnDecodeStreamData); } -DecoderStateMachine::DecoderStatus DecoderStateMachine::onMessageBegin() { - ENVOY_LOG(debug, "dubbo decoder: start deserializing messages, deserializer name {}", - deserializer_.name()); - return {ProtocolState::OnMessageEnd, - handler_->messageBegin(metadata_->message_type(), metadata_->request_id(), - metadata_->serialization_type())}; -} - -DecoderStateMachine::DecoderStatus DecoderStateMachine::onMessageEnd(Buffer::Instance& buffer, - int32_t message_size) { - ENVOY_LOG(debug, "dubbo decoder: expected body size is {}", message_size); +DecoderStateMachine::DecoderStatus +DecoderStateMachine::onDecodeStreamData(Buffer::Instance& buffer) { + ASSERT(active_stream_); - if (buffer.length() < static_cast(message_size)) { - ENVOY_LOG(debug, "dubbo decoder: need more data for {} deserialization, current size {}", - deserializer_.name(), buffer.length()); - return {ProtocolState::WaitForData}; + if (!protocol_.decodeData(buffer, active_stream_->context_, active_stream_->metadata_)) { + ENVOY_LOG(debug, "dubbo decoder: need more data for {} serialization, current size {}", + protocol_.serializer()->name(), buffer.length()); + return DecoderStatus(ProtocolState::WaitForData); } - switch (metadata_->message_type()) { - case MessageType::Oneway: - case MessageType::Request: - deserializer_.deserializeRpcInvocation(buffer, message_size, metadata_); - break; - case MessageType::Response: { - auto info = deserializer_.deserializeRpcResult(buffer, message_size); - if (info->hasException()) { - metadata_->setMessageType(MessageType::Exception); - } - break; - } - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } + active_stream_->context_->message_origin_data().move(buffer, + active_stream_->context_->body_size()); + active_stream_->onStreamDecoded(); + active_stream_ = nullptr; ENVOY_LOG(debug, "dubbo decoder: ends the deserialization of the message"); - return {ProtocolState::OnTransferBodyTo, handler_->messageEnd(metadata_)}; + return DecoderStatus(ProtocolState::Done); } DecoderStateMachine::DecoderStatus DecoderStateMachine::handleState(Buffer::Instance& buffer) { switch (state_) { - case ProtocolState::OnTransportBegin: - return onTransportBegin(buffer, context_); - case ProtocolState::OnTransferHeaderTo: - return onTransferHeaderTo(buffer, context_.header_size_); - case ProtocolState::OnMessageBegin: - return onMessageBegin(); - case ProtocolState::OnMessageEnd: - return onMessageEnd(buffer, context_.body_size_); - case ProtocolState::OnTransferBodyTo: - return onTransferBodyTo(buffer, context_.body_size_); - case ProtocolState::OnTransportEnd: - return onTransportEnd(); + case ProtocolState::OnDecodeStreamHeader: + return onDecodeStreamHeader(buffer); + case ProtocolState::OnDecodeStreamData: + return onDecodeStreamData(buffer); default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -114,11 +76,6 @@ ProtocolState DecoderStateMachine::run(Buffer::Instance& buffer) { } state_ = s.next_state_; - - ASSERT(s.filter_status_.has_value()); - if (s.filter_status_.value() == Network::FilterStatus::StopIteration) { - return ProtocolState::StopIteration; - } } return state_; @@ -126,11 +83,11 @@ ProtocolState DecoderStateMachine::run(Buffer::Instance& buffer) { using DecoderStateMachinePtr = std::unique_ptr; -Decoder::Decoder(Protocol& protocol, Deserializer& deserializer, - DecoderCallbacks& decoder_callbacks) - : deserializer_(deserializer), protocol_(protocol), decoder_callbacks_(decoder_callbacks) {} +DecoderBase::DecoderBase(Protocol& protocol) : protocol_(protocol) {} -Network::FilterStatus Decoder::onData(Buffer::Instance& data, bool& buffer_underflow) { +DecoderBase::~DecoderBase() { complete(); } + +FilterStatus DecoderBase::onData(Buffer::Instance& data, bool& buffer_underflow) { ENVOY_LOG(debug, "dubbo decoder: {} bytes available", data.length()); buffer_underflow = false; @@ -148,10 +105,7 @@ Network::FilterStatus Decoder::onData(Buffer::Instance& data, bool& buffer_under case ProtocolState::WaitForData: ENVOY_LOG(debug, "dubbo decoder: wait for data"); buffer_underflow = true; - return Network::FilterStatus::Continue; - case ProtocolState::StopIteration: - ENVOY_LOG(debug, "dubbo decoder: wait for continuation"); - return Network::FilterStatus::StopIteration; + return FilterStatus::Continue; default: break; } @@ -161,22 +115,22 @@ Network::FilterStatus Decoder::onData(Buffer::Instance& data, bool& buffer_under complete(); buffer_underflow = (data.length() == 0); ENVOY_LOG(debug, "dubbo decoder: data length {}", data.length()); - return Network::FilterStatus::Continue; + return FilterStatus::Continue; } -void Decoder::start() { - metadata_ = std::make_shared(); - state_machine_ = std::make_unique(protocol_, deserializer_, metadata_, - decoder_callbacks_); +void DecoderBase::start() { + state_machine_ = std::make_unique(protocol_, *this); decode_started_ = true; } -void Decoder::complete() { - metadata_.reset(); +void DecoderBase::complete() { state_machine_.reset(); + stream_.reset(); decode_started_ = false; } +void DecoderBase::reset() { complete(); } + } // namespace DubboProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.h b/source/extensions/filters/network/dubbo_proxy/decoder.h index 982b9dd31b9d..5b04d277876e 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.h +++ b/source/extensions/filters/network/dubbo_proxy/decoder.h @@ -6,8 +6,8 @@ #include "common/common/logger.h" #include "extensions/filters/network/dubbo_proxy/decoder_event_handler.h" -#include "extensions/filters/network/dubbo_proxy/deserializer.h" #include "extensions/filters/network/dubbo_proxy/protocol.h" +#include "extensions/filters/network/dubbo_proxy/serializer.h" namespace Envoy { namespace Extensions { @@ -17,12 +17,8 @@ namespace DubboProxy { #define ALL_PROTOCOL_STATES(FUNCTION) \ FUNCTION(StopIteration) \ FUNCTION(WaitForData) \ - FUNCTION(OnTransportBegin) \ - FUNCTION(OnTransportEnd) \ - FUNCTION(OnMessageBegin) \ - FUNCTION(OnMessageEnd) \ - FUNCTION(OnTransferHeaderTo) \ - FUNCTION(OnTransferBodyTo) \ + FUNCTION(OnDecodeStreamHeader) \ + FUNCTION(OnDecodeStreamData) \ FUNCTION(Done) /** @@ -45,12 +41,38 @@ class ProtocolStateNameValues { } }; +struct ActiveStream { + ActiveStream(StreamHandler& handler, MessageMetadataSharedPtr metadata, ContextSharedPtr context) + : handler_(handler), metadata_(metadata), context_(context) {} + ~ActiveStream() { + metadata_.reset(); + context_.reset(); + } + + void onStreamDecoded() { + ASSERT(metadata_ && context_); + handler_.onStreamDecoded(metadata_, context_); + } + + StreamHandler& handler_; + MessageMetadataSharedPtr metadata_; + ContextSharedPtr context_; +}; + +using ActiveStreamPtr = std::unique_ptr; + class DecoderStateMachine : public Logger::Loggable { public: - DecoderStateMachine(Protocol& protocol, Deserializer& deserializer, - MessageMetadataSharedPtr& metadata, DecoderCallbacks& decoder_callbacks) - : protocol_(protocol), deserializer_(deserializer), metadata_(metadata), - decoder_callbacks_(decoder_callbacks), state_(ProtocolState::OnTransportBegin) {} + class Delegate { + public: + virtual ~Delegate() = default; + virtual ActiveStream* newStream(MessageMetadataSharedPtr metadata, + ContextSharedPtr context) PURE; + virtual void onHeartbeat(MessageMetadataSharedPtr metadata) PURE; + }; + + DecoderStateMachine(Protocol& protocol, Delegate& delegate) + : protocol_(protocol), delegate_(delegate), state_(ProtocolState::OnDecodeStreamHeader) {} /** * Consumes as much data from the configured Buffer as possible and executes the decoding state @@ -77,45 +99,36 @@ class DecoderStateMachine : public Logger::Loggable { private: struct DecoderStatus { DecoderStatus() = default; - DecoderStatus(ProtocolState next_state) : next_state_(next_state){}; - DecoderStatus(ProtocolState next_state, Network::FilterStatus filter_status) + DecoderStatus(ProtocolState next_state) : next_state_(next_state), filter_status_{} {}; + DecoderStatus(ProtocolState next_state, FilterStatus filter_status) : next_state_(next_state), filter_status_(filter_status){}; ProtocolState next_state_; - absl::optional filter_status_; + absl::optional filter_status_; }; // These functions map directly to the matching ProtocolState values. Each returns the next state // or ProtocolState::WaitForData if more data is required. - DecoderStatus onTransportBegin(Buffer::Instance& buffer, Protocol::Context& context); - DecoderStatus onTransportEnd(); - DecoderStatus onTransferHeaderTo(Buffer::Instance& buffer, size_t length); - DecoderStatus onTransferBodyTo(Buffer::Instance& buffer, int32_t length); - DecoderStatus onMessageBegin(); - DecoderStatus onMessageEnd(Buffer::Instance& buffer, int32_t message_size); + DecoderStatus onDecodeStreamHeader(Buffer::Instance& buffer); + DecoderStatus onDecodeStreamData(Buffer::Instance& buffer); // handleState delegates to the appropriate method based on state_. DecoderStatus handleState(Buffer::Instance& buffer); Protocol& protocol_; - Deserializer& deserializer_; - MessageMetadataSharedPtr metadata_; - DecoderCallbacks& decoder_callbacks_; + Delegate& delegate_; ProtocolState state_; - Protocol::Context context_; - - DecoderEventHandler* handler_; + ActiveStream* active_stream_{nullptr}; }; using DecoderStateMachinePtr = std::unique_ptr; -/** - * Decoder encapsulates a configured and ProtocolPtr and SerializationPtr. - */ -class Decoder : public Logger::Loggable { +class DecoderBase : public DecoderStateMachine::Delegate, + public Logger::Loggable { public: - Decoder(Protocol& protocol, Deserializer& deserializer, DecoderCallbacks& decoder_callbacks); + DecoderBase(Protocol& protocol); + virtual ~DecoderBase(); /** * Drains data from the given buffer @@ -123,24 +136,60 @@ class Decoder : public Logger::Loggable { * @param data a Buffer containing Dubbo protocol data * @throw EnvoyException on Dubbo protocol errors */ - Network::FilterStatus onData(Buffer::Instance& data, bool& buffer_underflow); + FilterStatus onData(Buffer::Instance& data, bool& buffer_underflow); - const Deserializer& serializer() { return deserializer_; } const Protocol& protocol() { return protocol_; } -private: + // It is assumed that all of the protocol parsing are stateless, + // if there is a state of the need to provide the reset interface call here. + void reset(); + +protected: void start(); void complete(); - MessageMetadataSharedPtr metadata_; - Deserializer& deserializer_; Protocol& protocol_; + + ActiveStreamPtr stream_; DecoderStateMachinePtr state_machine_; - bool decode_started_ = false; - DecoderCallbacks& decoder_callbacks_; + + bool decode_started_{false}; +}; + +/** + * Decoder encapsulates a configured and ProtocolPtr and SerializationPtr. + */ +template class Decoder : public DecoderBase { +public: + Decoder(Protocol& protocol, T& callbacks) : DecoderBase(protocol), callbacks_(callbacks) {} + + ActiveStream* newStream(MessageMetadataSharedPtr metadata, ContextSharedPtr context) override { + ASSERT(!stream_); + stream_ = std::make_unique(callbacks_.newStream(), metadata, context); + return stream_.get(); + } + + void onHeartbeat(MessageMetadataSharedPtr metadata) override { callbacks_.onHeartbeat(metadata); } + +private: + T& callbacks_; +}; + +class RequestDecoder : public Decoder { +public: + RequestDecoder(Protocol& protocol, RequestDecoderCallbacks& callbacks) + : Decoder(protocol, callbacks) {} +}; + +using RequestDecoderPtr = std::unique_ptr; + +class ResponseDecoder : public Decoder { +public: + ResponseDecoder(Protocol& protocol, ResponseDecoderCallbacks& callbacks) + : Decoder(protocol, callbacks) {} }; -using DecoderPtr = std::unique_ptr; +using ResponseDecoderPtr = std::unique_ptr; } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/decoder_event_handler.h b/source/extensions/filters/network/dubbo_proxy/decoder_event_handler.h index 35b2caed3b01..0da2344ac123 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder_event_handler.h +++ b/source/extensions/filters/network/dubbo_proxy/decoder_event_handler.h @@ -1,7 +1,6 @@ #pragma once #include "envoy/common/pure.h" -#include "envoy/network/connection.h" #include "envoy/network/filter.h" #include "common/buffer/buffer_impl.h" @@ -14,101 +13,81 @@ namespace Extensions { namespace NetworkFilters { namespace DubboProxy { -/** - * This class provides the pass-through capability of the original data of - * the Dubbo protocol to improve the forwarding efficiency - * when no modification of the original data is required. - * Note: If the custom filter does not care about data transfer, - * then it does not need to care about this interface, - * which is currently used by router filter. - */ -class ProtocolDataPassthroughConverter { -public: - ProtocolDataPassthroughConverter() = default; - virtual ~ProtocolDataPassthroughConverter() = default; - - void initProtocolConverter(Buffer::Instance& buffer) { buffer_ = &buffer; } +enum class FilterStatus : uint8_t { + // Continue filter chain iteration. + Continue, + // Do not iterate to any of the remaining filters in the chain. Returning + // FilterDataStatus::Continue from decodeData()/encodeData() or calling + // continueDecoding()/continueEncoding() MUST be called if continued filter iteration is desired. + StopIteration, + // Indicates that a retry is required for the reply message received. + Retry, +}; - /** - * Transfer the original header data of the Dubbo protocol, - * it's called after the protocol's header data is parsed. - * @param header_buf raw header data - * @param size The size of the head. - */ - virtual Network::FilterStatus transferHeaderTo(Buffer::Instance& header_buf, size_t size) { - if (buffer_ != nullptr) { - buffer_->move(header_buf, size); - } - return Network::FilterStatus::Continue; - } +class StreamDecoder { +public: + virtual ~StreamDecoder() = default; /** - * Transfer the original body data of the Dubbo protocol - * it's called after the protocol's body data is parsed. - * @param header_buf raw body data - * @param size The size of the body. + * Indicates that the message had been decoded. + * @param metadata MessageMetadataSharedPtr describing the message + * @param ctx the message context information + * @return FilterStatus to indicate if filter chain iteration should continue */ - virtual Network::FilterStatus transferBodyTo(Buffer::Instance& body_buf, size_t size) { - if (buffer_ != nullptr) { - buffer_->move(body_buf, size); - } - return Network::FilterStatus::Continue; - } - -protected: - Buffer::Instance* buffer_{nullptr}; + virtual FilterStatus onMessageDecoded(MessageMetadataSharedPtr metadata, + ContextSharedPtr ctx) PURE; }; -class DecoderEventHandler : public ProtocolDataPassthroughConverter { +using StreamDecoderSharedPtr = std::shared_ptr; + +class StreamEncoder { public: - ~DecoderEventHandler() override = default; + virtual ~StreamEncoder() = default; /** - * Indicates the start of a Dubbo transport data was detected. Unframed transports generate - * simulated start messages. + * Indicates that the message had been encoded. + * @param metadata MessageMetadataSharedPtr describing the message + * @param ctx the message context information + * @return FilterStatus to indicate if filter chain iteration should continue */ - virtual Network::FilterStatus transportBegin() PURE; + virtual FilterStatus onMessageEncoded(MessageMetadataSharedPtr metadata, + ContextSharedPtr ctx) PURE; +}; - /** - * Indicates the end of a Dubbo transport data was detected. Unframed transport generate - * simulated complete messages. - */ - virtual Network::FilterStatus transportEnd() PURE; +using StreamEncoderSharedPtr = std::shared_ptr; - /** - * Indicates that the start of a Dubbo protocol message was detected. - * @param type the message type - * @param message_id the message identifier - * @param serialization_type the serialization type of the message - * @return FilterStatus to indicate if filter chain iteration should continue - */ - virtual Network::FilterStatus messageBegin(MessageType type, int64_t message_id, - SerializationType serialization_type) PURE; +class StreamHandler { +public: + virtual ~StreamHandler() = default; /** - * Indicates that the end of a Dubbo protocol message was detected. + * Indicates that the message had been decoded. * @param metadata MessageMetadataSharedPtr describing the message + * @param ctx the message context information * @return FilterStatus to indicate if filter chain iteration should continue */ - virtual Network::FilterStatus messageEnd(MessageMetadataSharedPtr metadata) PURE; + virtual void onStreamDecoded(MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) PURE; }; -class DecoderCallbacks { +using StreamDecoderSharedPtr = std::shared_ptr; + +class DecoderCallbacksBase { public: - virtual ~DecoderCallbacks() = default; + virtual ~DecoderCallbacksBase() = default; /** - * @return DecoderEventHandler& a new DecoderEventHandler for a message. + * @return StreamDecoder* a new StreamDecoder for a message. */ - virtual DecoderEventHandler* newDecoderEventHandler() PURE; + virtual StreamHandler& newStream() PURE; /** * Indicates that the message is a heartbeat. */ - virtual void onHeartbeat(MessageMetadataSharedPtr) {} + virtual void onHeartbeat(MessageMetadataSharedPtr) PURE; }; -using DecoderEventHandlerSharedPtr = std::shared_ptr; +class RequestDecoderCallbacks : public DecoderCallbacksBase {}; +class ResponseDecoderCallbacks : public DecoderCallbacksBase {}; } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/deserializer_impl.cc b/source/extensions/filters/network/dubbo_proxy/deserializer_impl.cc deleted file mode 100644 index 985c0d32fd97..000000000000 --- a/source/extensions/filters/network/dubbo_proxy/deserializer_impl.cc +++ /dev/null @@ -1,9 +0,0 @@ -#include "extensions/filters/network/dubbo_proxy/deserializer_impl.h" - -namespace Envoy { -namespace Extensions { -namespace NetworkFilters { -namespace DubboProxy {} // namespace DubboProxy -} // namespace NetworkFilters -} // namespace Extensions -} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/dubbo_proxy/deserializer_impl.h b/source/extensions/filters/network/dubbo_proxy/deserializer_impl.h deleted file mode 100644 index 252143c3454c..000000000000 --- a/source/extensions/filters/network/dubbo_proxy/deserializer_impl.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include "extensions/filters/network/dubbo_proxy/deserializer.h" - -namespace Envoy { -namespace Extensions { -namespace NetworkFilters { -namespace DubboProxy { - -class RpcResultImpl : public RpcResult { -public: - RpcResultImpl() {} - RpcResultImpl(bool has_exception) : has_exception_(has_exception) {} - virtual bool hasException() const override { return has_exception_; } - -private: - bool has_exception_ = false; -}; - -} // namespace DubboProxy -} // namespace NetworkFilters -} // namespace Extensions -} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.cc b/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.cc new file mode 100644 index 000000000000..19e40d284808 --- /dev/null +++ b/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.cc @@ -0,0 +1,118 @@ +#include "extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h" + +#include "envoy/common/exception.h" + +#include "common/common/assert.h" +#include "common/common/macros.h" + +#include "extensions/filters/network/dubbo_proxy/hessian_utils.h" +#include "extensions/filters/network/dubbo_proxy/message_impl.h" +#include "extensions/filters/network/dubbo_proxy/serializer.h" +#include "extensions/filters/network/dubbo_proxy/serializer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace DubboProxy { + +std::pair +DubboHessian2SerializerImpl::deserializeRpcInvocation(Buffer::Instance& buffer, + ContextSharedPtr context) { + size_t total_size = 0, size; + // TODO(zyfjeff): Add format checker + std::string dubbo_version = HessianUtils::peekString(buffer, &size); + total_size += size; + std::string service_name = HessianUtils::peekString(buffer, &size, total_size); + total_size += size; + std::string service_version = HessianUtils::peekString(buffer, &size, total_size); + total_size += size; + std::string method_name = HessianUtils::peekString(buffer, &size, total_size); + total_size += size; + + if (static_cast(context->body_size()) < total_size) { + throw EnvoyException(fmt::format("RpcInvocation size({}) large than body size({})", total_size, + context->body_size())); + } + + auto invo = std::make_shared(); + invo->setServiceName(service_name); + invo->setServiceVersion(service_version); + invo->setMethodName(method_name); + + return std::pair(invo, true); +} + +std::pair +DubboHessian2SerializerImpl::deserializeRpcResult(Buffer::Instance& buffer, + ContextSharedPtr context) { + ASSERT(buffer.length() >= context->body_size()); + size_t total_size = 0; + bool has_value = true; + + auto result = std::make_shared(); + RpcResponseType type = static_cast(HessianUtils::peekInt(buffer, &total_size)); + + switch (type) { + case RpcResponseType::ResponseWithException: + case RpcResponseType::ResponseWithExceptionWithAttachments: + case RpcResponseType::ResponseWithValue: + result->setException(true); + break; + case RpcResponseType::ResponseWithNullValue: + has_value = false; + FALLTHRU; + case RpcResponseType::ResponseValueWithAttachments: + case RpcResponseType::ResponseNullValueWithAttachments: + result->setException(false); + break; + default: + throw EnvoyException(fmt::format("not supported return type {}", static_cast(type))); + } + + if (context->body_size() < total_size) { + throw EnvoyException(fmt::format("RpcResult size({}) large than body size({})", total_size, + context->body_size())); + } + + if (!has_value && context->body_size() != total_size) { + throw EnvoyException( + fmt::format("RpcResult is no value, but the rest of the body size({}) not equal 0", + (context->body_size() - total_size))); + } + + return std::pair(result, true); +} + +size_t DubboHessian2SerializerImpl::serializeRpcResult(Buffer::Instance& output_buffer, + const std::string& content, + RpcResponseType type) { + size_t origin_length = output_buffer.length(); + + // The serialized response type is compact int. + size_t serialized_size = HessianUtils::writeInt( + output_buffer, static_cast::type>(type)); + + // Serialized response content. + serialized_size += HessianUtils::writeString(output_buffer, content); + + ASSERT((output_buffer.length() - origin_length) == serialized_size); + + return serialized_size; +} + +class DubboHessian2SerializerConfigFactory + : public SerializerFactoryBase { +public: + DubboHessian2SerializerConfigFactory() + : SerializerFactoryBase(ProtocolType::Dubbo, SerializationType::Hessian2) {} +}; + +/** + * Static registration for the Hessian protocol. @see RegisterFactory. + */ +REGISTER_FACTORY(DubboHessian2SerializerConfigFactory, NamedSerializerConfigFactory); + +} // namespace DubboProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h b/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h new file mode 100644 index 000000000000..eeffb0c1044e --- /dev/null +++ b/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h @@ -0,0 +1,30 @@ +#pragma once + +#include "extensions/filters/network/dubbo_proxy/serializer.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace DubboProxy { +class DubboHessian2SerializerImpl : public Serializer { +public: + ~DubboHessian2SerializerImpl() = default; + virtual const std::string& name() const override { + return ProtocolSerializerNames::get().fromType(ProtocolType::Dubbo, type()); + } + virtual SerializationType type() const override { return SerializationType::Hessian2; } + + virtual std::pair + deserializeRpcInvocation(Buffer::Instance& buffer, ContextSharedPtr context) override; + + virtual std::pair + deserializeRpcResult(Buffer::Instance& buffer, ContextSharedPtr context) override; + + virtual size_t serializeRpcResult(Buffer::Instance& output_buffer, const std::string& content, + RpcResponseType type) override; +}; + +} // namespace DubboProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.cc b/source/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.cc index f7b6f20a73f7..ff6531cff9cf 100644 --- a/source/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.cc +++ b/source/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.cc @@ -4,6 +4,9 @@ #include "common/common/assert.h" +#include "extensions/filters/network/dubbo_proxy/message_impl.h" +#include "extensions/filters/network/dubbo_proxy/serializer_impl.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -25,8 +28,7 @@ constexpr uint64_t BodySizeOffset = 12; // Consistent with the SerializationType bool isValidSerializationType(SerializationType type) { switch (type) { - case SerializationType::Hessian: - case SerializationType::Json: + case SerializationType::Hessian2: break; default: return false; @@ -64,7 +66,7 @@ void parseRequestInfoFromBuffer(Buffer::Instance& data, MessageMetadataSharedPtr static_cast::type>(type))); } - if (!is_two_way) { + if (!is_two_way && metadata->message_type() != MessageType::HeartbeatRequest) { metadata->setMessageType(MessageType::Oneway); } @@ -83,14 +85,14 @@ void parseResponseInfoFromBuffer(Buffer::Instance& buffer, MessageMetadataShared metadata->setResponseStatus(status); } -bool DubboProtocolImpl::decode(Buffer::Instance& buffer, Protocol::Context* context, - MessageMetadataSharedPtr metadata) { +std::pair +DubboProtocolImpl::decodeHeader(Buffer::Instance& buffer, MessageMetadataSharedPtr metadata) { if (!metadata) { throw EnvoyException("invalid metadata parameter"); } if (buffer.length() < DubboProtocolImpl::MessageSize) { - return false; + return std::pair(nullptr, false); } uint16_t magic_number = buffer.peekBEInt(); @@ -110,42 +112,101 @@ bool DubboProtocolImpl::decode(Buffer::Instance& buffer, Protocol::Context* cont throw EnvoyException(fmt::format("invalid dubbo message size {}", body_size)); } - metadata->setMessageType(type); metadata->setRequestId(request_id); if (type == MessageType::Request) { + if (is_event) { + type = MessageType::HeartbeatRequest; + } + metadata->setMessageType(type); parseRequestInfoFromBuffer(buffer, metadata); } else { + if (is_event) { + type = MessageType::HeartbeatResponse; + } + metadata->setMessageType(type); parseResponseInfoFromBuffer(buffer, metadata); } - context->header_size_ = DubboProtocolImpl::MessageSize; - context->body_size_ = body_size; - context->is_heartbeat_ = is_event; + auto context = std::make_shared(); + context->set_header_size(DubboProtocolImpl::MessageSize); + context->set_body_size(body_size); + context->set_heartbeat(is_event); + + return std::pair(context, true); +} + +bool DubboProtocolImpl::decodeData(Buffer::Instance& buffer, ContextSharedPtr context, + MessageMetadataSharedPtr metadata) { + ASSERT(serializer_); + + if ((buffer.length()) < static_cast(context->body_size())) { + return false; + } + + switch (metadata->message_type()) { + case MessageType::Oneway: + case MessageType::Request: { + auto ret = serializer_->deserializeRpcInvocation(buffer, context); + if (!ret.second) { + return false; + } + metadata->setInvocationInfo(ret.first); + break; + } + case MessageType::Response: { + auto ret = serializer_->deserializeRpcResult(buffer, context); + if (!ret.second) { + return false; + } + if (ret.first->hasException()) { + metadata->setMessageType(MessageType::Exception); + } + break; + } + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } return true; } -bool DubboProtocolImpl::encode(Buffer::Instance& buffer, int32_t body_size, - const MessageMetadata& metadata) { +bool DubboProtocolImpl::encode(Buffer::Instance& buffer, const MessageMetadata& metadata, + const std::string& content, RpcResponseType type) { + ASSERT(serializer_); + switch (metadata.message_type()) { - case MessageType::Response: { - ASSERT(metadata.response_status().has_value()); + case MessageType::HeartbeatResponse: { + ASSERT(metadata.hasResponseStatus()); + ASSERT(content.empty()); buffer.writeBEInt(MagicNumber); uint8_t flag = static_cast(metadata.serialization_type()); - if (metadata.is_event()) { - ASSERT(0 == body_size); - flag = flag ^ EventMask; - } + flag = flag ^ EventMask; buffer.writeByte(flag); - buffer.writeByte(static_cast(metadata.response_status().value())); + buffer.writeByte(static_cast(metadata.response_status())); buffer.writeBEInt(metadata.request_id()); - buffer.writeBEInt(body_size); + buffer.writeBEInt(0); return true; } - case MessageType::Request: { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + case MessageType::Response: { + ASSERT(metadata.hasResponseStatus()); + ASSERT(!content.empty()); + Buffer::OwnedImpl body_buffer; + size_t serialized_body_size = serializer_->serializeRpcResult(body_buffer, content, type); + + buffer.writeBEInt(MagicNumber); + buffer.writeByte(static_cast(metadata.serialization_type())); + buffer.writeByte(static_cast(metadata.response_status())); + buffer.writeBEInt(metadata.request_id()); + buffer.writeBEInt(serialized_body_size); + + buffer.move(body_buffer, serialized_body_size); + return true; } + case MessageType::Request: + case MessageType::Oneway: + case MessageType::Exception: + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; default: NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/source/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.h b/source/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.h index 6146df34f511..47d4bb0062f2 100644 --- a/source/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.h +++ b/source/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.h @@ -10,12 +10,18 @@ namespace DubboProxy { class DubboProtocolImpl : public Protocol { public: DubboProtocolImpl() = default; + ~DubboProtocolImpl() override = default; + const std::string& name() const override { return ProtocolNames::get().fromType(type()); } ProtocolType type() const override { return ProtocolType::Dubbo; } - bool decode(Buffer::Instance& buffer, Protocol::Context* context, - MessageMetadataSharedPtr metadata) override; - bool encode(Buffer::Instance& buffer, int32_t body_size, - const MessageMetadata& metadata) override; + + std::pair decodeHeader(Buffer::Instance& buffer, + MessageMetadataSharedPtr metadata) override; + bool decodeData(Buffer::Instance& buffer, ContextSharedPtr context, + MessageMetadataSharedPtr metadata) override; + + bool encode(Buffer::Instance& buffer, const MessageMetadata& metadata, const std::string& content, + RpcResponseType type) override; static constexpr uint8_t MessageSize = 16; static constexpr int32_t MaxBodySize = 16 * 1024 * 1024; diff --git a/source/extensions/filters/network/dubbo_proxy/filters/BUILD b/source/extensions/filters/network/dubbo_proxy/filters/BUILD index e9e5a42b7f40..253738c08779 100644 --- a/source/extensions/filters/network/dubbo_proxy/filters/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/filters/BUILD @@ -16,9 +16,9 @@ envoy_cc_library( "//include/envoy/network:connection_interface", "//include/envoy/stream_info:stream_info_interface", "//source/extensions/filters/network/dubbo_proxy:decoder_events_lib", - "//source/extensions/filters/network/dubbo_proxy:deserializer_interface", "//source/extensions/filters/network/dubbo_proxy:metadata_lib", "//source/extensions/filters/network/dubbo_proxy:protocol_interface", + "//source/extensions/filters/network/dubbo_proxy:serializer_interface", "//source/extensions/filters/network/dubbo_proxy/router:router_interface", ], ) diff --git a/source/extensions/filters/network/dubbo_proxy/filters/filter.h b/source/extensions/filters/network/dubbo_proxy/filters/filter.h index 5d73624d1a0a..eb7367a26ac7 100644 --- a/source/extensions/filters/network/dubbo_proxy/filters/filter.h +++ b/source/extensions/filters/network/dubbo_proxy/filters/filter.h @@ -9,11 +9,11 @@ #include "envoy/stream_info/stream_info.h" #include "extensions/filters/network/dubbo_proxy/decoder_event_handler.h" -#include "extensions/filters/network/dubbo_proxy/deserializer.h" #include "extensions/filters/network/dubbo_proxy/message.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" #include "extensions/filters/network/dubbo_proxy/protocol.h" #include "extensions/filters/network/dubbo_proxy/router/router.h" +#include "extensions/filters/network/dubbo_proxy/serializer.h" namespace Envoy { namespace Extensions { @@ -25,6 +25,7 @@ enum class UpstreamResponseStatus : uint8_t { MoreData = 0, // The upstream response requires more data. Complete = 1, // The upstream response is complete. Reset = 2, // The upstream response is invalid and its connection must be reset. + Retry = 3, // The upstream response is failure need to retry. }; class DirectResponse { @@ -51,7 +52,7 @@ class DirectResponse { * exception */ virtual ResponseType encode(MessageMetadata& metadata, Protocol& protocol, - Deserializer& deserializer, Buffer::Instance& buffer) const PURE; + Buffer::Instance& buffer) const PURE; }; using DirectResponsePtr = std::unique_ptr; @@ -59,9 +60,9 @@ using DirectResponsePtr = std::unique_ptr; /** * Decoder filter callbacks add additional callbacks. */ -class DecoderFilterCallbacks { +class FilterCallbacksBase { public: - virtual ~DecoderFilterCallbacks() = default; + virtual ~FilterCallbacksBase() = default; /** * @return uint64_t the ID of the originating request for logging purposes. @@ -78,15 +79,6 @@ class DecoderFilterCallbacks { */ virtual const Network::Connection* connection() const PURE; - /** - * Continue iterating through the filter chain with buffered data. This routine can only be - * called if the filter has previously returned StopIteration from one of the DecoderFilter - * methods. The connection manager will callbacks to the next filter in the chain. Further note - * that if the request is not complete, the calling filter may receive further callbacks and must - * return an appropriate status code depending on what the filter needs to do. - */ - virtual void continueDecoding() PURE; - /** * @return RouteConstSharedPtr the route for the current request. */ @@ -95,12 +87,44 @@ class DecoderFilterCallbacks { /** * @return SerializationType the originating protocol. */ - virtual SerializationType downstreamSerializationType() const PURE; + virtual SerializationType serializationType() const PURE; /** * @return ProtocolType the originating protocol. */ - virtual ProtocolType downstreamProtocolType() const PURE; + virtual ProtocolType protocolType() const PURE; + + /** + * @return StreamInfo for logging purposes. + */ + virtual StreamInfo::StreamInfo& streamInfo() PURE; + + /** + * @return Event::Dispatcher& the thread local dispatcher for allocating timers, etc. + */ + virtual Event::Dispatcher& dispatcher() PURE; + + /** + * Reset the underlying stream. + */ + virtual void resetStream() PURE; +}; + +/** + * Decoder filter callbacks add additional callbacks. + */ +class DecoderFilterCallbacks : public virtual FilterCallbacksBase { +public: + virtual ~DecoderFilterCallbacks() = default; + + /** + * Continue iterating through the filter chain with buffered data. This routine can only be + * called if the filter has previously returned StopIteration from one of the DecoderFilter + * methods. The connection manager will callbacks to the next filter in the chain. Further note + * that if the request is not complete, the calling filter may receive further callbacks and must + * return an appropriate status code depending on what the filter needs to do. + */ + virtual void continueDecoding() PURE; /** * Create a locally generated response using the provided response object. @@ -113,7 +137,7 @@ class DecoderFilterCallbacks { * @param transport_type TransportType the upstream is using * @param protocol_type ProtocolType the upstream is using */ - virtual void startUpstreamResponse(Deserializer& deserializer, Protocol& protocol) PURE; + virtual void startUpstreamResponse() PURE; /** * Called with upstream response data. @@ -127,24 +151,31 @@ class DecoderFilterCallbacks { * Reset the downstream connection. */ virtual void resetDownstreamConnection() PURE; +}; - /** - * @return StreamInfo for logging purposes. - */ - virtual StreamInfo::StreamInfo& streamInfo() PURE; +/** + * Encoder filter callbacks add additional callbacks. + */ +class EncoderFilterCallbacks : public virtual FilterCallbacksBase { +public: + virtual ~EncoderFilterCallbacks() = default; /** - * Reset the underlying stream. + * Continue iterating through the filter chain with buffered data. This routine can only be + * called if the filter has previously returned StopIteration from one of the DecoderFilter + * methods. The connection manager will callbacks to the next filter in the chain. Further note + * that if the request is not complete, the calling filter may receive further callbacks and must + * return an appropriate status code depending on what the filter needs to do. */ - virtual void resetStream() PURE; + virtual void continueEncoding() PURE; }; /** - * Decoder filter interface. + * Common base class for both decoder and encoder filters. */ -class DecoderFilter : public DecoderEventHandler { +class FilterBase { public: - virtual ~DecoderFilter() = default; + virtual ~FilterBase() {} /** * This routine is called prior to a filter being destroyed. This may happen after normal stream @@ -156,6 +187,14 @@ class DecoderFilter : public DecoderEventHandler { * onDestroy() invoked. */ virtual void onDestroy() PURE; +}; + +/** + * Decoder filter interface. + */ +class DecoderFilter : public StreamDecoder, public FilterBase { +public: + virtual ~DecoderFilter() = default; /** * Called by the connection manager once to initialize the filter decoder callbacks that the @@ -166,6 +205,29 @@ class DecoderFilter : public DecoderEventHandler { using DecoderFilterSharedPtr = std::shared_ptr; +/** + * Encoder filter interface. + */ +class EncoderFilter : public StreamEncoder, public FilterBase { +public: + virtual ~EncoderFilter() = default; + + /** + * Called by the connection manager once to initialize the filter encoder callbacks that the + * filter should use. Callbacks will not be invoked by the filter after onDestroy() is called. + */ + virtual void setEncoderFilterCallbacks(EncoderFilterCallbacks& callbacks) PURE; +}; + +using EncoderFilterSharedPtr = std::shared_ptr; + +/** + * A filter that handles both encoding and decoding. + */ +class CodecFilter : public virtual DecoderFilter, public virtual EncoderFilter {}; + +using CodecFilterSharedPtr = std::shared_ptr; + /** * These callbacks are provided by the connection manager to the factory so that the factory can * build the filter chain in an application specific way. @@ -179,6 +241,18 @@ class FilterChainFactoryCallbacks { * @param filter supplies the filter to add. */ virtual void addDecoderFilter(DecoderFilterSharedPtr filter) PURE; + + /** + * Add a encoder filter that is used when writing connection data. + * @param filter supplies the filter to add. + */ + virtual void addEncoderFilter(EncoderFilterSharedPtr filter) PURE; + + /** + * Add a decoder/encoder filter that is used both when reading and writing connection data. + * @param filter supplies the filter to add. + */ + virtual void addFilter(CodecFilterSharedPtr filter) PURE; }; /** diff --git a/source/extensions/filters/network/dubbo_proxy/heartbeat_response.cc b/source/extensions/filters/network/dubbo_proxy/heartbeat_response.cc index f966f9f86f8d..3d9f7a648844 100644 --- a/source/extensions/filters/network/dubbo_proxy/heartbeat_response.cc +++ b/source/extensions/filters/network/dubbo_proxy/heartbeat_response.cc @@ -6,14 +6,12 @@ namespace NetworkFilters { namespace DubboProxy { DubboFilters::DirectResponse::ResponseType -HeartbeatResponse::encode(MessageMetadata& metadata, DubboProxy::Protocol& protocol, Deserializer&, +HeartbeatResponse::encode(MessageMetadata& metadata, DubboProxy::Protocol& protocol, Buffer::Instance& buffer) const { - ASSERT(metadata.response_status().value() == ResponseStatus::Ok); - ASSERT(metadata.message_type() == MessageType::Response); - ASSERT(metadata.is_event()); + ASSERT(metadata.response_status() == ResponseStatus::Ok); + ASSERT(metadata.message_type() == MessageType::HeartbeatResponse); - const size_t serialized_body_size = 0; - if (!protocol.encode(buffer, serialized_body_size, metadata)) { + if (!protocol.encode(buffer, metadata, "")) { throw EnvoyException("failed to encode heartbeat message"); } diff --git a/source/extensions/filters/network/dubbo_proxy/heartbeat_response.h b/source/extensions/filters/network/dubbo_proxy/heartbeat_response.h index 4f53691c7f9b..7c76f6c0d674 100644 --- a/source/extensions/filters/network/dubbo_proxy/heartbeat_response.h +++ b/source/extensions/filters/network/dubbo_proxy/heartbeat_response.h @@ -1,9 +1,9 @@ #pragma once -#include "extensions/filters/network/dubbo_proxy/deserializer.h" #include "extensions/filters/network/dubbo_proxy/filters/filter.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" #include "extensions/filters/network/dubbo_proxy/protocol.h" +#include "extensions/filters/network/dubbo_proxy/serializer.h" namespace Envoy { namespace Extensions { @@ -16,7 +16,7 @@ struct HeartbeatResponse : public DubboFilters::DirectResponse, ~HeartbeatResponse() override = default; using ResponseType = DubboFilters::DirectResponse::ResponseType; - ResponseType encode(MessageMetadata& metadata, Protocol& protocol, Deserializer& deserializer, + ResponseType encode(MessageMetadata& metadata, Protocol& protocol, Buffer::Instance& buffer) const override; }; diff --git a/source/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.cc b/source/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.cc deleted file mode 100644 index e095ee4fe9bb..000000000000 --- a/source/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.cc +++ /dev/null @@ -1,111 +0,0 @@ -#include "extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h" - -#include "envoy/common/exception.h" - -#include "common/common/assert.h" -#include "common/common/macros.h" - -#include "extensions/filters/network/dubbo_proxy/deserializer.h" -#include "extensions/filters/network/dubbo_proxy/deserializer_impl.h" -#include "extensions/filters/network/dubbo_proxy/hessian_utils.h" - -namespace Envoy { -namespace Extensions { -namespace NetworkFilters { -namespace DubboProxy { - -void HessianDeserializerImpl::deserializeRpcInvocation(Buffer::Instance& buffer, size_t body_size, - MessageMetadataSharedPtr metadata) { - ASSERT(buffer.length() >= static_cast(body_size)); - size_t total_size = 0, size; - // TODO(zyfjeff): Add format checker - std::string dubbo_version = HessianUtils::peekString(buffer, &size); - total_size = total_size + size; - std::string service_name = HessianUtils::peekString(buffer, &size, total_size); - total_size = total_size + size; - std::string service_version = HessianUtils::peekString(buffer, &size, total_size); - total_size = total_size + size; - std::string method_name = HessianUtils::peekString(buffer, &size, total_size); - total_size = total_size + size; - - if (static_cast(body_size) < total_size) { - throw EnvoyException( - fmt::format("RpcInvocation size({}) large than body size({})", total_size, body_size)); - } - - metadata->setServiceName(service_name); - metadata->setServiceVersion(service_version); - metadata->setMethodName(method_name); -} - -RpcResultPtr HessianDeserializerImpl::deserializeRpcResult(Buffer::Instance& buffer, - size_t body_size) { - ASSERT(buffer.length() >= body_size); - size_t total_size = 0; - bool has_value = true; - - RpcResultPtr result; - RpcResponseType type = static_cast(HessianUtils::peekInt(buffer, &total_size)); - - switch (type) { - case RpcResponseType::ResponseWithException: - case RpcResponseType::ResponseWithExceptionWithAttachments: - case RpcResponseType::ResponseWithValue: - result = std::make_unique(true); - break; - case RpcResponseType::ResponseWithNullValue: - has_value = false; - FALLTHRU; - case RpcResponseType::ResponseValueWithAttachments: - case RpcResponseType::ResponseNullValueWithAttachments: - result = std::make_unique(); - break; - default: - throw EnvoyException(fmt::format("not supported return type {}", static_cast(type))); - } - - if (body_size < total_size) { - throw EnvoyException( - fmt::format("RpcResult size({}) large than body size({})", total_size, body_size)); - } - - if (!has_value && body_size != total_size) { - throw EnvoyException( - fmt::format("RpcResult is no value, but the rest of the body size({}) not equal 0", - (body_size - total_size))); - } - - return result; -} - -size_t HessianDeserializerImpl::serializeRpcResult(Buffer::Instance& output_buffer, - const std::string& content, - RpcResponseType type) { - size_t origin_length = output_buffer.length(); - - // The serialized response type is compact int. - size_t serialized_size = HessianUtils::writeInt( - output_buffer, static_cast::type>(type)); - - // Serialized response content. - serialized_size += HessianUtils::writeString(output_buffer, content); - - ASSERT((output_buffer.length() - origin_length) == serialized_size); - - return serialized_size; -} - -class HessianDeserializerConfigFactory : public DeserializerFactoryBase { -public: - HessianDeserializerConfigFactory() : DeserializerFactoryBase(SerializationType::Hessian) {} -}; - -/** - * Static registration for the Hessian protocol. @see RegisterFactory. - */ -REGISTER_FACTORY(HessianDeserializerConfigFactory, NamedDeserializerConfigFactory); - -} // namespace DubboProxy -} // namespace NetworkFilters -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h b/source/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h deleted file mode 100644 index 0e3dbe363f9a..000000000000 --- a/source/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include "extensions/filters/network/dubbo_proxy/deserializer.h" - -namespace Envoy { -namespace Extensions { -namespace NetworkFilters { -namespace DubboProxy { -class HessianDeserializerImpl : public Deserializer { -public: - HessianDeserializerImpl() {} - ~HessianDeserializerImpl() {} - virtual const std::string& name() const override { - return DeserializerNames::get().fromType(type()); - } - virtual SerializationType type() const override { return SerializationType::Hessian; } - virtual void deserializeRpcInvocation(Buffer::Instance& buffer, size_t body_size, - MessageMetadataSharedPtr metadata) override; - virtual RpcResultPtr deserializeRpcResult(Buffer::Instance& buffer, size_t body_size) override; - virtual size_t serializeRpcResult(Buffer::Instance& output_buffer, const std::string& content, - RpcResponseType type) override; -}; - -} // namespace DubboProxy -} // namespace NetworkFilters -} // namespace Extensions -} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/dubbo_proxy/message.h b/source/extensions/filters/network/dubbo_proxy/message.h index 30c5673fe309..95a435f5798d 100644 --- a/source/extensions/filters/network/dubbo_proxy/message.h +++ b/source/extensions/filters/network/dubbo_proxy/message.h @@ -5,15 +5,46 @@ #include "envoy/common/pure.h" +#include "common/buffer/buffer_impl.h" + +#include "absl/types/optional.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { namespace DubboProxy { +/** + * Stream reset reasons. + */ +enum class StreamResetReason : uint8_t { + // If a local codec level reset was sent on the stream. + LocalReset, + // If a local codec level refused stream reset was sent on the stream (allowing for retry). + LocalRefusedStreamReset, + // If a remote codec level reset was received on the stream. + RemoteReset, + // If a remote codec level refused stream reset was received on the stream (allowing for retry). + RemoteRefusedStreamReset, + // If the stream was locally reset by a connection pool due to an initial connection failure. + ConnectionFailure, + // If the stream was locally reset due to connection termination. + ConnectionTermination, + // The stream was reset because of a resource overflow. + Overflow +}; + +// Supported protocol type +enum class ProtocolType : uint8_t { + Dubbo = 1, + + // ATTENTION: MAKE SURE THIS REMAINS EQUAL TO THE LAST PROTOCOL TYPE + LastProtocolType = Dubbo, +}; + // Supported serialization type enum class SerializationType : uint8_t { - Hessian = 2, - Json = 6, + Hessian2 = 2, }; // Message Type @@ -22,9 +53,11 @@ enum class MessageType : uint8_t { Request = 1, Oneway = 2, Exception = 3, + HeartbeatRequest = 4, + HeartbeatResponse = 5, // ATTENTION: MAKE SURE THIS REMAINS EQUAL TO THE LAST MESSAGE TYPE - LastMessageType = Exception, + LastMessageType = HeartbeatResponse, }; /** @@ -53,32 +86,58 @@ enum class RpcResponseType : uint8_t { ResponseNullValueWithAttachments = 5, }; -class Message { +class Context { public: - virtual ~Message() = default; - virtual MessageType messageType() const PURE; - virtual int32_t bodySize() const PURE; - virtual bool isEvent() const PURE; - virtual int64_t requestId() const PURE; - virtual std::string toString() const PURE; + using AttachmentMap = std::unordered_map; + + bool hasAttachments() const { return attachments_.empty(); } + const AttachmentMap& attachments() const { return attachments_; } + + Buffer::Instance& message_origin_data() { return message_origin_buffer_; } + size_t message_size() const { return header_size() + body_size(); } + + virtual size_t body_size() const PURE; + virtual size_t header_size() const PURE; + +protected: + Context() = default; + virtual ~Context() { attachments_.clear(); } + + AttachmentMap attachments_; + Buffer::OwnedImpl message_origin_buffer_; }; -class RequestMessage : public virtual Message { +using ContextSharedPtr = std::shared_ptr; + +/** + * RpcInvocation represent an rpc call + * See + * https://github.com/apache/incubator-dubbo/blob/master/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/RpcInvocation.java + */ +class RpcInvocation { public: - ~RequestMessage() override = default; - virtual SerializationType serializationType() const PURE; - virtual bool isTwoWay() const PURE; + virtual ~RpcInvocation() = default; + + virtual const std::string& service_name() const PURE; + virtual const std::string& method_name() const PURE; + virtual const absl::optional& service_version() const PURE; + virtual const absl::optional& service_group() const PURE; }; -using RequestMessagePtr = std::unique_ptr; +using RpcInvocationSharedPtr = std::shared_ptr; -class ResponseMessage : public virtual Message { +/** + * RpcResult represent the result of an rpc call + * See + * https://github.com/apache/incubator-dubbo/blob/master/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/RpcResult.java + */ +class RpcResult { public: - ~ResponseMessage() override = default; - virtual ResponseStatus responseStatus() const PURE; + virtual ~RpcResult() = default; + virtual bool hasException() const PURE; }; -using ResponseMessagePtr = std::unique_ptr; +using RpcResultSharedPtr = std::shared_ptr; } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/message_impl.h b/source/extensions/filters/network/dubbo_proxy/message_impl.h new file mode 100644 index 000000000000..325633c859cc --- /dev/null +++ b/source/extensions/filters/network/dubbo_proxy/message_impl.h @@ -0,0 +1,65 @@ +#pragma once + +#include "extensions/filters/network/dubbo_proxy/message.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace DubboProxy { + +class ContextBase : public Context { +public: + ContextBase() = default; + ~ContextBase() override {} + + // Override from Context + size_t body_size() const override { return body_size_; } + size_t header_size() const override { return header_size_; } + + void set_body_size(size_t size) { body_size_ = size; } + void set_header_size(size_t size) { header_size_ = size; } + +protected: + size_t body_size_{0}; + size_t header_size_{0}; +}; + +class ContextImpl : public ContextBase { +public: + ContextImpl() = default; + ~ContextImpl() override {} + + bool is_heartbeat() const { return is_heartbeat_; } + void set_heartbeat(bool is_heartbeat) { is_heartbeat_ = is_heartbeat; } + +private: + bool is_heartbeat_{false}; +}; + +class RpcInvocationBase : public RpcInvocation { +public: + ~RpcInvocationBase() override = default; + + void setServiceName(const std::string& name) { service_name_ = name; } + const std::string& service_name() const override { return service_name_; } + + void setMethodName(const std::string& name) { method_name_ = name; } + const std::string& method_name() const override { return method_name_; } + + void setServiceVersion(const std::string& version) { service_version_ = version; } + const absl::optional& service_version() const override { return service_version_; } + + void setServiceGroup(const std::string& group) { group_ = group; } + const absl::optional& service_group() const override { return group_; } + +protected: + std::string service_name_; + std::string method_name_; + absl::optional service_version_; + absl::optional group_; +}; + +} // namespace DubboProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/dubbo_proxy/metadata.h b/source/extensions/filters/network/dubbo_proxy/metadata.h index a8251c75c021..41a7f3976f4d 100644 --- a/source/extensions/filters/network/dubbo_proxy/metadata.h +++ b/source/extensions/filters/network/dubbo_proxy/metadata.h @@ -19,23 +19,17 @@ namespace DubboProxy { class MessageMetadata { public: - // TODO(gengleilei) Add parameter data types and implement Dubbo data type mapping. - using ParameterValueMap = std::unordered_map; - using ParameterValueMapPtr = std::unique_ptr; - - using HeaderMapPtr = std::unique_ptr; - - void setServiceName(const std::string& name) { service_name_ = name; } - const std::string& service_name() const { return service_name_; } - - void setMethodName(const std::string& name) { method_name_ = name; } - const absl::optional& method_name() const { return method_name_; } + void setInvocationInfo(RpcInvocationSharedPtr invocation_info) { + invocation_info_ = invocation_info; + } + bool hasInvocationInfo() const { return invocation_info_ != nullptr; } + const RpcInvocation& invocation_info() const { return *invocation_info_; } - void setServiceVersion(const std::string& version) { service_version_ = version; } - const absl::optional& service_version() const { return service_version_; } + void setProtocolType(ProtocolType type) { proto_type_ = type; } + ProtocolType protocol_type() const { return proto_type_; } - void setServiceGroup(const std::string& group) { group_ = group; } - const absl::optional& service_group() const { return group_; } + void setProtocolVersion(uint8_t version) { protocol_version_ = version; } + uint8_t protocol_version() const { return protocol_version_; } void setMessageType(MessageType type) { message_type_ = type; } MessageType message_type() const { return message_type_; } @@ -43,80 +37,45 @@ class MessageMetadata { void setRequestId(int64_t id) { request_id_ = id; } int64_t request_id() const { return request_id_; } - void setSerializationType(SerializationType type) { serialization_type_ = type; } - SerializationType serialization_type() const { return serialization_type_; } + void setTimeout(uint32_t timeout) { timeout_ = timeout; } + absl::optional timeout() const { return timeout_; } void setTwoWayFlag(bool two_way) { is_two_way_ = two_way; } bool is_two_way() const { return is_two_way_; } - void setEventFlag(bool is_event) { is_event_ = is_event; } - bool is_event() const { return is_event_; } - - void setResponseStatus(ResponseStatus status) { response_status_ = status; } - const absl::optional& response_status() const { return response_status_; } - - void addParameterValue(uint32_t index, const std::string& value) { - assignParameterIfNeed(); - parameter_map_->emplace(index, value); - } - const std::string& getParameterValue(uint32_t index) const { - if (parameter_map_) { - auto itor = parameter_map_->find(index); - if (itor != parameter_map_->end()) { - return itor->second; - } - } - - return EMPTY_STRING; + template void setSerializationType(T type) { + ASSERT((std::is_same::type>::value)); + serialization_type_ = static_cast(type); } - bool hasParameters() const { return parameter_map_ != nullptr; } - const ParameterValueMap& parameters() { - ASSERT(hasParameters()); - return *parameter_map_; + template T serialization_type() const { + ASSERT((std::is_same::type>::value)); + return static_cast(serialization_type_); } - bool hasHeaders() const { return headers_ != nullptr; } - const Http::HeaderMap& headers() const { - ASSERT(hasHeaders()); - return *headers_; + template void setResponseStatus(T status) { + ASSERT((std::is_same::type>::value)); + response_status_ = static_cast(status); } - void addHeader(const std::string& key, const std::string& value) { - assignHeaderIfNeed(); - headers_->addCopy(Http::LowerCaseString(key), value); - } - void addHeaderReference(const Http::LowerCaseString& key, const std::string& value) { - assignHeaderIfNeed(); - headers_->addReference(key, value); + template T response_status() const { + ASSERT((std::is_same::type>::value)); + return static_cast(response_status_.value()); } + bool hasResponseStatus() const { return response_status_.has_value(); } private: - inline void assignHeaderIfNeed() { - if (!headers_) { - headers_ = std::make_unique(); - } - } - inline void assignParameterIfNeed() { - if (!parameter_map_) { - parameter_map_ = std::make_unique(); - } - } - bool is_two_way_{false}; - bool is_event_{false}; MessageType message_type_{MessageType::Request}; - SerializationType serialization_type_{SerializationType::Hessian}; - absl::optional response_status_; + ProtocolType proto_type_{ProtocolType::Dubbo}; - int64_t request_id_ = 0; + absl::optional response_status_; + absl::optional timeout_; + + RpcInvocationSharedPtr invocation_info_; - // Routing metadata. - std::string service_name_; - absl::optional method_name_; - absl::optional service_version_; - absl::optional group_; - ParameterValueMapPtr parameter_map_; - HeaderMapPtr headers_; // attachment + uint8_t serialization_type_{static_cast(SerializationType::Hessian2)}; + uint8_t protocol_version_{1}; + int64_t request_id_ = 0; }; using MessageMetadataSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/network/dubbo_proxy/protocol.h b/source/extensions/filters/network/dubbo_proxy/protocol.h index 85efff09249d..92f9584e3126 100644 --- a/source/extensions/filters/network/dubbo_proxy/protocol.h +++ b/source/extensions/filters/network/dubbo_proxy/protocol.h @@ -12,69 +12,33 @@ #include "extensions/filters/network/dubbo_proxy/message.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" +#include "extensions/filters/network/dubbo_proxy/serializer.h" namespace Envoy { namespace Extensions { namespace NetworkFilters { namespace DubboProxy { -enum class ProtocolType : uint8_t { - Dubbo = 0, - - // ATTENTION: MAKE SURE THIS REMAINS EQUAL TO THE LAST PROTOCOL TYPE - LastProtocolType = Dubbo, -}; - -/** - * Names of available Protocol implementations. - */ -class ProtocolNameValues { -public: - struct ProtocolTypeHash { - template std::size_t operator()(T t) const { return static_cast(t); } - }; - - using ProtocolTypeNameMap = std::unordered_map; - - const ProtocolTypeNameMap protocolTypeNameMap = { - {ProtocolType::Dubbo, "dubbo"}, - }; - - const std::string& fromType(ProtocolType type) const { - const auto& itor = protocolTypeNameMap.find(type); - if (itor != protocolTypeNameMap.end()) { - return itor->second; - } - - NOT_REACHED_GCOVR_EXCL_LINE; - } -}; - -using ProtocolNames = ConstSingleton; - -/** - * ProtocolCallbacks are Dubbo protocol-level callbacks. - */ -class ProtocolCallbacks { -public: - virtual ~ProtocolCallbacks() = default; - virtual void onRequestMessage(RequestMessagePtr&& req) PURE; - virtual void onResponseMessage(ResponseMessagePtr&& res) PURE; -}; - /** * See https://dubbo.incubator.apache.org/en-us/docs/dev/implementation.html */ class Protocol { public: - struct Context { - bool is_request_ = false; - size_t body_size_ = 0; - size_t header_size_ = 0; - bool is_heartbeat_ = false; - }; virtual ~Protocol() = default; Protocol() = default; + + /** + * @return Initializes the serializer used by the protocol codec + */ + void initSerializer(SerializationType type) { + serializer_ = NamedSerializerConfigFactory::getFactory(this->type(), type).createSerializer(); + } + + /** + * @return Serializer the protocol Serializer + */ + virtual Serializer* serializer() const { return serializer_.get(); } + virtual const std::string& name() const PURE; /** @@ -83,7 +47,21 @@ class Protocol { virtual ProtocolType type() const PURE; /* - * decodes the dubbo protocol message, potentially invoking callbacks. + * decodes the dubbo protocol message header. + * + * @param buffer the currently buffered dubbo data. + * @param metadata the meta data of current messages + * @return ContextSharedPtr save the context data of current messages, + * nullptr if more data is required. + * bool true if a complete message was successfully consumed, false if more data + * is required. + * @throws EnvoyException if the data is not valid for this protocol. + */ + virtual std::pair decodeHeader(Buffer::Instance& buffer, + MessageMetadataSharedPtr metadata) PURE; + + /* + * decodes the dubbo protocol message body, potentially invoking callbacks. * If successful, the message is removed from the buffer. * * @param buffer the currently buffered dubbo data. @@ -93,18 +71,24 @@ class Protocol { * is required. * @throws EnvoyException if the data is not valid for this protocol. */ - virtual bool decode(Buffer::Instance& buffer, Context* context, - MessageMetadataSharedPtr metadata) PURE; + virtual bool decodeData(Buffer::Instance& buffer, ContextSharedPtr context, + MessageMetadataSharedPtr metadata) PURE; /* * encodes the dubbo protocol message. * * @param buffer save the currently buffered dubbo data. * @param metadata the meta data of dubbo protocol + * @param content the body of dubbo protocol message + * @param type the type of dubbo protocol response message * @return bool true if the protocol coding succeeds. */ - virtual bool encode(Buffer::Instance& buffer, int32_t body_size, - const MessageMetadata& metadata) PURE; + virtual bool encode(Buffer::Instance& buffer, const MessageMetadata& metadata, + const std::string& content, + RpcResponseType type = RpcResponseType::ResponseWithValue) PURE; + +protected: + SerializerPtr serializer_; }; using ProtocolPtr = std::unique_ptr; @@ -119,9 +103,10 @@ class NamedProtocolConfigFactory { /** * Create a particular Dubbo protocol. + * @param serialization_type the serialization type of the protocol body. * @return protocol instance pointer. */ - virtual ProtocolPtr createProtocol() PURE; + virtual ProtocolPtr createProtocol(SerializationType serialization_type) PURE; /** * @return std::string the identifying name for a particular implementation of Dubbo protocol @@ -144,7 +129,11 @@ class NamedProtocolConfigFactory { * ProtocolFactoryBase provides a template for a trivial NamedProtocolConfigFactory. */ template class ProtocolFactoryBase : public NamedProtocolConfigFactory { - ProtocolPtr createProtocol() override { return std::make_unique(); } + ProtocolPtr createProtocol(SerializationType serialization_type) override { + auto protocol = std::make_unique(); + protocol->initSerializer(serialization_type); + return protocol; + } std::string name() override { return name_; } diff --git a/source/extensions/filters/network/dubbo_proxy/protocol_constants.h b/source/extensions/filters/network/dubbo_proxy/protocol_constants.h new file mode 100644 index 000000000000..138905d22c1e --- /dev/null +++ b/source/extensions/filters/network/dubbo_proxy/protocol_constants.h @@ -0,0 +1,98 @@ +#pragma once + +#include + +#include "common/common/assert.h" +#include "common/common/fmt.h" +#include "common/singleton/const_singleton.h" + +#include "extensions/filters/network/dubbo_proxy/message.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace DubboProxy { + +/** + * Names of available Protocol implementations. + */ +class ProtocolNameValues { +public: + struct ProtocolTypeHash { + template std::size_t operator()(T t) const { return static_cast(t); } + }; + + using ProtocolTypeNameMap = std::unordered_map; + + const ProtocolTypeNameMap protocolTypeNameMap = { + {ProtocolType::Dubbo, "dubbo"}, + }; + + const std::string& fromType(ProtocolType type) const { + const auto& itor = protocolTypeNameMap.find(type); + ASSERT(itor != protocolTypeNameMap.end()); + return itor->second; + } +}; + +using ProtocolNames = ConstSingleton; + +/** + * Names of available serializer implementations. + */ +class SerializerNameValues { +public: + struct SerializationTypeHash { + template std::size_t operator()(T t) const { return static_cast(t); } + }; + + using SerializerTypeNameMap = + std::unordered_map; + + const SerializerTypeNameMap serializerTypeNameMap = { + {SerializationType::Hessian2, "hessian2"}, + }; + + const std::string& fromType(SerializationType type) const { + const auto& itor = serializerTypeNameMap.find(type); + ASSERT(itor != serializerTypeNameMap.end()); + return itor->second; + } +}; + +using SerializerNames = ConstSingleton; + +class ProtocolSerializerNameValues { +public: + inline uint8_t generateKey(ProtocolType protocol_type, + SerializationType serialization_type) const { + return static_cast(serialization_type) ^ static_cast(protocol_type); + } + + inline std::string generateValue(ProtocolType protocol_type, + SerializationType serialization_type) const { + return fmt::format("{}.{}", ProtocolNames::get().fromType(protocol_type), + SerializerNames::get().fromType(serialization_type)); + } + +#define GENERATE_PAIR(X, Y) generateKey(X, Y), generateValue(X, Y) + + using ProtocolSerializerTypeNameMap = std::unordered_map; + + const ProtocolSerializerTypeNameMap protocolSerializerTypeNameMap = { + {GENERATE_PAIR(ProtocolType::Dubbo, SerializationType::Hessian2)}, + }; + + const std::string& fromType(ProtocolType protocol_type, SerializationType type) const { + const auto& itor = protocolSerializerTypeNameMap.find(generateKey(protocol_type, type)); + ASSERT(itor != protocolSerializerTypeNameMap.end()); + return itor->second; + } +}; + +using ProtocolSerializerNames = ConstSingleton; + +} // namespace DubboProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/dubbo_proxy/router/BUILD b/source/extensions/filters/network/dubbo_proxy/router/BUILD index 790e94d00e3a..0256ac5cdb31 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/BUILD +++ b/source/extensions/filters/network/dubbo_proxy/router/BUILD @@ -17,11 +17,25 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "route_matcher_interface", + hdrs = ["route.h"], + deps = [ + ":router_interface", + "//include/envoy/server:filter_config_interface", + "//source/common/config:utility_lib", + "//source/common/singleton:const_singleton", + "//source/extensions/filters/network/dubbo_proxy:metadata_lib", + "@envoy_api//envoy/config/filter/network/dubbo_proxy/v2alpha1:dubbo_proxy_cc", + ], +) + envoy_cc_library( name = "route_matcher", srcs = ["route_matcher.cc"], hdrs = ["route_matcher.h"], deps = [ + ":route_matcher_interface", ":router_interface", "//include/envoy/router:router_interface", "//source/common/common:logger_lib", @@ -29,6 +43,7 @@ envoy_cc_library( "//source/common/http:header_utility_lib", "//source/common/protobuf:utility_lib", "//source/extensions/filters/network/dubbo_proxy:metadata_lib", + "//source/extensions/filters/network/dubbo_proxy:serializer_interface", "@envoy_api//envoy/config/filter/network/dubbo_proxy/v2alpha1:dubbo_proxy_cc", ], ) @@ -62,8 +77,8 @@ envoy_cc_library( "//source/common/router:metadatamatchcriteria_lib", "//source/common/upstream:load_balancer_lib", "//source/extensions/filters/network/dubbo_proxy:app_exception_lib", - "//source/extensions/filters/network/dubbo_proxy:deserializer_interface", "//source/extensions/filters/network/dubbo_proxy:protocol_interface", + "//source/extensions/filters/network/dubbo_proxy:serializer_interface", "//source/extensions/filters/network/dubbo_proxy/filters:filter_interface", "@envoy_api//envoy/config/filter/network/dubbo_proxy/v2alpha1:dubbo_proxy_cc", ], diff --git a/source/extensions/filters/network/dubbo_proxy/router/config.cc b/source/extensions/filters/network/dubbo_proxy/router/config.cc index 4e4382a61bc3..a362dc7c86f3 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/config.cc +++ b/source/extensions/filters/network/dubbo_proxy/router/config.cc @@ -21,8 +21,7 @@ DubboFilters::FilterFactoryCb RouterFilterConfig::createFilterFactoryFromProtoTy /** * Static registration for the router filter. @see RegisterFactory. */ -static Registry::RegisterFactory - register_; +REGISTER_FACTORY(RouterFilterConfig, DubboFilters::NamedDubboFilterConfigFactory); } // namespace Router } // namespace DubboProxy diff --git a/source/extensions/filters/network/dubbo_proxy/router/route.h b/source/extensions/filters/network/dubbo_proxy/router/route.h new file mode 100644 index 000000000000..61844ce6a2bc --- /dev/null +++ b/source/extensions/filters/network/dubbo_proxy/router/route.h @@ -0,0 +1,121 @@ +#pragma once + +#include +#include + +#include "envoy/config/filter/network/dubbo_proxy/v2alpha1/route.pb.h" +#include "envoy/router/router.h" +#include "envoy/server/filter_config.h" + +#include "common/config/utility.h" +#include "common/singleton/const_singleton.h" + +#include "extensions/filters/network/dubbo_proxy/metadata.h" +#include "extensions/filters/network/dubbo_proxy/router/router.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace DubboProxy { +namespace Router { + +using RouteConfigurations = Protobuf::RepeatedPtrField< + ::envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration>; + +enum class RouteMatcherType : uint8_t { + Default, +}; + +/** + * Names of available Protocol implementations. + */ +class RouteMatcherNameValues { +public: + struct RouteMatcherTypeHash { + template std::size_t operator()(T t) const { return static_cast(t); } + }; + + using RouteMatcherNameMap = + std::unordered_map; + + const RouteMatcherNameMap routeMatcherNameMap = { + {RouteMatcherType::Default, "default"}, + }; + + const std::string& fromType(RouteMatcherType type) const { + const auto& itor = routeMatcherNameMap.find(type); + ASSERT(itor != routeMatcherNameMap.end()); + return itor->second; + } +}; + +using RouteMatcherNames = ConstSingleton; + +class RouteMatcher { +public: + virtual ~RouteMatcher() = default; + + virtual RouteConstSharedPtr route(const MessageMetadata& metadata, + uint64_t random_value) const PURE; +}; + +using RouteMatcherPtr = std::unique_ptr; +using RouteMatcherConstSharedPtr = std::shared_ptr; + +/** + * Implemented by each Dubbo protocol and registered via Registry::registerFactory or the + * convenience class RegisterFactory. + */ +class NamedRouteMatcherConfigFactory { +public: + virtual ~NamedRouteMatcherConfigFactory() = default; + + /** + * Create a particular Dubbo protocol. + * @param serialization_type the serialization type of the protocol body. + * @return protocol instance pointer. + */ + virtual RouteMatcherPtr createRouteMatcher(const RouteConfigurations& route_configs, + Server::Configuration::FactoryContext& context) PURE; + + /** + * @return std::string the identifying name for a particular implementation of Dubbo protocol + * produced by the factory. + */ + virtual std::string name() PURE; + + /** + * Convenience method to lookup a factory by type. + * @param RouteMatcherType the protocol type. + * @return NamedRouteMatcherConfigFactory& for the RouteMatcherType. + */ + static NamedRouteMatcherConfigFactory& getFactory(RouteMatcherType type) { + const std::string& name = RouteMatcherNames::get().fromType(type); + return Envoy::Config::Utility::getAndCheckFactory(name); + } +}; + +/** + * RouteMatcherFactoryBase provides a template for a trivial NamedProtocolConfigFactory. + */ +template +class RouteMatcherFactoryBase : public NamedRouteMatcherConfigFactory { + RouteMatcherPtr createRouteMatcher(const RouteConfigurations& route_configs, + Server::Configuration::FactoryContext& context) override { + return std::make_unique(route_configs, context); + } + + std::string name() override { return name_; } + +protected: + RouteMatcherFactoryBase(RouteMatcherType type) : name_(RouteMatcherNames::get().fromType(type)) {} + +private: + const std::string name_; +}; + +} // namespace Router +} // namespace DubboProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc index 512b38e43398..65daabe6f149 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc +++ b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc @@ -5,6 +5,8 @@ #include "common/protobuf/utility.h" +#include "extensions/filters/network/dubbo_proxy/serializer_impl.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -81,13 +83,16 @@ bool ParameterRouteEntryImpl::matchParameter(absl::string_view request_data, RouteConstSharedPtr ParameterRouteEntryImpl::matches(const MessageMetadata& metadata, uint64_t random_value) const { - if (!metadata.hasParameters()) { + ASSERT(metadata.hasInvocationInfo()); + const auto invocation = dynamic_cast(&metadata.invocation_info()); + ASSERT(invocation); + if (!invocation->hasParameters()) { return nullptr; } ENVOY_LOG(debug, "dubbo route matcher: parameter name match"); for (auto& config_data : parameter_data_list_) { - const std::string& data = metadata.getParameterValue(config_data.index_); + const std::string& data = invocation->getParameterValue(config_data.index_); if (data.empty()) { ENVOY_LOG(debug, "dubbo route matcher: parameter matching failed, there are no parameters in the " @@ -137,19 +142,23 @@ MethodRouteEntryImpl::~MethodRouteEntryImpl() {} RouteConstSharedPtr MethodRouteEntryImpl::matches(const MessageMetadata& metadata, uint64_t random_value) const { - if (metadata.hasHeaders() && !RouteEntryImplBase::headersMatch(metadata.headers())) { + ASSERT(metadata.hasInvocationInfo()); + const auto invocation = dynamic_cast(&metadata.invocation_info()); + ASSERT(invocation); + + if (invocation->hasHeaders() && !RouteEntryImplBase::headersMatch(invocation->headers())) { ENVOY_LOG(error, "dubbo route matcher: headers not match"); return nullptr; } - if (!metadata.method_name().has_value()) { + if (invocation->method_name().empty()) { ENVOY_LOG(error, "dubbo route matcher: there is no method name in the metadata"); return nullptr; } - if (!method_name_.match(metadata.method_name().value())) { + if (!method_name_.match(invocation->method_name())) { ENVOY_LOG(debug, "dubbo route matcher: method matching failed, input method '{}'", - metadata.method_name().value()); + invocation->method_name()); return nullptr; } @@ -161,7 +170,8 @@ RouteConstSharedPtr MethodRouteEntryImpl::matches(const MessageMetadata& metadat return clusterEntry(random_value); } -RouteMatcher::RouteMatcher(const RouteConfig& config) +SignleRouteMatcherImpl::SignleRouteMatcherImpl(const RouteConfig& config, + Server::Configuration::FactoryContext&) : service_name_(config.interface()), group_(config.group()), version_(config.version()) { using envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteMatch; @@ -171,13 +181,16 @@ RouteMatcher::RouteMatcher(const RouteConfig& config) ENVOY_LOG(debug, "dubbo route matcher: routes list size {}", routes_.size()); } -RouteConstSharedPtr RouteMatcher::route(const MessageMetadata& metadata, - uint64_t random_value) const { - if (service_name_ == metadata.service_name() && +RouteConstSharedPtr SignleRouteMatcherImpl::route(const MessageMetadata& metadata, + uint64_t random_value) const { + ASSERT(metadata.hasInvocationInfo()); + const auto& invocation = metadata.invocation_info(); + + if (service_name_ == invocation.service_name() && (group_.value().empty() || - (metadata.service_group().has_value() && metadata.service_group().value() == group_)) && - (version_.value().empty() || (metadata.service_version().has_value() && - metadata.service_version().value() == version_))) { + (invocation.service_group().has_value() && invocation.service_group().value() == group_)) && + (version_.value().empty() || (invocation.service_version().has_value() && + invocation.service_version().value() == version_))) { for (const auto& route : routes_) { RouteConstSharedPtr route_entry = route->matches(metadata, random_value); if (nullptr != route_entry) { @@ -191,9 +204,11 @@ RouteConstSharedPtr RouteMatcher::route(const MessageMetadata& metadata, return nullptr; } -MultiRouteMatcher::MultiRouteMatcher(const RouteConfigList& route_config_list) { +MultiRouteMatcher::MultiRouteMatcher(const RouteConfigList& route_config_list, + Server::Configuration::FactoryContext& context) { for (const auto& route_config : route_config_list) { - route_matcher_list_.emplace_back(std::make_unique(route_config)); + route_matcher_list_.emplace_back( + std::make_unique(route_config, context)); } ENVOY_LOG(debug, "route matcher list size {}", route_matcher_list_.size()); } @@ -210,6 +225,16 @@ RouteConstSharedPtr MultiRouteMatcher::route(const MessageMetadata& metadata, return nullptr; } +class DefaultRouteMatcherConfigFactory : public RouteMatcherFactoryBase { +public: + DefaultRouteMatcherConfigFactory() : RouteMatcherFactoryBase(RouteMatcherType::Default) {} +}; + +/** + * Static registration for the Dubbo protocol. @see RegisterFactory. + */ +REGISTER_FACTORY(DefaultRouteMatcherConfigFactory, NamedRouteMatcherConfigFactory); + } // namespace Router } // namespace DubboProxy } // namespace NetworkFilters diff --git a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h index cf01e91c9e28..551d658f6ba0 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h +++ b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h @@ -13,6 +13,7 @@ #include "common/protobuf/protobuf.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" +#include "extensions/filters/network/dubbo_proxy/router/route.h" #include "extensions/filters/network/dubbo_proxy/router/router.h" #include "absl/types/optional.h" @@ -126,12 +127,12 @@ class MethodRouteEntryImpl : public RouteEntryImplBase { std::shared_ptr parameter_route_; }; -class RouteMatcher : public Logger::Loggable { +class SignleRouteMatcherImpl : public RouteMatcher, public Logger::Loggable { public: using RouteConfig = envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration; - RouteMatcher(const RouteConfig& config); + SignleRouteMatcherImpl(const RouteConfig& config, Server::Configuration::FactoryContext& context); - RouteConstSharedPtr route(const MessageMetadata& metadata, uint64_t random_value) const; + RouteConstSharedPtr route(const MessageMetadata& metadata, uint64_t random_value) const override; private: std::vector routes_; @@ -140,16 +141,14 @@ class RouteMatcher : public Logger::Loggable { const absl::optional version_; }; -using RouteMatcherConstSharedPtr = std::shared_ptr; -using RouteMatcherPtr = std::unique_ptr; - -class MultiRouteMatcher : public Logger::Loggable { +class MultiRouteMatcher : public RouteMatcher, public Logger::Loggable { public: using RouteConfigList = Envoy::Protobuf::RepeatedPtrField< ::envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration>; - MultiRouteMatcher(const RouteConfigList& route_config_list); + MultiRouteMatcher(const RouteConfigList& route_config_list, + Server::Configuration::FactoryContext& context); - RouteConstSharedPtr route(const MessageMetadata& metadata, uint64_t random_value) const; + RouteConstSharedPtr route(const MessageMetadata& metadata, uint64_t random_value) const override; private: std::vector route_matcher_list_; diff --git a/source/extensions/filters/network/dubbo_proxy/router/router.h b/source/extensions/filters/network/dubbo_proxy/router/router.h index 790abe825d5e..37887be6058d 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/router.h +++ b/source/extensions/filters/network/dubbo_proxy/router/router.h @@ -32,6 +32,8 @@ class RouteEntry { virtual const Envoy::Router::MetadataMatchCriteria* metadataMatchCriteria() const PURE; }; +using RouteEntryPtr = std::shared_ptr; + /** * Route holds the RouteEntry for a request. */ @@ -46,6 +48,7 @@ class Route { }; using RouteConstSharedPtr = std::shared_ptr; +using RouteSharedPtr = std::shared_ptr; /** * The router configuration. diff --git a/source/extensions/filters/network/dubbo_proxy/router/router_impl.cc b/source/extensions/filters/network/dubbo_proxy/router/router_impl.cc index 7788ed82befc..82e12a3a8ebe 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/router_impl.cc +++ b/source/extensions/filters/network/dubbo_proxy/router/router_impl.cc @@ -23,47 +23,19 @@ void Router::setDecoderFilterCallbacks(DubboFilters::DecoderFilterCallbacks& cal callbacks_ = &callbacks; } -Network::FilterStatus Router::transportBegin() { - upstream_request_buffer_.drain(upstream_request_buffer_.length()); - ProtocolDataPassthroughConverter::initProtocolConverter(upstream_request_buffer_); - return Network::FilterStatus::Continue; -} - -Network::FilterStatus Router::transportEnd() { - // If the connection fails, the callback of the filter will be suspended, - // so it is impossible to call the transportEnd interface. - // the encodeData function will be called only if the connection is successful. - ASSERT(upstream_request_); - ASSERT(upstream_request_->conn_data_); - - upstream_request_->encodeData(upstream_request_buffer_); - - if (upstream_request_->metadata_->message_type() == MessageType::Oneway) { - // No response expected - upstream_request_->onResponseComplete(); - cleanup(); - ENVOY_LOG(debug, "dubbo upstream request: the message is one-way and no response is required"); - } - - filter_complete_ = true; - - return Network::FilterStatus::Continue; -} - -Network::FilterStatus Router::messageBegin(MessageType, int64_t, SerializationType) { - return Network::FilterStatus::Continue; -} +FilterStatus Router::onMessageDecoded(MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) { + ASSERT(metadata->hasInvocationInfo()); + const auto& invocation = metadata->invocation_info(); -Network::FilterStatus Router::messageEnd(MessageMetadataSharedPtr metadata) { route_ = callbacks_->route(); if (!route_) { ENVOY_STREAM_LOG(debug, "dubbo router: no cluster match for interface '{}'", *callbacks_, - metadata->service_name()); + invocation.service_name()); callbacks_->sendLocalReply(AppException(ResponseStatus::ServiceNotFound, fmt::format("dubbo router: no route for interface '{}'", - metadata->service_name())), + invocation.service_name())), false); - return Network::FilterStatus::StopIteration; + return FilterStatus::StopIteration; } route_entry_ = route_->routeEntry(); @@ -76,12 +48,12 @@ Network::FilterStatus Router::messageEnd(MessageMetadataSharedPtr metadata) { AppException(ResponseStatus::ServerError, fmt::format("dubbo router: unknown cluster '{}'", route_entry_->clusterName())), false); - return Network::FilterStatus::StopIteration; + return FilterStatus::StopIteration; } cluster_ = cluster->info(); ENVOY_STREAM_LOG(debug, "dubbo router: cluster '{}' match for interface '{}'", *callbacks_, - route_entry_->clusterName(), metadata->service_name()); + route_entry_->clusterName(), invocation.service_name()); if (cluster_->maintenanceMode()) { callbacks_->sendLocalReply( @@ -89,7 +61,7 @@ Network::FilterStatus Router::messageEnd(MessageMetadataSharedPtr metadata) { fmt::format("dubbo router: maintenance mode for cluster '{}'", route_entry_->clusterName())), false); - return Network::FilterStatus::StopIteration; + return FilterStatus::StopIteration; } Tcp::ConnectionPool::Instance* conn_pool = cluster_manager_.tcpConnPoolForCluster( @@ -100,14 +72,14 @@ Network::FilterStatus Router::messageEnd(MessageMetadataSharedPtr metadata) { ResponseStatus::ServerError, fmt::format("dubbo router: no healthy upstream for '{}'", route_entry_->clusterName())), false); - return Network::FilterStatus::StopIteration; + return FilterStatus::StopIteration; } ENVOY_STREAM_LOG(debug, "dubbo router: decoding request", *callbacks_); + upstream_request_buffer_.move(ctx->message_origin_data(), ctx->message_size()); - upstream_request_ = std::make_unique(*this, *conn_pool, metadata, - callbacks_->downstreamSerializationType(), - callbacks_->downstreamProtocolType()); + upstream_request_ = std::make_unique( + *this, *conn_pool, metadata, callbacks_->serializationType(), callbacks_->protocolType()); return upstream_request_->start(); } @@ -118,8 +90,7 @@ void Router::onUpstreamData(Buffer::Instance& data, bool end_stream) { // Handle normal response. if (!upstream_request_->response_started_) { - callbacks_->startUpstreamResponse(*upstream_request_->deserializer_.get(), - *upstream_request_->protocol_.get()); + callbacks_->startUpstreamResponse(); upstream_request_->response_started_ = true; } @@ -191,23 +162,22 @@ Router::UpstreamRequest::UpstreamRequest(Router& parent, Tcp::ConnectionPool::In SerializationType serialization_type, ProtocolType protocol_type) : parent_(parent), conn_pool_(pool), metadata_(metadata), - deserializer_( - NamedDeserializerConfigFactory::getFactory(serialization_type).createDeserializer()), - protocol_(NamedProtocolConfigFactory::getFactory(protocol_type).createProtocol()), + protocol_( + NamedProtocolConfigFactory::getFactory(protocol_type).createProtocol(serialization_type)), request_complete_(false), response_started_(false), response_complete_(false), stream_reset_(false) {} Router::UpstreamRequest::~UpstreamRequest() {} -Network::FilterStatus Router::UpstreamRequest::start() { +FilterStatus Router::UpstreamRequest::start() { Tcp::ConnectionPool::Cancellable* handle = conn_pool_.newConnection(*this); if (handle) { // Pause while we wait for a connection. conn_pool_handle_ = handle; - return Network::FilterStatus::StopIteration; + return FilterStatus::StopIteration; } - return Network::FilterStatus::Continue; + return FilterStatus::Continue; } void Router::UpstreamRequest::resetStream() { @@ -270,6 +240,7 @@ void Router::UpstreamRequest::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr conn_pool_handle_ = nullptr; onRequestStart(continue_decoding); + encodeData(parent_.upstream_request_buffer_); } void Router::UpstreamRequest::onRequestStart(bool continue_decoding) { diff --git a/source/extensions/filters/network/dubbo_proxy/router/router_impl.h b/source/extensions/filters/network/dubbo_proxy/router/router_impl.h index 63bcfa0e4ae4..c4840015a5b5 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/router_impl.h +++ b/source/extensions/filters/network/dubbo_proxy/router/router_impl.h @@ -24,16 +24,13 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, Logger::Loggable { public: Router(Upstream::ClusterManager& cluster_manager) : cluster_manager_(cluster_manager) {} - ~Router() {} + ~Router() override = default; // DubboFilters::DecoderFilter void onDestroy() override; void setDecoderFilterCallbacks(DubboFilters::DecoderFilterCallbacks& callbacks) override; - Network::FilterStatus transportBegin() override; - Network::FilterStatus transportEnd() override; - Network::FilterStatus messageBegin(MessageType type, int64_t message_id, - SerializationType serialization_type) override; - Network::FilterStatus messageEnd(MessageMetadataSharedPtr metadata) override; + + FilterStatus onMessageDecoded(MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) override; // Upstream::LoadBalancerContextBase const Envoy::Router::MetadataMatchCriteria* metadataMatchCriteria() override { return nullptr; } @@ -52,7 +49,7 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, ProtocolType protocol_type); ~UpstreamRequest(); - Network::FilterStatus start(); + FilterStatus start(); void resetStream(); void encodeData(Buffer::Instance& data); @@ -75,7 +72,7 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, Tcp::ConnectionPool::Cancellable* conn_pool_handle_{}; Tcp::ConnectionPool::ConnectionDataPtr conn_data_; Upstream::HostDescriptionConstSharedPtr upstream_host_; - DeserializerPtr deserializer_; + SerializerPtr serializer_; ProtocolPtr protocol_; bool request_complete_ : 1; diff --git a/source/extensions/filters/network/dubbo_proxy/serializer.h b/source/extensions/filters/network/dubbo_proxy/serializer.h new file mode 100644 index 000000000000..13b5dc8f0b3e --- /dev/null +++ b/source/extensions/filters/network/dubbo_proxy/serializer.h @@ -0,0 +1,128 @@ +#pragma once + +#include +#include + +#include "envoy/buffer/buffer.h" + +#include "common/common/assert.h" +#include "common/config/utility.h" +#include "common/singleton/const_singleton.h" + +#include "extensions/filters/network/dubbo_proxy/message.h" +#include "extensions/filters/network/dubbo_proxy/metadata.h" +#include "extensions/filters/network/dubbo_proxy/protocol_constants.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace DubboProxy { + +class Serializer { +public: + virtual ~Serializer() = default; + + /** + * Return this Serializer's name + * + * @return std::string containing the serialization name. + */ + virtual const std::string& name() const PURE; + + /** + * @return SerializationType the serializer type + */ + virtual SerializationType type() const PURE; + + /** + * deserialize an rpc call + * If successful, the RpcInvocation removed from the buffer + * + * @param buffer the currently buffered dubbo data + * @param context context information for RPC messages + * @return a pair containing the deserialized result of the message and the deserialized + * invocation information. + * @throws EnvoyException if the data is not valid for this serialization + */ + virtual std::pair + deserializeRpcInvocation(Buffer::Instance& buffer, ContextSharedPtr context) PURE; + + /** + * deserialize result of an rpc call + * + * @param buffer the currently buffered dubbo data + * @param context context information for RPC messages + * @return a pair containing the deserialized result of the message and the deserialized + * result information. + * @throws EnvoyException if the data is not valid for this serialization + */ + virtual std::pair deserializeRpcResult(Buffer::Instance& buffer, + ContextSharedPtr context) PURE; + + /** + * serialize result of an rpc call + * If successful, the output_buffer is written to the serialized data + * + * @param output_buffer store the serialized data + * @param content the rpc response content + * @param type the rpc response type + * @return size_t the length of the serialized content + */ + virtual size_t serializeRpcResult(Buffer::Instance& output_buffer, const std::string& content, + RpcResponseType type) PURE; +}; + +using SerializerPtr = std::unique_ptr; + +/** + * Implemented by each Dubbo serialize and registered via Registry::registerFactory or the + * convenience class RegisterFactory. + */ +class NamedSerializerConfigFactory { +public: + virtual ~NamedSerializerConfigFactory() = default; + + /** + * Create a particular Dubbo serializer. + * @return SerializerPtr the transport + */ + virtual SerializerPtr createSerializer() PURE; + + /** + * @return std::string the identifying name for a particular implementation of Dubbo serializer + * produced by the factory. + */ + virtual std::string name() PURE; + + /** + * Convenience method to lookup a factory by type. + * @param TransportType the transport type + * @return NamedSerializerConfigFactory& for the TransportType + */ + static NamedSerializerConfigFactory& getFactory(ProtocolType protocol_type, + SerializationType type) { + const std::string& name = ProtocolSerializerNames::get().fromType(protocol_type, type); + return Envoy::Config::Utility::getAndCheckFactory(name); + } +}; + +/** + * SerializerFactoryBase provides a template for a trivial NamedSerializerConfigFactory. + */ +template class SerializerFactoryBase : public NamedSerializerConfigFactory { + SerializerPtr createSerializer() override { return std::make_unique(); } + + std::string name() override { return name_; } + +protected: + SerializerFactoryBase(ProtocolType protocol_type, SerializationType type) + : name_(ProtocolSerializerNames::get().fromType(protocol_type, type)) {} + +private: + const std::string name_; +}; + +} // namespace DubboProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/dubbo_proxy/serializer_impl.cc b/source/extensions/filters/network/dubbo_proxy/serializer_impl.cc new file mode 100644 index 000000000000..550e847cc4d2 --- /dev/null +++ b/source/extensions/filters/network/dubbo_proxy/serializer_impl.cc @@ -0,0 +1,48 @@ +#include "extensions/filters/network/dubbo_proxy/serializer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace DubboProxy { + +void RpcInvocationImpl::addParameterValue(uint32_t index, const std::string& value) { + assignParameterIfNeed(); + parameter_map_->emplace(index, value); +} + +const std::string& RpcInvocationImpl::getParameterValue(uint32_t index) const { + if (parameter_map_) { + auto itor = parameter_map_->find(index); + if (itor != parameter_map_->end()) { + return itor->second; + } + } + + return EMPTY_STRING; +} + +const RpcInvocationImpl::ParameterValueMap& RpcInvocationImpl::parameters() { + ASSERT(hasParameters()); + return *parameter_map_; +} + +const Http::HeaderMap& RpcInvocationImpl::headers() const { + ASSERT(hasHeaders()); + return *headers_; +} + +void RpcInvocationImpl::addHeader(const std::string& key, const std::string& value) { + assignHeaderIfNeed(); + headers_->addCopy(Http::LowerCaseString(key), value); +} + +void RpcInvocationImpl::addHeaderReference(const Http::LowerCaseString& key, + const std::string& value) { + assignHeaderIfNeed(); + headers_->addReference(key, value); +} + +} // namespace DubboProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/dubbo_proxy/serializer_impl.h b/source/extensions/filters/network/dubbo_proxy/serializer_impl.h new file mode 100644 index 000000000000..e5ddae675dc8 --- /dev/null +++ b/source/extensions/filters/network/dubbo_proxy/serializer_impl.h @@ -0,0 +1,64 @@ +#pragma once + +#include "extensions/filters/network/dubbo_proxy/message_impl.h" +#include "extensions/filters/network/dubbo_proxy/serializer.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace DubboProxy { + +class RpcInvocationImpl : public RpcInvocationBase { +public: + // TODO(gengleilei) Add parameter data types and implement Dubbo data type mapping. + using ParameterValueMap = std::unordered_map; + using ParameterValueMapPtr = std::unique_ptr; + + using HeaderMapPtr = std::unique_ptr; + + RpcInvocationImpl() = default; + ~RpcInvocationImpl() override = default; + + void addParameterValue(uint32_t index, const std::string& value); + const ParameterValueMap& parameters(); + const std::string& getParameterValue(uint32_t index) const; + bool hasParameters() const { return parameter_map_ != nullptr; } + + void addHeader(const std::string& key, const std::string& value); + void addHeaderReference(const Http::LowerCaseString& key, const std::string& value); + const Http::HeaderMap& headers() const; + bool hasHeaders() const { return headers_ != nullptr; } + +private: + inline void assignHeaderIfNeed() { + if (!headers_) { + headers_ = std::make_unique(); + } + } + + inline void assignParameterIfNeed() { + if (!parameter_map_) { + parameter_map_ = std::make_unique(); + } + } + + ParameterValueMapPtr parameter_map_; + HeaderMapPtr headers_; // attachment +}; + +class RpcResultImpl : public RpcResult { +public: + RpcResultImpl() = default; + ~RpcResultImpl() override = default; + + bool hasException() const override { return has_exception_; } + void setException(bool has_exception) { has_exception_ = has_exception; } + +private: + bool has_exception_ = false; +}; + +} // namespace DubboProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/test/extensions/filters/network/dubbo_proxy/BUILD b/test/extensions/filters/network/dubbo_proxy/BUILD index 3d40c8ae9bb7..f59221ca29d9 100644 --- a/test/extensions/filters/network/dubbo_proxy/BUILD +++ b/test/extensions/filters/network/dubbo_proxy/BUILD @@ -21,8 +21,9 @@ envoy_cc_mock( "//source/common/protobuf", "//source/common/protobuf:utility_lib", "//source/extensions/filters/network/dubbo_proxy:decoder_events_lib", - "//source/extensions/filters/network/dubbo_proxy:deserializer_interface", + "//source/extensions/filters/network/dubbo_proxy:decoder_lib", "//source/extensions/filters/network/dubbo_proxy:protocol_interface", + "//source/extensions/filters/network/dubbo_proxy:serializer_interface", "//source/extensions/filters/network/dubbo_proxy/filters:factory_base_lib", "//source/extensions/filters/network/dubbo_proxy/filters:filter_interface", "//source/extensions/filters/network/dubbo_proxy/router:router_interface", @@ -68,13 +69,13 @@ envoy_extension_cc_test( ) envoy_extension_cc_test( - name = "hessian_deserializer_impl_test", - srcs = ["hessian_deserializer_impl_test.cc"], + name = "dubbo_hessian2_serializer_impl_test", + srcs = ["dubbo_hessian2_serializer_impl_test.cc"], extension_name = "envoy.filters.network.dubbo_proxy", deps = [ ":mocks_lib", ":utility_lib", - "//source/extensions/filters/network/dubbo_proxy:hessian_deserializer_impl_lib", + "//source/extensions/filters/network/dubbo_proxy:dubbo_hessian2_serializer_impl_lib", "//source/extensions/filters/network/dubbo_proxy:hessian_utils_lib", "//test/mocks/server:server_mocks", ], @@ -101,6 +102,7 @@ envoy_extension_cc_test( extension_name = "envoy.filters.network.dubbo_proxy", deps = [ "//source/extensions/filters/network/dubbo_proxy:metadata_lib", + "//source/extensions/filters/network/dubbo_proxy:serializer_interface", ], ) @@ -123,8 +125,8 @@ envoy_extension_cc_test( deps = [ ":mocks_lib", "//source/extensions/filters/network/dubbo_proxy:app_exception_lib", + "//source/extensions/filters/network/dubbo_proxy:dubbo_hessian2_serializer_impl_lib", "//source/extensions/filters/network/dubbo_proxy:dubbo_protocol_impl_lib", - "//source/extensions/filters/network/dubbo_proxy:hessian_deserializer_impl_lib", "//source/extensions/filters/network/dubbo_proxy:metadata_lib", "//source/extensions/filters/network/dubbo_proxy/router:config", "//test/mocks/server:server_mocks", @@ -140,8 +142,8 @@ envoy_extension_cc_test( ":mocks_lib", ":utility_lib", "//source/extensions/filters/network/dubbo_proxy:app_exception_lib", + "//source/extensions/filters/network/dubbo_proxy:dubbo_hessian2_serializer_impl_lib", "//source/extensions/filters/network/dubbo_proxy:dubbo_protocol_impl_lib", - "//source/extensions/filters/network/dubbo_proxy:hessian_deserializer_impl_lib", "//source/extensions/filters/network/dubbo_proxy:hessian_utils_lib", "//source/extensions/filters/network/dubbo_proxy:metadata_lib", ], @@ -180,8 +182,8 @@ envoy_extension_cc_test( ":utility_lib", "//source/extensions/filters/network/dubbo_proxy:config", "//source/extensions/filters/network/dubbo_proxy:conn_manager_lib", + "//source/extensions/filters/network/dubbo_proxy:dubbo_hessian2_serializer_impl_lib", "//source/extensions/filters/network/dubbo_proxy:dubbo_protocol_impl_lib", - "//source/extensions/filters/network/dubbo_proxy:hessian_deserializer_impl_lib", "//test/mocks/server:server_mocks", ], ) diff --git a/test/extensions/filters/network/dubbo_proxy/app_exception_test.cc b/test/extensions/filters/network/dubbo_proxy/app_exception_test.cc index 0fa7c8dd0e9f..3856f893bf2c 100644 --- a/test/extensions/filters/network/dubbo_proxy/app_exception_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/app_exception_test.cc @@ -1,10 +1,10 @@ #include "extensions/filters/network/dubbo_proxy/app_exception.h" -#include "extensions/filters/network/dubbo_proxy/deserializer_impl.h" +#include "extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h" #include "extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.h" #include "extensions/filters/network/dubbo_proxy/filters/filter.h" -#include "extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h" #include "extensions/filters/network/dubbo_proxy/hessian_utils.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" +#include "extensions/filters/network/dubbo_proxy/serializer_impl.h" #include "test/extensions/filters/network/dubbo_proxy/mocks.h" @@ -21,12 +21,12 @@ namespace DubboProxy { class AppExceptionTest : public testing::Test { public: - AppExceptionTest() : metadata_(std::make_shared()) {} + AppExceptionTest() : metadata_(std::make_shared()) { + protocol_.initSerializer(SerializationType::Hessian2); + } - HessianDeserializerImpl deserializer_; DubboProtocolImpl protocol_; MessageMetadataSharedPtr metadata_; - Protocol::Context context_; }; TEST_F(AppExceptionTest, Encode) { @@ -39,16 +39,19 @@ TEST_F(AppExceptionTest, Encode) { HessianUtils::writeInt(buffer, static_cast(app_exception.response_type_)); buffer.drain(buffer.length()); - metadata_->setSerializationType(SerializationType::Hessian); + metadata_->setSerializationType(SerializationType::Hessian2); metadata_->setRequestId(0); - EXPECT_EQ(app_exception.encode(*(metadata_.get()), protocol_, deserializer_, buffer), + EXPECT_EQ(app_exception.encode(*(metadata_.get()), protocol_, buffer), DubboFilters::DirectResponse::ResponseType::Exception); MessageMetadataSharedPtr metadata = std::make_shared(); - EXPECT_TRUE(protocol_.decode(buffer, &context_, metadata)); - EXPECT_EQ(expect_body_size, context_.body_size_); + auto result = protocol_.decodeHeader(buffer, metadata); + EXPECT_TRUE(result.second); + + const ContextImpl* context = static_cast(result.first.get()); + EXPECT_EQ(expect_body_size, context->body_size()); EXPECT_EQ(metadata->message_type(), MessageType::Response); - buffer.drain(context_.header_size_); + buffer.drain(context->header_size()); // Verify the response type and content. size_t hessian_int_size; @@ -61,17 +64,17 @@ TEST_F(AppExceptionTest, Encode) { EXPECT_EQ(buffer.length(), hessian_int_size + hessian_string_size); - auto result = deserializer_.deserializeRpcResult(buffer, context_.body_size_); - EXPECT_TRUE(result->hasException()); + auto rpc_result = protocol_.serializer()->deserializeRpcResult(buffer, result.first); + EXPECT_TRUE(rpc_result.second); + EXPECT_TRUE(rpc_result.first->hasException()); buffer.drain(buffer.length()); AppException new_app_exception(app_exception); EXPECT_EQ(new_app_exception.status_, ResponseStatus::ServiceNotFound); MockProtocol mock_protocol; - EXPECT_CALL(mock_protocol, encode(_, _, _)).WillOnce(Return(false)); - EXPECT_THROW(app_exception.encode(*(metadata_.get()), mock_protocol, deserializer_, buffer), - EnvoyException); + EXPECT_CALL(mock_protocol, encode(_, _, _, _)).WillOnce(Return(false)); + EXPECT_THROW(app_exception.encode(*(metadata_.get()), mock_protocol, buffer), EnvoyException); } } // namespace DubboProxy diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index 833bf34c1ad2..f099e1dae233 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -6,8 +6,9 @@ #include "extensions/filters/network/dubbo_proxy/app_exception.h" #include "extensions/filters/network/dubbo_proxy/config.h" #include "extensions/filters/network/dubbo_proxy/conn_manager.h" +#include "extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h" #include "extensions/filters/network/dubbo_proxy/dubbo_protocol_impl.h" -#include "extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h" +#include "extensions/filters/network/dubbo_proxy/message_impl.h" #include "test/extensions/filters/network/dubbo_proxy/mocks.h" #include "test/extensions/filters/network/dubbo_proxy/utility.h" @@ -46,14 +47,18 @@ class TestConfigImpl : public ConfigImpl { if (custom_filter_) { callbacks.addDecoderFilter(custom_filter_); } - callbacks.addDecoderFilter(decoder_filter_); - } - DeserializerPtr createDeserializer() override { - if (deserializer_) { - return DeserializerPtr{deserializer_}; + if (encoder_filter_) { + callbacks.addEncoderFilter(encoder_filter_); + } + + if (decoder_filter_) { + callbacks.addDecoderFilter(decoder_filter_); + } + + if (codec_filter_) { + callbacks.addFilter(codec_filter_); } - return ConfigImpl::createDeserializer(); } ProtocolPtr createProtocol() override { @@ -73,8 +78,10 @@ class TestConfigImpl : public ConfigImpl { DubboFilters::DecoderFilterSharedPtr custom_filter_; DubboFilters::DecoderFilterSharedPtr decoder_filter_; + DubboFilters::EncoderFilterSharedPtr encoder_filter_; + DubboFilters::CodecFilterSharedPtr codec_filter_; DubboFilterStats& stats_; - MockDeserializer* deserializer_{}; + MockSerializer* serializer_{}; MockProtocol* protocol_{}; std::shared_ptr route_; }; @@ -102,8 +109,8 @@ class ConnectionManagerTest : public testing::Test { decoder_filter_.reset(new NiceMock()); config_ = std::make_unique(proto_config_, factory_context_, decoder_filter_, stats_); - if (custom_deserializer_) { - config_->deserializer_ = custom_deserializer_; + if (custom_serializer_) { + config_->serializer_ = custom_serializer_; } if (custom_protocol_) { config_->protocol_ = custom_protocol_; @@ -112,8 +119,6 @@ class ConnectionManagerTest : public testing::Test { config_->custom_filter_ = custom_filter_; } - decoder_event_handler_.reset(new NiceMock()); - ON_CALL(random_, random()).WillByDefault(Return(42)); filter_ = std::make_unique( *config_, random_, filter_callbacks_.connection_.dispatcher_.timeSource()); @@ -272,14 +277,13 @@ class ConnectionManagerTest : public testing::Test { buffer.add(std::string{'\xda', '\xbb'}); buffer.add(static_cast(&msg_type), 1); - buffer.add(std::string{0x00}); + buffer.add(std::string{0x14}); addInt64(buffer, request_id); // Request Id buffer.add(std::string{0x00, 0x00, 0x00, 0x00}); // Body Length } NiceMock factory_context_; std::shared_ptr decoder_filter_; - std::shared_ptr decoder_event_handler_; Stats::IsolatedStoreImpl store_; DubboFilterStats stats_; ConfigDubboProxy proto_config_; @@ -291,7 +295,7 @@ class ConnectionManagerTest : public testing::Test { NiceMock filter_callbacks_; NiceMock random_; std::unique_ptr filter_; - MockDeserializer* custom_deserializer_{}; + MockSerializer* custom_serializer_{}; MockProtocol* custom_protocol_{}; DubboFilters::DecoderFilterSharedPtr custom_filter_; }; @@ -334,13 +338,14 @@ TEST_F(ConnectionManagerTest, OnDataHandlesHeartbeatEvent) { EXPECT_CALL(filter_callbacks_.connection_, write(_, false)) .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) -> void { ProtocolPtr protocol = filter_->config().createProtocol(); - Protocol::Context ctx; MessageMetadataSharedPtr metadata(std::make_shared()); - EXPECT_TRUE(protocol->decode(buffer, &ctx, metadata)); - EXPECT_TRUE(ctx.is_heartbeat_); - EXPECT_EQ(metadata->response_status().value(), ResponseStatus::Ok); - EXPECT_EQ(metadata->message_type(), MessageType::Response); - buffer.drain(ctx.header_size_); + auto result = protocol->decodeHeader(buffer, metadata); + EXPECT_TRUE(result.second); + const DubboProxy::ContextImpl& ctx = *static_cast(result.first.get()); + EXPECT_TRUE(ctx.is_heartbeat()); + EXPECT_EQ(metadata->response_status(), ResponseStatus::Ok); + EXPECT_EQ(metadata->message_type(), MessageType::HeartbeatResponse); + buffer.drain(ctx.header_size()); })); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); @@ -354,7 +359,7 @@ TEST_F(ConnectionManagerTest, HandlesHeartbeatWithException) { custom_protocol_ = new NiceMock(); initializeFilter(); - EXPECT_CALL(*custom_protocol_, encode(_, _, _)).WillOnce(Return(false)); + EXPECT_CALL(*custom_protocol_, encode(_, _, _, _)).WillOnce(Return(false)); MessageMetadataSharedPtr meta = std::make_shared(); EXPECT_THROW_WITH_MESSAGE(filter_->onHeartbeat(meta), EnvoyException, @@ -408,9 +413,7 @@ TEST_F(ConnectionManagerTest, OnDataHandlesProtocolErrorOnWrite) { // Disable sniffing writeInvalidRequestMessage(write_buffer_); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - callbacks->startUpstreamResponse(deserializer, protocol); + callbacks->startUpstreamResponse(); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); EXPECT_NE(DubboFilters::UpstreamResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); @@ -425,7 +428,7 @@ TEST_F(ConnectionManagerTest, OnDataStopsSniffingWithTooManyPendingCalls) { writeHessianRequestMessage(buffer_, false, false, i); } - EXPECT_CALL(*decoder_filter_, messageEnd(_)).Times(64); + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)).Times(64); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(64U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -455,9 +458,7 @@ TEST_F(ConnectionManagerTest, OnWriteHandlesResponse) { writeHessianResponseMessage(write_buffer_, false, request_id); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - callbacks->startUpstreamResponse(deserializer, protocol); + callbacks->startUpstreamResponse(); EXPECT_EQ(callbacks->requestId(), request_id); EXPECT_EQ(callbacks->connection(), &(filter_callbacks_.connection_)); @@ -491,9 +492,7 @@ TEST_F(ConnectionManagerTest, HandlesResponseContainExceptionInfo) { writeHessianExceptionResponseMessage(write_buffer_, false, 1); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - callbacks->startUpstreamResponse(deserializer, protocol); + callbacks->startUpstreamResponse(); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); @@ -523,9 +522,7 @@ TEST_F(ConnectionManagerTest, HandlesResponseError) { writeHessianErrorResponseMessage(write_buffer_, false, 1); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - callbacks->startUpstreamResponse(deserializer, protocol); + callbacks->startUpstreamResponse(); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); @@ -552,9 +549,7 @@ TEST_F(ConnectionManagerTest, OnWriteHandlesResponseException) { writeInvalidRequestMessage(write_buffer_); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - callbacks->startUpstreamResponse(deserializer, protocol); + callbacks->startUpstreamResponse(); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Reset, callbacks->upstreamData(write_buffer_)); @@ -582,18 +577,9 @@ TEST_F(ConnectionManagerTest, OnDataResumesWithNextFilter) { .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)); - ON_CALL(*filter, transferHeaderTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::Continue; - })); - ON_CALL(*filter, transferBodyTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::Continue; - })); - // First filter stops iteration. { - EXPECT_CALL(*filter, transportBegin()).WillOnce(Return(Network::FilterStatus::StopIteration)); + EXPECT_CALL(*filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::StopIteration)); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(0U, store_.counter("test.request").value()); EXPECT_EQ(1U, @@ -603,10 +589,8 @@ TEST_F(ConnectionManagerTest, OnDataResumesWithNextFilter) { // Resume processing. { InSequence s; - EXPECT_CALL(*decoder_filter_, transportBegin()) - .WillOnce(Return(Network::FilterStatus::Continue)); - EXPECT_CALL(*filter, messageEnd(_)).WillOnce(Return(Network::FilterStatus::Continue)); - EXPECT_CALL(*decoder_filter_, messageEnd(_)).WillOnce(Return(Network::FilterStatus::Continue)); + EXPECT_CALL(*filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); callbacks->continueDecoding(); } @@ -622,20 +606,9 @@ TEST_F(ConnectionManagerTest, OnDataHandlesDubboCallWithMultipleFilters) { writeHessianRequestMessage(buffer_, false, false, 0x0F); - ON_CALL(*filter, transferHeaderTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::Continue; - })); - ON_CALL(*filter, transferBodyTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::Continue; - })); - InSequence s; - EXPECT_CALL(*filter, transportBegin()).WillOnce(Return(Network::FilterStatus::Continue)); - EXPECT_CALL(*decoder_filter_, transportBegin()).WillOnce(Return(Network::FilterStatus::Continue)); - EXPECT_CALL(*filter, messageEnd(_)).WillOnce(Return(Network::FilterStatus::Continue)); - EXPECT_CALL(*decoder_filter_, messageEnd(_)).WillOnce(Return(Network::FilterStatus::Continue)); + EXPECT_CALL(*filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); @@ -659,11 +632,8 @@ TEST_F(ConnectionManagerTest, PipelinedRequestAndResponse) { EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(2); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - writeHessianResponseMessage(write_buffer_, false, 0x01); - callbacks.front()->startUpstreamResponse(deserializer, protocol); + callbacks.front()->startUpstreamResponse(); EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Complete, callbacks.front()->upstreamData(write_buffer_)); callbacks.pop_front(); @@ -671,7 +641,7 @@ TEST_F(ConnectionManagerTest, PipelinedRequestAndResponse) { EXPECT_EQ(1U, store_.counter("test.response_success").value()); writeHessianResponseMessage(write_buffer_, false, 0x02); - callbacks.front()->startUpstreamResponse(deserializer, protocol); + callbacks.front()->startUpstreamResponse(); EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Complete, callbacks.front()->upstreamData(write_buffer_)); callbacks.pop_front(); @@ -772,7 +742,6 @@ TEST_F(ConnectionManagerTest, OnEvent) { buffer_.drain(buffer_.length()); } } - TEST_F(ConnectionManagerTest, ResponseWithUnknownSequenceID) { initializeFilter(); @@ -785,9 +754,7 @@ TEST_F(ConnectionManagerTest, ResponseWithUnknownSequenceID) { writeHessianResponseMessage(write_buffer_, false, 10); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - callbacks->startUpstreamResponse(deserializer, protocol); + callbacks->startUpstreamResponse(); EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Reset, callbacks->upstreamData(write_buffer_)); EXPECT_EQ(1U, store_.counter("test.response_decoding_error").value()); @@ -800,15 +767,6 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalReply) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); - ON_CALL(*filter, transferHeaderTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::Continue; - })); - ON_CALL(*filter, transferBodyTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::Continue; - })); - DubboFilters::DecoderFilterCallbacks* callbacks{}; EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); @@ -816,19 +774,19 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalReply) { const std::string fake_response("mock dubbo response"); NiceMock direct_response; - EXPECT_CALL(direct_response, encode(_, _, _, _)) - .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Deserializer&, + EXPECT_CALL(direct_response, encode(_, _, _)) + .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Buffer::Instance& buffer) -> DubboFilters::DirectResponse::ResponseType { buffer.add(fake_response); return DubboFilters::DirectResponse::ResponseType::SuccessReply; })); // First filter sends local reply. - EXPECT_CALL(*filter, messageEnd(_)) - .WillOnce(Invoke([&](MessageMetadataSharedPtr) -> Network::FilterStatus { + EXPECT_CALL(*filter, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { callbacks->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); callbacks->sendLocalReply(direct_response, false); - return Network::FilterStatus::StopIteration; + return FilterStatus::StopIteration; })); EXPECT_CALL(filter_callbacks_.connection_, write(_, false)) .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) -> void { @@ -836,8 +794,8 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalReply) { })); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); - EXPECT_EQ(SerializationType::Hessian, callbacks->downstreamSerializationType()); - EXPECT_EQ(ProtocolType::Dubbo, callbacks->downstreamProtocolType()); + EXPECT_EQ(SerializationType::Hessian2, callbacks->serializationType()); + EXPECT_EQ(ProtocolType::Dubbo, callbacks->protocolType()); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -853,15 +811,6 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalErrorReply) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); - ON_CALL(*filter, transferHeaderTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::Continue; - })); - ON_CALL(*filter, transferBodyTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::Continue; - })); - DubboFilters::DecoderFilterCallbacks* callbacks{}; EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); @@ -869,18 +818,18 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalErrorReply) { const std::string fake_response("mock dubbo response"); NiceMock direct_response; - EXPECT_CALL(direct_response, encode(_, _, _, _)) - .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Deserializer&, + EXPECT_CALL(direct_response, encode(_, _, _)) + .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Buffer::Instance& buffer) -> DubboFilters::DirectResponse::ResponseType { buffer.add(fake_response); return DubboFilters::DirectResponse::ResponseType::ErrorReply; })); // First filter sends local reply. - EXPECT_CALL(*filter, messageEnd(_)) - .WillOnce(Invoke([&](MessageMetadataSharedPtr) -> Network::FilterStatus { + EXPECT_CALL(*filter, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { callbacks->sendLocalReply(direct_response, false); - return Network::FilterStatus::StopIteration; + return FilterStatus::StopIteration; })); EXPECT_CALL(filter_callbacks_.connection_, write(_, false)) .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) -> void { @@ -900,9 +849,9 @@ TEST_F(ConnectionManagerTest, TwoWayRequestWithEndStream) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 0x0F); - ON_CALL(*decoder_filter_, transferHeaderTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::StopIteration; + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { + return FilterStatus::StopIteration; })); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) @@ -916,15 +865,15 @@ TEST_F(ConnectionManagerTest, OneWayRequestWithEndStream) { initializeFilter(); writeHessianRequestMessage(buffer_, true, false, 0x0F); - EXPECT_CALL(*decoder_filter_, messageEnd(_)) - .WillOnce(Invoke([&](MessageMetadataSharedPtr) -> Network::FilterStatus { - return Network::FilterStatus::StopIteration; + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { + return FilterStatus::StopIteration; })); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) - .Times(0); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(0); + .Times(1); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); EXPECT_EQ(filter_->onData(buffer_, true), Network::FilterStatus::StopIteration); - EXPECT_EQ(0U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); + EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); } TEST_F(ConnectionManagerTest, EmptyRequestData) { @@ -940,9 +889,9 @@ TEST_F(ConnectionManagerTest, StopHandleRequest) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 0x0F); - ON_CALL(*decoder_filter_, transferHeaderTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance&, size_t) -> Network::FilterStatus { - return Network::FilterStatus::StopIteration; + ON_CALL(*decoder_filter_, onMessageDecoded(_, _)) + .WillByDefault(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { + return FilterStatus::StopIteration; })); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) @@ -973,8 +922,8 @@ TEST_F(ConnectionManagerTest, SendsLocalReplyWithCloseConnection) { const std::string fake_response("mock dubbo response"); NiceMock direct_response; - EXPECT_CALL(direct_response, encode(_, _, _, _)) - .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Deserializer&, + EXPECT_CALL(direct_response, encode(_, _, _)) + .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Buffer::Instance& buffer) -> DubboFilters::DirectResponse::ResponseType { buffer.add(fake_response); return DubboFilters::DirectResponse::ResponseType::ErrorReply; @@ -987,7 +936,7 @@ TEST_F(ConnectionManagerTest, SendsLocalReplyWithCloseConnection) { EXPECT_EQ(1U, store_.counter("test.local_response_error").value()); // The connection closed. - EXPECT_CALL(direct_response, encode(_, _, _, _)).Times(0); + EXPECT_CALL(direct_response, encode(_, _, _)).Times(0); filter_->sendLocalReply(metadata, direct_response, true); } @@ -995,19 +944,16 @@ TEST_F(ConnectionManagerTest, ContinueDecodingWithHalfClose) { initializeFilter(); writeHessianRequestMessage(buffer_, true, false, 0x0F); - EXPECT_CALL(*decoder_filter_, messageEnd(_)) - .WillOnce(Invoke([&](MessageMetadataSharedPtr) -> Network::FilterStatus { - return Network::FilterStatus::StopIteration; + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { + return FilterStatus::StopIteration; })); - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) - .Times(0); - EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(0); - EXPECT_EQ(filter_->onData(buffer_, true), Network::FilterStatus::StopIteration); - EXPECT_EQ(0U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) .Times(1); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_EQ(filter_->onData(buffer_, true), Network::FilterStatus::StopIteration); + EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); + filter_->continueDecoding(); } @@ -1030,11 +976,9 @@ TEST_F(ConnectionManagerTest, RoutingSuccess) { TEST_F(ConnectionManagerTest, RoutingFailure) { initializeFilter(); - writeHessianRequestMessage(buffer_, false, false, 0x0F); + writePartialHessianRequestMessage(buffer_, false, false, 0x0F, true); - EXPECT_CALL(*decoder_filter_, transportBegin()).WillOnce(Invoke([&]() -> Network::FilterStatus { - return Network::FilterStatus::StopIteration; - })); + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)).Times(0); DubboFilters::DecoderFilterCallbacks* callbacks{}; EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) @@ -1076,9 +1020,7 @@ TEST_F(ConnectionManagerTest, NeedMoreDataForHandleResponse) { writePartialHessianRequestMessage(write_buffer_, false, false, 0x0F, true); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - callbacks->startUpstreamResponse(deserializer, protocol); + callbacks->startUpstreamResponse(); EXPECT_EQ(DubboFilters::UpstreamResponseStatus::MoreData, callbacks->upstreamData(write_buffer_)); } @@ -1091,9 +1033,9 @@ TEST_F(ConnectionManagerTest, PendingMessageEnd) { DubboFilters::DecoderFilterCallbacks* callbacks{}; EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*decoder_filter_, messageEnd(_)) - .WillOnce(Invoke([&](MessageMetadataSharedPtr) -> Network::FilterStatus { - return Network::FilterStatus::StopIteration; + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { + return FilterStatus::StopIteration; })); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); @@ -1124,11 +1066,13 @@ serialization_type: Hessian2 DubboFilters::DecoderFilterCallbacks* callbacks{}; EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*decoder_filter_, messageEnd(_)) - .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata) -> Network::FilterStatus { - metadata->setServiceName("org.apache.dubbo.demo.DemoService"); - metadata->setMethodName("test"); - return Network::FilterStatus::StopIteration; + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata, ContextSharedPtr) -> FilterStatus { + auto invo = static_cast(&metadata->invocation_info()); + auto data = const_cast(invo); + data->setServiceName("org.apache.dubbo.demo.DemoService"); + data->setMethodName("test"); + return FilterStatus::StopIteration; })); EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); @@ -1153,9 +1097,7 @@ TEST_F(ConnectionManagerTest, TransportEndWithConnectionClose) { writeHessianResponseMessage(write_buffer_, false, 1); - DubboProtocolImpl protocol; - HessianDeserializerImpl deserializer; - callbacks->startUpstreamResponse(deserializer, protocol); + callbacks->startUpstreamResponse(); filter_callbacks_.connection_.close(Network::ConnectionCloseType::FlushWrite); @@ -1163,27 +1105,26 @@ TEST_F(ConnectionManagerTest, TransportEndWithConnectionClose) { EXPECT_EQ(1U, store_.counter("test.response_error_caused_connection_close").value()); } -TEST_F(ConnectionManagerTest, TransportBeginReturnStopIteration) { +TEST_F(ConnectionManagerTest, MessageDecodedReturnStopIteration) { initializeFilter(); DubboFilters::DecoderFilterCallbacks* callbacks{}; EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*decoder_filter_, transportBegin()).WillOnce(Invoke([&]() -> Network::FilterStatus { - return Network::FilterStatus::StopIteration; - })); - - EXPECT_CALL(*decoder_filter_, messageBegin(_, _, _)).Times(0); - EXPECT_CALL(*decoder_filter_, messageEnd(_)).Times(0); - EXPECT_CALL(*decoder_filter_, transferBodyTo(_, _)).Times(0); - EXPECT_CALL(*decoder_filter_, transportEnd()).Times(0); - // The sendLocalReply is not called and the message type is not oneway, // the ActiveMessage object is not destroyed. EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(0); writeHessianRequestMessage(buffer_, false, false, 1); + + size_t buf_size = buffer_.length(); + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr ctx) -> FilterStatus { + EXPECT_EQ(ctx->message_size(), buf_size); + return FilterStatus::StopIteration; + })); + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); // Buffer data should be consumed. @@ -1193,7 +1134,7 @@ TEST_F(ConnectionManagerTest, TransportBeginReturnStopIteration) { EXPECT_EQ(0U, store_.counter("test.request").value()); } -TEST_F(ConnectionManagerTest, SendLocalReplyInTransportBegin) { +TEST_F(ConnectionManagerTest, SendLocalReplyInMessageDecoded) { initializeFilter(); DubboFilters::DecoderFilterCallbacks* callbacks{}; @@ -1202,26 +1143,23 @@ TEST_F(ConnectionManagerTest, SendLocalReplyInTransportBegin) { const std::string fake_response("mock dubbo response"); NiceMock direct_response; - EXPECT_CALL(direct_response, encode(_, _, _, _)) - .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Deserializer&, + EXPECT_CALL(direct_response, encode(_, _, _)) + .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Buffer::Instance& buffer) -> DubboFilters::DirectResponse::ResponseType { buffer.add(fake_response); return DubboFilters::DirectResponse::ResponseType::ErrorReply; })); - EXPECT_CALL(*decoder_filter_, transportBegin()).WillOnce(Invoke([&]() -> Network::FilterStatus { - callbacks->sendLocalReply(direct_response, false); - return Network::FilterStatus::StopIteration; - })); - - EXPECT_CALL(*decoder_filter_, messageBegin(_, _, _)).Times(0); - EXPECT_CALL(*decoder_filter_, messageEnd(_)).Times(0); - EXPECT_CALL(*decoder_filter_, transferBodyTo(_, _)).Times(0); - EXPECT_CALL(*decoder_filter_, transportEnd()).Times(0); + EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { + callbacks->sendLocalReply(direct_response, false); + return FilterStatus::StopIteration; + })); // The sendLocalReply is called, the ActiveMessage object should be destroyed. EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); writeHessianRequestMessage(buffer_, false, false, 1); + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); // Buffer data should be consumed. @@ -1231,42 +1169,103 @@ TEST_F(ConnectionManagerTest, SendLocalReplyInTransportBegin) { EXPECT_EQ(1U, store_.counter("test.request").value()); } -TEST_F(ConnectionManagerTest, SendLocalReplyInMessageBegin) { +TEST_F(ConnectionManagerTest, HandleResponseWithEncoderFilter) { + uint64_t request_id = 100; initializeFilter(); + config_->encoder_filter_ = std::make_unique(); + auto mock_encoder_filter = + static_cast(config_->encoder_filter_.get()); + + writeHessianRequestMessage(buffer_, false, false, request_id); DubboFilters::DecoderFilterCallbacks* callbacks{}; EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - const std::string fake_response("mock dubbo response"); - NiceMock direct_response; - EXPECT_CALL(direct_response, encode(_, _, _, _)) - .WillOnce(Invoke([&](MessageMetadata&, Protocol&, Deserializer&, - Buffer::Instance& buffer) -> DubboFilters::DirectResponse::ResponseType { - buffer.add(fake_response); - return DubboFilters::DirectResponse::ResponseType::ErrorReply; - })); - EXPECT_CALL(*decoder_filter_, messageBegin(_, _, _)) - .WillOnce(Invoke([&](MessageType, int64_t, SerializationType) -> Network::FilterStatus { - callbacks->sendLocalReply(direct_response, false); - return Network::FilterStatus::StopIteration; - })); + EXPECT_CALL(*mock_encoder_filter, setEncoderFilterCallbacks(_)).Times(1); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(1U, store_.counter("test.request").value()); + EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); + + writeHessianResponseMessage(write_buffer_, false, request_id); - EXPECT_CALL(*decoder_filter_, messageEnd(_)).Times(0); - EXPECT_CALL(*decoder_filter_, transferBodyTo(_, _)).Times(0); - EXPECT_CALL(*decoder_filter_, transportEnd()).Times(0); + callbacks->startUpstreamResponse(); + + EXPECT_EQ(callbacks->requestId(), request_id); + EXPECT_EQ(callbacks->connection(), &(filter_callbacks_.connection_)); + EXPECT_GE(callbacks->streamId(), 0); + + size_t expect_response_length = write_buffer_.length(); + EXPECT_CALL(*mock_encoder_filter, onMessageEncoded(_, _)) + .WillOnce( + Invoke([&](MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) -> FilterStatus { + EXPECT_EQ(metadata->request_id(), request_id); + EXPECT_EQ(ctx->message_size(), expect_response_length); + return FilterStatus::Continue; + })); - // The sendLocalReply is called, the ActiveMessage object should be destroyed. EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); - writeHessianRequestMessage(buffer_, false, false, 1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); - // Buffer data should be consumed. - EXPECT_EQ(0, buffer_.length()); + EXPECT_EQ(1U, store_.counter("test.response").value()); + EXPECT_EQ(1U, store_.counter("test.response_success").value()); +} - // The finalizeRequest should be called. +TEST_F(ConnectionManagerTest, HandleResponseWithCodecFilter) { + uint64_t request_id = 100; + initializeFilter(); + config_->codec_filter_ = std::make_unique(); + auto mock_codec_filter = + static_cast(config_->codec_filter_.get()); + config_->decoder_filter_.reset(); + decoder_filter_.reset(); + EXPECT_EQ(config_->decoder_filter_.get(), nullptr); + + writeHessianRequestMessage(buffer_, false, false, request_id); + + DubboFilters::DecoderFilterCallbacks* callbacks{}; + EXPECT_CALL(*mock_codec_filter, setDecoderFilterCallbacks(_)) + .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); + EXPECT_CALL(*mock_codec_filter, onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata, ContextSharedPtr) -> FilterStatus { + EXPECT_EQ(metadata->request_id(), request_id); + return FilterStatus::Continue; + })); + + EXPECT_CALL(*mock_codec_filter, setEncoderFilterCallbacks(_)).Times(1); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); + EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); + + writeHessianResponseMessage(write_buffer_, false, request_id); + + callbacks->startUpstreamResponse(); + + EXPECT_EQ(callbacks->requestId(), request_id); + EXPECT_EQ(callbacks->connection(), &(filter_callbacks_.connection_)); + EXPECT_GE(callbacks->streamId(), 0); + + size_t expect_response_length = write_buffer_.length(); + EXPECT_CALL(*mock_codec_filter, onMessageEncoded(_, _)) + .WillOnce( + Invoke([&](MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) -> FilterStatus { + EXPECT_EQ(metadata->request_id(), request_id); + EXPECT_EQ(ctx->message_size(), expect_response_length); + return FilterStatus::Continue; + })); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); + EXPECT_CALL(*mock_codec_filter, onDestroy()).Times(1); + + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); + + EXPECT_EQ(1U, store_.counter("test.response").value()); + EXPECT_EQ(1U, store_.counter("test.response_success").value()); } } // namespace DubboProxy diff --git a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc index f33943c94100..ae9c291011c2 100644 --- a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc @@ -1,5 +1,6 @@ #include "extensions/filters/network/dubbo_proxy/decoder.h" -#include "extensions/filters/network/dubbo_proxy/deserializer_impl.h" +#include "extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h" +#include "extensions/filters/network/dubbo_proxy/message_impl.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" #include "test/extensions/filters/network/dubbo_proxy/mocks.h" @@ -22,33 +23,36 @@ namespace DubboProxy { class DecoderStateMachineTestBase { public: - DecoderStateMachineTestBase() : metadata_(std::make_shared()) { - context_.header_size_ = 16; - } - virtual ~DecoderStateMachineTestBase() = default; + DecoderStateMachineTestBase() = default; + virtual ~DecoderStateMachineTestBase() { active_stream_.reset(); } void initHandler() { - EXPECT_CALL(decoder_callback_, newDecoderEventHandler()) - .WillOnce(Invoke([this]() -> DecoderEventHandler* { return &handler_; })); + ON_CALL(delegate_, newStream(_, _)) + .WillByDefault(Invoke([this](MessageMetadataSharedPtr data, + ContextSharedPtr ctx) -> ActiveStream* { + this->active_stream_ = std::make_shared>(handler_, data, ctx); + return active_stream_.get(); + })); } - void initProtocolDecoder(MessageType type, int32_t body_size, bool is_heartbeat = false) { - EXPECT_CALL(protocol_, decode(_, _, _)) - .WillOnce(Invoke([=](Buffer::Instance&, Protocol::Context* context, - MessageMetadataSharedPtr metadata) -> bool { - context->is_heartbeat_ = is_heartbeat; - context->body_size_ = body_size; - metadata->setMessageType(type); - return true; - })); + void initProtocolDecoder(MessageType type, int32_t body_size) { + ON_CALL(protocol_, decodeHeader(_, _)) + .WillByDefault( + Invoke([=](Buffer::Instance&, + MessageMetadataSharedPtr metadata) -> std::pair { + auto context = std::make_shared(); + context->set_header_size(16); + context->set_body_size(body_size); + metadata->setMessageType(type); + + return std::pair(context, true); + })); } NiceMock protocol_; - NiceMock deserializer_; - NiceMock handler_; - NiceMock decoder_callback_; - MessageMetadataSharedPtr metadata_; - Protocol::Context context_; + NiceMock delegate_; + std::shared_ptr> active_stream_; + NiceMock handler_; }; class DubboDecoderStateMachineTest : public DecoderStateMachineTestBase, public testing::Test {}; @@ -59,163 +63,162 @@ class DubboDecoderTest : public testing::Test { virtual ~DubboDecoderTest() override = default; NiceMock protocol_; - NiceMock deserializer_; - NiceMock callbacks_; + NiceMock handler_; + NiceMock request_callbacks_; + NiceMock response_callbacks_; }; TEST_F(DubboDecoderStateMachineTest, EmptyData) { - EXPECT_CALL(protocol_, decode(_, _, _)).Times(1); - EXPECT_CALL(handler_, transferHeaderTo(_, _)).Times(0); - EXPECT_CALL(handler_, messageBegin(_, _, _)).Times(0); + EXPECT_CALL(protocol_, decodeHeader(_, _)).Times(1); + EXPECT_CALL(delegate_, newStream(_, _)).Times(0); + EXPECT_CALL(delegate_, onHeartbeat(_)).Times(0); - DecoderStateMachine dsm(protocol_, deserializer_, metadata_, decoder_callback_); + DecoderStateMachine dsm(protocol_, delegate_); Buffer::OwnedImpl buffer; EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); } TEST_F(DubboDecoderStateMachineTest, OnlyHaveHeaderData) { initHandler(); - initProtocolDecoder(MessageType::Request, 1, false); + initProtocolDecoder(MessageType::Request, 1); - EXPECT_CALL(handler_, transportBegin()).Times(1); - EXPECT_CALL(handler_, transferHeaderTo(_, _)).Times(1); - EXPECT_CALL(handler_, messageBegin(_, _, _)).Times(1); - EXPECT_CALL(handler_, messageEnd(_)).Times(0); + EXPECT_CALL(delegate_, onHeartbeat(_)).Times(0); + EXPECT_CALL(protocol_, decodeData(_, _, _)).WillOnce(Return(false)); Buffer::OwnedImpl buffer; - DecoderStateMachine dsm(protocol_, deserializer_, metadata_, decoder_callback_); + DecoderStateMachine dsm(protocol_, delegate_); EXPECT_EQ(dsm.run(buffer), ProtocolState::WaitForData); } TEST_F(DubboDecoderStateMachineTest, RequestMessageCallbacks) { initHandler(); - initProtocolDecoder(MessageType::Request, 0, false); + initProtocolDecoder(MessageType::Request, 0); - EXPECT_CALL(handler_, transportBegin()).Times(1); - EXPECT_CALL(handler_, transferHeaderTo(_, _)).Times(1); - EXPECT_CALL(handler_, messageBegin(_, _, _)).Times(1); - EXPECT_CALL(handler_, messageEnd(_)).Times(1); - EXPECT_CALL(handler_, transferBodyTo(_, _)).Times(1); - EXPECT_CALL(handler_, transportEnd()).Times(1); + EXPECT_CALL(delegate_, onHeartbeat(_)).Times(0); + EXPECT_CALL(protocol_, decodeData(_, _, _)).WillOnce(Return(true)); + EXPECT_CALL(handler_, onStreamDecoded(_, _)).Times(1); - EXPECT_CALL(deserializer_, deserializeRpcInvocation(_, _, _)).WillOnce(Return()); - - DecoderStateMachine dsm(protocol_, deserializer_, metadata_, decoder_callback_); + DecoderStateMachine dsm(protocol_, delegate_); Buffer::OwnedImpl buffer; EXPECT_EQ(dsm.run(buffer), ProtocolState::Done); + + EXPECT_EQ(active_stream_->metadata_->message_type(), MessageType::Request); } TEST_F(DubboDecoderStateMachineTest, ResponseMessageCallbacks) { initHandler(); - initProtocolDecoder(MessageType::Response, 0, false); - - EXPECT_CALL(handler_, transportBegin()).Times(1); - EXPECT_CALL(handler_, transferHeaderTo(_, _)).Times(1); - EXPECT_CALL(handler_, messageBegin(_, _, _)).Times(1); - EXPECT_CALL(handler_, messageEnd(_)).Times(1); - EXPECT_CALL(handler_, transferBodyTo(_, _)).Times(1); - EXPECT_CALL(handler_, transportEnd()).Times(1); - - EXPECT_CALL(deserializer_, deserializeRpcResult(_, _)) - .WillOnce(Invoke([](Buffer::Instance&, size_t) -> RpcResultPtr { - return std::make_unique(false); - })); + initProtocolDecoder(MessageType::Response, 0); + + EXPECT_CALL(delegate_, onHeartbeat(_)).Times(0); + EXPECT_CALL(protocol_, decodeData(_, _, _)).WillOnce(Return(true)); + EXPECT_CALL(handler_, onStreamDecoded(_, _)).Times(1); - DecoderStateMachine dsm(protocol_, deserializer_, metadata_, decoder_callback_); + DecoderStateMachine dsm(protocol_, delegate_); Buffer::OwnedImpl buffer; EXPECT_EQ(dsm.run(buffer), ProtocolState::Done); + + EXPECT_EQ(active_stream_->metadata_->message_type(), MessageType::Response); } -TEST_F(DubboDecoderStateMachineTest, DeserializeRpcInvocationException) { +TEST_F(DubboDecoderStateMachineTest, SerializeRpcInvocationException) { initHandler(); - initProtocolDecoder(MessageType::Request, 0, false); + initProtocolDecoder(MessageType::Request, 0); - EXPECT_CALL(handler_, messageEnd(_)).Times(0); - EXPECT_CALL(handler_, transferBodyTo(_, _)).Times(0); - EXPECT_CALL(handler_, transportEnd()).Times(0); + EXPECT_CALL(delegate_, newStream(_, _)).Times(1); + EXPECT_CALL(delegate_, onHeartbeat(_)).Times(0); + EXPECT_CALL(handler_, onStreamDecoded(_, _)).Times(0); - EXPECT_CALL(deserializer_, deserializeRpcInvocation(_, _, _)) - .WillOnce(Invoke([](Buffer::Instance&, int32_t, MessageMetadataSharedPtr) -> void { - throw EnvoyException(fmt::format("mock deserialize exception")); + EXPECT_CALL(protocol_, decodeData(_, _, _)) + .WillOnce(Invoke([&](Buffer::Instance&, ContextSharedPtr, MessageMetadataSharedPtr) -> bool { + throw EnvoyException(fmt::format("mock serialize exception")); })); - DecoderStateMachine dsm(protocol_, deserializer_, metadata_, decoder_callback_); + DecoderStateMachine dsm(protocol_, delegate_); Buffer::OwnedImpl buffer; - EXPECT_THROW_WITH_MESSAGE(dsm.run(buffer), EnvoyException, "mock deserialize exception"); - EXPECT_EQ(dsm.currentState(), ProtocolState::OnMessageEnd); + EXPECT_THROW_WITH_MESSAGE(dsm.run(buffer), EnvoyException, "mock serialize exception"); + EXPECT_EQ(dsm.currentState(), ProtocolState::OnDecodeStreamData); } -TEST_F(DubboDecoderStateMachineTest, DeserializeRpcResultException) { +TEST_F(DubboDecoderStateMachineTest, SerializeRpcResultException) { initHandler(); - initProtocolDecoder(MessageType::Response, 0, false); + initProtocolDecoder(MessageType::Response, 0); - EXPECT_CALL(handler_, messageEnd(_)).Times(0); - EXPECT_CALL(handler_, transferBodyTo(_, _)).Times(0); - EXPECT_CALL(handler_, transportEnd()).Times(0); + EXPECT_CALL(delegate_, newStream(_, _)).Times(1); + EXPECT_CALL(delegate_, onHeartbeat(_)).Times(0); + EXPECT_CALL(handler_, onStreamDecoded(_, _)).Times(0); - EXPECT_CALL(deserializer_, deserializeRpcResult(_, _)) - .WillOnce(Invoke([](Buffer::Instance&, size_t) -> RpcResultPtr { - throw EnvoyException(fmt::format("mock deserialize exception")); + EXPECT_CALL(protocol_, decodeData(_, _, _)) + .WillOnce(Invoke([&](Buffer::Instance&, ContextSharedPtr, MessageMetadataSharedPtr) -> bool { + throw EnvoyException(fmt::format("mock serialize exception")); })); - DecoderStateMachine dsm(protocol_, deserializer_, metadata_, decoder_callback_); + DecoderStateMachine dsm(protocol_, delegate_); Buffer::OwnedImpl buffer; - EXPECT_THROW_WITH_MESSAGE(dsm.run(buffer), EnvoyException, "mock deserialize exception"); - EXPECT_EQ(dsm.currentState(), ProtocolState::OnMessageEnd); + EXPECT_THROW_WITH_MESSAGE(dsm.run(buffer), EnvoyException, "mock serialize exception"); + EXPECT_EQ(dsm.currentState(), ProtocolState::OnDecodeStreamData); } TEST_F(DubboDecoderStateMachineTest, ProtocolDecodeException) { - EXPECT_CALL(decoder_callback_, newDecoderEventHandler()).Times(0); - EXPECT_CALL(protocol_, decode(_, _, _)) - .WillOnce(Invoke([](Buffer::Instance&, Protocol::Context*, MessageMetadataSharedPtr) -> bool { - throw EnvoyException(fmt::format("mock deserialize exception")); - })); + EXPECT_CALL(delegate_, newStream(_, _)).Times(0); + EXPECT_CALL(protocol_, decodeHeader(_, _)) + .WillOnce(Invoke( + [](Buffer::Instance&, MessageMetadataSharedPtr) -> std::pair { + throw EnvoyException(fmt::format("mock protocol decode exception")); + })); - DecoderStateMachine dsm(protocol_, deserializer_, metadata_, decoder_callback_); + DecoderStateMachine dsm(protocol_, delegate_); Buffer::OwnedImpl buffer; - EXPECT_THROW_WITH_MESSAGE(dsm.run(buffer), EnvoyException, "mock deserialize exception"); - EXPECT_EQ(dsm.currentState(), ProtocolState::OnTransportBegin); + EXPECT_THROW_WITH_MESSAGE(dsm.run(buffer), EnvoyException, "mock protocol decode exception"); + EXPECT_EQ(dsm.currentState(), ProtocolState::OnDecodeStreamHeader); } TEST_F(DubboDecoderTest, NeedMoreDataForProtocolHeader) { - EXPECT_CALL(protocol_, decode(_, _, _)) - .WillOnce(Invoke([](Buffer::Instance&, Protocol::Context*, MessageMetadataSharedPtr) -> bool { - return false; - })); - EXPECT_CALL(callbacks_, newDecoderEventHandler()).Times(0); + EXPECT_CALL(request_callbacks_, newStream()).Times(0); + EXPECT_CALL(protocol_, decodeHeader(_, _)) + .WillOnce(Invoke( + [](Buffer::Instance&, MessageMetadataSharedPtr) -> std::pair { + return std::pair(nullptr, false); + })); - Decoder decoder(protocol_, deserializer_, callbacks_); + RequestDecoder decoder(protocol_, request_callbacks_); Buffer::OwnedImpl buffer; bool buffer_underflow; - EXPECT_EQ(decoder.onData(buffer, buffer_underflow), Network::FilterStatus::Continue); + EXPECT_EQ(decoder.onData(buffer, buffer_underflow), FilterStatus::Continue); EXPECT_EQ(buffer_underflow, true); } TEST_F(DubboDecoderTest, NeedMoreDataForProtocolBody) { - EXPECT_CALL(protocol_, decode(_, _, _)) - .WillOnce(Invoke([](Buffer::Instance&, Protocol::Context* context, - MessageMetadataSharedPtr metadata) -> bool { - metadata->setMessageType(MessageType::Request); - context->body_size_ = 10; - return true; + EXPECT_CALL(protocol_, decodeHeader(_, _)) + .WillOnce(Invoke([](Buffer::Instance&, + MessageMetadataSharedPtr metadate) -> std::pair { + metadate->setMessageType(MessageType::Response); + auto context = std::make_shared(); + context->set_header_size(16); + context->set_body_size(10); + return std::pair(context, true); })); - EXPECT_CALL(callbacks_, newDecoderEventHandler()).Times(1); - EXPECT_CALL(callbacks_.handler_, transportBegin()).Times(1); - EXPECT_CALL(callbacks_.handler_, transferHeaderTo(_, _)).Times(1); - EXPECT_CALL(callbacks_.handler_, messageBegin(_, _, _)).Times(1); - EXPECT_CALL(callbacks_.handler_, messageEnd(_)).Times(0); - EXPECT_CALL(callbacks_.handler_, transferBodyTo(_, _)).Times(0); - EXPECT_CALL(callbacks_.handler_, transportEnd()).Times(0); + EXPECT_CALL(protocol_, decodeData(_, _, _)) + .WillOnce(Invoke([&](Buffer::Instance&, ContextSharedPtr, MessageMetadataSharedPtr) -> bool { + return false; + })); + + std::shared_ptr> active_stream; - Decoder decoder(protocol_, deserializer_, callbacks_); + EXPECT_CALL(response_callbacks_, newStream()).WillOnce(Invoke([this]() -> StreamHandler& { + return handler_; + })); + EXPECT_CALL(response_callbacks_, onHeartbeat(_)).Times(0); + EXPECT_CALL(handler_, onStreamDecoded(_, _)).Times(0); + + ResponseDecoder decoder(protocol_, response_callbacks_); Buffer::OwnedImpl buffer; bool buffer_underflow; - EXPECT_EQ(decoder.onData(buffer, buffer_underflow), Network::FilterStatus::Continue); + EXPECT_EQ(decoder.onData(buffer, buffer_underflow), FilterStatus::Continue); EXPECT_EQ(buffer_underflow, true); } @@ -223,30 +226,31 @@ TEST_F(DubboDecoderTest, decodeResponseMessage) { Buffer::OwnedImpl buffer; buffer.add(std::string({'\xda', '\xbb', '\xc2', 0x00})); - EXPECT_CALL(protocol_, decode(_, _, _)) - .WillOnce(Invoke([&](Buffer::Instance&, Protocol::Context* context, - MessageMetadataSharedPtr metadata) -> bool { - metadata->setMessageType(MessageType::Response); - context->body_size_ = buffer.length(); - return true; + EXPECT_CALL(protocol_, decodeHeader(_, _)) + .WillOnce(Invoke([](Buffer::Instance&, + MessageMetadataSharedPtr metadate) -> std::pair { + metadate->setMessageType(MessageType::Response); + auto context = std::make_shared(); + context->set_header_size(16); + context->set_body_size(10); + return std::pair(context, true); })); - EXPECT_CALL(deserializer_, deserializeRpcResult(_, _)) - .WillOnce(Invoke([](Buffer::Instance&, size_t) -> RpcResultPtr { - return std::make_unique(true); + EXPECT_CALL(protocol_, decodeData(_, _, _)) + .WillOnce(Invoke([&](Buffer::Instance&, ContextSharedPtr, MessageMetadataSharedPtr) -> bool { + return true; })); - EXPECT_CALL(callbacks_, newDecoderEventHandler()).Times(1); - EXPECT_CALL(callbacks_.handler_, transportBegin()).Times(1); - EXPECT_CALL(callbacks_.handler_, transferHeaderTo(_, _)).Times(1); - EXPECT_CALL(callbacks_.handler_, messageBegin(_, _, _)).Times(1); - EXPECT_CALL(callbacks_.handler_, messageEnd(_)).Times(1); - EXPECT_CALL(callbacks_.handler_, transferBodyTo(_, _)).Times(1); - EXPECT_CALL(callbacks_.handler_, transportEnd()).Times(1); - Decoder decoder(protocol_, deserializer_, callbacks_); + EXPECT_CALL(response_callbacks_, newStream()).WillOnce(Invoke([this]() -> StreamHandler& { + return handler_; + })); + EXPECT_CALL(response_callbacks_, onHeartbeat(_)).Times(0); + EXPECT_CALL(handler_, onStreamDecoded(_, _)).Times(1); + + ResponseDecoder decoder(protocol_, response_callbacks_); bool buffer_underflow; - EXPECT_EQ(decoder.onData(buffer, buffer_underflow), Network::FilterStatus::Continue); - EXPECT_EQ(buffer_underflow, false); + EXPECT_EQ(decoder.onData(buffer, buffer_underflow), FilterStatus::Continue); + EXPECT_EQ(buffer_underflow, true); } } // namespace DubboProxy diff --git a/test/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl_test.cc b/test/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl_test.cc similarity index 56% rename from test/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl_test.cc rename to test/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl_test.cc index 38a66a6ffdbf..3a74b52b3f89 100644 --- a/test/extensions/filters/network/dubbo_proxy/hessian_deserializer_impl_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl_test.cc @@ -1,5 +1,6 @@ -#include "extensions/filters/network/dubbo_proxy/hessian_deserializer_impl.h" +#include "extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h" #include "extensions/filters/network/dubbo_proxy/hessian_utils.h" +#include "extensions/filters/network/dubbo_proxy/message_impl.h" #include "test/extensions/filters/network/dubbo_proxy/mocks.h" #include "test/extensions/filters/network/dubbo_proxy/utility.h" @@ -17,12 +18,12 @@ namespace NetworkFilters { namespace DubboProxy { TEST(HessianProtocolTest, Name) { - HessianDeserializerImpl deserializer; - EXPECT_EQ(deserializer.name(), "hessian"); + DubboHessian2SerializerImpl serializer; + EXPECT_EQ(serializer.name(), "dubbo.hessian2"); } TEST(HessianProtocolTest, deserializeRpcInvocation) { - HessianDeserializerImpl deserializer; + DubboHessian2SerializerImpl serializer; { Buffer::OwnedImpl buffer; @@ -32,11 +33,15 @@ TEST(HessianProtocolTest, deserializeRpcInvocation) { 0x05, '0', '.', '0', '.', '0', // Service version 0x04, 't', 'e', 's', 't', // method name })); - MessageMetadataSharedPtr metadata = std::make_shared(); - deserializer.deserializeRpcInvocation(buffer, buffer.length(), metadata); - EXPECT_STREQ("test", metadata->method_name().value().c_str()); - EXPECT_STREQ("test", metadata->service_name().c_str()); - EXPECT_STREQ("0.0.0", metadata->service_version().value().c_str()); + std::shared_ptr context = std::make_shared(); + context->set_body_size(buffer.length()); + auto result = serializer.deserializeRpcInvocation(buffer, context); + EXPECT_TRUE(result.second); + + auto invo = result.first; + EXPECT_STREQ("test", invo->method_name().c_str()); + EXPECT_STREQ("test", invo->service_name().c_str()); + EXPECT_STREQ("0.0.0", invo->service_version().value().c_str()); } // incorrect body size @@ -50,15 +55,16 @@ TEST(HessianProtocolTest, deserializeRpcInvocation) { })); std::string exception_string = fmt::format("RpcInvocation size({}) large than body size({})", buffer.length(), buffer.length() - 1); - MessageMetadataSharedPtr metadata = std::make_shared(); - EXPECT_THROW_WITH_MESSAGE( - deserializer.deserializeRpcInvocation(buffer, buffer.length() - 1, metadata), - EnvoyException, exception_string); + std::shared_ptr context = std::make_shared(); + context->set_body_size(buffer.length() - 1); + EXPECT_THROW_WITH_MESSAGE(serializer.deserializeRpcInvocation(buffer, context), EnvoyException, + exception_string); } } TEST(HessianProtocolTest, deserializeRpcResult) { - HessianDeserializerImpl deserializer; + DubboHessian2SerializerImpl serializer; + std::shared_ptr context = std::make_shared(); { Buffer::OwnedImpl buffer; @@ -66,8 +72,10 @@ TEST(HessianProtocolTest, deserializeRpcResult) { '\x94', // return type 0x04, 't', 'e', 's', 't', // return body })); - auto result = deserializer.deserializeRpcResult(buffer, 4); - EXPECT_FALSE(result->hasException()); + context->set_body_size(4); + auto result = serializer.deserializeRpcResult(buffer, context); + EXPECT_TRUE(result.second); + EXPECT_FALSE(result.first->hasException()); } { @@ -76,8 +84,10 @@ TEST(HessianProtocolTest, deserializeRpcResult) { '\x93', // return type 0x04, 't', 'e', 's', 't', // return body })); - auto result = deserializer.deserializeRpcResult(buffer, 4); - EXPECT_TRUE(result->hasException()); + context->set_body_size(4); + auto result = serializer.deserializeRpcResult(buffer, context); + EXPECT_TRUE(result.second); + EXPECT_TRUE(result.first->hasException()); } { @@ -86,8 +96,10 @@ TEST(HessianProtocolTest, deserializeRpcResult) { '\x90', // return type 0x04, 't', 'e', 's', 't', // return body })); - auto result = deserializer.deserializeRpcResult(buffer, 4); - EXPECT_TRUE(result->hasException()); + context->set_body_size(4); + auto result = serializer.deserializeRpcResult(buffer, context); + EXPECT_TRUE(result.second); + EXPECT_TRUE(result.first->hasException()); } { @@ -96,8 +108,10 @@ TEST(HessianProtocolTest, deserializeRpcResult) { '\x91', // return type 0x04, 't', 'e', 's', 't', // return body })); - auto result = deserializer.deserializeRpcResult(buffer, 4); - EXPECT_TRUE(result->hasException()); + context->set_body_size(4); + auto result = serializer.deserializeRpcResult(buffer, context); + EXPECT_TRUE(result.second); + EXPECT_TRUE(result.first->hasException()); } // incorrect body size @@ -107,7 +121,8 @@ TEST(HessianProtocolTest, deserializeRpcResult) { '\x94', // return type 0x05, 't', 'e', 's', 't', // return body })); - EXPECT_THROW_WITH_MESSAGE(deserializer.deserializeRpcResult(buffer, 0), EnvoyException, + context->set_body_size(0); + EXPECT_THROW_WITH_MESSAGE(serializer.deserializeRpcResult(buffer, context), EnvoyException, "RpcResult size(1) large than body size(0)"); } @@ -118,8 +133,9 @@ TEST(HessianProtocolTest, deserializeRpcResult) { '\x96', // incorrect return type 0x05, 't', 'e', 's', 't', // return body })); - EXPECT_THROW_WITH_MESSAGE(deserializer.deserializeRpcResult(buffer, buffer.length()), - EnvoyException, "not supported return type 6"); + context->set_body_size(buffer.length()); + EXPECT_THROW_WITH_MESSAGE(serializer.deserializeRpcResult(buffer, context), EnvoyException, + "not supported return type 6"); } // incorrect value size @@ -132,25 +148,27 @@ TEST(HessianProtocolTest, deserializeRpcResult) { std::string exception_string = fmt::format("RpcResult is no value, but the rest of the body size({}) not equal 0", buffer.length() - 1); - EXPECT_THROW_WITH_MESSAGE(deserializer.deserializeRpcResult(buffer, buffer.length()), - EnvoyException, exception_string); + context->set_body_size(buffer.length()); + EXPECT_THROW_WITH_MESSAGE(serializer.deserializeRpcResult(buffer, context), EnvoyException, + exception_string); } } TEST(HessianProtocolTest, HessianDeserializerConfigFactory) { - auto deserializer = - NamedDeserializerConfigFactory::getFactory(SerializationType::Hessian).createDeserializer(); - EXPECT_EQ(deserializer->name(), "hessian"); - EXPECT_EQ(deserializer->type(), SerializationType::Hessian); + auto serializer = + NamedSerializerConfigFactory::getFactory(ProtocolType::Dubbo, SerializationType::Hessian2) + .createSerializer(); + EXPECT_EQ(serializer->name(), "dubbo.hessian2"); + EXPECT_EQ(serializer->type(), SerializationType::Hessian2); } TEST(HessianProtocolTest, serializeRpcResult) { Buffer::OwnedImpl buffer; std::string mock_response("invalid method name 'Add'"); RpcResponseType mock_response_type = RpcResponseType::ResponseWithException; - HessianDeserializerImpl deserializer; + DubboHessian2SerializerImpl serializer; - deserializer.serializeRpcResult(buffer, mock_response, mock_response_type); + EXPECT_NE(serializer.serializeRpcResult(buffer, mock_response, mock_response_type), 0); size_t hessian_int_size; int type_value = HessianUtils::peekInt(buffer, &hessian_int_size); @@ -163,8 +181,10 @@ TEST(HessianProtocolTest, serializeRpcResult) { EXPECT_EQ(buffer.length(), hessian_int_size + hessian_string_size); size_t body_size = mock_response.size() + sizeof(mock_response_type); - auto result = deserializer.deserializeRpcResult(buffer, body_size); - EXPECT_TRUE(result->hasException()); + std::shared_ptr context = std::make_shared(); + context->set_body_size(body_size); + auto result = serializer.deserializeRpcResult(buffer, context); + EXPECT_TRUE(result.first->hasException()); } } // namespace DubboProxy diff --git a/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc b/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc index f98cb0dee33c..3ee19707380a 100644 --- a/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc @@ -19,11 +19,13 @@ using testing::StrictMock; TEST(DubboProtocolImplTest, NotEnoughData) { Buffer::OwnedImpl buffer; DubboProtocolImpl dubbo_protocol; - Protocol::Context context; MessageMetadataSharedPtr metadata = std::make_shared(); - EXPECT_FALSE(dubbo_protocol.decode(buffer, &context, metadata)); + auto result = dubbo_protocol.decodeHeader(buffer, metadata); + EXPECT_FALSE(result.second); + buffer.add(std::string(15, 0x00)); - EXPECT_FALSE(dubbo_protocol.decode(buffer, &context, metadata)); + result = dubbo_protocol.decodeHeader(buffer, metadata); + EXPECT_FALSE(result.second); } TEST(DubboProtocolImplTest, Name) { @@ -36,37 +38,37 @@ TEST(DubboProtocolImplTest, Normal) { // Normal dubbo request message { Buffer::OwnedImpl buffer; - Protocol::Context context; MessageMetadataSharedPtr metadata = std::make_shared(); buffer.add(std::string({'\xda', '\xbb', '\xc2', 0x00})); addInt64(buffer, 1); addInt32(buffer, 1); - EXPECT_TRUE(dubbo_protocol.decode(buffer, &context, metadata)); + + auto result = dubbo_protocol.decodeHeader(buffer, metadata); + auto context = result.first; + EXPECT_TRUE(result.second); EXPECT_EQ(1, metadata->request_id()); - EXPECT_EQ(1, context.body_size_); - EXPECT_EQ(false, context.is_heartbeat_); + EXPECT_EQ(1, context->body_size()); EXPECT_EQ(MessageType::Request, metadata->message_type()); } // Normal dubbo response message { Buffer::OwnedImpl buffer; - Protocol::Context context; MessageMetadataSharedPtr metadata = std::make_shared(); buffer.add(std::string({'\xda', '\xbb', 0x42, 20})); addInt64(buffer, 1); addInt32(buffer, 1); - EXPECT_TRUE(dubbo_protocol.decode(buffer, &context, metadata)); + auto result = dubbo_protocol.decodeHeader(buffer, metadata); + auto context = result.first; + EXPECT_TRUE(result.second); EXPECT_EQ(1, metadata->request_id()); - EXPECT_EQ(1, context.body_size_); - EXPECT_EQ(false, context.is_heartbeat_); + EXPECT_EQ(1, context->body_size()); EXPECT_EQ(MessageType::Response, metadata->message_type()); } } TEST(DubboProtocolImplTest, InvalidProtocol) { DubboProtocolImpl dubbo_protocol; - Protocol::Context context; MessageMetadataSharedPtr metadata = std::make_shared(); // Invalid dubbo magic number @@ -74,7 +76,7 @@ TEST(DubboProtocolImplTest, InvalidProtocol) { Buffer::OwnedImpl buffer; addInt64(buffer, 0); addInt64(buffer, 0); - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, "invalid dubbo message magic number 0"); } @@ -86,7 +88,7 @@ TEST(DubboProtocolImplTest, InvalidProtocol) { addInt32(buffer, DubboProtocolImpl::MaxBodySize + 1); std::string exception_string = fmt::format("invalid dubbo message size {}", DubboProtocolImpl::MaxBodySize + 1); - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, exception_string); } @@ -96,7 +98,7 @@ TEST(DubboProtocolImplTest, InvalidProtocol) { buffer.add(std::string({'\xda', '\xbb', '\xc3', 0x00})); addInt64(buffer, 1); addInt32(buffer, 0xff); - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, "invalid dubbo message serialization type 3"); } @@ -106,60 +108,70 @@ TEST(DubboProtocolImplTest, InvalidProtocol) { buffer.add(std::string({'\xda', '\xbb', 0x42, 0x00})); addInt64(buffer, 1); addInt32(buffer, 0xff); - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, "invalid dubbo message response status 0"); } } TEST(DubboProtocolImplTest, DubboProtocolConfigFactory) { - auto protocol = NamedProtocolConfigFactory::getFactory(ProtocolType::Dubbo).createProtocol(); + auto protocol = NamedProtocolConfigFactory::getFactory(ProtocolType::Dubbo) + .createProtocol(SerializationType::Hessian2); EXPECT_EQ(protocol->name(), "dubbo"); EXPECT_EQ(protocol->type(), ProtocolType::Dubbo); + EXPECT_EQ(protocol->serializer()->type(), SerializationType::Hessian2); } TEST(DubboProtocolImplTest, encode) { MessageMetadata metadata; metadata.setMessageType(MessageType::Response); metadata.setResponseStatus(ResponseStatus::ServiceNotFound); - metadata.setSerializationType(SerializationType::Hessian); + metadata.setSerializationType(SerializationType::Hessian2); metadata.setRequestId(100); Buffer::OwnedImpl buffer; DubboProtocolImpl dubbo_protocol; - int32_t expect_body_size = 100; - EXPECT_TRUE(dubbo_protocol.encode(buffer, expect_body_size, metadata)); + dubbo_protocol.initSerializer(SerializationType::Hessian2); + std::string content("this is test data"); + EXPECT_TRUE(dubbo_protocol.encode(buffer, metadata, content, RpcResponseType::ResponseWithValue)); - Protocol::Context context; MessageMetadataSharedPtr output_metadata = std::make_shared(); - EXPECT_TRUE(dubbo_protocol.decode(buffer, &context, output_metadata)); + auto result = dubbo_protocol.decodeHeader(buffer, output_metadata); + EXPECT_TRUE(result.second); EXPECT_EQ(metadata.message_type(), output_metadata->message_type()); - EXPECT_EQ(metadata.response_status().value(), output_metadata->response_status().value()); + EXPECT_EQ(metadata.response_status(), output_metadata->response_status()); EXPECT_EQ(metadata.serialization_type(), output_metadata->serialization_type()); EXPECT_EQ(metadata.request_id(), output_metadata->request_id()); - EXPECT_EQ(context.body_size_, expect_body_size); + + Buffer::OwnedImpl body_buffer; + size_t serialized_body_size = dubbo_protocol.serializer()->serializeRpcResult( + body_buffer, content, RpcResponseType::ResponseWithValue); + auto context = result.first; + EXPECT_EQ(context->body_size(), serialized_body_size); + + buffer.drain(context->header_size()); + EXPECT_TRUE(dubbo_protocol.decodeData(buffer, context, output_metadata)); } TEST(DubboProtocolImplTest, decode) { Buffer::OwnedImpl buffer; MessageMetadataSharedPtr metadata; - Protocol::Context context; DubboProtocolImpl dubbo_protocol; // metadata is nullptr - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, "invalid metadata parameter"); metadata = std::make_shared(); // Invalid message header size - EXPECT_FALSE(dubbo_protocol.decode(buffer, &context, metadata)); + EXPECT_FALSE(dubbo_protocol.decodeHeader(buffer, metadata).second); // Invalid dubbo magic number { addInt64(buffer, 0); addInt64(buffer, 0); - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, "invalid dubbo message magic number 0"); buffer.drain(buffer.length()); } @@ -171,7 +183,7 @@ TEST(DubboProtocolImplTest, decode) { addInt32(buffer, DubboProtocolImpl::MaxBodySize + 1); std::string exception_string = fmt::format("invalid dubbo message size {}", DubboProtocolImpl::MaxBodySize + 1); - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, exception_string); buffer.drain(buffer.length()); } @@ -181,7 +193,7 @@ TEST(DubboProtocolImplTest, decode) { buffer.add(std::string({'\xda', '\xbb', '\xc3', 0x00})); addInt64(buffer, 1); addInt32(buffer, 0xff); - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, "invalid dubbo message serialization type 3"); buffer.drain(buffer.length()); } @@ -191,38 +203,38 @@ TEST(DubboProtocolImplTest, decode) { buffer.add(std::string({'\xda', '\xbb', 0x42, 0x00})); addInt64(buffer, 1); addInt32(buffer, 0xff); - EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decode(buffer, &context, metadata), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(dubbo_protocol.decodeHeader(buffer, metadata), EnvoyException, "invalid dubbo message response status 0"); buffer.drain(buffer.length()); } // The dubbo request message { - Protocol::Context context; buffer.add(std::string({'\xda', '\xbb', '\xc2', 0x00})); addInt64(buffer, 1); addInt32(buffer, 1); - EXPECT_TRUE(dubbo_protocol.decode(buffer, &context, metadata)); - EXPECT_EQ(1, context.body_size_); - EXPECT_FALSE(context.is_heartbeat_); + auto result = dubbo_protocol.decodeHeader(buffer, metadata); + EXPECT_TRUE(result.second); + auto context = result.first; + EXPECT_EQ(1, context->body_size()); EXPECT_EQ(MessageType::Request, metadata->message_type()); EXPECT_EQ(1, metadata->request_id()); - EXPECT_EQ(SerializationType::Hessian, metadata->serialization_type()); + EXPECT_EQ(SerializationType::Hessian2, metadata->serialization_type()); buffer.drain(buffer.length()); } // The One-way dubbo request message { - Protocol::Context context; buffer.add(std::string({'\xda', '\xbb', '\x82', 0x00})); addInt64(buffer, 1); addInt32(buffer, 1); - EXPECT_TRUE(dubbo_protocol.decode(buffer, &context, metadata)); - EXPECT_EQ(1, context.body_size_); - EXPECT_FALSE(context.is_heartbeat_); + auto result = dubbo_protocol.decodeHeader(buffer, metadata); + EXPECT_TRUE(result.second); + auto context = result.first; + EXPECT_EQ(1, context->body_size()); EXPECT_EQ(MessageType::Oneway, metadata->message_type()); EXPECT_EQ(1, metadata->request_id()); - EXPECT_EQ(SerializationType::Hessian, metadata->serialization_type()); + EXPECT_EQ(SerializationType::Hessian2, metadata->serialization_type()); } } diff --git a/test/extensions/filters/network/dubbo_proxy/metadata_test.cc b/test/extensions/filters/network/dubbo_proxy/metadata_test.cc index f89179cd637c..ab94547762f7 100644 --- a/test/extensions/filters/network/dubbo_proxy/metadata_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/metadata_test.cc @@ -1,4 +1,6 @@ +#include "extensions/filters/network/dubbo_proxy/message_impl.h" #include "extensions/filters/network/dubbo_proxy/metadata.h" +#include "extensions/filters/network/dubbo_proxy/serializer_impl.h" #include "gtest/gtest.h" @@ -9,43 +11,51 @@ namespace DubboProxy { TEST(MessageMetadataTest, Fields) { MessageMetadata metadata; + auto invo = std::make_shared(); - EXPECT_FALSE(metadata.method_name().has_value()); - EXPECT_THROW(metadata.method_name().value(), absl::bad_optional_access); - metadata.setMethodName("method"); - EXPECT_TRUE(metadata.method_name().has_value()); - EXPECT_EQ("method", metadata.method_name()); - - EXPECT_FALSE(metadata.service_version().has_value()); - EXPECT_THROW(metadata.service_version().value(), absl::bad_optional_access); - metadata.setServiceVersion("1.0.0"); - EXPECT_TRUE(metadata.service_version().has_value()); - EXPECT_EQ("1.0.0", metadata.service_version().value()); - - EXPECT_FALSE(metadata.service_group().has_value()); - EXPECT_THROW(metadata.service_group().value(), absl::bad_optional_access); - metadata.setServiceGroup("group"); - EXPECT_TRUE(metadata.service_group().has_value()); - EXPECT_EQ("group", metadata.service_group().value()); + EXPECT_FALSE(metadata.hasInvocationInfo()); + metadata.setInvocationInfo(invo); + EXPECT_TRUE(metadata.hasInvocationInfo()); + + EXPECT_THROW(metadata.timeout().value(), absl::bad_optional_access); + metadata.setTimeout(3); + EXPECT_TRUE(metadata.timeout().has_value()); + + invo->setMethodName("method"); + EXPECT_EQ("method", invo->method_name()); + + EXPECT_FALSE(invo->service_version().has_value()); + EXPECT_THROW(invo->service_version().value(), absl::bad_optional_access); + invo->setServiceVersion("1.0.0"); + EXPECT_TRUE(invo->service_version().has_value()); + EXPECT_EQ("1.0.0", invo->service_version().value()); + + EXPECT_FALSE(invo->service_group().has_value()); + EXPECT_THROW(invo->service_group().value(), absl::bad_optional_access); + invo->setServiceGroup("group"); + EXPECT_TRUE(invo->service_group().has_value()); + EXPECT_EQ("group", invo->service_group().value()); } TEST(MessageMetadataTest, Headers) { MessageMetadata metadata; + auto invo = std::make_shared(); - EXPECT_FALSE(metadata.hasHeaders()); - metadata.addHeader("k", "v"); - EXPECT_EQ(metadata.headers().size(), 1); + EXPECT_FALSE(invo->hasHeaders()); + invo->addHeader("k", "v"); + EXPECT_EQ(invo->headers().size(), 1); } TEST(MessageMetadataTest, Parameters) { MessageMetadata metadata; + auto invo = std::make_shared(); - EXPECT_FALSE(metadata.hasParameters()); - metadata.addParameterValue(0, "test"); - EXPECT_TRUE(metadata.hasParameters()); - EXPECT_EQ(metadata.parameters().size(), 1); - EXPECT_EQ(metadata.getParameterValue(0), "test"); - EXPECT_EQ(metadata.getParameterValue(1), ""); + EXPECT_FALSE(invo->hasParameters()); + invo->addParameterValue(0, "test"); + EXPECT_TRUE(invo->hasParameters()); + EXPECT_EQ(invo->parameters().size(), 1); + EXPECT_EQ(invo->getParameterValue(0), "test"); + EXPECT_EQ(invo->getParameterValue(1), ""); } } // namespace DubboProxy diff --git a/test/extensions/filters/network/dubbo_proxy/mocks.cc b/test/extensions/filters/network/dubbo_proxy/mocks.cc index 14a55f348d81..276d4cb3408f 100644 --- a/test/extensions/filters/network/dubbo_proxy/mocks.cc +++ b/test/extensions/filters/network/dubbo_proxy/mocks.cc @@ -17,52 +17,41 @@ namespace Extensions { namespace NetworkFilters { namespace DubboProxy { -MockDecoderEventHandler::MockDecoderEventHandler() { - ON_CALL(*this, transportBegin()).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, transportEnd()).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, messageBegin(_, _, _)).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, messageEnd(_)).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, transferHeaderTo(_, _)).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, transferBodyTo(_, _)).WillByDefault(Return(Network::FilterStatus::Continue)); +MockStreamDecoder::MockStreamDecoder() { + ON_CALL(*this, onMessageDecoded(_, _)).WillByDefault(Return(FilterStatus::Continue)); } -MockDecoderCallbacks::MockDecoderCallbacks() { - ON_CALL(*this, newDecoderEventHandler()).WillByDefault(Return(&handler_)); +MockStreamEncoder::MockStreamEncoder() { + ON_CALL(*this, onMessageEncoded(_, _)).WillByDefault(Return(FilterStatus::Continue)); +} + +MockRequestDecoderCallbacks::MockRequestDecoderCallbacks() { + ON_CALL(*this, newStream()).WillByDefault(ReturnRef(handler_)); +} + +MockResponseDecoderCallbacks::MockResponseDecoderCallbacks() { + ON_CALL(*this, newStream()).WillByDefault(ReturnRef(handler_)); } MockProtocol::MockProtocol() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, type()).WillByDefault(Return(type_)); + ON_CALL(*this, serializer()).WillByDefault(Return(&serializer_)); } MockProtocol::~MockProtocol() = default; -MockDeserializer::MockDeserializer() { +MockSerializer::MockSerializer() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, type()).WillByDefault(Return(type_)); } -MockDeserializer::~MockDeserializer() = default; +MockSerializer::~MockSerializer() = default; namespace DubboFilters { MockFilterChainFactoryCallbacks::MockFilterChainFactoryCallbacks() = default; MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() = default; -MockDecoderFilter::MockDecoderFilter() { - ON_CALL(*this, transportBegin()).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, transportEnd()).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, messageBegin(_, _, _)).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, messageEnd(_)).WillByDefault(Return(Network::FilterStatus::Continue)); - ON_CALL(*this, transferHeaderTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance& buf, size_t size) -> Network::FilterStatus { - buf.drain(size); - return Network::FilterStatus::Continue; - })); - ON_CALL(*this, transferBodyTo(_, _)) - .WillByDefault(Invoke([&](Buffer::Instance& buf, size_t size) -> Network::FilterStatus { - buf.drain(size); - return Network::FilterStatus::Continue; - })); -} +MockDecoderFilter::MockDecoderFilter() = default; MockDecoderFilter::~MockDecoderFilter() = default; MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { @@ -72,11 +61,26 @@ MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { ON_CALL(*this, connection()).WillByDefault(Return(&connection_)); ON_CALL(*this, route()).WillByDefault(Return(route_)); ON_CALL(*this, streamInfo()).WillByDefault(ReturnRef(stream_info_)); + ON_CALL(*this, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); } MockDecoderFilterCallbacks::~MockDecoderFilterCallbacks() = default; -MockDirectResponse::MockDirectResponse() = default; -MockDirectResponse::~MockDirectResponse() = default; +MockEncoderFilter::MockEncoderFilter() {} +MockEncoderFilter::~MockEncoderFilter() {} + +MockEncoderFilterCallbacks::MockEncoderFilterCallbacks() { + route_.reset(new NiceMock()); + + ON_CALL(*this, streamId()).WillByDefault(Return(stream_id_)); + ON_CALL(*this, connection()).WillByDefault(Return(&connection_)); + ON_CALL(*this, route()).WillByDefault(Return(route_)); + ON_CALL(*this, streamInfo()).WillByDefault(ReturnRef(stream_info_)); + ON_CALL(*this, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); +} +MockEncoderFilterCallbacks::~MockEncoderFilterCallbacks() {} + +MockCodecFilter::MockCodecFilter() {} +MockCodecFilter::~MockCodecFilter() {} MockFilterConfigFactory::MockFilterConfigFactory() : MockFactoryBase("envoy.filters.dubbo.mock_filter"), diff --git a/test/extensions/filters/network/dubbo_proxy/mocks.h b/test/extensions/filters/network/dubbo_proxy/mocks.h index 4de8c76982ca..884ab0f821ee 100644 --- a/test/extensions/filters/network/dubbo_proxy/mocks.h +++ b/test/extensions/filters/network/dubbo_proxy/mocks.h @@ -3,6 +3,7 @@ #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" +#include "extensions/filters/network/dubbo_proxy/decoder.h" #include "extensions/filters/network/dubbo_proxy/decoder_event_handler.h" #include "extensions/filters/network/dubbo_proxy/filters/factory_base.h" #include "extensions/filters/network/dubbo_proxy/filters/filter.h" @@ -22,40 +23,84 @@ namespace Extensions { namespace NetworkFilters { namespace DubboProxy { -class MockDecoderEventHandler : public DecoderEventHandler { +class MockStreamDecoder : public StreamDecoder { public: - MockDecoderEventHandler(); + MockStreamDecoder(); - MOCK_METHOD0(transportBegin, Network::FilterStatus()); - MOCK_METHOD0(transportEnd, Network::FilterStatus()); - MOCK_METHOD3(messageBegin, Network::FilterStatus(MessageType, int64_t, SerializationType)); - MOCK_METHOD1(messageEnd, Network::FilterStatus(MessageMetadataSharedPtr)); - MOCK_METHOD2(transferHeaderTo, Network::FilterStatus(Buffer::Instance&, size_t)); - MOCK_METHOD2(transferBodyTo, Network::FilterStatus(Buffer::Instance&, size_t)); + MOCK_METHOD2(onMessageDecoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); }; -class MockDecoderCallbacks : public DecoderCallbacks { +class MockStreamEncoder : public StreamEncoder { public: - MockDecoderCallbacks(); - ~MockDecoderCallbacks() = default; + MockStreamEncoder(); - MOCK_METHOD0(newDecoderEventHandler, DecoderEventHandler*()); + MOCK_METHOD2(onMessageEncoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); +}; + +class MockStreamHandler : public StreamHandler { +public: + MockStreamHandler() = default; + + MOCK_METHOD2(onStreamDecoded, void(MessageMetadataSharedPtr, ContextSharedPtr)); +}; + +class MockRequestDecoderCallbacks : public RequestDecoderCallbacks { +public: + MockRequestDecoderCallbacks(); + ~MockRequestDecoderCallbacks() = default; + + MOCK_METHOD0(newStream, StreamHandler&()); MOCK_METHOD1(onHeartbeat, void(MessageMetadataSharedPtr)); - MockDecoderEventHandler handler_; + MockStreamHandler handler_; }; +class MockResponseDecoderCallbacks : public ResponseDecoderCallbacks { +public: + MockResponseDecoderCallbacks(); + ~MockResponseDecoderCallbacks() = default; -class MockProtocolCallbacks : public ProtocolCallbacks { + MOCK_METHOD0(newStream, StreamHandler&()); + MOCK_METHOD1(onHeartbeat, void(MessageMetadataSharedPtr)); + + MockStreamHandler handler_; +}; + +class MockActiveStream : public ActiveStream { public: - MockProtocolCallbacks() = default; - ~MockProtocolCallbacks() = default; + MockActiveStream(StreamHandler& handler, MessageMetadataSharedPtr metadata, + ContextSharedPtr context) + : ActiveStream(handler, metadata, context) {} + ~MockActiveStream() = default; - void onRequestMessage(RequestMessagePtr&& req) override { onRequestMessageRvr(req.get()); } - void onResponseMessage(ResponseMessagePtr&& res) override { onResponseMessageRvr(res.get()); } + MOCK_METHOD2(newStream, ActiveStream*(MessageMetadataSharedPtr, ContextSharedPtr)); + MOCK_METHOD1(onHeartbeat, void(MessageMetadataSharedPtr)); +}; + +class MockDecoderStateMachineDelegate : public DecoderStateMachine::Delegate { +public: + MockDecoderStateMachineDelegate() = default; + ~MockDecoderStateMachineDelegate() = default; - // DubboProxy::ProtocolCallbacks - MOCK_METHOD1(onRequestMessageRvr, void(RequestMessage*)); - MOCK_METHOD1(onResponseMessageRvr, void(ResponseMessage*)); + MOCK_METHOD2(newStream, ActiveStream*(MessageMetadataSharedPtr, ContextSharedPtr)); + MOCK_METHOD1(onHeartbeat, void(MessageMetadataSharedPtr)); +}; + +class MockSerializer : public Serializer { +public: + MockSerializer(); + ~MockSerializer(); + + // DubboProxy::Serializer + MOCK_CONST_METHOD0(name, const std::string&()); + MOCK_CONST_METHOD0(type, SerializationType()); + MOCK_METHOD2(deserializeRpcInvocation, + std::pair(Buffer::Instance&, ContextSharedPtr)); + MOCK_METHOD2(deserializeRpcResult, + std::pair(Buffer::Instance&, ContextSharedPtr)); + MOCK_METHOD3(serializeRpcResult, size_t(Buffer::Instance&, const std::string&, RpcResponseType)); + + std::string name_{"mockDeserializer"}; + SerializationType type_{SerializationType::Hessian2}; }; class MockProtocol : public Protocol { @@ -65,28 +110,42 @@ class MockProtocol : public Protocol { MOCK_CONST_METHOD0(name, const std::string&()); MOCK_CONST_METHOD0(type, ProtocolType()); - MOCK_METHOD2(decode, bool(Buffer::Instance&, Context*)); - MOCK_METHOD3(decode, bool(Buffer::Instance&, Protocol::Context*, MessageMetadataSharedPtr)); - MOCK_METHOD3(encode, bool(Buffer::Instance&, int32_t, const MessageMetadata&)); + MOCK_CONST_METHOD0(serializer, Serializer*()); + MOCK_METHOD2(decodeHeader, + std::pair(Buffer::Instance&, MessageMetadataSharedPtr)); + MOCK_METHOD3(decodeData, bool(Buffer::Instance&, ContextSharedPtr, MessageMetadataSharedPtr)); + MOCK_METHOD4(encode, bool(Buffer::Instance&, const MessageMetadata&, const std::string&, + RpcResponseType)); std::string name_{"MockProtocol"}; ProtocolType type_{ProtocolType::Dubbo}; + NiceMock serializer_; }; -class MockDeserializer : public Deserializer { +class MockNamedSerializerConfigFactory : public NamedSerializerConfigFactory { public: - MockDeserializer(); - ~MockDeserializer(); + MockNamedSerializerConfigFactory(std::function f) : f_(f) {} - // DubboProxy::Deserializer - MOCK_CONST_METHOD0(name, const std::string&()); - MOCK_CONST_METHOD0(type, SerializationType()); - MOCK_METHOD3(deserializeRpcInvocation, void(Buffer::Instance&, size_t, MessageMetadataSharedPtr)); - MOCK_METHOD2(deserializeRpcResult, RpcResultPtr(Buffer::Instance&, size_t)); - MOCK_METHOD3(serializeRpcResult, size_t(Buffer::Instance&, const std::string&, RpcResponseType)); + SerializerPtr createSerializer() override { return SerializerPtr{f_()}; } + std::string name() override { + return SerializerNames::get().fromType(SerializationType::Hessian2); + } - std::string name_{"mockDeserializer"}; - SerializationType type_{SerializationType::Hessian}; + std::function f_; +}; + +class MockNamedProtocolConfigFactory : public NamedProtocolConfigFactory { +public: + MockNamedProtocolConfigFactory(std::function f) : f_(f) {} + + ProtocolPtr createProtocol(SerializationType serialization_type) override { + auto protocol = ProtocolPtr{f_()}; + protocol->initSerializer(serialization_type); + return protocol; + } + std::string name() override { return ProtocolNames::get().fromType(ProtocolType::Dubbo); } + + std::function f_; }; namespace Router { @@ -101,6 +160,8 @@ class MockFilterChainFactoryCallbacks : public FilterChainFactoryCallbacks { ~MockFilterChainFactoryCallbacks(); MOCK_METHOD1(addDecoderFilter, void(DecoderFilterSharedPtr)); + MOCK_METHOD1(addEncoderFilter, void(EncoderFilterSharedPtr)); + MOCK_METHOD1(addFilter, void(CodecFilterSharedPtr)); }; class MockDecoderFilter : public DecoderFilter { @@ -111,14 +172,7 @@ class MockDecoderFilter : public DecoderFilter { // DubboProxy::DubboFilters::DecoderFilter MOCK_METHOD0(onDestroy, void()); MOCK_METHOD1(setDecoderFilterCallbacks, void(DecoderFilterCallbacks& callbacks)); - - // DubboProxy::DecoderEventHandler - MOCK_METHOD0(transportBegin, Network::FilterStatus()); - MOCK_METHOD0(transportEnd, Network::FilterStatus()); - MOCK_METHOD3(messageBegin, Network::FilterStatus(MessageType, int64_t, SerializationType)); - MOCK_METHOD1(messageEnd, Network::FilterStatus(MessageMetadataSharedPtr)); - MOCK_METHOD2(transferHeaderTo, Network::FilterStatus(Buffer::Instance& buf, size_t size)); - MOCK_METHOD2(transferBodyTo, Network::FilterStatus(Buffer::Instance& buf, size_t size)); + MOCK_METHOD2(onMessageDecoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); }; class MockDecoderFilterCallbacks : public DecoderFilterCallbacks { @@ -132,28 +186,78 @@ class MockDecoderFilterCallbacks : public DecoderFilterCallbacks { MOCK_CONST_METHOD0(connection, const Network::Connection*()); MOCK_METHOD0(continueDecoding, void()); MOCK_METHOD0(route, Router::RouteConstSharedPtr()); - MOCK_CONST_METHOD0(downstreamSerializationType, SerializationType()); - MOCK_CONST_METHOD0(downstreamProtocolType, ProtocolType()); + MOCK_CONST_METHOD0(serializationType, SerializationType()); + MOCK_CONST_METHOD0(protocolType, ProtocolType()); MOCK_METHOD2(sendLocalReply, void(const DirectResponse&, bool)); - MOCK_METHOD2(startUpstreamResponse, void(Deserializer&, Protocol&)); + MOCK_METHOD0(startUpstreamResponse, void()); MOCK_METHOD1(upstreamData, UpstreamResponseStatus(Buffer::Instance&)); MOCK_METHOD0(resetDownstreamConnection, void()); MOCK_METHOD0(streamInfo, StreamInfo::StreamInfo&()); MOCK_METHOD0(resetStream, void()); + MOCK_METHOD0(dispatcher, Event::Dispatcher&()); uint64_t stream_id_{1}; NiceMock connection_; NiceMock stream_info_; std::shared_ptr route_; + NiceMock dispatcher_; +}; + +class MockEncoderFilter : public EncoderFilter { +public: + MockEncoderFilter(); + ~MockEncoderFilter(); + + // DubboProxy::DubboFilters::EncoderFilter + MOCK_METHOD0(onDestroy, void()); + MOCK_METHOD1(setEncoderFilterCallbacks, void(EncoderFilterCallbacks& callbacks)); + MOCK_METHOD2(onMessageEncoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); +}; + +class MockEncoderFilterCallbacks : public EncoderFilterCallbacks { +public: + MockEncoderFilterCallbacks(); + ~MockEncoderFilterCallbacks(); + + // DubboProxy::DubboFilters::MockEncoderFilterCallbacks + MOCK_CONST_METHOD0(requestId, uint64_t()); + MOCK_CONST_METHOD0(streamId, uint64_t()); + MOCK_CONST_METHOD0(connection, const Network::Connection*()); + MOCK_METHOD0(route, Router::RouteConstSharedPtr()); + MOCK_CONST_METHOD0(serializationType, SerializationType()); + MOCK_CONST_METHOD0(protocolType, ProtocolType()); + MOCK_METHOD0(streamInfo, StreamInfo::StreamInfo&()); + MOCK_METHOD0(resetStream, void()); + MOCK_METHOD0(dispatcher, Event::Dispatcher&()); + MOCK_METHOD0(continueEncoding, void()); + MOCK_METHOD0(continueDecoding, void()); + + uint64_t stream_id_{1}; + NiceMock connection_; + NiceMock stream_info_; + std::shared_ptr route_; + NiceMock dispatcher_; +}; + +class MockCodecFilter : public CodecFilter { +public: + MockCodecFilter(); + ~MockCodecFilter(); + + MOCK_METHOD0(onDestroy, void()); + MOCK_METHOD1(setEncoderFilterCallbacks, void(EncoderFilterCallbacks& callbacks)); + MOCK_METHOD2(onMessageEncoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); + MOCK_METHOD1(setDecoderFilterCallbacks, void(DecoderFilterCallbacks& callbacks)); + MOCK_METHOD2(onMessageDecoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); }; class MockDirectResponse : public DirectResponse { public: - MockDirectResponse(); - ~MockDirectResponse(); + MockDirectResponse() = default; + ~MockDirectResponse() = default; - MOCK_CONST_METHOD4(encode, DirectResponse::ResponseType(MessageMetadata&, Protocol&, - Deserializer&, Buffer::Instance&)); + MOCK_CONST_METHOD3(encode, + DirectResponse::ResponseType(MessageMetadata&, Protocol&, Buffer::Instance&)); }; template class MockFactoryBase : public NamedDubboFilterConfigFactory { diff --git a/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc b/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc index 69a3762604e4..623c6f657cd1 100644 --- a/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc @@ -6,9 +6,11 @@ #include "common/protobuf/protobuf.h" #include "extensions/filters/network/dubbo_proxy/router/route_matcher.h" +#include "extensions/filters/network/dubbo_proxy/serializer_impl.h" -#include "test/test_common/utility.h" +#include "test/mocks/server/mocks.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" namespace Envoy { @@ -53,28 +55,31 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setMethodName("test"); + metadata.setInvocationInfo(invo); + invo->setMethodName("test"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceName("unknown"); + invo->setServiceName("unknown"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceGroup("test"); + invo->setServiceGroup("test"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceVersion("1.0.0"); + invo->setServiceVersion("1.0.0"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); // Ignore version matches if there is no version field in the configuration information. - metadata.setServiceVersion("1.0.1"); + invo->setServiceVersion("1.0.1"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); - metadata.setServiceGroup("test_one"); + invo->setServiceGroup("test_one"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } @@ -97,16 +102,19 @@ group: test envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setMethodName("test"); - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + metadata.setInvocationInfo(invo); + invo->setMethodName("test"); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceGroup("test"); + invo->setServiceGroup("test"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceVersion("1.0.0"); + invo->setServiceVersion("1.0.0"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } @@ -128,21 +136,24 @@ version: 1.0.0 envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setMethodName("test"); - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + metadata.setInvocationInfo(invo); + invo->setMethodName("test"); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceGroup("test"); + invo->setServiceGroup("test"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceVersion("1.0.0"); + invo->setServiceVersion("1.0.0"); EXPECT_NE(nullptr, matcher.route(metadata, 0)); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); // Ignore group matches if there is no group field in the configuration information. - metadata.setServiceGroup("test_1"); + invo->setServiceGroup("test_1"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } @@ -164,19 +175,22 @@ group: HSF envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setMethodName("test"); - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + metadata.setInvocationInfo(invo); + invo->setMethodName("test"); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceGroup("test"); + invo->setServiceGroup("test"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceVersion("1.0.0"); + invo->setServiceVersion("1.0.0"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setServiceGroup("HSF"); + invo->setServiceGroup("HSF"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } } @@ -196,16 +210,19 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setMethodName("sub"); + invo->setMethodName("sub"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setMethodName("add"); + invo->setMethodName("add"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } @@ -224,16 +241,19 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setMethodName("sub"); + invo->setMethodName("sub"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setMethodName("add123test"); + invo->setMethodName("add123test"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } @@ -252,19 +272,22 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setMethodName("ab12test"); + invo->setMethodName("ab12test"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setMethodName("test12d2test"); + invo->setMethodName("test12d2test"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); - metadata.setMethodName("testme"); + invo->setMethodName("testme"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } @@ -283,19 +306,22 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setMethodName("12test"); + invo->setMethodName("12test"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); - metadata.setMethodName("456test"); + invo->setMethodName("456test"); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); - metadata.setMethodName("4567test"); + invo->setMethodName("4567test"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); } @@ -319,12 +345,15 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); - metadata.setMethodName("add"); - metadata.addParameterValue(0, "150"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); + invo->setMethodName("add"); + invo->addParameterValue(0, "150"); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } @@ -346,12 +375,15 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); - metadata.setMethodName("add"); - metadata.addParameterValue(1, "user_id:94562"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); + invo->setMethodName("add"); + invo->addParameterValue(1, "user_id:94562"); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } @@ -376,16 +408,19 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); - metadata.setMethodName("add"); - metadata.addHeader("custom", "123"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); + invo->setMethodName("add"); + invo->addHeader("custom", "123"); std::string test_value("123"); Envoy::Http::LowerCaseString test_key("custom1"); - metadata.addHeaderReference(test_key, test_value); + invo->addHeaderReference(test_key, test_value); - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); test_value = "456"; @@ -427,17 +462,20 @@ serialization_type: Hessian2 envoy::config::filter::network::dubbo_proxy::v2alpha1::DubboProxy config = parseDubboProxyFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); - metadata.setMethodName("add"); - metadata.addParameterValue(1, "user_id"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); + invo->setMethodName("add"); + invo->addParameterValue(1, "user_id"); - MultiRouteMatcher matcher(config.route_config()); + NiceMock context; + MultiRouteMatcher matcher(config.route_config(), context); EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); { envoy::config::filter::network::dubbo_proxy::v2alpha1::DubboProxy invalid_config; - MultiRouteMatcher matcher(invalid_config.route_config()); + MultiRouteMatcher matcher(invalid_config.route_config(), context); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); } } @@ -460,23 +498,28 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); - metadata.setMethodName("add"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); + invo->setMethodName("add"); // There is no parameter information in metadata. - RouteMatcher matcher(config); + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); // The parameter is empty. - metadata.addParameterValue(1, ""); + invo->addParameterValue(1, ""); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); { + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); - metadata.setMethodName("add"); - metadata.addParameterValue(1, "user_id:562"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); + invo->setMethodName("add"); + invo->addParameterValue(1, "user_id:562"); EXPECT_EQ(nullptr, matcher.route(metadata, 0)); } } @@ -516,12 +559,16 @@ interface: org.apache.dubbo.demo.DemoService envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); - RouteMatcher matcher(config); + auto invo = std::make_shared(); MessageMetadata metadata; - metadata.setServiceName("org.apache.dubbo.demo.DemoService"); + metadata.setInvocationInfo(invo); + invo->setServiceName("org.apache.dubbo.demo.DemoService"); + + NiceMock context; + SignleRouteMatcherImpl matcher(config, context); { - metadata.setMethodName("method1"); + invo->setMethodName("method1"); EXPECT_EQ("cluster1", matcher.route(metadata, 0)->routeEntry()->clusterName()); EXPECT_EQ("cluster1", matcher.route(metadata, 29)->routeEntry()->clusterName()); EXPECT_EQ("cluster2", matcher.route(metadata, 30)->routeEntry()->clusterName()); @@ -533,7 +580,7 @@ interface: org.apache.dubbo.demo.DemoService } { - metadata.setMethodName("method2"); + invo->setMethodName("method2"); EXPECT_EQ("cluster1", matcher.route(metadata, 0)->routeEntry()->clusterName()); EXPECT_EQ("cluster1", matcher.route(metadata, 1999)->routeEntry()->clusterName()); EXPECT_EQ("cluster2", matcher.route(metadata, 2000)->routeEntry()->clusterName()); @@ -565,7 +612,8 @@ name: config envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration config = parseRouteConfigurationFromV2Yaml(yaml); - EXPECT_THROW(RouteMatcher m(config), EnvoyException); + NiceMock context; + EXPECT_THROW(SignleRouteMatcherImpl m(config, context), EnvoyException); } } // namespace Router diff --git a/test/extensions/filters/network/dubbo_proxy/router_test.cc b/test/extensions/filters/network/dubbo_proxy/router_test.cc index 6376a8a96166..8caf2cfd4097 100644 --- a/test/extensions/filters/network/dubbo_proxy/router_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/router_test.cc @@ -1,7 +1,9 @@ #include "extensions/filters/network/dubbo_proxy/app_exception.h" -#include "extensions/filters/network/dubbo_proxy/deserializer.h" +#include "extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h" +#include "extensions/filters/network/dubbo_proxy/message_impl.h" #include "extensions/filters/network/dubbo_proxy/protocol.h" #include "extensions/filters/network/dubbo_proxy/router/router_impl.h" +#include "extensions/filters/network/dubbo_proxy/serializer_impl.h" #include "test/extensions/filters/network/dubbo_proxy/mocks.h" #include "test/mocks/network/mocks.h" @@ -30,23 +32,27 @@ namespace Router { namespace { -class TestNamedDeserializerConfigFactory : public NamedDeserializerConfigFactory { +class TestNamedSerializerConfigFactory : public NamedSerializerConfigFactory { public: - TestNamedDeserializerConfigFactory(std::function f) : f_(f) {} + TestNamedSerializerConfigFactory(std::function f) : f_(f) {} - DeserializerPtr createDeserializer() override { return DeserializerPtr{f_()}; } + SerializerPtr createSerializer() override { return SerializerPtr{f_()}; } std::string name() override { - return DeserializerNames::get().fromType(SerializationType::Hessian); + return SerializerNames::get().fromType(SerializationType::Hessian2); } - std::function f_; + std::function f_; }; class TestNamedProtocolConfigFactory : public NamedProtocolConfigFactory { public: TestNamedProtocolConfigFactory(std::function f) : f_(f) {} - ProtocolPtr createProtocol() override { return ProtocolPtr{f_()}; } + ProtocolPtr createProtocol(SerializationType serialization_type) override { + auto protocol = ProtocolPtr{f_()}; + protocol->initSerializer(serialization_type); + return protocol; + } std::string name() override { return ProtocolNames::get().fromType(ProtocolType::Dubbo); } std::function f_; @@ -57,13 +63,13 @@ class TestNamedProtocolConfigFactory : public NamedProtocolConfigFactory { class DubboRouterTestBase { public: DubboRouterTestBase() - : deserializer_factory_([&]() -> MockDeserializer* { - ASSERT(deserializer_ == nullptr); - deserializer_ = new NiceMock(); - if (mock_deserializer_cb_) { - mock_deserializer_cb_(deserializer_); + : serializer_factory_([&]() -> MockSerializer* { + ASSERT(serializer_ == nullptr); + serializer_ = new NiceMock(); + if (mock_serializer_cb_) { + mock_serializer_cb_(serializer_); } - return deserializer_; + return serializer_; }), protocol_factory_([&]() -> MockProtocol* { ASSERT(protocol_ == nullptr); @@ -73,7 +79,7 @@ class DubboRouterTestBase { } return protocol_; }), - deserializer_register_(deserializer_factory_), protocol_register_(protocol_factory_) {} + serializer_register_(serializer_factory_), protocol_register_(protocol_factory_) {} void initializeRouter() { route_ = new NiceMock(); @@ -90,9 +96,14 @@ class DubboRouterTestBase { msg_type_ = msg_type; metadata_.reset(new MessageMetadata()); - metadata_->setServiceName("test"); metadata_->setMessageType(msg_type_); metadata_->setRequestId(1); + + auto invo = std::make_shared(); + metadata_->setInvocationInfo(invo); + invo->setMethodName("test"); + + message_context_ = std::make_shared(); } void startRequest(MessageType msg_type) { @@ -102,13 +113,10 @@ class DubboRouterTestBase { EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(callbacks_, downstreamSerializationType()) - .WillOnce(Return(SerializationType::Hessian)); - EXPECT_CALL(callbacks_, downstreamProtocolType()).WillOnce(Return(ProtocolType::Dubbo)); + EXPECT_CALL(callbacks_, serializationType()).WillOnce(Return(SerializationType::Hessian2)); + EXPECT_CALL(callbacks_, protocolType()).WillOnce(Return(ProtocolType::Dubbo)); - EXPECT_EQ(Network::FilterStatus::Continue, - router_->messageBegin(msg_type, metadata_->request_id(), SerializationType::Hessian)); - EXPECT_EQ(Network::FilterStatus::StopIteration, router_->messageEnd(metadata_)); + EXPECT_EQ(FilterStatus::StopIteration, router_->onMessageDecoded(metadata_, message_context_)); EXPECT_CALL(callbacks_, connection()).WillRepeatedly(Return(&connection_)); EXPECT_EQ(&connection_, router_->downstreamConnection()); @@ -137,8 +145,6 @@ class DubboRouterTestBase { } void startRequestWithExistingConnection(MessageType msg_type) { - EXPECT_EQ(Network::FilterStatus::Continue, router_->transportBegin()); - EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); @@ -158,9 +164,8 @@ class DubboRouterTestBase { EXPECT_EQ(nullptr, router_->metadataMatchCriteria()); EXPECT_EQ(nullptr, router_->downstreamHeaders()); - EXPECT_CALL(callbacks_, downstreamSerializationType()) - .WillOnce(Return(SerializationType::Hessian)); - EXPECT_CALL(callbacks_, downstreamProtocolType()).WillOnce(Return(ProtocolType::Dubbo)); + EXPECT_CALL(callbacks_, serializationType()).WillOnce(Return(SerializationType::Hessian2)); + EXPECT_CALL(callbacks_, protocolType()).WillOnce(Return(ProtocolType::Dubbo)); EXPECT_CALL(callbacks_, continueDecoding()).Times(0); EXPECT_CALL(context_.cluster_manager_.tcp_conn_pool_, newConnection(_)) @@ -175,7 +180,7 @@ class DubboRouterTestBase { void returnResponse() { Buffer::OwnedImpl buffer; - EXPECT_CALL(callbacks_, startUpstreamResponse(_, _)); + EXPECT_CALL(callbacks_, startUpstreamResponse()); EXPECT_CALL(callbacks_, upstreamData(Ref(buffer))) .WillOnce(Return(DubboFilters::UpstreamResponseStatus::MoreData)); @@ -196,18 +201,18 @@ class DubboRouterTestBase { router_.reset(); } - TestNamedDeserializerConfigFactory deserializer_factory_; + TestNamedSerializerConfigFactory serializer_factory_; TestNamedProtocolConfigFactory protocol_factory_; - Registry::InjectFactory deserializer_register_; + Registry::InjectFactory serializer_register_; Registry::InjectFactory protocol_register_; - std::function mock_deserializer_cb_{}; + std::function mock_serializer_cb_{}; std::function mock_protocol_cb_{}; NiceMock context_; NiceMock connection_; NiceMock callbacks_; - NiceMock* deserializer_{}; + NiceMock* serializer_{}; NiceMock* protocol_{}; NiceMock* route_{}; NiceMock route_entry_; @@ -221,6 +226,7 @@ class DubboRouterTestBase { MessageType msg_type_{MessageType::Request}; MessageMetadataSharedPtr metadata_; + ContextSharedPtr message_context_; Tcp::ConnectionPool::UpstreamCallbacks* upstream_callbacks_{}; NiceMock upstream_connection_; @@ -293,7 +299,7 @@ TEST_F(DubboRouterTest, ClusterMaintenanceMode) { EXPECT_THAT(app_ex.what(), ContainsRegex(".*maintenance mode.*")); EXPECT_FALSE(end_stream); })); - EXPECT_EQ(Network::FilterStatus::StopIteration, router_->messageEnd(metadata_)); + EXPECT_EQ(FilterStatus::StopIteration, router_->onMessageDecoded(metadata_, message_context_)); } TEST_F(DubboRouterTest, NoHealthyHosts) { @@ -314,18 +320,18 @@ TEST_F(DubboRouterTest, NoHealthyHosts) { EXPECT_FALSE(end_stream); })); - EXPECT_EQ(Network::FilterStatus::StopIteration, router_->messageEnd(metadata_)); + EXPECT_EQ(FilterStatus::StopIteration, router_->onMessageDecoded(metadata_, message_context_)); } TEST_F(DubboRouterTest, PoolConnectionFailureWithOnewayMessage) { initializeRouter(); initializeMetadata(MessageType::Oneway); - EXPECT_CALL(callbacks_, downstreamSerializationType()) - .WillOnce(Return(SerializationType::Hessian)); + EXPECT_CALL(callbacks_, protocolType()).WillOnce(Return(ProtocolType::Dubbo)); + EXPECT_CALL(callbacks_, serializationType()).WillOnce(Return(SerializationType::Hessian2)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)).Times(0); EXPECT_CALL(callbacks_, resetStream()).Times(1); - EXPECT_EQ(Network::FilterStatus::StopIteration, router_->messageEnd(metadata_)); + EXPECT_EQ(FilterStatus::StopIteration, router_->onMessageDecoded(metadata_, message_context_)); context_.cluster_manager_.tcp_conn_pool_.poolFailure( Tcp::ConnectionPool::PoolFailureReason::RemoteConnectionFailure); @@ -345,7 +351,7 @@ TEST_F(DubboRouterTest, NoRoute) { EXPECT_THAT(app_ex.what(), ContainsRegex(".*no route.*")); EXPECT_FALSE(end_stream); })); - EXPECT_EQ(Network::FilterStatus::StopIteration, router_->messageEnd(metadata_)); + EXPECT_EQ(FilterStatus::StopIteration, router_->onMessageDecoded(metadata_, message_context_)); } TEST_F(DubboRouterTest, NoCluster) { @@ -363,24 +369,21 @@ TEST_F(DubboRouterTest, NoCluster) { EXPECT_THAT(app_ex.what(), ContainsRegex(".*unknown cluster.*")); EXPECT_FALSE(end_stream); })); - EXPECT_EQ(Network::FilterStatus::StopIteration, router_->messageEnd(metadata_)); + EXPECT_EQ(FilterStatus::StopIteration, router_->onMessageDecoded(metadata_, message_context_)); } TEST_F(DubboRouterTest, UnexpectedRouterDestroy) { initializeRouter(); initializeMetadata(MessageType::Request); EXPECT_CALL(upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); - startRequest(MessageType::Request); - - EXPECT_EQ(Network::FilterStatus::Continue, router_->transportBegin()); Buffer::OwnedImpl buffer; - buffer.add(std::string({'\xda', '\xbb', 0x42, 20})); - EXPECT_EQ(Network::FilterStatus::Continue, router_->transferHeaderTo(buffer, buffer.length())); - buffer.drain(buffer.length()); - buffer.add("test"); - EXPECT_EQ(Network::FilterStatus::Continue, router_->transferBodyTo(buffer, buffer.length())); + buffer.add(std::string({'\xda', '\xbb', 0x42, 20})); // Header + buffer.add("test"); // Body + auto ctx = static_cast(message_context_.get()); + ctx->message_origin_data().move(buffer, buffer.length()); + startRequest(MessageType::Request); connectUpstream(); destroyRouter(); } @@ -418,9 +421,6 @@ TEST_F(DubboRouterTest, OneWay) { startRequest(MessageType::Oneway); connectUpstream(); - - EXPECT_EQ(Network::FilterStatus::Continue, router_->transportEnd()); - destroyRouter(); } @@ -432,10 +432,6 @@ TEST_F(DubboRouterTest, Call) { startRequest(MessageType::Request); connectUpstream(); - - EXPECT_EQ(Network::FilterStatus::Continue, router_->transportBegin()); - EXPECT_EQ(Network::FilterStatus::Continue, router_->transportEnd()); - returnResponse(); destroyRouter(); } @@ -445,15 +441,12 @@ TEST_F(DubboRouterTest, DecoderFilterCallbacks) { initializeMetadata(MessageType::Request); EXPECT_CALL(upstream_connection_, write(_, false)); - EXPECT_CALL(callbacks_, startUpstreamResponse(_, _)).Times(1); + EXPECT_CALL(callbacks_, startUpstreamResponse()).Times(1); EXPECT_CALL(callbacks_, upstreamData(_)).Times(1); startRequest(MessageType::Request); connectUpstream(); - EXPECT_EQ(Network::FilterStatus::Continue, router_->transportBegin()); - EXPECT_EQ(Network::FilterStatus::Continue, router_->transportEnd()); - Buffer::OwnedImpl buffer; buffer.add(std::string("This is the test data")); router_->onUpstreamData(buffer, true); @@ -465,7 +458,7 @@ TEST_F(DubboRouterTest, UpstreamDataReset) { initializeRouter(); initializeMetadata(MessageType::Request); - EXPECT_CALL(callbacks_, startUpstreamResponse(_, _)).Times(1); + EXPECT_CALL(callbacks_, startUpstreamResponse()).Times(1); EXPECT_CALL(callbacks_, upstreamData(_)) .WillOnce(Return(DubboFilters::UpstreamResponseStatus::Reset)); EXPECT_CALL(upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); @@ -484,7 +477,7 @@ TEST_F(DubboRouterTest, StartRequestWithExistingConnection) { initializeRouter(); startRequestWithExistingConnection(MessageType::Request); - EXPECT_EQ(Network::FilterStatus::Continue, router_->messageEnd(metadata_)); + EXPECT_EQ(FilterStatus::Continue, router_->onMessageDecoded(metadata_, message_context_)); destroyRouter(); } @@ -511,7 +504,7 @@ TEST_F(DubboRouterTest, LocalClosedWhileResponseComplete) { initializeRouter(); initializeMetadata(MessageType::Request); - EXPECT_CALL(callbacks_, startUpstreamResponse(_, _)).Times(1); + EXPECT_CALL(callbacks_, startUpstreamResponse()).Times(1); EXPECT_CALL(callbacks_, upstreamData(_)) .WillOnce(Return(DubboFilters::UpstreamResponseStatus::Complete)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)).Times(0); From 2e8ddf3f1821816bfb6ae93352fcc5dcddbb058f Mon Sep 17 00:00:00 2001 From: Dylan Carney Date: Mon, 15 Jul 2019 14:28:01 -0700 Subject: [PATCH 173/542] http/1.1 codec: Rejects requests w/ invalid HTTP header values (#7306) Description: This PR addresses the behavior described in #7061 by modifying the HTTP/1.1 codec, such that it verifies the value of each HTTP header when onHeaderValue is called. The codec raises an exception, which results in a 400 response being returned if an invalid header value (per RFC 7230, section 3.2) is found. The actual validation is done via the nghttp2_check_header_value function, which is wrapped in Envoy::Http::HeaderUtility::headerIsValid. (NOTE: this has been discussed as something that nghttp2 could do itself, but the issue seems to have languished. Also note that Go handles this: Go uses the httpguts.ValidHeaderFieldValue function (which is analogous to nghttp2_check_header_value) to ensure that all the HTTP header values conform to the relevant RFC specs before an http.Transport instance will round-trip the request. Risk Level: Low/medium This stricter validation semantics are controlled with the envoy.reloadable_features.validate_header_values runtime-guarded feature. Originally, the PR used a new boolean on the HTTP connection manager proto to toggle the behavior, but during the course of PR review, it was decided that this would be more appropriate for a runtime guard. Testing: new unit tests, manual tests Release Notes: Updated Fixes #7061 Signed-off-by: Dylan Carney --- docs/root/intro/version_history.rst | 1 + source/common/http/BUILD | 4 + source/common/http/codec_client.cc | 5 +- source/common/http/codec_client.h | 3 +- source/common/http/conn_manager_config.h | 6 +- source/common/http/conn_manager_impl.cc | 7 +- source/common/http/conn_manager_impl.h | 2 + source/common/http/conn_manager_utility.cc | 7 +- source/common/http/conn_manager_utility.h | 2 +- source/common/http/header_utility.cc | 6 + source/common/http/header_utility.h | 7 ++ source/common/http/http1/BUILD | 2 + source/common/http/http1/codec_impl.cc | 53 +++++++-- source/common/http/http1/codec_impl.h | 11 ++ source/common/http/http1/conn_pool.cc | 8 +- source/common/http/http2/conn_pool.cc | 2 +- source/common/upstream/health_checker_impl.cc | 7 +- .../network/http_connection_manager/config.cc | 15 ++- .../network/http_connection_manager/config.h | 3 +- source/server/http/admin.cc | 5 +- source/server/http/admin.h | 3 +- .../http/conn_manager_impl_fuzz_test.cc | 2 +- test/common/http/conn_manager_impl_test.cc | 2 +- test/common/http/conn_manager_utility_test.cc | 10 +- test/common/http/header_utility_test.cc | 19 +++ test/common/http/http1/BUILD | 4 + test/common/http/http1/codec_impl_test.cc | 111 ++++++++++++++++-- test/integration/http_integration.cc | 4 +- test/integration/utility.cc | 2 +- 29 files changed, 261 insertions(+), 52 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 70e8fe3e8c87..82b9475ea32c 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -3,6 +3,7 @@ Version history 1.12.0 (pending) ================ +* http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. 1.11.0 (July 11, 2019) ====================== diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 88ff9675e93c..1dcf5045fa26 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -142,6 +142,7 @@ envoy_cc_library( ":conn_manager_config_interface", ":exception_lib", ":header_map_lib", + ":header_utility_lib", ":headers_lib", ":path_utility_lib", ":user_agent_lib", @@ -313,6 +314,9 @@ envoy_cc_library( name = "header_utility_lib", srcs = ["header_utility.cc"], hdrs = ["header_utility.h"], + external_deps = [ + "nghttp2", + ], deps = [ "//include/envoy/http:header_map_interface", "//include/envoy/json:json_object_interface", diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index 6c5488ef7883..fbfd421b0220 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -136,11 +136,12 @@ void CodecClient::onData(Buffer::Instance& data) { CodecClientProd::CodecClientProd(Type type, Network::ClientConnectionPtr&& connection, Upstream::HostDescriptionConstSharedPtr host, - Event::Dispatcher& dispatcher) + Event::Dispatcher& dispatcher, bool strict_header_validation) : CodecClient(type, std::move(connection), host, dispatcher) { switch (type) { case Type::HTTP1: { - codec_ = std::make_unique(*connection_, *this); + codec_ = std::make_unique(*connection_, *this, + strict_header_validation); break; } case Type::HTTP2: { diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index 4fa085d2e4fc..4bf2487da76c 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -241,7 +241,8 @@ using CodecClientPtr = std::unique_ptr; class CodecClientProd : public CodecClient { public: CodecClientProd(Type type, Network::ClientConnectionPtr&& connection, - Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher); + Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher, + bool strict_header_validation); }; } // namespace Http diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 5e08e267d5b9..78ebc7a19161 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -184,11 +184,15 @@ class ConnectionManagerConfig { * @param connection supplies the owning connection. * @param data supplies the currently available read data. * @param callbacks supplies the callbacks to install into the codec. + * @param strict_header_validation indicates whether or not the codec should validate the values + * of each HTTP header (NOTE: this argument only affects the H/1.1 codec; the H/2 codec always + * does this) * @return a codec or nullptr if no codec can be created. */ virtual ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - ServerConnectionCallbacks& callbacks) PURE; + ServerConnectionCallbacks& callbacks, + const bool strict_header_validation) PURE; /** * @return DateProvider& the date provider to use for diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index cf3e4f90ded6..998642cd2cd6 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -33,6 +33,7 @@ #include "common/http/path_utility.h" #include "common/http/utility.h" #include "common/network/utility.h" +#include "common/runtime/runtime_impl.h" #include "absl/strings/escaping.h" #include "absl/strings/match.h" @@ -111,7 +112,8 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfig& config, overload_manager ? overload_manager->getThreadLocalOverloadState().getState( Server::OverloadActionNames::get().DisableHttpKeepAlive) : Server::OverloadManager::getInactiveState()), - time_source_(time_source) {} + time_source_(time_source), strict_header_validation_(Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.strict_header_validation")) {} const HeaderMapImpl& ConnectionManagerImpl::continueHeader() { CONSTRUCT_ON_FIRST_USE(HeaderMapImpl, @@ -259,7 +261,8 @@ StreamDecoder& ConnectionManagerImpl::newStream(StreamEncoder& response_encoder, Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) { if (!codec_) { - codec_ = config_.createCodec(read_callbacks_->connection(), data, *this); + codec_ = + config_.createCodec(read_callbacks_->connection(), data, *this, strict_header_validation_); if (codec_->protocol() == Protocol::Http2) { stats_.named_.downstream_cx_http2_total_.inc(); stats_.named_.downstream_cx_http2_active_.inc(); diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 0a7139a40f12..e2a790b28683 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -678,6 +678,8 @@ class ConnectionManagerImpl : Logger::Loggable, const Server::OverloadActionState& overload_stop_accepting_requests_ref_; const Server::OverloadActionState& overload_disable_keepalive_ref_; TimeSource& time_source_; + + const bool strict_header_validation_; }; } // namespace Http diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index acc258499e56..6ec1bd5b9116 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -41,13 +41,14 @@ std::string ConnectionManagerUtility::determineNextProtocol(Network::Connection& ServerConnectionPtr ConnectionManagerUtility::autoCreateCodec( Network::Connection& connection, const Buffer::Instance& data, ServerConnectionCallbacks& callbacks, Stats::Scope& scope, const Http1Settings& http1_settings, - const Http2Settings& http2_settings, const uint32_t max_request_headers_kb) { + const Http2Settings& http2_settings, const uint32_t max_request_headers_kb, + bool strict_header_validation) { if (determineNextProtocol(connection, data) == Http2::ALPN_STRING) { return std::make_unique(connection, callbacks, scope, http2_settings, max_request_headers_kb); } else { - return std::make_unique(connection, callbacks, http1_settings, - max_request_headers_kb); + return std::make_unique( + connection, callbacks, http1_settings, max_request_headers_kb, strict_header_validation); } } diff --git a/source/common/http/conn_manager_utility.h b/source/common/http/conn_manager_utility.h index 802d0ef07add..c203fb154717 100644 --- a/source/common/http/conn_manager_utility.h +++ b/source/common/http/conn_manager_utility.h @@ -37,7 +37,7 @@ class ConnectionManagerUtility { autoCreateCodec(Network::Connection& connection, const Buffer::Instance& data, ServerConnectionCallbacks& callbacks, Stats::Scope& scope, const Http1Settings& http1_settings, const Http2Settings& http2_settings, - const uint32_t max_request_headers_kb); + const uint32_t max_request_headers_kb, bool strict_header_validation); /** * Mutates request headers in various ways. This functionality is broken out because of its diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc index 396fdb624f17..4cecc97178e7 100644 --- a/source/common/http/header_utility.cc +++ b/source/common/http/header_utility.cc @@ -6,6 +6,7 @@ #include "common/protobuf/utility.h" #include "absl/strings/match.h" +#include "nghttp2/nghttp2.h" namespace Envoy { namespace Http { @@ -117,6 +118,11 @@ bool HeaderUtility::matchHeaders(const Http::HeaderMap& request_headers, return match != header_data.invert_match_; } +bool HeaderUtility::headerIsValid(const absl::string_view header_value) { + return (nghttp2_check_header_value(reinterpret_cast(header_value.data()), + header_value.size()) != 0); +} + void HeaderUtility::addHeaders(Http::HeaderMap& headers, const Http::HeaderMap& headers_to_add) { headers_to_add.iterate( [](const Http::HeaderEntry& header, void* context) -> Http::HeaderMap::Iterate { diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h index 1e328691bf4c..d2b552da51fa 100644 --- a/source/common/http/header_utility.h +++ b/source/common/http/header_utility.h @@ -45,6 +45,13 @@ class HeaderUtility { static bool matchHeaders(const Http::HeaderMap& request_headers, const HeaderData& config_header); + /** + * Validates that a header value is valid, according to RFC 7230, section 3.2. + * http://tools.ietf.org/html/rfc7230#section-3.2 + * @return bool true if the header values are valid, according to the aforementioned RFC. + */ + static bool headerIsValid(const absl::string_view header_value); + /** * Add headers from one HeaderMap to another * @param headers target where headers will be added diff --git a/source/common/http/http1/BUILD b/source/common/http/http1/BUILD index edd4dff03073..22f47ba2f31f 100644 --- a/source/common/http/http1/BUILD +++ b/source/common/http/http1/BUILD @@ -27,6 +27,7 @@ envoy_cc_library( "//source/common/http:codes_lib", "//source/common/http:exception_lib", "//source/common/http:header_map_lib", + "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", ], @@ -55,6 +56,7 @@ envoy_cc_library( "//source/common/http:conn_pool_base_lib", "//source/common/http:headers_lib", "//source/common/network:utility_lib", + "//source/common/runtime:runtime_lib", "//source/common/upstream:upstream_lib", ], ) diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index edab3707018e..863606b32c6c 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -13,6 +13,7 @@ #include "common/common/stack_array.h" #include "common/common/utility.h" #include "common/http/exception.h" +#include "common/http/header_utility.h" #include "common/http/headers.h" #include "common/http/utility.h" @@ -319,11 +320,20 @@ const ToLowerTable& ConnectionImpl::toLowerTable() { return *table; } +// TODO(alyssawilk) The overloaded constructors here and on {Client,Server}ConnectionImpl +// can be cleaned up once "strict_header_validation" becomes the default behavior, rather +// than a runtime-guarded one. The overloads were a workaround for the fact that Runtime +// doesn't work from integration test call sites in scenarios where the required +// thread-local storage is not available. ConnectionImpl::ConnectionImpl(Network::Connection& connection, http_parser_type type, uint32_t max_headers_kb) + : ConnectionImpl::ConnectionImpl(connection, type, max_headers_kb, false) {} + +ConnectionImpl::ConnectionImpl(Network::Connection& connection, http_parser_type type, + uint32_t max_headers_kb, bool strict_header_validation) : connection_(connection), output_buffer_([&]() -> void { this->onBelowLowWatermark(); }, [&]() -> void { this->onAboveHighWatermark(); }), - max_headers_kb_(max_headers_kb) { + max_headers_kb_(max_headers_kb), strict_header_validation_(strict_header_validation) { output_buffer_.setWatermarks(connection.bufferLimit()); http_parser_init(&parser_, type); parser_.data = this; @@ -421,11 +431,21 @@ void ConnectionImpl::onHeaderValue(const char* data, size_t length) { // Ignore trailers. return; } - // http-parser should filter for this - // (https://tools.ietf.org/html/rfc7230#section-3.2.6), but it doesn't today. HeaderStrings - // have an invariant that they must not contain embedded zero characters - // (NUL, ASCII 0x0). - if (absl::string_view(data, length).find('\0') != absl::string_view::npos) { + + const absl::string_view header_value = absl::string_view(data, length); + + if (strict_header_validation_) { + if (!Http::HeaderUtility::headerIsValid(header_value)) { + ENVOY_CONN_LOG(debug, "invalid header value: {}", connection_, header_value); + error_code_ = Http::Code::BadRequest; + sendProtocolError(); + throw CodecProtocolException("http/1.1 protocol error: header value contains invalid chars"); + } + } else if (header_value.find('\0') != absl::string_view::npos) { + // http-parser should filter for this + // (https://tools.ietf.org/html/rfc7230#section-3.2.6), but it doesn't today. HeaderStrings + // have an invariant that they must not contain embedded zero characters + // (NUL, ASCII 0x0). throw CodecProtocolException("http/1.1 protocol error: header value contains NUL"); } @@ -496,8 +516,15 @@ void ConnectionImpl::onResetStreamBase(StreamResetReason reason) { ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, ServerConnectionCallbacks& callbacks, Http1Settings settings, uint32_t max_request_headers_kb) - : ConnectionImpl(connection, HTTP_REQUEST, max_request_headers_kb), callbacks_(callbacks), - codec_settings_(settings) {} + : ServerConnectionImpl::ServerConnectionImpl(connection, callbacks, settings, + max_request_headers_kb, false) {} + +ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, + ServerConnectionCallbacks& callbacks, + Http1Settings settings, uint32_t max_request_headers_kb, + bool strict_header_validation) + : ConnectionImpl(connection, HTTP_REQUEST, max_request_headers_kb, strict_header_validation), + callbacks_(callbacks), codec_settings_(settings) {} void ServerConnectionImpl::onEncodeComplete() { ASSERT(active_request_); @@ -668,8 +695,14 @@ void ServerConnectionImpl::onBelowLowWatermark() { } } -ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, ConnectionCallbacks&) - : ConnectionImpl(connection, HTTP_RESPONSE, MAX_RESPONSE_HEADERS_KB) {} +ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, + ConnectionCallbacks& callbacks) + : ClientConnectionImpl::ClientConnectionImpl(connection, callbacks, false) {} + +ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, ConnectionCallbacks&, + bool strict_header_validation) + : ConnectionImpl(connection, HTTP_RESPONSE, MAX_RESPONSE_HEADERS_KB, strict_header_validation) { +} bool ClientConnectionImpl::cannotHaveBody() { if ((!pending_responses_.empty() && pending_responses_.front().head_request_) || diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index 27b80e5e5b81..a9bf7111b73b 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -174,6 +174,8 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable(Http::CodecClient::Type::HTTP2, std::move(data.connection_), - data.host_description_, dispatcher_); + data.host_description_, dispatcher_, false); } std::ostream& operator<<(std::ostream& out, HealthState state) { diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 04edfdba79c4..f5596e83bfe0 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -360,21 +360,20 @@ void HttpConnectionManagerConfig::processFilter( filter_factories.push_back(callback); } -Http::ServerConnectionPtr -HttpConnectionManagerConfig::createCodec(Network::Connection& connection, - const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks) { +Http::ServerConnectionPtr HttpConnectionManagerConfig::createCodec( + Network::Connection& connection, const Buffer::Instance& data, + Http::ServerConnectionCallbacks& callbacks, const bool strict_header_validation) { switch (codec_type_) { case CodecType::HTTP1: return std::make_unique( - connection, callbacks, http1_settings_, maxRequestHeadersKb()); + connection, callbacks, http1_settings_, maxRequestHeadersKb(), strict_header_validation); case CodecType::HTTP2: return std::make_unique( connection, callbacks, context_.scope(), http2_settings_, maxRequestHeadersKb()); case CodecType::AUTO: - return Http::ConnectionManagerUtility::autoCreateCodec(connection, data, callbacks, - context_.scope(), http1_settings_, - http2_settings_, maxRequestHeadersKb()); + return Http::ConnectionManagerUtility::autoCreateCodec( + connection, data, callbacks, context_.scope(), http1_settings_, http2_settings_, + maxRequestHeadersKb(), strict_header_validation); } NOT_REACHED_GCOVR_EXCL_LINE; diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index 796b67fc9728..eb97a9bf21db 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -99,7 +99,8 @@ class HttpConnectionManagerConfig : Logger::Loggable, const std::list& accessLogs() override { return access_logs_; } Http::ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks) override; + Http::ServerConnectionCallbacks& callbacks, + const bool strict_header_validation) override; Http::DateProvider& dateProvider() override { return date_provider_; } std::chrono::milliseconds drainTimeout() override { return drain_timeout_; } FilterChainFactory& filterFactory() override { return *this; } diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 6537b978123b..598699859d90 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -1241,10 +1241,11 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server) Http::ServerConnectionPtr AdminImpl::createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks) { + Http::ServerConnectionCallbacks& callbacks, + const bool strict_header_validation) { return Http::ConnectionManagerUtility::autoCreateCodec( connection, data, callbacks, server_.stats(), Http::Http1Settings(), Http::Http2Settings(), - maxRequestHeadersKb()); + maxRequestHeadersKb(), strict_header_validation); } bool AdminImpl::createNetworkFilterChain(Network::Connection& connection, diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 6fc5d3a6e254..f13129a0e679 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -104,7 +104,8 @@ class AdminImpl : public Admin, const std::list& accessLogs() override { return access_logs_; } Http::ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks) override; + Http::ServerConnectionCallbacks& callbacks, + const bool strict_header_validation) override; Http::DateProvider& dateProvider() override { return date_provider_; } std::chrono::milliseconds drainTimeout() override { return std::chrono::milliseconds(100); } Http::FilterChainFactory& filterFactory() override { return *this; } diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 3c11424b5964..e15e88e97473 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -72,7 +72,7 @@ class FuzzConfig : public ConnectionManagerConfig { // Http::ConnectionManagerConfig const std::list& accessLogs() override { return access_logs_; } ServerConnectionPtr createCodec(Network::Connection&, const Buffer::Instance&, - ServerConnectionCallbacks&) override { + ServerConnectionCallbacks&, const bool) override { return ServerConnectionPtr{codec_}; } DateProvider& dateProvider() override { return date_provider_; } diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index ce4e8af46284..31865f3f7287 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -241,7 +241,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan // Http::ConnectionManagerConfig const std::list& accessLogs() override { return access_logs_; } ServerConnectionPtr createCodec(Network::Connection&, const Buffer::Instance&, - ServerConnectionCallbacks&) override { + ServerConnectionCallbacks&, const bool) override { return ServerConnectionPtr{codec_}; } DateProvider& dateProvider() override { return date_provider_; } diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 46937cf17ab0..48fa93d14174 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -43,13 +43,15 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { // Http::ConnectionManagerConfig ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& instance, - ServerConnectionCallbacks& callbacks) override { - return ServerConnectionPtr{createCodec_(connection, instance, callbacks)}; + ServerConnectionCallbacks& callbacks, + const bool strict_header_validation) override { + return ServerConnectionPtr{ + createCodec_(connection, instance, callbacks, strict_header_validation)}; } MOCK_METHOD0(accessLogs, const std::list&()); - MOCK_METHOD3(createCodec_, ServerConnection*(Network::Connection&, const Buffer::Instance&, - ServerConnectionCallbacks&)); + MOCK_METHOD4(createCodec_, ServerConnection*(Network::Connection&, const Buffer::Instance&, + ServerConnectionCallbacks&, const bool)); MOCK_METHOD0(dateProvider, DateProvider&()); MOCK_METHOD0(drainTimeout, std::chrono::milliseconds()); MOCK_METHOD0(filterFactory, FilterChainFactory&()); diff --git a/test/common/http/header_utility_test.cc b/test/common/http/header_utility_test.cc index b2fe1de01cd8..a20f4ca43f49 100644 --- a/test/common/http/header_utility_test.cc +++ b/test/common/http/header_utility_test.cc @@ -403,6 +403,25 @@ invert_match: true EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } +TEST(HeaderIsValidTest, InvalidHeaderValuesAreRejected) { + // ASCII values 1-31 are control characters (with the exception of ASCII + // values 9, 10, and 13 which are a horizontal tab, line feed, and carriage + // return, respectively), and are not valid in an HTTP header, per + // RFC 7230, section 3.2 + for (uint i = 0; i < 32; i++) { + if (i == 9) { + continue; + } + + EXPECT_FALSE(HeaderUtility::headerIsValid(std::string(1, i))); + } +} + +TEST(HeaderIsValidTest, ValidHeaderValuesAreAccepted) { + EXPECT_TRUE(HeaderUtility::headerIsValid("some-value")); + EXPECT_TRUE(HeaderUtility::headerIsValid("Some Other Value")); +} + TEST(HeaderAddTest, HeaderAdd) { TestHeaderMapImpl headers{{"myheader1", "123value"}}; TestHeaderMapImpl headers_to_add{{"myheader2", "456value"}}; diff --git a/test/common/http/http1/BUILD b/test/common/http/http1/BUILD index e685bff6f328..efee73f47d1b 100644 --- a/test/common/http/http1/BUILD +++ b/test/common/http/http1/BUILD @@ -21,7 +21,11 @@ envoy_cc_test( "//source/common/http/http1:codec_lib", "//test/mocks/buffer:buffer_mocks", "//test/mocks/http:http_mocks", + "//test/mocks/init:init_mocks", + "//test/mocks/local_info:local_info_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", ], ) diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index c65d0ba18cb3..1db1fee44f4e 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -8,11 +8,18 @@ #include "common/http/exception.h" #include "common/http/header_map_impl.h" #include "common/http/http1/codec_impl.h" +#include "common/runtime/runtime_impl.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/thread_local/mocks.h" #include "test/test_common/printers.h" +#include "test/test_common/utility.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -30,9 +37,23 @@ namespace Http1 { class Http1ServerConnectionImplTest : public testing::Test { public: + Http1ServerConnectionImplTest() : api_(Api::createApiForTest()) { + envoy::config::bootstrap::v2::LayeredRuntime config; + config.add_layers()->mutable_admin_layer(); + + // Create a runtime loader, so that tests can manually manipulate runtime + // guarded features. + loader_ = std::make_unique(Runtime::LoaderPtr{ + new Runtime::LoaderImpl(dispatcher_, tls_, config, local_info_, init_manager_, store_, + generator_, validation_visitor_, *api_)}); + } + void initialize() { - codec_ = std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_); + strict_header_validation_ = + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.strict_header_validation"); + codec_ = + std::make_unique(connection_, callbacks_, codec_settings_, + max_request_headers_kb_, strict_header_validation_); } NiceMock connection_; @@ -46,6 +67,16 @@ class Http1ServerConnectionImplTest : public testing::Test { protected: uint32_t max_request_headers_kb_{Http::DEFAULT_MAX_REQUEST_HEADERS_KB}; + bool strict_header_validation_; + Event::MockDispatcher dispatcher_; + NiceMock tls_; + Stats::IsolatedStoreImpl store_; + Runtime::MockRandomGenerator generator_; + Api::ApiPtr api_; + NiceMock local_info_; + Init::MockManager init_manager_; + NiceMock validation_visitor_; + std::unique_ptr loader_; }; void Http1ServerConnectionImplTest::expect400(Protocol p, bool allow_absolute_url, @@ -57,8 +88,9 @@ void Http1ServerConnectionImplTest::expect400(Protocol p, bool allow_absolute_ur if (allow_absolute_url) { codec_settings_.allow_absolute_url_ = allow_absolute_url; - codec_ = std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_); + codec_ = + std::make_unique(connection_, callbacks_, codec_settings_, + max_request_headers_kb_, strict_header_validation_); } Http::MockStreamDecoder decoder; @@ -77,8 +109,9 @@ void Http1ServerConnectionImplTest::expectHeadersTest(Protocol p, bool allow_abs // Make a new 'codec' with the right settings if (allow_absolute_url) { codec_settings_.allow_absolute_url_ = allow_absolute_url; - codec_ = std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_); + codec_ = + std::make_unique(connection_, callbacks_, codec_settings_, + max_request_headers_kb_, strict_header_validation_); } Http::MockStreamDecoder decoder; @@ -333,6 +366,43 @@ TEST_F(Http1ServerConnectionImplTest, HostHeaderTranslation) { EXPECT_EQ(0U, buffer.length()); } +// Ensures that requests with invalid HTTP header values are not rejected +// when the runtime guard is not enabled for the feature. +TEST_F(Http1ServerConnectionImplTest, HeaderInvalidCharsRuntimeGuard) { + // When the runtime-guarded feature is NOT enabled, invalid header values + // should be accepted by the codec. + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.strict_header_validation", "false"}}); + + initialize(); + + Http::MockStreamDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + + Buffer::OwnedImpl buffer( + absl::StrCat("GET / HTTP/1.1\r\nHOST: h.com\r\nfoo: ", std::string(1, 3), "\r\n")); + codec_->dispatch(buffer); +} + +// Ensures that requests with invalid HTTP header values are properly rejected +// when the runtime guard is enabled for the feature. +TEST_F(Http1ServerConnectionImplTest, HeaderInvalidCharsRejection) { + // When the runtime-guarded feature is enabled, invalid header values + // should result in a rejection. + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.strict_header_validation", "true"}}); + + initialize(); + + Http::MockStreamDecoder decoder; + EXPECT_CALL(callbacks_, newStream(_, _)).WillOnce(ReturnRef(decoder)); + + Buffer::OwnedImpl buffer( + absl::StrCat("GET / HTTP/1.1\r\nHOST: h.com\r\nfoo: ", std::string(1, 3), "\r\n")); + EXPECT_THROW_WITH_MESSAGE(codec_->dispatch(buffer), CodecProtocolException, + "http/1.1 protocol error: header value contains invalid chars"); +} + // Regression test for http-parser allowing embedded NULs in header values, // verify we reject them. TEST_F(Http1ServerConnectionImplTest, HeaderEmbeddedNulRejection) { @@ -784,11 +854,38 @@ TEST_F(Http1ServerConnectionImplTest, WatermarkTest) { class Http1ClientConnectionImplTest : public testing::Test { public: - void initialize() { codec_ = std::make_unique(connection_, callbacks_); } + Http1ClientConnectionImplTest() : api_(Api::createApiForTest()) { + envoy::config::bootstrap::v2::LayeredRuntime config; + + // Create a runtime loader, so that tests can manually manipulate runtime + // guarded features. + loader_ = std::make_unique(Runtime::LoaderPtr{ + new Runtime::LoaderImpl(dispatcher_, tls_, config, local_info_, init_manager_, store_, + generator_, validation_visitor_, *api_)}); + } + + void initialize() { + strict_header_validation_ = + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.strict_header_validation"); + codec_ = + std::make_unique(connection_, callbacks_, strict_header_validation_); + } NiceMock connection_; NiceMock callbacks_; std::unique_ptr codec_; + +protected: + Event::MockDispatcher dispatcher_; + NiceMock tls_; + Stats::IsolatedStoreImpl store_; + Runtime::MockRandomGenerator generator_; + Api::ApiPtr api_; + NiceMock local_info_; + Init::MockManager init_manager_; + NiceMock validation_visitor_; + std::unique_ptr loader_; + bool strict_header_validation_; }; TEST_F(Http1ClientConnectionImplTest, SimpleGet) { diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index fbd7308fd4a6..eec74c73d452 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -62,8 +62,8 @@ typeToCodecType(Http::CodecClient::Type type) { IntegrationCodecClient::IntegrationCodecClient( Event::Dispatcher& dispatcher, Network::ClientConnectionPtr&& conn, Upstream::HostDescriptionConstSharedPtr host_description, CodecClient::Type type) - : CodecClientProd(type, std::move(conn), host_description, dispatcher), dispatcher_(dispatcher), - callbacks_(*this), codec_callbacks_(*this) { + : CodecClientProd(type, std::move(conn), host_description, dispatcher, false), + dispatcher_(dispatcher), callbacks_(*this), codec_callbacks_(*this) { connection_->addConnectionCallbacks(callbacks_); setCodecConnectionCallbacks(codec_callbacks_); dispatcher.run(Event::Dispatcher::RunType::Block); diff --git a/test/integration/utility.cc b/test/integration/utility.cc index ecf5fd26ae7d..de4375118daf 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -76,7 +76,7 @@ IntegrationUtil::makeSingleRequest(const Network::Address::InstanceConstSharedPt type, dispatcher->createClientConnection(addr, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr), - host_description, *dispatcher); + host_description, *dispatcher, false); BufferingStreamDecoderPtr response(new BufferingStreamDecoder([&]() -> void { client.close(); dispatcher->exit(); From 101f8157fe06d78262d0b3c07cf1c2b7c8e72c98 Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Tue, 16 Jul 2019 01:24:37 +0300 Subject: [PATCH 174/542] tap: remove unused dependency. (#7580) Description: Remove an unused dependency from tap. This makes a build warning go away. Risk Level: low Testing: build Docs Changes: N/A Release Notes: N/A Signed-off-by: Ismo Puustinen --- api/envoy/service/tap/v2alpha/BUILD | 1 - api/envoy/service/tap/v2alpha/common.proto | 1 - 2 files changed, 2 deletions(-) diff --git a/api/envoy/service/tap/v2alpha/BUILD b/api/envoy/service/tap/v2alpha/BUILD index 6651b9c5e10d..63d11e80a755 100644 --- a/api/envoy/service/tap/v2alpha/BUILD +++ b/api/envoy/service/tap/v2alpha/BUILD @@ -10,7 +10,6 @@ api_proto_library_internal( "//envoy/api/v2/core:base", "//envoy/api/v2/core:grpc_service", "//envoy/api/v2/route", - "//envoy/type/matcher:string", ], ) diff --git a/api/envoy/service/tap/v2alpha/common.proto b/api/envoy/service/tap/v2alpha/common.proto index 3c7bf498f42a..54437b305469 100644 --- a/api/envoy/service/tap/v2alpha/common.proto +++ b/api/envoy/service/tap/v2alpha/common.proto @@ -3,7 +3,6 @@ syntax = "proto3"; import "envoy/api/v2/route/route.proto"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/core/grpc_service.proto"; -import "envoy/type/matcher/string.proto"; import "google/protobuf/wrappers.proto"; From 49635f47a42a6447047c2f97540041f6ca07a374 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 15 Jul 2019 17:12:42 -0700 Subject: [PATCH 175/542] dubbo: clang-tidy fix (#7587) Fix merge race of #7118 and #7447 Risk Level: Low Testing: n/a Docs Changes: n/a Release Notes: n/a Signed-off-by: Lizan Zhou --- source/extensions/filters/network/dubbo_proxy/decoder.cc | 8 ++++---- source/extensions/filters/network/dubbo_proxy/decoder.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.cc b/source/extensions/filters/network/dubbo_proxy/decoder.cc index a88ade7d9ed0..f7e9cfc84a24 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.cc +++ b/source/extensions/filters/network/dubbo_proxy/decoder.cc @@ -25,14 +25,14 @@ DecoderStateMachine::onDecodeStreamHeader(Buffer::Instance& buffer) { ENVOY_LOG(debug, "dubbo decoder: this is the {} heartbeat message", protocol_.name()); buffer.drain(context->header_size()); delegate_.onHeartbeat(metadata); - return DecoderStatus(ProtocolState::Done); + return {ProtocolState::Done}; } active_stream_ = delegate_.newStream(metadata, context); ASSERT(active_stream_); context->message_origin_data().move(buffer, context->header_size()); - return DecoderStatus(ProtocolState::OnDecodeStreamData); + return {ProtocolState::OnDecodeStreamData}; } DecoderStateMachine::DecoderStatus @@ -42,7 +42,7 @@ DecoderStateMachine::onDecodeStreamData(Buffer::Instance& buffer) { if (!protocol_.decodeData(buffer, active_stream_->context_, active_stream_->metadata_)) { ENVOY_LOG(debug, "dubbo decoder: need more data for {} serialization, current size {}", protocol_.serializer()->name(), buffer.length()); - return DecoderStatus(ProtocolState::WaitForData); + return {ProtocolState::WaitForData}; } active_stream_->context_->message_origin_data().move(buffer, @@ -51,7 +51,7 @@ DecoderStateMachine::onDecodeStreamData(Buffer::Instance& buffer) { active_stream_ = nullptr; ENVOY_LOG(debug, "dubbo decoder: ends the deserialization of the message"); - return DecoderStatus(ProtocolState::Done); + return {ProtocolState::Done}; } DecoderStateMachine::DecoderStatus DecoderStateMachine::handleState(Buffer::Instance& buffer) { diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.h b/source/extensions/filters/network/dubbo_proxy/decoder.h index 5b04d277876e..627dc314428b 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.h +++ b/source/extensions/filters/network/dubbo_proxy/decoder.h @@ -99,7 +99,7 @@ class DecoderStateMachine : public Logger::Loggable { private: struct DecoderStatus { DecoderStatus() = default; - DecoderStatus(ProtocolState next_state) : next_state_(next_state), filter_status_{} {}; + DecoderStatus(ProtocolState next_state) : next_state_(next_state){}; DecoderStatus(ProtocolState next_state, FilterStatus filter_status) : next_state_(next_state), filter_status_(filter_status){}; From cc155c1f1a1baf9dbd4a187e526e5ccd6f6c3370 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 15 Jul 2019 22:51:02 -0400 Subject: [PATCH 176/542] deflake ads_integration_test (#7584) Signed-off-by: Asra Ali --- test/integration/ads_integration_test.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 3745513ee431..507287362055 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -958,22 +958,42 @@ TEST_P(AdsIntegrationTest, XdsBatching) { TEST_P(AdsIntegrationTest, ListenerDrainBeforeServerStart) { initialize(); + // Initial request for cluster, response for cluster_0. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); + + // Initial request for load assignment for cluster_0, respond with version 1 + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", + {"cluster_0"}, {"cluster_0"}, {})); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, {buildClusterLoadAssignment("cluster_0")}, {}, "1"); + // Request for updates to cluster_0 version 1, no response + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); + // Initial request for any listener, respond with listener_0 version 1 + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); sendDiscoveryResponse( Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, {buildListener("listener_0", "route_config_0")}, {}, "1"); + + // Request for updates to load assignment version 1, no response + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", + {"cluster_0"}, {}, {})); + + // Initial request for route_config_0 (referenced by listener_0), respond with version 1 + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", + {"route_config_0"}, {"route_config_0"}, {})); + test_server_->waitForGaugeGe("listener_manager.total_listeners_active", 1); // Before server is started, even though listeners are added to active list // we mark them as "warming" in config dump since they're not initialized yet. EXPECT_EQ(getListenersConfigDump().dynamic_warming_listeners().size(), 1); // Remove listener. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, {}, "1"); test_server_->waitForGaugeEq("listener_manager.total_listeners_active", 0); } From 7a554179105917400b254065d9f97669b3af7025 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 15 Jul 2019 22:51:42 -0400 Subject: [PATCH 177/542] release: flipping deprecated features to be fatal-by-default (#7549) Signed-off-by: Alyssa Wilk --- GOVERNANCE.md | 2 +- api/envoy/api/v2/cds.proto | 2 +- configs/envoy_double_proxy_v2.template.yaml | 17 +++++++++++++---- configs/envoy_front_proxy_v2.template.yaml | 17 +++++++++++++---- .../envoy_service_to_service_v2.template.yaml | 17 +++++++++++++---- source/common/runtime/runtime_features.cc | 6 ++++++ 6 files changed, 47 insertions(+), 14 deletions(-) diff --git a/GOVERNANCE.md b/GOVERNANCE.md index aa5cb0da5972..737240ad62b6 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -103,7 +103,7 @@ or you can subscribe to the iCal feed [here](https://app.opsgenie.com/webcal/get following version. E.g., "1.7.0 (pending)". * Run the deprecate_versions.py script (e.g. `sh tools/deprecate_version/deprecate_version.sh`) to file tracking issues for code which can be removed. -* Run the deprecate_features.py script (e.g. `sh tools/deprecate_version/deprecate_features.sh`) +* Run the deprecate_features.py script (e.g. `sh tools/deprecate_features/deprecate_features.sh`) to make the last release's deprecated features fatal-by-default. Submit the resultant PR and send an email to envoy-announce. diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 83fe4329e15b..864290cdc42d 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -188,7 +188,7 @@ message Cluster { // **This field is deprecated**. Set the // :ref:`load_assignment` field instead. // - repeated core.Address hosts = 7 [deprecated = true]; + repeated core.Address hosts = 7; // Setting this is required for specifying members of // :ref:`STATIC`, diff --git a/configs/envoy_double_proxy_v2.template.yaml b/configs/envoy_double_proxy_v2.template.yaml index 2c08332f795d..9cff9a40a6c7 100644 --- a/configs/envoy_double_proxy_v2.template.yaml +++ b/configs/envoy_double_proxy_v2.template.yaml @@ -174,10 +174,19 @@ tracing: "@type": type.googleapis.com/envoy.config.trace.v2.LightstepConfig access_token_file: "/etc/envoy/lightstep_access_token" collector_cluster: lightstep_saas -runtime: - symlink_root: "/srv/runtime_data/current" - subdirectory: envoy - override_subdirectory: envoy_override +layered_runtime: + layers: + - name: root + disk_layer: + symlink_root: /srv/configset/envoydata/current + subdirectory: envoy + - name: override + disk_layer: + symlink_root: /srv/configset/envoydata/current + subdirectory: envoy_override + append_service_cluster: true + - name: admin + admin_layer: {} admin: access_log_path: "/var/log/envoy/admin_access.log" address: diff --git a/configs/envoy_front_proxy_v2.template.yaml b/configs/envoy_front_proxy_v2.template.yaml index 35f734f80ad2..c45bc1e24d0a 100644 --- a/configs/envoy_front_proxy_v2.template.yaml +++ b/configs/envoy_front_proxy_v2.template.yaml @@ -157,10 +157,19 @@ tracing: "@type": type.googleapis.com/envoy.config.trace.v2.LightstepConfig collector_cluster: lightstep_saas access_token_file: "/etc/envoy/lightstep_access_token" -runtime: - symlink_root: /srv/runtime_data/current - subdirectory: envoy - override_subdirectory: envoy_override +layered_runtime: + layers: + - name: root + disk_layer: + symlink_root: /srv/configset/envoydata/current + subdirectory: envoy + - name: override + disk_layer: + symlink_root: /srv/configset/envoydata/current + subdirectory: envoy_override + append_service_cluster: true + - name: admin + admin_layer: {} admin: access_log_path: /var/log/envoy/admin_access.log address: diff --git a/configs/envoy_service_to_service_v2.template.yaml b/configs/envoy_service_to_service_v2.template.yaml index 083a8c39a292..531322b3a14b 100644 --- a/configs/envoy_service_to_service_v2.template.yaml +++ b/configs/envoy_service_to_service_v2.template.yaml @@ -543,10 +543,19 @@ tracing: "@type": type.googleapis.com/envoy.config.trace.v2.LightstepConfig access_token_file: "/etc/envoy/lightstep_access_token" collector_cluster: lightstep_saas -runtime: - symlink_root: "/srv/runtime_data/current" - subdirectory: envoy - override_subdirectory: envoy_override +layered_runtime: + layers: + - name: root + disk_layer: + symlink_root: /srv/configset/envoydata/current + subdirectory: envoy + - name: override + disk_layer: + symlink_root: /srv/configset/envoydata/current + subdirectory: envoy_override + append_service_cluster: true + - name: admin + admin_layer: {} admin: access_log_path: /var/log/envoy/admin_access.log address: diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index bb1800a1b198..3a26e048119b 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -42,6 +42,12 @@ constexpr const char* disallowed_features[] = { // Acts as both a test entry for deprecated.proto and a marker for the Envoy // deprecation scripts. "envoy.deprecated_features.deprecated.proto:is_deprecated_fatal", + "envoy.deprecated_features.bootstrap.proto:runtime", + "envoy.deprecated_features.redis_proxy.proto:catch_all_cluster", + "envoy.deprecated_features.lds.proto:use_original_dst", + "envoy.deprecated_features.server_info.proto:max_stats", + "envoy.deprecated_features.redis_proxy.proto:cluster", + "envoy.deprecated_features.server_info.proto:max_obj_name_len", "envoy.deprecated_features.config_source.proto:UNSUPPORTED_REST_LEGACY", "envoy.deprecated_features.ext_authz.proto:use_alpha", "envoy.deprecated_features.route.proto:enabled", From 42706efea87eda276cd650db99bc318319176a98 Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Mon, 15 Jul 2019 23:10:49 -0700 Subject: [PATCH 178/542] config: async data access for local and remote data source (#7399) Description: Add an async data source which supports fetching data from local and remote data source. The async data provider guarantees that data is access before `init manager` is ready. Risk Level: Low Testing: Unit test Docs Changes: N/A Release Notes: Added Fixes #7311 Signed-off-by: crazyxy --- api/envoy/api/v2/core/BUILD | 6 +- api/envoy/api/v2/core/base.proto | 24 ++ docs/root/intro/version_history.rst | 1 + source/common/config/BUILD | 20 ++ source/common/config/datasource.h | 73 ++++ source/common/config/remote_data_fetcher.cc | 75 ++++ source/common/config/remote_data_fetcher.h | 82 +++++ test/common/config/BUILD | 14 + test/common/config/datasource_test.cc | 379 ++++++++++++++++++++ 9 files changed, 673 insertions(+), 1 deletion(-) create mode 100644 source/common/config/remote_data_fetcher.cc create mode 100644 source/common/config/remote_data_fetcher.h create mode 100644 test/common/config/datasource_test.cc diff --git a/api/envoy/api/v2/core/BUILD b/api/envoy/api/v2/core/BUILD index b324a8ad0190..b3d2be2301f1 100644 --- a/api/envoy/api/v2/core/BUILD +++ b/api/envoy/api/v2/core/BUILD @@ -38,6 +38,7 @@ api_proto_library_internal( ":friends", ], deps = [ + ":http_uri", "//envoy/type:percent", ], ) @@ -45,7 +46,10 @@ api_proto_library_internal( api_go_proto_library( name = "base", proto = ":base", - deps = ["//envoy/type:percent_go_proto"], + deps = [ + ":http_uri_go_proto", + "//envoy/type:percent_go_proto", + ], ) api_proto_library_internal( diff --git a/api/envoy/api/v2/core/base.proto b/api/envoy/api/v2/core/base.proto index 949cbfc94afd..23704f765502 100644 --- a/api/envoy/api/v2/core/base.proto +++ b/api/envoy/api/v2/core/base.proto @@ -7,6 +7,8 @@ option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.core"; option go_package = "core"; +import "envoy/api/v2/core/http_uri.proto"; + import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; @@ -187,6 +189,28 @@ message DataSource { } } +// The message specifies how to fetch data from remote and how to verify it. +message RemoteDataSource { + // The HTTP URI to fetch the remote data. + HttpUri http_uri = 1 [(validate.rules).message.required = true]; + + // SHA256 string for verifying data. + string sha256 = 2 [(validate.rules).string.min_bytes = 1]; +} + +// Async data source which support async data fetch. +message AsyncDataSource { + oneof specifier { + option (validate.required) = true; + + // Local async data source. + DataSource local = 1; + + // Remote async data source. + RemoteDataSource remote = 2; + } +} + // Configuration for transport socket in :ref:`listeners ` and // :ref:`clusters `. If the configuration is // empty, a default transport socket implementation and configuration will be diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 82b9475ea32c..566ef6d895c0 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -3,6 +3,7 @@ Version history 1.12.0 (pending) ================ +* config: async data access for local and remote data source. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. 1.11.0 (July 11, 2019) diff --git a/source/common/config/BUILD b/source/common/config/BUILD index c7fb5b067410..8ebd3aedae14 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -73,8 +73,15 @@ envoy_cc_library( srcs = ["datasource.cc"], hdrs = ["datasource.h"], deps = [ + ":remote_data_fetcher_lib", "//include/envoy/api:api_interface", + "//include/envoy/init:manager_interface", + "//include/envoy/upstream:cluster_manager_interface", + "//source/common/common:empty_string", + "//source/common/init:target_lib", + "//source/common/protobuf:utility_lib", "@envoy_api//envoy/api/v2/core:base_cc", + "@envoy_api//envoy/api/v2/core:http_uri_cc", ], ) @@ -413,3 +420,16 @@ envoy_cc_library( "//source/common/protobuf", ], ) + +envoy_cc_library( + name = "remote_data_fetcher_lib", + srcs = ["remote_data_fetcher.cc"], + hdrs = ["remote_data_fetcher.h"], + deps = [ + "//include/envoy/upstream:cluster_manager_interface", + "//source/common/common:hex_lib", + "//source/common/crypto:utility_lib", + "//source/common/http:utility_lib", + "@envoy_api//envoy/api/v2/core:http_uri_cc", + ], +) diff --git a/source/common/config/datasource.h b/source/common/config/datasource.h index 8c1312165f92..991f7c2e6034 100644 --- a/source/common/config/datasource.h +++ b/source/common/config/datasource.h @@ -2,6 +2,13 @@ #include "envoy/api/api.h" #include "envoy/api/v2/core/base.pb.h" +#include "envoy/init/manager.h" +#include "envoy/upstream/cluster_manager.h" + +#include "common/common/empty_string.h" +#include "common/common/enum_to_int.h" +#include "common/config/remote_data_fetcher.h" +#include "common/init/target_impl.h" #include "absl/types/optional.h" @@ -25,6 +32,72 @@ std::string read(const envoy::api::v2::core::DataSource& source, bool allow_empt */ absl::optional getPath(const envoy::api::v2::core::DataSource& source); +/** + * Callback for async data source. + */ +using AsyncDataSourceCb = std::function; + +class LocalAsyncDataProvider { +public: + LocalAsyncDataProvider(Init::Manager& manager, const envoy::api::v2::core::DataSource& source, + bool allow_empty, Api::Api& api, AsyncDataSourceCb&& callback) + : init_target_("LocalAsyncDataProvider", [this, &source, allow_empty, &api, callback]() { + callback(DataSource::read(source, allow_empty, api)); + init_target_.ready(); + }) { + manager.add(init_target_); + } + + ~LocalAsyncDataProvider() { init_target_.ready(); } + +private: + Init::TargetImpl init_target_; +}; + +using LocalAsyncDataProviderPtr = std::unique_ptr; + +class RemoteAsyncDataProvider : public Config::DataFetcher::RemoteDataFetcherCallback { +public: + RemoteAsyncDataProvider(Upstream::ClusterManager& cm, Init::Manager& manager, + const envoy::api::v2::core::RemoteDataSource& source, bool allow_empty, + AsyncDataSourceCb&& callback) + : allow_empty_(allow_empty), callback_(std::move(callback)), + fetcher_(std::make_unique(cm, source.http_uri(), + source.sha256(), *this)), + init_target_("RemoteAsyncDataProvider", [this]() { start(); }) { + manager.add(init_target_); + } + + ~RemoteAsyncDataProvider() override { init_target_.ready(); } + + // Config::DataFetcher::RemoteDataFetcherCallback + void onSuccess(const std::string& data) override { + callback_(data); + init_target_.ready(); + } + + // Config::DataFetcher::RemoteDataFetcherCallback + void onFailure(Config::DataFetcher::FailureReason failure) override { + if (allow_empty_) { + callback_(EMPTY_STRING); + init_target_.ready(); + } else { + throw EnvoyException( + fmt::format("Failed to fetch remote data. Failure reason: {}", enumToInt(failure))); + } + } + +private: + void start() { fetcher_->fetch(); } + + bool allow_empty_; + AsyncDataSourceCb callback_; + const Config::DataFetcher::RemoteDataFetcherPtr fetcher_; + Init::TargetImpl init_target_; +}; + +using RemoteAsyncDataProviderPtr = std::unique_ptr; + } // namespace DataSource } // namespace Config } // namespace Envoy diff --git a/source/common/config/remote_data_fetcher.cc b/source/common/config/remote_data_fetcher.cc new file mode 100644 index 000000000000..8e79e35fc8d9 --- /dev/null +++ b/source/common/config/remote_data_fetcher.cc @@ -0,0 +1,75 @@ +#include "common/config/remote_data_fetcher.h" + +#include "common/common/enum_to_int.h" +#include "common/common/hex.h" +#include "common/crypto/utility.h" +#include "common/http/headers.h" +#include "common/http/utility.h" + +namespace Envoy { +namespace Config { +namespace DataFetcher { + +RemoteDataFetcher::RemoteDataFetcher(Upstream::ClusterManager& cm, + const ::envoy::api::v2::core::HttpUri& uri, + const std::string& content_hash, + RemoteDataFetcherCallback& callback) + : cm_(cm), uri_(uri), content_hash_(content_hash), callback_(callback) {} + +RemoteDataFetcher::~RemoteDataFetcher() { cancel(); } + +void RemoteDataFetcher::cancel() { + if (request_) { + request_->cancel(); + ENVOY_LOG(debug, "fetch remote data [uri = {}]: canceled", uri_.uri()); + } + + request_ = nullptr; +} + +void RemoteDataFetcher::fetch() { + Http::MessagePtr message = Http::Utility::prepareHeaders(uri_); + message->headers().insertMethod().value().setReference(Http::Headers::get().MethodValues.Get); + ENVOY_LOG(debug, "fetch remote data from [uri = {}]: start", uri_.uri()); + request_ = cm_.httpAsyncClientForCluster(uri_.cluster()) + .send(std::move(message), *this, + Http::AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds( + DurationUtil::durationToMilliseconds(uri_.timeout())))); +} + +void RemoteDataFetcher::onSuccess(Http::MessagePtr&& response) { + const uint64_t status_code = Http::Utility::getResponseStatus(response->headers()); + if (status_code == enumToInt(Http::Code::OK)) { + ENVOY_LOG(debug, "fetch remote data [uri = {}]: success", uri_.uri()); + if (response->body()) { + const auto content_hash = + Hex::encode(Envoy::Common::Crypto::Utility::getSha256Digest(*response->body())); + + if (content_hash_ != content_hash) { + ENVOY_LOG(debug, "fetch remote data [uri = {}]: data is invalid", uri_.uri()); + callback_.onFailure(FailureReason::InvalidData); + } else { + callback_.onSuccess(response->body()->toString()); + } + } else { + ENVOY_LOG(debug, "fetch remote data [uri = {}]: body is empty", uri_.uri()); + callback_.onFailure(FailureReason::Network); + } + } else { + ENVOY_LOG(debug, "fetch remote data [uri = {}]: response status code {}", uri_.uri(), + status_code); + callback_.onFailure(FailureReason::Network); + } + + request_ = nullptr; +} + +void RemoteDataFetcher::onFailure(Http::AsyncClient::FailureReason reason) { + ENVOY_LOG(debug, "fetch remote data [uri = {}]: network error {}", uri_.uri(), enumToInt(reason)); + request_ = nullptr; + callback_.onFailure(FailureReason::Network); +} + +} // namespace DataFetcher +} // namespace Config +} // namespace Envoy \ No newline at end of file diff --git a/source/common/config/remote_data_fetcher.h b/source/common/config/remote_data_fetcher.h new file mode 100644 index 000000000000..687dfe8a7ef9 --- /dev/null +++ b/source/common/config/remote_data_fetcher.h @@ -0,0 +1,82 @@ +#pragma once + +#include "envoy/api/v2/core/http_uri.pb.h" +#include "envoy/common/pure.h" +#include "envoy/upstream/cluster_manager.h" + +namespace Envoy { +namespace Config { +namespace DataFetcher { + +/** + * Failure reason. + */ +enum class FailureReason { + /* A network error occurred causing remote data retrieval failure. */ + Network, + /* A failure occurred when trying to verify remote data using sha256. */ + InvalidData, +}; + +/** + * Callback used by remote data fetcher. + */ +class RemoteDataFetcherCallback { +public: + virtual ~RemoteDataFetcherCallback() = default; + + /** + * This function will be called when data is fetched successfully from remote. + * @param data remote data + */ + virtual void onSuccess(const std::string& data) PURE; + + /** + * This function is called when error happens during fetching data. + * @param reason failure reason. + */ + virtual void onFailure(FailureReason reason) PURE; +}; + +/** + * Remote data fetcher. + */ +class RemoteDataFetcher : public Logger::Loggable, + public Http::AsyncClient::Callbacks { +public: + RemoteDataFetcher(Upstream::ClusterManager& cm, const ::envoy::api::v2::core::HttpUri& uri, + const std::string& content_hash, RemoteDataFetcherCallback& callback); + + virtual ~RemoteDataFetcher() override; + + // Http::AsyncClient::Callbacks + void onSuccess(Http::MessagePtr&& response) override; + void onFailure(Http::AsyncClient::FailureReason reason) override; + + /** + * Fetch data from remote. + * @param uri remote URI + * @param content_hash for verifying data integrity + * @param callback callback when fetch is done. + */ + void fetch(); + + /** + * Cancel the fetch. + */ + void cancel(); + +private: + Upstream::ClusterManager& cm_; + const envoy::api::v2::core::HttpUri& uri_; + const std::string content_hash_; + RemoteDataFetcherCallback& callback_; + + Http::AsyncClient::Request* request_{}; +}; + +using RemoteDataFetcherPtr = std::unique_ptr; + +} // namespace DataFetcher +} // namespace Config +} // namespace Envoy \ No newline at end of file diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 7e254c5ee8c7..ebc6d79e5330 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -301,3 +301,17 @@ envoy_cc_test( "//test/test_common:simulated_time_system_lib", ], ) + +envoy_cc_test( + name = "datasource_test", + srcs = ["datasource_test.cc"], + deps = [ + "//source/common/common:empty_string", + "//source/common/config:datasource_lib", + "//source/common/protobuf:utility_lib", + "//test/mocks/server:server_mocks", + "//test/mocks/upstream:upstream_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2/core:base_cc", + ], +) diff --git a/test/common/config/datasource_test.cc b/test/common/config/datasource_test.cc new file mode 100644 index 000000000000..4621c2e521db --- /dev/null +++ b/test/common/config/datasource_test.cc @@ -0,0 +1,379 @@ +#include "envoy/api/v2/core/base.pb.h" +#include "envoy/api/v2/core/base.pb.validate.h" + +#include "common/common/empty_string.h" +#include "common/config/datasource.h" +#include "common/protobuf/protobuf.h" + +#include "test/mocks/server/mocks.h" +#include "test/mocks/upstream/mocks.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Config { +namespace { + +class AsyncDataSourceTest : public testing::Test { +protected: + AsyncDataSourceTest() : api_(Api::createApiForTest()) {} + + using AsyncDataSourcePb = envoy::api::v2::core::AsyncDataSource; + + NiceMock cm_; + Init::MockManager init_manager_; + Init::ExpectableWatcherImpl init_watcher_; + Init::TargetHandlePtr init_target_handle_; + Api::ApiPtr api_; +}; + +TEST_F(AsyncDataSourceTest, loadLocalDataSource) { + AsyncDataSourcePb config; + + std::string yaml = R"EOF( + local: + inline_string: + xxxxxx + )EOF"; + TestUtility::loadFromYaml(yaml, config); + EXPECT_TRUE(config.has_local()); + + std::string async_data; + + EXPECT_CALL(init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + auto provider = std::make_unique( + init_manager_, config.local(), true, *api_, [&](const std::string& data) { + EXPECT_EQ(init_manager_.state(), Init::Manager::State::Initializing); + EXPECT_EQ(data, "xxxxxx"); + async_data = data; + }); + + EXPECT_CALL(init_manager_, state()).WillOnce(Return(Init::Manager::State::Initializing)); + EXPECT_CALL(init_watcher_, ready()); + + init_target_handle_->initialize(init_watcher_); + + EXPECT_EQ(async_data, "xxxxxx"); + EXPECT_NE(nullptr, provider.get()); +} + +TEST_F(AsyncDataSourceTest, loadRemoteDataSourceReturnFailure) { + AsyncDataSourcePb config; + + std::string yaml = R"EOF( + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + sha256: + xxxxxx + )EOF"; + TestUtility::loadFromYaml(yaml, config); + EXPECT_TRUE(config.has_remote()); + + EXPECT_CALL(cm_, httpAsyncClientForCluster("cluster_1")).WillOnce(ReturnRef(cm_.async_client_)); + EXPECT_CALL(cm_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::MessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callbacks.onFailure(Envoy::Http::AsyncClient::FailureReason::Reset); + return nullptr; + })); + + EXPECT_CALL(init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + std::string async_data = "non-empty"; + auto provider = std::make_unique( + cm_, init_manager_, config.remote(), true, [&](const std::string& data) { + EXPECT_EQ(init_manager_.state(), Init::Manager::State::Initializing); + EXPECT_EQ(data, EMPTY_STRING); + async_data = data; + }); + + EXPECT_CALL(init_manager_, state()).WillOnce(Return(Init::Manager::State::Initializing)); + EXPECT_CALL(init_watcher_, ready()); + + init_target_handle_->initialize(init_watcher_); + + EXPECT_EQ(async_data, EMPTY_STRING); + EXPECT_NE(nullptr, provider.get()); +} + +TEST_F(AsyncDataSourceTest, loadRemoteDataSourceSuccessWith503) { + AsyncDataSourcePb config; + + std::string yaml = R"EOF( + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + sha256: + xxxxxx + )EOF"; + TestUtility::loadFromYaml(yaml, config); + EXPECT_TRUE(config.has_remote()); + + EXPECT_CALL(cm_, httpAsyncClientForCluster("cluster_1")).WillOnce(ReturnRef(cm_.async_client_)); + EXPECT_CALL(cm_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::MessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callbacks.onSuccess(Http::MessagePtr{new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "503"}}})}); + return nullptr; + })); + + EXPECT_CALL(init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + std::string async_data = "non-empty"; + auto provider = std::make_unique( + cm_, init_manager_, config.remote(), true, [&](const std::string& data) { + EXPECT_EQ(init_manager_.state(), Init::Manager::State::Initializing); + EXPECT_EQ(data, EMPTY_STRING); + async_data = data; + }); + + EXPECT_CALL(init_manager_, state()).WillOnce(Return(Init::Manager::State::Initializing)); + EXPECT_CALL(init_watcher_, ready()); + + init_target_handle_->initialize(init_watcher_); + EXPECT_EQ(async_data, EMPTY_STRING); + EXPECT_NE(nullptr, provider.get()); +} + +TEST_F(AsyncDataSourceTest, loadRemoteDataSourceSuccessWithEmptyBody) { + AsyncDataSourcePb config; + + std::string yaml = R"EOF( + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + sha256: + xxxxxx + )EOF"; + TestUtility::loadFromYaml(yaml, config); + EXPECT_TRUE(config.has_remote()); + + EXPECT_CALL(cm_, httpAsyncClientForCluster("cluster_1")).WillOnce(ReturnRef(cm_.async_client_)); + EXPECT_CALL(cm_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::MessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callbacks.onSuccess(Http::MessagePtr{new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})}); + return nullptr; + })); + + EXPECT_CALL(init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + std::string async_data = "non-empty"; + auto provider = std::make_unique( + cm_, init_manager_, config.remote(), true, [&](const std::string& data) { + EXPECT_EQ(init_manager_.state(), Init::Manager::State::Initializing); + EXPECT_EQ(data, EMPTY_STRING); + async_data = data; + }); + + EXPECT_CALL(init_manager_, state()).WillOnce(Return(Init::Manager::State::Initializing)); + EXPECT_CALL(init_watcher_, ready()); + + init_target_handle_->initialize(init_watcher_); + + EXPECT_EQ(async_data, EMPTY_STRING); + EXPECT_NE(nullptr, provider.get()); +} + +TEST_F(AsyncDataSourceTest, loadRemoteDataSourceSuccessIncorrectSha256) { + AsyncDataSourcePb config; + + std::string yaml = R"EOF( + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + sha256: + xxxxxx + )EOF"; + TestUtility::loadFromYaml(yaml, config); + EXPECT_TRUE(config.has_remote()); + + const std::string body = "hello world"; + + EXPECT_CALL(cm_, httpAsyncClientForCluster("cluster_1")).WillOnce(ReturnRef(cm_.async_client_)); + EXPECT_CALL(cm_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::MessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + Http::MessagePtr response(new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); + response->body() = std::make_unique(body); + + callbacks.onSuccess(std::move(response)); + return nullptr; + })); + + EXPECT_CALL(init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + std::string async_data = "non-empty"; + auto provider = std::make_unique( + cm_, init_manager_, config.remote(), true, [&](const std::string& data) { + EXPECT_EQ(init_manager_.state(), Init::Manager::State::Initializing); + EXPECT_EQ(data, EMPTY_STRING); + async_data = data; + }); + + EXPECT_CALL(init_manager_, state()).WillOnce(Return(Init::Manager::State::Initializing)); + EXPECT_CALL(init_watcher_, ready()); + + init_target_handle_->initialize(init_watcher_); + EXPECT_EQ(async_data, EMPTY_STRING); + EXPECT_NE(nullptr, provider.get()); +} + +TEST_F(AsyncDataSourceTest, loadRemoteDataSourceSuccess) { + AsyncDataSourcePb config; + + std::string yaml = R"EOF( + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + sha256: + b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 + )EOF"; + TestUtility::loadFromYaml(yaml, config); + EXPECT_TRUE(config.has_remote()); + + const std::string body = "hello world"; + + EXPECT_CALL(cm_, httpAsyncClientForCluster("cluster_1")).WillOnce(ReturnRef(cm_.async_client_)); + EXPECT_CALL(cm_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::MessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + Http::MessagePtr response(new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); + response->body() = std::make_unique(body); + + callbacks.onSuccess(std::move(response)); + return nullptr; + })); + + EXPECT_CALL(init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + std::string async_data = "non-empty"; + auto provider = std::make_unique( + cm_, init_manager_, config.remote(), true, [&](const std::string& data) { + EXPECT_EQ(init_manager_.state(), Init::Manager::State::Initializing); + EXPECT_EQ(data, body); + async_data = data; + }); + + EXPECT_CALL(init_manager_, state()).WillOnce(Return(Init::Manager::State::Initializing)); + EXPECT_CALL(init_watcher_, ready()); + + init_target_handle_->initialize(init_watcher_); + EXPECT_EQ(async_data, body); + EXPECT_NE(nullptr, provider.get()); +} + +TEST_F(AsyncDataSourceTest, loadRemoteDataSourceExpectNetworkFailure) { + AsyncDataSourcePb config; + + std::string yaml = R"EOF( + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + sha256: + xxxxxx + )EOF"; + TestUtility::loadFromYaml(yaml, config); + EXPECT_TRUE(config.has_remote()); + + EXPECT_CALL(cm_, httpAsyncClientForCluster("cluster_1")).WillOnce(ReturnRef(cm_.async_client_)); + EXPECT_CALL(cm_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::MessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callbacks.onSuccess(Http::MessagePtr{new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "503"}}})}); + return nullptr; + })); + + EXPECT_CALL(init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + auto provider = std::make_unique( + cm_, init_manager_, config.remote(), false, [](const std::string&) {}); + + EXPECT_THROW_WITH_MESSAGE(init_target_handle_->initialize(init_watcher_), EnvoyException, + "Failed to fetch remote data. Failure reason: 0"); + EXPECT_NE(nullptr, provider.get()); + EXPECT_CALL(init_watcher_, ready()); +} + +TEST_F(AsyncDataSourceTest, loadRemoteDataSourceExpectInvalidData) { + AsyncDataSourcePb config; + + std::string yaml = R"EOF( + remote: + http_uri: + uri: https://example.com/data + cluster: cluster_1 + sha256: + xxxxxx + )EOF"; + TestUtility::loadFromYaml(yaml, config); + EXPECT_TRUE(config.has_remote()); + + const std::string body = "hello world"; + + EXPECT_CALL(cm_, httpAsyncClientForCluster("cluster_1")).WillOnce(ReturnRef(cm_.async_client_)); + EXPECT_CALL(cm_.async_client_, send_(_, _, _)) + .WillOnce( + Invoke([&](Http::MessagePtr&, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + Http::MessagePtr response(new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); + response->body() = std::make_unique(body); + + callbacks.onSuccess(std::move(response)); + return nullptr; + })); + + EXPECT_CALL(init_manager_, add(_)).WillOnce(Invoke([this](const Init::Target& target) { + init_target_handle_ = target.createHandle("test"); + })); + + auto provider = std::make_unique( + cm_, init_manager_, config.remote(), false, [](const std::string&) {}); + + EXPECT_THROW_WITH_MESSAGE(init_target_handle_->initialize(init_watcher_), EnvoyException, + "Failed to fetch remote data. Failure reason: 1"); + EXPECT_NE(nullptr, provider.get()); + EXPECT_CALL(init_watcher_, ready()); +} + +} // namespace +} // namespace Config +} // namespace Envoy \ No newline at end of file From 71e36ad8b24b829a2923d06486978fe46e2fd026 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 16 Jul 2019 01:23:03 -0700 Subject: [PATCH 179/542] Revert "ci: Enable bazel limited download flags (#7563)" (#7593) This reverts commit 32bfd01da2a95c51e4ae18d134449ceb0123efb8. Signed-off-by: Keith Smiley --- ci/setup_cache.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ci/setup_cache.sh b/ci/setup_cache.sh index f97aa9b34963..3e10133e717d 100755 --- a/ci/setup_cache.sh +++ b/ci/setup_cache.sh @@ -17,13 +17,6 @@ if [[ ! -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then echo "${GCP_SERVICE_ACCOUNT_KEY}" | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_FILE}" fi -if [[ -n "${BAZEL_REMOTE_CACHE}" ]]; then - export BAZEL_BUILD_EXTRA_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS} \ - --experimental_inmemory_dotd_files \ - --experimental_inmemory_jdeps_files \ - --experimental_remote_download_outputs=toplevel" -fi - if [[ "${BAZEL_REMOTE_CACHE}" =~ ^http ]]; then if [[ ! -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then export BAZEL_BUILD_EXTRA_OPTIONS="${BAZEL_BUILD_EXTRA_OPTIONS} \ From 29a1620251f44e400348a88a0048b5be3ffc738f Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 16 Jul 2019 09:13:11 -0400 Subject: [PATCH 180/542] test: adding tests for set-cookie being handled correctly (#7541) Risk Level: n/a (test only) Testing: yes Docs Changes: no Release Notes: no #7488 Signed-off-by: Alyssa Wilk --- source/common/http/header_utility.cc | 16 ++++++++++++++ source/common/http/header_utility.h | 11 ++++++++++ test/common/http/BUILD | 1 + test/common/http/header_map_impl_test.cc | 19 ++++++++++++++++ test/common/http/header_utility_test.cc | 20 +++++++++++++++++ test/common/router/header_formatter_test.cc | 17 ++++++++++++++ test/integration/protocol_integration_test.cc | 22 +++++++++++++++++++ 7 files changed, 106 insertions(+) diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc index 4cecc97178e7..b6af8ab75f00 100644 --- a/source/common/http/header_utility.cc +++ b/source/common/http/header_utility.cc @@ -65,6 +65,22 @@ HeaderUtility::HeaderData::HeaderData(const Json::Object& config) return header_matcher; }()) {} +void HeaderUtility::getAllOfHeader(const Http::HeaderMap& headers, absl::string_view key, + std::vector& out) { + auto args = std::make_pair(LowerCaseString(std::string(key)), &out); + + headers.iterate( + [](const HeaderEntry& header, void* context) -> Envoy::Http::HeaderMap::Iterate { + auto key_ret = + static_cast*>*>(context); + if (header.key() == key_ret->first.get().c_str()) { + key_ret->second->emplace_back(header.value().getStringView()); + } + return Envoy::Http::HeaderMap::Iterate::Continue; + }, + &args); +} + bool HeaderUtility::matchHeaders(const Http::HeaderMap& request_headers, const std::vector& config_headers) { // No headers to match is considered a match. diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h index d2b552da51fa..7d3e13853059 100644 --- a/source/common/http/header_utility.h +++ b/source/common/http/header_utility.h @@ -18,6 +18,17 @@ class HeaderUtility { public: enum class HeaderMatchType { Value, Regex, Range, Present, Prefix, Suffix }; + /* Get all instances of the header key specified, and return the values in the vector provided. + * + * This should not be used for inline headers, as it turns a constant time lookup into O(n). + * + * @param headers the headers to return keys from + * @param key the header key to return values for + * @param out the vector to return values in + */ + static void getAllOfHeader(const Http::HeaderMap& headers, absl::string_view key, + std::vector& out); + // A HeaderData specifies one of exact value or regex or range element // to match in a request's header, specified in the header_match_type_ member. // It is the runtime equivalent of the HeaderMatchSpecifier proto in RDS API. diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 4f5ac9cad9f9..5cfeaff671fa 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -252,6 +252,7 @@ envoy_cc_test( srcs = ["header_map_impl_test.cc"], deps = [ "//source/common/http:header_map_lib", + "//source/common/http:header_utility_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/common/http/header_map_impl_test.cc b/test/common/http/header_map_impl_test.cc index da9dc1cccf7b..5ec8f8ab5f28 100644 --- a/test/common/http/header_map_impl_test.cc +++ b/test/common/http/header_map_impl_test.cc @@ -2,6 +2,7 @@ #include #include "common/http/header_map_impl.h" +#include "common/http/header_utility.h" #include "test/test_common/printers.h" #include "test/test_common/utility.h" @@ -557,6 +558,24 @@ TEST(HeaderMapImplTest, DoubleInlineAdd) { } } +// Per https://github.com/envoyproxy/envoy/issues/7488 make sure we don't +// combine set-cookie headers +TEST(HeaderMapImplTest, DoubleCookieAdd) { + HeaderMapImpl headers; + const std::string foo("foo"); + const std::string bar("bar"); + const LowerCaseString& set_cookie = Http::Headers::get().SetCookie; + headers.addReference(set_cookie, foo); + headers.addReference(set_cookie, bar); + EXPECT_EQ(2UL, headers.size()); + + std::vector out; + Http::HeaderUtility::getAllOfHeader(headers, "set-cookie", out); + ASSERT_EQ(out.size(), 2); + ASSERT_EQ(out[0], "foo"); + ASSERT_EQ(out[1], "bar"); +} + TEST(HeaderMapImplTest, DoubleInlineSet) { HeaderMapImpl headers; headers.setReferenceKey(Headers::get().ContentType, "blah"); diff --git a/test/common/http/header_utility_test.cc b/test/common/http/header_utility_test.cc index a20f4ca43f49..97f6b95b34b6 100644 --- a/test/common/http/header_utility_test.cc +++ b/test/common/http/header_utility_test.cc @@ -147,6 +147,26 @@ invert_match: true EXPECT_EQ(true, header_data.invert_match_); } +TEST(HeaderDataConstructorTest, GetAllOfHeader) { + TestHeaderMapImpl headers{{"foo", "val1"}, {"bar", "bar2"}, {"foo", "eep, bar"}, {"foo", ""}}; + + std::vector foo_out; + Http::HeaderUtility::getAllOfHeader(headers, "foo", foo_out); + ASSERT_EQ(foo_out.size(), 3); + ASSERT_EQ(foo_out[0], "val1"); + ASSERT_EQ(foo_out[1], "eep, bar"); + ASSERT_EQ(foo_out[2], ""); + + std::vector bar_out; + Http::HeaderUtility::getAllOfHeader(headers, "bar", bar_out); + ASSERT_EQ(bar_out.size(), 1); + ASSERT_EQ(bar_out[0], "bar2"); + + std::vector eep_out; + Http::HeaderUtility::getAllOfHeader(headers, "eep", eep_out); + ASSERT_EQ(eep_out.size(), 0); +} + TEST(MatchHeadersTest, MayMatchOneOrMoreRequestHeader) { TestHeaderMapImpl headers{{"some-header", "a"}, {"other-header", "b"}}; diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index e0820fedf79e..3ab5ead740d7 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -1068,6 +1068,14 @@ match: { prefix: "/new_endpoint" } key: "x-request-start-default" value: "%START_TIME%" append: true + - header: + key: "set-cookie" + value: "foo" + - header: + key: "set-cookie" + value: "bar" + append: true + response_headers_to_remove: ["x-nope"] )EOF"; @@ -1097,6 +1105,15 @@ response_headers_to_remove: ["x-nope"] EXPECT_TRUE(header_map.has("x-request-start-range")); EXPECT_EQ("123456000, 1, 12, 123, 1234, 12345, 123456, 1234560, 12345600, 123456000", header_map.get_("x-request-start-range")); + EXPECT_EQ("foo", header_map.get_("set-cookie")); + + // Per https://github.com/envoyproxy/envoy/issues/7488 make sure we don't + // combine set-cookie headers + std::vector out; + Http::HeaderUtility::getAllOfHeader(header_map, "set-cookie", out); + ASSERT_EQ(out.size(), 2); + ASSERT_EQ(out[0], "foo"); + ASSERT_EQ(out[1], "bar"); } TEST(HeaderParserTest, EvaluateRequestHeadersRemoveBeforeAdd) { diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 1e7c6bcb0b72..ae7801e4176e 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -1020,6 +1020,28 @@ name: encode-headers-return-stop-all-filter EXPECT_EQ(count_ * size_ + added_decoded_data_size_, response->body().size()); } +// Per https://github.com/envoyproxy/envoy/issues/7488 make sure we don't +// combine set-cookie headers +TEST_P(ProtocolIntegrationTest, MultipleSetCookies) { + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + Http::TestHeaderMapImpl response_headers{ + {":status", "200"}, {"set-cookie", "foo"}, {"set-cookie", "bar"}}; + + auto response = sendRequestAndWaitForResponse(default_request_headers_, 0, response_headers, 0); + + ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + + std::vector out; + Http::HeaderUtility::getAllOfHeader(response->headers(), "set-cookie", out); + ASSERT_EQ(out.size(), 2); + ASSERT_EQ(out[0], "foo"); + ASSERT_EQ(out[1], "bar"); +} + // For tests which focus on downstream-to-Envoy behavior, and don't need to be // run with both HTTP/1 and HTTP/2 upstreams. INSTANTIATE_TEST_SUITE_P(Protocols, DownstreamProtocolIntegrationTest, From cd620ad1227d5ef5908036468e4212a4b1f22666 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 16 Jul 2019 06:48:12 -0700 Subject: [PATCH 181/542] coverage: use LLVM and bazel native coverage (#7552) Signed-off-by: Lizan Zhou --- DEVELOPER.md | 4 +- bazel/README.md | 6 +- bazel/external/gcovr.BUILD | 24 ----- bazel/repositories.bzl | 14 --- bazel/repository_locations.bzl | 12 --- ci/build_setup.sh | 15 --- ci/do_ci.sh | 17 +--- test/coverage/gcc_only_test/BUILD | 16 --- test/coverage/gcc_only_test/gcc_only_test.cc | 12 --- test/coverage/gen_build.sh | 7 -- test/run_envoy_bazel_coverage.sh | 102 ++++--------------- 11 files changed, 27 insertions(+), 202 deletions(-) delete mode 100644 bazel/external/gcovr.BUILD delete mode 100644 test/coverage/gcc_only_test/BUILD delete mode 100644 test/coverage/gcc_only_test/gcc_only_test.cc diff --git a/DEVELOPER.md b/DEVELOPER.md index ee76c5746310..465644c0e02c 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -2,7 +2,9 @@ Envoy is built using the Bazel build system. CircleCI builds, tests, and runs coverage against all pull requests and the master branch. -To get started building Envoy locally, see the [Bazel quick start](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#quick-start-bazel-build-for-developers). To run tests, there are Bazel [targets](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#testing-envoy-with-bazel) for Google Test. To generate a coverage report, use the tooling for [gcovr](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#coverage-builds). +To get started building Envoy locally, see the [Bazel quick start](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#quick-start-bazel-build-for-developers). +To run tests, there are Bazel [targets](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#testing-envoy-with-bazel) for Google Test. +To generate a coverage report, there is a [coverage build script](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#coverage-builds). If you plan to contribute to Envoy, you may find it useful to install the Envoy [development support toolchain](https://github.com/envoyproxy/envoy/blob/master/support/README.md), which helps automate parts of the development process, particularly those involving code review. diff --git a/bazel/README.md b/bazel/README.md index 25743ffdc097..657177dbccfd 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -435,10 +435,8 @@ https://github.com/bazelbuild/bazel/issues/2805. # Coverage builds -To generate coverage results, make sure you have -[`gcovr`](https://github.com/gcovr/gcovr) 3.3 in your `PATH` (or set `GCOVR` to -point at it) and are using a GCC toolchain (clang does not work currently, see -https://github.com/envoyproxy/envoy/issues/1000). Then run: +To generate coverage results, make sure you are using a clang toolchain and have `llvm-cov` and +`llvm-profdata` in your `PATH`. Then run: ``` test/run_envoy_bazel_coverage.sh diff --git a/bazel/external/gcovr.BUILD b/bazel/external/gcovr.BUILD deleted file mode 100644 index 0215a25da848..000000000000 --- a/bazel/external/gcovr.BUILD +++ /dev/null @@ -1,24 +0,0 @@ -licenses(["notice"]) # Apache 2 - -load("@subpar//:subpar.bzl", "par_binary") - -# gcovr is difficult to run from a CI environment because it has hard -# assumptions about its local working directory, which interact poorly -# with `bazel run`. To make gcovr more mobile, we package it into a -# .par file (a mostly-hermetic "Python binary"). -par_binary( - name = "gcovr", - srcs = [":renamed_gcovr.py"], - main = ":renamed_gcovr.py", - python_version = "PY2", - visibility = ["//visibility:public"], -) - -# par_binary expects its `srcs` to contain only *.py files, but gcovr is -# distributed as a script with no filename extension. Rename it here. -genrule( - name = "gcovr_to_exec_py", - srcs = ["scripts/gcovr"], - outs = ["renamed_gcovr.py"], - cmd = "cat $(location scripts/gcovr) > $(location renamed_gcovr.py)", -) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index ed14486d3a5e..027d69baa27f 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -140,7 +140,6 @@ def envoy_dependencies(skip_targets = []): _com_github_envoyproxy_sqlparser() _com_github_fmtlib_fmt() _com_github_gabime_spdlog() - _com_github_gcovr_gcovr() _com_github_google_benchmark() _com_github_google_jwt_verify() _com_github_google_libprotobuf_mutator() @@ -164,9 +163,6 @@ def envoy_dependencies(skip_targets = []): _io_opentracing_cpp() _net_zlib() - # Used for bundling gcovr into a relocatable .par file. - _repository_impl("subpar") - _python_deps() _cc_deps() _go_deps(skip_targets) @@ -257,16 +253,6 @@ def _com_github_gabime_spdlog(): actual = "@com_github_gabime_spdlog//:spdlog", ) -def _com_github_gcovr_gcovr(): - _repository_impl( - name = "com_github_gcovr_gcovr", - build_file = "@envoy//bazel/external:gcovr.BUILD", - ) - native.bind( - name = "gcovr", - actual = "@com_github_gcovr_gcovr//:gcovr", - ) - def _com_github_google_benchmark(): location = REPOSITORY_LOCATIONS["com_github_google_benchmark"] http_archive( diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 383f40e90c60..d809a6f87192 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -66,11 +66,6 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "spdlog-1.3.1", urls = ["https://github.com/gabime/spdlog/archive/v1.3.1.tar.gz"], ), - com_github_gcovr_gcovr = dict( - sha256 = "1c52a71f245adfe1b45e30fbe5015337fe66546f17f40038b3969b7b42acceed", - strip_prefix = "gcovr-3.4", - urls = ["https://github.com/gcovr/gcovr/archive/3.4.tar.gz"], - ), com_github_google_libprotobuf_mutator = dict( sha256 = "97b3639630040f41c45f45838ab00b78909e6b4cb69c8028e01302bea5b79495", strip_prefix = "libprotobuf-mutator-c3d2faf04a1070b0b852b0efdef81e1a81ba925e", @@ -229,13 +224,6 @@ REPOSITORY_LOCATIONS = dict( sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", urls = ["https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55"], ), - # I'd love to name this `com_github_google_subpar`, but something in the Subpar - # code assumes its repository name is just `subpar`. - subpar = dict( - sha256 = "b80297a1b8d38027a86836dbadc22f55dc3ecad56728175381aa6330705ac10f", - strip_prefix = "subpar-2.0.0", - urls = ["https://github.com/google/subpar/archive/2.0.0.tar.gz"], - ), io_opencensus_cpp = dict( sha256 = "d6d68704c419a9e892bd1f942e09509ebc5a318499a1abcf2c09734e5dc56e19", strip_prefix = "opencensus-cpp-1145dd77ffb7a2845c71c8e6ca188ef55e4ff607", diff --git a/ci/build_setup.sh b/ci/build_setup.sh index a8069429be0d..8323eb744011 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -40,21 +40,6 @@ then fi export ENVOY_FILTER_EXAMPLE_SRCDIR="${BUILD_DIR}/envoy-filter-example" -# Make sure that /source doesn't contain /build on the underlying host -# filesystem, including via hard links or symlinks. We can get into weird -# loops with Bazel symlinking and gcovr's path traversal if this is true, so -# best to keep /source and /build in distinct directories on the host -# filesystem. -SENTINEL="${BUILD_DIR}"/bazel.sentinel -touch "${SENTINEL}" -if [[ -n "$(find -L "${ENVOY_SRCDIR}" -name "$(basename "${SENTINEL}")")" ]] -then - rm -f "${SENTINEL}" - echo "/source mount must not contain /build mount" - exit 1 -fi -rm -f "${SENTINEL}" - # Environment setup. export USER=bazel export TEST_TMPDIR=${BUILD_DIR}/tmp diff --git a/ci/do_ci.sh b/ci/do_ci.sh index ec4d05496390..d9a16221574a 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -241,22 +241,11 @@ elif [[ "$CI_TARGET" == "bazel.api" ]]; then @envoy_api//tools:tap2pcap_test exit 0 elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then - setup_gcc_toolchain + setup_clang_toolchain echo "bazel coverage build with tests ${TEST_TARGETS}" - # gcovr is a pain to run with `bazel run`, so package it up into a - # relocatable and hermetic-ish .par file. - bazel build --python_version=PY2 @com_github_gcovr_gcovr//:gcovr.par - export GCOVR="/tmp/gcovr.par" - cp -f "${ENVOY_SRCDIR}/bazel-bin/external/com_github_gcovr_gcovr/gcovr.par" ${GCOVR} - - # Reduce the amount of memory and number of cores Bazel tries to use to - # prevent it from launching too many subprocesses. This should prevent the - # system from running out of memory and killing tasks. See discussion on - # https://github.com/envoyproxy/envoy/pull/5611. - # TODO(akonradi): use --local_cpu_resources flag once Bazel has a release - # after 0.21. - [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_resources=12288,4,1" + # LLVM coverage is a memory hog too. + [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_cpu_resources=6" test/run_envoy_bazel_coverage.sh ${TEST_TARGETS} collect_build_profile coverage diff --git a/test/coverage/gcc_only_test/BUILD b/test/coverage/gcc_only_test/BUILD deleted file mode 100644 index 1482a741b3e3..000000000000 --- a/test/coverage/gcc_only_test/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -licenses(["notice"]) # Apache 2 - -load( - "//bazel:envoy_build_system.bzl", - "envoy_cc_test", - "envoy_package", -) - -envoy_package() - -envoy_cc_test( - name = "gcc_only_test", - srcs = ["gcc_only_test.cc"], - coverage = False, - tags = ["manual"], -) diff --git a/test/coverage/gcc_only_test/gcc_only_test.cc b/test/coverage/gcc_only_test/gcc_only_test.cc deleted file mode 100644 index 31346323d20e..000000000000 --- a/test/coverage/gcc_only_test/gcc_only_test.cc +++ /dev/null @@ -1,12 +0,0 @@ -#include "gtest/gtest.h" - -namespace Envoy { - -TEST(GccOnly, CompilerCheck) { -#if defined(__clang__) or not defined(__GNUC__) - // clang is incompatible with gcov. - FAIL() << "GCC is required for coverage runs"; -#endif -} - -} // namespace Envoy diff --git a/test/coverage/gen_build.sh b/test/coverage/gen_build.sh index 5532e78a5e95..5998e3eefd49 100755 --- a/test/coverage/gen_build.sh +++ b/test/coverage/gen_build.sh @@ -44,13 +44,6 @@ if [ -n "${EXTRA_QUERY_PATHS}" ]; then TARGETS="$TARGETS $("${BAZEL_BIN}" query ${BAZEL_QUERY_OPTIONS} "attr('tags', 'coverage_test_lib', ${EXTRA_QUERY_PATHS})" | grep "^//")" fi -# gcov requires gcc -if [ "${NO_GCOV}" != 1 ] -then - # Here we use the synthetic library target created by envoy_build_system.bzl - TARGETS="${TARGETS} ${REPOSITORY}//test/coverage/gcc_only_test:gcc_only_test_lib_internal_only" -fi - ( cat << EOF # This file is generated by test/coverage/gen_build.sh automatically prior to diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 04499887105d..a3542492ef2c 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -3,21 +3,11 @@ set -e [[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" -[[ -z "${GCOVR_DIR}" ]] && GCOVR_DIR="${SRCDIR}/bazel-$(basename "${SRCDIR}")" -[[ -z "${TESTLOGS_DIR}" ]] && TESTLOGS_DIR="${SRCDIR}/bazel-testlogs" -[[ -z "${BAZEL_COVERAGE}" ]] && BAZEL_COVERAGE=bazel -[[ -z "${GCOVR}" ]] && GCOVR=gcovr -[[ -z "${WORKSPACE}" ]] && WORKSPACE=envoy [[ -z "${VALIDATE_COVERAGE}" ]] && VALIDATE_COVERAGE=true echo "Starting run_envoy_bazel_coverage.sh..." echo " PWD=$(pwd)" echo " SRCDIR=${SRCDIR}" -echo " GCOVR_DIR=${GCOVR_DIR}" -echo " TESTLOGS_DIR=${TESTLOGS_DIR}" -echo " BAZEL_COVERAGE=${BAZEL_COVERAGE}" -echo " GCOVR=${GCOVR}" -echo " WORKSPACE=${WORKSPACE}" echo " VALIDATE_COVERAGE=${VALIDATE_COVERAGE}" # This is the target that will be run to generate coverage data. It can be overridden by consumer @@ -31,91 +21,37 @@ else COVERAGE_TARGETS=//test/... fi -# This is where we are going to copy the .gcno files into. -GCNO_ROOT=bazel-out/k8-dbg/bin/test/coverage/coverage_tests.runfiles/"${WORKSPACE}" -echo " GCNO_ROOT=${GCNO_ROOT}" - -echo "Cleaning .gcno from previous coverage runs..." -NUM_PREVIOUS_GCNO_FILES=0 -for f in $(find -L "${GCNO_ROOT}" -name "*.gcno") -do - rm -f "${f}" - let NUM_PREVIOUS_GCNO_FILES=NUM_PREVIOUS_GCNO_FILES+1 -done -echo "Cleanup completed. ${NUM_PREVIOUS_GCNO_FILES} files deleted." - # Make sure //test/coverage:coverage_tests is up-to-date. SCRIPT_DIR="$(realpath "$(dirname "$0")")" -(BAZEL_BIN="${BAZEL_COVERAGE}" "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS}) - -echo "Cleaning .gcda/.gcov from previous coverage runs..." -NUM_PREVIOUS_GCOV_FILES=0 -for f in $(find -L "${GCOVR_DIR}" -name "*.gcda" -o -name "*.gcov") -do - rm -f "${f}" - let NUM_PREVIOUS_GCOV_FILES=NUM_PREVIOUS_GCOV_FILES+1 -done -echo "Cleanup completed. ${NUM_PREVIOUS_GCOV_FILES} files deleted." +"${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} -# Force dbg for path consistency later, don't include debug code in coverage. -BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} -c dbg --copt=-DNDEBUG" - -# Run all tests under "bazel test", no sandbox. We're going to generate the -# .gcda inplace in the bazel-out/ directory. This is in contrast to the "bazel -# coverage" method, which is currently broken for C++ (see -# https://github.com/bazelbuild/bazel/issues/1118). This works today as we have -# a single coverage test binary and do not require the "bazel coverage" support -# for collecting multiple traces and glueing them together. -"${BAZEL_COVERAGE}" test //test/coverage:coverage_tests ${BAZEL_BUILD_OPTIONS} \ - --cache_test_results=no --cxxopt="--coverage" --cxxopt="-DENVOY_CONFIG_COVERAGE=1" \ - --linkopt="--coverage" --define ENVOY_CONFIG_COVERAGE=1 --test_output=streamed \ - --strategy=Genrule=standalone --spawn_strategy=standalone --test_timeout=4000 \ - --test_arg="--log-path /dev/null" --test_arg="-l trace" - -# The Bazel build has a lot of whack in it, in particular generated files, headers from external -# deps, etc. So, we exclude this from gcov to avoid false reporting of these files in the html and -# stats. The #foo# pattern is because gcov produces files such as -# bazel-out#local-fastbuild#bin#external#spdlog_git#_virtual_includes#spdlog#spdlog#details#pattern_formatter_impl.h.gcov. -# To find these while modifying this regex, perform a gcov run with -k set. -[[ -z "${GCOVR_EXCLUDE_REGEX}" ]] && GCOVR_EXCLUDE_REGEX=".*pb\\..*|test#.*|.*#test#.*|external#.*|.*#external#.*|.*#prebuilt#.*|.*#config_validation#.*|.*#chromium_url#.*" -[[ -z "${GCOVR_EXCLUDE_DIR}" ]] && GCOVR_EXCLUDE_DIR=".*/external/.*" +BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ + -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \ + --test_timeout=2000 --cxxopt="-DENVOY_CONFIG_COVERAGE=1" --test_output=streamed \ + --test_arg="--log-path /dev/null" --test_arg="-l trace" --test_env=HEAPCHECK= \ + //test/coverage:coverage_tests COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" -COVERAGE_SUMMARY="${COVERAGE_DIR}/coverage_summary.txt" - -# Copy .gcno objects into the same location that we find the .gcda. -# TODO(htuch): Should use rsync, but there are some symlink loops to fight. -echo "Finding and copying .gcno files in GCOVR_DIR: ${GCOVR_DIR}" -mkdir -p ${GCNO_ROOT} -NUM_GCNO_FILES=0 -for f in $(find -L bazel-out/ -name "*.gcno") -do - cp --parents "$f" ${GCNO_ROOT}/ - let NUM_GCNO_FILES=NUM_GCNO_FILES+1 -done -echo "OK: copied ${NUM_GCNO_FILES} .gcno files" -# gcovr is extremely picky about where it is run and where the paths of the -# original source are relative to its execution location. -cd -P "${GCOVR_DIR}" -echo "Running gcovr in $(pwd)..." -time "${GCOVR}" -v --gcov-exclude="${GCOVR_EXCLUDE_REGEX}" \ - --exclude-directories="${GCOVR_EXCLUDE_DIR}" -r . \ - --html --html-details --exclude-unreachable-branches --print-summary \ - -o "${COVERAGE_DIR}"/coverage.html > "${COVERAGE_SUMMARY}" +COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp)" +COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests" +COVERAGE_DATA="bazel-out/k8-fastbuild/testlogs/test/coverage/coverage_tests/coverage.dat" -# Clean up the generated test/coverage/BUILD file: subsequent bazel invocations -# can choke on it if it references things that changed since the last coverage -# run. -rm "${SRCDIR}"/test/coverage/BUILD +echo "Generating report..." +llvm-cov show "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" -Xdemangler=c++filt \ + -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -output-dir=${COVERAGE_DIR} -format=html +sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html" +sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}" if [ "$VALIDATE_COVERAGE" == "true" ] then - COVERAGE_VALUE=$(grep -Po 'lines: \K(\d|\.)*' "${COVERAGE_SUMMARY}") - COVERAGE_THRESHOLD=97.5 + COVERAGE_VALUE=$(llvm-cov export "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \ + -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -summary-only | \ + python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['totals']['lines']['percent'])") + COVERAGE_THRESHOLD=96.9 COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc) if test ${COVERAGE_FAILED} -eq 1; then echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD} @@ -123,5 +59,5 @@ then else echo Code coverage ${COVERAGE_VALUE} is good and higher than limit of ${COVERAGE_THRESHOLD} fi - echo "HTML coverage report is in ${COVERAGE_DIR}/coverage.html" fi +echo "HTML coverage report is in ${COVERAGE_DIR}/index.html" From 79c165f2d9d29e5af3f93f2baa51f1b29b2cdefe Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Tue, 16 Jul 2019 18:48:09 +0300 Subject: [PATCH 182/542] dispatcher: fix comment. (#7597) Signed-off-by: Ismo Puustinen --- source/common/event/dispatcher_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index 51f71e8ad807..3777762265b1 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -91,7 +91,7 @@ class DispatcherImpl : Logger::Loggable, void runPostCallbacks(); // Validate that an operation is thread safe, i.e. it's invoked on the same thread that the - // dispatcher run loop is executing on. We allow run_tid_ == nullptr for tests where we don't + // dispatcher run loop is executing on. We allow run_tid_ to be empty for tests where we don't // invoke run(). bool isThreadSafe() const { return run_tid_.isEmpty() || run_tid_ == api_.threadFactory().currentThreadId(); From 14dd85d969325bb76feb85035a1668c6cbd8f6e2 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 16 Jul 2019 13:04:21 -0700 Subject: [PATCH 183/542] build: remove cc_configure and cc_wrapper (#7555) Use BAZEL_LINKLIBS to statically link libstdc++ and libc++, get rid of cc_wrapper. Requires Bazel >= 0.28. Additional compiler options which was in cc_wrapeer moved into envoy_copts with select. Side effect: static linking libstdc++ applies to all binaries built with the toolchain, so tests are now statically linked with libstdc++. Risk Level: Med, affecting depending projects Testing: CI Signed-off-by: Lizan Zhou --- .bazelrc | 6 + WORKSPACE | 3 - bazel/BUILD | 14 +++ bazel/cc_configure.bzl | 79 ------------- bazel/cc_wrapper.py | 128 --------------------- bazel/envoy_binary.bzl | 38 +++--- bazel/envoy_internal.bzl | 10 +- bazel/envoy_test.bzl | 3 +- bazel/repositories.bzl | 6 - ci/WORKSPACE.filter.example | 3 - ci/build_setup.sh | 2 + test/integration/stats_integration_test.cc | 3 +- 12 files changed, 47 insertions(+), 248 deletions(-) delete mode 100644 bazel/cc_configure.bzl delete mode 100755 bazel/cc_wrapper.py diff --git a/.bazelrc b/.bazelrc index a365b1d064bc..771d01e27b3f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -8,8 +8,12 @@ startup --host_jvm_args=-Xmx512m build --workspace_status_command=bazel/get_workspace_status build --experimental_remap_main_repo build --host_force_python=PY2 +build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a +build --action_env=BAZEL_LINKOPTS=-lm:-static-libgcc # Basic ASAN/UBSAN that works for gcc +build:asan --action_env=BAZEL_LINKLIBS= +build:asan --action_env=BAZEL_LINKOPTS=-lstdc++:-lm build:asan --define ENVOY_CONFIG_ASAN=1 build:asan --copt -fsanitize=address,undefined build:asan --linkopt -fsanitize=address,undefined @@ -60,6 +64,8 @@ build:clang-msan --copt -fsanitize-memory-track-origins=2 build:libc++ --action_env=CC build:libc++ --action_env=CXX build:libc++ --action_env=CXXFLAGS=-stdlib=libc++ +build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ +build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a:-lm build:libc++ --action_env=PATH build:libc++ --host_linkopt=-fuse-ld=lld build:libc++ --define force_libcpp=enabled diff --git a/WORKSPACE b/WORKSPACE index 5609189bd56d..bc3ce13bd3a7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -5,7 +5,6 @@ load("//bazel:api_repositories.bzl", "envoy_api_dependencies") envoy_api_dependencies() load("//bazel:repositories.bzl", "GO_VERSION", "envoy_dependencies") -load("//bazel:cc_configure.bzl", "cc_configure") envoy_dependencies() @@ -13,8 +12,6 @@ load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependen rules_foreign_cc_dependencies() -cc_configure() - load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") go_rules_dependencies() diff --git a/bazel/BUILD b/bazel/BUILD index c288f351054f..a574dc1eb27a 100755 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -85,6 +85,20 @@ config_setting( values = {"define": "ENVOY_CONFIG_COVERAGE=1"}, ) +config_setting( + name = "clang_build", + flag_values = { + "@bazel_tools//tools/cpp:compiler": "clang", + }, +) + +config_setting( + name = "gcc_build", + flag_values = { + "@bazel_tools//tools/cpp:compiler": "gcc", + }, +) + config_setting( name = "disable_tcmalloc", values = {"define": "tcmalloc=disabled"}, diff --git a/bazel/cc_configure.bzl b/bazel/cc_configure.bzl deleted file mode 100644 index 30275140b917..000000000000 --- a/bazel/cc_configure.bzl +++ /dev/null @@ -1,79 +0,0 @@ -load("@bazel_tools//tools/cpp:cc_configure.bzl", _upstream_cc_autoconf_impl = "cc_autoconf_impl") -load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") -load("@bazel_tools//tools/cpp:unix_cc_configure.bzl", "find_cc") - -def _build_envoy_cc_wrapper(repository_ctx): - real_cc = find_cc(repository_ctx, {}) - - # Copy our CC wrapper script into @local_config_cc, with the true paths - # to the C and C++ compiler injected in. The wrapper will use these paths - # to invoke the compiler after deciding which one is correct for the current - # invocation. - # - # Since the script is Python, we can inject values using `repr(str(value))` - # and escaping will be handled correctly. - repository_ctx.template("extra_tools/envoy_cc_wrapper", repository_ctx.attr._envoy_cc_wrapper, { - "{ENVOY_REAL_CC}": repr(str(real_cc)), - "{ENVOY_CFLAGS}": repr(str(repository_ctx.os.environ.get("CFLAGS", ""))), - "{ENVOY_CXXFLAGS}": repr(str(repository_ctx.os.environ.get("CXXFLAGS", ""))), - }) - return repository_ctx.path("extra_tools/envoy_cc_wrapper") - -def _needs_envoy_cc_wrapper(repository_ctx): - # When building for Linux we set additional C++ compiler options that aren't - # handled well by Bazel, so we need a wrapper around $CC to fix its - # compiler invocations. - cpu_value = get_cpu_value(repository_ctx) - return cpu_value not in ["freebsd", "x64_windows", "darwin"] - -def cc_autoconf_impl(repository_ctx): - overriden_tools = {} - if _needs_envoy_cc_wrapper(repository_ctx): - # Bazel uses "gcc" as a generic name for all C and C++ compilers. - overriden_tools["gcc"] = _build_envoy_cc_wrapper(repository_ctx) - return _upstream_cc_autoconf_impl(repository_ctx, overriden_tools = overriden_tools) - -cc_autoconf = repository_rule( - implementation = cc_autoconf_impl, - attrs = { - "_envoy_cc_wrapper": attr.label(default = "@envoy//bazel:cc_wrapper.py"), - }, - environ = [ - "ABI_LIBC_VERSION", - "ABI_VERSION", - "BAZEL_COMPILER", - "BAZEL_HOST_SYSTEM", - "BAZEL_CXXOPTS", - "BAZEL_LINKOPTS", - "BAZEL_PYTHON", - "BAZEL_SH", - "BAZEL_TARGET_CPU", - "BAZEL_TARGET_LIBC", - "BAZEL_TARGET_SYSTEM", - "BAZEL_USE_CPP_ONLY_TOOLCHAIN", - "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN", - "BAZEL_USE_LLVM_NATIVE_COVERAGE", - "BAZEL_VC", - "BAZEL_VS", - "BAZEL_LLVM", - "USE_CLANG_CL", - "CC", - "CFLAGS", - "CXXFLAGS", - "CC_CONFIGURE_DEBUG", - "CC_TOOLCHAIN_NAME", - "CPLUS_INCLUDE_PATH", - "GCOV", - "HOMEBREW_RUBY_PATH", - "SYSTEMROOT", - "VS90COMNTOOLS", - "VS100COMNTOOLS", - "VS110COMNTOOLS", - "VS120COMNTOOLS", - "VS140COMNTOOLS", - ], -) - -def cc_configure(): - cc_autoconf(name = "local_config_cc") - native.bind(name = "cc_toolchain", actual = "@local_config_cc//:toolchain") diff --git a/bazel/cc_wrapper.py b/bazel/cc_wrapper.py deleted file mode 100755 index 41029a05eb4b..000000000000 --- a/bazel/cc_wrapper.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/python3 -import contextlib -import os -import shlex -import sys -import tempfile - -compiler = {ENVOY_REAL_CC} -envoy_cflags = {ENVOY_CFLAGS} -envoy_cxxflags = {ENVOY_CXXFLAGS} - - -@contextlib.contextmanager -def closing_fd(fd): - try: - yield fd - finally: - os.close(fd) - - -def sanitize_flagfile(in_path, out_fd): - with open(in_path, "rb") as in_fp: - for line in in_fp: - if line != "-lstdc++\n": - os.write(out_fd, line) - elif "-stdlib=libc++" in envoy_cxxflags: - os.write(out_fd, "-l:libc++.a\n-l:libc++abi.a\n") - - -# Is the arg a flag indicating that we're building for C++ (rather than C)? -def is_cpp_flag(arg): - return arg in ["-static-libstdc++", "-stdlib=libc++", "-lstdc++", "-lc++" - ] or arg.startswith("-std=c++") or arg.startswith("-std=gnu++") - - -def modify_driver_args(input_driver_flags): - # Detect if we're building for C++ or vanilla C. - if any(map(is_cpp_flag, input_driver_flags)): - # Append CXXFLAGS to all C++ targets (this is mostly for dependencies). - argv = shlex.split(envoy_cxxflags) - else: - # Append CFLAGS to all C targets (this is mostly for dependencies). - argv = shlex.split(envoy_cflags) - - # Either: - # a) remove all occurrences of -lstdc++ (when statically linking against libstdc++), - # b) replace all occurrences of -lstdc++ with -lc++ (when linking against libc++). - if "-static-libstdc++" in input_driver_flags or "-stdlib=libc++" in envoy_cxxflags: - for arg in input_driver_flags: - if arg in ("-lstdc++", "-static-libstdc++"): - pass - elif arg.startswith("-Wl,@"): - # tempfile.mkstemp will write to the out-of-sandbox tempdir - # unless the user has explicitly set environment variables - # before starting Bazel. But here in $PWD is the Bazel sandbox, - # which will be deleted automatically after the compiler exits. - (flagfile_fd, flagfile_path) = tempfile.mkstemp(dir="./", suffix=".linker-params") - with closing_fd(flagfile_fd): - sanitize_flagfile(arg[len("-Wl,@"):], flagfile_fd) - argv.append("-Wl,@" + flagfile_path) - else: - argv.append(arg) - else: - argv += input_driver_flags - - # This flags should after all libraries - if "-static-libstdc++" in input_driver_flags: - argv.append("-l:libstdc++.a") - if "-lstdc++" in input_driver_flags and "-stdlib=libc++" in envoy_cxxflags: - argv.append("-l:libc++.a") - argv.append("-l:libc++abi.a") - - # Bazel will add -fuse-ld=gold in some cases, gcc/clang will take the last -fuse-ld argument, - # so whenever we see lld once, add it to the end. - if "-fuse-ld=lld" in argv: - argv.append("-fuse-ld=lld") - - # Add compiler-specific options - if "clang" in compiler: - # This ensures that STL symbols are included. - # See https://github.com/envoyproxy/envoy/issues/1341 - argv.append("-fno-limit-debug-info") - argv.append("-Wthread-safety") - argv.append("-Wgnu-conditional-omitted-operand") - elif "gcc" in compiler or "g++" in compiler: - # -Wmaybe-initialized is warning about many uses of absl::optional. Disable - # to prevent build breakage. This option does not exist in clang, so setting - # it in clang builds causes a build error because of unknown command line - # flag. - # See https://github.com/envoyproxy/envoy/issues/2987 - argv.append("-Wno-maybe-uninitialized") - - return argv - - -def main(): - # Append CXXFLAGS to correctly detect include paths for either libstdc++ or libc++. - if sys.argv[1:5] == ["-E", "-xc++", "-", "-v"]: - os.execv(compiler, [compiler] + sys.argv[1:] + shlex.split(envoy_cxxflags)) - - if sys.argv[1].startswith("@"): - # Read flags from file - flagfile_path = sys.argv[1][1:] - with open(flagfile_path, "r") as fd: - input_driver_flags = fd.read().splitlines() - - # Compute new args - new_driver_args = modify_driver_args(input_driver_flags) - - # Write args to temp file - (new_flagfile_fd, new_flagfile_path) = tempfile.mkstemp(dir="./", suffix=".linker-params") - - with closing_fd(new_flagfile_fd): - for arg in new_driver_args: - os.write(new_flagfile_fd, bytes(str(arg + "\n").encode("utf-8"))) - - # Provide new arguments using the temp file containing the args - new_args = ["@" + new_flagfile_path] - else: - # TODO(https://github.com/bazelbuild/bazel/issues/7687): Remove this branch - # when Bazel 0.27 is released. - new_args = modify_driver_args(sys.argv[1:]) - - os.execv(compiler, [compiler] + new_args) - - -if __name__ == "__main__": - main() diff --git a/bazel/envoy_binary.bzl b/bazel/envoy_binary.bzl index a7a369e8c16b..edcdd09ae03b 100644 --- a/bazel/envoy_binary.bzl +++ b/bazel/envoy_binary.bzl @@ -4,7 +4,6 @@ load( ":envoy_internal.bzl", "envoy_copts", "envoy_external_dep_path", - "envoy_static_link_libstdcpp_linkopts", "tcmalloc_external_dep", ) @@ -50,25 +49,24 @@ def _envoy_select_exported_symbols(xs): # Compute the final linkopts based on various options. def _envoy_linkopts(): return select({ - # The macOS system library transitively links common libraries (e.g., pthread). - "@envoy//bazel:apple": [ - # See note here: https://luajit.org/install.html - "-pagezero_size 10000", - "-image_base 100000000", - ], - "@envoy//bazel:windows_x86_64": [ - "-DEFAULTLIB:advapi32.lib", - "-DEFAULTLIB:ws2_32.lib", - "-WX", - ], - "//conditions:default": [ - "-pthread", - "-lrt", - "-ldl", - "-Wl,--hash-style=gnu", - ], - }) + envoy_static_link_libstdcpp_linkopts() + \ - _envoy_select_exported_symbols(["-Wl,-E"]) + # The macOS system library transitively links common libraries (e.g., pthread). + "@envoy//bazel:apple": [ + # See note here: https://luajit.org/install.html + "-pagezero_size 10000", + "-image_base 100000000", + ], + "@envoy//bazel:windows_x86_64": [ + "-DEFAULTLIB:advapi32.lib", + "-DEFAULTLIB:ws2_32.lib", + "-WX", + ], + "//conditions:default": [ + "-pthread", + "-lrt", + "-ldl", + "-Wl,--hash-style=gnu", + ], + }) + _envoy_select_exported_symbols(["-Wl,-E"]) def _envoy_stamped_deps(): return select({ diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 4ea71659d295..325ea94f5596 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -40,6 +40,10 @@ def envoy_copts(repository, test = False): repository + "//bazel:windows_opt_build": [], repository + "//bazel:windows_fastbuild_build": [], repository + "//bazel:windows_dbg_build": [], + }) + select({ + repository + "//bazel:clang_build": ["-fno-limit-debug-info", "-Wgnu-conditional-omitted-operand"], + repository + "//bazel:gcc_build": ["-Wno-maybe-uninitialized"], + "//conditions:default": [], }) + select({ repository + "//bazel:disable_tcmalloc": ["-DABSL_MALLOC_HOOK_MMAP_DISABLE"], "//conditions:default": ["-DTCMALLOC"], @@ -82,12 +86,6 @@ def envoy_select_force_libcpp(if_libcpp, default = None): "//conditions:default": default or [], }) -def envoy_static_link_libstdcpp_linkopts(): - return envoy_select_force_libcpp( - ["-stdlib=libc++", "-l:libc++.a", "-l:libc++abi.a", "-static-libgcc"], - ["-static-libstdc++", "-static-libgcc"], - ) - # Dependencies on tcmalloc_and_profiler should be wrapped with this function. def tcmalloc_external_dep(repository): return select({ diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index ea564d619fa8..c836096913fa 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -8,7 +8,6 @@ load( "envoy_external_dep_path", "envoy_linkstatic", "envoy_select_force_libcpp", - "envoy_static_link_libstdcpp_linkopts", "tcmalloc_external_dep", ) @@ -205,7 +204,7 @@ def envoy_cc_test_binary( envoy_cc_binary( name, testonly = 1, - linkopts = _envoy_test_linkopts() + envoy_static_link_libstdcpp_linkopts(), + linkopts = _envoy_test_linkopts(), **kargs ) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 027d69baa27f..1c0e6abb57dd 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -2,12 +2,6 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load(":genrule_repository.bzl", "genrule_repository") load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive") load(":repository_locations.bzl", "REPOSITORY_LOCATIONS") -load( - "@bazel_tools//tools/cpp:windows_cc_configure.bzl", - "find_vc_path", - "setup_vc_env_vars", -) -load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_env_var") load("@envoy_api//bazel:repositories.bzl", "api_dependencies") # dict of {build recipe name: longform extension name,} diff --git a/ci/WORKSPACE.filter.example b/ci/WORKSPACE.filter.example index 4eb98345a13f..5013077c2468 100644 --- a/ci/WORKSPACE.filter.example +++ b/ci/WORKSPACE.filter.example @@ -9,7 +9,6 @@ load("@envoy//bazel:api_repositories.bzl", "envoy_api_dependencies") envoy_api_dependencies() load("@envoy//bazel:repositories.bzl", "envoy_dependencies", "GO_VERSION") -load("@envoy//bazel:cc_configure.bzl", "cc_configure") envoy_dependencies() @@ -17,8 +16,6 @@ envoy_dependencies() load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") rules_foreign_cc_dependencies() -cc_configure() - load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") go_rules_dependencies() go_register_toolchains(go_version = GO_VERSION) diff --git a/ci/build_setup.sh b/ci/build_setup.sh index 8323eb744011..f5ae34ac5fc9 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -13,6 +13,7 @@ echo "ENVOY_SRCDIR=${ENVOY_SRCDIR}" function setup_gcc_toolchain() { export CC=gcc export CXX=g++ + export BAZEL_COMPILER=gcc echo "$CC/$CXX toolchain configured" } @@ -20,6 +21,7 @@ function setup_clang_toolchain() { export PATH=/usr/lib/llvm-8/bin:$PATH export CC=clang export CXX=clang++ + export BAZEL_COMPILER=clang export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer echo "$CC/$CXX toolchain configured" } diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index a404e8de89d5..ad2b87484f58 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -210,6 +210,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // 2019/06/29 7364 45685 46000 combine 2 levels of stat ref-counting into 1 // 2019/06/30 7428 42742 43000 remove stats multiple inheritance, inline HeapStatData // 2019/07/06 7477 42742 43000 fork gauge representation to drop pending_increment_ + // 2019/07/15 7555 42806 43000 static link libstdc++ in tests // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -219,7 +220,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 42742); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 42806); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 43000); } From c53b12dc52744f523b5d7431514cffd2ee4c33c4 Mon Sep 17 00:00:00 2001 From: Auni Ahsan Date: Tue, 16 Jul 2019 19:51:41 -0400 Subject: [PATCH 184/542] access log: Add AccessLoggers::Common::ImplBase base class to handle AccessLog::Filter evaluation (#7359) Introduce AccessLoggers::Common::ImplBase in the access_loggers extension to handle converting request/response/trailer HeaderMap pointer-to-reference conversion and evaluation of access log filters. Refactors existing loggers to inherit from the common base class. Risk Level: Low. No behavior change intended. Testing: Added unit tests for base class, existing tests suffice for other loggers. Docs Changes: n/a Release Notes: n/a Fixes: #7325 Signed-off-by: Auni Ahsan --- CODEOWNERS | 2 + source/common/access_log/BUILD | 54 ++++++------- source/common/access_log/access_log_impl.cc | 4 +- source/extensions/access_loggers/common/BUILD | 23 ++++++ .../access_loggers/common/access_log_base.cc | 34 +++++++++ .../access_loggers/common/access_log_base.h | 55 ++++++++++++++ source/extensions/access_loggers/file/BUILD | 3 +- .../file/file_access_log_impl.cc | 31 ++------ .../file/file_access_log_impl.h | 15 ++-- .../extensions/access_loggers/http_grpc/BUILD | 2 +- .../http_grpc/grpc_access_log_impl.cc | 75 +++++++------------ .../http_grpc/grpc_access_log_impl.h | 16 ++-- source/extensions/filters/http/tap/BUILD | 1 + .../extensions/filters/http/tap/tap_filter.h | 1 + test/extensions/access_loggers/common/BUILD | 19 +++++ .../common/access_log_base_test.cc | 57 ++++++++++++++ 16 files changed, 272 insertions(+), 120 deletions(-) create mode 100644 source/extensions/access_loggers/common/BUILD create mode 100644 source/extensions/access_loggers/common/access_log_base.cc create mode 100644 source/extensions/access_loggers/common/access_log_base.h create mode 100644 test/extensions/access_loggers/common/BUILD create mode 100644 test/extensions/access_loggers/common/access_log_base_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index e6d99af3b562..f908e99f3780 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -4,6 +4,8 @@ # api /api/ @envoyproxy/api-shepherds +# access loggers +/*/extensions/access_loggers/common @auni53 @zuercher # csrf extension /*/extensions/filters/http/csrf @dschaller @mattklein123 # original_src http filter extension diff --git a/source/common/access_log/BUILD b/source/common/access_log/BUILD index 2de71c912d53..8506985f2676 100644 --- a/source/common/access_log/BUILD +++ b/source/common/access_log/BUILD @@ -8,33 +8,6 @@ load( envoy_package() -envoy_cc_library( - name = "access_log_formatter_lib", - srcs = ["access_log_formatter.cc"], - hdrs = ["access_log_formatter.h"], - deps = [ - "//include/envoy/access_log:access_log_interface", - "//include/envoy/stream_info:stream_info_interface", - "//source/common/common:assert_lib", - "//source/common/common:utility_lib", - "//source/common/config:metadata_lib", - "//source/common/http:utility_lib", - "//source/common/stream_info:utility_lib", - ], -) - -envoy_cc_library( - name = "access_log_manager_lib", - srcs = ["access_log_manager_impl.cc"], - hdrs = ["access_log_manager_impl.h"], - deps = [ - "//include/envoy/access_log:access_log_interface", - "//include/envoy/api:api_interface", - "//source/common/buffer:buffer_lib", - "//source/common/common:thread_lib", - ], -) - envoy_cc_library( name = "access_log_lib", srcs = ["access_log_impl.cc"], @@ -61,3 +34,30 @@ envoy_cc_library( "@envoy_api//envoy/config/filter/accesslog/v2:accesslog_cc", ], ) + +envoy_cc_library( + name = "access_log_formatter_lib", + srcs = ["access_log_formatter.cc"], + hdrs = ["access_log_formatter.h"], + deps = [ + "//include/envoy/access_log:access_log_interface", + "//include/envoy/stream_info:stream_info_interface", + "//source/common/common:assert_lib", + "//source/common/common:utility_lib", + "//source/common/config:metadata_lib", + "//source/common/http:utility_lib", + "//source/common/stream_info:utility_lib", + ], +) + +envoy_cc_library( + name = "access_log_manager_lib", + srcs = ["access_log_manager_impl.cc"], + hdrs = ["access_log_manager_impl.h"], + deps = [ + "//include/envoy/access_log:access_log_interface", + "//include/envoy/api:api_interface", + "//source/common/buffer:buffer_lib", + "//source/common/common:thread_lib", + ], +) diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc index aac76ae28817..33fcd5485d04 100644 --- a/source/common/access_log/access_log_impl.cc +++ b/source/common/access_log/access_log_impl.cc @@ -114,9 +114,9 @@ RuntimeFilter::RuntimeFilter(const envoy::config::filter::accesslog::v2::Runtime percent_(config.percent_sampled()), use_independent_randomness_(config.use_independent_randomness()) {} -bool RuntimeFilter::evaluate(const StreamInfo::StreamInfo&, const Http::HeaderMap& request_header, +bool RuntimeFilter::evaluate(const StreamInfo::StreamInfo&, const Http::HeaderMap& request_headers, const Http::HeaderMap&, const Http::HeaderMap&) { - const Http::HeaderEntry* uuid = request_header.RequestId(); + const Http::HeaderEntry* uuid = request_headers.RequestId(); uint64_t random_value; // TODO(dnoe): Migrate uuidModBy to take string_view (#6580) if (use_independent_randomness_ || uuid == nullptr || diff --git a/source/extensions/access_loggers/common/BUILD b/source/extensions/access_loggers/common/BUILD new file mode 100644 index 000000000000..daa8a198e578 --- /dev/null +++ b/source/extensions/access_loggers/common/BUILD @@ -0,0 +1,23 @@ +licenses(["notice"]) # Apache 2 + +# Base class for implementations of AccessLog::Instance. + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "access_log_base", + srcs = ["access_log_base.cc"], + hdrs = ["access_log_base.h"], + deps = [ + "//include/envoy/access_log:access_log_interface", + "//source/common/access_log:access_log_lib", + "//source/common/http:header_map_lib", + "//source/common/singleton:const_singleton", + ], +) diff --git a/source/extensions/access_loggers/common/access_log_base.cc b/source/extensions/access_loggers/common/access_log_base.cc new file mode 100644 index 000000000000..77b10388537a --- /dev/null +++ b/source/extensions/access_loggers/common/access_log_base.cc @@ -0,0 +1,34 @@ +#include "extensions/access_loggers/common/access_log_base.h" + +#include "common/http/header_map_impl.h" +#include "common/singleton/const_singleton.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace Common { + +void ImplBase::log(const Http::HeaderMap* request_headers, const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info) { + ConstSingleton empty_headers; + if (!request_headers) { + request_headers = &empty_headers.get(); + } + if (!response_headers) { + response_headers = &empty_headers.get(); + } + if (!response_trailers) { + response_trailers = &empty_headers.get(); + } + if (filter_ && + !filter_->evaluate(stream_info, *request_headers, *response_headers, *response_trailers)) { + return; + } + return emitLog(*request_headers, *response_headers, *response_trailers, stream_info); +} + +} // namespace Common +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/common/access_log_base.h b/source/extensions/access_loggers/common/access_log_base.h new file mode 100644 index 000000000000..9a6d6caa6668 --- /dev/null +++ b/source/extensions/access_loggers/common/access_log_base.h @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include +#include + +#include "envoy/access_log/access_log.h" +#include "envoy/config/filter/accesslog/v2/accesslog.pb.h" +#include "envoy/runtime/runtime.h" +#include "envoy/server/access_log_config.h" + +#include "common/http/header_utility.h" +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace Common { + +/** + * Base implementation of Accesslog::Instance handles common filter logic. + */ +class ImplBase : public AccessLog::Instance { +public: + ImplBase(AccessLog::FilterPtr filter) : filter_(std::move(filter)) {} + + /** + * Log a completed request if the underlying AccessLog `filter_` allows it. + */ + void log(const Http::HeaderMap* request_headers, const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info) override; + +private: + /** + * Log a completed request. + * @param request_headers supplies the incoming request headers after filtering. + * @param response_headers supplies response headers. + * @param response_trailers supplies response trailers. + * @param stream_info supplies additional information about the request not + * contained in the request headers. + */ + virtual void emitLog(const Http::HeaderMap& request_headers, + const Http::HeaderMap& response_headers, + const Http::HeaderMap& response_trailers, + const StreamInfo::StreamInfo& stream_info) PURE; + + AccessLog::FilterPtr filter_; +}; + +} // namespace Common +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/file/BUILD b/source/extensions/access_loggers/file/BUILD index e83a33f02a9f..3ea8c8819c30 100644 --- a/source/extensions/access_loggers/file/BUILD +++ b/source/extensions/access_loggers/file/BUILD @@ -16,8 +16,7 @@ envoy_cc_library( srcs = ["file_access_log_impl.cc"], hdrs = ["file_access_log_impl.h"], deps = [ - "//include/envoy/access_log:access_log_interface", - "//source/common/http:header_map_lib", + "//source/extensions/access_loggers/common:access_log_base", ], ) diff --git a/source/extensions/access_loggers/file/file_access_log_impl.cc b/source/extensions/access_loggers/file/file_access_log_impl.cc index 75409f34dadc..410204d3ad86 100644 --- a/source/extensions/access_loggers/file/file_access_log_impl.cc +++ b/source/extensions/access_loggers/file/file_access_log_impl.cc @@ -1,7 +1,5 @@ #include "extensions/access_loggers/file/file_access_log_impl.h" -#include "common/http/header_map_impl.h" - namespace Envoy { namespace Extensions { namespace AccessLoggers { @@ -10,33 +8,16 @@ namespace File { FileAccessLog::FileAccessLog(const std::string& access_log_path, AccessLog::FilterPtr&& filter, AccessLog::FormatterPtr&& formatter, AccessLog::AccessLogManager& log_manager) - : filter_(std::move(filter)), formatter_(std::move(formatter)) { + : ImplBase(std::move(filter)), formatter_(std::move(formatter)) { log_file_ = log_manager.createAccessLog(access_log_path); } -void FileAccessLog::log(const Http::HeaderMap* request_headers, - const Http::HeaderMap* response_headers, - const Http::HeaderMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) { - static Http::HeaderMapImpl empty_headers; - if (!request_headers) { - request_headers = &empty_headers; - } - if (!response_headers) { - response_headers = &empty_headers; - } - if (!response_trailers) { - response_trailers = &empty_headers; - } - - if (filter_) { - if (!filter_->evaluate(stream_info, *request_headers, *response_headers, *response_trailers)) { - return; - } - } - +void FileAccessLog::emitLog(const Http::HeaderMap& request_headers, + const Http::HeaderMap& response_headers, + const Http::HeaderMap& response_trailers, + const StreamInfo::StreamInfo& stream_info) { log_file_->write( - formatter_->format(*request_headers, *response_headers, *response_trailers, stream_info)); + formatter_->format(request_headers, response_headers, response_trailers, stream_info)); } } // namespace File diff --git a/source/extensions/access_loggers/file/file_access_log_impl.h b/source/extensions/access_loggers/file/file_access_log_impl.h index b14b396befd8..06546d6aba9a 100644 --- a/source/extensions/access_loggers/file/file_access_log_impl.h +++ b/source/extensions/access_loggers/file/file_access_log_impl.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/access_log/access_log.h" +#include "extensions/access_loggers/common/access_log_base.h" namespace Envoy { namespace Extensions { @@ -10,19 +10,18 @@ namespace File { /** * Access log Instance that writes logs to a file. */ -class FileAccessLog : public AccessLog::Instance { +class FileAccessLog : public Common::ImplBase { public: FileAccessLog(const std::string& access_log_path, AccessLog::FilterPtr&& filter, AccessLog::FormatterPtr&& formatter, AccessLog::AccessLogManager& log_manager); - // AccessLog::Instance - void log(const Http::HeaderMap* request_headers, const Http::HeaderMap* response_headers, - const Http::HeaderMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) override; - private: + // Common::ImplBase + void emitLog(const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers, + const Http::HeaderMap& response_trailers, + const StreamInfo::StreamInfo& stream_info) override; + AccessLog::AccessLogFileSharedPtr log_file_; - AccessLog::FilterPtr filter_; AccessLog::FormatterPtr formatter_; }; diff --git a/source/extensions/access_loggers/http_grpc/BUILD b/source/extensions/access_loggers/http_grpc/BUILD index d57adbd547ef..941d950ff9c4 100644 --- a/source/extensions/access_loggers/http_grpc/BUILD +++ b/source/extensions/access_loggers/http_grpc/BUILD @@ -16,7 +16,6 @@ envoy_cc_library( srcs = ["grpc_access_log_impl.cc"], hdrs = ["grpc_access_log_impl.h"], deps = [ - "//include/envoy/access_log:access_log_interface", "//include/envoy/grpc:async_client_interface", "//include/envoy/grpc:async_client_manager_interface", "//include/envoy/singleton:instance_interface", @@ -26,6 +25,7 @@ envoy_cc_library( "//source/common/grpc:async_client_lib", "//source/common/grpc:typed_async_client_lib", "//source/common/network:utility_lib", + "//source/extensions/access_loggers/common:access_log_base", "@envoy_api//envoy/config/accesslog/v2:als_cc", "@envoy_api//envoy/config/filter/accesslog/v2:accesslog_cc", "@envoy_api//envoy/service/accesslog/v2:als_cc", diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc index c9c94146ef9c..953e2e74e45e 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc @@ -3,7 +3,6 @@ #include "envoy/upstream/upstream.h" #include "common/common/assert.h" -#include "common/http/header_map_impl.h" #include "common/network/utility.h" #include "common/stream_info/utility.h" @@ -89,7 +88,7 @@ HttpGrpcAccessLog::HttpGrpcAccessLog( AccessLog::FilterPtr&& filter, const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig& config, GrpcAccessLogStreamerSharedPtr grpc_access_log_streamer) - : filter_(std::move(filter)), config_(config), + : Common::ImplBase(std::move(filter)), config_(config), grpc_access_log_streamer_(grpc_access_log_streamer) { for (const auto& header : config_.additional_request_headers_to_log()) { request_headers_to_log_.emplace_back(header); @@ -186,27 +185,10 @@ void HttpGrpcAccessLog::responseFlagsToAccessLogResponseFlags( } } -void HttpGrpcAccessLog::log(const Http::HeaderMap* request_headers, - const Http::HeaderMap* response_headers, - const Http::HeaderMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) { - static Http::HeaderMapImpl empty_headers; - if (!request_headers) { - request_headers = &empty_headers; - } - if (!response_headers) { - response_headers = &empty_headers; - } - if (!response_trailers) { - response_trailers = &empty_headers; - } - - if (filter_) { - if (!filter_->evaluate(stream_info, *request_headers, *response_headers, *response_trailers)) { - return; - } - } - +void HttpGrpcAccessLog::emitLog(const Http::HeaderMap& request_headers, + const Http::HeaderMap& response_headers, + const Http::HeaderMap& response_trailers, + const StreamInfo::StreamInfo& stream_info) { envoy::service::accesslog::v2::StreamAccessLogsMessage message; auto* log_entry = message.mutable_http_logs()->add_log_entry(); @@ -340,50 +322,49 @@ void HttpGrpcAccessLog::log(const Http::HeaderMap* request_headers, // HTTP request properties. // TODO(mattklein123): Populate port field. auto* request_properties = log_entry->mutable_request(); - if (request_headers->Scheme() != nullptr) { - request_properties->set_scheme(std::string(request_headers->Scheme()->value().getStringView())); + if (request_headers.Scheme() != nullptr) { + request_properties->set_scheme(std::string(request_headers.Scheme()->value().getStringView())); } - if (request_headers->Host() != nullptr) { - request_properties->set_authority( - std::string(request_headers->Host()->value().getStringView())); + if (request_headers.Host() != nullptr) { + request_properties->set_authority(std::string(request_headers.Host()->value().getStringView())); } - if (request_headers->Path() != nullptr) { - request_properties->set_path(std::string(request_headers->Path()->value().getStringView())); + if (request_headers.Path() != nullptr) { + request_properties->set_path(std::string(request_headers.Path()->value().getStringView())); } - if (request_headers->UserAgent() != nullptr) { + if (request_headers.UserAgent() != nullptr) { request_properties->set_user_agent( - std::string(request_headers->UserAgent()->value().getStringView())); + std::string(request_headers.UserAgent()->value().getStringView())); } - if (request_headers->Referer() != nullptr) { + if (request_headers.Referer() != nullptr) { request_properties->set_referer( - std::string(request_headers->Referer()->value().getStringView())); + std::string(request_headers.Referer()->value().getStringView())); } - if (request_headers->ForwardedFor() != nullptr) { + if (request_headers.ForwardedFor() != nullptr) { request_properties->set_forwarded_for( - std::string(request_headers->ForwardedFor()->value().getStringView())); + std::string(request_headers.ForwardedFor()->value().getStringView())); } - if (request_headers->RequestId() != nullptr) { + if (request_headers.RequestId() != nullptr) { request_properties->set_request_id( - std::string(request_headers->RequestId()->value().getStringView())); + std::string(request_headers.RequestId()->value().getStringView())); } - if (request_headers->EnvoyOriginalPath() != nullptr) { + if (request_headers.EnvoyOriginalPath() != nullptr) { request_properties->set_original_path( - std::string(request_headers->EnvoyOriginalPath()->value().getStringView())); + std::string(request_headers.EnvoyOriginalPath()->value().getStringView())); } - request_properties->set_request_headers_bytes(request_headers->byteSize()); + request_properties->set_request_headers_bytes(request_headers.byteSize()); request_properties->set_request_body_bytes(stream_info.bytesReceived()); - if (request_headers->Method() != nullptr) { + if (request_headers.Method() != nullptr) { envoy::api::v2::core::RequestMethod method = envoy::api::v2::core::RequestMethod::METHOD_UNSPECIFIED; envoy::api::v2::core::RequestMethod_Parse( - std::string(request_headers->Method()->value().getStringView()), &method); + std::string(request_headers.Method()->value().getStringView()), &method); request_properties->set_request_method(method); } if (!request_headers_to_log_.empty()) { auto* logged_headers = request_properties->mutable_request_headers(); for (const auto& header : request_headers_to_log_) { - const Http::HeaderEntry* entry = request_headers->get(header); + const Http::HeaderEntry* entry = request_headers.get(header); if (entry != nullptr) { logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); } @@ -398,13 +379,13 @@ void HttpGrpcAccessLog::log(const Http::HeaderMap* request_headers, if (stream_info.responseCodeDetails()) { response_properties->set_response_code_details(stream_info.responseCodeDetails().value()); } - response_properties->set_response_headers_bytes(response_headers->byteSize()); + response_properties->set_response_headers_bytes(response_headers.byteSize()); response_properties->set_response_body_bytes(stream_info.bytesSent()); if (!response_headers_to_log_.empty()) { auto* logged_headers = response_properties->mutable_response_headers(); for (const auto& header : response_headers_to_log_) { - const Http::HeaderEntry* entry = response_headers->get(header); + const Http::HeaderEntry* entry = response_headers.get(header); if (entry != nullptr) { logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); } @@ -415,7 +396,7 @@ void HttpGrpcAccessLog::log(const Http::HeaderMap* request_headers, auto* logged_headers = response_properties->mutable_response_trailers(); for (const auto& header : response_trailers_to_log_) { - const Http::HeaderEntry* entry = response_trailers->get(header); + const Http::HeaderEntry* entry = response_trailers.get(header); if (entry != nullptr) { logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); } diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h index e8c6acde5e19..cf81c7990f40 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h @@ -3,7 +3,6 @@ #include #include -#include "envoy/access_log/access_log.h" #include "envoy/config/accesslog/v2/als.pb.h" #include "envoy/config/filter/accesslog/v2/accesslog.pb.h" #include "envoy/grpc/async_client.h" @@ -15,6 +14,8 @@ #include "common/grpc/typed_async_client.h" +#include "extensions/access_loggers/common/access_log_base.h" + namespace Envoy { namespace Extensions { namespace AccessLoggers { @@ -115,7 +116,7 @@ class GrpcAccessLogStreamerImpl : public Singleton::Instance, public GrpcAccessL /** * Access log Instance that streams HTTP logs over gRPC. */ -class HttpGrpcAccessLog : public AccessLog::Instance { +class HttpGrpcAccessLog : public Common::ImplBase { public: HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig& config, @@ -125,13 +126,12 @@ class HttpGrpcAccessLog : public AccessLog::Instance { envoy::data::accesslog::v2::AccessLogCommon& common_access_log, const StreamInfo::StreamInfo& stream_info); - // AccessLog::Instance - void log(const Http::HeaderMap* request_headers, const Http::HeaderMap* response_headers, - const Http::HeaderMap* response_trailers, - const StreamInfo::StreamInfo& stream_info) override; - private: - AccessLog::FilterPtr filter_; + // Common::ImplBase + void emitLog(const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers, + const Http::HeaderMap& response_trailers, + const StreamInfo::StreamInfo& stream_info) override; + const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config_; GrpcAccessLogStreamerSharedPtr grpc_access_log_streamer_; std::vector request_headers_to_log_; diff --git a/source/extensions/filters/http/tap/BUILD b/source/extensions/filters/http/tap/BUILD index 40e7a1f8f6ac..0581a9f7d2c6 100644 --- a/source/extensions/filters/http/tap/BUILD +++ b/source/extensions/filters/http/tap/BUILD @@ -39,6 +39,7 @@ envoy_cc_library( hdrs = ["tap_filter.h"], deps = [ ":tap_config_interface", + "//include/envoy/access_log:access_log_interface", "//include/envoy/http:filter_interface", "//source/extensions/common/tap:extension_config_base", "@envoy_api//envoy/config/filter/http/tap/v2alpha:tap_cc", diff --git a/source/extensions/filters/http/tap/tap_filter.h b/source/extensions/filters/http/tap/tap_filter.h index a8c5d0d7f5d7..79058b0a244a 100644 --- a/source/extensions/filters/http/tap/tap_filter.h +++ b/source/extensions/filters/http/tap/tap_filter.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/access_log/access_log.h" #include "envoy/config/filter/http/tap/v2alpha/tap.pb.h" #include "envoy/http/filter.h" #include "envoy/stats/scope.h" diff --git a/test/extensions/access_loggers/common/BUILD b/test/extensions/access_loggers/common/BUILD new file mode 100644 index 000000000000..9dbb3c91c70f --- /dev/null +++ b/test/extensions/access_loggers/common/BUILD @@ -0,0 +1,19 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +envoy_package() + +envoy_cc_test( + name = "access_log_base_test", + srcs = ["access_log_base_test.cc"], + deps = [ + "//source/extensions/access_loggers/common:access_log_base", + "//test/mocks/access_log:access_log_mocks", + "//test/mocks/stream_info:stream_info_mocks", + ], +) diff --git a/test/extensions/access_loggers/common/access_log_base_test.cc b/test/extensions/access_loggers/common/access_log_base_test.cc new file mode 100644 index 000000000000..52fcb4a6bcdb --- /dev/null +++ b/test/extensions/access_loggers/common/access_log_base_test.cc @@ -0,0 +1,57 @@ +#include "extensions/access_loggers/common/access_log_base.h" + +#include "test/mocks/access_log/mocks.h" +#include "test/mocks/stream_info/mocks.h" + +#include "gmock/gmock.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace Common { +namespace { + +using AccessLog::FilterPtr; +using AccessLog::MockFilter; +using testing::_; +using testing::Return; + +class TestImpl : public ImplBase { +public: + TestImpl(FilterPtr filter) : ImplBase(std::move(filter)) {} + + int count() { return count_; }; + +private: + void emitLog(const Http::HeaderMap&, const Http::HeaderMap&, const Http::HeaderMap&, + const StreamInfo::StreamInfo&) override { + count_++; + } + + int count_ = 0; +}; + +TEST(AccessLogBaseTest, NoFilter) { + StreamInfo::MockStreamInfo stream_info; + TestImpl logger(nullptr); + EXPECT_EQ(logger.count(), 0); + logger.log(nullptr, nullptr, nullptr, stream_info); + EXPECT_EQ(logger.count(), 1); +} + +TEST(AccessLogBaseTest, FilterReject) { + StreamInfo::MockStreamInfo stream_info; + + std::unique_ptr filter = std::make_unique(); + EXPECT_CALL(*filter, evaluate(_, _, _, _)).WillOnce(Return(false)); + TestImpl logger(std::move(filter)); + EXPECT_EQ(logger.count(), 0); + logger.log(nullptr, nullptr, nullptr, stream_info); + EXPECT_EQ(logger.count(), 0); +} + +} // namespace +} // namespace Common +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy From 6ca8500b39568049647f4f40a599b8f8bd2afc3d Mon Sep 17 00:00:00 2001 From: asraa Date: Tue, 16 Jul 2019 19:55:31 -0400 Subject: [PATCH 185/542] test: separate AdsIntegrationTest to a library (#7581) Moves AdsIntegrationTest to it's own file. I'd like to reuse the class for the xDS fuzzer (#7543), since the setup and utilities are the same. Risk Level: Low Testing: n/a Docs Changes: n/a Release Notes: n/a Signed-off-by: Asra Ali --- test/integration/BUILD | 38 ++- test/integration/ads_integration.cc | 280 +++++++++++++++++++++ test/integration/ads_integration.h | 71 ++++++ test/integration/ads_integration_test.cc | 296 +---------------------- 4 files changed, 383 insertions(+), 302 deletions(-) create mode 100644 test/integration/ads_integration.cc create mode 100644 test/integration/ads_integration.h diff --git a/test/integration/BUILD b/test/integration/BUILD index 708030bc66fb..64949483c84e 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -17,9 +17,14 @@ load( envoy_package() -envoy_cc_test( - name = "ads_integration_test", - srcs = ["ads_integration_test.cc"], +envoy_cc_test_library( + name = "ads_integration_lib", + srcs = [ + "ads_integration.cc", + ], + hdrs = [ + "ads_integration.h", + ], data = [ "//test/config/integration/certs", ], @@ -28,13 +33,28 @@ envoy_cc_test( "//source/common/config:protobuf_link_hacks", "//source/common/config:resources_lib", "//source/common/protobuf:utility_lib", - "//source/extensions/transport_sockets/tls:config", - "//source/extensions/transport_sockets/tls:context_config_lib", - "//source/extensions/transport_sockets/tls:context_lib", - "//source/extensions/transport_sockets/tls:ssl_socket_lib", "//test/common/grpc:grpc_client_integration_lib", - "//test/mocks/runtime:runtime_mocks", - "//test/mocks/server:server_mocks", + "//test/test_common:network_utility_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:cds_cc", + "@envoy_api//envoy/api/v2:discovery_cc", + "@envoy_api//envoy/api/v2:eds_cc", + "@envoy_api//envoy/api/v2:lds_cc", + "@envoy_api//envoy/api/v2:rds_cc", + "@envoy_api//envoy/service/discovery/v2:ads_cc", + ], +) + +envoy_cc_test( + name = "ads_integration_test", + srcs = ["ads_integration_test.cc"], + deps = [ + ":ads_integration_lib", + ":http_integration_lib", + "//source/common/config:protobuf_link_hacks", + "//source/common/config:resources_lib", + "//source/common/protobuf:utility_lib", + "//test/common/grpc:grpc_client_integration_lib", "//test/test_common:network_utility_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/api/v2:cds_cc", diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc new file mode 100644 index 000000000000..a950bc8e5899 --- /dev/null +++ b/test/integration/ads_integration.cc @@ -0,0 +1,280 @@ +#include "test/integration/ads_integration.h" + +#include "envoy/api/v2/cds.pb.h" +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/api/v2/eds.pb.h" +#include "envoy/api/v2/lds.pb.h" +#include "envoy/api/v2/rds.pb.h" + +#include "common/config/protobuf_link_hacks.h" +#include "common/config/resources.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +#include "test/test_common/network_utility.h" +#include "test/test_common/utility.h" + +using testing::AssertionFailure; +using testing::AssertionResult; +using testing::AssertionSuccess; +using testing::IsSubstring; + +namespace Envoy { + +AdsIntegrationTest::AdsIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { + use_lds_ = false; + create_xds_upstream_ = true; + tls_xds_upstream_ = true; +} + +void AdsIntegrationTest::TearDown() { + cleanUpXdsConnection(); + test_server_.reset(); + fake_upstreams_.clear(); +} + +envoy::api::v2::Cluster AdsIntegrationTest::buildCluster(const std::string& name) { + return TestUtility::parseYaml(fmt::format(R"EOF( + name: {} + connect_timeout: 5s + type: EDS + eds_cluster_config: {{ eds_config: {{ ads: {{}} }} }} + lb_policy: ROUND_ROBIN + http2_protocol_options: {{}} + )EOF", + name)); +} + +envoy::api::v2::ClusterLoadAssignment +AdsIntegrationTest::buildClusterLoadAssignment(const std::string& name) { + return TestUtility::parseYaml( + fmt::format(R"EOF( + cluster_name: {} + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {} + port_value: {} + )EOF", + name, Network::Test::getLoopbackAddressString(ipVersion()), + fake_upstreams_[0]->localAddress()->ip()->port())); +} + +envoy::api::v2::Listener AdsIntegrationTest::buildListener(const std::string& name, + const std::string& route_config, + const std::string& stat_prefix) { + return TestUtility::parseYaml(fmt::format( + R"EOF( + name: {} + address: + socket_address: + address: {} + port_value: 0 + filter_chains: + filters: + - name: envoy.http_connection_manager + config: + stat_prefix: {} + codec_type: HTTP2 + rds: + route_config_name: {} + config_source: {{ ads: {{}} }} + http_filters: [{{ name: envoy.router }}] + )EOF", + name, Network::Test::getLoopbackAddressString(ipVersion()), stat_prefix, route_config)); +} + +envoy::api::v2::RouteConfiguration +AdsIntegrationTest::buildRouteConfig(const std::string& name, const std::string& cluster) { + return TestUtility::parseYaml(fmt::format(R"EOF( + name: {} + virtual_hosts: + - name: integration + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: {} }} + )EOF", + name, cluster)); +} + +void AdsIntegrationTest::makeSingleRequest() { + registerTestServerPorts({"http"}); + testRouterHeaderOnlyRequestAndResponse(); + cleanupUpstreamAndDownstream(); +} + +void AdsIntegrationTest::initialize() { initializeAds(false); } + +void AdsIntegrationTest::initializeAds(const bool rate_limiting) { + config_helper_.addConfigModifier([this, &rate_limiting]( + envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + auto* ads_config = bootstrap.mutable_dynamic_resources()->mutable_ads_config(); + if (rate_limiting) { + ads_config->mutable_rate_limit_settings(); + } + auto* grpc_service = ads_config->add_grpc_services(); + setGrpcService(*grpc_service, "ads_cluster", xds_upstream_->localAddress()); + auto* ads_cluster = bootstrap.mutable_static_resources()->add_clusters(); + ads_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + ads_cluster->set_name("ads_cluster"); + auto* context = ads_cluster->mutable_tls_context(); + auto* validation_context = context->mutable_common_tls_context()->mutable_validation_context(); + validation_context->mutable_trusted_ca()->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); + validation_context->add_verify_subject_alt_name("foo.lyft.com"); + if (clientType() == Grpc::ClientType::GoogleGrpc) { + auto* google_grpc = grpc_service->mutable_google_grpc(); + auto* ssl_creds = google_grpc->mutable_channel_credentials()->mutable_ssl_credentials(); + ssl_creds->mutable_root_certs()->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); + } + }); + setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + HttpIntegrationTest::initialize(); + if (xds_stream_ == nullptr) { + createXdsConnection(); + AssertionResult result = xds_connection_->waitForNewStream(*dispatcher_, xds_stream_); + RELEASE_ASSERT(result, result.message()); + xds_stream_->startGrpcStream(); + } +} + +void AdsIntegrationTest::testBasicFlow() { + // Send initial configuration, validate we can process a request. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("cluster_0")}, + {buildCluster("cluster_0")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", + {"cluster_0"}, {"cluster_0"}, {})); + sendDiscoveryResponse( + Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, + {buildClusterLoadAssignment("cluster_0")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); + sendDiscoveryResponse( + Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, + {buildListener("listener_0", "route_config_0")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", + {"cluster_0"}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", + {"route_config_0"}, {"route_config_0"}, {})); + sendDiscoveryResponse( + Config::TypeUrl::get().RouteConfiguration, {buildRouteConfig("route_config_0", "cluster_0")}, + {buildRouteConfig("route_config_0", "cluster_0")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "1", + {"route_config_0"}, {}, {})); + + test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); + makeSingleRequest(); + const ProtobufWkt::Timestamp first_active_listener_ts_1 = + getListenersConfigDump().dynamic_active_listeners()[0].last_updated(); + const ProtobufWkt::Timestamp first_active_cluster_ts_1 = + getClustersConfigDump().dynamic_active_clusters()[0].last_updated(); + const ProtobufWkt::Timestamp first_route_config_ts_1 = + getRoutesConfigDump().dynamic_route_configs()[0].last_updated(); + + // Upgrade RDS/CDS/EDS to a newer config, validate we can process a request. + sendDiscoveryResponse( + Config::TypeUrl::get().Cluster, {buildCluster("cluster_1"), buildCluster("cluster_2")}, + {buildCluster("cluster_1"), buildCluster("cluster_2")}, {"cluster_0"}, "2"); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2); + sendDiscoveryResponse( + Config::TypeUrl::get().ClusterLoadAssignment, + {buildClusterLoadAssignment("cluster_1"), buildClusterLoadAssignment("cluster_2")}, + {buildClusterLoadAssignment("cluster_1"), buildClusterLoadAssignment("cluster_2")}, + {"cluster_0"}, "2"); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", + {"cluster_2", "cluster_1"}, {"cluster_2", "cluster_1"}, + {"cluster_0"})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "2", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", + {"cluster_2", "cluster_1"}, {}, {})); + sendDiscoveryResponse( + Config::TypeUrl::get().RouteConfiguration, {buildRouteConfig("route_config_0", "cluster_1")}, + {buildRouteConfig("route_config_0", "cluster_1")}, {}, "2"); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "2", + {"route_config_0"}, {}, {})); + + makeSingleRequest(); + const ProtobufWkt::Timestamp first_active_listener_ts_2 = + getListenersConfigDump().dynamic_active_listeners()[0].last_updated(); + const ProtobufWkt::Timestamp first_active_cluster_ts_2 = + getClustersConfigDump().dynamic_active_clusters()[0].last_updated(); + const ProtobufWkt::Timestamp first_route_config_ts_2 = + getRoutesConfigDump().dynamic_route_configs()[0].last_updated(); + + // Upgrade LDS/RDS, validate we can process a request. + sendDiscoveryResponse(Config::TypeUrl::get().Listener, + {buildListener("listener_1", "route_config_1"), + buildListener("listener_2", "route_config_2")}, + {buildListener("listener_1", "route_config_1"), + buildListener("listener_2", "route_config_2")}, + {"listener_0"}, "2"); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "2", + {"route_config_2", "route_config_1", "route_config_0"}, + {"route_config_2", "route_config_1"}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "2", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "2", + {"route_config_2", "route_config_1"}, {}, + {"route_config_0"})); + sendDiscoveryResponse( + Config::TypeUrl::get().RouteConfiguration, + {buildRouteConfig("route_config_1", "cluster_1"), + buildRouteConfig("route_config_2", "cluster_1")}, + {buildRouteConfig("route_config_1", "cluster_1"), + buildRouteConfig("route_config_2", "cluster_1")}, + {"route_config_0"}, "3"); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "3", + {"route_config_2", "route_config_1"}, {}, {})); + + test_server_->waitForCounterGe("listener_manager.listener_create_success", 2); + makeSingleRequest(); + const ProtobufWkt::Timestamp first_active_listener_ts_3 = + getListenersConfigDump().dynamic_active_listeners()[0].last_updated(); + const ProtobufWkt::Timestamp first_active_cluster_ts_3 = + getClustersConfigDump().dynamic_active_clusters()[0].last_updated(); + const ProtobufWkt::Timestamp first_route_config_ts_3 = + getRoutesConfigDump().dynamic_route_configs()[0].last_updated(); + + // Expect last_updated timestamps to be updated in a predictable way + // For the listener configs in this example, 1 == 2 < 3. + EXPECT_EQ(first_active_listener_ts_2, first_active_listener_ts_1); + EXPECT_GT(first_active_listener_ts_3, first_active_listener_ts_2); + // For the cluster configs in this example, 1 < 2 == 3. + EXPECT_GT(first_active_cluster_ts_2, first_active_cluster_ts_1); + EXPECT_EQ(first_active_cluster_ts_3, first_active_cluster_ts_2); + // For the route configs in this example, 1 < 2 < 3. + EXPECT_GT(first_route_config_ts_2, first_route_config_ts_1); + EXPECT_GT(first_route_config_ts_3, first_route_config_ts_2); +} + +envoy::admin::v2alpha::ClustersConfigDump AdsIntegrationTest::getClustersConfigDump() { + auto message_ptr = + test_server_->server().admin().getConfigTracker().getCallbacksMap().at("clusters")(); + return dynamic_cast(*message_ptr); +} + +envoy::admin::v2alpha::ListenersConfigDump AdsIntegrationTest::getListenersConfigDump() { + auto message_ptr = + test_server_->server().admin().getConfigTracker().getCallbacksMap().at("listeners")(); + return dynamic_cast(*message_ptr); +} + +envoy::admin::v2alpha::RoutesConfigDump AdsIntegrationTest::getRoutesConfigDump() { + auto message_ptr = + test_server_->server().admin().getConfigTracker().getCallbacksMap().at("routes")(); + return dynamic_cast(*message_ptr); +} + +} // namespace Envoy diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h new file mode 100644 index 000000000000..c03e42e645bf --- /dev/null +++ b/test/integration/ads_integration.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include + +#include "envoy/admin/v2alpha/config_dump.pb.h" +#include "envoy/api/v2/cds.pb.h" +#include "envoy/api/v2/eds.pb.h" +#include "envoy/api/v2/lds.pb.h" +#include "envoy/api/v2/rds.pb.h" + +#include "test/common/grpc/grpc_client_integration.h" +#include "test/integration/http_integration.h" + +namespace Envoy { +static const std::string& AdsIntegrationConfig() { + CONSTRUCT_ON_FIRST_USE(std::string, R"EOF( +dynamic_resources: + lds_config: {ads: {}} + cds_config: {ads: {}} + ads_config: + api_type: GRPC +static_resources: + clusters: + name: dummy_cluster + connect_timeout: { seconds: 5 } + type: STATIC + hosts: + socket_address: + address: 127.0.0.1 + port_value: 0 + lb_policy: ROUND_ROBIN + http2_protocol_options: {} +admin: + access_log_path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 0 +)EOF"); +} + +class AdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public HttpIntegrationTest { +public: + AdsIntegrationTest(); + + void TearDown() override; + + envoy::api::v2::Cluster buildCluster(const std::string& name); + + envoy::api::v2::ClusterLoadAssignment buildClusterLoadAssignment(const std::string& name); + + envoy::api::v2::Listener buildListener(const std::string& name, const std::string& route_config, + const std::string& stat_prefix = "ads_test"); + + envoy::api::v2::RouteConfiguration buildRouteConfig(const std::string& name, + const std::string& cluster); + + void makeSingleRequest(); + + void initialize() override; + void initializeAds(const bool rate_limiting); + + void testBasicFlow(); + + envoy::admin::v2alpha::ClustersConfigDump getClustersConfigDump(); + envoy::admin::v2alpha::ListenersConfigDump getListenersConfigDump(); + envoy::admin::v2alpha::RoutesConfigDump getRoutesConfigDump(); +}; + +} // namespace Envoy diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 507287362055..e87fdff3f6f4 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -7,7 +7,6 @@ #include "envoy/api/v2/route/route.pb.h" #include "envoy/grpc/status.h" #include "envoy/service/discovery/v2/ads.pb.h" -#include "envoy/stats/scope.h" #include "common/config/protobuf_link_hacks.h" #include "common/config/resources.h" @@ -15,11 +14,10 @@ #include "common/protobuf/utility.h" #include "test/common/grpc/grpc_client_integration.h" +#include "test/integration/ads_integration.h" #include "test/integration/http_integration.h" #include "test/integration/utility.h" -#include "test/mocks/server/mocks.h" #include "test/test_common/network_utility.h" -#include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -30,293 +28,6 @@ using testing::AssertionSuccess; using testing::IsSubstring; namespace Envoy { -namespace { - -const std::string config = R"EOF( -dynamic_resources: - lds_config: {ads: {}} - cds_config: {ads: {}} - ads_config: - api_type: GRPC -static_resources: - clusters: - name: dummy_cluster - connect_timeout: { seconds: 5 } - type: STATIC - hosts: - socket_address: - address: 127.0.0.1 - port_value: 0 - lb_policy: ROUND_ROBIN - http2_protocol_options: {} -admin: - access_log_path: /dev/null - address: - socket_address: - address: 127.0.0.1 - port_value: 0 -)EOF"; - -class AdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public HttpIntegrationTest { -public: - AdsIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), config) { - use_lds_ = false; - create_xds_upstream_ = true; - tls_xds_upstream_ = true; - } - - void TearDown() override { - cleanUpXdsConnection(); - test_server_.reset(); - fake_upstreams_.clear(); - } - - envoy::api::v2::Cluster buildCluster(const std::string& name) { - return TestUtility::parseYaml(fmt::format(R"EOF( - name: {} - connect_timeout: 5s - type: EDS - eds_cluster_config: {{ eds_config: {{ ads: {{}} }} }} - lb_policy: ROUND_ROBIN - http2_protocol_options: {{}} - )EOF", - name)); - } - - envoy::api::v2::ClusterLoadAssignment buildClusterLoadAssignment(const std::string& name) { - return TestUtility::parseYaml( - fmt::format(R"EOF( - cluster_name: {} - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: {} - port_value: {} - )EOF", - name, Network::Test::getLoopbackAddressString(ipVersion()), - fake_upstreams_[0]->localAddress()->ip()->port())); - } - - envoy::api::v2::Listener buildListener(const std::string& name, const std::string& route_config, - const std::string& stat_prefix = "ads_test") { - return TestUtility::parseYaml(fmt::format( - R"EOF( - name: {} - address: - socket_address: - address: {} - port_value: 0 - filter_chains: - filters: - - name: envoy.http_connection_manager - config: - stat_prefix: {} - codec_type: HTTP2 - rds: - route_config_name: {} - config_source: {{ ads: {{}} }} - http_filters: [{{ name: envoy.router }}] - )EOF", - name, Network::Test::getLoopbackAddressString(ipVersion()), stat_prefix, route_config)); - } - - envoy::api::v2::RouteConfiguration buildRouteConfig(const std::string& name, - const std::string& cluster) { - return TestUtility::parseYaml(fmt::format(R"EOF( - name: {} - virtual_hosts: - - name: integration - domains: ["*"] - routes: - - match: {{ prefix: "/" }} - route: {{ cluster: {} }} - )EOF", - name, cluster)); - } - - void makeSingleRequest() { - registerTestServerPorts({"http"}); - testRouterHeaderOnlyRequestAndResponse(); - cleanupUpstreamAndDownstream(); - } - - void initialize() override { initializeAds(false); } - - void initializeAds(const bool rate_limiting) { - config_helper_.addConfigModifier( - [this, &rate_limiting](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { - auto* ads_config = bootstrap.mutable_dynamic_resources()->mutable_ads_config(); - if (rate_limiting) { - ads_config->mutable_rate_limit_settings(); - } - auto* grpc_service = ads_config->add_grpc_services(); - setGrpcService(*grpc_service, "ads_cluster", xds_upstream_->localAddress()); - auto* ads_cluster = bootstrap.mutable_static_resources()->add_clusters(); - ads_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); - ads_cluster->set_name("ads_cluster"); - auto* context = ads_cluster->mutable_tls_context(); - auto* validation_context = - context->mutable_common_tls_context()->mutable_validation_context(); - validation_context->mutable_trusted_ca()->set_filename( - TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); - validation_context->add_verify_subject_alt_name("foo.lyft.com"); - if (clientType() == Grpc::ClientType::GoogleGrpc) { - auto* google_grpc = grpc_service->mutable_google_grpc(); - auto* ssl_creds = google_grpc->mutable_channel_credentials()->mutable_ssl_credentials(); - ssl_creds->mutable_root_certs()->set_filename( - TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); - } - }); - setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); - HttpIntegrationTest::initialize(); - if (xds_stream_ == nullptr) { - createXdsConnection(); - AssertionResult result = xds_connection_->waitForNewStream(*dispatcher_, xds_stream_); - RELEASE_ASSERT(result, result.message()); - xds_stream_->startGrpcStream(); - } - } - - void testBasicFlow() { - // Send initial configuration, validate we can process a request. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); - sendDiscoveryResponse(Config::TypeUrl::get().Cluster, - {buildCluster("cluster_0")}, - {buildCluster("cluster_0")}, {}, "1"); - - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", - {"cluster_0"}, {"cluster_0"}, {})); - sendDiscoveryResponse( - Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, - {buildClusterLoadAssignment("cluster_0")}, {}, "1"); - - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); - sendDiscoveryResponse( - Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, - {buildListener("listener_0", "route_config_0")}, {}, "1"); - - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", - {"cluster_0"}, {}, {})); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", - {"route_config_0"}, {"route_config_0"}, {})); - sendDiscoveryResponse( - Config::TypeUrl::get().RouteConfiguration, - {buildRouteConfig("route_config_0", "cluster_0")}, - {buildRouteConfig("route_config_0", "cluster_0")}, {}, "1"); - - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "1", - {"route_config_0"}, {}, {})); - - test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); - makeSingleRequest(); - const ProtobufWkt::Timestamp first_active_listener_ts_1 = - getListenersConfigDump().dynamic_active_listeners()[0].last_updated(); - const ProtobufWkt::Timestamp first_active_cluster_ts_1 = - getClustersConfigDump().dynamic_active_clusters()[0].last_updated(); - const ProtobufWkt::Timestamp first_route_config_ts_1 = - getRoutesConfigDump().dynamic_route_configs()[0].last_updated(); - - // Upgrade RDS/CDS/EDS to a newer config, validate we can process a request. - sendDiscoveryResponse( - Config::TypeUrl::get().Cluster, {buildCluster("cluster_1"), buildCluster("cluster_2")}, - {buildCluster("cluster_1"), buildCluster("cluster_2")}, {"cluster_0"}, "2"); - test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2); - sendDiscoveryResponse( - Config::TypeUrl::get().ClusterLoadAssignment, - {buildClusterLoadAssignment("cluster_1"), buildClusterLoadAssignment("cluster_2")}, - {buildClusterLoadAssignment("cluster_1"), buildClusterLoadAssignment("cluster_2")}, - {"cluster_0"}, "2"); - test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", - {"cluster_2", "cluster_1"}, {"cluster_2", "cluster_1"}, - {"cluster_0"})); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "2", {}, {}, {})); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", - {"cluster_2", "cluster_1"}, {}, {})); - sendDiscoveryResponse( - Config::TypeUrl::get().RouteConfiguration, - {buildRouteConfig("route_config_0", "cluster_1")}, - {buildRouteConfig("route_config_0", "cluster_1")}, {}, "2"); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "2", - {"route_config_0"}, {}, {})); - - makeSingleRequest(); - const ProtobufWkt::Timestamp first_active_listener_ts_2 = - getListenersConfigDump().dynamic_active_listeners()[0].last_updated(); - const ProtobufWkt::Timestamp first_active_cluster_ts_2 = - getClustersConfigDump().dynamic_active_clusters()[0].last_updated(); - const ProtobufWkt::Timestamp first_route_config_ts_2 = - getRoutesConfigDump().dynamic_route_configs()[0].last_updated(); - - // Upgrade LDS/RDS, validate we can process a request. - sendDiscoveryResponse(Config::TypeUrl::get().Listener, - {buildListener("listener_1", "route_config_1"), - buildListener("listener_2", "route_config_2")}, - {buildListener("listener_1", "route_config_1"), - buildListener("listener_2", "route_config_2")}, - {"listener_0"}, "2"); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "2", - {"route_config_2", "route_config_1", "route_config_0"}, - {"route_config_2", "route_config_1"}, {})); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "2", {}, {}, {})); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "2", - {"route_config_2", "route_config_1"}, {}, - {"route_config_0"})); - sendDiscoveryResponse( - Config::TypeUrl::get().RouteConfiguration, - {buildRouteConfig("route_config_1", "cluster_1"), - buildRouteConfig("route_config_2", "cluster_1")}, - {buildRouteConfig("route_config_1", "cluster_1"), - buildRouteConfig("route_config_2", "cluster_1")}, - {"route_config_0"}, "3"); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "3", - {"route_config_2", "route_config_1"}, {}, {})); - - test_server_->waitForCounterGe("listener_manager.listener_create_success", 2); - makeSingleRequest(); - const ProtobufWkt::Timestamp first_active_listener_ts_3 = - getListenersConfigDump().dynamic_active_listeners()[0].last_updated(); - const ProtobufWkt::Timestamp first_active_cluster_ts_3 = - getClustersConfigDump().dynamic_active_clusters()[0].last_updated(); - const ProtobufWkt::Timestamp first_route_config_ts_3 = - getRoutesConfigDump().dynamic_route_configs()[0].last_updated(); - - // Expect last_updated timestamps to be updated in a predictable way - // For the listener configs in this example, 1 == 2 < 3. - EXPECT_EQ(first_active_listener_ts_2, first_active_listener_ts_1); - EXPECT_GT(first_active_listener_ts_3, first_active_listener_ts_2); - // For the cluster configs in this example, 1 < 2 == 3. - EXPECT_GT(first_active_cluster_ts_2, first_active_cluster_ts_1); - EXPECT_EQ(first_active_cluster_ts_3, first_active_cluster_ts_2); - // For the route configs in this example, 1 < 2 < 3. - EXPECT_GT(first_route_config_ts_2, first_route_config_ts_1); - EXPECT_GT(first_route_config_ts_3, first_route_config_ts_2); - } - - envoy::admin::v2alpha::ClustersConfigDump getClustersConfigDump() { - auto message_ptr = - test_server_->server().admin().getConfigTracker().getCallbacksMap().at("clusters")(); - return dynamic_cast(*message_ptr); - } - - envoy::admin::v2alpha::ListenersConfigDump getListenersConfigDump() { - auto message_ptr = - test_server_->server().admin().getConfigTracker().getCallbacksMap().at("listeners")(); - return dynamic_cast(*message_ptr); - } - - envoy::admin::v2alpha::RoutesConfigDump getRoutesConfigDump() { - auto message_ptr = - test_server_->server().admin().getConfigTracker().getCallbacksMap().at("routes")(); - return dynamic_cast(*message_ptr); - } - - Extensions::TransportSockets::Tls::ContextManagerImpl context_manager_{timeSystem()}; -}; INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsIntegrationTest, GRPC_CLIENT_INTEGRATION_PARAMS); @@ -825,7 +536,7 @@ class AdsFailIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public HttpIntegrationTest { public: AdsFailIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), config) { + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { create_xds_upstream_ = true; use_lds_ = false; } @@ -866,7 +577,7 @@ class AdsConfigIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public HttpIntegrationTest { public: AdsConfigIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), config) { + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { create_xds_upstream_ = true; use_lds_ = false; } @@ -998,5 +709,4 @@ TEST_P(AdsIntegrationTest, ListenerDrainBeforeServerStart) { test_server_->waitForGaugeEq("listener_manager.total_listeners_active", 0); } -} // namespace } // namespace Envoy From ea3ebca3b6d84a8b29c35ca03fa3666a0f4951c9 Mon Sep 17 00:00:00 2001 From: easy Date: Wed, 17 Jul 2019 13:44:05 +1000 Subject: [PATCH 186/542] Set zipkin_service_name to LocalInfo::clusterName(). (#7354) Signed-off-by: Emil Mikulic --- api/envoy/config/trace/v2/trace.proto | 3 +-- source/extensions/tracers/opencensus/config.cc | 2 +- .../tracers/opencensus/opencensus_tracer_impl.cc | 7 ++++--- .../tracers/opencensus/opencensus_tracer_impl.h | 5 ++++- test/extensions/tracers/opencensus/BUILD | 1 + test/extensions/tracers/opencensus/config_test.cc | 1 - test/extensions/tracers/opencensus/tracer_test.cc | 10 +++++++--- 7 files changed, 18 insertions(+), 11 deletions(-) diff --git a/api/envoy/config/trace/v2/trace.proto b/api/envoy/config/trace/v2/trace.proto index ac2a9cbb7649..3fe4d1fff66f 100644 --- a/api/envoy/config/trace/v2/trace.proto +++ b/api/envoy/config/trace/v2/trace.proto @@ -130,8 +130,7 @@ message OpenCensusConfig { // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" string zipkin_url = 6; - // The Zipkin service name. - string zipkin_service_name = 7; + reserved 7; // Formerly zipkin_service_name. enum TraceContext { // No-op default, no trace context is utilized. diff --git a/source/extensions/tracers/opencensus/config.cc b/source/extensions/tracers/opencensus/config.cc index 038c329aff18..ec0e766e532d 100644 --- a/source/extensions/tracers/opencensus/config.cc +++ b/source/extensions/tracers/opencensus/config.cc @@ -16,7 +16,7 @@ OpenCensusTracerFactory::OpenCensusTracerFactory() : FactoryBase(TracerNames::ge Tracing::HttpTracerPtr OpenCensusTracerFactory::createHttpTracerTyped( const envoy::config::trace::v2::OpenCensusConfig& proto_config, Server::Instance& server) { - Tracing::DriverPtr driver = std::make_unique(proto_config); + Tracing::DriverPtr driver = std::make_unique(proto_config, server.localInfo()); return std::make_unique(std::move(driver), server.localInfo()); } diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 6a93dbbffd90..12c0cfd19f54 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -187,8 +187,9 @@ void Span::setSampled(bool sampled) { span_.AddAnnotation("setSampled", {{"sampl } // namespace -Driver::Driver(const envoy::config::trace::v2::OpenCensusConfig& oc_config) - : oc_config_(oc_config) { +Driver::Driver(const envoy::config::trace::v2::OpenCensusConfig& oc_config, + const LocalInfo::LocalInfo& localinfo) + : oc_config_(oc_config), local_info_(localinfo) { if (oc_config.has_trace_config()) { applyTraceConfig(oc_config.trace_config()); } @@ -202,7 +203,7 @@ Driver::Driver(const envoy::config::trace::v2::OpenCensusConfig& oc_config) } if (oc_config.zipkin_exporter_enabled()) { ::opencensus::exporters::trace::ZipkinExporterOptions opts(oc_config.zipkin_url()); - opts.service_name = oc_config.zipkin_service_name(); + opts.service_name = local_info_.clusterName(); ::opencensus::exporters::trace::ZipkinExporter::Register(opts); } } diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.h b/source/extensions/tracers/opencensus/opencensus_tracer_impl.h index cb5990bd763a..1b334e928172 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.h +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/config/trace/v2/trace.pb.validate.h" +#include "envoy/local_info/local_info.h" #include "envoy/tracing/http_tracer.h" #include "common/common/logger.h" @@ -15,7 +16,8 @@ namespace OpenCensus { */ class Driver : public Tracing::Driver, Logger::Loggable { public: - Driver(const envoy::config::trace::v2::OpenCensusConfig& oc_config); + Driver(const envoy::config::trace::v2::OpenCensusConfig& oc_config, + const LocalInfo::LocalInfo& localinfo); /** * Implements the abstract Driver's startSpan operation. @@ -28,6 +30,7 @@ class Driver : public Tracing::Driver, Logger::Loggable { void applyTraceConfig(const opencensus::proto::trace::v1::TraceConfig& config); const envoy::config::trace::v2::OpenCensusConfig oc_config_; + const LocalInfo::LocalInfo& local_info_; }; } // namespace OpenCensus diff --git a/test/extensions/tracers/opencensus/BUILD b/test/extensions/tracers/opencensus/BUILD index d4d7d4cfb079..03b343f5884b 100644 --- a/test/extensions/tracers/opencensus/BUILD +++ b/test/extensions/tracers/opencensus/BUILD @@ -19,6 +19,7 @@ envoy_extension_cc_test( deps = [ "//source/extensions/tracers/opencensus:opencensus_tracer_impl", "//test/mocks/http:http_mocks", + "//test/mocks/local_info:local_info_mocks", "//test/mocks/tracing:tracing_mocks", ], ) diff --git a/test/extensions/tracers/opencensus/config_test.cc b/test/extensions/tracers/opencensus/config_test.cc index 31442eee7007..1f549cd4afb1 100644 --- a/test/extensions/tracers/opencensus/config_test.cc +++ b/test/extensions/tracers/opencensus/config_test.cc @@ -50,7 +50,6 @@ TEST(OpenCensusTracerConfigTest, OpenCensusHttpTracerWithTypedConfig) { stackdriver_project_id: test_project_id zipkin_exporter_enabled: true zipkin_url: http://127.0.0.1:9411/api/v2/spans - zipkin_service_name: test_service incoming_trace_context: trace_context incoming_trace_context: grpc_trace_bin incoming_trace_context: cloud_trace_context diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index a223b08e3be5..83d622b70bcf 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -12,6 +12,7 @@ #include "extensions/tracers/opencensus/opencensus_tracer_impl.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/local_info/mocks.h" #include "test/mocks/tracing/mocks.h" #include "gmock/gmock.h" @@ -101,7 +102,8 @@ void registerSpanCatcher() { TEST(OpenCensusTracerTest, Span) { registerSpanCatcher(); OpenCensusConfig oc_config; - std::unique_ptr driver(new OpenCensus::Driver(oc_config)); + NiceMock local_info; + std::unique_ptr driver(new OpenCensus::Driver(oc_config, local_info)); NiceMock config; Http::TestHeaderMapImpl request_headers{ @@ -166,6 +168,7 @@ TEST(OpenCensusTracerTest, PropagateTraceContext) { // The test calls the helper with each kind of incoming context in turn. auto helper = [](const std::string& header, const std::string& value) { OpenCensusConfig oc_config; + NiceMock local_info; oc_config.add_incoming_trace_context(OpenCensusConfig::NONE); oc_config.add_incoming_trace_context(OpenCensusConfig::TRACE_CONTEXT); oc_config.add_incoming_trace_context(OpenCensusConfig::GRPC_TRACE_BIN); @@ -174,7 +177,7 @@ TEST(OpenCensusTracerTest, PropagateTraceContext) { oc_config.add_outgoing_trace_context(OpenCensusConfig::TRACE_CONTEXT); oc_config.add_outgoing_trace_context(OpenCensusConfig::GRPC_TRACE_BIN); oc_config.add_outgoing_trace_context(OpenCensusConfig::CLOUD_TRACE_CONTEXT); - std::unique_ptr driver(new OpenCensus::Driver(oc_config)); + std::unique_ptr driver(new OpenCensus::Driver(oc_config, local_info)); NiceMock config; Http::TestHeaderMapImpl request_headers{ {":path", "/"}, @@ -239,7 +242,8 @@ namespace { // the exporter (either zero or one). int SamplerTestHelper(const OpenCensusConfig& oc_config) { registerSpanCatcher(); - std::unique_ptr driver(new OpenCensus::Driver(oc_config)); + NiceMock local_info; + std::unique_ptr driver(new OpenCensus::Driver(oc_config, local_info)); auto span = ::opencensus::trace::Span::StartSpan("test_span"); span.End(); // Retrieve SpanData from the OpenCensus trace exporter. From 1179d6e1aa4189ec3e84b93428043ae1412e027d Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 17 Jul 2019 01:14:57 -0700 Subject: [PATCH 187/542] fix coverage publish script (#7608) #7552 breaks master coverage build. Risk Level: Low Testing: cannot be tested via presubmit only Docs Changes: n/a Release Notes: n/a Signed-off-by: Lizan Zhou --- ci/coverage_publish.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/coverage_publish.sh b/ci/coverage_publish.sh index a1e08fbc4e95..ad4b05ba3019 100755 --- a/ci/coverage_publish.sh +++ b/ci/coverage_publish.sh @@ -14,7 +14,7 @@ then echo "Uploading coverage report..." [[ -z "${ENVOY_BUILD_DIR}" ]] && ENVOY_BUILD_DIR=/build - COVERAGE_FILE="${ENVOY_BUILD_DIR}/envoy/generated/coverage/coverage.html" + COVERAGE_FILE="${ENVOY_BUILD_DIR}/envoy/generated/coverage/index.html" if [ ! -f "${COVERAGE_FILE}" ]; then echo "ERROR: Coverage file not found." @@ -27,7 +27,7 @@ then pip install awscli --upgrade aws s3 cp "${COVERAGE_DIR}" "s3://${S3_LOCATION}" --recursive --acl public-read --quiet --sse - echo "Coverage report for branch '${BRANCH_NAME}': https://s3.amazonaws.com/${S3_LOCATION}/coverage.html" + echo "Coverage report for branch '${BRANCH_NAME}': https://s3.amazonaws.com/${S3_LOCATION}/index.html" else echo "Coverage report will not be uploaded for this build." fi From a8e13646852e11c08e69b75e61337164b52cac7c Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Wed, 17 Jul 2019 08:11:35 -0700 Subject: [PATCH 188/542] admin: Add socket options (#7562) Signed-off-by: Ruslan Nigmatullin --- api/envoy/config/bootstrap/v2/bootstrap.proto | 4 ++ docs/root/intro/version_history.rst | 1 + include/envoy/server/admin.h | 1 + include/envoy/server/configuration.h | 5 +++ source/server/BUILD | 1 + source/server/config_validation/admin.cc | 1 + source/server/config_validation/admin.h | 1 + source/server/configuration_impl.cc | 7 +++ source/server/configuration_impl.h | 2 + source/server/http/admin.cc | 3 +- source/server/http/admin.h | 1 + source/server/server.cc | 1 + test/mocks/server/mocks.h | 3 +- test/server/BUILD | 1 + test/server/configuration_impl_test.cc | 44 +++++++++++++++++++ test/server/http/admin_test.cc | 4 +- ...e_bootstrap_with_admin_socket_options.yaml | 19 ++++++++ test/server/server_test.cc | 15 +++++++ test/test_common/environment.cc | 8 ++++ 19 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 test/server/node_bootstrap_with_admin_socket_options.yaml diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index ab841058c0b1..01ea30d2bcf3 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -161,6 +161,10 @@ message Admin { // The TCP address that the administration server will listen on. // If not specified, Envoy will not start an administration server. envoy.api.v2.core.Address address = 3; + + // Additional socket options that may not be present in Envoy source code or + // precompiled binaries. + repeated envoy.api.v2.core.SocketOption socket_options = 4; } // Cluster manager :ref:`architecture overview `. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 566ef6d895c0..c81ece57fdfd 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -3,6 +3,7 @@ Version history 1.12.0 (pending) ================ +* admin: added ability to configure listener :ref:`socket options `. * config: async data access for local and remote data source. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. diff --git a/include/envoy/server/admin.h b/include/envoy/server/admin.h index 7e5a4b5f33c2..56cabe8ab462 100644 --- a/include/envoy/server/admin.h +++ b/include/envoy/server/admin.h @@ -124,6 +124,7 @@ class Admin { virtual void startHttpListener(const std::string& access_log_path_, const std::string& address_out_path, Network::Address::InstanceConstSharedPtr address, + const Network::Socket::OptionsSharedPtr& socket_options, Stats::ScopePtr&& listener_scope) PURE; /** diff --git a/include/envoy/server/configuration.h b/include/envoy/server/configuration.h index df2022ba9cee..21a2e583fbad 100644 --- a/include/envoy/server/configuration.h +++ b/include/envoy/server/configuration.h @@ -92,6 +92,11 @@ class Admin { * @return Network::Address::InstanceConstSharedPtr the server address. */ virtual Network::Address::InstanceConstSharedPtr address() PURE; + + /** + * @return Network::Address::OptionsSharedPtr the list of listener socket options. + */ + virtual Network::Socket::OptionsSharedPtr socketOptions() PURE; }; /** diff --git a/source/server/BUILD b/source/server/BUILD index 9eb61a7fca69..538f25f922be 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -41,6 +41,7 @@ envoy_cc_library( "//source/common/config:runtime_utility_lib", "//source/common/config:utility_lib", "//source/common/network:resolver_lib", + "//source/common/network:socket_option_factory_lib", "//source/common/network:utility_lib", "//source/common/protobuf:utility_lib", "//source/common/tracing:http_tracer_lib", diff --git a/source/server/config_validation/admin.cc b/source/server/config_validation/admin.cc index 8535122d2eca..365309c17427 100644 --- a/source/server/config_validation/admin.cc +++ b/source/server/config_validation/admin.cc @@ -15,6 +15,7 @@ ConfigTracker& ValidationAdmin::getConfigTracker() { return config_tracker_; } void ValidationAdmin::startHttpListener(const std::string&, const std::string&, Network::Address::InstanceConstSharedPtr, + const Network::Socket::OptionsSharedPtr&, Stats::ScopePtr&&) { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } diff --git a/source/server/config_validation/admin.h b/source/server/config_validation/admin.h index 44863e2dfff3..8ff101dce813 100644 --- a/source/server/config_validation/admin.h +++ b/source/server/config_validation/admin.h @@ -22,6 +22,7 @@ class ValidationAdmin : public Admin { ConfigTracker& getConfigTracker() override; void startHttpListener(const std::string& access_log_path, const std::string& address_out_path, Network::Address::InstanceConstSharedPtr address, + const Network::Socket::OptionsSharedPtr&, Stats::ScopePtr&& listener_scope) override; Http::Code request(absl::string_view path_and_query, absl::string_view method, Http::HeaderMap& response_headers, std::string& body) override; diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc index 6f53db07d801..20c87a41e039 100644 --- a/source/server/configuration_impl.cc +++ b/source/server/configuration_impl.cc @@ -17,6 +17,7 @@ #include "common/common/utility.h" #include "common/config/runtime_utility.h" #include "common/config/utility.h" +#include "common/network/socket_option_factory.h" #include "common/protobuf/utility.h" #include "common/runtime/runtime_impl.h" #include "common/tracing/http_tracer_impl.h" @@ -133,6 +134,12 @@ InitialImpl::InitialImpl(const envoy::config::bootstrap::v2::Bootstrap& bootstra if (admin.has_address()) { admin_.address_ = Network::Address::resolveProtoAddress(admin.address()); } + admin_.socket_options_ = std::make_shared>(); + if (!admin.socket_options().empty()) { + Network::Socket::appendOptions( + admin_.socket_options_, + Network::SocketOptionFactory::buildLiteralOptions(admin.socket_options())); + } if (!bootstrap.flags_path().empty()) { flags_path_ = bootstrap.flags_path(); diff --git a/source/server/configuration_impl.h b/source/server/configuration_impl.h index 772c3c332546..4aa4852eb0ca 100644 --- a/source/server/configuration_impl.h +++ b/source/server/configuration_impl.h @@ -159,10 +159,12 @@ class InitialImpl : public Initial { const std::string& accessLogPath() override { return access_log_path_; } const std::string& profilePath() override { return profile_path_; } Network::Address::InstanceConstSharedPtr address() override { return address_; } + Network::Socket::OptionsSharedPtr socketOptions() override { return socket_options_; } std::string access_log_path_; std::string profile_path_; Network::Address::InstanceConstSharedPtr address_; + Network::Socket::OptionsSharedPtr socket_options_; }; AdminImpl admin_; diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 598699859d90..17e655af186f 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -1167,13 +1167,14 @@ AdminImpl::NullRouteConfigProvider::NullRouteConfigProvider(TimeSource& time_sou void AdminImpl::startHttpListener(const std::string& access_log_path, const std::string& address_out_path, Network::Address::InstanceConstSharedPtr address, + const Network::Socket::OptionsSharedPtr& socket_options, Stats::ScopePtr&& listener_scope) { // TODO(mattklein123): Allow admin to use normal access logger extension loading and avoid the // hard dependency here. access_logs_.emplace_back(new Extensions::AccessLoggers::File::FileAccessLog( access_log_path, {}, AccessLog::AccessLogFormatUtils::defaultAccessLogFormatter(), server_.accessLogManager())); - socket_ = std::make_unique(address, nullptr, true); + socket_ = std::make_unique(address, socket_options, true); listener_ = std::make_unique(*this, std::move(listener_scope)); if (!address_out_path.empty()) { std::ofstream address_out_file(address_out_path); diff --git a/source/server/http/admin.h b/source/server/http/admin.h index f13129a0e679..664a7f27884e 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -76,6 +76,7 @@ class AdminImpl : public Admin, void startHttpListener(const std::string& access_log_path, const std::string& address_out_path, Network::Address::InstanceConstSharedPtr address, + const Network::Socket::OptionsSharedPtr& socket_options, Stats::ScopePtr&& listener_scope) override; // Network::FilterChainManager diff --git a/source/server/server.cc b/source/server/server.cc index 072a7612ec2b..c9d49732c1a2 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -322,6 +322,7 @@ void InstanceImpl::initialize(const Options& options, ENVOY_LOG(info, "admin address: {}", initial_config.admin().address()->asString()); admin_->startHttpListener(initial_config.admin().accessLogPath(), options.adminAddressPath(), initial_config.admin().address(), + initial_config.admin().socketOptions(), stats_store_.createScope("listener.admin.")); } else { ENVOY_LOG(warn, "No admin address given, so no admin HTTP server started."); diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 6ab3baac302c..f61b4be09e99 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -128,9 +128,10 @@ class MockAdmin : public Admin { MOCK_METHOD1(removeHandler, bool(const std::string& prefix)); MOCK_METHOD0(socket, Network::Socket&()); MOCK_METHOD0(getConfigTracker, ConfigTracker&()); - MOCK_METHOD4(startHttpListener, + MOCK_METHOD5(startHttpListener, void(const std::string& access_log_path, const std::string& address_out_path, Network::Address::InstanceConstSharedPtr address, + const Network::Socket::OptionsSharedPtr& socket_options, Stats::ScopePtr&& listener_scope)); MOCK_METHOD4(request, Http::Code(absl::string_view path_and_query, absl::string_view method, Http::HeaderMap& response_headers, std::string& body)); diff --git a/test/server/BUILD b/test/server/BUILD index a6601e6ca5c4..490eccbac91d 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -254,6 +254,7 @@ envoy_cc_test( ":invalid_runtime_bootstrap.yaml", ":node_bootstrap.yaml", ":node_bootstrap_no_admin_port.yaml", + ":node_bootstrap_with_admin_socket_options.yaml", ":node_bootstrap_without_access_log.yaml", ":runtime_bootstrap.yaml", ":runtime_test_data", diff --git a/test/server/configuration_impl_test.cc b/test/server/configuration_impl_test.cc index 732c47126c51..c2c4247b1f87 100644 --- a/test/server/configuration_impl_test.cc +++ b/test/server/configuration_impl_test.cc @@ -459,6 +459,50 @@ TEST(InitialImplTest, DeprecatedRuntimeTranslation) { EXPECT_THAT(config.runtime(), ProtoEq(expected_runtime)); } +TEST_F(ConfigurationImplTest, AdminSocketOptions) { + std::string json = R"EOF( + { + "admin": { + "access_log_path": "/dev/null", + "address": { + "socket_address": { + "address": "1.2.3.4", + "port_value": 5678 + } + }, + "socket_options": [ + { + "level": 1, + "name": 2, + "int_value": 3, + "state": "STATE_PREBIND" + }, + { + "level": 4, + "name": 5, + "int_value": 6, + "state": "STATE_BOUND" + }, + ] + } + } + )EOF"; + + auto bootstrap = Upstream::parseBootstrapFromV2Json(json); + InitialImpl config(bootstrap); + Network::MockListenSocket socket_mock; + + ASSERT_EQ(config.admin().socketOptions()->size(), 2); + auto detail = config.admin().socketOptions()->at(0)->getOptionDetails( + socket_mock, envoy::api::v2::core::SocketOption::STATE_PREBIND); + ASSERT_NE(detail, absl::nullopt); + EXPECT_EQ(detail->name_, Envoy::Network::SocketOptionName(1, 2, "1/2")); + detail = config.admin().socketOptions()->at(1)->getOptionDetails( + socket_mock, envoy::api::v2::core::SocketOption::STATE_BOUND); + ASSERT_NE(detail, absl::nullopt); + EXPECT_EQ(detail->name_, Envoy::Network::SocketOptionName(4, 5, "4/5")); +} + } // namespace } // namespace Configuration } // namespace Server diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index 274f4dfcd524..e89d132cb5cb 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -586,7 +586,7 @@ class AdminInstanceTest : public testing::TestWithParamadmin().socket().localAddress(); + EXPECT_THAT_THROWS_MESSAGE(std::make_unique(address, nullptr, true), + EnvoyException, HasSubstr("Address already in use")); + auto options = std::make_shared(); + options->emplace_back(std::make_shared( + envoy::api::v2::core::SocketOption::STATE_PREBIND, + ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_REUSEPORT), 1)); + EXPECT_NO_THROW(std::make_unique(address, options, true)); +} + // Empty bootstrap succeeds. TEST_P(ServerInstanceImplTest, EmptyBootstrap) { options_.service_cluster_name_ = "some_cluster_name"; diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index ee770090818d..a83054bdcdb2 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -234,6 +234,14 @@ std::string TestEnvironment::substitute(const std::string& str, break; } + // Substitute socket options arguments. + const std::regex sol_socket_regex(R"(\{\{ sol_socket \}\})"); + out_json_string = + std::regex_replace(out_json_string, sol_socket_regex, std::to_string(SOL_SOCKET)); + const std::regex so_reuseport_regex(R"(\{\{ so_reuseport \}\})"); + out_json_string = + std::regex_replace(out_json_string, so_reuseport_regex, std::to_string(SO_REUSEPORT)); + return out_json_string; } From 5e95cc3e545deba4a42c629ef9c7d8ccd8ec271d Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Wed, 17 Jul 2019 08:11:57 -0700 Subject: [PATCH 189/542] clang-tidy: bugprone-unused-raii (#7605) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + test/common/stats/tag_producer_impl_test.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index 0692d1c459ee..6d73fdb3994b 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -17,6 +17,7 @@ WarningsAsErrors: 'abseil-duration-*, abseil-string-find-startswith, abseil-upgrade-duration-conversions, bugprone-assert-side-effect, + bugprone-unused-raii, bugprone-use-after-move, modernize-make-shared, modernize-make-unique, diff --git a/test/common/stats/tag_producer_impl_test.cc b/test/common/stats/tag_producer_impl_test.cc index d97819ed4e9c..7dd44f8cbcec 100644 --- a/test/common/stats/tag_producer_impl_test.cc +++ b/test/common/stats/tag_producer_impl_test.cc @@ -17,7 +17,7 @@ TEST(TagProducerTest, CheckConstructor) { auto& tag_specifier1 = *stats_config.mutable_stats_tags()->Add(); tag_specifier1.set_tag_name("test.x"); tag_specifier1.set_fixed_value("xxx"); - TagProducerImpl{stats_config}; + EXPECT_NO_THROW(TagProducerImpl{stats_config}); // Should raise an error when duplicate tag names are specified. auto& tag_specifier2 = *stats_config.mutable_stats_tags()->Add(); From 0a27d181fd8da8a5f290c87f72f76ac012cff7fd Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Wed, 17 Jul 2019 11:08:45 -0700 Subject: [PATCH 190/542] test: remove unused function in lua_integration_test.cc (#7590) Risk Level: low Testing: existing tests Docs Changes: N/A Release Notes: N/A Signed-off-by: Yan Xue --- test/extensions/filters/http/lua/lua_integration_test.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/lua/lua_integration_test.cc b/test/extensions/filters/http/lua/lua_integration_test.cc index 4149d4333852..db599dee1302 100644 --- a/test/extensions/filters/http/lua/lua_integration_test.cc +++ b/test/extensions/filters/http/lua/lua_integration_test.cc @@ -445,7 +445,7 @@ name: envoy.lua request_handle:headers():add("signature_verification", "rejected") end - request_handle:releasePublicKey(pubkey) + request_handle:headers():add("verification", "done") end )EOF"; @@ -473,6 +473,11 @@ name: envoy.lua ->value() .getStringView()); + EXPECT_EQ("done", upstream_request_->headers() + .get(Http::LowerCaseString("verification")) + ->value() + .getStringView()); + upstream_request_->encodeHeaders(default_response_headers_, true); response->waitForEndStream(); From 470375919acadd1aa7de1fb87ef68840b5c0c7eb Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Wed, 17 Jul 2019 14:12:18 -0400 Subject: [PATCH 191/542] stats: rename StatDataAllocator and friends, paying down some tech-debt. (#7591) Rename *StatDataAllocator to Allocator and AllocatorImpl, fixing filenames as well. This cleans up the last bit of tech-debt left from #7109. Risk Level: low Testing: //test/... Docs Changes: n/a Release Notes: n/a Signed-off-by: Joshua Marantz --- include/envoy/server/hot_restart.h | 2 +- include/envoy/stats/BUILD | 2 +- .../{stat_data_allocator.h => allocator.h} | 5 ++-- include/envoy/stats/stats.h | 2 +- source/common/stats/BUILD | 10 +++---- .../{heap_stat_data.cc => allocator_impl.cc} | 29 +++++++++---------- .../{heap_stat_data.h => allocator_impl.h} | 12 ++++---- source/common/stats/isolated_store_impl.h | 4 +-- source/common/stats/metric_impl.h | 2 +- source/common/stats/thread_local_store.cc | 9 +++--- source/common/stats/thread_local_store.h | 10 +++---- source/exe/main_common.h | 2 +- source/server/BUILD | 4 +-- source/server/hot_restart_impl.h | 2 +- source/server/hot_restart_nop_impl.h | 2 +- test/common/stats/BUILD | 8 ++--- ...locator_test.cc => allocator_impl_test.cc} | 14 ++++----- test/common/stats/metric_impl_test.cc | 4 +-- test/common/stats/stat_merger_test.cc | 2 +- .../stats/thread_local_store_speed_test.cc | 4 +-- test/common/stats/thread_local_store_test.cc | 16 +++++----- test/integration/server.cc | 2 +- test/mocks/server/mocks.h | 4 +-- test/server/http/admin_test.cc | 4 +-- 24 files changed, 75 insertions(+), 80 deletions(-) rename include/envoy/stats/{stat_data_allocator.h => allocator.h} (92%) rename source/common/stats/{heap_stat_data.cc => allocator_impl.cc} (86%) rename source/common/stats/{heap_stat_data.h => allocator_impl.h} (88%) rename test/common/stats/{heap_allocator_test.cc => allocator_impl_test.cc} (85%) diff --git a/include/envoy/server/hot_restart.h b/include/envoy/server/hot_restart.h index 916646f5c087..16c182c8da3c 100644 --- a/include/envoy/server/hot_restart.h +++ b/include/envoy/server/hot_restart.h @@ -5,7 +5,7 @@ #include "envoy/common/pure.h" #include "envoy/event/dispatcher.h" -#include "envoy/stats/stat_data_allocator.h" +#include "envoy/stats/allocator.h" #include "envoy/stats/store.h" #include "envoy/thread/thread.h" diff --git a/include/envoy/stats/BUILD b/include/envoy/stats/BUILD index 2dcf68988def..b6a0389265fa 100644 --- a/include/envoy/stats/BUILD +++ b/include/envoy/stats/BUILD @@ -18,10 +18,10 @@ envoy_cc_library( envoy_cc_library( name = "stats_interface", hdrs = [ + "allocator.h", "histogram.h", "scope.h", "sink.h", - "stat_data_allocator.h", "stats.h", "stats_matcher.h", "store.h", diff --git a/include/envoy/stats/stat_data_allocator.h b/include/envoy/stats/allocator.h similarity index 92% rename from include/envoy/stats/stat_data_allocator.h rename to include/envoy/stats/allocator.h index 2fc94861828d..f05f6df4f89a 100644 --- a/include/envoy/stats/stat_data_allocator.h +++ b/include/envoy/stats/allocator.h @@ -23,11 +23,10 @@ namespace Stats { * be created utilizing a single fixed-size block suitable for * shared-memory, or in the heap, allowing for pointers and sharing of * substrings, with an opportunity for reduced memory consumption. - * TODO(fredlas) this interface can be deleted now that the shared memory version is gone. */ -class StatDataAllocator { +class Allocator { public: - virtual ~StatDataAllocator() = default; + virtual ~Allocator() = default; /** * @param name the full name of the stat. diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index a61249ba6e50..a610a9d48f9c 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -15,7 +15,7 @@ namespace Envoy { namespace Stats { -class StatDataAllocator; +class Allocator; struct Tag; /** diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index 16c0174eaab0..cb88ed2dc976 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -9,9 +9,9 @@ load( envoy_package() envoy_cc_library( - name = "heap_stat_data_lib", - srcs = ["heap_stat_data.cc"], - hdrs = ["heap_stat_data.h"], + name = "allocator_lib", + srcs = ["allocator_impl.cc"], + hdrs = ["allocator_impl.h"], deps = [ ":metric_impl_lib", ":stat_merger_lib", @@ -51,7 +51,7 @@ envoy_cc_library( ":stats_lib", ":store_impl_lib", "//include/envoy/stats:stats_macros", - "//source/common/stats:heap_stat_data_lib", + "//source/common/stats:allocator_lib", ], ) @@ -202,7 +202,7 @@ envoy_cc_library( srcs = ["thread_local_store.cc"], hdrs = ["thread_local_store.h"], deps = [ - ":heap_stat_data_lib", + ":allocator_lib", ":null_counter_lib", ":null_gauge_lib", ":scope_prefixer_lib", diff --git a/source/common/stats/heap_stat_data.cc b/source/common/stats/allocator_impl.cc similarity index 86% rename from source/common/stats/heap_stat_data.cc rename to source/common/stats/allocator_impl.cc index 2bb3d7646db8..6fbf0da19a0b 100644 --- a/source/common/stats/heap_stat_data.cc +++ b/source/common/stats/allocator_impl.cc @@ -1,4 +1,4 @@ -#include "common/stats/heap_stat_data.h" +#include "common/stats/allocator_impl.h" #include @@ -20,25 +20,25 @@ namespace Envoy { namespace Stats { -HeapStatDataAllocator::~HeapStatDataAllocator() { +AllocatorImpl::~AllocatorImpl() { ASSERT(counters_.empty()); ASSERT(gauges_.empty()); } -void HeapStatDataAllocator::removeCounterFromSet(Counter* counter) { +void AllocatorImpl::removeCounterFromSet(Counter* counter) { Thread::LockGuard lock(mutex_); const size_t count = counters_.erase(counter->statName()); ASSERT(count == 1); } -void HeapStatDataAllocator::removeGaugeFromSet(Gauge* gauge) { +void AllocatorImpl::removeGaugeFromSet(Gauge* gauge) { Thread::LockGuard lock(mutex_); const size_t count = gauges_.erase(gauge->statName()); ASSERT(count == 1); } #ifndef ENVOY_CONFIG_COVERAGE -void HeapStatDataAllocator::debugPrint() { +void AllocatorImpl::debugPrint() { Thread::LockGuard lock(mutex_); for (Counter* counter : counters_) { ENVOY_LOG_MISC(info, "counter: {}", symbolTable().toString(counter->statName())); @@ -59,7 +59,7 @@ void HeapStatDataAllocator::debugPrint() { // wasted in the alignment padding next to flags_. template class StatsSharedImpl : public MetricImpl { public: - StatsSharedImpl(StatName name, HeapStatDataAllocator& alloc, absl::string_view tag_extracted_name, + StatsSharedImpl(StatName name, AllocatorImpl& alloc, absl::string_view tag_extracted_name, const std::vector& tags) : MetricImpl(name, tag_extracted_name, tags, alloc.symbolTable()), alloc_(alloc) {} @@ -84,7 +84,7 @@ template class StatsSharedImpl : public MetricImpl uint32_t use_count() const override { return ref_count_; } protected: - HeapStatDataAllocator& alloc_; + AllocatorImpl& alloc_; // Holds backing store shared by both CounterImpl and GaugeImpl. CounterImpl // adds another field, pending_increment_, that is not used in Gauge. @@ -95,7 +95,7 @@ template class StatsSharedImpl : public MetricImpl class CounterImpl : public StatsSharedImpl { public: - CounterImpl(StatName name, HeapStatDataAllocator& alloc, absl::string_view tag_extracted_name, + CounterImpl(StatName name, AllocatorImpl& alloc, absl::string_view tag_extracted_name, const std::vector& tags) : StatsSharedImpl(name, alloc, tag_extracted_name, tags) {} ~CounterImpl() override { alloc_.removeCounterFromSet(this); } @@ -119,7 +119,7 @@ class CounterImpl : public StatsSharedImpl { class GaugeImpl : public StatsSharedImpl { public: - GaugeImpl(StatName name, HeapStatDataAllocator& alloc, absl::string_view tag_extracted_name, + GaugeImpl(StatName name, AllocatorImpl& alloc, absl::string_view tag_extracted_name, const std::vector& tags, ImportMode import_mode) : StatsSharedImpl(name, alloc, tag_extracted_name, tags) { switch (import_mode) { @@ -194,9 +194,8 @@ class GaugeImpl : public StatsSharedImpl { } }; -CounterSharedPtr HeapStatDataAllocator::makeCounter(StatName name, - absl::string_view tag_extracted_name, - const std::vector& tags) { +CounterSharedPtr AllocatorImpl::makeCounter(StatName name, absl::string_view tag_extracted_name, + const std::vector& tags) { Thread::LockGuard lock(mutex_); ASSERT(gauges_.find(name) == gauges_.end()); auto iter = counters_.find(name); @@ -208,9 +207,9 @@ CounterSharedPtr HeapStatDataAllocator::makeCounter(StatName name, return counter; } -GaugeSharedPtr HeapStatDataAllocator::makeGauge(StatName name, absl::string_view tag_extracted_name, - const std::vector& tags, - Gauge::ImportMode import_mode) { +GaugeSharedPtr AllocatorImpl::makeGauge(StatName name, absl::string_view tag_extracted_name, + const std::vector& tags, + Gauge::ImportMode import_mode) { Thread::LockGuard lock(mutex_); ASSERT(counters_.find(name) == counters_.end()); auto iter = gauges_.find(name); diff --git a/source/common/stats/heap_stat_data.h b/source/common/stats/allocator_impl.h similarity index 88% rename from source/common/stats/heap_stat_data.h rename to source/common/stats/allocator_impl.h index c15f37a59958..5d5e82721b6d 100644 --- a/source/common/stats/heap_stat_data.h +++ b/source/common/stats/allocator_impl.h @@ -1,10 +1,8 @@ -// TODO(jmarantz): rename this file and class to heap_allocator.h. - #pragma once #include -#include "envoy/stats/stat_data_allocator.h" +#include "envoy/stats/allocator.h" #include "envoy/stats/stats.h" #include "envoy/stats/symbol_table.h" @@ -16,15 +14,15 @@ namespace Envoy { namespace Stats { -class HeapStatDataAllocator : public StatDataAllocator { +class AllocatorImpl : public Allocator { public: - HeapStatDataAllocator(SymbolTable& symbol_table) : symbol_table_(symbol_table) {} - ~HeapStatDataAllocator() override; + AllocatorImpl(SymbolTable& symbol_table) : symbol_table_(symbol_table) {} + ~AllocatorImpl() override; void removeCounterFromSet(Counter* counter); void removeGaugeFromSet(Gauge* gauge); - // StatDataAllocator + // Allocator CounterSharedPtr makeCounter(StatName name, absl::string_view tag_extracted_name, const std::vector& tags) override; GaugeSharedPtr makeGauge(StatName name, absl::string_view tag_extracted_name, diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index ba492db0c040..ae22d13dfdba 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -9,7 +9,7 @@ #include "envoy/stats/store.h" #include "common/common/utility.h" -#include "common/stats/heap_stat_data.h" +#include "common/stats/allocator_impl.h" #include "common/stats/null_counter.h" #include "common/stats/null_gauge.h" #include "common/stats/store_impl.h" @@ -142,7 +142,7 @@ class IsolatedStoreImpl : public StoreImpl { IsolatedStoreImpl(std::unique_ptr&& symbol_table); std::unique_ptr symbol_table_storage_; - HeapStatDataAllocator alloc_; + AllocatorImpl alloc_; IsolatedStatsCache counters_; IsolatedStatsCache gauges_; IsolatedStatsCache histograms_; diff --git a/source/common/stats/metric_impl.h b/source/common/stats/metric_impl.h index 60e121976fad..2caf82f1863f 100644 --- a/source/common/stats/metric_impl.h +++ b/source/common/stats/metric_impl.h @@ -3,7 +3,7 @@ #include #include -#include "envoy/stats/stat_data_allocator.h" +#include "envoy/stats/allocator.h" #include "envoy/stats/stats.h" #include "envoy/stats/tag.h" diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 6de61634cd10..155cfefda757 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -6,9 +6,9 @@ #include #include +#include "envoy/stats/allocator.h" #include "envoy/stats/histogram.h" #include "envoy/stats/sink.h" -#include "envoy/stats/stat_data_allocator.h" #include "envoy/stats/stats.h" #include "common/common/lock_guard.h" @@ -20,7 +20,7 @@ namespace Envoy { namespace Stats { -ThreadLocalStoreImpl::ThreadLocalStoreImpl(StatDataAllocator& alloc) +ThreadLocalStoreImpl::ThreadLocalStoreImpl(Allocator& alloc) : alloc_(alloc), default_scope_(createScope("")), tag_producer_(std::make_unique()), stats_matcher_(std::make_unique()), heap_allocator_(alloc.symbolTable()), @@ -400,7 +400,7 @@ Counter& ThreadLocalStoreImpl::ScopeImpl::counterFromStatName(StatName name) { return safeMakeStat( final_stat_name, central_cache_.counters_, central_cache_.rejected_stats_, - [](StatDataAllocator& allocator, StatName name, absl::string_view tag_extracted_name, + [](Allocator& allocator, StatName name, absl::string_view tag_extracted_name, const std::vector& tags) -> CounterSharedPtr { return allocator.makeCounter(name, tag_extracted_name, tags); }, @@ -450,8 +450,7 @@ Gauge& ThreadLocalStoreImpl::ScopeImpl::gaugeFromStatName(StatName name, Gauge& gauge = safeMakeStat( final_stat_name, central_cache_.gauges_, central_cache_.rejected_stats_, - [import_mode](StatDataAllocator& allocator, StatName name, - absl::string_view tag_extracted_name, + [import_mode](Allocator& allocator, StatName name, absl::string_view tag_extracted_name, const std::vector& tags) -> GaugeSharedPtr { return allocator.makeGauge(name, tag_extracted_name, tags, import_mode); }, diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index 95babe5efafe..b10e7babac88 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -9,7 +9,7 @@ #include "envoy/thread_local/thread_local.h" #include "common/common/hash.h" -#include "common/stats/heap_stat_data.h" +#include "common/stats/allocator_impl.h" #include "common/stats/histogram_impl.h" #include "common/stats/null_counter.h" #include "common/stats/null_gauge.h" @@ -140,7 +140,7 @@ class TlsScope : public Scope { */ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRoot { public: - ThreadLocalStoreImpl(StatDataAllocator& alloc); + ThreadLocalStoreImpl(Allocator& alloc); ~ThreadLocalStoreImpl() override; // Stats::Scope @@ -280,7 +280,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo findHistogram(StatName name) const override; template - using MakeStatFn = std::function(StatDataAllocator&, StatName name, + using MakeStatFn = std::function(Allocator&, StatName name, absl::string_view tag_extracted_name, const std::vector& tags)>; @@ -349,7 +349,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo bool checkAndRememberRejection(StatName name, StatNameStorageSet& central_rejected_stats, StatNameHashSet* tls_rejected_stats); - StatDataAllocator& alloc_; + Allocator& alloc_; Event::Dispatcher* main_thread_dispatcher_{}; ThreadLocal::SlotPtr tls_; mutable Thread::MutexBasicLockable lock_; @@ -361,7 +361,7 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo std::atomic threading_ever_initialized_{}; std::atomic shutting_down_{}; std::atomic merge_in_progress_{}; - HeapStatDataAllocator heap_allocator_; + AllocatorImpl heap_allocator_; NullCounterImpl null_counter_; NullGaugeImpl null_gauge_; diff --git a/source/exe/main_common.h b/source/exe/main_common.h index a8944fd69b34..5faf31b547cc 100644 --- a/source/exe/main_common.h +++ b/source/exe/main_common.h @@ -71,7 +71,7 @@ class MainCommonBase { Server::ComponentFactory& component_factory_; Thread::ThreadFactory& thread_factory_; Filesystem::Instance& file_system_; - Stats::HeapStatDataAllocator stats_allocator_; + Stats::AllocatorImpl stats_allocator_; std::unique_ptr tls_; std::unique_ptr restarter_; diff --git a/source/server/BUILD b/source/server/BUILD index 538f25f922be..7cb1a512905e 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -168,7 +168,7 @@ envoy_cc_library( "//include/envoy/server:options_interface", "//source/common/api:os_sys_calls_lib", "//source/common/common:assert_lib", - "//source/common/stats:heap_stat_data_lib", + "//source/common/stats:allocator_lib", ], ) @@ -177,7 +177,7 @@ envoy_cc_library( hdrs = ["hot_restart_nop_impl.h"], deps = [ "//include/envoy/server:hot_restart_interface", - "//source/common/stats:heap_stat_data_lib", + "//source/common/stats:allocator_lib", ], ) diff --git a/source/server/hot_restart_impl.h b/source/server/hot_restart_impl.h index 0244c20f6927..b8cb4c636e22 100644 --- a/source/server/hot_restart_impl.h +++ b/source/server/hot_restart_impl.h @@ -12,7 +12,7 @@ #include "envoy/server/hot_restart.h" #include "common/common/assert.h" -#include "common/stats/heap_stat_data.h" +#include "common/stats/allocator_impl.h" #include "server/hot_restarting_child.h" #include "server/hot_restarting_parent.h" diff --git a/source/server/hot_restart_nop_impl.h b/source/server/hot_restart_nop_impl.h index dc4e0662270d..205097649b81 100644 --- a/source/server/hot_restart_nop_impl.h +++ b/source/server/hot_restart_nop_impl.h @@ -5,7 +5,7 @@ #include "envoy/server/hot_restart.h" #include "common/common/thread.h" -#include "common/stats/heap_stat_data.h" +#include "common/stats/allocator_impl.h" namespace Envoy { namespace Server { diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index c6a724d90ca5..6ab96a754d42 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -11,11 +11,11 @@ load( envoy_package() envoy_cc_test( - name = "heap_allocator_test", - srcs = ["heap_allocator_test.cc"], + name = "allocator_impl_test", + srcs = ["allocator_impl_test.cc"], deps = [ + "//source/common/stats:allocator_lib", "//source/common/stats:fake_symbol_table_lib", - "//source/common/stats:heap_stat_data_lib", "//test/test_common:logging_lib", ], ) @@ -32,8 +32,8 @@ envoy_cc_test( name = "metric_impl_test", srcs = ["metric_impl_test.cc"], deps = [ + "//source/common/stats:allocator_lib", "//source/common/stats:fake_symbol_table_lib", - "//source/common/stats:heap_stat_data_lib", "//source/common/stats:utility_lib", "//test/test_common:logging_lib", ], diff --git a/test/common/stats/heap_allocator_test.cc b/test/common/stats/allocator_impl_test.cc similarity index 85% rename from test/common/stats/heap_allocator_test.cc rename to test/common/stats/allocator_impl_test.cc index 770257332d99..99327899f2a6 100644 --- a/test/common/stats/heap_allocator_test.cc +++ b/test/common/stats/allocator_impl_test.cc @@ -1,7 +1,7 @@ #include +#include "common/stats/allocator_impl.h" #include "common/stats/fake_symbol_table_impl.h" -#include "common/stats/heap_stat_data.h" #include "test/test_common/logging.h" @@ -11,10 +11,10 @@ namespace Envoy { namespace Stats { namespace { -class HeapAllocatorTest : public testing::Test { +class AllocatorImplTest : public testing::Test { protected: - HeapAllocatorTest() : alloc_(symbol_table_), pool_(symbol_table_) {} - ~HeapAllocatorTest() { clearStorage(); } + AllocatorImplTest() : alloc_(symbol_table_), pool_(symbol_table_) {} + ~AllocatorImplTest() { clearStorage(); } StatNameStorage makeStatStorage(absl::string_view name) { return StatNameStorage(name, symbol_table_); @@ -28,12 +28,12 @@ class HeapAllocatorTest : public testing::Test { } FakeSymbolTableImpl symbol_table_; - HeapStatDataAllocator alloc_; + AllocatorImpl alloc_; StatNamePool pool_; }; // Allocate 2 counters of the same name, and you'll get the same object. -TEST_F(HeapAllocatorTest, CountersWithSameName) { +TEST_F(AllocatorImplTest, CountersWithSameName) { StatName counter_name = makeStat("counter.name"); CounterSharedPtr c1 = alloc_.makeCounter(counter_name, "", std::vector()); EXPECT_EQ(1, c1->use_count()); @@ -51,7 +51,7 @@ TEST_F(HeapAllocatorTest, CountersWithSameName) { EXPECT_EQ(2, c2->value()); } -TEST_F(HeapAllocatorTest, GaugesWithSameName) { +TEST_F(AllocatorImplTest, GaugesWithSameName) { StatName gauge_name = makeStat("gauges.name"); GaugeSharedPtr g1 = alloc_.makeGauge(gauge_name, "", std::vector(), Gauge::ImportMode::Accumulate); diff --git a/test/common/stats/metric_impl_test.cc b/test/common/stats/metric_impl_test.cc index 493444fd1d0e..ed3a10abd537 100644 --- a/test/common/stats/metric_impl_test.cc +++ b/test/common/stats/metric_impl_test.cc @@ -1,7 +1,7 @@ #include +#include "common/stats/allocator_impl.h" #include "common/stats/fake_symbol_table_impl.h" -#include "common/stats/heap_stat_data.h" #include "common/stats/utility.h" #include "test/test_common/logging.h" @@ -25,7 +25,7 @@ class MetricImplTest : public testing::Test { } FakeSymbolTableImpl symbol_table_; - HeapStatDataAllocator alloc_; + AllocatorImpl alloc_; StatNamePool pool_; }; diff --git a/test/common/stats/stat_merger_test.cc b/test/common/stats/stat_merger_test.cc index e91cb3dda992..9ebac5f2a408 100644 --- a/test/common/stats/stat_merger_test.cc +++ b/test/common/stats/stat_merger_test.cc @@ -183,7 +183,7 @@ TEST_F(StatMergerTest, gaugeMergeImportMode) { class StatMergerThreadLocalTest : public testing::Test { protected: FakeSymbolTableImpl symbol_table_; - HeapStatDataAllocator alloc_{symbol_table_}; + AllocatorImpl alloc_{symbol_table_}; ThreadLocalStoreImpl store_{alloc_}; }; diff --git a/test/common/stats/thread_local_store_speed_test.cc b/test/common/stats/thread_local_store_speed_test.cc index 30413ff6b765..877c6bc7bb09 100644 --- a/test/common/stats/thread_local_store_speed_test.cc +++ b/test/common/stats/thread_local_store_speed_test.cc @@ -4,8 +4,8 @@ #include "common/common/logger.h" #include "common/common/thread.h" #include "common/event/dispatcher_impl.h" +#include "common/stats/allocator_impl.h" #include "common/stats/fake_symbol_table_impl.h" -#include "common/stats/heap_stat_data.h" #include "common/stats/tag_producer_impl.h" #include "common/stats/thread_local_store.h" #include "common/thread_local/thread_local_impl.h" @@ -56,7 +56,7 @@ class ThreadLocalStorePerf { private: Stats::FakeSymbolTableImpl symbol_table_; Event::SimulatedTimeSystem time_system_; - Stats::HeapStatDataAllocator heap_alloc_; + Stats::AllocatorImpl heap_alloc_; Stats::ThreadLocalStoreImpl store_; Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index edd3ba983591..711d1c459e46 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -43,7 +43,7 @@ class StatsThreadLocalStoreTest : public testing::Test { store_->addSink(sink_); } - void resetStoreWithAlloc(StatDataAllocator& alloc) { + void resetStoreWithAlloc(Allocator& alloc) { store_ = std::make_unique(alloc); store_->addSink(sink_); } @@ -51,7 +51,7 @@ class StatsThreadLocalStoreTest : public testing::Test { Stats::FakeSymbolTableImpl symbol_table_; NiceMock main_thread_dispatcher_; NiceMock tls_; - HeapStatDataAllocator alloc_; + AllocatorImpl alloc_; MockSink sink_; std::unique_ptr store_; }; @@ -169,7 +169,7 @@ class HistogramTest : public testing::Test { FakeSymbolTableImpl symbol_table_; NiceMock main_thread_dispatcher_; NiceMock tls_; - HeapStatDataAllocator alloc_; + AllocatorImpl alloc_; MockSink sink_; std::unique_ptr store_; InSequence s; @@ -461,7 +461,7 @@ class LookupWithStatNameTest : public testing::Test { StatName makeStatName(absl::string_view name) { return pool_.add(name); } Stats::FakeSymbolTableImpl symbol_table_; - HeapStatDataAllocator alloc_; + AllocatorImpl alloc_; ThreadLocalStoreImpl store_; StatNamePool pool_; }; @@ -758,7 +758,7 @@ class RememberStatsMatcherTest : public testing::TestWithParam { Stats::FakeSymbolTableImpl symbol_table_; NiceMock main_thread_dispatcher_; NiceMock tls_; - HeapStatDataAllocator heap_alloc_; + AllocatorImpl heap_alloc_; ThreadLocalStoreImpl store_; ScopePtr scope_; }; @@ -849,7 +849,7 @@ TEST_F(StatsThreadLocalStoreTest, NonHotRestartNoTruncation) { TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTls) { MockSink sink; Stats::FakeSymbolTableImpl symbol_table; - HeapStatDataAllocator alloc(symbol_table); + AllocatorImpl alloc(symbol_table); ThreadLocalStoreImpl store(alloc); store.addSink(sink); @@ -867,7 +867,7 @@ TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTls) { TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithTls) { Stats::FakeSymbolTableImpl symbol_table; - HeapStatDataAllocator alloc(symbol_table); + AllocatorImpl alloc(symbol_table); NiceMock main_thread_dispatcher; NiceMock tls; ThreadLocalStoreImpl store(alloc); @@ -933,7 +933,7 @@ TEST(ThreadLocalStoreThreadTest, ConstructDestruct) { Api::ApiPtr api = Api::createApiForTest(); Event::DispatcherPtr dispatcher = api->allocateDispatcher(); NiceMock tls; - HeapStatDataAllocator alloc(symbol_table); + AllocatorImpl alloc(symbol_table); ThreadLocalStoreImpl store(alloc); store.initializeThreading(*dispatcher, tls); diff --git a/test/integration/server.cc b/test/integration/server.cc index ddd12b31191d..382332aa80c3 100644 --- a/test/integration/server.cc +++ b/test/integration/server.cc @@ -180,7 +180,7 @@ void IntegrationTestServerImpl::createAndRunEnvoyServer( Stats::FakeSymbolTableImpl symbol_table; Server::HotRestartNopImpl restarter; ThreadLocal::InstanceImpl tls; - Stats::HeapStatDataAllocator stats_allocator(symbol_table); + Stats::AllocatorImpl stats_allocator(symbol_table); Stats::ThreadLocalStoreImpl stat_store(stats_allocator); std::unique_ptr process_context; if (process_object.has_value()) { diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index f61b4be09e99..7ab95b1df378 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -207,13 +207,13 @@ class MockHotRestart : public HotRestart { MOCK_METHOD0(version, std::string()); MOCK_METHOD0(logLock, Thread::BasicLockable&()); MOCK_METHOD0(accessLogLock, Thread::BasicLockable&()); - MOCK_METHOD0(statsAllocator, Stats::StatDataAllocator&()); + MOCK_METHOD0(statsAllocator, Stats::Allocator&()); private: Test::Global symbol_table_; Thread::MutexBasicLockable log_lock_; Thread::MutexBasicLockable access_log_lock_; - Stats::HeapStatDataAllocator stats_allocator_; + Stats::AllocatorImpl stats_allocator_; }; class MockListenerComponentFactory : public ListenerComponentFactory { diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index e89d132cb5cb..c70a39ad2d18 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -66,7 +66,7 @@ class AdminStatsTest : public testing::TestWithParam main_thread_dispatcher_; NiceMock tls_; - Stats::HeapStatDataAllocator alloc_; + Stats::AllocatorImpl alloc_; Stats::MockSink sink_; std::unique_ptr store_; }; @@ -1397,7 +1397,7 @@ class PrometheusStatsFormatterTest : public testing::Test { } Stats::FakeSymbolTableImpl symbol_table_; - Stats::HeapStatDataAllocator alloc_; + Stats::AllocatorImpl alloc_; std::vector counters_; std::vector gauges_; std::vector histograms_; From ef11902b3db2f3618f39648b8694050ae609b330 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 17 Jul 2019 11:48:46 -0700 Subject: [PATCH 192/542] ci: speed up release and mac run (#7612) Description: Pulling out from #7536, allowing RBE not download test binaries. Let bazel schedule test runs at same time at build proceeds to speed up. Also let us collect Bazel profile on release compiles with test. Risk Level: Low Testing: Docs Changes: Release Notes: Signed-off-by: Lizan Zhou --- ci/do_ci.sh | 6 ------ ci/mac_ci_steps.sh | 4 +++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index d9a16221574a..e47f66563bec 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -104,12 +104,6 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then bazel_binary_build release echo "Testing ${TEST_TARGETS}" - if [[ "$TEST_TARGETS" == "//test/..." ]]; then - # We have various test binaries in the test directory such as tools, benchmarks, etc. We - # run a build pass to make sure they compile. - bazel build ${BAZEL_BUILD_OPTIONS} -c opt //include/... //source/exe:envoy-static //test/... - fi - # Now run all of the tests which should already be compiled. bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c opt ${TEST_TARGETS} exit 0 elif [[ "$CI_TARGET" == "bazel.release.server_only" ]]; then diff --git a/ci/mac_ci_steps.sh b/ci/mac_ci_steps.sh index 7d322dafee2a..552f9d7957ad 100755 --- a/ci/mac_ci_steps.sh +++ b/ci/mac_ci_steps.sh @@ -29,5 +29,7 @@ else TEST_TARGETS=//test/... fi -bazel build ${BAZEL_BUILD_OPTIONS} //source/exe:envoy-static ${TEST_TARGETS} +if [[ "$TEST_TARGETS" == "//test/..." ]]; then + bazel build ${BAZEL_BUILD_OPTIONS} //source/exe:envoy-static +fi bazel test ${BAZEL_BUILD_OPTIONS} ${TEST_TARGETS} From ad9926fb29dc047d7583ab5b0217a3c379f14200 Mon Sep 17 00:00:00 2001 From: mnktsts2 Date: Thu, 18 Jul 2019 03:51:18 +0900 Subject: [PATCH 193/542] load balancer: fix?: panic mode is disabled only when healthy_panic_threshold is 0% (#7478) Currently, in load_balancer_impl.cc: recalculatePerPriorityPanic(), even if common_lb_config.healthy_panic_threshold is 0%, a load balancer enters panic mode whenever normalized_total_availability is 0%. I guess, a user who intentionally set to healthy_panic_threshold = 0 expects to immediately return error responses if there is no available host checked by a load balancer. (In fact, current load_balancer_impl.cc: isGlobalPanic() decide not to enter panic mode whenever healthy_panic_threshold is 0%.) So I suggest that panic mode is disabled only when healthy_panic_threshold is 0%. I want this change for automatic degenerating lower priority or optional back-end services. Risk Level: Low (It seems that setting healthy_panic_threshold == 0 is a special case originally. It won't happen unless a user intend to disable panic mode, because default value of healthy_panic_threshold is 50%.) Testing: unit and Integration tests with ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.release' Docs Changes: inline Signed-off-by: mnktsts2 --- api/envoy/api/v2/cds.proto | 1 + .../upstream/load_balancing/panic_threshold.rst | 9 ++++++++- source/common/upstream/load_balancer_impl.cc | 10 +++++++--- test/common/upstream/load_balancer_impl_test.cc | 15 +++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 864290cdc42d..44d2d7501479 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -517,6 +517,7 @@ message Cluster { message CommonLbConfig { // Configures the :ref:`healthy panic threshold `. // If not specified, the default is 50%. + // To disable panic mode, set to 0%. // // .. note:: // The specified percent will be truncated to the nearest 1%. diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst b/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst index 80b062593a29..e24022a8f07b 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst @@ -66,5 +66,12 @@ priority. | 5% | 65% | 7% | YES | 93% | NO | 98% | +-------------+-------------+----------+--------------+----------+--------------+-------------+ -Note that panic thresholds can be configured *per-priority*. +Panic mode can be disabled by setting the panic threshold to 0%. + +If all hosts become unhealthy normalized total health is 0%, and if the panic threshold is above 0% +all traffic will be redirected to P=0. +However, if the panic threshold is 0% for any priority, that priority will never enter panic mode. +In this case if all hosts are unhealthy, Envoy will fail to select a host and will instead immediately +return error responses with "503 - no healthy upstream". +Note that panic thresholds can be configured *per-priority*. diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 3969ed5dc308..e4b653d928dc 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -117,8 +117,8 @@ LoadBalancerBase::LoadBalancerBase(const PrioritySet& priority_set, ClusterStats // - normalized total health is < 100%. There are not enough healthy hosts to handle the load. // Continue distributing the load among priority sets, but turn on panic mode for a given priority // if # of healthy hosts in priority set is low. -// - normalized total health is 0%. All hosts are down. Redirect 100% of traffic to P=0 and enable -// panic mode. +// - normalized total health is 0%. All hosts are down. Redirect 100% of traffic to P=0. +// And if panic threshold > 0% then enable panic mode for P=0, otherwise disable. void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, const PrioritySet& priority_set, @@ -219,7 +219,11 @@ void LoadBalancerBase::recalculatePerPriorityPanic() { const uint32_t normalized_total_availability = calculateNormalizedTotalAvailability(per_priority_health_, per_priority_degraded_); - if (normalized_total_availability == 0) { + const uint64_t panic_threshold = std::min( + 100, runtime_.snapshot().getInteger(RuntimePanicThreshold, default_healthy_panic_percent_)); + + // Panic mode is disabled only when panic_threshold is 0%. + if (panic_threshold > 0 && normalized_total_availability == 0) { // Everything is terrible. All load should be to P=0. Turn on panic mode. ASSERT(per_priority_load_.healthy_priority_load_.get()[0] == 100); per_priority_panic_[0] = true; diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index e9d26eb922cc..5a42f7bf5edf 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -829,6 +829,20 @@ TEST_P(RoundRobinLoadBalancerTest, MaxUnhealthyPanic) { EXPECT_EQ(3UL, stats_.lb_healthy_panic_.value()); } +// Ensure if the panic threshold is 0%, panic mode is disabled. +TEST_P(RoundRobinLoadBalancerTest, DisablePanicMode) { + hostSet().healthy_hosts_ = {}; + hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; + + common_config_.mutable_healthy_panic_threshold()->set_value(0); + + init(false); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillRepeatedly(Return(0)); + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); + EXPECT_EQ(0UL, stats_.lb_healthy_panic_.value()); +} + // Test of host set selection with host filter TEST_P(RoundRobinLoadBalancerTest, HostSelectionWithFilter) { NiceMock context; @@ -1144,6 +1158,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingLocalEmpty) { HostsPerLocalitySharedPtr local_hosts_per_locality = makeHostsPerLocality({{}, {}}); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillOnce(Return(50)) .WillOnce(Return(50)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) .WillOnce(Return(true)); From d63befed30383202bb28f6e7e86bb7cded1e7632 Mon Sep 17 00:00:00 2001 From: Elisha Ziskind Date: Wed, 17 Jul 2019 16:02:56 -0400 Subject: [PATCH 194/542] server: skip lifecycle notifications on early envoy shutdown (#7585) Skip lifecycle notifications which take a completion callback when envoy shuts down early before starting worker threads. This is needed because these registrations are typically implemented using Slot::runOnAllThreads which will not work correctly if worker threads have not been started. Risk Level: low Testing: unit tests Signed-off-by: Elisha Ziskind --- include/envoy/server/lifecycle_notifier.h | 3 ++ source/server/server.cc | 21 ++++++++---- source/server/server.h | 1 + test/server/server_test.cc | 40 +++++++++++++++++++++++ 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/include/envoy/server/lifecycle_notifier.h b/include/envoy/server/lifecycle_notifier.h index cd829f61b991..dbda0e2e0055 100644 --- a/include/envoy/server/lifecycle_notifier.h +++ b/include/envoy/server/lifecycle_notifier.h @@ -26,6 +26,9 @@ class ServerLifecycleNotifier { * This provides listeners a last chance to run a callback on the main dispatcher. * Note: the server will wait for callbacks that registered to take a completion * before exiting the dispatcher loop. + * Note: callbacks that registered with a completion will only be notified for this + * stage if the server did not prematurely shutdown before fully starting up (specifically + * if the server shutdown before worker threads were started). */ ShutdownExit }; diff --git a/source/server/server.cc b/source/server/server.cc index c9d49732c1a2..117cd8a6b60f 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -57,8 +57,8 @@ InstanceImpl::InstanceImpl(const Options& options, Event::TimeSystem& time_syste ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system, std::unique_ptr process_context) - : secret_manager_(std::make_unique()), shutdown_(false), - options_(options), time_source_(time_system), restarter_(restarter), + : secret_manager_(std::make_unique()), workers_started_(false), + shutdown_(false), options_(options), time_source_(time_system), restarter_(restarter), start_time_(time(nullptr)), original_start_time_(start_time_), stats_store_(store), thread_local_(tls), api_(new Api::Impl(thread_factory, store, time_system, file_system)), dispatcher_(api_->allocateDispatcher()), @@ -422,6 +422,7 @@ void InstanceImpl::initialize(const Options& options, void InstanceImpl::startWorkers() { listener_manager_->startWorkers(*guard_dog_); initialization_timer_->complete(); + workers_started_ = true; // At this point we are ready to take traffic and all listening ports are up. Notify our parent // if applicable that they can stop listening and drain. restarter_.drainParentListeners(); @@ -614,11 +615,17 @@ void InstanceImpl::notifyCallbacksForStage(Stage stage, Event::PostCb completion delete cb; }); - auto it2 = stage_completable_callbacks_.find(stage); - if (it2 != stage_completable_callbacks_.end()) { - ENVOY_LOG(info, "Notifying {} callback(s) with completion.", it2->second.size()); - for (const StageCallbackWithCompletion& callback : it2->second) { - callback([cb_guard] { (*cb_guard)(); }); + // Registrations which take a completion callback are typically implemented by executing a + // callback on all worker threads using Slot::runOnAllThreads which will hang indefinitely if + // worker threads have not been started so we need to skip notifications if envoy is shutdown + // early before workers have started. + if (workers_started_) { + auto it2 = stage_completable_callbacks_.find(stage); + if (it2 != stage_completable_callbacks_.end()) { + ENVOY_LOG(info, "Notifying {} callback(s) with completion.", it2->second.size()); + for (const StageCallbackWithCompletion& callback : it2->second) { + callback([cb_guard] { (*cb_guard)(); }); + } } } } diff --git a/source/server/server.h b/source/server/server.h index fba73e1b2e58..90128e9b2deb 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -233,6 +233,7 @@ class InstanceImpl : Logger::Loggable, // - There may be active clusters referencing it in config_.cluster_manager_. // - There may be active connections referencing it. std::unique_ptr secret_manager_; + bool workers_started_; bool shutdown_; const Options& options_; TimeSource& time_source_; diff --git a/test/server/server_test.cc b/test/server/server_test.cc index d724441907b4..7c754040455a 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -375,6 +375,46 @@ TEST_P(ServerInstanceImplTest, LifecycleNotifications) { server_thread->join(); } +// A test target which never signals that it is ready. +class NeverReadyTarget : public Init::TargetImpl { +public: + NeverReadyTarget(absl::Notification& initialized) + : Init::TargetImpl("test", [this] { initialize(); }), initialized_(initialized) {} + +private: + void initialize() { initialized_.Notify(); } + + absl::Notification& initialized_; +}; + +TEST_P(ServerInstanceImplTest, NoLifecycleNotificationOnEarlyShutdown) { + absl::Notification initialized; + + auto server_thread = Thread::threadFactoryForTest().createThread([&] { + initialize("test/server/node_bootstrap.yaml"); + + // This shutdown notification should never be called because we will shutdown + // early before the init manager finishes initializing and therefore before + // the server starts worker threads. + auto shutdown_handle = server_->registerCallback(ServerLifecycleNotifier::Stage::ShutdownExit, + [&](Event::PostCb) { FAIL(); }); + NeverReadyTarget target(initialized); + server_->initManager().add(target); + server_->run(); + + shutdown_handle = nullptr; + server_ = nullptr; + thread_local_ = nullptr; + }); + + // Wait until the init manager starts initializing targets... + initialized.WaitForNotification(); + + // Now shutdown the main dispatcher and trigger server lifecycle notifications. + server_->dispatcher().post([&] { server_->shutdown(); }); + server_thread->join(); +} + TEST_P(ServerInstanceImplTest, V2ConfigOnly) { options_.service_cluster_name_ = "some_cluster_name"; options_.service_node_name_ = "some_node_name"; From bcc66c6b74c365d1d2834cfe15b847ae13be0eb6 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 17 Jul 2019 15:33:19 -0700 Subject: [PATCH 195/542] build: remove ranlib workaround (#7620) This has now been fixed upstream Signed-off-by: Keith Smiley --- bazel/foreign_cc/BUILD | 20 -------------------- bazel/repository_locations.bzl | 8 ++++---- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index f6049369a9ad..82fcae17b6a6 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -52,10 +52,6 @@ envoy_cmake_external( "CARES_SHARED": "no", "CARES_STATIC": "on", "CMAKE_BUILD_TYPE": "RelWithDebInfo", - # Disable ranlib because it is not handled by bazel, and therefore - # doesn't respect custom toolchains such as the Android NDK, - # see https://github.com/bazelbuild/rules_foreign_cc/issues/252 - "CMAKE_RANLIB": "", }, copy_pdb = True, lib_source = "@com_github_c_ares_c_ares//:all", @@ -95,10 +91,6 @@ envoy_cmake_external( "EVENT__DISABLE_TESTS": "on", "EVENT__LIBRARY_TYPE": "STATIC", "CMAKE_BUILD_TYPE": "Release", - # Disable ranlib because it is not handled by bazel, and therefore - # doesn't respect custom toolchains such as the Android NDK, - # see https://github.com/bazelbuild/rules_foreign_cc/issues/252 - "CMAKE_RANLIB": "", # Force _GNU_SOURCE on for Android builds. This would be contained in # a 'select' but the downstream macro uses a select on all of these # options, and they cannot be nested. @@ -126,10 +118,6 @@ envoy_cmake_external( "CMAKE_BUILD_TYPE": "RelWithDebInfo", "CMAKE_INSTALL_LIBDIR": "lib", "CMAKE_CXX_COMPILER_FORCED": "on", - # Disable ranlib because it is not handled by bazel, and therefore - # doesn't respect custom toolchains such as the Android NDK, - # see https://github.com/bazelbuild/rules_foreign_cc/issues/252 - "CMAKE_RANLIB": "", }, cmake_files_dir = "$BUILD_TMPDIR/lib/CMakeFiles", copy_pdb = True, @@ -149,10 +137,6 @@ envoy_cmake_external( "YAML_CPP_BUILD_TOOLS": "off", "CMAKE_BUILD_TYPE": "RelWithDebInfo", "CMAKE_CXX_COMPILER_FORCED": "on", - # Disable ranlib because it is not handled by bazel, and therefore - # doesn't respect custom toolchains such as the Android NDK, - # see https://github.com/bazelbuild/rules_foreign_cc/issues/252 - "CMAKE_RANLIB": "", }, lib_source = "@com_github_jbeder_yaml_cpp//:all", static_libraries = select({ @@ -167,10 +151,6 @@ envoy_cmake_external( name = "zlib", cache_entries = { "CMAKE_BUILD_TYPE": "RelWithDebInfo", - # Disable ranlib because it is not handled by bazel, and therefore - # doesn't respect custom toolchains such as the Android NDK, - # see https://github.com/bazelbuild/rules_foreign_cc/issues/252 - "CMAKE_RANLIB": "", }, copy_pdb = True, lib_source = "@net_zlib//:all", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index d809a6f87192..5cee046006e9 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -215,10 +215,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.18.5/rules_go-0.18.5.tar.gz"], ), rules_foreign_cc = dict( - sha256 = "980c1b74f5c18ea099889b0fb0479ee34b8a02845d3d302ecb16b15d73d624c8", - strip_prefix = "rules_foreign_cc-a0dc109915cea85909bef586e2b2a9bbdc6c8ff5", - # 2019-06-04 - urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/a0dc109915cea85909bef586e2b2a9bbdc6c8ff5.tar.gz"], + sha256 = "c957e6663094a1478c43330c1bbfa71afeaf1ab86b7565233783301240c7a0ab", + strip_prefix = "rules_foreign_cc-a209b642c7687a8894c19b3dd40e43e6d3f38e83", + # 2019-07-17 + urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/a209b642c7687a8894c19b3dd40e43e6d3f38e83.tar.gz"], ), six_archive = dict( sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", From 170c89eb0b2afb7a39d44d0f8dfb77444ffc038f Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Thu, 18 Jul 2019 09:41:51 -0700 Subject: [PATCH 196/542] listener: keep ListenerFactoryContext small (#7528) Signed-off-by: Yuchen Dai --- include/envoy/server/filter_config.h | 7 -- source/server/listener_manager_impl.h | 17 +-- test/mocks/server/mocks.h | 8 -- test/server/listener_manager_impl_test.cc | 142 +--------------------- 4 files changed, 11 insertions(+), 163 deletions(-) diff --git a/include/envoy/server/filter_config.h b/include/envoy/server/filter_config.h index cf99d734ad49..46d5fbac6165 100644 --- a/include/envoy/server/filter_config.h +++ b/include/envoy/server/filter_config.h @@ -173,13 +173,6 @@ class FactoryContext { class ListenerFactoryContext : public virtual FactoryContext { public: - /** - * Store socket options to be set on the listen socket before listening. - */ - virtual void addListenSocketOption(const Network::Socket::OptionConstSharedPtr& option) PURE; - - virtual void addListenSocketOptions(const Network::Socket::OptionsSharedPtr& options) PURE; - /** * Give access to the listener configuration */ diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index f302ff7391db..c3c9055ae4fd 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -305,14 +305,6 @@ class ListenerImpl : public Network::ListenerConfig, std::make_shared>(); } } - void addListenSocketOption(const Network::Socket::OptionConstSharedPtr& option) override { - ensureSocketOptions(); - listen_socket_options_->emplace_back(std::move(option)); - } - void addListenSocketOptions(const Network::Socket::OptionsSharedPtr& options) override { - ensureSocketOptions(); - Network::Socket::appendOptions(listen_socket_options_, options); - } const Network::ListenerConfig& listenerConfig() const override { return *this; } ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { return parent_.server_.messageValidationVisitor(); @@ -336,6 +328,15 @@ class ListenerImpl : public Network::ListenerConfig, SystemTime last_updated_; private: + void addListenSocketOption(const Network::Socket::OptionConstSharedPtr& option) { + ensureSocketOptions(); + listen_socket_options_->emplace_back(std::move(option)); + } + void addListenSocketOptions(const Network::Socket::OptionsSharedPtr& options) { + ensureSocketOptions(); + Network::Socket::appendOptions(listen_socket_options_, options); + } + ListenerManagerImpl& parent_; Network::Address::InstanceConstSharedPtr address_; FilterChainManagerImpl filter_chain_manager_; diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 7ab95b1df378..707fc9ebdf0c 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -517,14 +517,6 @@ class MockListenerFactoryContext : public MockFactoryContext, public ListenerFac MockListenerFactoryContext(); ~MockListenerFactoryContext() override; - void addListenSocketOption(const Network::Socket::OptionConstSharedPtr& option) override { - addListenSocketOption_(option); - } - MOCK_METHOD1(addListenSocketOption_, void(const Network::Socket::OptionConstSharedPtr&)); - void addListenSocketOptions(const Network::Socket::OptionsSharedPtr& options) override { - addListenSocketOptions_(options); - } - MOCK_METHOD1(addListenSocketOptions_, void(const Network::Socket::OptionsSharedPtr&)); const Network::ListenerConfig& listenerConfig() const override { return _listenerConfig_; } MOCK_CONST_METHOD0(listenerConfig_, const Network::ListenerConfig&()); diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 46c733415434..18df21b388e1 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -2686,30 +2686,12 @@ class OriginalDstTestFilter : public Extensions::ListenerFilters::OriginalDst::O }; TEST_F(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilter) { - // Static scope required for the io_handle to be in scope for the lambda below - // and for the final check at the end of this test. - static int fd; - fd = -1; - // temporary io_handle to test result of socket creation - Network::IoHandlePtr io_handle_tmp = std::make_unique(0); - EXPECT_CALL(*listener_factory_.socket_, ioHandle()).WillOnce(ReturnRef(*io_handle_tmp)); - class OriginalDstTestConfigFactory : public Configuration::NamedListenerFilterConfigFactory { public: // NamedListenerFilterConfigFactory Network::ListenerFilterFactoryCb createFilterFactoryFromProto(const Protobuf::Message&, - Configuration::ListenerFactoryContext& context) override { - auto option = std::make_unique(); - EXPECT_CALL(*option, setOption(_, envoy::api::v2::core::SocketOption::STATE_PREBIND)) - .WillOnce(Return(true)); - EXPECT_CALL(*option, setOption(_, envoy::api::v2::core::SocketOption::STATE_BOUND)) - .WillOnce(Invoke( - [](Network::Socket& socket, envoy::api::v2::core::SocketOption::SocketState) -> bool { - fd = socket.ioHandle().fd(); - return true; - })); - context.addListenSocketOption(std::move(option)); + Configuration::ListenerFactoryContext&) override { return [](Network::ListenerFilterManager& filter_manager) -> void { filter_manager.addAcceptFilter(std::make_unique()); }; @@ -2767,57 +2749,6 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilter) { EXPECT_TRUE(filterChainFactory.createListenerFilterChain(manager)); EXPECT_TRUE(socket.localAddressRestored()); EXPECT_EQ("127.0.0.2:2345", socket.localAddress()->asString()); - EXPECT_NE(fd, -1); - io_handle_tmp->close(); -} - -TEST_F(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterOptionFail) { - class OriginalDstTestConfigFactory : public Configuration::NamedListenerFilterConfigFactory { - public: - // NamedListenerFilterConfigFactory - Network::ListenerFilterFactoryCb - createFilterFactoryFromProto(const Protobuf::Message&, - Configuration::ListenerFactoryContext& context) override { - auto option = std::make_unique(); - EXPECT_CALL(*option, setOption(_, envoy::api::v2::core::SocketOption::STATE_PREBIND)) - .WillOnce(Return(false)); - context.addListenSocketOption(std::move(option)); - return [](Network::ListenerFilterManager& filter_manager) -> void { - filter_manager.addAcceptFilter(std::make_unique()); - }; - } - - ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); - } - - std::string name() override { return "testfail.listener.original_dst"; } - }; - - /** - * Static registration for the original dst filter. @see RegisterFactory. - */ - static Registry::RegisterFactory - registered_; - - const std::string yaml = TestEnvironment::substitute(R"EOF( - name: "socketOptionFailListener" - address: - socket_address: { address: 127.0.0.1, port_value: 1111 } - filter_chains: {} - listener_filters: - - name: "testfail.listener.original_dst" - config: {} - )EOF", - Network::Address::IpVersion::v4); - - EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); - - EXPECT_THROW_WITH_MESSAGE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(yaml), "", true), - EnvoyException, - "MockListenerComponentFactory: Setting socket options failed"); - EXPECT_EQ(0U, manager_->listeners().size()); } class OriginalDstTestFilterIPv6 @@ -2829,30 +2760,12 @@ class OriginalDstTestFilterIPv6 }; TEST_F(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterIPv6) { - // Static scope required for the io_handle to be in scope for the lambda below - // and for the final check at the end of this test. - static int fd; - fd = -1; - // temporary io_handle to test result of socket creation - Network::IoHandlePtr io_handle_tmp = std::make_unique(0); - EXPECT_CALL(*listener_factory_.socket_, ioHandle()).WillOnce(ReturnRef(*io_handle_tmp)); - class OriginalDstTestConfigFactory : public Configuration::NamedListenerFilterConfigFactory { public: // NamedListenerFilterConfigFactory Network::ListenerFilterFactoryCb createFilterFactoryFromProto(const Protobuf::Message&, - Configuration::ListenerFactoryContext& context) override { - auto option = std::make_unique(); - EXPECT_CALL(*option, setOption(_, envoy::api::v2::core::SocketOption::STATE_PREBIND)) - .WillOnce(Return(true)); - EXPECT_CALL(*option, setOption(_, envoy::api::v2::core::SocketOption::STATE_BOUND)) - .WillOnce(Invoke( - [](Network::Socket& socket, envoy::api::v2::core::SocketOption::SocketState) -> bool { - fd = socket.ioHandle().fd(); - return true; - })); - context.addListenSocketOption(std::move(option)); + Configuration::ListenerFactoryContext&) override { return [](Network::ListenerFilterManager& filter_manager) -> void { filter_manager.addAcceptFilter(std::make_unique()); }; @@ -2910,57 +2823,6 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterIPv6) { EXPECT_TRUE(filterChainFactory.createListenerFilterChain(manager)); EXPECT_TRUE(socket.localAddressRestored()); EXPECT_EQ("[1::2]:2345", socket.localAddress()->asString()); - EXPECT_NE(fd, -1); - io_handle_tmp->close(); -} - -TEST_F(ListenerManagerImplWithRealFiltersTest, OriginalDstTestFilterOptionFailIPv6) { - class OriginalDstTestConfigFactory : public Configuration::NamedListenerFilterConfigFactory { - public: - // NamedListenerFilterConfigFactory - Network::ListenerFilterFactoryCb - createFilterFactoryFromProto(const Protobuf::Message&, - Configuration::ListenerFactoryContext& context) override { - auto option = std::make_unique(); - EXPECT_CALL(*option, setOption(_, envoy::api::v2::core::SocketOption::STATE_PREBIND)) - .WillOnce(Return(false)); - context.addListenSocketOption(std::move(option)); - return [](Network::ListenerFilterManager& filter_manager) -> void { - filter_manager.addAcceptFilter(std::make_unique()); - }; - } - - ProtobufTypes::MessagePtr createEmptyConfigProto() override { - return std::make_unique(); - } - - std::string name() override { return "testfail.listener.original_dstipv6"; } - }; - - /** - * Static registration for the original dst filter. @see RegisterFactory. - */ - static Registry::RegisterFactory - registered_; - - const std::string yaml = TestEnvironment::substitute(R"EOF( - name: "socketOptionFailListener" - address: - socket_address: { address: ::0001, port_value: 1111 } - filter_chains: {} - listener_filters: - - name: "testfail.listener.original_dstipv6" - config: {} - )EOF", - Network::Address::IpVersion::v6); - - EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); - - EXPECT_THROW_WITH_MESSAGE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(yaml), "", true), - EnvoyException, - "MockListenerComponentFactory: Setting socket options failed"); - EXPECT_EQ(0U, manager_->listeners().size()); } // Validate that when neither transparent nor freebind is not set in the From aa24d0892610f1b6cbfb91b1d03c9b26f3724dc7 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 18 Jul 2019 10:12:10 -0700 Subject: [PATCH 197/542] ci: use local_memory_estimate (#7558) Signed-off-by: Lizan Zhou --- .bazelrc | 7 +------ ci/build_setup.sh | 2 +- ci/do_ci.sh | 6 ++++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.bazelrc b/.bazelrc index 771d01e27b3f..aaab93c120aa 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,12 +1,7 @@ # Envoy specific Bazel build/test options. - -# Bazel doesn't need more than 200MB of memory based on memory profiling: -# https://docs.bazel.build/versions/master/skylark/performance.html#memory-profiling -# Limiting JVM heapsize here to let it do GC more when approaching the limit to -# leave room for compiler/linker. -startup --host_jvm_args=-Xmx512m build --workspace_status_command=bazel/get_workspace_status build --experimental_remap_main_repo +build --experimental_local_memory_estimate build --host_force_python=PY2 build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a build --action_env=BAZEL_LINKOPTS=-lm:-static-libgcc diff --git a/ci/build_setup.sh b/ci/build_setup.sh index f5ae34ac5fc9..fd2a94f05a8b 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -58,7 +58,7 @@ fi export BAZEL_QUERY_OPTIONS="${BAZEL_OPTIONS}" export BAZEL_BUILD_OPTIONS="--strategy=Genrule=standalone --spawn_strategy=standalone \ --verbose_failures ${BAZEL_OPTIONS} --action_env=HOME --action_env=PYTHONUSERBASE \ - --jobs=${NUM_CPUS} --show_task_finish --experimental_generate_json_trace_profile \ + --local_cpu_resources=${NUM_CPUS} --show_task_finish --experimental_generate_json_trace_profile \ --test_env=HOME --test_env=PYTHONUSERBASE --cache_test_results=no --test_output=all \ ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}" diff --git a/ci/do_ci.sh b/ci/do_ci.sh index e47f66563bec..daf1c2a15e56 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -238,8 +238,10 @@ elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then setup_clang_toolchain echo "bazel coverage build with tests ${TEST_TARGETS}" - # LLVM coverage is a memory hog too. - [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_cpu_resources=6" + # Reduce the amount of memory Bazel tries to use to prevent it from launching too many subprocesses. + # This should prevent the system from running out of memory and killing tasks. See discussion on + # https://github.com/envoyproxy/envoy/pull/5611. + [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_ram_resources=12288" test/run_envoy_bazel_coverage.sh ${TEST_TARGETS} collect_build_profile coverage From 9f6594148306a9a86a18b14ff1c35ab4d19a0282 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Thu, 18 Jul 2019 10:28:33 -0700 Subject: [PATCH 198/542] clang-tidy: modernize-deprecated-headers (#7626) Description: Use headers such as `` as opposed to ``. These headers were marked deprecated in C++11/14 - https://clang.llvm.org/extra/clang-tidy/checks/modernize-deprecated-headers.html Risk Level: Low Testing: existing Docs Changes: N/A Release Notes: N/A Signed-off-by: Derek Argueta --- .clang-tidy | 1 + source/common/network/io_socket_handle_impl.cc | 3 +-- source/common/router/config_utility.h | 3 +-- source/common/router/header_parser.cc | 3 +-- source/common/signal/signal_action.h | 2 +- source/common/stats/isolated_store_impl.cc | 3 +-- source/common/stats/isolated_store_impl.h | 3 +-- source/common/stats/tag_extractor_impl.cc | 3 +-- .../filters/listener/proxy_protocol/proxy_protocol.cc | 2 +- source/extensions/filters/network/dubbo_proxy/conn_manager.cc | 2 +- source/extensions/filters/network/thrift_proxy/metadata.h | 3 +-- source/server/server.cc | 3 +-- test/common/signal/signals_test.cc | 3 ++- test/integration/tcp_dump.cc | 2 +- 14 files changed, 15 insertions(+), 21 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 6d73fdb3994b..ffdc2d0dd973 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -19,6 +19,7 @@ WarningsAsErrors: 'abseil-duration-*, bugprone-assert-side-effect, bugprone-unused-raii, bugprone-use-after-move, + modernize-deprecated-headers, modernize-make-shared, modernize-make-unique, modernize-return-braced-init-list, diff --git a/source/common/network/io_socket_handle_impl.cc b/source/common/network/io_socket_handle_impl.cc index f2442973bd11..cce957655380 100644 --- a/source/common/network/io_socket_handle_impl.cc +++ b/source/common/network/io_socket_handle_impl.cc @@ -1,7 +1,6 @@ #include "common/network/io_socket_handle_impl.h" -#include - +#include #include #include "envoy/buffer/buffer.h" diff --git a/source/common/router/config_utility.h b/source/common/router/config_utility.h index adfb1e9d2818..4a753964059f 100644 --- a/source/common/router/config_utility.h +++ b/source/common/router/config_utility.h @@ -1,7 +1,6 @@ #pragma once -#include - +#include #include #include #include diff --git a/source/common/router/header_parser.cc b/source/common/router/header_parser.cc index 05022605347c..e5d71aa9ef1f 100644 --- a/source/common/router/header_parser.cc +++ b/source/common/router/header_parser.cc @@ -1,7 +1,6 @@ #include "common/router/header_parser.h" -#include - +#include #include #include diff --git a/source/common/signal/signal_action.h b/source/common/signal/signal_action.h index 12fcb1805a7b..b7ae17cd426b 100644 --- a/source/common/signal/signal_action.h +++ b/source/common/signal/signal_action.h @@ -1,9 +1,9 @@ #pragma once -#include #include #include +#include #include #include "common/common/non_copyable.h" diff --git a/source/common/stats/isolated_store_impl.cc b/source/common/stats/isolated_store_impl.cc index 83a99379c587..6f2da0f899af 100644 --- a/source/common/stats/isolated_store_impl.cc +++ b/source/common/stats/isolated_store_impl.cc @@ -1,8 +1,7 @@ #include "common/stats/isolated_store_impl.h" -#include - #include +#include #include #include "common/common/utility.h" diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index ae22d13dfdba..ce6ddaca703b 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -1,8 +1,7 @@ #pragma once -#include - #include +#include #include #include "envoy/stats/stats.h" diff --git a/source/common/stats/tag_extractor_impl.cc b/source/common/stats/tag_extractor_impl.cc index acf440fa695e..7bb02e6f53e1 100644 --- a/source/common/stats/tag_extractor_impl.cc +++ b/source/common/stats/tag_extractor_impl.cc @@ -1,7 +1,6 @@ #include "common/stats/tag_extractor_impl.h" -#include - +#include #include #include "envoy/common/exception.h" diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc index 9c6fb7b93997..f4dce6138b9c 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol.cc @@ -1,11 +1,11 @@ #include "extensions/filters/listener/proxy_protocol/proxy_protocol.h" -#include #include #include #include #include +#include #include #include diff --git a/source/extensions/filters/network/dubbo_proxy/conn_manager.cc b/source/extensions/filters/network/dubbo_proxy/conn_manager.cc index 534b3d292964..b9a5f27d278f 100644 --- a/source/extensions/filters/network/dubbo_proxy/conn_manager.cc +++ b/source/extensions/filters/network/dubbo_proxy/conn_manager.cc @@ -1,6 +1,6 @@ #include "extensions/filters/network/dubbo_proxy/conn_manager.h" -#include +#include #include "envoy/common/exception.h" diff --git a/source/extensions/filters/network/thrift_proxy/metadata.h b/source/extensions/filters/network/thrift_proxy/metadata.h index 5c1469e86e3f..731514756656 100644 --- a/source/extensions/filters/network/thrift_proxy/metadata.h +++ b/source/extensions/filters/network/thrift_proxy/metadata.h @@ -1,8 +1,7 @@ #pragma once -#include - #include +#include #include #include #include diff --git a/source/server/server.cc b/source/server/server.cc index 117cd8a6b60f..107e99beeb1f 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -1,8 +1,7 @@ #include "server/server.h" -#include - #include +#include #include #include #include diff --git a/test/common/signal/signals_test.cc b/test/common/signal/signals_test.cc index 72b28c7bbf8b..0dda4fe752c8 100644 --- a/test/common/signal/signals_test.cc +++ b/test/common/signal/signals_test.cc @@ -1,6 +1,7 @@ -#include #include +#include + #include "common/signal/signal_action.h" #include "test/test_common/utility.h" diff --git a/test/integration/tcp_dump.cc b/test/integration/tcp_dump.cc index 1c3b76f124b7..95cd8b55c0e0 100644 --- a/test/integration/tcp_dump.cc +++ b/test/integration/tcp_dump.cc @@ -1,10 +1,10 @@ #include "test/integration/tcp_dump.h" -#include #include #include #include +#include #include #include "common/common/assert.h" From e4e1e627882e085dce8c84ffb77a33daf4c558d1 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Thu, 18 Jul 2019 13:57:11 -0700 Subject: [PATCH 199/542] release: un-deprecate "use_original_dst". (#7623) "use_original_dst" and "bind_to_port" are two complementary parts of the "virtual listeners" feature, and they should be deprecated together. However, they are both currently exempted (see: #5355), so revert the change from #7549. Signed-off-by: Piotr Sikora --- source/common/runtime/runtime_features.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 3a26e048119b..37eb7bb8cdc5 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -44,7 +44,6 @@ constexpr const char* disallowed_features[] = { "envoy.deprecated_features.deprecated.proto:is_deprecated_fatal", "envoy.deprecated_features.bootstrap.proto:runtime", "envoy.deprecated_features.redis_proxy.proto:catch_all_cluster", - "envoy.deprecated_features.lds.proto:use_original_dst", "envoy.deprecated_features.server_info.proto:max_stats", "envoy.deprecated_features.redis_proxy.proto:cluster", "envoy.deprecated_features.server_info.proto:max_obj_name_len", From 14c5371735e34ca3659b8332a19de24b5c776163 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 18 Jul 2019 14:42:19 -0700 Subject: [PATCH 200/542] ci: enable clang based RBE in Azure pipelines (#7536) Description: Minimum configuration to enable RBE and run release pipeline in Azure pipelines. Risk Level: Low Testing: Docs Changes: N/A Release Notes: N/A Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 27 +- .azure-pipelines/macos.yml | 2 +- .bazelrc | 44 + .circleci/config.yml | 1 + WORKSPACE | 4 + bazel/BUILD | 1 + bazel/repositories.bzl | 1 + bazel/repository_locations.bzl | 8 + bazel/toolchains/BUILD | 17 + bazel/toolchains/README.md | 13 + bazel/toolchains/configs/.gitignore | 2 + .../configs/clang/bazel_0.28.0/cc/BUILD | 148 ++ .../cc/armeabi_cc_toolchain_config.bzl | 82 ++ .../bazel_0.28.0/cc/cc_toolchain_config.bzl | 1202 +++++++++++++++++ .../clang/bazel_0.28.0/cc/cc_wrapper.sh | 25 + .../configs/clang/bazel_0.28.0/config/BUILD | 53 + .../configs/clang/bazel_0.28.0/java/BUILD | 25 + .../configs/gcc/bazel_0.28.0/cc/BUILD | 148 ++ .../cc/armeabi_cc_toolchain_config.bzl | 82 ++ .../bazel_0.28.0/cc/cc_toolchain_config.bzl | 1202 +++++++++++++++++ .../configs/gcc/bazel_0.28.0/cc/cc_wrapper.sh | 25 + .../configs/gcc/bazel_0.28.0/config/BUILD | 53 + .../configs/gcc/bazel_0.28.0/java/BUILD | 25 + bazel/toolchains/configs/versions.bzl | 17 + bazel/toolchains/empty.bzl | 18 + bazel/toolchains/rbe_toolchains_config.bzl | 80 ++ bazel/toolchains/regenerate.sh | 12 + ci/WORKSPACE.filter.example | 2 +- ci/build_setup.sh | 5 +- ci/do_ci.sh | 4 +- ci/setup_cache.sh | 2 +- tools/check_format.py | 3 +- tools/envoy_build_fixer.py | 2 +- 33 files changed, 3315 insertions(+), 20 deletions(-) create mode 100644 bazel/toolchains/BUILD create mode 100644 bazel/toolchains/README.md create mode 100644 bazel/toolchains/configs/.gitignore create mode 100755 bazel/toolchains/configs/clang/bazel_0.28.0/cc/BUILD create mode 100755 bazel/toolchains/configs/clang/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl create mode 100755 bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_toolchain_config.bzl create mode 100755 bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_wrapper.sh create mode 100644 bazel/toolchains/configs/clang/bazel_0.28.0/config/BUILD create mode 100644 bazel/toolchains/configs/clang/bazel_0.28.0/java/BUILD create mode 100755 bazel/toolchains/configs/gcc/bazel_0.28.0/cc/BUILD create mode 100755 bazel/toolchains/configs/gcc/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl create mode 100755 bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_toolchain_config.bzl create mode 100755 bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_wrapper.sh create mode 100644 bazel/toolchains/configs/gcc/bazel_0.28.0/config/BUILD create mode 100644 bazel/toolchains/configs/gcc/bazel_0.28.0/java/BUILD create mode 100644 bazel/toolchains/configs/versions.bzl create mode 100644 bazel/toolchains/empty.bzl create mode 100644 bazel/toolchains/rbe_toolchains_config.bzl create mode 100755 bazel/toolchains/regenerate.sh diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index 725e299b19cb..d8a0ed07eca9 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -10,20 +10,27 @@ jobs: vmImage: 'Ubuntu 16.04' container: envoy-build steps: - # bazel.dev isn't for CI purpose but we use it here to experiment with Azure Pipeline - # as it builds faster, before Azure can provision larger VM or we can use RBE. - - script: ci/do_ci.sh bazel.dev + - bash: | + echo "disk space at beginning of build:" + df -h + + - script: ci/do_ci.sh bazel.release env: ENVOY_SRCDIR: $(Build.SourcesDirectory) BUILD_DIR: $(Build.StagingDirectory) - BAZEL_EXTRA_TEST_OPTIONS: "--test_env=ENVOY_IP_TEST_VERSIONS=v4only" - BAZEL_REMOTE_CACHE: remotebuildexecution.googleapis.com + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-clang --config=remote-ci --jobs=100 --curses=no" + # //test/integration:echo_integration_test is marked exclusive so it will be run locally, no IPv6 locally in container. + BAZEL_EXTRA_TEST_OPTIONS: "--test_filter=-IpVersions/EchoIntegrationTest.*/IPv6" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - task: PublishTestResults@2 - inputs: - testResultsFiles: '**/testlogs/**/test.xml' - testRunTitle: 'bazel.dev' - searchFolder: $(Build.StagingDirectory) + - bash: | + echo "disk space at end of build:" + df -h condition: always() + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/envoy" + artifactName: 'envoy' diff --git a/.azure-pipelines/macos.yml b/.azure-pipelines/macos.yml index 8aa255052dd1..20655177351d 100644 --- a/.azure-pipelines/macos.yml +++ b/.azure-pipelines/macos.yml @@ -15,7 +15,7 @@ jobs: - script: ./ci/mac_ci_steps.sh displayName: 'Run Mac CI' env: - BAZEL_REMOTE_CACHE: remotebuildexecution.googleapis.com + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) diff --git a/.bazelrc b/.bazelrc index aaab93c120aa..f8a14fadc686 100644 --- a/.bazelrc +++ b/.bazelrc @@ -70,3 +70,47 @@ build:sizeopt -c opt --copt -Os # Test options build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH + +# Remote execution: https://docs.bazel.build/versions/master/remote-execution.html +build:rbe-toolchain --host_javabase=@rbe_ubuntu_clang//java:jdk +build:rbe-toolchain --javabase=@rbe_ubuntu_clang//java:jdk +build:rbe-toolchain --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:rbe-toolchain --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:rbe-toolchain --host_platform=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform +build:rbe-toolchain --platforms=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform +build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +build:rbe-toolchain --crosstool_top=@rbe_ubuntu_clang//cc:toolchain +build:rbe-toolchain --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain +build:rbe-toolchain --linkopt=-fuse-ld=lld +build:rbe-toolchain --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin + +build:remote --spawn_strategy=remote,sandboxed,local +build:remote --strategy=Javac=remote,sandboxed,local +build:remote --strategy=Closure=remote,sandboxed,local +build:remote --strategy=Genrule=remote,sandboxed,local +build:remote --remote_timeout=3600 +build:remote --auth_enabled=true +build:remote --experimental_inmemory_jdeps_files +build:remote --experimental_inmemory_dotd_files +build:remote --experimental_remote_download_outputs=toplevel +test:remote --experimental_remote_download_outputs=minimal + +build:remote-clang --config=remote +build:remote-clang --config=rbe-toolchain + +# Docker sandbox +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build:cfc514546bc0284536893cca5fa43d7128edcd35 +build:docker-sandbox --spawn_strategy=docker +build:docker-sandbox --strategy=Javac=docker +build:docker-sandbox --strategy=Closure=docker +build:docker-sandbox --strategy=Genrule=docker +build:docker-sandbox --define=EXECUTOR=remote +build:docker-sandbox --experimental_docker_verbose +build:docker-sandbox --experimental_enable_docker_sandbox + +build:docker-clang --config=docker-sandbox +build:docker-clang --config=rbe-toolchain + +# CI configurations +build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com +build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com diff --git a/.circleci/config.yml b/.circleci/config.yml index 69c8a7ed7437..ea49aaaa7018 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,7 @@ executors: ubuntu-build: description: "A regular build executor based on ubuntu image" docker: + # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here. - image: envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d resource_class: xlarge working_directory: /source diff --git a/WORKSPACE b/WORKSPACE index bc3ce13bd3a7..b10f47cf7026 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -17,3 +17,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe go_rules_dependencies() go_register_toolchains(go_version = GO_VERSION) + +load("@envoy//bazel/toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config") + +rbe_toolchains_config() diff --git a/bazel/BUILD b/bazel/BUILD index a574dc1eb27a..d78d94046059 100755 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -5,6 +5,7 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "gen_sh_test_runner.sh", "sh_test_wrapper.sh", + "cc_wrapper.py", ]) genrule( diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 1c0e6abb57dd..fe86b72263d8 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -156,6 +156,7 @@ def envoy_dependencies(skip_targets = []): _com_lightstep_tracer_cpp() _io_opentracing_cpp() _net_zlib() + _repository_impl("bazel_toolchains") _python_deps() _cc_deps() diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 5cee046006e9..dece1e411e59 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -3,6 +3,14 @@ REPOSITORY_LOCATIONS = dict( sha256 = "3c681998538231a2d24d0c07ed5a7658cb72bfb5fd4bf9911157c0e9ac6a2687", urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.17.0/bazel-gazelle-0.17.0.tar.gz"], ), + bazel_toolchains = dict( + sha256 = "68e7678473090542e679ce7e6aa8a3ba5669577dede2b404f9865d556bd99f10", + strip_prefix = "bazel-toolchains-0.28.0", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/0.28.0.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/0.28.0.tar.gz", + ], + ), boringssl = dict( # Use commits from branch "chromium-stable-with-bazel" sha256 = "448773376d063b1e9a19e4fd41002d1a31a968a13be20b3b66ecd4aab9cf14a8", diff --git a/bazel/toolchains/BUILD b/bazel/toolchains/BUILD new file mode 100644 index 000000000000..e6a683365028 --- /dev/null +++ b/bazel/toolchains/BUILD @@ -0,0 +1,17 @@ +licenses(["notice"]) # Apache 2 + +platform( + name = "rbe_ubuntu_clang_platform", + parents = ["@rbe_ubuntu_clang//config:platform"], + remote_execution_properties = """ + {PARENT_REMOTE_EXECUTION_PROPERTIES} + properties: { + name: "dockerAddCapabilities" + value: "SYS_PTRACE,NET_RAW,NET_ADMIN" + } + properties: { + name: "dockerNetwork" + value: "standard" + } + """, +) diff --git a/bazel/toolchains/README.md b/bazel/toolchains/README.md new file mode 100644 index 000000000000..38e01600e86e --- /dev/null +++ b/bazel/toolchains/README.md @@ -0,0 +1,13 @@ +# Bazel Toolchains + +This directory contains toolchains config generated for Bazel [RBE](https://docs.bazel.build/versions/master/remote-execution.html) and +[Docker sandbox](https://docs.bazel.build/versions/master/remote-execution-sandbox.html). + +To regenerate toolchain configs, update the docker image information in `rbe_toolchains_config.bzl` and run following command in an +environment with the latest Bazel and Docker installed: + +``` +bazel/toolchains/regenerate.sh +``` + +This will generate configs in `bazel/toolchains/configs`, check in those files so they can be used in CI. diff --git a/bazel/toolchains/configs/.gitignore b/bazel/toolchains/configs/.gitignore new file mode 100644 index 000000000000..9683e8aa4d54 --- /dev/null +++ b/bazel/toolchains/configs/.gitignore @@ -0,0 +1,2 @@ +# RBE autoconfig generator will generate a .bazelrc but we don't need it. +.latest.bazelrc diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/BUILD b/bazel/toolchains/configs/clang/bazel_0.28.0/cc/BUILD new file mode 100755 index 000000000000..7de0f2682246 --- /dev/null +++ b/bazel/toolchains/configs/clang/bazel_0.28.0/cc/BUILD @@ -0,0 +1,148 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes. + +package(default_visibility = ["//visibility:public"]) + +load(":cc_toolchain_config.bzl", "cc_toolchain_config") +load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") + +licenses(["notice"]) # Apache 2.0 + +cc_library( + name = "malloc", +) + +filegroup( + name = "empty", + srcs = [], +) + +filegroup( + name = "cc_wrapper", + srcs = ["cc_wrapper.sh"], +) + +filegroup( + name = "compiler_deps", + srcs = glob(["extra_tools/**"], allow_empty = True) + [":empty"], +) + +# This is the entry point for --crosstool_top. Toolchains are found +# by lopping off the name of --crosstool_top and searching for +# the "${CPU}" entry in the toolchains attribute. +cc_toolchain_suite( + name = "toolchain", + toolchains = { + "k8|clang": ":cc-compiler-k8", + "k8": ":cc-compiler-k8", + "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a", + "armeabi-v7a": ":cc-compiler-armeabi-v7a", + }, +) + +cc_toolchain( + name = "cc-compiler-k8", + toolchain_identifier = "local", + toolchain_config = ":local", + all_files = ":compiler_deps", + ar_files = ":compiler_deps", + as_files = ":compiler_deps", + compiler_files = ":compiler_deps", + dwp_files = ":empty", + linker_files = ":compiler_deps", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 1, +) + +cc_toolchain_config( + name = "local", + cpu = "k8", + compiler = "clang", + toolchain_identifier = "local", + host_system_name = "local", + target_system_name = "local", + target_libc = "local", + abi_version = "local", + abi_libc_version = "local", + cxx_builtin_include_directories = ["/usr/local/include", + "/usr/lib/llvm-8/lib/clang/8.0.1/include", + "/usr/include/x86_64-linux-gnu", + "/usr/include", + "/usr/include/c++/7.4.0", + "/usr/include/x86_64-linux-gnu/c++/7.4.0", + "/usr/include/c++/7.4.0/backward", + "/usr/include/clang/8.0.1/include"], + tool_paths = {"ar": "/usr/bin/ar", + "ld": "/usr/bin/ld", + "cpp": "/usr/bin/cpp", + "gcc": "/usr/lib/llvm-8/bin/clang", + "dwp": "/usr/bin/dwp", + "gcov": "/usr/lib/llvm-8/bin/llvm-profdata", + "nm": "/usr/bin/nm", + "objcopy": "/usr/bin/objcopy", + "objdump": "/usr/bin/objdump", + "strip": "/usr/bin/strip"}, + compile_flags = ["-U_FORTIFY_SOURCE", + "-fstack-protector", + "-Wall", + "-Wthread-safety", + "-Wself-assign", + "-fcolor-diagnostics", + "-fno-omit-frame-pointer"], + opt_compile_flags = ["-g0", + "-O2", + "-D_FORTIFY_SOURCE=1", + "-DNDEBUG", + "-ffunction-sections", + "-fdata-sections"], + dbg_compile_flags = ["-g"], + cxx_flags = ["-std=c++0x"], + link_flags = ["-fuse-ld=gold", + "-Wl,-no-as-needed", + "-Wl,-z,relro,-z,now", + "-B/usr/lib/llvm-8/bin", + "-lm", + "-static-libgcc"], + link_libs = ["-l:libstdc++.a"], + opt_link_flags = ["-Wl,--gc-sections"], + unfiltered_compile_flags = ["-no-canonical-prefixes", + "-Wno-builtin-macro-redefined", + "-D__DATE__=\"redacted\"", + "-D__TIMESTAMP__=\"redacted\"", + "-D__TIME__=\"redacted\""], + coverage_compile_flags = ["-fprofile-instr-generate", "-fcoverage-mapping"], + coverage_link_flags = ["-fprofile-instr-generate"], + supports_start_end_lib = True, +) + +# Android tooling requires a default toolchain for the armeabi-v7a cpu. +cc_toolchain( + name = "cc-compiler-armeabi-v7a", + toolchain_identifier = "stub_armeabi-v7a", + toolchain_config = ":stub_armeabi-v7a", + all_files = ":empty", + ar_files = ":empty", + as_files = ":empty", + compiler_files = ":empty", + dwp_files = ":empty", + linker_files = ":empty", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 1, +) + +armeabi_cc_toolchain_config(name = "stub_armeabi-v7a") diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/clang/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl new file mode 100755 index 000000000000..94e0720bf6c9 --- /dev/null +++ b/bazel/toolchains/configs/clang/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl @@ -0,0 +1,82 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Starlark cc_toolchain configuration rule""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "feature", + "tool_path", +) + +def _impl(ctx): + toolchain_identifier = "stub_armeabi-v7a" + host_system_name = "armeabi-v7a" + target_system_name = "armeabi-v7a" + target_cpu = "armeabi-v7a" + target_libc = "armeabi-v7a" + compiler = "compiler" + abi_version = "armeabi-v7a" + abi_libc_version = "armeabi-v7a" + cc_target_os = None + builtin_sysroot = None + action_configs = [] + + supports_pic_feature = feature(name = "supports_pic", enabled = True) + supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) + features = [supports_dynamic_linker_feature, supports_pic_feature] + + cxx_builtin_include_directories = [] + artifact_name_patterns = [] + make_variables = [] + + tool_paths = [ + tool_path(name = "ar", path = "/bin/false"), + tool_path(name = "compat-ld", path = "/bin/false"), + tool_path(name = "cpp", path = "/bin/false"), + tool_path(name = "dwp", path = "/bin/false"), + tool_path(name = "gcc", path = "/bin/false"), + tool_path(name = "gcov", path = "/bin/false"), + tool_path(name = "ld", path = "/bin/false"), + tool_path(name = "nm", path = "/bin/false"), + tool_path(name = "objcopy", path = "/bin/false"), + tool_path(name = "objdump", path = "/bin/false"), + tool_path(name = "strip", path = "/bin/false"), + ] + + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + artifact_name_patterns = artifact_name_patterns, + cxx_builtin_include_directories = cxx_builtin_include_directories, + toolchain_identifier = toolchain_identifier, + host_system_name = host_system_name, + target_system_name = target_system_name, + target_cpu = target_cpu, + target_libc = target_libc, + compiler = compiler, + abi_version = abi_version, + abi_libc_version = abi_libc_version, + tool_paths = tool_paths, + make_variables = make_variables, + builtin_sysroot = builtin_sysroot, + cc_target_os = cc_target_os, + ) + +armeabi_cc_toolchain_config = rule( + implementation = _impl, + attrs = {}, + provides = [CcToolchainConfigInfo], +) diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_toolchain_config.bzl new file mode 100755 index 000000000000..f2b12d962963 --- /dev/null +++ b/bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_toolchain_config.bzl @@ -0,0 +1,1202 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Starlark cc_toolchain configuration rule""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "feature", + "feature_set", + "flag_group", + "flag_set", + "tool_path", + "variable_with_value", + "with_feature_set", +) +load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") + +all_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.lto_backend, +] + +all_cpp_compile_actions = [ + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, +] + +preprocessor_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, +] + +codegen_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, +] + +all_link_actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, +] + +def _impl(ctx): + tool_paths = [ + tool_path(name = name, path = path) + for name, path in ctx.attr.tool_paths.items() + ] + action_configs = [] + + supports_pic_feature = feature( + name = "supports_pic", + enabled = True, + ) + supports_start_end_lib_feature = feature( + name = "supports_start_end_lib", + enabled = True, + ) + + default_compile_flags_feature = feature( + name = "default_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.compile_flags, + ), + ] if ctx.attr.compile_flags else []), + ), + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.dbg_compile_flags, + ), + ] if ctx.attr.dbg_compile_flags else []), + with_features = [with_feature_set(features = ["dbg"])], + ), + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.opt_compile_flags, + ), + ] if ctx.attr.opt_compile_flags else []), + with_features = [with_feature_set(features = ["opt"])], + ), + flag_set( + actions = [ + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.cxx_flags, + ), + ] if ctx.attr.cxx_flags else []), + ), + ], + ) + + default_link_flags_feature = feature( + name = "default_link_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = ([ + flag_group( + flags = ctx.attr.link_flags, + ), + ] if ctx.attr.link_flags else []), + ), + flag_set( + actions = all_link_actions, + flag_groups = ([ + flag_group( + flags = ctx.attr.opt_link_flags, + ), + ] if ctx.attr.opt_link_flags else []), + with_features = [with_feature_set(features = ["opt"])], + ), + ], + ) + + dbg_feature = feature(name = "dbg") + + opt_feature = feature(name = "opt") + + sysroot_feature = feature( + name = "sysroot", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = ["--sysroot=%{sysroot}"], + expand_if_available = "sysroot", + ), + ], + ), + ], + ) + + fdo_optimize_feature = feature( + name = "fdo_optimize", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-use=%{fdo_profile_path}", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) + + user_compile_flags_feature = feature( + name = "user_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["%{user_compile_flags}"], + iterate_over = "user_compile_flags", + expand_if_available = "user_compile_flags", + ), + ], + ), + ], + ) + + unfiltered_compile_flags_feature = feature( + name = "unfiltered_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.unfiltered_compile_flags, + ), + ] if ctx.attr.unfiltered_compile_flags else []), + ), + ], + ) + + library_search_directories_feature = feature( + name = "library_search_directories", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-L%{library_search_directories}"], + iterate_over = "library_search_directories", + expand_if_available = "library_search_directories", + ), + ], + ), + ], + ) + + static_libgcc_feature = feature( + name = "static_libgcc", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ], + flag_groups = [flag_group(flags = ["-static-libgcc"])], + with_features = [ + with_feature_set(features = ["static_link_cpp_runtimes"]), + ], + ), + ], + ) + + pic_feature = feature( + name = "pic", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = [ + flag_group(flags = ["-fPIC"], expand_if_available = "pic"), + ], + ), + ], + ) + + per_object_debug_info_feature = feature( + name = "per_object_debug_info", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ], + flag_groups = [ + flag_group( + flags = ["-gsplit-dwarf"], + expand_if_available = "per_object_debug_info_file", + ), + ], + ), + ], + ) + + preprocessor_defines_feature = feature( + name = "preprocessor_defines", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-D%{preprocessor_defines}"], + iterate_over = "preprocessor_defines", + ), + ], + ), + ], + ) + + cs_fdo_optimize_feature = feature( + name = "cs_fdo_optimize", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.lto_backend], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-use=%{fdo_profile_path}", + "-Xclang-only=-Wno-profile-instr-unprofiled", + "-Xclang-only=-Wno-profile-instr-out-of-date", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["csprofile"], + ) + + autofdo_feature = feature( + name = "autofdo", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-fauto-profile=%{fdo_profile_path}", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + runtime_library_search_directories_feature = feature( + name = "runtime_library_search_directories", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "runtime_library_search_directories", + flag_groups = [ + flag_group( + flags = [ + "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_true = "is_cc_test", + ), + flag_group( + flags = [ + "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_false = "is_cc_test", + ), + ], + expand_if_available = + "runtime_library_search_directories", + ), + ], + with_features = [ + with_feature_set(features = ["static_link_cpp_runtimes"]), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "runtime_library_search_directories", + flag_groups = [ + flag_group( + flags = [ + "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", + ], + ), + ], + expand_if_available = + "runtime_library_search_directories", + ), + ], + with_features = [ + with_feature_set( + not_features = ["static_link_cpp_runtimes"], + ), + ], + ), + ], + ) + + fission_support_feature = feature( + name = "fission_support", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-Wl,--gdb-index"], + expand_if_available = "is_using_fission", + ), + ], + ), + ], + ) + + shared_flag_feature = feature( + name = "shared_flag", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [flag_group(flags = ["-shared"])], + ), + ], + ) + + random_seed_feature = feature( + name = "random_seed", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = [ + flag_group( + flags = ["-frandom-seed=%{output_file}"], + expand_if_available = "output_file", + ), + ], + ), + ], + ) + + includes_feature = feature( + name = "includes", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = ["-include", "%{includes}"], + iterate_over = "includes", + expand_if_available = "includes", + ), + ], + ), + ], + ) + + fdo_instrument_feature = feature( + name = "fdo_instrument", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-generate=%{fdo_instrument_path}", + "-fno-data-sections", + ], + expand_if_available = "fdo_instrument_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + cs_fdo_instrument_feature = feature( + name = "cs_fdo_instrument", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.lto_backend, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = [ + "-fcs-profile-generate=%{cs_fdo_instrument_path}", + ], + expand_if_available = "cs_fdo_instrument_path", + ), + ], + ), + ], + provides = ["csprofile"], + ) + + include_paths_feature = feature( + name = "include_paths", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = ["-iquote", "%{quote_include_paths}"], + iterate_over = "quote_include_paths", + ), + flag_group( + flags = ["-I%{include_paths}"], + iterate_over = "include_paths", + ), + flag_group( + flags = ["-isystem", "%{system_include_paths}"], + iterate_over = "system_include_paths", + ), + ], + ), + ], + ) + + symbol_counts_feature = feature( + name = "symbol_counts", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = [ + "-Wl,--print-symbol-counts=%{symbol_counts_output}", + ], + expand_if_available = "symbol_counts_output", + ), + ], + ), + ], + ) + + llvm_coverage_map_format_feature = feature( + name = "llvm_coverage_map_format", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-instr-generate", + "-fcoverage-mapping", + ], + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + "objc-executable", + "objc++-executable", + ], + flag_groups = [ + flag_group(flags = ["-fprofile-instr-generate"]), + ], + ), + ], + requires = [feature_set(features = ["coverage"])], + provides = ["profile"], + ) + + strip_debug_symbols_feature = feature( + name = "strip_debug_symbols", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-Wl,-S"], + expand_if_available = "strip_debug_symbols", + ), + ], + ), + ], + ) + + build_interface_libraries_feature = feature( + name = "build_interface_libraries", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = [ + "%{generate_interface_library}", + "%{interface_library_builder_path}", + "%{interface_library_input_path}", + "%{interface_library_output_path}", + ], + expand_if_available = "generate_interface_library", + ), + ], + with_features = [ + with_feature_set( + features = ["supports_interface_shared_libraries"], + ), + ], + ), + ], + ) + + libraries_to_link_feature = feature( + name = "libraries_to_link", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["-Wl,--start-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["-Wl,-whole-archive"], + expand_if_true = + "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "interface_library", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "static_library", + ), + ), + flag_group( + flags = ["-l%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "dynamic_library", + ), + ), + flag_group( + flags = ["-l:%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "versioned_dynamic_library", + ), + ), + flag_group( + flags = ["-Wl,-no-whole-archive"], + expand_if_true = "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["-Wl,--end-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + flag_group( + flags = ["-Wl,@%{thinlto_param_file}"], + expand_if_true = "thinlto_param_file", + ), + ], + ), + ], + ) + + user_link_flags_feature = feature( + name = "user_link_flags", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{user_link_flags}"], + iterate_over = "user_link_flags", + expand_if_available = "user_link_flags", + ), + ] + ([flag_group(flags = ctx.attr.link_libs)] if ctx.attr.link_libs else []), + ), + ], + ) + + fdo_prefetch_hints_feature = feature( + name = "fdo_prefetch_hints", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.lto_backend, + ], + flag_groups = [ + flag_group( + flags = [ + "-Xclang-only=-mllvm", + "-Xclang-only=-prefetch-hints-file=%{fdo_prefetch_hints_path}", + ], + expand_if_available = "fdo_prefetch_hints_path", + ), + ], + ), + ], + ) + + linkstamps_feature = feature( + name = "linkstamps", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{linkstamp_paths}"], + iterate_over = "linkstamp_paths", + expand_if_available = "linkstamp_paths", + ), + ], + ), + ], + ) + + gcc_coverage_map_format_feature = feature( + name = "gcc_coverage_map_format", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + "objc-executable", + "objc++-executable", + ], + flag_groups = [ + flag_group( + flags = ["-fprofile-arcs", "-ftest-coverage"], + expand_if_available = "gcov_gcno_file", + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [flag_group(flags = ["--coverage"])], + ), + ], + requires = [feature_set(features = ["coverage"])], + provides = ["profile"], + ) + + archiver_flags_feature = feature( + name = "archiver_flags", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group(flags = ["rcsD"]), + flag_group( + flags = ["%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + flag_set( + actions = [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + ], + ), + ], + ) + + force_pic_flags_feature = feature( + name = "force_pic_flags", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.cpp_link_executable], + flag_groups = [ + flag_group( + flags = ["-pie"], + expand_if_available = "force_pic", + ), + ], + ), + ], + ) + + dependency_file_feature = feature( + name = "dependency_file", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-MD", "-MF", "%{dependency_file}"], + expand_if_available = "dependency_file", + ), + ], + ), + ], + ) + + dynamic_library_linker_tool_feature = feature( + name = "dynamic_library_linker_tool", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = [" + cppLinkDynamicLibraryToolPath + "], + expand_if_available = "generate_interface_library", + ), + ], + with_features = [ + with_feature_set( + features = ["supports_interface_shared_libraries"], + ), + ], + ), + ], + ) + + output_execpath_flags_feature = feature( + name = "output_execpath_flags", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = ["-o", "%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + ], + ) + + # Note that we also set --coverage for c++-link-nodeps-dynamic-library. The + # generated code contains references to gcov symbols, and the dynamic linker + # can't resolve them unless the library is linked against gcov. + coverage_feature = feature( + name = "coverage", + provides = ["profile"], + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = ([ + flag_group(flags = ctx.attr.coverage_compile_flags), + ] if ctx.attr.coverage_compile_flags else []), + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = ([ + flag_group(flags = ctx.attr.coverage_link_flags), + ] if ctx.attr.coverage_link_flags else []), + ), + ], + ) + + is_linux = ctx.attr.target_libc != "macosx" + + # TODO(#8303): Mac crosstool should also declare every feature. + if is_linux: + features = [ + dependency_file_feature, + random_seed_feature, + pic_feature, + per_object_debug_info_feature, + preprocessor_defines_feature, + includes_feature, + include_paths_feature, + fdo_instrument_feature, + cs_fdo_instrument_feature, + cs_fdo_optimize_feature, + fdo_prefetch_hints_feature, + autofdo_feature, + build_interface_libraries_feature, + dynamic_library_linker_tool_feature, + symbol_counts_feature, + shared_flag_feature, + linkstamps_feature, + output_execpath_flags_feature, + runtime_library_search_directories_feature, + library_search_directories_feature, + archiver_flags_feature, + force_pic_flags_feature, + fission_support_feature, + strip_debug_symbols_feature, + coverage_feature, + supports_pic_feature, + ] + ( + [ + supports_start_end_lib_feature, + ] if ctx.attr.supports_start_end_lib else [] + ) + [ + default_compile_flags_feature, + default_link_flags_feature, + libraries_to_link_feature, + user_link_flags_feature, + static_libgcc_feature, + fdo_optimize_feature, + supports_dynamic_linker_feature, + dbg_feature, + opt_feature, + user_compile_flags_feature, + sysroot_feature, + unfiltered_compile_flags_feature, + ] + else: + features = [ + supports_pic_feature, + ] + ( + [ + supports_start_end_lib_feature, + ] if ctx.attr.supports_start_end_lib else [] + ) + [ + coverage_feature, + default_compile_flags_feature, + default_link_flags_feature, + fdo_optimize_feature, + supports_dynamic_linker_feature, + dbg_feature, + opt_feature, + user_compile_flags_feature, + sysroot_feature, + unfiltered_compile_flags_feature, + ] + + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + cxx_builtin_include_directories = ctx.attr.cxx_builtin_include_directories, + toolchain_identifier = ctx.attr.toolchain_identifier, + host_system_name = ctx.attr.host_system_name, + target_system_name = ctx.attr.target_system_name, + target_cpu = ctx.attr.cpu, + target_libc = ctx.attr.target_libc, + compiler = ctx.attr.compiler, + abi_version = ctx.attr.abi_version, + abi_libc_version = ctx.attr.abi_libc_version, + tool_paths = tool_paths, + ) + +cc_toolchain_config = rule( + implementation = _impl, + attrs = { + "cpu": attr.string(mandatory = True), + "compiler": attr.string(mandatory = True), + "toolchain_identifier": attr.string(mandatory = True), + "host_system_name": attr.string(mandatory = True), + "target_system_name": attr.string(mandatory = True), + "target_libc": attr.string(mandatory = True), + "abi_version": attr.string(mandatory = True), + "abi_libc_version": attr.string(mandatory = True), + "cxx_builtin_include_directories": attr.string_list(), + "tool_paths": attr.string_dict(), + "compile_flags": attr.string_list(), + "dbg_compile_flags": attr.string_list(), + "opt_compile_flags": attr.string_list(), + "cxx_flags": attr.string_list(), + "link_flags": attr.string_list(), + "link_libs": attr.string_list(), + "opt_link_flags": attr.string_list(), + "unfiltered_compile_flags": attr.string_list(), + "coverage_compile_flags": attr.string_list(), + "coverage_link_flags": attr.string_list(), + "supports_start_end_lib": attr.bool(), + }, + provides = [CcToolchainConfigInfo], +) diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_wrapper.sh b/bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_wrapper.sh new file mode 100755 index 000000000000..b7ff6355883c --- /dev/null +++ b/bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_wrapper.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2015 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Ship the environment to the C++ action +# +set -eu + +# Set-up the environment + + +# Call the C++ compiler +/usr/lib/llvm-8/bin/clang "$@" diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/config/BUILD b/bazel/toolchains/configs/clang/bazel_0.28.0/config/BUILD new file mode 100644 index 000000000000..ac2d4d5f1c55 --- /dev/null +++ b/bazel/toolchains/configs/clang/bazel_0.28.0/config/BUILD @@ -0,0 +1,53 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated by an rbe_autoconfig repository rule +# and should not be modified directly. +# See @bazel_toolchains//rules:rbe_repo.bzl + +package(default_visibility = ["//visibility:public"]) + +toolchain( + name = "cc-toolchain", + exec_compatible_with = [ + "@bazel_tools//platforms:x86_64", + "@bazel_tools//platforms:linux", + "@bazel_tools//tools/cpp:clang", + ], + target_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + toolchain = "//bazel/toolchains/configs/clang/bazel_0.28.0/cc:cc-compiler-k8", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +platform( + name = "platform", + constraint_values = [ + "@bazel_tools//platforms:x86_64", + "@bazel_tools//platforms:linux", + "@bazel_tools//tools/cpp:clang", + ], + remote_execution_properties = """ + properties: { + name: "container-image" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f" + } + properties { + name: "OSFamily" + value: "Linux" + } + """, +) diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/java/BUILD b/bazel/toolchains/configs/clang/bazel_0.28.0/java/BUILD new file mode 100644 index 000000000000..7c273a5b0e49 --- /dev/null +++ b/bazel/toolchains/configs/clang/bazel_0.28.0/java/BUILD @@ -0,0 +1,25 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated by an rbe_autoconfig repository rule +# and should not be modified directly. +# See @bazel_toolchains//rules:rbe_repo.bzl + +package(default_visibility = ["//visibility:public"]) + +java_runtime( + name = "jdk", + srcs = [], + java_home = "/usr/lib/jvm/java-8-openjdk-amd64", +) diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/BUILD b/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/BUILD new file mode 100755 index 000000000000..eb9ab72263f1 --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/BUILD @@ -0,0 +1,148 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes. + +package(default_visibility = ["//visibility:public"]) + +load(":cc_toolchain_config.bzl", "cc_toolchain_config") +load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") + +licenses(["notice"]) # Apache 2.0 + +cc_library( + name = "malloc", +) + +filegroup( + name = "empty", + srcs = [], +) + +filegroup( + name = "cc_wrapper", + srcs = ["cc_wrapper.sh"], +) + +filegroup( + name = "compiler_deps", + srcs = glob(["extra_tools/**"], allow_empty = True) + [":empty"], +) + +# This is the entry point for --crosstool_top. Toolchains are found +# by lopping off the name of --crosstool_top and searching for +# the "${CPU}" entry in the toolchains attribute. +cc_toolchain_suite( + name = "toolchain", + toolchains = { + "k8|gcc": ":cc-compiler-k8", + "k8": ":cc-compiler-k8", + "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a", + "armeabi-v7a": ":cc-compiler-armeabi-v7a", + }, +) + +cc_toolchain( + name = "cc-compiler-k8", + toolchain_identifier = "local", + toolchain_config = ":local", + all_files = ":compiler_deps", + ar_files = ":compiler_deps", + as_files = ":compiler_deps", + compiler_files = ":compiler_deps", + dwp_files = ":empty", + linker_files = ":compiler_deps", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 1, +) + +cc_toolchain_config( + name = "local", + cpu = "k8", + compiler = "gcc", + toolchain_identifier = "local", + host_system_name = "local", + target_system_name = "local", + target_libc = "local", + abi_version = "local", + abi_libc_version = "local", + cxx_builtin_include_directories = ["/usr/lib/gcc/x86_64-linux-gnu/7/include", + "/usr/local/include", + "/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed", + "/usr/include/x86_64-linux-gnu", + "/usr/include", + "/usr/include/c++/7", + "/usr/include/x86_64-linux-gnu/c++/7", + "/usr/include/c++/7/backward"], + tool_paths = {"ar": "/usr/bin/ar", + "ld": "/usr/bin/ld", + "cpp": "/usr/bin/cpp", + "gcc": "/usr/bin/gcc", + "dwp": "/usr/bin/dwp", + "gcov": "/usr/bin/gcov", + "nm": "/usr/bin/nm", + "objcopy": "/usr/bin/objcopy", + "objdump": "/usr/bin/objdump", + "strip": "/usr/bin/strip"}, + compile_flags = ["-U_FORTIFY_SOURCE", + "-fstack-protector", + "-Wall", + "-Wunused-but-set-parameter", + "-Wno-free-nonheap-object", + "-fno-omit-frame-pointer"], + opt_compile_flags = ["-g0", + "-O2", + "-D_FORTIFY_SOURCE=1", + "-DNDEBUG", + "-ffunction-sections", + "-fdata-sections"], + dbg_compile_flags = ["-g"], + cxx_flags = ["-std=c++0x"], + link_flags = ["-fuse-ld=gold", + "-Wl,-no-as-needed", + "-Wl,-z,relro,-z,now", + "-B/usr/bin", + "-pass-exit-codes", + "-lm", + "-static-libgcc"], + link_libs = ["-l:libstdc++.a"], + opt_link_flags = ["-Wl,--gc-sections"], + unfiltered_compile_flags = ["-fno-canonical-system-headers", + "-Wno-builtin-macro-redefined", + "-D__DATE__=\"redacted\"", + "-D__TIMESTAMP__=\"redacted\"", + "-D__TIME__=\"redacted\""], + coverage_compile_flags = ["--coverage"], + coverage_link_flags = ["--coverage"], + supports_start_end_lib = True, +) + +# Android tooling requires a default toolchain for the armeabi-v7a cpu. +cc_toolchain( + name = "cc-compiler-armeabi-v7a", + toolchain_identifier = "stub_armeabi-v7a", + toolchain_config = ":stub_armeabi-v7a", + all_files = ":empty", + ar_files = ":empty", + as_files = ":empty", + compiler_files = ":empty", + dwp_files = ":empty", + linker_files = ":empty", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 1, +) + +armeabi_cc_toolchain_config(name = "stub_armeabi-v7a") diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl new file mode 100755 index 000000000000..94e0720bf6c9 --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl @@ -0,0 +1,82 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Starlark cc_toolchain configuration rule""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "feature", + "tool_path", +) + +def _impl(ctx): + toolchain_identifier = "stub_armeabi-v7a" + host_system_name = "armeabi-v7a" + target_system_name = "armeabi-v7a" + target_cpu = "armeabi-v7a" + target_libc = "armeabi-v7a" + compiler = "compiler" + abi_version = "armeabi-v7a" + abi_libc_version = "armeabi-v7a" + cc_target_os = None + builtin_sysroot = None + action_configs = [] + + supports_pic_feature = feature(name = "supports_pic", enabled = True) + supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) + features = [supports_dynamic_linker_feature, supports_pic_feature] + + cxx_builtin_include_directories = [] + artifact_name_patterns = [] + make_variables = [] + + tool_paths = [ + tool_path(name = "ar", path = "/bin/false"), + tool_path(name = "compat-ld", path = "/bin/false"), + tool_path(name = "cpp", path = "/bin/false"), + tool_path(name = "dwp", path = "/bin/false"), + tool_path(name = "gcc", path = "/bin/false"), + tool_path(name = "gcov", path = "/bin/false"), + tool_path(name = "ld", path = "/bin/false"), + tool_path(name = "nm", path = "/bin/false"), + tool_path(name = "objcopy", path = "/bin/false"), + tool_path(name = "objdump", path = "/bin/false"), + tool_path(name = "strip", path = "/bin/false"), + ] + + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + artifact_name_patterns = artifact_name_patterns, + cxx_builtin_include_directories = cxx_builtin_include_directories, + toolchain_identifier = toolchain_identifier, + host_system_name = host_system_name, + target_system_name = target_system_name, + target_cpu = target_cpu, + target_libc = target_libc, + compiler = compiler, + abi_version = abi_version, + abi_libc_version = abi_libc_version, + tool_paths = tool_paths, + make_variables = make_variables, + builtin_sysroot = builtin_sysroot, + cc_target_os = cc_target_os, + ) + +armeabi_cc_toolchain_config = rule( + implementation = _impl, + attrs = {}, + provides = [CcToolchainConfigInfo], +) diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_toolchain_config.bzl new file mode 100755 index 000000000000..f2b12d962963 --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_toolchain_config.bzl @@ -0,0 +1,1202 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Starlark cc_toolchain configuration rule""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "feature", + "feature_set", + "flag_group", + "flag_set", + "tool_path", + "variable_with_value", + "with_feature_set", +) +load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") + +all_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.lto_backend, +] + +all_cpp_compile_actions = [ + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, +] + +preprocessor_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, +] + +codegen_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, +] + +all_link_actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, +] + +def _impl(ctx): + tool_paths = [ + tool_path(name = name, path = path) + for name, path in ctx.attr.tool_paths.items() + ] + action_configs = [] + + supports_pic_feature = feature( + name = "supports_pic", + enabled = True, + ) + supports_start_end_lib_feature = feature( + name = "supports_start_end_lib", + enabled = True, + ) + + default_compile_flags_feature = feature( + name = "default_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.compile_flags, + ), + ] if ctx.attr.compile_flags else []), + ), + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.dbg_compile_flags, + ), + ] if ctx.attr.dbg_compile_flags else []), + with_features = [with_feature_set(features = ["dbg"])], + ), + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.opt_compile_flags, + ), + ] if ctx.attr.opt_compile_flags else []), + with_features = [with_feature_set(features = ["opt"])], + ), + flag_set( + actions = [ + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.cxx_flags, + ), + ] if ctx.attr.cxx_flags else []), + ), + ], + ) + + default_link_flags_feature = feature( + name = "default_link_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = ([ + flag_group( + flags = ctx.attr.link_flags, + ), + ] if ctx.attr.link_flags else []), + ), + flag_set( + actions = all_link_actions, + flag_groups = ([ + flag_group( + flags = ctx.attr.opt_link_flags, + ), + ] if ctx.attr.opt_link_flags else []), + with_features = [with_feature_set(features = ["opt"])], + ), + ], + ) + + dbg_feature = feature(name = "dbg") + + opt_feature = feature(name = "opt") + + sysroot_feature = feature( + name = "sysroot", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = ["--sysroot=%{sysroot}"], + expand_if_available = "sysroot", + ), + ], + ), + ], + ) + + fdo_optimize_feature = feature( + name = "fdo_optimize", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-use=%{fdo_profile_path}", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) + + user_compile_flags_feature = feature( + name = "user_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["%{user_compile_flags}"], + iterate_over = "user_compile_flags", + expand_if_available = "user_compile_flags", + ), + ], + ), + ], + ) + + unfiltered_compile_flags_feature = feature( + name = "unfiltered_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.unfiltered_compile_flags, + ), + ] if ctx.attr.unfiltered_compile_flags else []), + ), + ], + ) + + library_search_directories_feature = feature( + name = "library_search_directories", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-L%{library_search_directories}"], + iterate_over = "library_search_directories", + expand_if_available = "library_search_directories", + ), + ], + ), + ], + ) + + static_libgcc_feature = feature( + name = "static_libgcc", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ], + flag_groups = [flag_group(flags = ["-static-libgcc"])], + with_features = [ + with_feature_set(features = ["static_link_cpp_runtimes"]), + ], + ), + ], + ) + + pic_feature = feature( + name = "pic", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = [ + flag_group(flags = ["-fPIC"], expand_if_available = "pic"), + ], + ), + ], + ) + + per_object_debug_info_feature = feature( + name = "per_object_debug_info", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ], + flag_groups = [ + flag_group( + flags = ["-gsplit-dwarf"], + expand_if_available = "per_object_debug_info_file", + ), + ], + ), + ], + ) + + preprocessor_defines_feature = feature( + name = "preprocessor_defines", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-D%{preprocessor_defines}"], + iterate_over = "preprocessor_defines", + ), + ], + ), + ], + ) + + cs_fdo_optimize_feature = feature( + name = "cs_fdo_optimize", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.lto_backend], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-use=%{fdo_profile_path}", + "-Xclang-only=-Wno-profile-instr-unprofiled", + "-Xclang-only=-Wno-profile-instr-out-of-date", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["csprofile"], + ) + + autofdo_feature = feature( + name = "autofdo", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-fauto-profile=%{fdo_profile_path}", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + runtime_library_search_directories_feature = feature( + name = "runtime_library_search_directories", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "runtime_library_search_directories", + flag_groups = [ + flag_group( + flags = [ + "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_true = "is_cc_test", + ), + flag_group( + flags = [ + "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_false = "is_cc_test", + ), + ], + expand_if_available = + "runtime_library_search_directories", + ), + ], + with_features = [ + with_feature_set(features = ["static_link_cpp_runtimes"]), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "runtime_library_search_directories", + flag_groups = [ + flag_group( + flags = [ + "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", + ], + ), + ], + expand_if_available = + "runtime_library_search_directories", + ), + ], + with_features = [ + with_feature_set( + not_features = ["static_link_cpp_runtimes"], + ), + ], + ), + ], + ) + + fission_support_feature = feature( + name = "fission_support", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-Wl,--gdb-index"], + expand_if_available = "is_using_fission", + ), + ], + ), + ], + ) + + shared_flag_feature = feature( + name = "shared_flag", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [flag_group(flags = ["-shared"])], + ), + ], + ) + + random_seed_feature = feature( + name = "random_seed", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = [ + flag_group( + flags = ["-frandom-seed=%{output_file}"], + expand_if_available = "output_file", + ), + ], + ), + ], + ) + + includes_feature = feature( + name = "includes", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = ["-include", "%{includes}"], + iterate_over = "includes", + expand_if_available = "includes", + ), + ], + ), + ], + ) + + fdo_instrument_feature = feature( + name = "fdo_instrument", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-generate=%{fdo_instrument_path}", + "-fno-data-sections", + ], + expand_if_available = "fdo_instrument_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + cs_fdo_instrument_feature = feature( + name = "cs_fdo_instrument", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.lto_backend, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = [ + "-fcs-profile-generate=%{cs_fdo_instrument_path}", + ], + expand_if_available = "cs_fdo_instrument_path", + ), + ], + ), + ], + provides = ["csprofile"], + ) + + include_paths_feature = feature( + name = "include_paths", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = ["-iquote", "%{quote_include_paths}"], + iterate_over = "quote_include_paths", + ), + flag_group( + flags = ["-I%{include_paths}"], + iterate_over = "include_paths", + ), + flag_group( + flags = ["-isystem", "%{system_include_paths}"], + iterate_over = "system_include_paths", + ), + ], + ), + ], + ) + + symbol_counts_feature = feature( + name = "symbol_counts", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = [ + "-Wl,--print-symbol-counts=%{symbol_counts_output}", + ], + expand_if_available = "symbol_counts_output", + ), + ], + ), + ], + ) + + llvm_coverage_map_format_feature = feature( + name = "llvm_coverage_map_format", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-instr-generate", + "-fcoverage-mapping", + ], + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + "objc-executable", + "objc++-executable", + ], + flag_groups = [ + flag_group(flags = ["-fprofile-instr-generate"]), + ], + ), + ], + requires = [feature_set(features = ["coverage"])], + provides = ["profile"], + ) + + strip_debug_symbols_feature = feature( + name = "strip_debug_symbols", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-Wl,-S"], + expand_if_available = "strip_debug_symbols", + ), + ], + ), + ], + ) + + build_interface_libraries_feature = feature( + name = "build_interface_libraries", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = [ + "%{generate_interface_library}", + "%{interface_library_builder_path}", + "%{interface_library_input_path}", + "%{interface_library_output_path}", + ], + expand_if_available = "generate_interface_library", + ), + ], + with_features = [ + with_feature_set( + features = ["supports_interface_shared_libraries"], + ), + ], + ), + ], + ) + + libraries_to_link_feature = feature( + name = "libraries_to_link", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["-Wl,--start-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["-Wl,-whole-archive"], + expand_if_true = + "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "interface_library", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "static_library", + ), + ), + flag_group( + flags = ["-l%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "dynamic_library", + ), + ), + flag_group( + flags = ["-l:%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "versioned_dynamic_library", + ), + ), + flag_group( + flags = ["-Wl,-no-whole-archive"], + expand_if_true = "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["-Wl,--end-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + flag_group( + flags = ["-Wl,@%{thinlto_param_file}"], + expand_if_true = "thinlto_param_file", + ), + ], + ), + ], + ) + + user_link_flags_feature = feature( + name = "user_link_flags", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{user_link_flags}"], + iterate_over = "user_link_flags", + expand_if_available = "user_link_flags", + ), + ] + ([flag_group(flags = ctx.attr.link_libs)] if ctx.attr.link_libs else []), + ), + ], + ) + + fdo_prefetch_hints_feature = feature( + name = "fdo_prefetch_hints", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.lto_backend, + ], + flag_groups = [ + flag_group( + flags = [ + "-Xclang-only=-mllvm", + "-Xclang-only=-prefetch-hints-file=%{fdo_prefetch_hints_path}", + ], + expand_if_available = "fdo_prefetch_hints_path", + ), + ], + ), + ], + ) + + linkstamps_feature = feature( + name = "linkstamps", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{linkstamp_paths}"], + iterate_over = "linkstamp_paths", + expand_if_available = "linkstamp_paths", + ), + ], + ), + ], + ) + + gcc_coverage_map_format_feature = feature( + name = "gcc_coverage_map_format", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + "objc-executable", + "objc++-executable", + ], + flag_groups = [ + flag_group( + flags = ["-fprofile-arcs", "-ftest-coverage"], + expand_if_available = "gcov_gcno_file", + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [flag_group(flags = ["--coverage"])], + ), + ], + requires = [feature_set(features = ["coverage"])], + provides = ["profile"], + ) + + archiver_flags_feature = feature( + name = "archiver_flags", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group(flags = ["rcsD"]), + flag_group( + flags = ["%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + flag_set( + actions = [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + ], + ), + ], + ) + + force_pic_flags_feature = feature( + name = "force_pic_flags", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.cpp_link_executable], + flag_groups = [ + flag_group( + flags = ["-pie"], + expand_if_available = "force_pic", + ), + ], + ), + ], + ) + + dependency_file_feature = feature( + name = "dependency_file", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-MD", "-MF", "%{dependency_file}"], + expand_if_available = "dependency_file", + ), + ], + ), + ], + ) + + dynamic_library_linker_tool_feature = feature( + name = "dynamic_library_linker_tool", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = [" + cppLinkDynamicLibraryToolPath + "], + expand_if_available = "generate_interface_library", + ), + ], + with_features = [ + with_feature_set( + features = ["supports_interface_shared_libraries"], + ), + ], + ), + ], + ) + + output_execpath_flags_feature = feature( + name = "output_execpath_flags", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = ["-o", "%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + ], + ) + + # Note that we also set --coverage for c++-link-nodeps-dynamic-library. The + # generated code contains references to gcov symbols, and the dynamic linker + # can't resolve them unless the library is linked against gcov. + coverage_feature = feature( + name = "coverage", + provides = ["profile"], + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = ([ + flag_group(flags = ctx.attr.coverage_compile_flags), + ] if ctx.attr.coverage_compile_flags else []), + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = ([ + flag_group(flags = ctx.attr.coverage_link_flags), + ] if ctx.attr.coverage_link_flags else []), + ), + ], + ) + + is_linux = ctx.attr.target_libc != "macosx" + + # TODO(#8303): Mac crosstool should also declare every feature. + if is_linux: + features = [ + dependency_file_feature, + random_seed_feature, + pic_feature, + per_object_debug_info_feature, + preprocessor_defines_feature, + includes_feature, + include_paths_feature, + fdo_instrument_feature, + cs_fdo_instrument_feature, + cs_fdo_optimize_feature, + fdo_prefetch_hints_feature, + autofdo_feature, + build_interface_libraries_feature, + dynamic_library_linker_tool_feature, + symbol_counts_feature, + shared_flag_feature, + linkstamps_feature, + output_execpath_flags_feature, + runtime_library_search_directories_feature, + library_search_directories_feature, + archiver_flags_feature, + force_pic_flags_feature, + fission_support_feature, + strip_debug_symbols_feature, + coverage_feature, + supports_pic_feature, + ] + ( + [ + supports_start_end_lib_feature, + ] if ctx.attr.supports_start_end_lib else [] + ) + [ + default_compile_flags_feature, + default_link_flags_feature, + libraries_to_link_feature, + user_link_flags_feature, + static_libgcc_feature, + fdo_optimize_feature, + supports_dynamic_linker_feature, + dbg_feature, + opt_feature, + user_compile_flags_feature, + sysroot_feature, + unfiltered_compile_flags_feature, + ] + else: + features = [ + supports_pic_feature, + ] + ( + [ + supports_start_end_lib_feature, + ] if ctx.attr.supports_start_end_lib else [] + ) + [ + coverage_feature, + default_compile_flags_feature, + default_link_flags_feature, + fdo_optimize_feature, + supports_dynamic_linker_feature, + dbg_feature, + opt_feature, + user_compile_flags_feature, + sysroot_feature, + unfiltered_compile_flags_feature, + ] + + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + cxx_builtin_include_directories = ctx.attr.cxx_builtin_include_directories, + toolchain_identifier = ctx.attr.toolchain_identifier, + host_system_name = ctx.attr.host_system_name, + target_system_name = ctx.attr.target_system_name, + target_cpu = ctx.attr.cpu, + target_libc = ctx.attr.target_libc, + compiler = ctx.attr.compiler, + abi_version = ctx.attr.abi_version, + abi_libc_version = ctx.attr.abi_libc_version, + tool_paths = tool_paths, + ) + +cc_toolchain_config = rule( + implementation = _impl, + attrs = { + "cpu": attr.string(mandatory = True), + "compiler": attr.string(mandatory = True), + "toolchain_identifier": attr.string(mandatory = True), + "host_system_name": attr.string(mandatory = True), + "target_system_name": attr.string(mandatory = True), + "target_libc": attr.string(mandatory = True), + "abi_version": attr.string(mandatory = True), + "abi_libc_version": attr.string(mandatory = True), + "cxx_builtin_include_directories": attr.string_list(), + "tool_paths": attr.string_dict(), + "compile_flags": attr.string_list(), + "dbg_compile_flags": attr.string_list(), + "opt_compile_flags": attr.string_list(), + "cxx_flags": attr.string_list(), + "link_flags": attr.string_list(), + "link_libs": attr.string_list(), + "opt_link_flags": attr.string_list(), + "unfiltered_compile_flags": attr.string_list(), + "coverage_compile_flags": attr.string_list(), + "coverage_link_flags": attr.string_list(), + "supports_start_end_lib": attr.bool(), + }, + provides = [CcToolchainConfigInfo], +) diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_wrapper.sh b/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_wrapper.sh new file mode 100755 index 000000000000..f246528abf2e --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_wrapper.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright 2015 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Ship the environment to the C++ action +# +set -eu + +# Set-up the environment + + +# Call the C++ compiler +/usr/bin/gcc "$@" diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/config/BUILD b/bazel/toolchains/configs/gcc/bazel_0.28.0/config/BUILD new file mode 100644 index 000000000000..2d431bd2ce03 --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.28.0/config/BUILD @@ -0,0 +1,53 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated by an rbe_autoconfig repository rule +# and should not be modified directly. +# See @bazel_toolchains//rules:rbe_repo.bzl + +package(default_visibility = ["//visibility:public"]) + +toolchain( + name = "cc-toolchain", + exec_compatible_with = [ + "@bazel_tools//platforms:x86_64", + "@bazel_tools//platforms:linux", + "@bazel_tools//tools/cpp:clang", + ], + target_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + toolchain = "//bazel/toolchains/configs/gcc/bazel_0.28.0/cc:cc-compiler-k8", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +platform( + name = "platform", + constraint_values = [ + "@bazel_tools//platforms:x86_64", + "@bazel_tools//platforms:linux", + "@bazel_tools//tools/cpp:clang", + ], + remote_execution_properties = """ + properties: { + name: "container-image" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f" + } + properties { + name: "OSFamily" + value: "Linux" + } + """, +) diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/java/BUILD b/bazel/toolchains/configs/gcc/bazel_0.28.0/java/BUILD new file mode 100644 index 000000000000..7c273a5b0e49 --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.28.0/java/BUILD @@ -0,0 +1,25 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated by an rbe_autoconfig repository rule +# and should not be modified directly. +# See @bazel_toolchains//rules:rbe_repo.bzl + +package(default_visibility = ["//visibility:public"]) + +java_runtime( + name = "jdk", + srcs = [], + java_home = "/usr/lib/jvm/java-8-openjdk-amd64", +) diff --git a/bazel/toolchains/configs/versions.bzl b/bazel/toolchains/configs/versions.bzl new file mode 100644 index 000000000000..6704b14fa542 --- /dev/null +++ b/bazel/toolchains/configs/versions.bzl @@ -0,0 +1,17 @@ +# Generated file, do not modify by hand +# Generated by 'rbe_ubuntu_gcc_gen' rbe_autoconfig rule +"""Definitions to be used in rbe_repo attr of an rbe_autoconf rule """ +toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", name = "clang") +toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", name = "gcc") +_TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1] +_BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.0": ["clang", "gcc"]} +LATEST = "sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f" +_CONTAINER_TO_CONFIG_SPEC_NAMES = {"sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f": ["clang", "gcc"]} +_DEFAULT_TOOLCHAIN_CONFIG_SPEC = toolchain_config_spec0 +TOOLCHAIN_CONFIG_AUTOGEN_SPEC = struct( + bazel_to_config_spec_names_map = _BAZEL_TO_CONFIG_SPEC_NAMES, + container_to_config_spec_names_map = _CONTAINER_TO_CONFIG_SPEC_NAMES, + default_toolchain_config_spec = _DEFAULT_TOOLCHAIN_CONFIG_SPEC, + latest_container = LATEST, + toolchain_config_specs = _TOOLCHAIN_CONFIG_SPECS, + ) \ No newline at end of file diff --git a/bazel/toolchains/empty.bzl b/bazel/toolchains/empty.bzl new file mode 100644 index 000000000000..3fc95e435327 --- /dev/null +++ b/bazel/toolchains/empty.bzl @@ -0,0 +1,18 @@ +_BAZEL_TO_CONFIG_SPEC_NAMES = {} + +# sha256 digest of the latest version of the toolchain container. +LATEST = "" + +_CONTAINER_TO_CONFIG_SPEC_NAMES = {} + +_DEFAULT_TOOLCHAIN_CONFIG_SPEC = "" + +_TOOLCHAIN_CONFIG_SPECS = [] + +TOOLCHAIN_CONFIG_AUTOGEN_SPEC = struct( + bazel_to_config_spec_names_map = _BAZEL_TO_CONFIG_SPEC_NAMES, + container_to_config_spec_names_map = _CONTAINER_TO_CONFIG_SPEC_NAMES, + default_toolchain_config_spec = _DEFAULT_TOOLCHAIN_CONFIG_SPEC, + latest_container = LATEST, + toolchain_config_specs = _TOOLCHAIN_CONFIG_SPECS, +) diff --git a/bazel/toolchains/rbe_toolchains_config.bzl b/bazel/toolchains/rbe_toolchains_config.bzl new file mode 100644 index 000000000000..c1f32be5d78a --- /dev/null +++ b/bazel/toolchains/rbe_toolchains_config.bzl @@ -0,0 +1,80 @@ +load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") +load("@envoy//bazel/toolchains:configs/versions.bzl", _generated_toolchain_config_suite_autogen_spec = "TOOLCHAIN_CONFIG_AUTOGEN_SPEC") + +_ENVOY_BUILD_IMAGE_REGISTRY = "gcr.io" +_ENVOY_BUILD_IMAGE_REPOSITORY = "envoy-ci/envoy-build" +_ENVOY_BUILD_IMAGE_DIGEST = "sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f" +_ENVOY_BUILD_IMAGE_JAVA_HOME = "/usr/lib/jvm/java-8-openjdk-amd64" +_CONFIGS_OUTPUT_BASE = "bazel/toolchains/configs" + +_CLANG_ENV = { + "BAZEL_COMPILER": "clang", + "BAZEL_LINKLIBS": "-l%:libstdc++.a", + "BAZEL_LINKOPTS": "-lm:-static-libgcc", + "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", + "GCOV": "llvm-profdata", + "CC": "clang", + "CXX": "clang++", + "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", +} + +_GCC_ENV = { + "BAZEL_COMPILER": "gcc", + "BAZEL_LINKLIBS": "-l%:libstdc++.a", + "BAZEL_LINKOPTS": "-lm:-static-libgcc", + "CC": "gcc", + "CXX": "g++", + "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", +} + +_TOOLCHAIN_CONFIG_SUITE_SPEC = { + "container_registry": _ENVOY_BUILD_IMAGE_REGISTRY, + "container_repo": _ENVOY_BUILD_IMAGE_REPOSITORY, + "output_base": _CONFIGS_OUTPUT_BASE, + "repo_name": "envoy", + "toolchain_config_suite_autogen_spec": _generated_toolchain_config_suite_autogen_spec, +} + +def _rbe_toolchains_generator(): + rbe_autoconfig( + name = "rbe_ubuntu_clang_gen", + digest = _ENVOY_BUILD_IMAGE_DIGEST, + export_configs = True, + java_home = _ENVOY_BUILD_IMAGE_JAVA_HOME, + registry = _ENVOY_BUILD_IMAGE_REGISTRY, + repository = _ENVOY_BUILD_IMAGE_REPOSITORY, + env = _CLANG_ENV, + toolchain_config_spec_name = "clang", + toolchain_config_suite_spec = _TOOLCHAIN_CONFIG_SUITE_SPEC, + use_checked_in_confs = "False", + ) + + rbe_autoconfig( + name = "rbe_ubuntu_gcc_gen", + digest = _ENVOY_BUILD_IMAGE_DIGEST, + export_configs = True, + java_home = _ENVOY_BUILD_IMAGE_JAVA_HOME, + registry = _ENVOY_BUILD_IMAGE_REGISTRY, + repository = _ENVOY_BUILD_IMAGE_REPOSITORY, + env = _GCC_ENV, + toolchain_config_spec_name = "gcc", + toolchain_config_suite_spec = _TOOLCHAIN_CONFIG_SUITE_SPEC, + use_checked_in_confs = "False", + ) + +def _generated_rbe_toolchains(): + rbe_autoconfig( + name = "rbe_ubuntu_clang", + digest = _ENVOY_BUILD_IMAGE_DIGEST, + export_configs = True, + java_home = _ENVOY_BUILD_IMAGE_JAVA_HOME, + registry = _ENVOY_BUILD_IMAGE_REGISTRY, + repository = _ENVOY_BUILD_IMAGE_REPOSITORY, + toolchain_config_spec_name = "clang", + toolchain_config_suite_spec = _TOOLCHAIN_CONFIG_SUITE_SPEC, + use_checked_in_confs = "Force", + ) + +def rbe_toolchains_config(): + _rbe_toolchains_generator() + _generated_rbe_toolchains() diff --git a/bazel/toolchains/regenerate.sh b/bazel/toolchains/regenerate.sh new file mode 100755 index 000000000000..c71a0506d178 --- /dev/null +++ b/bazel/toolchains/regenerate.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +export RBE_AUTOCONF_ROOT=$(bazel info workspace) + +rm -rf "${RBE_AUTOCONF_ROOT}/bazel/toolchains/configs/*" +cp -vf "${RBE_AUTOCONF_ROOT}/bazel/toolchains/empty.bzl" "${RBE_AUTOCONF_ROOT}/bazel/toolchains/configs/versions.bzl" + +# Bazel query is the right command so bazel won't fail itself. +bazel query "@rbe_ubuntu_clang_gen//..." +bazel query "@rbe_ubuntu_gcc_gen//..." diff --git a/ci/WORKSPACE.filter.example b/ci/WORKSPACE.filter.example index 5013077c2468..a4d245957f5c 100644 --- a/ci/WORKSPACE.filter.example +++ b/ci/WORKSPACE.filter.example @@ -2,7 +2,7 @@ workspace(name = "envoy_filter_example") local_repository( name = "envoy", - path = "/source", + path = "{ENVOY_SRCDIR}", ) load("@envoy//bazel:api_repositories.bzl", "envoy_api_dependencies") diff --git a/ci/build_setup.sh b/ci/build_setup.sh index fd2a94f05a8b..d397349da3eb 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -56,8 +56,7 @@ fi # Not sandboxing, since non-privileged Docker can't do nested namespaces. export BAZEL_QUERY_OPTIONS="${BAZEL_OPTIONS}" -export BAZEL_BUILD_OPTIONS="--strategy=Genrule=standalone --spawn_strategy=standalone \ - --verbose_failures ${BAZEL_OPTIONS} --action_env=HOME --action_env=PYTHONUSERBASE \ +export BAZEL_BUILD_OPTIONS="--verbose_failures ${BAZEL_OPTIONS} --action_env=HOME --action_env=PYTHONUSERBASE \ --local_cpu_resources=${NUM_CPUS} --show_task_finish --experimental_generate_json_trace_profile \ --test_env=HOME --test_env=PYTHONUSERBASE --cache_test_results=no --test_output=all \ ${BAZEL_BUILD_EXTRA_OPTIONS} ${BAZEL_EXTRA_TEST_OPTIONS}" @@ -73,7 +72,7 @@ if [ "$1" != "-nofetch" ]; then # This is the hash on https://github.com/envoyproxy/envoy-filter-example.git we pin to. (cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" && git fetch origin && git checkout -f dcd3374baa9365ab7ab505018232994d6c8a8d81) - cp -f "${ENVOY_SRCDIR}"/ci/WORKSPACE.filter.example "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/WORKSPACE + sed -e "s|{ENVOY_SRCDIR}|${ENVOY_SRCDIR}|" "${ENVOY_SRCDIR}"/ci/WORKSPACE.filter.example > "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/WORKSPACE fi # Also setup some space for building Envoy standalone. diff --git a/ci/do_ci.sh b/ci/do_ci.sh index daf1c2a15e56..030b413e30a8 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -153,8 +153,8 @@ elif [[ "$CI_TARGET" == "bazel.asan" ]]; then rm -rf "${TAP_TMP}" mkdir -p "${TAP_TMP}" bazel_with_collection test ${BAZEL_BUILD_OPTIONS} -c dbg --config=clang-asan \ - //test/extensions/transport_sockets/tls/integration:ssl_integration_test \ - --test_env=TAP_PATH="${TAP_TMP}/tap" + --strategy=TestRunner=local --test_env=TAP_PATH="${TAP_TMP}/tap" \ + //test/extensions/transport_sockets/tls/integration:ssl_integration_test # Verify that some pb_text files have been created. We can't check for pcap, # since tcpdump is not available in general due to CircleCI lack of support # for privileged Docker executors. diff --git a/ci/setup_cache.sh b/ci/setup_cache.sh index 3e10133e717d..699961bbb082 100755 --- a/ci/setup_cache.sh +++ b/ci/setup_cache.sh @@ -33,7 +33,7 @@ elif [[ ! -z "${BAZEL_REMOTE_CACHE}" ]]; then --remote_cache=${BAZEL_REMOTE_CACHE} \ --remote_instance_name=${BAZEL_REMOTE_INSTANCE} \ --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_FILE} \ - --tls_enabled=true --auth_enabled=true" + --auth_enabled=true" echo "Set up bazel remote read/write cache at ${BAZEL_REMOTE_CACHE} instance: ${BAZEL_REMOTE_INSTANCE}." else echo "No remote cache bucket is set, skipping setup remote cache." diff --git a/tools/check_format.py b/tools/check_format.py index d05194f627d2..4e9db37bf505 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -16,7 +16,8 @@ EXCLUDED_PREFIXES = ("./generated/", "./thirdparty/", "./build", "./.git/", "./bazel-", "./.cache", "./source/extensions/extensions_build_config.bzl", - "./tools/testdata/check_format/", "./tools/pyformat/") + "./bazel/toolchains/configs/", "./tools/testdata/check_format/", + "./tools/pyformat/") SUFFIXES = ("BUILD", "WORKSPACE", ".bzl", ".cc", ".h", ".java", ".m", ".md", ".mm", ".proto", ".rst") DOCS_SUFFIX = (".md", ".rst") diff --git a/tools/envoy_build_fixer.py b/tools/envoy_build_fixer.py index 4114ce552f3b..fa4f3c14e251 100755 --- a/tools/envoy_build_fixer.py +++ b/tools/envoy_build_fixer.py @@ -26,7 +26,7 @@ def FixBuild(path): if line != '\n': outlines.append('\n') first = False - if path.startswith('./bazel/external/'): + if path.startswith('./bazel/external/') or path.startswith('./bazel/toolchains/'): outlines.append(line) continue if line.startswith('package(') and not path.endswith('bazel/BUILD') and not path.endswith( From 1278529f5fc1b01e1d33d5097fc40da5111062b6 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Thu, 18 Jul 2019 20:31:05 -0700 Subject: [PATCH 201/542] clang-tidy: modernize-use-equals-default (#7638) Mass translation to = default; done using clang-tidy's -fix flag and enable as a clang-tidy error. https://clang.llvm.org/extra/clang-tidy/checks/modernize-use-equals-default.html Risk Level: low Testing: existing Signed-off-by: Derek Argueta --- .clang-tidy | 1 + source/common/access_log/access_log_impl.h | 2 +- .../common/config/delta_subscription_state.h | 2 +- source/common/grpc/codec.cc | 2 +- source/common/grpc/typed_async_client.h | 4 +- source/common/network/cidr_range.cc | 8 +- source/common/network/io_socket_error_impl.h | 2 +- source/common/protobuf/utility.h | 2 +- source/common/router/config_impl.h | 2 +- source/common/router/header_parser.h | 2 +- source/common/stats/tag_producer_impl.h | 2 +- source/common/upstream/load_balancer_impl.h | 2 +- .../original_src/original_src_socket_option.h | 2 +- .../http/dynamo/dynamo_request_parser.h | 2 +- .../http/ip_tagging/ip_tagging_filter.cc | 2 +- .../filters/http/squash/squash_filter.cc | 2 +- .../network/dubbo_proxy/filters/filter.h | 2 +- .../network/dubbo_proxy/message_impl.h | 4 +- .../dubbo_proxy/router/route_matcher.cc | 4 +- .../network/dubbo_proxy/router/router_impl.cc | 2 +- .../filters/network/ext_authz/ext_authz.h | 2 +- .../filters/network/kafka/serialization.h | 2 +- .../filters/network/mongo_proxy/bson_impl.h | 2 +- .../filters/network/rbac/rbac_filter.h | 2 +- .../thrift_proxy/compact_protocol_impl.h | 2 +- .../network/thrift_proxy/conn_manager.cc | 2 +- .../thrift_proxy/framed_transport_impl.h | 2 +- .../filters/network/thrift_proxy/metadata.h | 2 +- .../thrift_proxy/router/router_impl.cc | 2 +- .../network/thrift_proxy/thrift_object_impl.h | 4 +- .../filters/network/thrift_proxy/tracing.h | 8 +- .../thrift_proxy/twitter_protocol_impl.cc | 2 +- .../thrift_proxy/unframed_transport_impl.h | 2 +- .../extensions/tracers/zipkin/span_buffer.h | 2 +- .../tracers/zipkin/span_context_extractor.cc | 2 +- source/server/http/admin.h | 4 +- .../compressor/zlib_compressor_impl_test.cc | 2 +- .../config/delta_subscription_impl_test.cc | 2 +- test/common/network/address_impl_test.cc | 2 +- test/common/network/connection_impl_test.cc | 2 +- test/common/network/dns_impl_test.cc | 2 +- test/common/secret/sds_api_test.cc | 4 +- test/common/tcp/conn_pool_test.cc | 2 +- test/common/upstream/eds_test.cc | 2 +- .../upstream/health_checker_impl_test.cc | 2 +- .../filters/http/common/aws/mocks.cc | 8 +- .../filters/http/common/aws/mocks.h | 2 +- .../filters/network/dubbo_proxy/mocks.cc | 10 +-- .../kafka/kafka_request_parser_test.cc | 2 +- .../kafka/kafka_response_parser_test.cc | 2 +- .../filters/network/redis_proxy/mocks.cc | 22 ++--- .../header_transport_impl_test.cc | 4 +- .../filters/network/thrift_proxy/mocks.cc | 42 +++++----- .../quiche/platform/quic_epoll_clock.h | 2 +- .../quiche/platform/quic_platform_test.cc | 2 +- .../fixed_heap/fixed_heap_monitor_test.cc | 2 +- .../noop_transport_socket_callbacks_test.cc | 2 +- .../filters/process_context_filter.h | 2 +- test/integration/utility.cc | 2 +- test/mocks/access_log/mocks.cc | 14 ++-- test/mocks/api/mocks.cc | 6 +- test/mocks/buffer/mocks.cc | 4 +- test/mocks/common.cc | 8 +- test/mocks/config/mocks.h | 2 +- test/mocks/event/mocks.cc | 10 +-- test/mocks/filesystem/mocks.cc | 2 +- test/mocks/grpc/mocks.cc | 16 ++-- test/mocks/http/stream.cc | 2 +- test/mocks/http/stream_decoder.cc | 4 +- test/mocks/http/stream_encoder.cc | 2 +- test/mocks/local_info/mocks.cc | 2 +- test/mocks/network/connection.cc | 10 +-- test/mocks/network/mocks.cc | 84 +++++++++---------- test/mocks/network/mocks.h | 4 +- test/mocks/protobuf/mocks.cc | 4 +- test/mocks/router/mocks.cc | 48 +++++------ test/mocks/runtime/mocks.cc | 10 +-- test/mocks/secret/mocks.cc | 6 +- test/mocks/ssl/mocks.cc | 20 ++--- test/mocks/stream_info/mocks.cc | 2 +- test/mocks/tcp/mocks.cc | 12 +-- test/mocks/tracing/mocks.cc | 14 ++-- test/mocks/upstream/cluster_info.cc | 6 +- test/mocks/upstream/host.cc | 18 ++-- test/test_common/logging.cc | 2 +- 85 files changed, 264 insertions(+), 265 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index ffdc2d0dd973..981246fbb172 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -23,6 +23,7 @@ WarningsAsErrors: 'abseil-duration-*, modernize-make-shared, modernize-make-unique, modernize-return-braced-init-list, + modernize-use-equals-default, modernize-use-using, performance-faster-string-find, performance-for-range-copy, diff --git a/source/common/access_log/access_log_impl.h b/source/common/access_log/access_log_impl.h index 3635957aa95f..0941602a30b0 100644 --- a/source/common/access_log/access_log_impl.h +++ b/source/common/access_log/access_log_impl.h @@ -121,7 +121,7 @@ class OrFilter : public OperatorFilter { */ class NotHealthCheckFilter : public Filter { public: - NotHealthCheckFilter() {} + NotHealthCheckFilter() = default; // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::HeaderMap& request_headers, diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 5fbb6f79f5a1..f21e0b895b9e 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -56,7 +56,7 @@ class DeltaSubscriptionState : public Logger::Loggable { public: explicit ResourceVersion(absl::string_view version) : version_(version) {} // Builds a ResourceVersion in the waitingForServer state. - ResourceVersion() {} + ResourceVersion() = default; // If true, we currently have no version of this resource - we are waiting for the server to // provide us with one. diff --git a/source/common/grpc/codec.cc b/source/common/grpc/codec.cc index 94a0666a17e6..3362bf736061 100644 --- a/source/common/grpc/codec.cc +++ b/source/common/grpc/codec.cc @@ -11,7 +11,7 @@ namespace Envoy { namespace Grpc { -Encoder::Encoder() {} +Encoder::Encoder() = default; void Encoder::newFrame(uint8_t flags, uint64_t length, std::array& output) { output[0] = flags; diff --git a/source/common/grpc/typed_async_client.h b/source/common/grpc/typed_async_client.h index 910ba7c5731f..48ad6d06df70 100644 --- a/source/common/grpc/typed_async_client.h +++ b/source/common/grpc/typed_async_client.h @@ -29,7 +29,7 @@ AsyncRequest* sendUntyped(RawAsyncClient* client, const Protobuf::MethodDescript */ template class AsyncStream /* : public RawAsyncStream */ { public: - AsyncStream() {} + AsyncStream() = default; AsyncStream(RawAsyncStream* stream) : stream_(stream) {} AsyncStream(const AsyncStream& other) = default; void sendMessage(const Request& request, bool end_stream) { @@ -93,7 +93,7 @@ template class AsyncStreamCallbacks : public RawAsyncStreamC template class AsyncClient /* : public RawAsyncClient )*/ { public: - AsyncClient() {} + AsyncClient() = default; AsyncClient(RawAsyncClientPtr&& client) : client_(std::move(client)) {} virtual ~AsyncClient() = default; diff --git a/source/common/network/cidr_range.cc b/source/common/network/cidr_range.cc index e8ee2c7d6537..50b33dccbd64 100644 --- a/source/common/network/cidr_range.cc +++ b/source/common/network/cidr_range.cc @@ -34,13 +34,9 @@ CidrRange::CidrRange(InstanceConstSharedPtr address, int length) } } -CidrRange::CidrRange(const CidrRange& other) : address_(other.address_), length_(other.length_) {} +CidrRange::CidrRange(const CidrRange& other) = default; -CidrRange& CidrRange::operator=(const CidrRange& other) { - address_ = other.address_; - length_ = other.length_; - return *this; -} +CidrRange& CidrRange::operator=(const CidrRange& other) = default; bool CidrRange::operator==(const CidrRange& other) const { // Lengths must be the same, and must be valid (i.e. not -1). diff --git a/source/common/network/io_socket_error_impl.h b/source/common/network/io_socket_error_impl.h index a28e5c841eec..aa8f362dc8ca 100644 --- a/source/common/network/io_socket_error_impl.h +++ b/source/common/network/io_socket_error_impl.h @@ -11,7 +11,7 @@ class IoSocketError : public Api::IoError { public: explicit IoSocketError(int sys_errno) : errno_(sys_errno) {} - ~IoSocketError() override {} + ~IoSocketError() override = default; Api::IoError::IoErrorCode getErrorCode() const override; std::string getErrorDetails() const override; diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 5adba55f0f66..f18683d8bb36 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -374,7 +374,7 @@ class ValueUtil { class HashedValue { public: HashedValue(const ProtobufWkt::Value& value) : value_(value), hash_(ValueUtil::hash(value)){}; - HashedValue(const HashedValue& v) : value_(v.value_), hash_(v.hash_){}; + HashedValue(const HashedValue& v) = default; const ProtobufWkt::Value& value() const { return value_; } std::size_t hash() const { return hash_; } diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 8ec35a02330c..3fcbd480359b 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -228,7 +228,7 @@ class RetryPolicyImpl : public RetryPolicy { public: RetryPolicyImpl(const envoy::api::v2::route::RetryPolicy& retry_policy, ProtobufMessage::ValidationVisitor& validation_visitor); - RetryPolicyImpl() {} + RetryPolicyImpl() = default; // Router::RetryPolicy std::chrono::milliseconds perTryTimeout() const override { return per_try_timeout_; } diff --git a/source/common/router/header_parser.h b/source/common/router/header_parser.h index b64ebd169c4b..725f78a5e97c 100644 --- a/source/common/router/header_parser.h +++ b/source/common/router/header_parser.h @@ -42,7 +42,7 @@ class HeaderParser { void evaluateHeaders(Http::HeaderMap& headers, const StreamInfo::StreamInfo& stream_info) const; protected: - HeaderParser() {} + HeaderParser() = default; private: std::vector> headers_to_add_; diff --git a/source/common/stats/tag_producer_impl.h b/source/common/stats/tag_producer_impl.h index 505cb71929aa..6a4fcb07a6ef 100644 --- a/source/common/stats/tag_producer_impl.h +++ b/source/common/stats/tag_producer_impl.h @@ -29,7 +29,7 @@ namespace Stats { class TagProducerImpl : public TagProducer { public: TagProducerImpl(const envoy::config::metrics::v2::StatsConfig& config); - TagProducerImpl() {} + TagProducerImpl() = default; /** * Take a metric name and a vector then add proper tags into the vector and diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 1ee7e2b02672..79e3a1eca77d 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -182,7 +182,7 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase { LocalityDegradedHosts, }; - HostsSource() {} + HostsSource() = default; HostsSource(uint32_t priority, SourceType source_type) : priority_(priority), source_type_(source_type) { diff --git a/source/extensions/filters/common/original_src/original_src_socket_option.h b/source/extensions/filters/common/original_src/original_src_socket_option.h index 2659354d8598..e030456c2089 100644 --- a/source/extensions/filters/common/original_src/original_src_socket_option.h +++ b/source/extensions/filters/common/original_src/original_src_socket_option.h @@ -18,7 +18,7 @@ class OriginalSrcSocketOption : public Network::Socket::Option { * Constructs a socket option which will set the socket to use source @c src_address */ OriginalSrcSocketOption(Network::Address::InstanceConstSharedPtr src_address); - ~OriginalSrcSocketOption() {} + ~OriginalSrcSocketOption() = default; /** * Updates the source address of the socket to match `src_address_`. diff --git a/source/extensions/filters/http/dynamo/dynamo_request_parser.h b/source/extensions/filters/http/dynamo/dynamo_request_parser.h index e913e740e2b5..c7bd669b0bec 100644 --- a/source/extensions/filters/http/dynamo/dynamo_request_parser.h +++ b/source/extensions/filters/http/dynamo/dynamo_request_parser.h @@ -107,7 +107,7 @@ class RequestParser { // http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html static const std::vector SUPPORTED_ERROR_TYPES; - RequestParser() {} + RequestParser() = default; }; } // namespace Dynamo diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 9b43ab02a121..1d82239c7dfd 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -12,7 +12,7 @@ namespace IpTagging { IpTaggingFilter::IpTaggingFilter(IpTaggingFilterConfigSharedPtr config) : config_(config) {} -IpTaggingFilter::~IpTaggingFilter() {} +IpTaggingFilter::~IpTaggingFilter() = default; void IpTaggingFilter::onDestroy() {} diff --git a/source/extensions/filters/http/squash/squash_filter.cc b/source/extensions/filters/http/squash/squash_filter.cc index be4d4e095c9b..5a3b7d7b12cc 100644 --- a/source/extensions/filters/http/squash/squash_filter.cc +++ b/source/extensions/filters/http/squash/squash_filter.cc @@ -129,7 +129,7 @@ SquashFilter::SquashFilter(SquashFilterConfigSharedPtr config, Upstream::Cluster std::bind(&SquashFilter::onGetAttachmentFailure, this, _1)), cm_(cm), decoder_callbacks_(nullptr) {} -SquashFilter::~SquashFilter() {} +SquashFilter::~SquashFilter() = default; void SquashFilter::onDestroy() { cleanup(); } diff --git a/source/extensions/filters/network/dubbo_proxy/filters/filter.h b/source/extensions/filters/network/dubbo_proxy/filters/filter.h index eb7367a26ac7..5b4a5e5625c3 100644 --- a/source/extensions/filters/network/dubbo_proxy/filters/filter.h +++ b/source/extensions/filters/network/dubbo_proxy/filters/filter.h @@ -175,7 +175,7 @@ class EncoderFilterCallbacks : public virtual FilterCallbacksBase { */ class FilterBase { public: - virtual ~FilterBase() {} + virtual ~FilterBase() = default; /** * This routine is called prior to a filter being destroyed. This may happen after normal stream diff --git a/source/extensions/filters/network/dubbo_proxy/message_impl.h b/source/extensions/filters/network/dubbo_proxy/message_impl.h index 325633c859cc..1fc20c5f7a11 100644 --- a/source/extensions/filters/network/dubbo_proxy/message_impl.h +++ b/source/extensions/filters/network/dubbo_proxy/message_impl.h @@ -10,7 +10,7 @@ namespace DubboProxy { class ContextBase : public Context { public: ContextBase() = default; - ~ContextBase() override {} + ~ContextBase() override = default; // Override from Context size_t body_size() const override { return body_size_; } @@ -27,7 +27,7 @@ class ContextBase : public Context { class ContextImpl : public ContextBase { public: ContextImpl() = default; - ~ContextImpl() override {} + ~ContextImpl() override = default; bool is_heartbeat() const { return is_heartbeat_; } void set_heartbeat(bool is_heartbeat) { is_heartbeat_ = is_heartbeat; } diff --git a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc index 65daabe6f149..81eac5f0f396 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc +++ b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc @@ -64,7 +64,7 @@ ParameterRouteEntryImpl::ParameterRouteEntryImpl( } } -ParameterRouteEntryImpl::~ParameterRouteEntryImpl() {} +ParameterRouteEntryImpl::~ParameterRouteEntryImpl() = default; bool ParameterRouteEntryImpl::matchParameter(absl::string_view request_data, const ParameterData& config_data) const { @@ -138,7 +138,7 @@ MethodRouteEntryImpl::MethodRouteEntryImpl( } } -MethodRouteEntryImpl::~MethodRouteEntryImpl() {} +MethodRouteEntryImpl::~MethodRouteEntryImpl() = default; RouteConstSharedPtr MethodRouteEntryImpl::matches(const MessageMetadata& metadata, uint64_t random_value) const { diff --git a/source/extensions/filters/network/dubbo_proxy/router/router_impl.cc b/source/extensions/filters/network/dubbo_proxy/router/router_impl.cc index 82e12a3a8ebe..7e985e71a59d 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/router_impl.cc +++ b/source/extensions/filters/network/dubbo_proxy/router/router_impl.cc @@ -167,7 +167,7 @@ Router::UpstreamRequest::UpstreamRequest(Router& parent, Tcp::ConnectionPool::In request_complete_(false), response_started_(false), response_complete_(false), stream_reset_(false) {} -Router::UpstreamRequest::~UpstreamRequest() {} +Router::UpstreamRequest::~UpstreamRequest() = default; FilterStatus Router::UpstreamRequest::start() { Tcp::ConnectionPool::Cancellable* handle = conn_pool_.newConnection(*this); diff --git a/source/extensions/filters/network/ext_authz/ext_authz.h b/source/extensions/filters/network/ext_authz/ext_authz.h index b259c62f2f3d..276509c5d8e0 100644 --- a/source/extensions/filters/network/ext_authz/ext_authz.h +++ b/source/extensions/filters/network/ext_authz/ext_authz.h @@ -73,7 +73,7 @@ class Filter : public Network::ReadFilter, public: Filter(ConfigSharedPtr config, Filters::Common::ExtAuthz::ClientPtr&& client) : config_(config), client_(std::move(client)) {} - ~Filter() {} + ~Filter() = default; // Network::ReadFilter Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index 430cbe29e706..ea0ce7ca9029 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -153,7 +153,7 @@ class Int64Deserializer : public IntDeserializer { */ class BooleanDeserializer : public Deserializer { public: - BooleanDeserializer(){}; + BooleanDeserializer() = default; uint32_t feed(absl::string_view& data) override { return buffer_.feed(data); } diff --git a/source/extensions/filters/network/mongo_proxy/bson_impl.h b/source/extensions/filters/network/mongo_proxy/bson_impl.h index f929c6f4e45b..18d8ddc9ab9f 100644 --- a/source/extensions/filters/network/mongo_proxy/bson_impl.h +++ b/source/extensions/filters/network/mongo_proxy/bson_impl.h @@ -268,7 +268,7 @@ class DocumentImpl : public Document, const std::list& values() const override { return fields_; } private: - DocumentImpl() {} + DocumentImpl() = default; void fromBuffer(Buffer::Instance& data); diff --git a/source/extensions/filters/network/rbac/rbac_filter.h b/source/extensions/filters/network/rbac/rbac_filter.h index 8d9402e99680..3fffefd23e93 100644 --- a/source/extensions/filters/network/rbac/rbac_filter.h +++ b/source/extensions/filters/network/rbac/rbac_filter.h @@ -55,7 +55,7 @@ class RoleBasedAccessControlFilter : public Network::ReadFilter, public: RoleBasedAccessControlFilter(RoleBasedAccessControlFilterConfigSharedPtr config) : config_(config) {} - ~RoleBasedAccessControlFilter() {} + ~RoleBasedAccessControlFilter() = default; // Network::ReadFilter Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; diff --git a/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h b/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h index fa88a0c98d80..5d69037d68e2 100644 --- a/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h +++ b/source/extensions/filters/network/thrift_proxy/compact_protocol_impl.h @@ -21,7 +21,7 @@ namespace ThriftProxy { */ class CompactProtocolImpl : public Protocol { public: - CompactProtocolImpl() {} + CompactProtocolImpl() = default; // Protocol const std::string& name() const override { return ProtocolNames::get().COMPACT; } diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.cc b/source/extensions/filters/network/thrift_proxy/conn_manager.cc index 0edb3d6cbb04..509d2f42d3e1 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.cc +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.cc @@ -19,7 +19,7 @@ ConnectionManager::ConnectionManager(Config& config, Runtime::RandomGenerator& r decoder_(std::make_unique(*transport_, *protocol_, *this)), random_generator_(random_generator), time_source_(time_source) {} -ConnectionManager::~ConnectionManager() {} +ConnectionManager::~ConnectionManager() = default; Network::FilterStatus ConnectionManager::onData(Buffer::Instance& data, bool end_stream) { request_buffer_.move(data); diff --git a/source/extensions/filters/network/thrift_proxy/framed_transport_impl.h b/source/extensions/filters/network/thrift_proxy/framed_transport_impl.h index 5adf153a1f0e..786bfce0680b 100644 --- a/source/extensions/filters/network/thrift_proxy/framed_transport_impl.h +++ b/source/extensions/filters/network/thrift_proxy/framed_transport_impl.h @@ -17,7 +17,7 @@ namespace ThriftProxy { */ class FramedTransportImpl : public Transport { public: - FramedTransportImpl() {} + FramedTransportImpl() = default; // Transport const std::string& name() const override { return TransportNames::get().FRAMED; } diff --git a/source/extensions/filters/network/thrift_proxy/metadata.h b/source/extensions/filters/network/thrift_proxy/metadata.h index 731514756656..7ee3e68f297f 100644 --- a/source/extensions/filters/network/thrift_proxy/metadata.h +++ b/source/extensions/filters/network/thrift_proxy/metadata.h @@ -29,7 +29,7 @@ namespace ThriftProxy { */ class MessageMetadata { public: - MessageMetadata() {} + MessageMetadata() = default; bool hasFrameSize() const { return frame_size_.has_value(); } uint32_t frameSize() const { return frame_size_.value(); } diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc index a5112e9a0cc2..1fc44b5f78ae 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc @@ -368,7 +368,7 @@ Router::UpstreamRequest::UpstreamRequest(Router& parent, Tcp::ConnectionPool::In protocol_(NamedProtocolConfigFactory::getFactory(protocol_type).createProtocol()), request_complete_(false), response_started_(false), response_complete_(false) {} -Router::UpstreamRequest::~UpstreamRequest() {} +Router::UpstreamRequest::~UpstreamRequest() = default; FilterStatus Router::UpstreamRequest::start() { Tcp::ConnectionPool::Cancellable* handle = conn_pool_.newConnection(*this); diff --git a/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h b/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h index 8c87c6c8b040..39c31b3c79a0 100644 --- a/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h +++ b/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h @@ -18,7 +18,7 @@ namespace ThriftProxy { class ThriftBase : public DecoderEventHandler { public: ThriftBase(ThriftBase* parent); - ~ThriftBase() {} + ~ThriftBase() = default; // DecoderEventHandler FilterStatus transportBegin(MessageMetadataSharedPtr) override { return FilterStatus::Continue; } @@ -61,7 +61,7 @@ class ThriftValueBase : public ThriftValue, public ThriftBase { public: ThriftValueBase(ThriftBase* parent, FieldType value_type) : ThriftBase(parent), value_type_(value_type) {} - ~ThriftValueBase() {} + ~ThriftValueBase() = default; // ThriftValue FieldType type() const override { return value_type_; } diff --git a/source/extensions/filters/network/thrift_proxy/tracing.h b/source/extensions/filters/network/thrift_proxy/tracing.h index da0f1d52fc5c..e5f26a51e24a 100644 --- a/source/extensions/filters/network/thrift_proxy/tracing.h +++ b/source/extensions/filters/network/thrift_proxy/tracing.h @@ -17,7 +17,7 @@ class Endpoint { public: Endpoint(int32_t ipv4, int16_t port, const std::string& service_name) : ipv4_(ipv4), port_(port), service_name_(service_name) {} - Endpoint() {} + Endpoint() = default; int32_t ipv4_{0}; int16_t port_{0}; @@ -31,7 +31,7 @@ class Annotation { public: Annotation(int64_t timestamp, const std::string& value, absl::optional host) : timestamp_(timestamp), value_(value), host_(host) {} - Annotation() {} + Annotation() = default; int64_t timestamp_{0}; std::string value_; @@ -60,7 +60,7 @@ class BinaryAnnotation { BinaryAnnotation(const std::string& key, const std::string& value, AnnotationType annotation_type, absl::optional host) : key_(key), value_(value), annotation_type_(annotation_type), host_(host) {} - BinaryAnnotation() {} + BinaryAnnotation() = default; std::string key_; std::string value_; @@ -80,7 +80,7 @@ class Span { : trace_id_(trace_id), name_(name), span_id_(span_id), parent_span_id_(parent_span_id), annotations_(std::move(annotations)), binary_annotations_(std::move(binary_annotations)), debug_(debug) {} - Span() {} + Span() = default; int64_t trace_id_{0}; std::string name_; diff --git a/source/extensions/filters/network/thrift_proxy/twitter_protocol_impl.cc b/source/extensions/filters/network/thrift_proxy/twitter_protocol_impl.cc index 77e6862dac14..37a2961f891e 100644 --- a/source/extensions/filters/network/thrift_proxy/twitter_protocol_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/twitter_protocol_impl.cc @@ -166,7 +166,7 @@ class ClientId { */ class UpgradeReply : public DirectResponse, public ThriftObject { public: - UpgradeReply() {} + UpgradeReply() = default; UpgradeReply(Transport& transport) : thrift_obj_(std::make_unique(transport, protocol_)) {} diff --git a/source/extensions/filters/network/thrift_proxy/unframed_transport_impl.h b/source/extensions/filters/network/thrift_proxy/unframed_transport_impl.h index e3d5556fadf9..5266d1eaad7e 100644 --- a/source/extensions/filters/network/thrift_proxy/unframed_transport_impl.h +++ b/source/extensions/filters/network/thrift_proxy/unframed_transport_impl.h @@ -17,7 +17,7 @@ namespace ThriftProxy { */ class UnframedTransportImpl : public Transport { public: - UnframedTransportImpl() {} + UnframedTransportImpl() = default; // Transport const std::string& name() const override { return TransportNames::get().UNFRAMED; } diff --git a/source/extensions/tracers/zipkin/span_buffer.h b/source/extensions/tracers/zipkin/span_buffer.h index 2e9f6b90a69f..a67479c644a4 100644 --- a/source/extensions/tracers/zipkin/span_buffer.h +++ b/source/extensions/tracers/zipkin/span_buffer.h @@ -17,7 +17,7 @@ class SpanBuffer { * Constructor that creates an empty buffer. Space needs to be allocated by invoking * the method allocateBuffer(size). */ - SpanBuffer() {} + SpanBuffer() = default; /** * Constructor that initializes a buffer with the given size. diff --git a/source/extensions/tracers/zipkin/span_context_extractor.cc b/source/extensions/tracers/zipkin/span_context_extractor.cc index 50a057ae45af..747efc999525 100644 --- a/source/extensions/tracers/zipkin/span_context_extractor.cc +++ b/source/extensions/tracers/zipkin/span_context_extractor.cc @@ -32,7 +32,7 @@ bool getSamplingFlags(char c, const Tracing::Decision tracing_decision) { SpanContextExtractor::SpanContextExtractor(Http::HeaderMap& request_headers) : request_headers_(request_headers) {} -SpanContextExtractor::~SpanContextExtractor() {} +SpanContextExtractor::~SpanContextExtractor() = default; bool SpanContextExtractor::extractSampled(const Tracing::Decision tracing_decision) { bool sampled(false); diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 664a7f27884e..9d1da1e8700d 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -328,7 +328,9 @@ class AdminImpl : public Admin, class AdminFilterChain : public Network::FilterChain { public: - AdminFilterChain() {} + // We can't use the default constructor because transport_socket_factory_ doesn't have a + // default constructor. + AdminFilterChain() {} // NOLINT(modernize-use-equals-default) // Network::FilterChain const Network::TransportSocketFactory& transportSocketFactory() const override { diff --git a/test/common/compressor/zlib_compressor_impl_test.cc b/test/common/compressor/zlib_compressor_impl_test.cc index 0d0c4457cb89..112325f6e377 100644 --- a/test/common/compressor/zlib_compressor_impl_test.cc +++ b/test/common/compressor/zlib_compressor_impl_test.cc @@ -70,7 +70,7 @@ class ZlibCompressorImplTest : public testing::Test { class ZlibCompressorImplTester : public ZlibCompressorImpl { public: - ZlibCompressorImplTester() {} + ZlibCompressorImplTester() = default; ZlibCompressorImplTester(uint64_t chunk_size) : ZlibCompressorImpl(chunk_size) {} void compressThenFlush(Buffer::OwnedImpl& buffer) { compress(buffer, State::Flush); } void finish(Buffer::OwnedImpl& buffer) { compress(buffer, State::Finish); } diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 55731c26a88a..26d7daca0df3 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -8,7 +8,7 @@ namespace { class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public testing::Test { protected: - DeltaSubscriptionImplTest() {} + DeltaSubscriptionImplTest() = default; }; TEST_F(DeltaSubscriptionImplTest, UpdateResourcesCausesRequest) { diff --git a/test/common/network/address_impl_test.cc b/test/common/network/address_impl_test.cc index 27298c383d9d..f3a069cb1b87 100644 --- a/test/common/network/address_impl_test.cc +++ b/test/common/network/address_impl_test.cc @@ -415,7 +415,7 @@ struct TestCase { TestCase() = default; TestCase(enum InstanceType type, const std::string& address, uint32_t port) : address_(address), type_(type), port_(port) {} - TestCase(const TestCase& rhs) : address_(rhs.address_), type_(rhs.type_), port_(rhs.port_) {} + TestCase(const TestCase& rhs) = default; bool operator==(const TestCase& rhs) { return (type_ == rhs.type_ && address_ == rhs.address_ && port_ == rhs.port_); diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 8ddfb9f31535..dd7f751d55ed 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -1335,7 +1335,7 @@ TEST_P(ConnectionImplTest, DelayedCloseTimeoutNullStats) { class FakeReadFilter : public Network::ReadFilter { public: - FakeReadFilter() {} + FakeReadFilter() = default; ~FakeReadFilter() { EXPECT_TRUE(callbacks_ != nullptr); // The purpose is to verify that when FilterManger is destructed, ConnectionSocketImpl is not diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index 2909bcb7f85f..3b83558e378e 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -362,7 +362,7 @@ class CustomInstance : public Address::Instance { CustomInstance(const std::string& address, uint32_t port) : instance_(address, port) { antagonistic_name_ = fmt::format("{}:borked_port_{}", address, port); } - ~CustomInstance() override {} + ~CustomInstance() override = default; // Address::Instance bool operator==(const Address::Instance& rhs) const override { diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index 7a2d0abe50d9..ae59bf16a54c 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -240,8 +240,8 @@ class CvcValidationCallback { class MockCvcValidationCallback : public CvcValidationCallback { public: - MockCvcValidationCallback() {} - ~MockCvcValidationCallback() {} + MockCvcValidationCallback() = default; + ~MockCvcValidationCallback() = default; MOCK_METHOD1(validateCvc, void(const envoy::api::v2::auth::CertificateValidationContext&)); }; diff --git a/test/common/tcp/conn_pool_test.cc b/test/common/tcp/conn_pool_test.cc index 149354109c05..68153686e52b 100644 --- a/test/common/tcp/conn_pool_test.cc +++ b/test/common/tcp/conn_pool_test.cc @@ -180,7 +180,7 @@ class TcpConnPoolImplDestructorTest : public testing::Test { Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000"), Upstream::ResourcePriority::Default, nullptr, nullptr)} {} - ~TcpConnPoolImplDestructorTest() {} + ~TcpConnPoolImplDestructorTest() = default; void prepareConn() { connection_ = new NiceMock(); diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index 864b7295558f..83f8d3f27c2f 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -116,7 +116,7 @@ class EdsTest : public testing::Test { class EdsWithHealthCheckUpdateTest : public EdsTest { protected: - EdsWithHealthCheckUpdateTest() {} + EdsWithHealthCheckUpdateTest() = default; // Build the initial cluster with some endpoints. void initializeCluster(const std::vector endpoint_ports, diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index f1d28d67d9b6..95054a43fcf8 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -2832,7 +2832,7 @@ class TestGrpcHealthCheckerImpl : public GrpcHealthCheckerImpl { class GrpcHealthCheckerImplTestBase { public: struct TestSession { - TestSession() {} + TestSession() = default; Event::MockTimer* interval_timer_{}; Event::MockTimer* timeout_timer_{}; diff --git a/test/extensions/filters/http/common/aws/mocks.cc b/test/extensions/filters/http/common/aws/mocks.cc index ee5048203bb1..3e68170b4b6e 100644 --- a/test/extensions/filters/http/common/aws/mocks.cc +++ b/test/extensions/filters/http/common/aws/mocks.cc @@ -6,13 +6,13 @@ namespace HttpFilters { namespace Common { namespace Aws { -MockCredentialsProvider::MockCredentialsProvider() {} +MockCredentialsProvider::MockCredentialsProvider() = default; -MockCredentialsProvider::~MockCredentialsProvider() {} +MockCredentialsProvider::~MockCredentialsProvider() = default; -MockSigner::MockSigner() {} +MockSigner::MockSigner() = default; -MockSigner::~MockSigner() {} +MockSigner::~MockSigner() = default; } // namespace Aws } // namespace Common diff --git a/test/extensions/filters/http/common/aws/mocks.h b/test/extensions/filters/http/common/aws/mocks.h index a19f906f238b..0e4ced802301 100644 --- a/test/extensions/filters/http/common/aws/mocks.h +++ b/test/extensions/filters/http/common/aws/mocks.h @@ -29,7 +29,7 @@ class MockSigner : public Signer { class MockMetadataFetcher { public: - virtual ~MockMetadataFetcher(){}; + virtual ~MockMetadataFetcher() = default; MOCK_CONST_METHOD3(fetch, absl::optional(const std::string&, const std::string&, const absl::optional&)); diff --git a/test/extensions/filters/network/dubbo_proxy/mocks.cc b/test/extensions/filters/network/dubbo_proxy/mocks.cc index 276d4cb3408f..2e93c8eebbc3 100644 --- a/test/extensions/filters/network/dubbo_proxy/mocks.cc +++ b/test/extensions/filters/network/dubbo_proxy/mocks.cc @@ -65,8 +65,8 @@ MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { } MockDecoderFilterCallbacks::~MockDecoderFilterCallbacks() = default; -MockEncoderFilter::MockEncoderFilter() {} -MockEncoderFilter::~MockEncoderFilter() {} +MockEncoderFilter::MockEncoderFilter() = default; +MockEncoderFilter::~MockEncoderFilter() = default; MockEncoderFilterCallbacks::MockEncoderFilterCallbacks() { route_.reset(new NiceMock()); @@ -77,10 +77,10 @@ MockEncoderFilterCallbacks::MockEncoderFilterCallbacks() { ON_CALL(*this, streamInfo()).WillByDefault(ReturnRef(stream_info_)); ON_CALL(*this, dispatcher()).WillByDefault(ReturnRef(dispatcher_)); } -MockEncoderFilterCallbacks::~MockEncoderFilterCallbacks() {} +MockEncoderFilterCallbacks::~MockEncoderFilterCallbacks() = default; -MockCodecFilter::MockCodecFilter() {} -MockCodecFilter::~MockCodecFilter() {} +MockCodecFilter::MockCodecFilter() = default; +MockCodecFilter::~MockCodecFilter() = default; MockFilterConfigFactory::MockFilterConfigFactory() : MockFactoryBase("envoy.filters.dubbo.mock_filter"), diff --git a/test/extensions/filters/network/kafka/kafka_request_parser_test.cc b/test/extensions/filters/network/kafka/kafka_request_parser_test.cc index a70797742806..fda8f7770e6c 100644 --- a/test/extensions/filters/network/kafka/kafka_request_parser_test.cc +++ b/test/extensions/filters/network/kafka/kafka_request_parser_test.cc @@ -21,7 +21,7 @@ class KafkaRequestParserTest : public testing::Test, public BufferBasedTest {}; class MockRequestParserResolver : public RequestParserResolver { public: - MockRequestParserResolver(){}; + MockRequestParserResolver() = default; MOCK_CONST_METHOD3(createParser, RequestParserSharedPtr(int16_t, int16_t, RequestContextSharedPtr)); }; diff --git a/test/extensions/filters/network/kafka/kafka_response_parser_test.cc b/test/extensions/filters/network/kafka/kafka_response_parser_test.cc index de1ab3edf013..ea7fd9699126 100644 --- a/test/extensions/filters/network/kafka/kafka_response_parser_test.cc +++ b/test/extensions/filters/network/kafka/kafka_response_parser_test.cc @@ -21,7 +21,7 @@ class KafkaResponseParserTest : public testing::Test, public BufferBasedTest {}; class MockResponseParserResolver : public ResponseParserResolver { public: - MockResponseParserResolver(){}; + MockResponseParserResolver() = default; MOCK_CONST_METHOD1(createParser, ResponseParserSharedPtr(ResponseContextSharedPtr)); }; diff --git a/test/extensions/filters/network/redis_proxy/mocks.cc b/test/extensions/filters/network/redis_proxy/mocks.cc index b3da756d65c3..f06a6bd9b0b9 100644 --- a/test/extensions/filters/network/redis_proxy/mocks.cc +++ b/test/extensions/filters/network/redis_proxy/mocks.cc @@ -10,32 +10,32 @@ namespace Extensions { namespace NetworkFilters { namespace RedisProxy { -MockRouter::MockRouter() {} -MockRouter::~MockRouter() {} +MockRouter::MockRouter() = default; +MockRouter::~MockRouter() = default; MockRoute::MockRoute(ConnPool::InstanceSharedPtr conn_pool) : conn_pool_(std::move(conn_pool)) { ON_CALL(*this, upstream()).WillByDefault(Return(conn_pool_)); ON_CALL(*this, mirrorPolicies()).WillByDefault(ReturnRef(policies_)); } -MockRoute::~MockRoute() {} +MockRoute::~MockRoute() = default; namespace ConnPool { -MockInstance::MockInstance() {} -MockInstance::~MockInstance() {} +MockInstance::MockInstance() = default; +MockInstance::~MockInstance() = default; } // namespace ConnPool namespace CommandSplitter { -MockSplitRequest::MockSplitRequest() {} -MockSplitRequest::~MockSplitRequest() {} +MockSplitRequest::MockSplitRequest() = default; +MockSplitRequest::~MockSplitRequest() = default; -MockSplitCallbacks::MockSplitCallbacks() {} -MockSplitCallbacks::~MockSplitCallbacks() {} +MockSplitCallbacks::MockSplitCallbacks() = default; +MockSplitCallbacks::~MockSplitCallbacks() = default; -MockInstance::MockInstance() {} -MockInstance::~MockInstance() {} +MockInstance::MockInstance() = default; +MockInstance::~MockInstance() = default; } // namespace CommandSplitter } // namespace RedisProxy diff --git a/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc b/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc index 261eb6c80adb..c29cf0a02220 100644 --- a/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc @@ -24,8 +24,8 @@ namespace { class MockBuffer : public Envoy::MockBuffer { public: - MockBuffer() {} - ~MockBuffer() {} + MockBuffer() = default; + ~MockBuffer() = default; MOCK_CONST_METHOD0(length, uint64_t()); }; diff --git a/test/extensions/filters/network/thrift_proxy/mocks.cc b/test/extensions/filters/network/thrift_proxy/mocks.cc index e32c0a654782..3f591919a4fc 100644 --- a/test/extensions/filters/network/thrift_proxy/mocks.cc +++ b/test/extensions/filters/network/thrift_proxy/mocks.cc @@ -18,14 +18,14 @@ namespace Extensions { namespace NetworkFilters { namespace ThriftProxy { -MockConfig::MockConfig() {} -MockConfig::~MockConfig() {} +MockConfig::MockConfig() = default; +MockConfig::~MockConfig() = default; MockTransport::MockTransport() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, type()).WillByDefault(Return(type_)); } -MockTransport::~MockTransport() {} +MockTransport::~MockTransport() = default; MockProtocol::MockProtocol() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); @@ -35,24 +35,24 @@ MockProtocol::MockProtocol() { })); ON_CALL(*this, supportsUpgrade()).WillByDefault(Return(false)); } -MockProtocol::~MockProtocol() {} +MockProtocol::~MockProtocol() = default; -MockDecoderCallbacks::MockDecoderCallbacks() {} -MockDecoderCallbacks::~MockDecoderCallbacks() {} +MockDecoderCallbacks::MockDecoderCallbacks() = default; +MockDecoderCallbacks::~MockDecoderCallbacks() = default; -MockDecoderEventHandler::MockDecoderEventHandler() {} -MockDecoderEventHandler::~MockDecoderEventHandler() {} +MockDecoderEventHandler::MockDecoderEventHandler() = default; +MockDecoderEventHandler::~MockDecoderEventHandler() = default; -MockDirectResponse::MockDirectResponse() {} -MockDirectResponse::~MockDirectResponse() {} +MockDirectResponse::MockDirectResponse() = default; +MockDirectResponse::~MockDirectResponse() = default; -MockThriftObject::MockThriftObject() {} -MockThriftObject::~MockThriftObject() {} +MockThriftObject::MockThriftObject() = default; +MockThriftObject::~MockThriftObject() = default; namespace ThriftFilters { -MockFilterChainFactoryCallbacks::MockFilterChainFactoryCallbacks() {} -MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() {} +MockFilterChainFactoryCallbacks::MockFilterChainFactoryCallbacks() = default; +MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() = default; MockDecoderFilter::MockDecoderFilter() { ON_CALL(*this, transportBegin(_)).WillByDefault(Return(FilterStatus::Continue)); @@ -77,7 +77,7 @@ MockDecoderFilter::MockDecoderFilter() { ON_CALL(*this, setBegin(_, _)).WillByDefault(Return(FilterStatus::Continue)); ON_CALL(*this, setEnd()).WillByDefault(Return(FilterStatus::Continue)); } -MockDecoderFilter::~MockDecoderFilter() {} +MockDecoderFilter::~MockDecoderFilter() = default; MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { route_.reset(new NiceMock()); @@ -87,14 +87,14 @@ MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { ON_CALL(*this, route()).WillByDefault(Return(route_)); ON_CALL(*this, streamInfo()).WillByDefault(ReturnRef(stream_info_)); } -MockDecoderFilterCallbacks::~MockDecoderFilterCallbacks() {} +MockDecoderFilterCallbacks::~MockDecoderFilterCallbacks() = default; MockFilterConfigFactory::MockFilterConfigFactory() : FactoryBase("envoy.filters.thrift.mock_filter") { mock_filter_.reset(new NiceMock()); } -MockFilterConfigFactory::~MockFilterConfigFactory() {} +MockFilterConfigFactory::~MockFilterConfigFactory() = default; FilterFactoryCb MockFilterConfigFactory::createFilterFactoryFromProtoTyped( const ProtobufWkt::Struct& proto_config, const std::string& stat_prefix, @@ -116,22 +116,22 @@ namespace Router { MockRateLimitPolicyEntry::MockRateLimitPolicyEntry() { ON_CALL(*this, disableKey()).WillByDefault(ReturnRef(disable_key_)); } -MockRateLimitPolicyEntry::~MockRateLimitPolicyEntry() {} +MockRateLimitPolicyEntry::~MockRateLimitPolicyEntry() = default; MockRateLimitPolicy::MockRateLimitPolicy() { ON_CALL(*this, empty()).WillByDefault(Return(true)); ON_CALL(*this, getApplicableRateLimit(_)).WillByDefault(ReturnRef(rate_limit_policy_entry_)); } -MockRateLimitPolicy::~MockRateLimitPolicy() {} +MockRateLimitPolicy::~MockRateLimitPolicy() = default; MockRouteEntry::MockRouteEntry() { ON_CALL(*this, clusterName()).WillByDefault(ReturnRef(cluster_name_)); ON_CALL(*this, rateLimitPolicy()).WillByDefault(ReturnRef(rate_limit_policy_)); } -MockRouteEntry::~MockRouteEntry() {} +MockRouteEntry::~MockRouteEntry() = default; MockRoute::MockRoute() { ON_CALL(*this, routeEntry()).WillByDefault(Return(&route_entry_)); } -MockRoute::~MockRoute() {} +MockRoute::~MockRoute() = default; } // namespace Router } // namespace ThriftProxy diff --git a/test/extensions/quic_listeners/quiche/platform/quic_epoll_clock.h b/test/extensions/quic_listeners/quiche/platform/quic_epoll_clock.h index fc605c9a2611..f1446dbb4bee 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_epoll_clock.h +++ b/test/extensions/quic_listeners/quiche/platform/quic_epoll_clock.h @@ -21,7 +21,7 @@ class QuicEpollClock : public QuicClock { QuicEpollClock(const QuicEpollClock&) = delete; QuicEpollClock& operator=(const QuicEpollClock&) = delete; - ~QuicEpollClock() override {} + ~QuicEpollClock() override = default; // Returns the approximate current time as a QuicTime object. QuicTime ApproximateNow() const override; diff --git a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc index 8fcb0ff33365..34e406378117 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc @@ -726,7 +726,7 @@ TEST_F(QuicPlatformTest, TestQuicOptional) { class QuicMemSliceTest : public Envoy::Buffer::BufferImplementationParamTest { public: - ~QuicMemSliceTest() override {} + ~QuicMemSliceTest() override = default; }; INSTANTIATE_TEST_SUITE_P(QuicMemSliceTests, QuicMemSliceTest, diff --git a/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc b/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc index 697e556590cf..d8c82b845017 100644 --- a/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc +++ b/test/extensions/resource_monitors/fixed_heap/fixed_heap_monitor_test.cc @@ -12,7 +12,7 @@ namespace { class MockMemoryStatsReader : public MemoryStatsReader { public: - MockMemoryStatsReader() {} + MockMemoryStatsReader() = default; MOCK_METHOD0(reservedHeapBytes, uint64_t()); MOCK_METHOD0(unmappedHeapBytes, uint64_t()); diff --git a/test/extensions/transport_sockets/alts/noop_transport_socket_callbacks_test.cc b/test/extensions/transport_sockets/alts/noop_transport_socket_callbacks_test.cc index 2ba6dc28f4ad..507448d70d47 100644 --- a/test/extensions/transport_sockets/alts/noop_transport_socket_callbacks_test.cc +++ b/test/extensions/transport_sockets/alts/noop_transport_socket_callbacks_test.cc @@ -19,7 +19,7 @@ class TestTransportSocketCallbacks : public Network::TransportSocketCallbacks { explicit TestTransportSocketCallbacks(Network::Connection& connection) : io_handle_(std::make_unique()), connection_(connection) {} - ~TestTransportSocketCallbacks() override {} + ~TestTransportSocketCallbacks() override = default; Network::IoHandle& ioHandle() override { return *io_handle_; } const Network::IoHandle& ioHandle() const override { return *io_handle_; } Network::Connection& connection() override { return connection_; } diff --git a/test/integration/filters/process_context_filter.h b/test/integration/filters/process_context_filter.h index 7e1fde35ff64..16e4cff2a6c9 100644 --- a/test/integration/filters/process_context_filter.h +++ b/test/integration/filters/process_context_filter.h @@ -7,7 +7,7 @@ namespace Envoy { class ProcessObjectForFilter : public ProcessObject { public: explicit ProcessObjectForFilter(bool is_healthy) : is_healthy_(is_healthy) {} - ~ProcessObjectForFilter() override {} + ~ProcessObjectForFilter() override = default; bool isHealthy() { return is_healthy_; } diff --git a/test/integration/utility.cc b/test/integration/utility.cc index de4375118daf..59b336b38d6d 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -129,7 +129,7 @@ RawConnectionDriver::RawConnectionDriver(uint32_t port, Buffer::Instance& initia client_->connect(); } -RawConnectionDriver::~RawConnectionDriver() {} +RawConnectionDriver::~RawConnectionDriver() = default; void RawConnectionDriver::run(Event::Dispatcher::RunType run_type) { dispatcher_->run(run_type); } diff --git a/test/mocks/access_log/mocks.cc b/test/mocks/access_log/mocks.cc index 0220ae53f450..33c7b022fd31 100644 --- a/test/mocks/access_log/mocks.cc +++ b/test/mocks/access_log/mocks.cc @@ -10,20 +10,20 @@ using testing::ReturnRef; namespace Envoy { namespace AccessLog { -MockAccessLogFile::MockAccessLogFile() {} -MockAccessLogFile::~MockAccessLogFile() {} +MockAccessLogFile::MockAccessLogFile() = default; +MockAccessLogFile::~MockAccessLogFile() = default; -MockFilter::MockFilter() {} -MockFilter::~MockFilter() {} +MockFilter::MockFilter() = default; +MockFilter::~MockFilter() = default; MockAccessLogManager::MockAccessLogManager() { ON_CALL(*this, createAccessLog(_)).WillByDefault(Return(file_)); } -MockAccessLogManager::~MockAccessLogManager() {} +MockAccessLogManager::~MockAccessLogManager() = default; -MockInstance::MockInstance() {} -MockInstance::~MockInstance() {} +MockInstance::MockInstance() = default; +MockInstance::~MockInstance() = default; } // namespace AccessLog } // namespace Envoy diff --git a/test/mocks/api/mocks.cc b/test/mocks/api/mocks.cc index b3eb6296544d..77f6a4463dd3 100644 --- a/test/mocks/api/mocks.cc +++ b/test/mocks/api/mocks.cc @@ -17,7 +17,7 @@ MockApi::MockApi() { ON_CALL(*this, rootScope()).WillByDefault(ReturnRef(stats_store_)); } -MockApi::~MockApi() {} +MockApi::~MockApi() = default; Event::DispatcherPtr MockApi::allocateDispatcher() { return Event::DispatcherPtr{allocateDispatcher_(time_system_)}; @@ -26,9 +26,9 @@ Event::DispatcherPtr MockApi::allocateDispatcher(Buffer::WatermarkFactoryPtr&& w return Event::DispatcherPtr{allocateDispatcher_(std::move(watermark_factory), time_system_)}; } -MockOsSysCalls::MockOsSysCalls() {} +MockOsSysCalls::MockOsSysCalls() = default; -MockOsSysCalls::~MockOsSysCalls() {} +MockOsSysCalls::~MockOsSysCalls() = default; SysCallIntResult MockOsSysCalls::setsockopt(int sockfd, int level, int optname, const void* optval, socklen_t optlen) { diff --git a/test/mocks/buffer/mocks.cc b/test/mocks/buffer/mocks.cc index 1aae98289c2c..64459da03703 100644 --- a/test/mocks/buffer/mocks.cc +++ b/test/mocks/buffer/mocks.cc @@ -22,7 +22,7 @@ MockBufferBase::MockBufferBase(std::function, std::fu template <> MockBufferBase::MockBufferBase() : Buffer::OwnedImpl() {} -MockBufferFactory::MockBufferFactory() {} -MockBufferFactory::~MockBufferFactory() {} +MockBufferFactory::MockBufferFactory() = default; +MockBufferFactory::~MockBufferFactory() = default; } // namespace Envoy diff --git a/test/mocks/common.cc b/test/mocks/common.cc index f0b878da8477..ba36e63fc102 100644 --- a/test/mocks/common.cc +++ b/test/mocks/common.cc @@ -1,10 +1,10 @@ #include "test/mocks/common.h" namespace Envoy { -ReadyWatcher::ReadyWatcher() {} -ReadyWatcher::~ReadyWatcher() {} +ReadyWatcher::ReadyWatcher() = default; +ReadyWatcher::~ReadyWatcher() = default; -MockTimeSystem::MockTimeSystem() {} -MockTimeSystem::~MockTimeSystem() {} +MockTimeSystem::MockTimeSystem() = default; +MockTimeSystem::~MockTimeSystem() = default; } // namespace Envoy diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 8396fff4fc31..8ced0b8a96a5 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -25,7 +25,7 @@ template class MockSubscriptionCallbacks : public Subscript return resourceName_(TestUtility::anyConvert(resource)); })); } - ~MockSubscriptionCallbacks() override {} + ~MockSubscriptionCallbacks() override = default; static std::string resourceName_(const envoy::api::v2::ClusterLoadAssignment& resource) { return resource.cluster_name(); } diff --git a/test/mocks/event/mocks.cc b/test/mocks/event/mocks.cc index f1d5cdcadb19..0cd2f11d33be 100644 --- a/test/mocks/event/mocks.cc +++ b/test/mocks/event/mocks.cc @@ -24,7 +24,7 @@ MockDispatcher::MockDispatcher() { ON_CALL(*this, post(_)).WillByDefault(Invoke([](PostCb cb) -> void { cb(); })); } -MockDispatcher::~MockDispatcher() {} +MockDispatcher::~MockDispatcher() = default; MockTimer::MockTimer() { ON_CALL(*this, enableTimer(_)).WillByDefault(Assign(&enabled_, true)); @@ -41,7 +41,7 @@ MockTimer::MockTimer(MockDispatcher* dispatcher) : MockTimer() { .RetiresOnSaturation(); } -MockTimer::~MockTimer() {} +MockTimer::~MockTimer() = default; MockSignalEvent::MockSignalEvent(MockDispatcher* dispatcher) { EXPECT_CALL(*dispatcher, listenForSignal_(_, _)) @@ -49,10 +49,10 @@ MockSignalEvent::MockSignalEvent(MockDispatcher* dispatcher) { .RetiresOnSaturation(); } -MockSignalEvent::~MockSignalEvent() {} +MockSignalEvent::~MockSignalEvent() = default; -MockFileEvent::MockFileEvent() {} -MockFileEvent::~MockFileEvent() {} +MockFileEvent::MockFileEvent() = default; +MockFileEvent::~MockFileEvent() = default; } // namespace Event } // namespace Envoy diff --git a/test/mocks/filesystem/mocks.cc b/test/mocks/filesystem/mocks.cc index a9eca9c4815c..5aa5806a26a0 100644 --- a/test/mocks/filesystem/mocks.cc +++ b/test/mocks/filesystem/mocks.cc @@ -7,7 +7,7 @@ namespace Envoy { namespace Filesystem { MockFile::MockFile() : num_opens_(0), num_writes_(0), is_open_(false) {} -MockFile::~MockFile() {} +MockFile::~MockFile() = default; Api::IoCallBoolResult MockFile::open() { Thread::LockGuard lock(open_mutex_); diff --git a/test/mocks/grpc/mocks.cc b/test/mocks/grpc/mocks.cc index 1eccf7ea0870..07b3e3b348dc 100644 --- a/test/mocks/grpc/mocks.cc +++ b/test/mocks/grpc/mocks.cc @@ -3,17 +3,17 @@ namespace Envoy { namespace Grpc { -MockAsyncRequest::MockAsyncRequest() {} -MockAsyncRequest::~MockAsyncRequest() {} +MockAsyncRequest::MockAsyncRequest() = default; +MockAsyncRequest::~MockAsyncRequest() = default; -MockAsyncStream::MockAsyncStream() {} -MockAsyncStream::~MockAsyncStream() {} +MockAsyncStream::MockAsyncStream() = default; +MockAsyncStream::~MockAsyncStream() = default; -MockAsyncClientFactory::MockAsyncClientFactory() {} -MockAsyncClientFactory::~MockAsyncClientFactory() {} +MockAsyncClientFactory::MockAsyncClientFactory() = default; +MockAsyncClientFactory::~MockAsyncClientFactory() = default; -MockAsyncClientManager::MockAsyncClientManager() {} -MockAsyncClientManager::~MockAsyncClientManager() {} +MockAsyncClientManager::MockAsyncClientManager() = default; +MockAsyncClientManager::~MockAsyncClientManager() = default; } // namespace Grpc } // namespace Envoy diff --git a/test/mocks/http/stream.cc b/test/mocks/http/stream.cc index ef881fb8618c..95ef73f56c42 100644 --- a/test/mocks/http/stream.cc +++ b/test/mocks/http/stream.cc @@ -22,7 +22,7 @@ MockStream::MockStream() { })); } -MockStream::~MockStream() {} +MockStream::~MockStream() = default; } // namespace Http } // namespace Envoy diff --git a/test/mocks/http/stream_decoder.cc b/test/mocks/http/stream_decoder.cc index 592a55ce96a6..3c2c75aed33b 100644 --- a/test/mocks/http/stream_decoder.cc +++ b/test/mocks/http/stream_decoder.cc @@ -3,8 +3,8 @@ namespace Envoy { namespace Http { -MockStreamDecoder::MockStreamDecoder() {} -MockStreamDecoder::~MockStreamDecoder() {} +MockStreamDecoder::MockStreamDecoder() = default; +MockStreamDecoder::~MockStreamDecoder() = default; } // namespace Http } // namespace Envoy diff --git a/test/mocks/http/stream_encoder.cc b/test/mocks/http/stream_encoder.cc index da99c0b97035..98fdec782184 100644 --- a/test/mocks/http/stream_encoder.cc +++ b/test/mocks/http/stream_encoder.cc @@ -7,7 +7,7 @@ MockStreamEncoder::MockStreamEncoder() { ON_CALL(*this, getStream()).WillByDefault(ReturnRef(stream_)); } -MockStreamEncoder::~MockStreamEncoder() {} +MockStreamEncoder::~MockStreamEncoder() = default; } // namespace Http } // namespace Envoy diff --git a/test/mocks/local_info/mocks.cc b/test/mocks/local_info/mocks.cc index 2f3a2e8ff4b7..7faff3e6b8b8 100644 --- a/test/mocks/local_info/mocks.cc +++ b/test/mocks/local_info/mocks.cc @@ -23,7 +23,7 @@ MockLocalInfo::MockLocalInfo() : address_(new Network::Address::Ipv4Instance("12 ON_CALL(*this, node()).WillByDefault(ReturnRef(node_)); } -MockLocalInfo::~MockLocalInfo() {} +MockLocalInfo::~MockLocalInfo() = default; } // namespace LocalInfo } // namespace Envoy diff --git a/test/mocks/network/connection.cc b/test/mocks/network/connection.cc index f329be222c68..915eb0e86c48 100644 --- a/test/mocks/network/connection.cc +++ b/test/mocks/network/connection.cc @@ -9,8 +9,8 @@ using testing::ReturnRef; namespace Envoy { namespace Network { -MockConnectionCallbacks::MockConnectionCallbacks() {} -MockConnectionCallbacks::~MockConnectionCallbacks() {} +MockConnectionCallbacks::MockConnectionCallbacks() = default; +MockConnectionCallbacks::~MockConnectionCallbacks() = default; uint64_t MockConnectionBase::next_id_; @@ -79,7 +79,7 @@ MockConnection::MockConnection() { remote_address_ = Utility::resolveUrl("tcp://10.0.0.3:50000"); initializeMockConnection(*this); } -MockConnection::~MockConnection() {} +MockConnection::~MockConnection() = default; MockClientConnection::MockClientConnection() { remote_address_ = Utility::resolveUrl("tcp://10.0.0.1:443"); @@ -87,7 +87,7 @@ MockClientConnection::MockClientConnection() { initializeMockConnection(*this); } -MockClientConnection::~MockClientConnection() {} +MockClientConnection::~MockClientConnection() = default; MockFilterManagerConnection::MockFilterManagerConnection() { remote_address_ = Utility::resolveUrl("tcp://10.0.0.3:50000"); @@ -98,7 +98,7 @@ MockFilterManagerConnection::MockFilterManagerConnection() { buffer.drain(buffer.length()); })); } -MockFilterManagerConnection::~MockFilterManagerConnection() {} +MockFilterManagerConnection::~MockFilterManagerConnection() = default; } // namespace Network } // namespace Envoy diff --git a/test/mocks/network/mocks.cc b/test/mocks/network/mocks.cc index b3307515d5cc..6070d90c3fb2 100644 --- a/test/mocks/network/mocks.cc +++ b/test/mocks/network/mocks.cc @@ -30,22 +30,22 @@ MockListenerConfig::MockListenerConfig() { ON_CALL(*this, listenerScope()).WillByDefault(ReturnRef(scope_)); ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); } -MockListenerConfig::~MockListenerConfig() {} +MockListenerConfig::~MockListenerConfig() = default; -MockActiveDnsQuery::MockActiveDnsQuery() {} -MockActiveDnsQuery::~MockActiveDnsQuery() {} +MockActiveDnsQuery::MockActiveDnsQuery() = default; +MockActiveDnsQuery::~MockActiveDnsQuery() = default; MockDnsResolver::MockDnsResolver() { ON_CALL(*this, resolve(_, _, _)).WillByDefault(Return(&active_query_)); } -MockDnsResolver::~MockDnsResolver() {} +MockDnsResolver::~MockDnsResolver() = default; MockAddressResolver::MockAddressResolver() { ON_CALL(*this, name()).WillByDefault(Return("envoy.mock.resolver")); } -MockAddressResolver::~MockAddressResolver() {} +MockAddressResolver::~MockAddressResolver() = default; MockReadFilterCallbacks::MockReadFilterCallbacks() { ON_CALL(*this, connection()).WillByDefault(ReturnRef(connection_)); @@ -53,7 +53,7 @@ MockReadFilterCallbacks::MockReadFilterCallbacks() { ON_CALL(*this, upstreamHost(_)).WillByDefault(SaveArg<0>(&host_)); } -MockReadFilterCallbacks::~MockReadFilterCallbacks() {} +MockReadFilterCallbacks::~MockReadFilterCallbacks() = default; MockReadFilter::MockReadFilter() { ON_CALL(*this, onData(_, _)).WillByDefault(Return(FilterStatus::StopIteration)); @@ -62,20 +62,20 @@ MockReadFilter::MockReadFilter() { Invoke([this](ReadFilterCallbacks& callbacks) -> void { callbacks_ = &callbacks; })); } -MockReadFilter::~MockReadFilter() {} +MockReadFilter::~MockReadFilter() = default; MockWriteFilterCallbacks::MockWriteFilterCallbacks() { ON_CALL(*this, connection()).WillByDefault(ReturnRef(connection_)); } -MockWriteFilterCallbacks::~MockWriteFilterCallbacks() {} +MockWriteFilterCallbacks::~MockWriteFilterCallbacks() = default; MockWriteFilter::MockWriteFilter() { EXPECT_CALL(*this, initializeWriteFilterCallbacks(_)) .WillOnce(Invoke( [this](WriteFilterCallbacks& callbacks) -> void { write_callbacks_ = &callbacks; })); } -MockWriteFilter::~MockWriteFilter() {} +MockWriteFilter::~MockWriteFilter() = default; MockFilter::MockFilter() { EXPECT_CALL(*this, initializeReadFilterCallbacks(_)) @@ -86,39 +86,39 @@ MockFilter::MockFilter() { [this](WriteFilterCallbacks& callbacks) -> void { write_callbacks_ = &callbacks; })); } -MockFilter::~MockFilter() {} +MockFilter::~MockFilter() = default; -MockListenerCallbacks::MockListenerCallbacks() {} -MockListenerCallbacks::~MockListenerCallbacks() {} +MockListenerCallbacks::MockListenerCallbacks() = default; +MockListenerCallbacks::~MockListenerCallbacks() = default; -MockUdpListenerCallbacks::MockUdpListenerCallbacks() {} -MockUdpListenerCallbacks::~MockUdpListenerCallbacks() {} +MockUdpListenerCallbacks::MockUdpListenerCallbacks() = default; +MockUdpListenerCallbacks::~MockUdpListenerCallbacks() = default; -MockDrainDecision::MockDrainDecision() {} -MockDrainDecision::~MockDrainDecision() {} +MockDrainDecision::MockDrainDecision() = default; +MockDrainDecision::~MockDrainDecision() = default; -MockListenerFilter::MockListenerFilter() {} -MockListenerFilter::~MockListenerFilter() {} +MockListenerFilter::MockListenerFilter() = default; +MockListenerFilter::~MockListenerFilter() = default; MockListenerFilterCallbacks::MockListenerFilterCallbacks() { ON_CALL(*this, socket()).WillByDefault(ReturnRef(socket_)); } -MockListenerFilterCallbacks::~MockListenerFilterCallbacks() {} +MockListenerFilterCallbacks::~MockListenerFilterCallbacks() = default; -MockListenerFilterManager::MockListenerFilterManager() {} -MockListenerFilterManager::~MockListenerFilterManager() {} +MockListenerFilterManager::MockListenerFilterManager() = default; +MockListenerFilterManager::~MockListenerFilterManager() = default; -MockFilterChain::MockFilterChain() {} -MockFilterChain::~MockFilterChain() {} +MockFilterChain::MockFilterChain() = default; +MockFilterChain::~MockFilterChain() = default; -MockFilterChainManager::MockFilterChainManager() {} -MockFilterChainManager::~MockFilterChainManager() {} +MockFilterChainManager::MockFilterChainManager() = default; +MockFilterChainManager::~MockFilterChainManager() = default; MockFilterChainFactory::MockFilterChainFactory() { ON_CALL(*this, createListenerFilterChain(_)).WillByDefault(Return(true)); ON_CALL(*this, createUdpListenerFilterChain(_, _)).WillByDefault(Return(true)); } -MockFilterChainFactory::~MockFilterChainFactory() {} +MockFilterChainFactory::~MockFilterChainFactory() = default; MockListenSocket::MockListenSocket() : io_handle_(std::make_unique()), @@ -133,7 +133,7 @@ MockSocketOption::MockSocketOption() { ON_CALL(*this, setOption(_, _)).WillByDefault(Return(true)); } -MockSocketOption::~MockSocketOption() {} +MockSocketOption::~MockSocketOption() = default; MockConnectionSocket::MockConnectionSocket() : io_handle_(std::make_unique()), @@ -145,46 +145,46 @@ MockConnectionSocket::MockConnectionSocket() ON_CALL(testing::Const(*this), ioHandle()).WillByDefault(ReturnRef(*io_handle_)); } -MockListener::MockListener() {} +MockListener::MockListener() = default; MockListener::~MockListener() { onDestroy(); } -MockConnectionHandler::MockConnectionHandler() {} -MockConnectionHandler::~MockConnectionHandler() {} +MockConnectionHandler::MockConnectionHandler() = default; +MockConnectionHandler::~MockConnectionHandler() = default; -MockIp::MockIp() {} -MockIp::~MockIp() {} +MockIp::MockIp() = default; +MockIp::~MockIp() = default; -MockResolvedAddress::~MockResolvedAddress() {} +MockResolvedAddress::~MockResolvedAddress() = default; MockTransportSocket::MockTransportSocket() { ON_CALL(*this, setTransportSocketCallbacks(_)) .WillByDefault(Invoke([&](TransportSocketCallbacks& callbacks) { callbacks_ = &callbacks; })); } -MockTransportSocket::~MockTransportSocket() {} +MockTransportSocket::~MockTransportSocket() = default; -MockTransportSocketFactory::MockTransportSocketFactory() {} -MockTransportSocketFactory::~MockTransportSocketFactory() {} +MockTransportSocketFactory::MockTransportSocketFactory() = default; +MockTransportSocketFactory::~MockTransportSocketFactory() = default; MockTransportSocketCallbacks::MockTransportSocketCallbacks() { ON_CALL(*this, connection()).WillByDefault(ReturnRef(connection_)); } -MockTransportSocketCallbacks::~MockTransportSocketCallbacks() {} +MockTransportSocketCallbacks::~MockTransportSocketCallbacks() = default; -MockUdpListener::MockUdpListener() {} +MockUdpListener::MockUdpListener() = default; MockUdpListener::~MockUdpListener() { onDestroy(); } MockUdpReadFilterCallbacks::MockUdpReadFilterCallbacks() { ON_CALL(*this, udpListener()).WillByDefault(ReturnRef(udp_listener_)); } -MockUdpReadFilterCallbacks::~MockUdpReadFilterCallbacks() {} +MockUdpReadFilterCallbacks::~MockUdpReadFilterCallbacks() = default; MockUdpListenerReadFilter::MockUdpListenerReadFilter(UdpReadFilterCallbacks& callbacks) : UdpListenerReadFilter(callbacks) {} -MockUdpListenerReadFilter::~MockUdpListenerReadFilter() {} +MockUdpListenerReadFilter::~MockUdpListenerReadFilter() = default; -MockUdpListenerFilterManager::MockUdpListenerFilterManager() {} -MockUdpListenerFilterManager::~MockUdpListenerFilterManager() {} +MockUdpListenerFilterManager::MockUdpListenerFilterManager() = default; +MockUdpListenerFilterManager::~MockUdpListenerFilterManager() = default; } // namespace Network } // namespace Envoy diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 80fec6e54c42..28e12f6f4d02 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -216,7 +216,7 @@ class MockFilterChainFactory : public FilterChainFactory { class MockListenSocket : public Socket { public: MockListenSocket(); - ~MockListenSocket() override {} + ~MockListenSocket() override = default; void addOption(const Socket::OptionConstSharedPtr& option) override { addOption_(option); } void addOptions(const Socket::OptionsSharedPtr& options) override { addOptions_(options); } @@ -252,7 +252,7 @@ class MockSocketOption : public Socket::Option { class MockConnectionSocket : public ConnectionSocket { public: MockConnectionSocket(); - ~MockConnectionSocket() override {} + ~MockConnectionSocket() override = default; void addOption(const Socket::OptionConstSharedPtr& option) override { addOption_(option); } void addOptions(const Socket::OptionsSharedPtr& options) override { addOptions_(options); } diff --git a/test/mocks/protobuf/mocks.cc b/test/mocks/protobuf/mocks.cc index 95799456341c..908b08864fab 100644 --- a/test/mocks/protobuf/mocks.cc +++ b/test/mocks/protobuf/mocks.cc @@ -3,9 +3,9 @@ namespace Envoy { namespace ProtobufMessage { -MockValidationVisitor::MockValidationVisitor() {} +MockValidationVisitor::MockValidationVisitor() = default; -MockValidationVisitor::~MockValidationVisitor() {} +MockValidationVisitor::~MockValidationVisitor() = default; } // namespace ProtobufMessage } // namespace Envoy diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index f99496356e95..827e9c84f875 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -15,10 +15,10 @@ using testing::SaveArg; namespace Envoy { namespace Router { -MockDirectResponseEntry::MockDirectResponseEntry() {} -MockDirectResponseEntry::~MockDirectResponseEntry() {} +MockDirectResponseEntry::MockDirectResponseEntry() = default; +MockDirectResponseEntry::~MockDirectResponseEntry() = default; -MockRetryState::MockRetryState() {} +MockRetryState::MockRetryState() = default; void MockRetryState::expectHeadersRetry() { EXPECT_CALL(*this, shouldRetryHeaders(_, _)) @@ -35,43 +35,43 @@ void MockRetryState::expectResetRetry() { .WillOnce(DoAll(SaveArg<1>(&callback_), Return(RetryStatus::Yes))); } -MockRetryState::~MockRetryState() {} +MockRetryState::~MockRetryState() = default; MockRateLimitPolicyEntry::MockRateLimitPolicyEntry() { ON_CALL(*this, disableKey()).WillByDefault(ReturnRef(disable_key_)); } -MockRateLimitPolicyEntry::~MockRateLimitPolicyEntry() {} +MockRateLimitPolicyEntry::~MockRateLimitPolicyEntry() = default; MockRateLimitPolicy::MockRateLimitPolicy() { ON_CALL(*this, getApplicableRateLimit(_)).WillByDefault(ReturnRef(rate_limit_policy_entry_)); ON_CALL(*this, empty()).WillByDefault(Return(true)); } -MockRateLimitPolicy::~MockRateLimitPolicy() {} +MockRateLimitPolicy::~MockRateLimitPolicy() = default; -MockShadowWriter::MockShadowWriter() {} -MockShadowWriter::~MockShadowWriter() {} +MockShadowWriter::MockShadowWriter() = default; +MockShadowWriter::~MockShadowWriter() = default; MockVirtualHost::MockVirtualHost() { ON_CALL(*this, name()).WillByDefault(ReturnRef(name_)); ON_CALL(*this, rateLimitPolicy()).WillByDefault(ReturnRef(rate_limit_policy_)); } -MockVirtualHost::~MockVirtualHost() {} +MockVirtualHost::~MockVirtualHost() = default; -MockHashPolicy::MockHashPolicy() {} -MockHashPolicy::~MockHashPolicy() {} +MockHashPolicy::MockHashPolicy() = default; +MockHashPolicy::~MockHashPolicy() = default; -MockMetadataMatchCriteria::MockMetadataMatchCriteria() {} -MockMetadataMatchCriteria::~MockMetadataMatchCriteria() {} +MockMetadataMatchCriteria::MockMetadataMatchCriteria() = default; +MockMetadataMatchCriteria::~MockMetadataMatchCriteria() = default; MockPathMatchCriterion::MockPathMatchCriterion() { ON_CALL(*this, matchType()).WillByDefault(ReturnPointee(&type_)); ON_CALL(*this, matcher()).WillByDefault(ReturnPointee(&matcher_)); } -MockPathMatchCriterion::~MockPathMatchCriterion() {} +MockPathMatchCriterion::~MockPathMatchCriterion() = default; MockRouteEntry::MockRouteEntry() { ON_CALL(*this, clusterName()).WillByDefault(ReturnRef(cluster_name_)); @@ -90,7 +90,7 @@ MockRouteEntry::MockRouteEntry() { ON_CALL(*this, routeName()).WillByDefault(ReturnRef(route_name_)); } -MockRouteEntry::~MockRouteEntry() {} +MockRouteEntry::~MockRouteEntry() = default; MockConfig::MockConfig() : route_(new NiceMock()) { ON_CALL(*this, route(_, _)).WillByDefault(Return(route_)); @@ -99,28 +99,28 @@ MockConfig::MockConfig() : route_(new NiceMock()) { ON_CALL(*this, usesVhds()).WillByDefault(Return(false)); } -MockConfig::~MockConfig() {} +MockConfig::~MockConfig() = default; MockDecorator::MockDecorator() { ON_CALL(*this, getOperation()).WillByDefault(ReturnRef(operation_)); } -MockDecorator::~MockDecorator() {} +MockDecorator::~MockDecorator() = default; -MockRouteTracing::MockRouteTracing() {} -MockRouteTracing::~MockRouteTracing() {} +MockRouteTracing::MockRouteTracing() = default; +MockRouteTracing::~MockRouteTracing() = default; MockRoute::MockRoute() { ON_CALL(*this, routeEntry()).WillByDefault(Return(&route_entry_)); ON_CALL(*this, decorator()).WillByDefault(Return(&decorator_)); ON_CALL(*this, tracingConfig()).WillByDefault(Return(nullptr)); } -MockRoute::~MockRoute() {} +MockRoute::~MockRoute() = default; -MockRouteConfigProviderManager::MockRouteConfigProviderManager() {} -MockRouteConfigProviderManager::~MockRouteConfigProviderManager() {} +MockRouteConfigProviderManager::MockRouteConfigProviderManager() = default; +MockRouteConfigProviderManager::~MockRouteConfigProviderManager() = default; -MockScopedConfig::MockScopedConfig() {} -MockScopedConfig::~MockScopedConfig() {} +MockScopedConfig::MockScopedConfig() = default; +MockScopedConfig::~MockScopedConfig() = default; } // namespace Router } // namespace Envoy diff --git a/test/mocks/runtime/mocks.cc b/test/mocks/runtime/mocks.cc index acc39e811d86..c0415b560e44 100644 --- a/test/mocks/runtime/mocks.cc +++ b/test/mocks/runtime/mocks.cc @@ -12,19 +12,19 @@ namespace Runtime { MockRandomGenerator::MockRandomGenerator() { ON_CALL(*this, uuid()).WillByDefault(Return(uuid_)); } -MockRandomGenerator::~MockRandomGenerator() {} +MockRandomGenerator::~MockRandomGenerator() = default; MockSnapshot::MockSnapshot() { ON_CALL(*this, getInteger(_, _)).WillByDefault(ReturnArg<1>()); } -MockSnapshot::~MockSnapshot() {} +MockSnapshot::~MockSnapshot() = default; MockLoader::MockLoader() { ON_CALL(*this, snapshot()).WillByDefault(ReturnRef(snapshot_)); } -MockLoader::~MockLoader() {} +MockLoader::~MockLoader() = default; -MockOverrideLayer::MockOverrideLayer() {} +MockOverrideLayer::MockOverrideLayer() = default; -MockOverrideLayer::~MockOverrideLayer() {} +MockOverrideLayer::~MockOverrideLayer() = default; } // namespace Runtime } // namespace Envoy diff --git a/test/mocks/secret/mocks.cc b/test/mocks/secret/mocks.cc index 3bacd5fe9894..30fd6ae5d622 100644 --- a/test/mocks/secret/mocks.cc +++ b/test/mocks/secret/mocks.cc @@ -21,11 +21,11 @@ MockSecretManager::MockSecretManager() { })); } -MockSecretManager::~MockSecretManager() {} +MockSecretManager::~MockSecretManager() = default; -MockSecretCallbacks::MockSecretCallbacks() {} +MockSecretCallbacks::MockSecretCallbacks() = default; -MockSecretCallbacks::~MockSecretCallbacks() {} +MockSecretCallbacks::~MockSecretCallbacks() = default; } // namespace Secret } // namespace Envoy diff --git a/test/mocks/ssl/mocks.cc b/test/mocks/ssl/mocks.cc index 3318d4ec9804..72702de823ed 100644 --- a/test/mocks/ssl/mocks.cc +++ b/test/mocks/ssl/mocks.cc @@ -3,20 +3,20 @@ namespace Envoy { namespace Ssl { -MockContextManager::MockContextManager() {} -MockContextManager::~MockContextManager() {} +MockContextManager::MockContextManager() = default; +MockContextManager::~MockContextManager() = default; -MockConnectionInfo::MockConnectionInfo() {} -MockConnectionInfo::~MockConnectionInfo() {} +MockConnectionInfo::MockConnectionInfo() = default; +MockConnectionInfo::~MockConnectionInfo() = default; -MockClientContext::MockClientContext() {} -MockClientContext::~MockClientContext() {} +MockClientContext::MockClientContext() = default; +MockClientContext::~MockClientContext() = default; -MockClientContextConfig::MockClientContextConfig() {} -MockClientContextConfig::~MockClientContextConfig() {} +MockClientContextConfig::MockClientContextConfig() = default; +MockClientContextConfig::~MockClientContextConfig() = default; -MockServerContextConfig::MockServerContextConfig() {} -MockServerContextConfig::~MockServerContextConfig() {} +MockServerContextConfig::MockServerContextConfig() = default; +MockServerContextConfig::~MockServerContextConfig() = default; } // namespace Ssl } // namespace Envoy diff --git a/test/mocks/stream_info/mocks.cc b/test/mocks/stream_info/mocks.cc index 0d102036dd90..6c842c9ecb3c 100644 --- a/test/mocks/stream_info/mocks.cc +++ b/test/mocks/stream_info/mocks.cc @@ -96,7 +96,7 @@ MockStreamInfo::MockStreamInfo() .WillByDefault(ReturnRef(upstream_transport_failure_reason_)); } -MockStreamInfo::~MockStreamInfo() {} +MockStreamInfo::~MockStreamInfo() = default; } // namespace StreamInfo } // namespace Envoy diff --git a/test/mocks/tcp/mocks.cc b/test/mocks/tcp/mocks.cc index d9ffcc2f79e7..be7f9046fdb1 100644 --- a/test/mocks/tcp/mocks.cc +++ b/test/mocks/tcp/mocks.cc @@ -12,13 +12,13 @@ namespace Envoy { namespace Tcp { namespace ConnectionPool { -MockCancellable::MockCancellable() {} -MockCancellable::~MockCancellable() {} +MockCancellable::MockCancellable() = default; +MockCancellable::~MockCancellable() = default; -MockUpstreamCallbacks::MockUpstreamCallbacks() {} -MockUpstreamCallbacks::~MockUpstreamCallbacks() {} +MockUpstreamCallbacks::MockUpstreamCallbacks() = default; +MockUpstreamCallbacks::~MockUpstreamCallbacks() = default; -MockConnectionData::MockConnectionData() {} +MockConnectionData::MockConnectionData() = default; MockConnectionData::~MockConnectionData() { if (release_callback_) { release_callback_(); @@ -30,7 +30,7 @@ MockInstance::MockInstance() { return newConnectionImpl(cb); })); } -MockInstance::~MockInstance() {} +MockInstance::~MockInstance() = default; MockCancellable* MockInstance::newConnectionImpl(Callbacks& cb) { handles_.emplace_back(); diff --git a/test/mocks/tracing/mocks.cc b/test/mocks/tracing/mocks.cc index 04e9df4ae7aa..db9be3902320 100644 --- a/test/mocks/tracing/mocks.cc +++ b/test/mocks/tracing/mocks.cc @@ -9,21 +9,21 @@ using testing::ReturnRef; namespace Envoy { namespace Tracing { -MockSpan::MockSpan() {} -MockSpan::~MockSpan() {} +MockSpan::MockSpan() = default; +MockSpan::~MockSpan() = default; MockConfig::MockConfig() { ON_CALL(*this, operationName()).WillByDefault(Return(operation_name_)); ON_CALL(*this, requestHeadersForTags()).WillByDefault(ReturnRef(headers_)); ON_CALL(*this, verbose()).WillByDefault(Return(verbose_)); } -MockConfig::~MockConfig() {} +MockConfig::~MockConfig() = default; -MockHttpTracer::MockHttpTracer() {} -MockHttpTracer::~MockHttpTracer() {} +MockHttpTracer::MockHttpTracer() = default; +MockHttpTracer::~MockHttpTracer() = default; -MockDriver::MockDriver() {} -MockDriver::~MockDriver() {} +MockDriver::MockDriver() = default; +MockDriver::~MockDriver() = default; } // namespace Tracing } // namespace Envoy diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index d34c8fa20fe1..c139f9b2c8e0 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -25,13 +25,13 @@ MockLoadBalancerSubsetInfo::MockLoadBalancerSubsetInfo() { ON_CALL(*this, subsetSelectors()).WillByDefault(ReturnRef(subset_selectors_)); } -MockLoadBalancerSubsetInfo::~MockLoadBalancerSubsetInfo() {} +MockLoadBalancerSubsetInfo::~MockLoadBalancerSubsetInfo() = default; MockIdleTimeEnabledClusterInfo::MockIdleTimeEnabledClusterInfo() { ON_CALL(*this, idleTimeout()).WillByDefault(Return(std::chrono::milliseconds(1000))); } -MockIdleTimeEnabledClusterInfo::~MockIdleTimeEnabledClusterInfo() {} +MockIdleTimeEnabledClusterInfo::~MockIdleTimeEnabledClusterInfo() = default; MockClusterInfo::MockClusterInfo() : stats_(ClusterInfoImpl::generateStats(stats_store_)), @@ -82,7 +82,7 @@ MockClusterInfo::MockClusterInfo() ON_CALL(*this, clusterType()).WillByDefault(ReturnRef(cluster_type_)); } -MockClusterInfo::~MockClusterInfo() {} +MockClusterInfo::~MockClusterInfo() = default; } // namespace Upstream } // namespace Envoy diff --git a/test/mocks/upstream/host.cc b/test/mocks/upstream/host.cc index 536428cba59a..10eb8d9bb151 100644 --- a/test/mocks/upstream/host.cc +++ b/test/mocks/upstream/host.cc @@ -11,11 +11,11 @@ namespace Envoy { namespace Upstream { namespace Outlier { -MockDetectorHostMonitor::MockDetectorHostMonitor() {} -MockDetectorHostMonitor::~MockDetectorHostMonitor() {} +MockDetectorHostMonitor::MockDetectorHostMonitor() = default; +MockDetectorHostMonitor::~MockDetectorHostMonitor() = default; -MockEventLogger::MockEventLogger() {} -MockEventLogger::~MockEventLogger() {} +MockEventLogger::MockEventLogger() = default; +MockEventLogger::~MockEventLogger() = default; MockDetector::MockDetector() { ON_CALL(*this, addChangedStateCb(_)).WillByDefault(Invoke([this](ChangeStateCb cb) -> void { @@ -23,12 +23,12 @@ MockDetector::MockDetector() { })); } -MockDetector::~MockDetector() {} +MockDetector::~MockDetector() = default; } // namespace Outlier -MockHealthCheckHostMonitor::MockHealthCheckHostMonitor() {} -MockHealthCheckHostMonitor::~MockHealthCheckHostMonitor() {} +MockHealthCheckHostMonitor::MockHealthCheckHostMonitor() = default; +MockHealthCheckHostMonitor::~MockHealthCheckHostMonitor() = default; MockHostDescription::MockHostDescription() : address_(Network::Utility::resolveUrl("tcp://10.0.0.1:443")) { @@ -40,7 +40,7 @@ MockHostDescription::MockHostDescription() ON_CALL(*this, healthChecker()).WillByDefault(ReturnRef(health_checker_)); } -MockHostDescription::~MockHostDescription() {} +MockHostDescription::~MockHostDescription() = default; MockHost::MockHost() { ON_CALL(*this, cluster()).WillByDefault(ReturnRef(cluster_)); @@ -49,7 +49,7 @@ MockHost::MockHost() { ON_CALL(*this, warmed()).WillByDefault(Return(true)); } -MockHost::~MockHost() {} +MockHost::~MockHost() = default; } // namespace Upstream } // namespace Envoy diff --git a/test/test_common/logging.cc b/test/test_common/logging.cc index d53a175f3c52..636bb56c4bad 100644 --- a/test/test_common/logging.cc +++ b/test/test_common/logging.cc @@ -23,7 +23,7 @@ LogLevelSetter::~LogLevelSetter() { LogRecordingSink::LogRecordingSink(Logger::DelegatingLogSinkPtr log_sink) : Logger::SinkDelegate(log_sink) {} -LogRecordingSink::~LogRecordingSink() {} +LogRecordingSink::~LogRecordingSink() = default; void LogRecordingSink::log(absl::string_view msg) { previous_delegate()->log(msg); From 50fb561c23078e01efd4dc3852812efbf5ca8936 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 18 Jul 2019 21:11:21 -0700 Subject: [PATCH 202/542] ci: bring JVM heap limit back (#7639) Fixes some CI flakes in ASAN/TSAN caused by #7558. Signed-off-by: Lizan Zhou --- .bazelrc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.bazelrc b/.bazelrc index f8a14fadc686..3a507d5ca130 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,4 +1,15 @@ # Envoy specific Bazel build/test options. + +# Bazel doesn't need more than 200MB of memory for local build based on memory profiling: +# https://docs.bazel.build/versions/master/skylark/performance.html#memory-profiling +# The default JVM max heapsize is 1/4 of physical memory up to 32GB which could be large +# enough to consume all memory constrained by cgroup in large host, which is the case in CircleCI. +# Limiting JVM heapsize here to let it do GC more when approaching the limit to +# leave room for compiler/linker. +# The number 2G is choosed heuristically to both support in CircleCI and large enough for RBE. +# Startup options cannot be selected via config. +startup --host_jvm_args=-Xmx2g + build --workspace_status_command=bazel/get_workspace_status build --experimental_remap_main_repo build --experimental_local_memory_estimate From 50109476364f0859432e71e08222f341a58a696d Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Fri, 19 Jul 2019 22:15:08 +0530 Subject: [PATCH 203/542] upstream: improve original dst logs (#7633) Currently when original dst validation occurs, it does not very clear what lb type_cluster type combination is causing that. This PR improves that log line. Risk Level: Low Testing: Added automated tests Docs Changes: N/A Release Notes: N/A Signed-off-by: Rama Chavali --- source/common/upstream/original_dst_cluster.cc | 5 ++++- source/common/upstream/upstream_impl.cc | 5 ++++- test/common/upstream/cluster_manager_impl_test.cc | 6 ++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/source/common/upstream/original_dst_cluster.cc b/source/common/upstream/original_dst_cluster.cc index e061a7c390cc..9cb8414aa166 100644 --- a/source/common/upstream/original_dst_cluster.cc +++ b/source/common/upstream/original_dst_cluster.cc @@ -196,7 +196,10 @@ OriginalDstClusterFactory::createClusterImpl( Stats::ScopePtr&& stats_scope) { if (cluster.lb_policy() != envoy::api::v2::Cluster::ORIGINAL_DST_LB) { throw EnvoyException(fmt::format( - "cluster: cluster type 'original_dst' may only be used with LB type 'original_dst_lb'")); + "cluster: LB policy {} is not valid for Cluster type {}. Only 'original_dst_lb' " + "is allowed with cluster type 'original_dst'", + envoy::api::v2::Cluster_LbPolicy_Name(cluster.lb_policy()), + envoy::api::v2::Cluster_DiscoveryType_Name(cluster.type()))); } if (cluster.has_lb_subset_config() && cluster.lb_subset_config().subset_selectors_size() != 0) { throw EnvoyException( diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 03ca1318c195..a8f3d695ece2 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -595,7 +595,10 @@ ClusterInfoImpl::ClusterInfoImpl(const envoy::api::v2::Cluster& config, case envoy::api::v2::Cluster::ORIGINAL_DST_LB: if (config.type() != envoy::api::v2::Cluster::ORIGINAL_DST) { throw EnvoyException(fmt::format( - "cluster: LB type 'original_dst_lb' may only be used with cluster type 'original_dst'")); + "cluster: LB policy {} is not valid for Cluster type {}. Only 'original_dst_lb' " + "is allowed with cluster type 'original_dst'", + envoy::api::v2::Cluster_LbPolicy_Name(config.lb_policy()), + envoy::api::v2::Cluster_DiscoveryType_Name(config.type()))); } lb_type_ = LoadBalancerType::OriginalDst; break; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index badbc3df0394..e9aa3293d89a 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -543,7 +543,8 @@ TEST_F(ClusterManagerImplTest, OriginalDstLbRestriction) { EXPECT_THROW_WITH_MESSAGE( create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, - "cluster: cluster type 'original_dst' may only be used with LB type 'original_dst_lb'"); + "cluster: LB policy ROUND_ROBIN is not valid for Cluster type ORIGINAL_DST. Only " + "'original_dst_lb' is allowed with cluster type 'original_dst'"); } TEST_F(ClusterManagerImplTest, OriginalDstLbRestriction2) { @@ -566,7 +567,8 @@ TEST_F(ClusterManagerImplTest, OriginalDstLbRestriction2) { EXPECT_THROW_WITH_MESSAGE( create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, - "cluster: LB type 'original_dst_lb' may only be used with cluster type 'original_dst'"); + "cluster: LB policy ORIGINAL_DST_LB is not valid for Cluster type STATIC. Only " + "'original_dst_lb' is allowed with cluster type 'original_dst'"); } TEST_F(ClusterManagerImplTest, SubsetLoadBalancerInitialization) { From c2967a82c9e55dd77bbbc932e4907b204b87cb45 Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Fri, 19 Jul 2019 10:47:40 -0700 Subject: [PATCH 204/542] listener filter: new listener filter for inspecting http protocol (#7559) Description: new listener filter for inspecting http protocol. - Http1x: check request line - Http2: check connection preface Pros: Performance; Cons: False positive possibility Risk Level: low Testing: unit test, manual test Docs Changes: Added Release Notes: Added #Issue: #7527 Signed-off-by: Yan Xue --- CODEOWNERS | 2 + .../listener_filters/http_inspector.rst | 38 +++ .../listener_filters/listener_filters.rst | 1 + docs/root/intro/version_history.rst | 1 + source/extensions/extensions_build_config.bzl | 10 +- .../filters/listener/http_inspector/BUILD | 38 +++ .../filters/listener/http_inspector/config.cc | 43 +++ .../listener/http_inspector/http_inspector.cc | 145 +++++++++ .../listener/http_inspector/http_inspector.h | 83 +++++ .../filters/listener/well_known_names.h | 2 + .../filters/listener/http_inspector/BUILD | 43 +++ .../http_inspector_config_test.cc | 52 ++++ .../http_inspector/http_inspector_test.cc | 286 ++++++++++++++++++ tools/spelling_dictionary.txt | 1 + 14 files changed, 739 insertions(+), 6 deletions(-) create mode 100644 docs/root/configuration/listener_filters/http_inspector.rst create mode 100644 source/extensions/filters/listener/http_inspector/BUILD create mode 100644 source/extensions/filters/listener/http_inspector/config.cc create mode 100644 source/extensions/filters/listener/http_inspector/http_inspector.cc create mode 100644 source/extensions/filters/listener/http_inspector/http_inspector.h create mode 100644 test/extensions/filters/listener/http_inspector/BUILD create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_config_test.cc create mode 100644 test/extensions/filters/listener/http_inspector/http_inspector_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index f908e99f3780..a252439569d6 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -44,3 +44,5 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/http/dynamic_forward_proxy @mattklein123 @alyssawilk # omit_canary_hosts retry predicate /*/extensions/retry/host/omit_canary_hosts @sriduth @snowp +# http inspector +/*/extensions/filters/listener/http_inspector @crazyxy @PiotrSikora @lizan diff --git a/docs/root/configuration/listener_filters/http_inspector.rst b/docs/root/configuration/listener_filters/http_inspector.rst new file mode 100644 index 000000000000..5972e0ad89cd --- /dev/null +++ b/docs/root/configuration/listener_filters/http_inspector.rst @@ -0,0 +1,38 @@ +.. _config_listener_filters_http_inspector: + +HTTP Inspector +============== + +HTTP Inspector listener filter allows detecting whether the application protocol appears to be HTTP, +and if it is HTTP, it detects the HTTP protocol (HTTP/1.x or HTTP/2) further. This can be used to select a +:ref:`FilterChain ` via the :ref:`application_protocols ` +of a :ref:`FilterChainMatch `. + +* :ref:`v2 API reference ` +* This filter should be configured with the name *envoy.listener.http_inspector*. + +Example +------- + +A sample filter configuration could be: + +.. code-block:: yaml + + listener_filters: + - name: "envoy.listener.http_inspector" + typed_config: {} + +Statistics +---------- + +This filter has a statistics tree rooted at *http_inspector* with the following statistics: + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + read_error, Counter, Total read errors + http10_found, Counter, Total number of times HTTP/1.0 was found + http11_found, Counter, Total number of times HTTP/1.1 was found + http2_found, Counter, Total number of times HTTP/2 was found + http_not_found, Counter, Total number of times HTTP protocol was not found diff --git a/docs/root/configuration/listener_filters/listener_filters.rst b/docs/root/configuration/listener_filters/listener_filters.rst index 7357976c500b..0bed75eb9fc6 100644 --- a/docs/root/configuration/listener_filters/listener_filters.rst +++ b/docs/root/configuration/listener_filters/listener_filters.rst @@ -8,6 +8,7 @@ Envoy has the following builtin listener filters. .. toctree:: :maxdepth: 2 + http_inspector original_dst_filter original_src_filter proxy_protocol diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index c81ece57fdfd..ab727239f444 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -5,6 +5,7 @@ Version history ================ * admin: added ability to configure listener :ref:`socket options `. * config: async data access for local and remote data source. +* listeners: added :ref:`HTTP inspector listener filter `. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. 1.11.0 (July 11, 2019) diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 8e3970090ba2..c18d6d6fa47e 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -57,16 +57,14 @@ EXTENSIONS = { # Listener filters # - # NOTE: The proxy_protocol filter is implicitly loaded if proxy_protocol functionality is - # configured on the listener. Do not remove it in that case or configs will fail to load. - "envoy.filters.listener.proxy_protocol": "//source/extensions/filters/listener/proxy_protocol:config", - + "envoy.filters.listener.http_inspector": "//source/extensions/filters/listener/http_inspector:config", # NOTE: The original_dst filter is implicitly loaded if original_dst functionality is # configured on the listener. Do not remove it in that case or configs will fail to load. "envoy.filters.listener.original_dst": "//source/extensions/filters/listener/original_dst:config", - "envoy.filters.listener.original_src": "//source/extensions/filters/listener/original_src:config", - + # NOTE: The proxy_protocol filter is implicitly loaded if proxy_protocol functionality is + # configured on the listener. Do not remove it in that case or configs will fail to load. + "envoy.filters.listener.proxy_protocol": "//source/extensions/filters/listener/proxy_protocol:config", "envoy.filters.listener.tls_inspector": "//source/extensions/filters/listener/tls_inspector:config", # diff --git a/source/extensions/filters/listener/http_inspector/BUILD b/source/extensions/filters/listener/http_inspector/BUILD new file mode 100644 index 000000000000..1d5b13218f00 --- /dev/null +++ b/source/extensions/filters/listener/http_inspector/BUILD @@ -0,0 +1,38 @@ +licenses(["notice"]) # Apache 2 + +# HTTP inspector filter for sniffing HTTP protocol and setting HTTP version to a FilterChain. + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "http_inspector_lib", + srcs = ["http_inspector.cc"], + hdrs = ["http_inspector.h"], + deps = [ + "//include/envoy/event:dispatcher_interface", + "//include/envoy/event:timer_interface", + "//include/envoy/network:filter_interface", + "//include/envoy/network:listen_socket_interface", + "//source/common/api:os_sys_calls_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/http:headers_lib", + "//source/extensions/transport_sockets:well_known_names", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + deps = [ + ":http_inspector_lib", + "//include/envoy/registry", + "//include/envoy/server:filter_config_interface", + "//source/extensions/filters/listener:well_known_names", + ], +) diff --git a/source/extensions/filters/listener/http_inspector/config.cc b/source/extensions/filters/listener/http_inspector/config.cc new file mode 100644 index 000000000000..3a0833d6721f --- /dev/null +++ b/source/extensions/filters/listener/http_inspector/config.cc @@ -0,0 +1,43 @@ +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "extensions/filters/listener/http_inspector/http_inspector.h" +#include "extensions/filters/listener/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { +namespace HttpInspector { + +/** + * Config registration for the Http inspector filter. @see NamedNetworkFilterConfigFactory. + */ +class HttpInspectorConfigFactory : public Server::Configuration::NamedListenerFilterConfigFactory { +public: + // NamedListenerFilterConfigFactory + Network::ListenerFilterFactoryCb + createFilterFactoryFromProto(const Protobuf::Message&, + Server::Configuration::ListenerFactoryContext& context) override { + ConfigSharedPtr config(std::make_shared(context.scope())); + return [config](Network::ListenerFilterManager& filter_manager) -> void { + filter_manager.addAcceptFilter(std::make_unique(config)); + }; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() override { return ListenerFilterNames::get().HttpInspector; } +}; + +/** + * Static registration for the http inspector filter. @see RegisterFactory. + */ +REGISTER_FACTORY(HttpInspectorConfigFactory, + Server::Configuration::NamedListenerFilterConfigFactory); + +} // namespace HttpInspector +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/listener/http_inspector/http_inspector.cc b/source/extensions/filters/listener/http_inspector/http_inspector.cc new file mode 100644 index 000000000000..332149a86813 --- /dev/null +++ b/source/extensions/filters/listener/http_inspector/http_inspector.cc @@ -0,0 +1,145 @@ +#include "extensions/filters/listener/http_inspector/http_inspector.h" + +#include "envoy/event/dispatcher.h" +#include "envoy/network/listen_socket.h" +#include "envoy/stats/scope.h" + +#include "common/api/os_sys_calls_impl.h" +#include "common/common/assert.h" +#include "common/common/macros.h" +#include "common/http/headers.h" + +#include "extensions/transport_sockets/well_known_names.h" + +#include "absl/strings/match.h" +#include "absl/strings/str_split.h" + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { +namespace HttpInspector { + +Config::Config(Stats::Scope& scope) + : stats_{ALL_HTTP_INSPECTOR_STATS(POOL_COUNTER_PREFIX(scope, "http_inspector."))} {} + +const absl::string_view Filter::HTTP2_CONNECTION_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; +thread_local uint8_t Filter::buf_[Config::MAX_INSPECT_SIZE]; + +Filter::Filter(const ConfigSharedPtr config) : config_(config) {} + +Network::FilterStatus Filter::onAccept(Network::ListenerFilterCallbacks& cb) { + ENVOY_LOG(debug, "http inspector: new connection accepted"); + + Network::ConnectionSocket& socket = cb.socket(); + + absl::string_view transport_protocol = socket.detectedTransportProtocol(); + if (!transport_protocol.empty() && + transport_protocol != TransportSockets::TransportSocketNames::get().RawBuffer) { + ENVOY_LOG(trace, "http inspector: cannot inspect http protocol with transport socket {}", + transport_protocol); + return Network::FilterStatus::Continue; + } + + ASSERT(file_event_ == nullptr); + + file_event_ = cb.dispatcher().createFileEvent( + socket.ioHandle().fd(), + [this](uint32_t events) { + ASSERT(events == Event::FileReadyType::Read); + onRead(); + }, + Event::FileTriggerType::Edge, Event::FileReadyType::Read); + + cb_ = &cb; + return Network::FilterStatus::StopIteration; +} + +void Filter::onRead() { + auto& os_syscalls = Api::OsSysCallsSingleton::get(); + const Api::SysCallSizeResult result = + os_syscalls.recv(cb_->socket().ioHandle().fd(), buf_, Config::MAX_INSPECT_SIZE, MSG_PEEK); + ENVOY_LOG(trace, "http inspector: recv: {}", result.rc_); + if (result.rc_ == -1 && result.errno_ == EAGAIN) { + return; + } else if (result.rc_ < 0) { + config_->stats().read_error_.inc(); + done(false); + return; + } + + parseHttpHeader(absl::string_view(reinterpret_cast(buf_), result.rc_)); +} + +void Filter::parseHttpHeader(absl::string_view data) { + if (absl::StartsWith(data, Filter::HTTP2_CONNECTION_PREFACE)) { + ENVOY_LOG(trace, "http inspector: http2 connection preface found"); + protocol_ = "HTTP/2"; + done(true); + } else { + size_t pos = data.find_first_of("\r\n"); + + // Cannot find \r or \n + if (pos == absl::string_view::npos) { + ENVOY_LOG(trace, "http inspector: no request line detected"); + return done(false); + } + + absl::string_view request_line = data.substr(0, pos); + std::vector fields = absl::StrSplit(request_line, absl::MaxSplits(' ', 4)); + + // Method SP Request-URI SP HTTP-Version + if (fields.size() != 3) { + ENVOY_LOG(trace, "http inspector: invalid http1x request line"); + return done(false); + } + + if (httpProtocols().count(fields[2]) == 0) { + ENVOY_LOG(trace, "http inspector: protocol: {} not valid", fields[2]); + return done(false); + } + + ENVOY_LOG(trace, "http inspector: method: {}, request uri: {}, protocol: {}", fields[0], + fields[1], fields[2]); + + protocol_ = fields[2]; + done(true); + } +} + +void Filter::done(bool success) { + ENVOY_LOG(trace, "http inspector: done: {}", success); + + if (success) { + absl::string_view protocol; + if (protocol_ == Http::Headers::get().ProtocolStrings.Http10String) { + config_->stats().http10_found_.inc(); + protocol = "http/1.0"; + } else if (protocol_ == Http::Headers::get().ProtocolStrings.Http11String) { + config_->stats().http11_found_.inc(); + protocol = "http/1.1"; + } else { + ASSERT(protocol_ == "HTTP/2"); + config_->stats().http2_found_.inc(); + protocol = "h2"; + } + + cb_->socket().setRequestedApplicationProtocols({protocol}); + } else { + config_->stats().http_not_found_.inc(); + } + + file_event_.reset(); + // Do not skip following listener filters. + cb_->continueFilterChain(true); +} + +const absl::flat_hash_set& Filter::httpProtocols() const { + CONSTRUCT_ON_FIRST_USE(absl::flat_hash_set, + Http::Headers::get().ProtocolStrings.Http10String, + Http::Headers::get().ProtocolStrings.Http11String); +} + +} // namespace HttpInspector +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/listener/http_inspector/http_inspector.h b/source/extensions/filters/listener/http_inspector/http_inspector.h new file mode 100644 index 000000000000..cb284910a0d1 --- /dev/null +++ b/source/extensions/filters/listener/http_inspector/http_inspector.h @@ -0,0 +1,83 @@ +#pragma once + +#include "envoy/event/file_event.h" +#include "envoy/event/timer.h" +#include "envoy/network/filter.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" + +#include "common/common/logger.h" + +#include "absl/container/flat_hash_set.h" + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { +namespace HttpInspector { + +/** + * All stats for the http inspector. @see stats_macros.h + */ +#define ALL_HTTP_INSPECTOR_STATS(COUNTER) \ + COUNTER(read_error) \ + COUNTER(http10_found) \ + COUNTER(http11_found) \ + COUNTER(http2_found) \ + COUNTER(http_not_found) + +/** + * Definition of all stats for the Http inspector. @see stats_macros.h + */ +struct HttpInspectorStats { + ALL_HTTP_INSPECTOR_STATS(GENERATE_COUNTER_STRUCT) +}; + +/** + * Global configuration for http inspector. + */ +class Config { +public: + Config(Stats::Scope& scope); + + const HttpInspectorStats& stats() const { return stats_; } + + static constexpr uint32_t MAX_INSPECT_SIZE = 8192; + +private: + HttpInspectorStats stats_; +}; + +using ConfigSharedPtr = std::shared_ptr; + +/** + * Http inspector listener filter. + */ +class Filter : public Network::ListenerFilter, Logger::Loggable { +public: + Filter(const ConfigSharedPtr config); + + // Network::ListenerFilter + Network::FilterStatus onAccept(Network::ListenerFilterCallbacks& cb) override; + +private: + static const absl::string_view HTTP2_CONNECTION_PREFACE; + + void onRead(); + void done(bool success); + void parseHttpHeader(absl::string_view data); + + const absl::flat_hash_set& httpProtocols() const; + + ConfigSharedPtr config_; + Network::ListenerFilterCallbacks* cb_{nullptr}; + Event::FileEventPtr file_event_; + absl::string_view protocol_; + + // Use static thread_local to avoid allocating buffer over and over again. + static thread_local uint8_t buf_[Config::MAX_INSPECT_SIZE]; +}; + +} // namespace HttpInspector +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/listener/well_known_names.h b/source/extensions/filters/listener/well_known_names.h index f7745ed2bba8..fa12272c598e 100644 --- a/source/extensions/filters/listener/well_known_names.h +++ b/source/extensions/filters/listener/well_known_names.h @@ -14,6 +14,8 @@ namespace ListenerFilters { */ class ListenerFilterNameValues { public: + // HTTP Inspector listener filter + const std::string HttpInspector = "envoy.listener.http_inspector"; // Original destination listener filter const std::string OriginalDst = "envoy.listener.original_dst"; // Original source listener filter diff --git a/test/extensions/filters/listener/http_inspector/BUILD b/test/extensions/filters/listener/http_inspector/BUILD new file mode 100644 index 000000000000..fcce5eea9449 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/BUILD @@ -0,0 +1,43 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "http_inspector_test", + srcs = ["http_inspector_test.cc"], + extension_name = "envoy.filters.listener.http_inspector", + deps = [ + "//source/common/common:hex_lib", + "//source/extensions/filters/listener/http_inspector:http_inspector_lib", + "//test/mocks/api:api_mocks", + "//test/mocks/network:network_mocks", + "//test/mocks/stats:stats_mocks", + "//test/test_common:threadsafe_singleton_injector_lib", + ], +) + +envoy_extension_cc_test( + name = "http_inspector_config_test", + srcs = ["http_inspector_config_test.cc"], + extension_name = "envoy.filters.listener.http_inspector", + deps = [ + "//source/extensions/filters/listener:well_known_names", + "//source/extensions/filters/listener/http_inspector:config", + "//source/extensions/filters/listener/http_inspector:http_inspector_lib", + "//test/mocks/api:api_mocks", + "//test/mocks/network:network_mocks", + "//test/mocks/server:server_mocks", + "//test/mocks/stats:stats_mocks", + "//test/test_common:threadsafe_singleton_injector_lib", + ], +) diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_config_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_config_test.cc new file mode 100644 index 000000000000..4e647709f7d8 --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_config_test.cc @@ -0,0 +1,52 @@ +#include "extensions/filters/listener/http_inspector/http_inspector.h" +#include "extensions/filters/listener/well_known_names.h" + +#include "test/mocks/server/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::Invoke; + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { +namespace HttpInspector { +namespace { + +TEST(HttpInspectorConfigFactoryTest, TestCreateFactory) { + Server::Configuration::NamedListenerFilterConfigFactory* factory = + Registry::FactoryRegistry:: + getFactory(ListenerFilters::ListenerFilterNames::get().HttpInspector); + + EXPECT_EQ(factory->name(), ListenerFilters::ListenerFilterNames::get().HttpInspector); + + std::string yaml = R"EOF( + {} +)EOF"; + + ProtobufTypes::MessagePtr proto_config = factory->createEmptyConfigProto(); + TestUtility::loadFromYaml(yaml, *proto_config); + + Server::Configuration::MockListenerFactoryContext context; + EXPECT_CALL(context, scope()).Times(1); + Network::ListenerFilterFactoryCb cb = + factory->createFilterFactoryFromProto(*proto_config, context); + + Network::MockListenerFilterManager manager; + Network::ListenerFilterPtr added_filter; + EXPECT_CALL(manager, addAcceptFilter_(_)) + .WillOnce(Invoke([&added_filter](Network::ListenerFilterPtr& filter) { + added_filter = std::move(filter); + })); + cb(manager); + + // Make sure we actually create the correct type! + EXPECT_NE(dynamic_cast(added_filter.get()), nullptr); +} + +} // namespace +} // namespace HttpInspector +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc new file mode 100644 index 000000000000..f1aa3012c7bf --- /dev/null +++ b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc @@ -0,0 +1,286 @@ +#include "common/common/hex.h" +#include "common/network/io_socket_handle_impl.h" + +#include "extensions/filters/listener/http_inspector/http_inspector.h" + +#include "test/mocks/api/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/stats/mocks.h" +#include "test/test_common/threadsafe_singleton_injector.h" + +#include "gtest/gtest.h" + +using testing::_; +using testing::AtLeast; +using testing::Eq; +using testing::InSequence; +using testing::Invoke; +using testing::InvokeWithoutArgs; +using testing::NiceMock; +using testing::Return; +using testing::ReturnNew; +using testing::ReturnRef; +using testing::SaveArg; + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { +namespace HttpInspector { +namespace { + +class HttpInspectorTest : public testing::Test { +public: + HttpInspectorTest() + : cfg_(std::make_shared(store_)), + io_handle_(std::make_unique(42)) {} + ~HttpInspectorTest() { io_handle_->close(); } + + void init() { + filter_ = std::make_unique(cfg_); + + EXPECT_CALL(cb_, socket()).WillRepeatedly(ReturnRef(socket_)); + EXPECT_CALL(socket_, detectedTransportProtocol()).WillRepeatedly(Return("raw_buffer")); + EXPECT_CALL(cb_, dispatcher()).WillRepeatedly(ReturnRef(dispatcher_)); + EXPECT_CALL(socket_, ioHandle()).WillRepeatedly(ReturnRef(*io_handle_)); + + EXPECT_CALL(dispatcher_, + createFileEvent_(_, _, Event::FileTriggerType::Edge, Event::FileReadyType::Read)) + .WillOnce( + DoAll(SaveArg<1>(&file_event_callback_), ReturnNew>())); + filter_->onAccept(cb_); + } + + NiceMock os_sys_calls_; + TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; + Stats::IsolatedStoreImpl store_; + ConfigSharedPtr cfg_; + std::unique_ptr filter_; + Network::MockListenerFilterCallbacks cb_; + Network::MockConnectionSocket socket_; + NiceMock dispatcher_; + Event::FileReadyCb file_event_callback_; + Network::IoHandlePtr io_handle_; +}; + +TEST_F(HttpInspectorTest, SkipHttpInspectForTLS) { + filter_ = std::make_unique(cfg_); + + EXPECT_CALL(cb_, socket()).WillRepeatedly(ReturnRef(socket_)); + EXPECT_CALL(socket_, detectedTransportProtocol()).WillRepeatedly(Return("TLS")); + EXPECT_EQ(filter_->onAccept(cb_), Network::FilterStatus::Continue); +} + +TEST_F(HttpInspectorTest, InspectHttp10) { + init(); + absl::string_view header = + "GET /anything HTTP/1.0\r\nhost: google.com\r\nuser-agent: curl/7.64.0\r\naccept: " + "*/*\r\nx-forwarded-proto: http\r\nx-request-id: " + "a52df4a0-ed00-4a19-86a7-80e5049c6c84\r\nx-envoy-expected-rq-timeout-ms: " + "15000\r\ncontent-length: 0\r\n\r\n"; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= header.size()); + memcpy(buffer, header.data(), header.size()); + return Api::SysCallSizeResult{ssize_t(header.size()), 0}; + })); + + const std::vector alpn_protos{absl::string_view("http/1.0")}; + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(alpn_protos)); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().http10_found_.value()); +} + +TEST_F(HttpInspectorTest, InspectHttp11) { + init(); + absl::string_view header = + "GET /anything HTTP/1.1\r\nhost: google.com\r\nuser-agent: curl/7.64.0\r\naccept: " + "*/*\r\nx-forwarded-proto: http\r\nx-request-id: " + "a52df4a0-ed00-4a19-86a7-80e5049c6c84\r\nx-envoy-expected-rq-timeout-ms: " + "15000\r\ncontent-length: 0\r\n\r\n"; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= header.size()); + memcpy(buffer, header.data(), header.size()); + return Api::SysCallSizeResult{ssize_t(header.size()), 0}; + })); + + const std::vector alpn_protos{absl::string_view("http/1.1")}; + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(alpn_protos)); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().http11_found_.value()); +} + +TEST_F(HttpInspectorTest, InvalidHttpMethod) { + init(); + absl::string_view header = "BAD /anything HTTP/1.1\r\n"; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= header.size()); + memcpy(buffer, header.data(), header.size()); + return Api::SysCallSizeResult{ssize_t(header.size()), 0}; + })); + + const std::vector alpn_protos{absl::string_view("http/1.1")}; + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(alpn_protos)); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().http11_found_.value()); +} + +TEST_F(HttpInspectorTest, InvalidHttpRequestLine) { + init(); + absl::string_view header = "BAD /anything HTTP/1.1"; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= header.size()); + memcpy(buffer, header.data(), header.size()); + return Api::SysCallSizeResult{ssize_t(header.size()), 0}; + })); + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().http_not_found_.value()); +} + +TEST_F(HttpInspectorTest, UnsupportedHttpProtocol) { + init(); + absl::string_view header = "GET /anything HTTP/0.9\r\n"; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= header.size()); + memcpy(buffer, header.data(), header.size()); + return Api::SysCallSizeResult{ssize_t(header.size()), 0}; + })); + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().http_not_found_.value()); +} + +TEST_F(HttpInspectorTest, InvalidRequestLine) { + init(); + absl::string_view header = "GET /anything HTTP/1.1 BadRequestLine\r\n"; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= header.size()); + memcpy(buffer, header.data(), header.size()); + return Api::SysCallSizeResult{ssize_t(header.size()), 0}; + })); + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().http_not_found_.value()); +} + +TEST_F(HttpInspectorTest, InspectHttp2) { + init(); + + std::string header = + "505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00000c04000000000000041000000000020000000000" + "00040800000000000fff000100007d010500000001418aa0e41d139d09b8f0000f048860757a4ce6aa660582867a" + "8825b650c3abb8d2e053032a2f2a408df2b4a7b3c0ec90b22d5d8749ff839d29af4089f2b585ed6950958d279a18" + "9e03f1ca5582265f59a75b0ac3111959c7e49004908db6e83f4096f2b16aee7f4b17cd65224b22d6765926a4a7b5" + "2b528f840b60003f"; + std::vector data = Hex::decode(header); + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&data](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= data.size()); + memcpy(buffer, data.data(), data.size()); + return Api::SysCallSizeResult{ssize_t(data.size()), 0}; + })); + + const std::vector alpn_protos{absl::string_view("h2")}; + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(alpn_protos)); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().http2_found_.value()); +} + +TEST_F(HttpInspectorTest, InvalidConnectionPreface) { + init(); + + std::string header = "505249202a20485454502f322e300d0a"; + std::vector data = Hex::decode(header); + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&data](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= data.size()); + memcpy(buffer, data.data(), data.size()); + return Api::SysCallSizeResult{ssize_t(data.size()), 0}; + })); + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().http_not_found_.value()); +} + +TEST_F(HttpInspectorTest, ReadError) { + init(); + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)).WillOnce(InvokeWithoutArgs([]() { + return Api::SysCallSizeResult{ssize_t(-1), ENOTSUP}; + })); + EXPECT_CALL(cb_, continueFilterChain(true)); + file_event_callback_(Event::FileReadyType::Read); + EXPECT_EQ(1, cfg_->stats().read_error_.value()); +} + +TEST_F(HttpInspectorTest, MultipleReads) { + + init(); + const std::vector alpn_protos = {absl::string_view("h2")}; + + const std::string header = + "505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00000c04000000000000041000000000020000000000" + "00040800000000000fff000100007d010500000001418aa0e41d139d09b8f0000f048860757a4ce6aa660582867a" + "8825b650c3abb8d2e053032a2f2a408df2b4a7b3c0ec90b22d5d8749ff839d29af4089f2b585ed6950958d279a18" + "9e03f1ca5582265f59a75b0ac3111959c7e49004908db6e83f4096f2b16aee7f4b17cd65224b22d6765926a4a7b5" + "2b528f840b60003f"; + const std::vector data = Hex::decode(header); + { + InSequence s; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)).WillOnce(InvokeWithoutArgs([]() { + return Api::SysCallSizeResult{ssize_t(-1), EAGAIN}; + })); + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&data](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= data.size()); + memcpy(buffer, data.data(), data.size()); + return Api::SysCallSizeResult{ssize_t(data.size()), 0}; + })); + } + + bool got_continue = false; + EXPECT_CALL(socket_, setRequestedApplicationProtocols(alpn_protos)); + EXPECT_CALL(cb_, continueFilterChain(true)).WillOnce(InvokeWithoutArgs([&got_continue]() { + got_continue = true; + })); + while (!got_continue) { + file_event_callback_(Event::FileReadyType::Read); + } + EXPECT_EQ(1, cfg_->stats().http2_found_.value()); +} + +} // namespace +} // namespace HttpInspector +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index f7e6dfa0bbe6..051ce386f181 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -298,6 +298,7 @@ XDS decRefCount getaddrinfo sendto +ssize upcasts vip xDSes From 875960fecbb12b7586983d3a833eb8717087ad5f Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 19 Jul 2019 11:15:55 -0700 Subject: [PATCH 205/542] bazelci: fix failures in ubuntu 18.04 (#7648) Signed-off-by: Lizan Zhou --- .bazelci/presubmit.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 02b2cfc5dc4b..6c5da1e60f0b 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -1,8 +1,8 @@ --- tasks: - ubuntu_1804: - platform: ubuntu1804 + release: + platform: ubuntu1604 build_targets: - - "//source/..." + - "//source/exe:envoy-static" test_targets: - "//test/..." From b94a964ff8f1481e5d7f348a52b3c1349f1e50e4 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 19 Jul 2019 11:16:29 -0700 Subject: [PATCH 206/542] ci: full IPv6 tests in Azure pipelines (#7640) Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 35 ++++++++++++++++++++++------------- ci/WORKSPACE.filter.example | 4 ++++ ci/run_envoy_docker.sh | 8 ++++++-- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index d8a0ed07eca9..0515ec656aa5 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -1,36 +1,45 @@ -resources: - containers: - - container: envoy-build - image: envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d - jobs: -- job: EnvoyFullTest +- job: bazel + strategy: + matrix: + release: + CI_TARGET: 'bazel.release' + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. timeoutInMinutes: 360 pool: vmImage: 'Ubuntu 16.04' - container: envoy-build steps: - bash: | echo "disk space at beginning of build:" df -h + displayName: "Check disk space at beginning" + + - bash: | + sudo mkdir -p /etc/docker + echo '{ + "ipv6": true, + "fixed-cidr-v6": "2001:db8:1::/64" + }' | sudo tee /etc/docker/daemon.json + sudo service docker restart + displayName: "Enable IPv6" - - script: ci/do_ci.sh bazel.release + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh $(CI_TARGET)' + workingDirectory: $(Build.SourcesDirectory) env: - ENVOY_SRCDIR: $(Build.SourcesDirectory) - BUILD_DIR: $(Build.StagingDirectory) + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-clang --config=remote-ci --jobs=100 --curses=no" - # //test/integration:echo_integration_test is marked exclusive so it will be run locally, no IPv6 locally in container. - BAZEL_EXTRA_TEST_OPTIONS: "--test_filter=-IpVersions/EchoIntegrationTest.*/IPv6" BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Run CI script" - bash: | echo "disk space at end of build:" df -h + displayName: "Check disk space at end" condition: always() - task: PublishBuildArtifacts@1 inputs: pathtoPublish: "$(Build.StagingDirectory)/envoy" - artifactName: 'envoy' + artifactName: $(CI_TARGET) diff --git a/ci/WORKSPACE.filter.example b/ci/WORKSPACE.filter.example index a4d245957f5c..f11c8e7911be 100644 --- a/ci/WORKSPACE.filter.example +++ b/ci/WORKSPACE.filter.example @@ -19,3 +19,7 @@ rules_foreign_cc_dependencies() load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") go_rules_dependencies() go_register_toolchains(go_version = GO_VERSION) + +load("@envoy//bazel/toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config") + +rbe_toolchains_config() diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index c6b91fae5f09..af828943fb53 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -18,10 +18,14 @@ USER_GROUP=root [[ -f .git ]] && [[ ! -d .git ]] && GIT_VOLUME_OPTION="-v $(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)" +[[ -t 1 ]] && DOCKER_TTY_OPTION=-it + mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" # Since we specify an explicit hash, docker-run will pull from the remote repo if missing. -docker run --rm -t -i -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} \ +docker run --rm ${DOCKER_TTY_OPTION} -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} \ -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build ${GIT_VOLUME_OPTION} \ - -v "$PWD":/source -e NUM_CPUS --cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN "${IMAGE_NAME}":"${IMAGE_ID}" \ + -e BAZEL_BUILD_EXTRA_OPTIONS -e BAZEL_EXTRA_TEST_OPTIONS -e BAZEL_REMOTE_CACHE \ + -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS \ + -v "$PWD":/source --cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN "${IMAGE_NAME}":"${IMAGE_ID}" \ /bin/bash -lc "groupadd --gid $(id -g) -f envoygroup && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home \ --home-dir /source envoybuild && usermod -a -G pcap envoybuild && su envoybuild -c \"cd source && $*\"" From 8d2eb9d81411d25a1a3b3101ece780afcc18c521 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 19 Jul 2019 11:24:19 -0700 Subject: [PATCH 207/542] coverage: shard coverage tests (#7635) Signed-off-by: Lizan Zhou --- test/coverage/gen_build.sh | 10 +++++++--- test/run_envoy_bazel_coverage.sh | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/coverage/gen_build.sh b/test/coverage/gen_build.sh index 5998e3eefd49..fb79e074745a 100755 --- a/test/coverage/gen_build.sh +++ b/test/coverage/gen_build.sh @@ -67,10 +67,14 @@ EOF done cat << EOF ], - tags = ["manual"], + # no-remote due to https://github.com/bazelbuild/bazel/issues/4685 + tags = ["manual", "no-remote"], coverage = False, - # Needed when invoking external shell tests etc. - local = True, + # Due to the nature of coverage_tests, the shard of coverage_tests are very uneven, some of + # shard can take 100s and some takes only 10s, so we use the maximum sharding to here to let + # Bazel scheduling them across CPU cores. + # Sharding can be disabled by --test_sharding_strategy=disabled. + shard_count = 50, ) EOF diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index a3542492ef2c..08c63609b661 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -27,7 +27,7 @@ SCRIPT_DIR="$(realpath "$(dirname "$0")")" BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \ - --test_timeout=2000 --cxxopt="-DENVOY_CONFIG_COVERAGE=1" --test_output=streamed \ + --test_timeout=2000 --cxxopt="-DENVOY_CONFIG_COVERAGE=1" --test_output=errors \ --test_arg="--log-path /dev/null" --test_arg="-l trace" --test_env=HEAPCHECK= \ //test/coverage:coverage_tests @@ -36,7 +36,10 @@ mkdir -p "${COVERAGE_DIR}" COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp)" COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests" -COVERAGE_DATA="bazel-out/k8-fastbuild/testlogs/test/coverage/coverage_tests/coverage.dat" +COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat" + +echo "Merging coverage data..." +llvm-profdata merge -sparse -o ${COVERAGE_DATA} $(find -L bazel-out/k8-fastbuild/testlogs/test/coverage/coverage_tests/ -name coverage.dat) echo "Generating report..." llvm-cov show "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" -Xdemangler=c++filt \ From 6feb15c6c7203f19b8b507cd0fab76a5b0b9cb2a Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Fri, 19 Jul 2019 14:26:10 -0400 Subject: [PATCH 208/542] test: Use an env-var rather than compile define to control exact memory comparisons. (#7642) * Use an env-var rather than compile define to control exact memory comparisons. Signed-off-by: Joshua Marantz --- ci/do_ci.sh | 2 +- test/common/stats/stat_test_utility.cc | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 030b413e30a8..3ca2968b2c45 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -97,7 +97,7 @@ if [[ "$CI_TARGET" == "bazel.release" ]]; then # toolchain is kept consistent. This ifdef is checked in # test/common/stats/stat_test_utility.cc when computing # Stats::TestUtil::MemoryTest::mode(). - BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --cxxopt=-DMEMORY_TEST_EXACT=1" + BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --test_env=ENVOY_MEMORY_TEST_EXACT=true" setup_clang_toolchain echo "bazel release build with tests..." diff --git a/test/common/stats/stat_test_utility.cc b/test/common/stats/stat_test_utility.cc index 7a9241d2c5b4..ef13cdab3748 100644 --- a/test/common/stats/stat_test_utility.cc +++ b/test/common/stats/stat_test_utility.cc @@ -118,18 +118,19 @@ MemoryTest::Mode MemoryTest::mode() { const size_t end_mem = Memory::Stats::totalCurrentlyAllocated(); bool can_measure_memory = end_mem > start_mem; -#if defined(MEMORY_TEST_EXACT) // Set in "ci/do_ci.sh" for 'release' tests. - RELEASE_ASSERT(can_measure_memory, "Compilation is set up for canonical memory measurements, " - "but memory measurement looks broken"); - return Mode::Canonical; -#else - // Different versions of STL and other compiler/architecture differences may - // also impact memory usage, so when not compiling with MEMORY_TEST_EXACT, - // memory comparisons must be given some slack. There have recently emerged - // some memory-allocation differences between development and Envoy CI and - // Bazel CI (which compiles Envoy as a test of Bazel). - return can_measure_memory ? Mode::Approximate : Mode::Disabled; -#endif + if (getenv("ENVOY_MEMORY_TEST_EXACT") != nullptr) { // Set in "ci/do_ci.sh" for 'release' tests. + RELEASE_ASSERT(can_measure_memory, + "$ENVOY_MEMORY_TEST_EXACT is set for canonical memory measurements, " + "but memory measurement looks broken"); + return Mode::Canonical; + } else { + // Different versions of STL and other compiler/architecture differences may + // also impact memory usage, so when not compiling with MEMORY_TEST_EXACT, + // memory comparisons must be given some slack. There have recently emerged + // some memory-allocation differences between development and Envoy CI and + // Bazel CI (which compiles Envoy as a test of Bazel). + return can_measure_memory ? Mode::Approximate : Mode::Disabled; + } #endif } From 6e6a7ccfeb57f79569feff2d7da128361fcacc33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Fri, 19 Jul 2019 16:27:33 -0400 Subject: [PATCH 209/542] server: more constness in server.cc (#7645) Risk LeveL :low Testing: existing Doc Changes: n/a Release Notes: n/a Signed-off-by: Raul Gutierrez Segales --- source/server/server.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/server/server.cc b/source/server/server.cc index 107e99beeb1f..f595909746c0 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -510,8 +510,9 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch void InstanceImpl::run() { // RunHelper exists primarily to facilitate testing of how we respond to early shutdown during // startup (see RunHelperTest in server_test.cc). - auto run_helper = RunHelper(*this, options_, *dispatcher_, clusterManager(), access_log_manager_, - init_manager_, overloadManager(), [this] { startWorkers(); }); + const auto run_helper = + RunHelper(*this, options_, *dispatcher_, clusterManager(), access_log_manager_, init_manager_, + overloadManager(), [this] { startWorkers(); }); // Run the main dispatch loop waiting to exit. ENVOY_LOG(info, "starting main dispatch loop"); @@ -598,7 +599,7 @@ InstanceImpl::registerCallback(Stage stage, StageCallbackWithCompletion callback void InstanceImpl::notifyCallbacksForStage(Stage stage, Event::PostCb completion_cb) { ASSERT(std::this_thread::get_id() == main_thread_id_); - auto it = stage_callbacks_.find(stage); + const auto it = stage_callbacks_.find(stage); if (it != stage_callbacks_.end()) { for (const StageCallback& callback : it->second) { callback(); @@ -619,7 +620,7 @@ void InstanceImpl::notifyCallbacksForStage(Stage stage, Event::PostCb completion // worker threads have not been started so we need to skip notifications if envoy is shutdown // early before workers have started. if (workers_started_) { - auto it2 = stage_completable_callbacks_.find(stage); + const auto it2 = stage_completable_callbacks_.find(stage); if (it2 != stage_completable_callbacks_.end()) { ENVOY_LOG(info, "Notifying {} callback(s) with completion.", it2->second.size()); for (const StageCallbackWithCompletion& callback : it2->second) { From 3b1a79f57d1ea2f774395803888d68dbcbb59275 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Fri, 19 Jul 2019 17:42:33 -0400 Subject: [PATCH 210/542] Add type nicknames for the absl::optional types for various stats for their find*() functions. (#7654) Description: The types for the findCounter() and sibling methods are quite awkward to type and read. Moreover https://google.github.io/styleguide/cppguide.html#auto discourages use of 'auto' for temp variables to receive the return value. These new type names are in the spirit of CounterSharedPtr et al. Risk Level: low Testing: //test/... Docs Changes: n/a Release Notes: n/a Signed-off-by: Joshua Marantz --- include/envoy/stats/scope.h | 11 ++++++----- include/envoy/stats/stats.h | 1 - source/common/stats/isolated_store_impl.h | 13 +++---------- source/common/stats/scope_prefixer.cc | 12 +++--------- source/common/stats/scope_prefixer.h | 7 +++---- source/common/stats/stat_merger.cc | 3 +-- source/common/stats/thread_local_store.cc | 9 +++------ source/common/stats/thread_local_store.h | 20 +++++++++----------- test/common/stats/stat_merger_test.cc | 2 +- test/integration/filters/eds_ready_filter.cc | 3 +-- test/integration/server.h | 14 ++++++-------- test/mocks/stats/mocks.h | 7 +++---- 12 files changed, 39 insertions(+), 63 deletions(-) diff --git a/include/envoy/stats/scope.h b/include/envoy/stats/scope.h index ef01045374f1..1d122ddf6544 100644 --- a/include/envoy/stats/scope.h +++ b/include/envoy/stats/scope.h @@ -19,6 +19,9 @@ class Histogram; class Scope; class NullGaugeImpl; +using OptionalCounter = absl::optional>; +using OptionalGauge = absl::optional>; +using OptionalHistogram = absl::optional>; using ScopePtr = std::unique_ptr; using ScopeSharedPtr = std::shared_ptr; @@ -93,22 +96,20 @@ class Scope { * @param The name of the stat, obtained from the SymbolTable. * @return a reference to a counter within the scope's namespace, if it exists. */ - virtual absl::optional> - findCounter(StatName name) const PURE; + virtual OptionalCounter findCounter(StatName name) const PURE; /** * @param The name of the stat, obtained from the SymbolTable. * @return a reference to a gauge within the scope's namespace, if it exists. */ - virtual absl::optional> findGauge(StatName name) const PURE; + virtual OptionalGauge findGauge(StatName name) const PURE; /** * @param The name of the stat, obtained from the SymbolTable. * @return a reference to a histogram within the scope's namespace, if it * exists. */ - virtual absl::optional> - findHistogram(StatName name) const PURE; + virtual OptionalHistogram findHistogram(StatName name) const PURE; /** * @return a reference to the symbol table. diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index a610a9d48f9c..21d19ffa21b4 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -10,7 +10,6 @@ #include "envoy/stats/symbol_table.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" namespace Envoy { namespace Stats { diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index ce6ddaca703b..6a11ceec2a7c 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -99,16 +99,9 @@ class IsolatedStoreImpl : public StoreImpl { Histogram& histogram = histograms_.get(name); return histogram; } - absl::optional> findCounter(StatName name) const override { - return counters_.find(name); - } - absl::optional> findGauge(StatName name) const override { - return gauges_.find(name); - } - absl::optional> - findHistogram(StatName name) const override { - return histograms_.find(name); - } + OptionalCounter findCounter(StatName name) const override { return counters_.find(name); } + OptionalGauge findGauge(StatName name) const override { return gauges_.find(name); } + OptionalHistogram findHistogram(StatName name) const override { return histograms_.find(name); } // Stats::Store std::vector counters() const override { return counters_.toVector(); } diff --git a/source/common/stats/scope_prefixer.cc b/source/common/stats/scope_prefixer.cc index 4efb49058909..f83265b0511e 100644 --- a/source/common/stats/scope_prefixer.cc +++ b/source/common/stats/scope_prefixer.cc @@ -44,17 +44,11 @@ Histogram& ScopePrefixer::histogramFromStatName(StatName name) { return scope_.histogramFromStatName(StatName(stat_name_storage.get())); } -absl::optional> -ScopePrefixer::findCounter(StatName name) const { - return scope_.findCounter(name); -} +OptionalCounter ScopePrefixer::findCounter(StatName name) const { return scope_.findCounter(name); } -absl::optional> ScopePrefixer::findGauge(StatName name) const { - return scope_.findGauge(name); -} +OptionalGauge ScopePrefixer::findGauge(StatName name) const { return scope_.findGauge(name); } -absl::optional> -ScopePrefixer::findHistogram(StatName name) const { +OptionalHistogram ScopePrefixer::findHistogram(StatName name) const { return scope_.findHistogram(name); } diff --git a/source/common/stats/scope_prefixer.h b/source/common/stats/scope_prefixer.h index 5d775c539b2e..d4c63df99d8b 100644 --- a/source/common/stats/scope_prefixer.h +++ b/source/common/stats/scope_prefixer.h @@ -35,10 +35,9 @@ class ScopePrefixer : public Scope { return histogramFromStatName(storage.statName()); } - absl::optional> findCounter(StatName name) const override; - absl::optional> findGauge(StatName name) const override; - absl::optional> - findHistogram(StatName name) const override; + OptionalCounter findCounter(StatName name) const override; + OptionalGauge findGauge(StatName name) const override; + OptionalHistogram findHistogram(StatName name) const override; const SymbolTable& constSymbolTable() const override { return scope_.constSymbolTable(); } virtual SymbolTable& symbolTable() override { return scope_.symbolTable(); } diff --git a/source/common/stats/stat_merger.cc b/source/common/stats/stat_merger.cc index db2e7d49a319..a53e32d93fbc 100644 --- a/source/common/stats/stat_merger.cc +++ b/source/common/stats/stat_merger.cc @@ -35,8 +35,7 @@ void StatMerger::mergeGauges(const Protobuf::Map& gauges) StatNameManagedStorage storage(gauge.first, temp_scope_->symbolTable()); StatName stat_name = storage.statName(); - absl::optional> gauge_opt = - temp_scope_->findGauge(stat_name); + OptionalGauge gauge_opt = temp_scope_->findGauge(stat_name); Gauge::ImportMode import_mode = Gauge::ImportMode::Uninitialized; if (gauge_opt) { diff --git a/source/common/stats/thread_local_store.cc b/source/common/stats/thread_local_store.cc index 155cfefda757..9ae746cb2c04 100644 --- a/source/common/stats/thread_local_store.cc +++ b/source/common/stats/thread_local_store.cc @@ -513,18 +513,15 @@ Histogram& ThreadLocalStoreImpl::ScopeImpl::histogramFromStatName(StatName name) return **central_ref; } -absl::optional> -ThreadLocalStoreImpl::ScopeImpl::findCounter(StatName name) const { +OptionalCounter ThreadLocalStoreImpl::ScopeImpl::findCounter(StatName name) const { return findStatLockHeld(name, central_cache_.counters_); } -absl::optional> -ThreadLocalStoreImpl::ScopeImpl::findGauge(StatName name) const { +OptionalGauge ThreadLocalStoreImpl::ScopeImpl::findGauge(StatName name) const { return findStatLockHeld(name, central_cache_.gauges_); } -absl::optional> -ThreadLocalStoreImpl::ScopeImpl::findHistogram(StatName name) const { +OptionalHistogram ThreadLocalStoreImpl::ScopeImpl::findHistogram(StatName name) const { auto iter = central_cache_.histograms_.find(name); if (iter == central_cache_.histograms_.end()) { return absl::nullopt; diff --git a/source/common/stats/thread_local_store.h b/source/common/stats/thread_local_store.h index b10e7babac88..b4bd9766f9a4 100644 --- a/source/common/stats/thread_local_store.h +++ b/source/common/stats/thread_local_store.h @@ -166,8 +166,8 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo const SymbolTable& constSymbolTable() const override { return alloc_.constSymbolTable(); } SymbolTable& symbolTable() override { return alloc_.symbolTable(); } const TagProducer& tagProducer() const { return *tag_producer_; } - absl::optional> findCounter(StatName name) const override { - absl::optional> found_counter; + OptionalCounter findCounter(StatName name) const override { + OptionalCounter found_counter; Thread::LockGuard lock(lock_); for (ScopeImpl* scope : scopes_) { found_counter = scope->findCounter(name); @@ -177,8 +177,8 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo } return absl::nullopt; } - absl::optional> findGauge(StatName name) const override { - absl::optional> found_gauge; + OptionalGauge findGauge(StatName name) const override { + OptionalGauge found_gauge; Thread::LockGuard lock(lock_); for (ScopeImpl* scope : scopes_) { found_gauge = scope->findGauge(name); @@ -188,9 +188,8 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo } return absl::nullopt; } - absl::optional> - findHistogram(StatName name) const override { - absl::optional> found_histogram; + OptionalHistogram findHistogram(StatName name) const override { + OptionalHistogram found_histogram; Thread::LockGuard lock(lock_); for (ScopeImpl* scope : scopes_) { found_histogram = scope->findHistogram(name); @@ -274,10 +273,9 @@ class ThreadLocalStoreImpl : Logger::Loggable, public StoreRo // NOTE: The find methods assume that `name` is fully-qualified. // Implementations will not add the scope prefix. - absl::optional> findCounter(StatName name) const override; - absl::optional> findGauge(StatName name) const override; - absl::optional> - findHistogram(StatName name) const override; + OptionalCounter findCounter(StatName name) const override; + OptionalGauge findGauge(StatName name) const override; + OptionalHistogram findHistogram(StatName name) const override; template using MakeStatFn = std::function(Allocator&, StatName name, diff --git a/test/common/stats/stat_merger_test.cc b/test/common/stats/stat_merger_test.cc index 9ebac5f2a408..f56e8d98c718 100644 --- a/test/common/stats/stat_merger_test.cc +++ b/test/common/stats/stat_merger_test.cc @@ -196,7 +196,7 @@ TEST_F(StatMergerThreadLocalTest, filterOutUninitializedGauges) { // We don't get "newgauge1" in the aggregated list, but we *do* get it if we try to // find it by name. - absl::optional> find = store_.findGauge(g1.statName()); + OptionalGauge find = store_.findGauge(g1.statName()); ASSERT_TRUE(find); EXPECT_EQ(&g1, &(find->get())); } diff --git a/test/integration/filters/eds_ready_filter.cc b/test/integration/filters/eds_ready_filter.cc index c1fbf66e9b53..665c0232161c 100644 --- a/test/integration/filters/eds_ready_filter.cc +++ b/test/integration/filters/eds_ready_filter.cc @@ -20,8 +20,7 @@ class EdsReadyFilter : public Http::PassThroughFilter { EdsReadyFilter(const Stats::Scope& root_scope, Stats::SymbolTable& symbol_table) : root_scope_(root_scope), stat_name_("cluster.cluster_0.membership_healthy", symbol_table) {} Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap&, bool) override { - absl::optional> gauge = - root_scope_.findGauge(stat_name_.statName()); + Stats::OptionalGauge gauge = root_scope_.findGauge(stat_name_.statName()); if (!gauge.has_value()) { decoder_callbacks_->sendLocalReply(Envoy::Http::Code::InternalServerError, "Couldn't find stat", nullptr, absl::nullopt, ""); diff --git a/test/integration/server.h b/test/integration/server.h index 1b77f511fbae..3cc7489cad55 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -109,16 +109,15 @@ class TestScopeWrapper : public Scope { return histogramFromStatName(storage.statName()); } - absl::optional> findCounter(StatName name) const override { + OptionalCounter findCounter(StatName name) const override { Thread::LockGuard lock(lock_); return wrapped_scope_->findCounter(name); } - absl::optional> findGauge(StatName name) const override { + OptionalGauge findGauge(StatName name) const override { Thread::LockGuard lock(lock_); return wrapped_scope_->findGauge(name); } - absl::optional> - findHistogram(StatName name) const override { + OptionalHistogram findHistogram(StatName name) const override { Thread::LockGuard lock(lock_); return wrapped_scope_->findHistogram(name); } @@ -170,16 +169,15 @@ class TestIsolatedStoreImpl : public StoreRoot { Thread::LockGuard lock(lock_); return store_.histogram(name); } - absl::optional> findCounter(StatName name) const override { + OptionalCounter findCounter(StatName name) const override { Thread::LockGuard lock(lock_); return store_.findCounter(name); } - absl::optional> findGauge(StatName name) const override { + OptionalGauge findGauge(StatName name) const override { Thread::LockGuard lock(lock_); return store_.findGauge(name); } - absl::optional> - findHistogram(StatName name) const override { + OptionalHistogram findHistogram(StatName name) const override { Thread::LockGuard lock(lock_); return store_.findHistogram(name); } diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index 987e86c43fa7..d418ad430f2b 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -271,10 +271,9 @@ class MockStore : public SymbolTableProvider, public StoreImpl { MOCK_METHOD1(histogram, Histogram&(const std::string& name)); MOCK_CONST_METHOD0(histograms, std::vector()); - MOCK_CONST_METHOD1(findCounter, absl::optional>(StatName)); - MOCK_CONST_METHOD1(findGauge, absl::optional>(StatName)); - MOCK_CONST_METHOD1(findHistogram, - absl::optional>(StatName)); + MOCK_CONST_METHOD1(findCounter, OptionalCounter(StatName)); + MOCK_CONST_METHOD1(findGauge, OptionalGauge(StatName)); + MOCK_CONST_METHOD1(findHistogram, OptionalHistogram(StatName)); Counter& counterFromStatName(StatName name) override { return counter(symbol_table_->toString(name)); From 99a34dc40a21499df84f80296e272b77506245ed Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Fri, 19 Jul 2019 22:02:33 -0700 Subject: [PATCH 211/542] clang-tidy: modernize-use-override (#7658) Description: Adds `modernize-use-override` as an error and fixes existing issues (automatically applied using `-fix`). https://clang.llvm.org/extra/clang-tidy/checks/modernize-use-override.html Risk Level: Low Testing: Existing Docs Changes: N/A Release Notes: N/A Signed-off-by: Derek Argueta --- .clang-tidy | 1 + include/envoy/stats/stats.h | 2 +- .../access_log/access_log_manager_impl.h | 2 +- source/common/common/assert.cc | 2 +- source/common/config/grpc_mux_impl.h | 2 +- source/common/config/remote_data_fetcher.h | 2 +- source/common/event/dispatcher_impl.h | 2 +- .../posix/directory_iterator_impl.h | 2 +- source/common/grpc/google_async_client_impl.h | 4 +- source/common/http/codec_client.h | 2 +- source/common/http/http1/codec_impl.h | 2 +- source/common/http/http2/codec_impl.h | 2 +- source/common/http/http2/conn_pool.h | 4 +- source/common/network/listen_socket_impl.h | 2 +- source/common/router/rds_impl.h | 2 +- source/common/router/retry_state_impl.h | 2 +- source/common/router/router.h | 4 +- source/common/stats/histogram_impl.h | 4 +- source/common/stats/scope_prefixer.h | 2 +- source/common/tcp/conn_pool.h | 10 ++-- source/common/tcp_proxy/tcp_proxy.h | 4 +- .../common/thread_local/thread_local_impl.h | 4 +- source/common/upstream/cluster_factory_impl.h | 2 +- source/common/upstream/cluster_manager_impl.h | 4 +- .../upstream/health_checker_base_impl.h | 2 +- source/common/upstream/load_balancer_impl.h | 2 +- .../common/upstream/outlier_detection_impl.h | 2 +- .../common/upstream/resource_manager_impl.h | 2 +- source/common/upstream/upstream_impl.h | 13 ++-- .../dynamic_forward_proxy/dns_cache_impl.h | 4 +- .../dns_cache_manager_impl.h | 4 +- .../common/ext_authz/ext_authz_grpc_impl.h | 2 +- .../common/ext_authz/ext_authz_http_impl.h | 2 +- .../original_src/original_src_socket_option.h | 2 +- .../filters/common/ratelimit/ratelimit_impl.h | 2 +- .../common/aws/credentials_provider_impl.h | 2 +- .../filters/http/common/jwks_fetcher.cc | 11 ++-- .../filters/http/fault/fault_filter.h | 2 +- .../transcoder_input_stream_impl.h | 4 +- .../header_to_metadata_filter.h | 2 +- .../http/ip_tagging/ip_tagging_filter.h | 2 +- .../network/common/redis/client_impl.h | 4 +- .../filters/network/dubbo_proxy/decoder.h | 2 +- .../dubbo_hessian2_serializer_impl.h | 16 ++--- .../network/dubbo_proxy/filters/filter.h | 8 +-- .../dubbo_proxy/router/route_matcher.h | 2 +- .../network/dubbo_proxy/router/router_impl.h | 2 +- .../filters/network/ext_authz/ext_authz.h | 2 +- .../extensions/filters/network/kafka/codec.h | 2 +- .../filters/network/mongo_proxy/proxy.h | 2 +- .../filters/network/rbac/rbac_filter.h | 2 +- .../redis_proxy/command_splitter_impl.h | 6 +- .../network/redis_proxy/conn_pool_impl.h | 2 +- .../network/redis_proxy/proxy_filter.h | 4 +- .../network/thrift_proxy/conn_manager.h | 4 +- .../network/thrift_proxy/thrift_object_impl.h | 4 +- .../grpc_credentials/example/config.h | 2 +- .../extensions/health_checkers/redis/redis.h | 2 +- .../stat_sinks/common/statsd/statsd.h | 4 +- .../extensions/tracers/common/factory_base.h | 4 +- .../tracers/lightstep/lightstep_tracer_impl.h | 2 +- .../transport_sockets/alts/config.cc | 2 +- .../transport_sockets/alts/tsi_handshaker.h | 2 +- .../transport_sockets/alts/tsi_socket.h | 2 +- .../tls/context_manager_impl.h | 2 +- source/server/config_validation/connection.h | 2 +- source/server/http/config_tracker_impl.h | 2 +- source/server/listener_manager_impl.h | 2 +- source/server/options_impl.h | 2 +- source/server/overload_manager_impl.cc | 4 +- .../config/delta_subscription_test_harness.h | 2 +- .../filesystem_subscription_test_harness.h | 2 +- .../config/http_subscription_test_harness.h | 2 +- test/common/config/metadata_test.cc | 5 +- test/common/event/dispatcher_impl_test.cc | 2 +- .../grpc/grpc_client_integration_test.cc | 2 +- .../grpc_client_integration_test_harness.h | 2 +- test/common/http/codec_client_test.cc | 2 +- test/common/http/common.h | 2 +- test/common/http/conn_manager_impl_test.cc | 2 +- test/common/http/http1/conn_pool_test.cc | 4 +- test/common/http/http2/conn_pool_test.cc | 2 +- test/common/network/connection_impl_test.cc | 12 ++-- .../network/filter_manager_impl_test.cc | 2 +- test/common/router/config_impl_test.cc | 4 +- test/common/router/rds_impl_test.cc | 6 +- test/common/secret/sds_api_test.cc | 2 +- test/common/signal/signals_test.cc | 2 +- test/common/singleton/manager_impl_test.cc | 2 +- .../singleton/threadsafe_singleton_test.cc | 2 +- test/common/stats/allocator_impl_test.cc | 2 +- test/common/stats/metric_impl_test.cc | 2 +- .../stream_info/filter_state_impl_test.cc | 2 +- test/common/tcp/conn_pool_test.cc | 8 +-- test/common/tcp_proxy/tcp_proxy_test.cc | 2 +- .../thread_local/thread_local_impl_test.cc | 2 +- .../upstream/cluster_manager_impl_test.cc | 4 +- test/common/upstream/upstream_impl_test.cc | 4 +- test/exe/main_common_test.cc | 2 +- test/extensions/clusters/redis/mocks.h | 2 +- .../dns_cache_impl_test.cc | 2 +- .../common/dynamic_forward_proxy/mocks.h | 12 ++-- test/extensions/common/tap/admin_test.cc | 4 +- test/extensions/common/tap/common.h | 4 +- .../filters/common/ext_authz/mocks.h | 4 +- .../extensions/filters/common/lua/lua_test.cc | 2 +- .../filters/common/lua/wrappers_test.cc | 4 +- .../filters/common/ratelimit/mocks.h | 2 +- .../common/ratelimit/ratelimit_impl_test.cc | 2 +- .../aws/credentials_provider_impl_test.cc | 8 +-- .../filters/http/common/aws/mocks.h | 4 +- .../common/aws/region_provider_impl_test.cc | 2 +- test/extensions/filters/http/common/mock.h | 2 +- .../proxy_filter_test.cc | 2 +- .../filters/http/dynamo/dynamo_filter_test.cc | 2 +- .../json_transcoder_filter_test.cc | 2 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 2 +- .../http/jwt_authn/filter_integration_test.cc | 3 +- test/extensions/filters/http/jwt_authn/mock.h | 2 +- .../filters/http/lua/lua_filter_test.cc | 2 +- .../filters/http/rbac/rbac_filter_test.cc | 2 +- .../http_inspector/http_inspector_test.cc | 2 +- .../tls_inspector/tls_inspector_test.cc | 2 +- .../client_ssl_auth/client_ssl_auth_test.cc | 2 +- .../network/common/redis/client_impl_test.cc | 2 +- .../filters/network/common/redis/mocks.h | 10 ++-- .../network/dubbo_proxy/conn_manager_test.cc | 4 +- .../network/dubbo_proxy/decoder_test.cc | 2 +- .../filters/network/dubbo_proxy/mocks.h | 30 +++++----- .../network/ext_authz/ext_authz_test.cc | 2 +- .../network/kafka/serialization_utilities.h | 6 +- .../network/ratelimit/ratelimit_test.cc | 2 +- .../redis_proxy/command_lookup_speed_test.cc | 2 +- .../filters/network/redis_proxy/mocks.h | 12 ++-- .../header_transport_impl_test.cc | 2 +- .../filters/network/thrift_proxy/mocks.h | 30 +++++----- ...le_based_metadata_grpc_credentials_test.cc | 2 +- .../quiche/platform/quic_mock_log_impl.h | 2 +- .../quiche/platform/quic_platform_test.cc | 2 +- .../grpc_metrics_service_impl_test.cc | 2 +- test/extensions/tracers/zipkin/tracer_test.cc | 2 +- test/integration/autonomous_upstream.h | 4 +- test/integration/fake_upstream.h | 2 +- .../filters/add_trailers_filter.cc | 7 ++- .../filters/clear_route_cache_filter.cc | 3 +- test/integration/filters/common.h | 3 +- .../filters/modify_buffer_filter.cc | 7 ++- test/integration/filters/pause_filter.cc | 3 +- .../filters/random_pause_filter.cc | 3 +- .../filters/request_metadata_filter.cc | 3 +- .../filters/response_metadata_filter.cc | 3 +- .../stop_iteration_and_continue_filter.cc | 5 +- test/integration/http_integration.h | 2 +- test/integration/server.h | 2 +- test/mocks/access_log/mocks.h | 8 +-- test/mocks/api/mocks.h | 4 +- test/mocks/buffer/mocks.h | 2 +- test/mocks/common.h | 2 +- test/mocks/event/mocks.h | 8 +-- test/mocks/grpc/mocks.h | 10 ++-- test/mocks/http/conn_pool.h | 4 +- test/mocks/http/mocks.h | 34 +++++------ test/mocks/http/stream_encoder.h | 2 +- test/mocks/local_info/mocks.h | 2 +- test/mocks/network/connection.h | 8 +-- test/mocks/network/mocks.h | 60 +++++++++---------- test/mocks/protobuf/mocks.h | 2 +- test/mocks/router/mocks.h | 32 +++++----- test/mocks/runtime/mocks.h | 6 +- test/mocks/secret/mocks.h | 4 +- test/mocks/ssl/mocks.h | 10 ++-- test/mocks/stream_info/mocks.h | 2 +- test/mocks/tcp/mocks.h | 8 +-- test/mocks/thread_local/mocks.h | 4 +- test/mocks/tracing/mocks.h | 8 +-- test/mocks/upstream/cluster_info.h | 6 +- test/mocks/upstream/host.h | 12 ++-- test/mocks/upstream/load_balancer_context.h | 2 +- test/mocks/upstream/mocks.h | 36 +++++------ test/server/filter_chain_benchmark_test.cc | 2 +- test/server/guarddog_impl_test.cc | 4 +- test/server/http/config_tracker_impl_test.cc | 2 +- test/test_common/logging.h | 2 +- test/test_common/simulated_time_system.cc | 2 +- test/test_common/test_time_system.h | 2 +- 185 files changed, 430 insertions(+), 412 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 981246fbb172..3e39eb682ee8 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -24,6 +24,7 @@ WarningsAsErrors: 'abseil-duration-*, modernize-make-unique, modernize-return-braced-init-list, modernize-use-equals-default, + modernize-use-override, modernize-use-using, performance-faster-string-find, performance-for-range-copy, diff --git a/include/envoy/stats/stats.h b/include/envoy/stats/stats.h index 21d19ffa21b4..f92715d9406b 100644 --- a/include/envoy/stats/stats.h +++ b/include/envoy/stats/stats.h @@ -22,7 +22,7 @@ struct Tag; */ class Metric : public RefcountInterface { public: - virtual ~Metric() = default; + ~Metric() override = default; /** * Returns the full name of the Metric. This is intended for most uses, such * as streaming out the name to a stats sink or admin request, or comparing diff --git a/source/common/access_log/access_log_manager_impl.h b/source/common/access_log/access_log_manager_impl.h index df09d2ad7863..4a8dedcdf367 100644 --- a/source/common/access_log/access_log_manager_impl.h +++ b/source/common/access_log/access_log_manager_impl.h @@ -64,7 +64,7 @@ class AccessLogFileImpl : public AccessLogFile { Thread::BasicLockable& lock, AccessLogFileStats& stats_, std::chrono::milliseconds flush_interval_msec, Thread::ThreadFactory& thread_factory); - ~AccessLogFileImpl(); + ~AccessLogFileImpl() override; // AccessLog::AccessLogFile void write(absl::string_view data) override; diff --git a/source/common/common/assert.cc b/source/common/common/assert.cc index cb01008c121e..ab4b1b8776a4 100644 --- a/source/common/common/assert.cc +++ b/source/common/common/assert.cc @@ -10,7 +10,7 @@ class ActionRegistrationImpl : public ActionRegistration { debug_assertion_failure_record_action_ = action; } - ~ActionRegistrationImpl() { + ~ActionRegistrationImpl() override { ASSERT(debug_assertion_failure_record_action_ != nullptr); debug_assertion_failure_record_action_ = nullptr; } diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 1181ef0dfec5..33ef94150822 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -29,7 +29,7 @@ class GrpcMuxImpl : public GrpcMux, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings); - ~GrpcMuxImpl(); + ~GrpcMuxImpl() override; void start() override; GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, diff --git a/source/common/config/remote_data_fetcher.h b/source/common/config/remote_data_fetcher.h index 687dfe8a7ef9..6455e44abf1b 100644 --- a/source/common/config/remote_data_fetcher.h +++ b/source/common/config/remote_data_fetcher.h @@ -47,7 +47,7 @@ class RemoteDataFetcher : public Logger::Loggable, RemoteDataFetcher(Upstream::ClusterManager& cm, const ::envoy::api::v2::core::HttpUri& uri, const std::string& content_hash, RemoteDataFetcherCallback& callback); - virtual ~RemoteDataFetcher() override; + ~RemoteDataFetcher() override; // Http::AsyncClient::Callbacks void onSuccess(Http::MessagePtr&& response) override; diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index 3777762265b1..508e18e2a312 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -33,7 +33,7 @@ class DispatcherImpl : Logger::Loggable, DispatcherImpl(Api::Api& api, Event::TimeSystem& time_system); DispatcherImpl(Buffer::WatermarkFactoryPtr&& factory, Api::Api& api, Event::TimeSystem& time_system); - ~DispatcherImpl(); + ~DispatcherImpl() override; /** * @return event_base& the libevent base. diff --git a/source/common/filesystem/posix/directory_iterator_impl.h b/source/common/filesystem/posix/directory_iterator_impl.h index 513b5f98406d..889ffef1b5ac 100644 --- a/source/common/filesystem/posix/directory_iterator_impl.h +++ b/source/common/filesystem/posix/directory_iterator_impl.h @@ -15,7 +15,7 @@ class DirectoryIteratorImpl : public DirectoryIterator { DirectoryIteratorImpl() : directory_path_(""), dir_(nullptr), os_sys_calls_(Api::OsSysCallsSingleton::get()) {} - ~DirectoryIteratorImpl(); + ~DirectoryIteratorImpl() override; DirectoryIteratorImpl& operator++() override; diff --git a/source/common/grpc/google_async_client_impl.h b/source/common/grpc/google_async_client_impl.h index b14456235b21..f2e9eb651b7e 100644 --- a/source/common/grpc/google_async_client_impl.h +++ b/source/common/grpc/google_async_client_impl.h @@ -66,7 +66,7 @@ class GoogleAsyncClientThreadLocal : public ThreadLocal::ThreadLocalObject, Logger::Loggable { public: GoogleAsyncClientThreadLocal(Api::Api& api); - ~GoogleAsyncClientThreadLocal(); + ~GoogleAsyncClientThreadLocal() override; grpc::CompletionQueue& completionQueue() { return cq_; } @@ -199,7 +199,7 @@ class GoogleAsyncStreamImpl : public RawAsyncStream, GoogleAsyncStreamImpl(GoogleAsyncClientImpl& parent, absl::string_view service_full_name, absl::string_view method_name, RawAsyncStreamCallbacks& callbacks, const absl::optional& timeout); - ~GoogleAsyncStreamImpl(); + ~GoogleAsyncStreamImpl() override; virtual void initialize(bool buffer_body_for_retry); diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index 4bf2487da76c..cc27050e8fbe 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -53,7 +53,7 @@ class CodecClient : Logger::Loggable, */ enum class Type { HTTP1, HTTP2 }; - ~CodecClient(); + ~CodecClient() override; /** * Add a connection callback to the underlying network connection. diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index a9bf7111b73b..a15bdbe913ff 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -300,7 +300,7 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { Http1Settings settings, uint32_t max_request_headers_kb, bool strict_header_validation); - virtual bool supports_http_10() override { return codec_settings_.accept_http_10_; } + bool supports_http_10() override { return codec_settings_.accept_http_10_; } private: /** diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 0b8c8bd0c4be..c8df6ee4499d 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -80,7 +80,7 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable, downstream_end_stream_(false), do_shadowing_(false), is_retry_(false), attempting_internal_redirect_with_complete_stream_(false) {} - ~Filter(); + ~Filter() override; // Http::StreamFilterBase void onDestroy() override; @@ -357,7 +357,7 @@ class Filter : Logger::Loggable, public Http::ConnectionPool::Callbacks, public LinkedObject { UpstreamRequest(Filter& parent, Http::ConnectionPool::Instance& pool); - ~UpstreamRequest(); + ~UpstreamRequest() override; void encodeHeaders(bool end_stream); void encodeData(Buffer::Instance& data, bool end_stream); diff --git a/source/common/stats/histogram_impl.h b/source/common/stats/histogram_impl.h index 926b3496d2b0..0177d9f3317e 100644 --- a/source/common/stats/histogram_impl.h +++ b/source/common/stats/histogram_impl.h @@ -72,7 +72,7 @@ class HistogramImpl : public HistogramImplHelper { const std::vector& tags) : HistogramImplHelper(name, tag_extracted_name, tags, parent.symbolTable()), parent_(parent) { } - ~HistogramImpl() { + ~HistogramImpl() override { // We must explicitly free the StatName here in order to supply the // SymbolTable reference. An RAII alternative would be to store a // reference to the SymbolTable in MetricImpl, which would cost 8 bytes @@ -99,7 +99,7 @@ class NullHistogramImpl : public HistogramImplHelper { public: explicit NullHistogramImpl(SymbolTable& symbol_table) : HistogramImplHelper(symbol_table), symbol_table_(symbol_table) {} - ~NullHistogramImpl() { MetricImpl::clear(symbol_table_); } + ~NullHistogramImpl() override { MetricImpl::clear(symbol_table_); } bool used() const override { return false; } SymbolTable& symbolTable() override { return symbol_table_; } diff --git a/source/common/stats/scope_prefixer.h b/source/common/stats/scope_prefixer.h index d4c63df99d8b..f85cfc3ff098 100644 --- a/source/common/stats/scope_prefixer.h +++ b/source/common/stats/scope_prefixer.h @@ -40,7 +40,7 @@ class ScopePrefixer : public Scope { OptionalHistogram findHistogram(StatName name) const override; const SymbolTable& constSymbolTable() const override { return scope_.constSymbolTable(); } - virtual SymbolTable& symbolTable() override { return scope_.symbolTable(); } + SymbolTable& symbolTable() override { return scope_.symbolTable(); } NullGaugeImpl& nullGauge(const std::string& str) override { return scope_.nullGauge(str); } diff --git a/source/common/tcp/conn_pool.h b/source/common/tcp/conn_pool.h index ff28802476cf..217b79cfdfbb 100644 --- a/source/common/tcp/conn_pool.h +++ b/source/common/tcp/conn_pool.h @@ -25,7 +25,7 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: const Network::ConnectionSocket::OptionsSharedPtr& options, Network::TransportSocketOptionsSharedPtr transport_socket_options); - ~ConnPoolImpl(); + ~ConnPoolImpl() override; // ConnectionPool::Instance void addDrainedCallback(DrainedCb cb) override; @@ -59,7 +59,7 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: struct ConnectionDataImpl : public ConnectionPool::ConnectionData { ConnectionDataImpl(ConnectionWrapperSharedPtr wrapper) : wrapper_(std::move(wrapper)) {} - ~ConnectionDataImpl() { wrapper_->release(false); } + ~ConnectionDataImpl() override { wrapper_->release(false); } // ConnectionPool::ConnectionData Network::ClientConnection& connection() override { return wrapper_->connection(); } @@ -80,7 +80,7 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: ConnReadFilter(ActiveConn& parent) : parent_(parent) {} // Network::ReadFilter - Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) { + Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override { parent_.onUpstreamData(data, end_stream); return Network::FilterStatus::StopIteration; } @@ -92,7 +92,7 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: public Network::ConnectionCallbacks, public Event::DeferredDeletable { ActiveConn(ConnPoolImpl& parent); - ~ActiveConn(); + ~ActiveConn() override; void onConnectTimeout(); void onUpstreamData(Buffer::Instance& data, bool end_stream); @@ -122,7 +122,7 @@ class ConnPoolImpl : Logger::Loggable, public ConnectionPool:: struct PendingRequest : LinkedObject, public ConnectionPool::Cancellable { PendingRequest(ConnPoolImpl& parent, ConnectionPool::Callbacks& callbacks); - ~PendingRequest(); + ~PendingRequest() override; // ConnectionPool::Cancellable void cancel(ConnectionPool::CancelPolicy cancel_policy) override { diff --git a/source/common/tcp_proxy/tcp_proxy.h b/source/common/tcp_proxy/tcp_proxy.h index 44cdd35aeacf..5a8534a56287 100644 --- a/source/common/tcp_proxy/tcp_proxy.h +++ b/source/common/tcp_proxy/tcp_proxy.h @@ -178,7 +178,7 @@ class Filter : public Network::ReadFilter, public: Filter(ConfigSharedPtr config, Upstream::ClusterManager& cluster_manager, TimeSource& time_source); - ~Filter(); + ~Filter() override; // Network::ReadFilter Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; @@ -315,7 +315,7 @@ using DrainerPtr = std::unique_ptr; class UpstreamDrainManager : public ThreadLocal::ThreadLocalObject { public: - ~UpstreamDrainManager(); + ~UpstreamDrainManager() override; void add(const Config::SharedConfigSharedPtr& config, Tcp::ConnectionPool::ConnectionDataPtr&& upstream_conn_data, const std::shared_ptr& callbacks, diff --git a/source/common/thread_local/thread_local_impl.h b/source/common/thread_local/thread_local_impl.h index 820cd1504a95..25e346809f5a 100644 --- a/source/common/thread_local/thread_local_impl.h +++ b/source/common/thread_local/thread_local_impl.h @@ -18,7 +18,7 @@ namespace ThreadLocal { class InstanceImpl : Logger::Loggable, public Instance { public: InstanceImpl() : main_thread_id_(std::this_thread::get_id()) {} - ~InstanceImpl(); + ~InstanceImpl() override; // ThreadLocal::Instance SlotPtr allocateSlot() override; @@ -30,7 +30,7 @@ class InstanceImpl : Logger::Loggable, public Instance { private: struct SlotImpl : public Slot { SlotImpl(InstanceImpl& parent, uint64_t index) : parent_(parent), index_(index) {} - ~SlotImpl() { parent_.removeSlot(*this); } + ~SlotImpl() override { parent_.removeSlot(*this); } // ThreadLocal::Slot ThreadLocalObjectSharedPtr get() override; diff --git a/source/common/upstream/cluster_factory_impl.h b/source/common/upstream/cluster_factory_impl.h index cb5128a53763..ae709eb094c2 100644 --- a/source/common/upstream/cluster_factory_impl.h +++ b/source/common/upstream/cluster_factory_impl.h @@ -169,7 +169,7 @@ template class ConfigurableClusterFactoryBase : public Clust ConfigurableClusterFactoryBase(const std::string& name) : ClusterFactoryImplBase(name) {} private: - virtual std::pair + std::pair createClusterImpl(const envoy::api::v2::Cluster& cluster, ClusterFactoryContext& context, Server::Configuration::TransportSocketFactoryContext& socket_factory_context, Stats::ScopePtr&& stats_scope) override { diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index abe595033747..055cb9fd2cce 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -293,7 +293,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable& local_cluster_name); - ~ThreadLocalClusterManagerImpl(); + ~ThreadLocalClusterManagerImpl() override; void drainConnPools(const HostVector& hosts); void drainConnPools(HostSharedPtr old_host, ConnPoolsContainer& container); void clearContainer(HostSharedPtr old_host, ConnPoolsContainer& container); diff --git a/source/common/upstream/health_checker_base_impl.h b/source/common/upstream/health_checker_base_impl.h index 39aa9a8687aa..5008cf9d3907 100644 --- a/source/common/upstream/health_checker_base_impl.h +++ b/source/common/upstream/health_checker_base_impl.h @@ -46,7 +46,7 @@ class HealthCheckerImplBase : public HealthChecker, protected: class ActiveHealthCheckSession : public Event::DeferredDeletable { public: - virtual ~ActiveHealthCheckSession(); + ~ActiveHealthCheckSession() override; HealthTransition setUnhealthy(envoy::data::core::v2alpha::HealthCheckFailureType type); void onDeferredDeleteBase(); void start() { onInitialInterval(); } diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 79e3a1eca77d..518ea5071121 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -162,7 +162,7 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase { ClusterStats& stats, Runtime::Loader& runtime, Runtime::RandomGenerator& random, const envoy::api::v2::Cluster::CommonLbConfig& common_config); - ~ZoneAwareLoadBalancerBase(); + ~ZoneAwareLoadBalancerBase() override; // When deciding which hosts to use on an LB decision, we need to know how to index into the // priority_set. This priority_set cursor is used by ZoneAwareLoadBalancerBase subclasses, e.g. diff --git a/source/common/upstream/outlier_detection_impl.h b/source/common/upstream/outlier_detection_impl.h index 3a03a7556c00..2b703c2adba4 100644 --- a/source/common/upstream/outlier_detection_impl.h +++ b/source/common/upstream/outlier_detection_impl.h @@ -290,7 +290,7 @@ class DetectorImpl : public Detector, public std::enable_shared_from_this(new_metadata); } @@ -483,13 +483,12 @@ class PrioritySetImpl : public PrioritySet { ASSERT(!parent_.batch_update_); parent_.batch_update_ = true; } - ~BatchUpdateScope() { parent_.batch_update_ = false; } + ~BatchUpdateScope() override { parent_.batch_update_ = false; } - virtual void updateHosts(uint32_t priority, - PrioritySet::UpdateHostsParams&& update_hosts_params, - LocalityWeightsConstSharedPtr locality_weights, - const HostVector& hosts_added, const HostVector& hosts_removed, - absl::optional overprovisioning_factor) override; + void updateHosts(uint32_t priority, PrioritySet::UpdateHostsParams&& update_hosts_params, + LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, + const HostVector& hosts_removed, + absl::optional overprovisioning_factor) override; std::unordered_set all_hosts_added_; std::unordered_set all_hosts_removed_; diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index 440617b61c70..af3b81704b38 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -39,7 +39,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable& timeout, bool use_alpha); - ~GrpcClientImpl(); + ~GrpcClientImpl() override; // ExtAuthz::Client void cancel() override; diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h index 376ccdc64c70..a26d5e4532f4 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h @@ -129,7 +129,7 @@ class RawHttpClientImpl : public Client, Logger::Loggable { public: explicit RawHttpClientImpl(Upstream::ClusterManager& cm, ClientConfigSharedPtr config); - ~RawHttpClientImpl(); + ~RawHttpClientImpl() override; // ExtAuthz::Client void cancel() override; diff --git a/source/extensions/filters/common/original_src/original_src_socket_option.h b/source/extensions/filters/common/original_src/original_src_socket_option.h index e030456c2089..3e428bf9b329 100644 --- a/source/extensions/filters/common/original_src/original_src_socket_option.h +++ b/source/extensions/filters/common/original_src/original_src_socket_option.h @@ -18,7 +18,7 @@ class OriginalSrcSocketOption : public Network::Socket::Option { * Constructs a socket option which will set the socket to use source @c src_address */ OriginalSrcSocketOption(Network::Address::InstanceConstSharedPtr src_address); - ~OriginalSrcSocketOption() = default; + ~OriginalSrcSocketOption() override = default; /** * Updates the source address of the socket to match `src_address_`. diff --git a/source/extensions/filters/common/ratelimit/ratelimit_impl.h b/source/extensions/filters/common/ratelimit/ratelimit_impl.h index 11ca4c4396b0..fcd4cb520b1d 100644 --- a/source/extensions/filters/common/ratelimit/ratelimit_impl.h +++ b/source/extensions/filters/common/ratelimit/ratelimit_impl.h @@ -47,7 +47,7 @@ class GrpcClientImpl : public Client, public: GrpcClientImpl(Grpc::RawAsyncClientPtr&& async_client, const absl::optional& timeout); - ~GrpcClientImpl(); + ~GrpcClientImpl() override; static void createRequest(envoy::service::ratelimit::v2::RateLimitRequest& request, const std::string& domain, diff --git a/source/extensions/filters/http/common/aws/credentials_provider_impl.h b/source/extensions/filters/http/common/aws/credentials_provider_impl.h index 31e4d25d5d87..5f9b3816cfe0 100644 --- a/source/extensions/filters/http/common/aws/credentials_provider_impl.h +++ b/source/extensions/filters/http/common/aws/credentials_provider_impl.h @@ -98,7 +98,7 @@ class TaskRoleCredentialsProvider : public MetadataCredentialsProviderBase { class CredentialsProviderChain : public CredentialsProvider, public Logger::Loggable { public: - virtual ~CredentialsProviderChain() = default; + ~CredentialsProviderChain() override = default; void add(const CredentialsProviderSharedPtr& credentials_provider) { providers_.emplace_back(credentials_provider); diff --git a/source/extensions/filters/http/common/jwks_fetcher.cc b/source/extensions/filters/http/common/jwks_fetcher.cc index d5c7a6c26d90..0eba021cf41e 100644 --- a/source/extensions/filters/http/common/jwks_fetcher.cc +++ b/source/extensions/filters/http/common/jwks_fetcher.cc @@ -18,9 +18,9 @@ class JwksFetcherImpl : public JwksFetcher, public: JwksFetcherImpl(Upstream::ClusterManager& cm) : cm_(cm) { ENVOY_LOG(trace, "{}", __func__); } - ~JwksFetcherImpl() { cancel(); } + ~JwksFetcherImpl() override { cancel(); } - void cancel() { + void cancel() override { if (request_ && !complete_) { request_->cancel(); ENVOY_LOG(debug, "fetch pubkey [uri = {}]: canceled", uri_->uri()); @@ -28,7 +28,8 @@ class JwksFetcherImpl : public JwksFetcher, reset(); } - void fetch(const ::envoy::api::v2::core::HttpUri& uri, JwksFetcher::JwksReceiver& receiver) { + void fetch(const ::envoy::api::v2::core::HttpUri& uri, + JwksFetcher::JwksReceiver& receiver) override { ENVOY_LOG(trace, "{}", __func__); ASSERT(!receiver_); complete_ = false; @@ -44,7 +45,7 @@ class JwksFetcherImpl : public JwksFetcher, } // HTTP async receive methods - void onSuccess(Http::MessagePtr&& response) { + void onSuccess(Http::MessagePtr&& response) override { ENVOY_LOG(trace, "{}", __func__); complete_ = true; const uint64_t status_code = Http::Utility::getResponseStatus(response->headers()); @@ -74,7 +75,7 @@ class JwksFetcherImpl : public JwksFetcher, reset(); } - void onFailure(Http::AsyncClient::FailureReason reason) { + void onFailure(Http::AsyncClient::FailureReason reason) override { ENVOY_LOG(debug, "{}: fetch pubkey [uri = {}]: network error {}", __func__, uri_->uri(), enumToInt(reason)); complete_ = true; diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index b350ce61ce64..2a592c98a0f7 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -168,7 +168,7 @@ class StreamRateLimiter : Logger::Loggable { class FaultFilter : public Http::StreamFilter, Logger::Loggable { public: FaultFilter(FaultFilterConfigSharedPtr config); - ~FaultFilter(); + ~FaultFilter() override; // Http::StreamFilterBase void onDestroy() override; diff --git a/source/extensions/filters/http/grpc_json_transcoder/transcoder_input_stream_impl.h b/source/extensions/filters/http/grpc_json_transcoder/transcoder_input_stream_impl.h index 7c4dfa4dc3f9..f5b4159be0d1 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/transcoder_input_stream_impl.h +++ b/source/extensions/filters/http/grpc_json_transcoder/transcoder_input_stream_impl.h @@ -13,8 +13,8 @@ class TranscoderInputStreamImpl : public Buffer::ZeroCopyInputStreamImpl, public google::grpc::transcoding::TranscoderInputStream { public: // TranscoderInputStream - virtual int64_t BytesAvailable() const override; - virtual bool Finished() const override; + int64_t BytesAvailable() const override; + bool Finished() const override; }; } // namespace GrpcJsonTranscoder diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h index 188124e9d16f..e10da6e3eba9 100644 --- a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h +++ b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h @@ -66,7 +66,7 @@ class HeaderToMetadataFilter : public Http::StreamFilter, public Logger::Loggable { public: HeaderToMetadataFilter(const ConfigSharedPtr config); - ~HeaderToMetadataFilter(); + ~HeaderToMetadataFilter() override; // Http::StreamFilterBase void onDestroy() override {} diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 14a35a9a914b..3eaa7f22c841 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -107,7 +107,7 @@ using IpTaggingFilterConfigSharedPtr = std::shared_ptr; class IpTaggingFilter : public Http::StreamDecoderFilter { public: IpTaggingFilter(IpTaggingFilterConfigSharedPtr config); - ~IpTaggingFilter(); + ~IpTaggingFilter() override; // Http::StreamFilterBase void onDestroy() override; diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index d88fc4c93e23..ffeb0ee458d8 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -65,7 +65,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config); - ~ClientImpl(); + ~ClientImpl() override; // Client void addConnectionCallbacks(Network::ConnectionCallbacks& callbacks) override { @@ -93,7 +93,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne struct PendingRequest : public PoolRequest { PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks); - ~PendingRequest(); + ~PendingRequest() override; // PoolRequest void cancel() override; diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.h b/source/extensions/filters/network/dubbo_proxy/decoder.h index 627dc314428b..2723633c79a6 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.h +++ b/source/extensions/filters/network/dubbo_proxy/decoder.h @@ -128,7 +128,7 @@ class DecoderBase : public DecoderStateMachine::Delegate, public Logger::Loggable { public: DecoderBase(Protocol& protocol); - virtual ~DecoderBase(); + ~DecoderBase() override; /** * Drains data from the given buffer diff --git a/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h b/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h index eeffb0c1044e..5af5e5d59622 100644 --- a/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h +++ b/source/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl.h @@ -8,20 +8,20 @@ namespace NetworkFilters { namespace DubboProxy { class DubboHessian2SerializerImpl : public Serializer { public: - ~DubboHessian2SerializerImpl() = default; - virtual const std::string& name() const override { + ~DubboHessian2SerializerImpl() override = default; + const std::string& name() const override { return ProtocolSerializerNames::get().fromType(ProtocolType::Dubbo, type()); } - virtual SerializationType type() const override { return SerializationType::Hessian2; } + SerializationType type() const override { return SerializationType::Hessian2; } - virtual std::pair + std::pair deserializeRpcInvocation(Buffer::Instance& buffer, ContextSharedPtr context) override; - virtual std::pair - deserializeRpcResult(Buffer::Instance& buffer, ContextSharedPtr context) override; + std::pair deserializeRpcResult(Buffer::Instance& buffer, + ContextSharedPtr context) override; - virtual size_t serializeRpcResult(Buffer::Instance& output_buffer, const std::string& content, - RpcResponseType type) override; + size_t serializeRpcResult(Buffer::Instance& output_buffer, const std::string& content, + RpcResponseType type) override; }; } // namespace DubboProxy diff --git a/source/extensions/filters/network/dubbo_proxy/filters/filter.h b/source/extensions/filters/network/dubbo_proxy/filters/filter.h index 5b4a5e5625c3..7ff4cbf5a030 100644 --- a/source/extensions/filters/network/dubbo_proxy/filters/filter.h +++ b/source/extensions/filters/network/dubbo_proxy/filters/filter.h @@ -115,7 +115,7 @@ class FilterCallbacksBase { */ class DecoderFilterCallbacks : public virtual FilterCallbacksBase { public: - virtual ~DecoderFilterCallbacks() = default; + ~DecoderFilterCallbacks() override = default; /** * Continue iterating through the filter chain with buffered data. This routine can only be @@ -158,7 +158,7 @@ class DecoderFilterCallbacks : public virtual FilterCallbacksBase { */ class EncoderFilterCallbacks : public virtual FilterCallbacksBase { public: - virtual ~EncoderFilterCallbacks() = default; + ~EncoderFilterCallbacks() override = default; /** * Continue iterating through the filter chain with buffered data. This routine can only be @@ -194,7 +194,7 @@ class FilterBase { */ class DecoderFilter : public StreamDecoder, public FilterBase { public: - virtual ~DecoderFilter() = default; + ~DecoderFilter() override = default; /** * Called by the connection manager once to initialize the filter decoder callbacks that the @@ -210,7 +210,7 @@ using DecoderFilterSharedPtr = std::shared_ptr; */ class EncoderFilter : public StreamEncoder, public FilterBase { public: - virtual ~EncoderFilter() = default; + ~EncoderFilter() override = default; /** * Called by the connection manager once to initialize the filter encoder callbacks that the diff --git a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h index 551d658f6ba0..caa8b9ca3112 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h +++ b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h @@ -30,7 +30,7 @@ class RouteEntryImplBase : public RouteEntry, public Logger::Loggable { public: RouteEntryImplBase(const envoy::config::filter::network::dubbo_proxy::v2alpha1::Route& route); - virtual ~RouteEntryImplBase() = default; + ~RouteEntryImplBase() override = default; // Router::RouteEntry const std::string& clusterName() const override; diff --git a/source/extensions/filters/network/dubbo_proxy/router/router_impl.h b/source/extensions/filters/network/dubbo_proxy/router/router_impl.h index c4840015a5b5..9fc078a5aa4b 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/router_impl.h +++ b/source/extensions/filters/network/dubbo_proxy/router/router_impl.h @@ -47,7 +47,7 @@ class Router : public Tcp::ConnectionPool::UpstreamCallbacks, UpstreamRequest(Router& parent, Tcp::ConnectionPool::Instance& pool, MessageMetadataSharedPtr& metadata, SerializationType serialization_type, ProtocolType protocol_type); - ~UpstreamRequest(); + ~UpstreamRequest() override; FilterStatus start(); void resetStream(); diff --git a/source/extensions/filters/network/ext_authz/ext_authz.h b/source/extensions/filters/network/ext_authz/ext_authz.h index 276509c5d8e0..a0a963750be1 100644 --- a/source/extensions/filters/network/ext_authz/ext_authz.h +++ b/source/extensions/filters/network/ext_authz/ext_authz.h @@ -73,7 +73,7 @@ class Filter : public Network::ReadFilter, public: Filter(ConfigSharedPtr config, Filters::Common::ExtAuthz::ClientPtr&& client) : config_(config), client_(std::move(client)) {} - ~Filter() = default; + ~Filter() override = default; // Network::ReadFilter Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; diff --git a/source/extensions/filters/network/kafka/codec.h b/source/extensions/filters/network/kafka/codec.h index 6dfe638d31d0..54e5709981be 100644 --- a/source/extensions/filters/network/kafka/codec.h +++ b/source/extensions/filters/network/kafka/codec.h @@ -48,7 +48,7 @@ template class MessageCallback template class AbstractMessageDecoder : public MessageDecoder { public: - virtual ~AbstractMessageDecoder() = default; + ~AbstractMessageDecoder() override = default; /** * Creates a decoder that will invoke given callbacks when a message has been parsed. diff --git a/source/extensions/filters/network/mongo_proxy/proxy.h b/source/extensions/filters/network/mongo_proxy/proxy.h index 35cdc29c17d1..f81ef86017ef 100644 --- a/source/extensions/filters/network/mongo_proxy/proxy.h +++ b/source/extensions/filters/network/mongo_proxy/proxy.h @@ -111,7 +111,7 @@ class ProxyFilter : public Network::Filter, const Filters::Common::Fault::FaultDelayConfigSharedPtr& fault_config, const Network::DrainDecision& drain_decision, TimeSource& time_system, bool emit_dynamic_metadata); - ~ProxyFilter(); + ~ProxyFilter() override; virtual DecoderPtr createDecoder(DecoderCallbacks& callbacks) PURE; diff --git a/source/extensions/filters/network/rbac/rbac_filter.h b/source/extensions/filters/network/rbac/rbac_filter.h index 3fffefd23e93..b548424c63de 100644 --- a/source/extensions/filters/network/rbac/rbac_filter.h +++ b/source/extensions/filters/network/rbac/rbac_filter.h @@ -55,7 +55,7 @@ class RoleBasedAccessControlFilter : public Network::ReadFilter, public: RoleBasedAccessControlFilter(RoleBasedAccessControlFilterConfigSharedPtr config) : config_(config) {} - ~RoleBasedAccessControlFilter() = default; + ~RoleBasedAccessControlFilter() override = default; // Network::ReadFilter Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; diff --git a/source/extensions/filters/network/redis_proxy/command_splitter_impl.h b/source/extensions/filters/network/redis_proxy/command_splitter_impl.h index e530fe8d3fdf..f4ee4c6f79d4 100644 --- a/source/extensions/filters/network/redis_proxy/command_splitter_impl.h +++ b/source/extensions/filters/network/redis_proxy/command_splitter_impl.h @@ -100,7 +100,7 @@ class SplitRequestBase : public SplitRequest { */ class SingleServerRequest : public SplitRequestBase, public Common::Redis::Client::PoolCallbacks { public: - ~SingleServerRequest(); + ~SingleServerRequest() override; // Common::Redis::Client::PoolCallbacks void onResponse(Common::Redis::RespValuePtr&& response) override; @@ -158,7 +158,7 @@ class EvalRequest : public SingleServerRequest { */ class FragmentedRequest : public SplitRequestBase { public: - ~FragmentedRequest(); + ~FragmentedRequest() override; // RedisProxy::CommandSplitter::SplitRequest void cancel() override; @@ -277,7 +277,7 @@ class CommandHandlerFactory : public CommandHandler, CommandHandlerBase { CommandHandlerFactory(Router& router) : CommandHandlerBase(router) {} SplitRequestPtr startRequest(Common::Redis::RespValuePtr&& request, SplitCallbacks& callbacks, CommandStats& command_stats, TimeSource& time_source, - bool latency_in_micros) { + bool latency_in_micros) override { return RequestClass::create(router_, std::move(request), callbacks, command_stats, time_source, latency_in_micros); } diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index ff5b52ed5067..91ea51f3e752 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -84,7 +84,7 @@ class InstanceImpl : public Instance { struct ThreadLocalPool : public ThreadLocal::ThreadLocalObject, public Upstream::ClusterUpdateCallbacks { ThreadLocalPool(InstanceImpl& parent, Event::Dispatcher& dispatcher, std::string cluster_name); - ~ThreadLocalPool(); + ~ThreadLocalPool() override; ThreadLocalActiveClientPtr& threadLocalActiveClient(Upstream::HostConstSharedPtr host); Common::Redis::Client::PoolRequest* makeRequest(const std::string& key, const Common::Redis::RespValue& request, diff --git a/source/extensions/filters/network/redis_proxy/proxy_filter.h b/source/extensions/filters/network/redis_proxy/proxy_filter.h index d916a97f6fad..aefa3b9a7efa 100644 --- a/source/extensions/filters/network/redis_proxy/proxy_filter.h +++ b/source/extensions/filters/network/redis_proxy/proxy_filter.h @@ -75,7 +75,7 @@ class ProxyFilter : public Network::ReadFilter, public: ProxyFilter(Common::Redis::DecoderFactory& factory, Common::Redis::EncoderPtr&& encoder, CommandSplitter::Instance& splitter, ProxyFilterConfigSharedPtr config); - ~ProxyFilter(); + ~ProxyFilter() override; // Network::ReadFilter void initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) override; @@ -95,7 +95,7 @@ class ProxyFilter : public Network::ReadFilter, private: struct PendingRequest : public CommandSplitter::SplitCallbacks { PendingRequest(ProxyFilter& parent); - ~PendingRequest(); + ~PendingRequest() override; // RedisProxy::CommandSplitter::SplitCallbacks bool connectionAllowed() override { return parent_.connectionAllowed(); } diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.h b/source/extensions/filters/network/thrift_proxy/conn_manager.h index afdcbcc8ff34..2188ad7ee9eb 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.h +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.h @@ -61,7 +61,7 @@ class ConnectionManager : public Network::ReadFilter, public: ConnectionManager(Config& config, Runtime::RandomGenerator& random_generator, TimeSource& time_system); - ~ConnectionManager(); + ~ConnectionManager() override; // Network::ReadFilter Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; @@ -164,7 +164,7 @@ class ConnectionManager : public Network::ReadFilter, stream_info_.setDownstreamRemoteAddress( parent_.read_callbacks_->connection().remoteAddress()); } - ~ActiveRpc() { + ~ActiveRpc() override { request_timer_->complete(); parent_.stats_.request_active_.dec(); diff --git a/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h b/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h index 39c31b3c79a0..d6961ce87357 100644 --- a/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h +++ b/source/extensions/filters/network/thrift_proxy/thrift_object_impl.h @@ -18,7 +18,7 @@ namespace ThriftProxy { class ThriftBase : public DecoderEventHandler { public: ThriftBase(ThriftBase* parent); - ~ThriftBase() = default; + ~ThriftBase() override = default; // DecoderEventHandler FilterStatus transportBegin(MessageMetadataSharedPtr) override { return FilterStatus::Continue; } @@ -61,7 +61,7 @@ class ThriftValueBase : public ThriftValue, public ThriftBase { public: ThriftValueBase(ThriftBase* parent, FieldType value_type) : ThriftBase(parent), value_type_(value_type) {} - ~ThriftValueBase() = default; + ~ThriftValueBase() override = default; // ThriftValue FieldType type() const override { return value_type_; } diff --git a/source/extensions/grpc_credentials/example/config.h b/source/extensions/grpc_credentials/example/config.h index 06f2d2f2b498..f10557a30a55 100644 --- a/source/extensions/grpc_credentials/example/config.h +++ b/source/extensions/grpc_credentials/example/config.h @@ -27,7 +27,7 @@ namespace Example { */ class AccessTokenExampleGrpcCredentialsFactory : public Grpc::GoogleGrpcCredentialsFactory { public: - virtual std::shared_ptr + std::shared_ptr getChannelCredentials(const envoy::api::v2::core::GrpcService& grpc_service_config, Api::Api& api) override; diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index 63f9dd0de0d5..346b27e9b951 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -51,7 +51,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { public Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks, public Network::ConnectionCallbacks { RedisActiveHealthCheckSession(RedisHealthChecker& parent, const Upstream::HostSharedPtr& host); - ~RedisActiveHealthCheckSession(); + ~RedisActiveHealthCheckSession() override; // ActiveHealthCheckSession void onInterval() override; diff --git a/source/extensions/stat_sinks/common/statsd/statsd.h b/source/extensions/stat_sinks/common/statsd/statsd.h index a13cd0a730bc..a6eb91a62750 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.h +++ b/source/extensions/stat_sinks/common/statsd/statsd.h @@ -30,7 +30,7 @@ class Writer : public ThreadLocal::ThreadLocalObject { Writer(Network::Address::InstanceConstSharedPtr address); // For testing. Writer() : io_handle_(std::make_unique()) {} - virtual ~Writer(); + ~Writer() override; virtual void write(const std::string& message); // Called in unit test to validate address. @@ -98,7 +98,7 @@ class TcpStatsdSink : public Stats::Sink { private: struct TlsSink : public ThreadLocal::ThreadLocalObject, public Network::ConnectionCallbacks { TlsSink(TcpStatsdSink& parent, Event::Dispatcher& dispatcher); - ~TlsSink(); + ~TlsSink() override; void beginFlush(bool expect_empty_buffer); void checkSize(); diff --git a/source/extensions/tracers/common/factory_base.h b/source/extensions/tracers/common/factory_base.h index 67191834c65b..6b57de7b3c35 100644 --- a/source/extensions/tracers/common/factory_base.h +++ b/source/extensions/tracers/common/factory_base.h @@ -15,8 +15,8 @@ namespace Common { template class FactoryBase : public Server::Configuration::TracerFactory { public: // Server::Configuration::TracerFactory - virtual Tracing::HttpTracerPtr createHttpTracer(const Protobuf::Message& config, - Server::Instance& server) override { + Tracing::HttpTracerPtr createHttpTracer(const Protobuf::Message& config, + Server::Instance& server) override { return createHttpTracerTyped(MessageUtil::downcastAndValidate(config), server); } diff --git a/source/extensions/tracers/lightstep/lightstep_tracer_impl.h b/source/extensions/tracers/lightstep/lightstep_tracer_impl.h index d6046ccd4e40..6988ce504bcc 100644 --- a/source/extensions/tracers/lightstep/lightstep_tracer_impl.h +++ b/source/extensions/tracers/lightstep/lightstep_tracer_impl.h @@ -74,7 +74,7 @@ class LightStepDriver : public Common::Ot::OpenTracingDriver { public: explicit LightStepTransporter(LightStepDriver& driver); - ~LightStepTransporter(); + ~LightStepTransporter() override; // lightstep::AsyncTransporter void Send(const Protobuf::Message& request, Protobuf::Message& response, diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index 1549a2a33d77..a85bb9d7fa85 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -64,7 +64,7 @@ class AltsSharedState : public Singleton::Instance { public: AltsSharedState() { grpc_alts_shared_resource_dedicated_init(); } - ~AltsSharedState() { grpc_alts_shared_resource_dedicated_shutdown(); } + ~AltsSharedState() override { grpc_alts_shared_resource_dedicated_shutdown(); } }; SINGLETON_MANAGER_REGISTRATION(alts_shared_state); diff --git a/source/extensions/transport_sockets/alts/tsi_handshaker.h b/source/extensions/transport_sockets/alts/tsi_handshaker.h index d578493573ec..71cd6989e71a 100644 --- a/source/extensions/transport_sockets/alts/tsi_handshaker.h +++ b/source/extensions/transport_sockets/alts/tsi_handshaker.h @@ -51,7 +51,7 @@ class TsiHandshakerCallbacks { class TsiHandshaker final : public Event::DeferredDeletable { public: explicit TsiHandshaker(CHandshakerPtr&& handshaker, Event::Dispatcher& dispatcher); - ~TsiHandshaker(); + ~TsiHandshaker() override; /** * Conduct next step of handshake, see diff --git a/source/extensions/transport_sockets/alts/tsi_socket.h b/source/extensions/transport_sockets/alts/tsi_socket.h index 31b4571db096..bbd7b19fadcb 100644 --- a/source/extensions/transport_sockets/alts/tsi_socket.h +++ b/source/extensions/transport_sockets/alts/tsi_socket.h @@ -51,7 +51,7 @@ class TsiSocket : public Network::TransportSocket, * The connection will be closed immediately if it returns false. */ TsiSocket(HandshakerFactory handshaker_factory, HandshakeValidator handshake_validator); - virtual ~TsiSocket(); + ~TsiSocket() override; // Network::TransportSocket void setTransportSocketCallbacks(Envoy::Network::TransportSocketCallbacks& callbacks) override; diff --git a/source/extensions/transport_sockets/tls/context_manager_impl.h b/source/extensions/transport_sockets/tls/context_manager_impl.h index 2bb82342940c..88ef9e67d546 100644 --- a/source/extensions/transport_sockets/tls/context_manager_impl.h +++ b/source/extensions/transport_sockets/tls/context_manager_impl.h @@ -22,7 +22,7 @@ namespace Tls { class ContextManagerImpl final : public Envoy::Ssl::ContextManager { public: ContextManagerImpl(TimeSource& time_source) : time_source_(time_source) {} - ~ContextManagerImpl(); + ~ContextManagerImpl() override; // Ssl::ContextManager Ssl::ClientContextSharedPtr diff --git a/source/server/config_validation/connection.h b/source/server/config_validation/connection.h index 7c6d5b6db062..6fb9839ff890 100644 --- a/source/server/config_validation/connection.h +++ b/source/server/config_validation/connection.h @@ -24,7 +24,7 @@ class ConfigValidateConnection : public Network::ClientConnectionImpl { // Unit tests may instantiate it without proper event machine and leave opened sockets. // Do some cleanup before invoking base class's destructor. - virtual ~ConfigValidateConnection() { close(ConnectionCloseType::NoFlush); } + ~ConfigValidateConnection() override { close(ConnectionCloseType::NoFlush); } // connect may be called in config verification mode. // It is redefined as no-op. Calling parent's method triggers connection to upstream host. diff --git a/source/server/http/config_tracker_impl.h b/source/server/http/config_tracker_impl.h index 0bcb840b7478..3c8ab1c156bf 100644 --- a/source/server/http/config_tracker_impl.h +++ b/source/server/http/config_tracker_impl.h @@ -22,7 +22,7 @@ class ConfigTrackerImpl : public ConfigTracker { class EntryOwnerImpl : public ConfigTracker::EntryOwner { public: EntryOwnerImpl(const std::shared_ptr& map, const std::string& key); - ~EntryOwnerImpl(); + ~EntryOwnerImpl() override; private: std::shared_ptr map_; diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index c3c9055ae4fd..43edef1e2f3e 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -226,7 +226,7 @@ class ListenerImpl : public Network::ListenerConfig, ListenerImpl(const envoy::api::v2::Listener& config, const std::string& version_info, ListenerManagerImpl& parent, const std::string& name, bool modifiable, bool workers_started, uint64_t hash); - ~ListenerImpl(); + ~ListenerImpl() override; /** * Helper functions to determine whether a listener is blocked for update or remove. diff --git a/source/server/options_impl.h b/source/server/options_impl.h index cddf660f79f4..3269050f5fc4 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -103,7 +103,7 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable= threshold_; } + bool isFired() const override { return value_.has_value() && value_ >= threshold_; } private: const double threshold_; diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index e14599210884..a2431c154bc8 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -38,7 +38,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { rate_limit_settings_, callbacks_, stats_, init_fetch_timeout); } - ~DeltaSubscriptionTestHarness() { + ~DeltaSubscriptionTestHarness() override { while (!nonce_acks_required_.empty()) { EXPECT_FALSE(nonce_acks_sent_.empty()); EXPECT_EQ(nonce_acks_required_.front(), nonce_acks_sent_.front()); diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 9a14fc73dd5c..28dea555e0d5 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -32,7 +32,7 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { api_(Api::createApiForTest(stats_store_)), dispatcher_(api_->allocateDispatcher()), subscription_(*dispatcher_, path_, callbacks_, stats_, validation_visitor_, *api_) {} - ~FilesystemSubscriptionTestHarness() { + ~FilesystemSubscriptionTestHarness() override { if (::access(path_.c_str(), F_OK) != -1) { EXPECT_EQ(0, ::unlink(path_.c_str())); } diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index c769a40cc4ff..ecf5c056950e 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -51,7 +51,7 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { init_fetch_timeout, validation_visitor_); } - ~HttpSubscriptionTestHarness() { + ~HttpSubscriptionTestHarness() override { // Stop subscribing on the way out. if (request_in_progress_) { EXPECT_CALL(http_request_, cancel()); diff --git a/test/common/config/metadata_test.cc b/test/common/config/metadata_test.cc index 28d5cb67b4a8..885d69f82117 100644 --- a/test/common/config/metadata_test.cc +++ b/test/common/config/metadata_test.cc @@ -61,9 +61,10 @@ class TypedMetadataTest : public testing::Test { class FooFactory : public TypedMetadataFactory::TypedMetadataFactory { public: - const std::string name() const { return "foo"; } + const std::string name() const override { return "foo"; } // Throws EnvoyException (conversion failure) if d is empty. - std::unique_ptr parse(const ProtobufWkt::Struct& d) const { + std::unique_ptr + parse(const ProtobufWkt::Struct& d) const override { if (d.fields().find("name") != d.fields().end()) { return std::make_unique(d.fields().at("name").string_value()); } diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index 97a36cc43c03..b6fc1de1b124 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -28,7 +28,7 @@ namespace { class TestDeferredDeletable : public DeferredDeletable { public: TestDeferredDeletable(std::function on_destroy) : on_destroy_(on_destroy) {} - ~TestDeferredDeletable() { on_destroy_(); } + ~TestDeferredDeletable() override { on_destroy_(); } private: std::function on_destroy_; diff --git a/test/common/grpc/grpc_client_integration_test.cc b/test/common/grpc/grpc_client_integration_test.cc index 5044cace4d40..f7b7505b4ad8 100644 --- a/test/common/grpc/grpc_client_integration_test.cc +++ b/test/common/grpc/grpc_client_integration_test.cc @@ -406,7 +406,7 @@ class GrpcAccessTokenClientIntegrationTest : public GrpcSslClientIntegrationTest } } - virtual envoy::api::v2::core::GrpcService createGoogleGrpcConfig() override { + envoy::api::v2::core::GrpcService createGoogleGrpcConfig() override { auto config = GrpcClientIntegrationTest::createGoogleGrpcConfig(); auto* google_grpc = config.mutable_google_grpc(); google_grpc->set_credentials_factory_name(credentials_factory_name_); diff --git a/test/common/grpc/grpc_client_integration_test_harness.h b/test/common/grpc/grpc_client_integration_test_harness.h index ad8a997d770b..997bae8b6bb8 100644 --- a/test/common/grpc/grpc_client_integration_test_harness.h +++ b/test/common/grpc/grpc_client_integration_test_harness.h @@ -471,7 +471,7 @@ class GrpcSslClientIntegrationTest : public GrpcClientIntegrationTest { mock_cluster_info_->transport_socket_factory_.reset(); } - virtual envoy::api::v2::core::GrpcService createGoogleGrpcConfig() override { + envoy::api::v2::core::GrpcService createGoogleGrpcConfig() override { auto config = GrpcClientIntegrationTest::createGoogleGrpcConfig(); TestUtility::setTestSslGoogleGrpcConfig(config, use_client_cert_); return config; diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 911b8a64c55f..7fc083eba090 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -58,7 +58,7 @@ class CodecClientTest : public testing::Test { dispatcher_); } - ~CodecClientTest() { EXPECT_EQ(0U, client_->numActiveRequests()); } + ~CodecClientTest() override { EXPECT_EQ(0U, client_->numActiveRequests()); } Event::MockDispatcher dispatcher_; Network::MockClientConnection* connection_; diff --git a/test/common/http/common.h b/test/common/http/common.h index e2c4d4a5e12c..9aa57ef87c8e 100644 --- a/test/common/http/common.h +++ b/test/common/http/common.h @@ -24,7 +24,7 @@ class CodecClientForTest : public Http::CodecClient { destroy_cb_(destroy_cb) { codec_.reset(codec); } - ~CodecClientForTest() { + ~CodecClientForTest() override { if (destroy_cb_) { destroy_cb_(this); } diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 31865f3f7287..7f2abde9ba58 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -89,7 +89,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan EXPECT_CALL(response_encoder_, getStream()).Times(AtLeast(0)); } - ~HttpConnectionManagerImplTest() { + ~HttpConnectionManagerImplTest() override { filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); } diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc index 002aecc3ce52..df25c3ee3bf5 100644 --- a/test/common/http/http1/conn_pool_test.cc +++ b/test/common/http/http1/conn_pool_test.cc @@ -50,7 +50,7 @@ class ConnPoolImplForTest : public ConnPoolImpl { api_(Api::createApiForTest()), mock_dispatcher_(dispatcher), mock_upstream_ready_timer_(upstream_ready_timer) {} - ~ConnPoolImplForTest() { + ~ConnPoolImplForTest() override { EXPECT_EQ(0U, ready_clients_.size()); EXPECT_EQ(0U, busy_clients_.size()); EXPECT_EQ(0U, pending_requests_.size()); @@ -128,7 +128,7 @@ class Http1ConnPoolImplTest : public testing::Test { : upstream_ready_timer_(new NiceMock(&dispatcher_)), conn_pool_(dispatcher_, cluster_, upstream_ready_timer_) {} - ~Http1ConnPoolImplTest() { + ~Http1ConnPoolImplTest() override { EXPECT_TRUE(TestUtility::gaugesZeroed(cluster_->stats_store_.gauges())); } diff --git a/test/common/http/http2/conn_pool_test.cc b/test/common/http/http2/conn_pool_test.cc index b63d04c9afa5..9a17581301c7 100644 --- a/test/common/http/http2/conn_pool_test.cc +++ b/test/common/http/http2/conn_pool_test.cc @@ -67,7 +67,7 @@ class Http2ConnPoolImplTest : public testing::Test { : api_(Api::createApiForTest(stats_store_)), pool_(dispatcher_, host_, Upstream::ResourcePriority::Default, nullptr) {} - ~Http2ConnPoolImplTest() { + ~Http2ConnPoolImplTest() override { EXPECT_TRUE(TestUtility::gaugesZeroed(cluster_->stats_store_.gauges())); } diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index dd7f751d55ed..0526b0ec97dc 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -1336,21 +1336,23 @@ TEST_P(ConnectionImplTest, DelayedCloseTimeoutNullStats) { class FakeReadFilter : public Network::ReadFilter { public: FakeReadFilter() = default; - ~FakeReadFilter() { + ~FakeReadFilter() override { EXPECT_TRUE(callbacks_ != nullptr); // The purpose is to verify that when FilterManger is destructed, ConnectionSocketImpl is not // destructed, and ConnectionSocketImpl can still be accessed via ReadFilterCallbacks. EXPECT_TRUE(callbacks_->connection().state() != Network::Connection::State::Open); } - Network::FilterStatus onData(Buffer::Instance& data, bool) { + Network::FilterStatus onData(Buffer::Instance& data, bool) override { data.drain(data.length()); return Network::FilterStatus::Continue; } - Network::FilterStatus onNewConnection() { return Network::FilterStatus::Continue; } + Network::FilterStatus onNewConnection() override { return Network::FilterStatus::Continue; } - void initializeReadFilterCallbacks(ReadFilterCallbacks& callbacks) { callbacks_ = &callbacks; } + void initializeReadFilterCallbacks(ReadFilterCallbacks& callbacks) override { + callbacks_ = &callbacks; + } private: ReadFilterCallbacks* callbacks_{nullptr}; @@ -1380,7 +1382,7 @@ class MockTransportConnectionImplTest : public testing::Test { connection_->addConnectionCallbacks(callbacks_); } - ~MockTransportConnectionImplTest() { connection_->close(ConnectionCloseType::NoFlush); } + ~MockTransportConnectionImplTest() override { connection_->close(ConnectionCloseType::NoFlush); } // This may be invoked for doWrite() on the transport to simulate all the data // being written. diff --git a/test/common/network/filter_manager_impl_test.cc b/test/common/network/filter_manager_impl_test.cc index cccbac597e73..f713a79a63e3 100644 --- a/test/common/network/filter_manager_impl_test.cc +++ b/test/common/network/filter_manager_impl_test.cc @@ -56,7 +56,7 @@ class NetworkFilterManagerTest : public testing::Test { class LocalMockFilter : public MockFilter { public: - ~LocalMockFilter() { + ~LocalMockFilter() override { // Make sure the upstream host is still valid in the filter destructor. callbacks_->upstreamHost()->address(); } diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 755ef68d98d9..5af81cbc1ff9 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -3521,10 +3521,10 @@ struct Baz : public Envoy::Config::TypedMetadata::Object { }; class BazFactory : public HttpRouteTypedMetadataFactory { public: - const std::string name() const { return "baz"; } + const std::string name() const override { return "baz"; } // Returns nullptr (conversion failure) if d is empty. std::unique_ptr - parse(const ProtobufWkt::Struct& d) const { + parse(const ProtobufWkt::Struct& d) const override { if (d.fields().find("name") != d.fields().end()) { return std::make_unique(d.fields().at("name").string_value()); } diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index daa93436b370..303589eb9750 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -75,7 +75,7 @@ class RdsImplTest : public RdsTestBase { route_config_provider_manager_ = std::make_unique(factory_context_.admin_); } - ~RdsImplTest() { factory_context_.thread_local_.shutdownThread(); } + ~RdsImplTest() override { factory_context_.thread_local_.shutdownThread(); } void setup() { const std::string config_json = R"EOF( @@ -276,7 +276,9 @@ class RouteConfigProviderManagerImplTest : public RdsTestBase { std::make_unique(factory_context_.admin_); } - ~RouteConfigProviderManagerImplTest() { factory_context_.thread_local_.shutdownThread(); } + ~RouteConfigProviderManagerImplTest() override { + factory_context_.thread_local_.shutdownThread(); + } envoy::config::filter::network::http_connection_manager::v2::Rds rds_; std::unique_ptr route_config_provider_manager_; diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index ae59bf16a54c..5db20fa89e3b 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -241,7 +241,7 @@ class CvcValidationCallback { class MockCvcValidationCallback : public CvcValidationCallback { public: MockCvcValidationCallback() = default; - ~MockCvcValidationCallback() = default; + ~MockCvcValidationCallback() override = default; MOCK_METHOD1(validateCvc, void(const envoy::api::v2::auth::CertificateValidationContext&)); }; diff --git a/test/common/signal/signals_test.cc b/test/common/signal/signals_test.cc index 0dda4fe752c8..98753f047d0f 100644 --- a/test/common/signal/signals_test.cc +++ b/test/common/signal/signals_test.cc @@ -36,7 +36,7 @@ TEST(SignalsDeathTest, InvalidAddressDeathTest) { } class TestFatalErrorHandler : public FatalErrorHandlerInterface { - virtual void onFatalError() const override { std::cerr << "HERE!"; } + void onFatalError() const override { std::cerr << "HERE!"; } }; TEST(SignalsDeathTest, RegisteredHandlerTest) { diff --git a/test/common/singleton/manager_impl_test.cc b/test/common/singleton/manager_impl_test.cc index 14b3d9eacad2..95b3b768a151 100644 --- a/test/common/singleton/manager_impl_test.cc +++ b/test/common/singleton/manager_impl_test.cc @@ -29,7 +29,7 @@ static Registry::RegisterFactory on_destructor) : id_(id), on_destructor_(on_destructor) {} - ~TestConnectionState() { on_destructor_(); } + ~TestConnectionState() override { on_destructor_(); } int id_; std::function on_destructor_; @@ -79,7 +79,7 @@ class ConnPoolImplForTest : public ConnPoolImpl { Upstream::ResourcePriority::Default, nullptr, nullptr), mock_dispatcher_(dispatcher), mock_upstream_ready_timer_(upstream_ready_timer) {} - ~ConnPoolImplForTest() { + ~ConnPoolImplForTest() override { EXPECT_EQ(0U, ready_conns_.size()); EXPECT_EQ(0U, busy_conns_.size()); EXPECT_EQ(0U, pending_requests_.size()); @@ -158,7 +158,7 @@ class TcpConnPoolImplTest : public testing::Test { : upstream_ready_timer_(new NiceMock(&dispatcher_)), conn_pool_(dispatcher_, cluster_, upstream_ready_timer_) {} - ~TcpConnPoolImplTest() { + ~TcpConnPoolImplTest() override { EXPECT_TRUE(TestUtility::gaugesZeroed(cluster_->stats_store_.gauges())); } @@ -180,7 +180,7 @@ class TcpConnPoolImplDestructorTest : public testing::Test { Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000"), Upstream::ResourcePriority::Default, nullptr, nullptr)} {} - ~TcpConnPoolImplDestructorTest() = default; + ~TcpConnPoolImplDestructorTest() override = default; void prepareConn() { connection_ = new NiceMock(); diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index ec747aa3dca3..dbfb840d619c 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -357,7 +357,7 @@ class TcpProxyTest : public testing::Test { .WillByDefault(SaveArg<0>(&access_log_data_)); } - ~TcpProxyTest() { + ~TcpProxyTest() override { if (filter_ != nullptr) { filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); } diff --git a/test/common/thread_local/thread_local_impl_test.cc b/test/common/thread_local/thread_local_impl_test.cc index 49ba114a2daf..5a678c839384 100644 --- a/test/common/thread_local/thread_local_impl_test.cc +++ b/test/common/thread_local/thread_local_impl_test.cc @@ -18,7 +18,7 @@ namespace { class TestThreadLocalObject : public ThreadLocalObject { public: - ~TestThreadLocalObject() { onDestroy(); } + ~TestThreadLocalObject() override { onDestroy(); } MOCK_METHOD0(onDestroy, void()); }; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index e9aa3293d89a..039490906a11 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -2130,14 +2130,14 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemoveDefaultPriority) { class MockConnPoolWithDestroy : public Http::ConnectionPool::MockInstance { public: - ~MockConnPoolWithDestroy() { onDestroy(); } + ~MockConnPoolWithDestroy() override { onDestroy(); } MOCK_METHOD0(onDestroy, void()); }; class MockTcpConnPoolWithDestroy : public Tcp::ConnectionPool::MockInstance { public: - ~MockTcpConnPoolWithDestroy() { onDestroy(); } + ~MockTcpConnPoolWithDestroy() override { onDestroy(); } MOCK_METHOD0(onDestroy, void()); }; diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index b0b8094c55b2..688e53b06e7b 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -1872,10 +1872,10 @@ struct Baz : public Envoy::Config::TypedMetadata::Object { class BazFactory : public ClusterTypedMetadataFactory { public: - const std::string name() const { return "baz"; } + const std::string name() const override { return "baz"; } // Returns nullptr (conversion failure) if d is empty. std::unique_ptr - parse(const ProtobufWkt::Struct& d) const { + parse(const ProtobufWkt::Struct& d) const override { if (d.fields().find("name") != d.fields().end()) { return std::make_unique(d.fields().at("name").string_value()); } diff --git a/test/exe/main_common_test.cc b/test/exe/main_common_test.cc index baeeb243976a..1d412d13a6e1 100644 --- a/test/exe/main_common_test.cc +++ b/test/exe/main_common_test.cc @@ -210,7 +210,7 @@ class AdminRequestTest : public MainCommonTest { main_common_->dispatcherForTest().post([this, &done] { struct Sacrifice : Event::DeferredDeletable { Sacrifice(absl::Notification& notify) : notify_(notify) {} - ~Sacrifice() { notify_.Notify(); } + ~Sacrifice() override { notify_.Notify(); } absl::Notification& notify_; }; auto sacrifice = std::make_unique(done); diff --git a/test/extensions/clusters/redis/mocks.h b/test/extensions/clusters/redis/mocks.h index 3f4940836f91..7269548b1b2c 100644 --- a/test/extensions/clusters/redis/mocks.h +++ b/test/extensions/clusters/redis/mocks.h @@ -15,7 +15,7 @@ namespace Redis { class MockClusterSlotUpdateCallBack : public ClusterSlotUpdateCallBack { public: MockClusterSlotUpdateCallBack(); - ~MockClusterSlotUpdateCallBack() = default; + ~MockClusterSlotUpdateCallBack() override = default; MOCK_METHOD2(onClusterSlotUpdate, bool(const std::vector&, Upstream::HostMap)); }; diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index 3fe6ddad32d3..36ebfec8391c 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -28,7 +28,7 @@ class DnsCacheImplTest : public testing::Test, public Event::TestUsingSimulatedT update_callbacks_handle_ = dns_cache_->addUpdateCallbacks(update_callbacks_); } - ~DnsCacheImplTest() { + ~DnsCacheImplTest() override { dns_cache_.reset(); EXPECT_EQ(0, TestUtility::findGauge(store_, "dns_cache.foo.num_hosts")->value()); } diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.h b/test/extensions/common/dynamic_forward_proxy/mocks.h index 0c65895c1f7d..0ac9e476deac 100644 --- a/test/extensions/common/dynamic_forward_proxy/mocks.h +++ b/test/extensions/common/dynamic_forward_proxy/mocks.h @@ -12,7 +12,7 @@ namespace DynamicForwardProxy { class MockDnsCache : public DnsCache { public: MockDnsCache(); - ~MockDnsCache(); + ~MockDnsCache() override; struct MockLoadDnsCacheEntryResult { LoadDnsCacheEntryStatus status_; @@ -38,7 +38,7 @@ class MockDnsCache : public DnsCache { class MockLoadDnsCacheEntryHandle : public DnsCache::LoadDnsCacheEntryHandle { public: MockLoadDnsCacheEntryHandle(); - ~MockLoadDnsCacheEntryHandle(); + ~MockLoadDnsCacheEntryHandle() override; MOCK_METHOD0(onDestroy, void()); }; @@ -46,7 +46,7 @@ class MockLoadDnsCacheEntryHandle : public DnsCache::LoadDnsCacheEntryHandle { class MockDnsCacheManager : public DnsCacheManager { public: MockDnsCacheManager(); - ~MockDnsCacheManager(); + ~MockDnsCacheManager() override; MOCK_METHOD1( getCache, @@ -59,7 +59,7 @@ class MockDnsCacheManager : public DnsCacheManager { class MockDnsHostInfo : public DnsHostInfo { public: MockDnsHostInfo(); - ~MockDnsHostInfo(); + ~MockDnsHostInfo() override; MOCK_METHOD0(address, Network::Address::InstanceConstSharedPtr()); MOCK_METHOD0(resolvedHost, const std::string&()); @@ -72,7 +72,7 @@ class MockDnsHostInfo : public DnsHostInfo { class MockUpdateCallbacks : public DnsCache::UpdateCallbacks { public: MockUpdateCallbacks(); - ~MockUpdateCallbacks(); + ~MockUpdateCallbacks() override; MOCK_METHOD2(onDnsHostAddOrUpdate, void(const std::string& host, const DnsHostInfoSharedPtr& address)); @@ -82,7 +82,7 @@ class MockUpdateCallbacks : public DnsCache::UpdateCallbacks { class MockLoadDnsCacheEntryCallbacks : public DnsCache::LoadDnsCacheEntryCallbacks { public: MockLoadDnsCacheEntryCallbacks(); - ~MockLoadDnsCacheEntryCallbacks(); + ~MockLoadDnsCacheEntryCallbacks() override; MOCK_METHOD0(onLoadDnsCacheComplete, void()); }; diff --git a/test/extensions/common/tap/admin_test.cc b/test/extensions/common/tap/admin_test.cc index 8af525043213..4c7428a62820 100644 --- a/test/extensions/common/tap/admin_test.cc +++ b/test/extensions/common/tap/admin_test.cc @@ -30,7 +30,9 @@ class AdminHandlerTest : public testing::Test { handler_ = std::make_unique(admin_, main_thread_dispatcher_); } - ~AdminHandlerTest() { EXPECT_CALL(admin_, removeHandler("/tap")).WillOnce(Return(true)); } + ~AdminHandlerTest() override { + EXPECT_CALL(admin_, removeHandler("/tap")).WillOnce(Return(true)); + } Server::MockAdmin admin_; Event::MockDispatcher main_thread_dispatcher_; diff --git a/test/extensions/common/tap/common.h b/test/extensions/common/tap/common.h index b98ed66f60b6..dda23a913472 100644 --- a/test/extensions/common/tap/common.h +++ b/test/extensions/common/tap/common.h @@ -37,7 +37,7 @@ namespace Tap { class MockPerTapSinkHandleManager : public PerTapSinkHandleManager { public: MockPerTapSinkHandleManager(); - ~MockPerTapSinkHandleManager(); + ~MockPerTapSinkHandleManager() override; void submitTrace(TraceWrapperPtr&& trace) override { submitTrace_(*trace); } @@ -47,7 +47,7 @@ class MockPerTapSinkHandleManager : public PerTapSinkHandleManager { class MockMatcher : public Matcher { public: using Matcher::Matcher; - ~MockMatcher(); + ~MockMatcher() override; MOCK_CONST_METHOD1(onNewStream, void(MatchStatusVector& statuses)); MOCK_CONST_METHOD2(onHttpRequestHeaders, diff --git a/test/extensions/filters/common/ext_authz/mocks.h b/test/extensions/filters/common/ext_authz/mocks.h index 75ffe3d391c8..8d2dd1a31365 100644 --- a/test/extensions/filters/common/ext_authz/mocks.h +++ b/test/extensions/filters/common/ext_authz/mocks.h @@ -16,7 +16,7 @@ namespace ExtAuthz { class MockClient : public Client { public: MockClient(); - ~MockClient(); + ~MockClient() override; // ExtAuthz::Client MOCK_METHOD0(cancel, void()); @@ -28,7 +28,7 @@ class MockClient : public Client { class MockRequestCallbacks : public RequestCallbacks { public: MockRequestCallbacks(); - ~MockRequestCallbacks(); + ~MockRequestCallbacks() override; void onComplete(ResponsePtr&& response) override { onComplete_(response); } diff --git a/test/extensions/filters/common/lua/lua_test.cc b/test/extensions/filters/common/lua/lua_test.cc index b2996f1b92aa..5c430403793a 100644 --- a/test/extensions/filters/common/lua/lua_test.cc +++ b/test/extensions/filters/common/lua/lua_test.cc @@ -23,7 +23,7 @@ namespace { // not aligned by Envoy. See https://github.com/envoyproxy/envoy/issues/5551 for details. class alignas(32) TestObject : public BaseLuaObject { public: - ~TestObject() { onDestroy(); } + ~TestObject() override { onDestroy(); } static ExportedFunctions exportedFunctions() { return {{"testCall", static_luaTestCall}}; } diff --git a/test/extensions/filters/common/lua/wrappers_test.cc b/test/extensions/filters/common/lua/wrappers_test.cc index 46428f22aec7..926db9ce3d5e 100644 --- a/test/extensions/filters/common/lua/wrappers_test.cc +++ b/test/extensions/filters/common/lua/wrappers_test.cc @@ -18,7 +18,7 @@ class LuaBufferWrapperTest : public LuaWrappersTestBase {}; class LuaMetadataMapWrapperTest : public LuaWrappersTestBase { public: - virtual void setup(const std::string& script) { + void setup(const std::string& script) override { LuaWrappersTestBase::setup(script); state_->registerType(); } @@ -32,7 +32,7 @@ class LuaMetadataMapWrapperTest : public LuaWrappersTestBase class LuaConnectionWrapperTest : public LuaWrappersTestBase { public: - virtual void setup(const std::string& script) { + void setup(const std::string& script) override { LuaWrappersTestBase::setup(script); state_->registerType(); } diff --git a/test/extensions/filters/common/ratelimit/mocks.h b/test/extensions/filters/common/ratelimit/mocks.h index accaa311aa4c..4c84e385eef2 100644 --- a/test/extensions/filters/common/ratelimit/mocks.h +++ b/test/extensions/filters/common/ratelimit/mocks.h @@ -18,7 +18,7 @@ namespace RateLimit { class MockClient : public Client { public: MockClient(); - ~MockClient(); + ~MockClient() override; // RateLimit::Client MOCK_METHOD0(cancel, void()); diff --git a/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc b/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc index 85c704dfa2de..97a5863c0bb6 100644 --- a/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc +++ b/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc @@ -36,7 +36,7 @@ namespace { class MockRequestCallbacks : public RequestCallbacks { public: - void complete(LimitStatus status, Http::HeaderMapPtr&& headers) { + void complete(LimitStatus status, Http::HeaderMapPtr&& headers) override { complete_(status, headers.get()); } diff --git a/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc b/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc index d4c22e6215af..b72ea27af87f 100644 --- a/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc +++ b/test/extensions/filters/http/common/aws/credentials_provider_impl_test.cc @@ -20,7 +20,7 @@ namespace Aws { class EvironmentCredentialsProviderTest : public testing::Test { public: - ~EvironmentCredentialsProviderTest() { + ~EvironmentCredentialsProviderTest() override { TestEnvironment::unsetEnvVar("AWS_ACCESS_KEY_ID"); TestEnvironment::unsetEnvVar("AWS_SECRET_ACCESS_KEY"); TestEnvironment::unsetEnvVar("AWS_SESSION_TOKEN"); @@ -330,7 +330,7 @@ class DefaultCredentialsProviderChainTest : public testing::Test { EXPECT_CALL(factories_, createEnvironmentCredentialsProvider()); } - ~DefaultCredentialsProviderChainTest() { + ~DefaultCredentialsProviderChainTest() override { TestEnvironment::unsetEnvVar("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"); TestEnvironment::unsetEnvVar("AWS_CONTAINER_CREDENTIALS_FULL_URI"); TestEnvironment::unsetEnvVar("AWS_CONTAINER_AUTHORIZATION_TOKEN"); @@ -349,9 +349,9 @@ class DefaultCredentialsProviderChainTest : public testing::Test { Api::Api&, const MetadataCredentialsProviderBase::MetadataFetcher& fetcher)); - virtual CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( + CredentialsProviderSharedPtr createTaskRoleCredentialsProvider( Api::Api& api, const MetadataCredentialsProviderBase::MetadataFetcher& metadata_fetcher, - const std::string& credential_uri, const std::string& authorization_token) const { + const std::string& credential_uri, const std::string& authorization_token) const override { return createTaskRoleCredentialsProviderMock(api, metadata_fetcher, credential_uri, authorization_token); } diff --git a/test/extensions/filters/http/common/aws/mocks.h b/test/extensions/filters/http/common/aws/mocks.h index 0e4ced802301..df97224efb5e 100644 --- a/test/extensions/filters/http/common/aws/mocks.h +++ b/test/extensions/filters/http/common/aws/mocks.h @@ -14,7 +14,7 @@ namespace Aws { class MockCredentialsProvider : public CredentialsProvider { public: MockCredentialsProvider(); - ~MockCredentialsProvider(); + ~MockCredentialsProvider() override; MOCK_METHOD0(getCredentials, Credentials()); }; @@ -22,7 +22,7 @@ class MockCredentialsProvider : public CredentialsProvider { class MockSigner : public Signer { public: MockSigner(); - ~MockSigner(); + ~MockSigner() override; MOCK_METHOD2(sign, void(Http::Message&, bool)); }; diff --git a/test/extensions/filters/http/common/aws/region_provider_impl_test.cc b/test/extensions/filters/http/common/aws/region_provider_impl_test.cc index 42ce4d907bef..7cb14439a521 100644 --- a/test/extensions/filters/http/common/aws/region_provider_impl_test.cc +++ b/test/extensions/filters/http/common/aws/region_provider_impl_test.cc @@ -12,7 +12,7 @@ namespace Aws { class EnvironmentRegionProviderTest : public testing::Test { public: - virtual ~EnvironmentRegionProviderTest() { TestEnvironment::unsetEnvVar("AWS_REGION"); } + ~EnvironmentRegionProviderTest() override { TestEnvironment::unsetEnvVar("AWS_REGION"); } EnvironmentRegionProvider provider_; }; diff --git a/test/extensions/filters/http/common/mock.h b/test/extensions/filters/http/common/mock.h index 2e16a0c2774f..a60fc266aa06 100644 --- a/test/extensions/filters/http/common/mock.h +++ b/test/extensions/filters/http/common/mock.h @@ -46,7 +46,7 @@ class MockJwksReceiver : public JwksFetcher::JwksReceiver { * Expectations and assertions should be made on onJwksSuccessImpl in place * of onJwksSuccess. */ - void onJwksSuccess(google::jwt_verify::JwksPtr&& jwks) { + void onJwksSuccess(google::jwt_verify::JwksPtr&& jwks) override { ASSERT(jwks); onJwksSuccessImpl(*jwks.get()); } diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc index 4718dee34720..b741cbc592f3 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_test.cc @@ -40,7 +40,7 @@ class ProxyFilterTest : public testing::Test, cm_.thread_local_cluster_.cluster_.info_->resetResourceManager(0, 1, 0, 0, 0); } - ~ProxyFilterTest() { + ~ProxyFilterTest() override { EXPECT_TRUE( cm_.thread_local_cluster_.cluster_.info_->resource_manager_->pendingRequests().canCreate()); } diff --git a/test/extensions/filters/http/dynamo/dynamo_filter_test.cc b/test/extensions/filters/http/dynamo/dynamo_filter_test.cc index f7021a9b05e9..a21e4734963f 100644 --- a/test/extensions/filters/http/dynamo/dynamo_filter_test.cc +++ b/test/extensions/filters/http/dynamo/dynamo_filter_test.cc @@ -41,7 +41,7 @@ class DynamoFilterTest : public testing::Test { filter_->setEncoderFilterCallbacks(encoder_callbacks_); } - ~DynamoFilterTest() { filter_->onDestroy(); } + ~DynamoFilterTest() override { filter_->onDestroy(); } std::unique_ptr filter_; NiceMock loader_; diff --git a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc index 437bc9d373b4..40181c29e9b7 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc @@ -740,7 +740,7 @@ class GrpcJsonTranscoderFilterPrintTest filter_->setEncoderFilterCallbacks(encoder_callbacks_); } - ~GrpcJsonTranscoderFilterPrintTest() { + ~GrpcJsonTranscoderFilterPrintTest() override { delete filter_; delete config_; } diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 5dfaeaa7cd1b..50679f090c42 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -50,7 +50,7 @@ request_type: internal filter_->setDecoderFilterCallbacks(filter_callbacks_); } - ~IpTaggingFilterTest() { filter_->onDestroy(); } + ~IpTaggingFilterTest() override { filter_->onDestroy(); } IpTaggingFilterConfigSharedPtr config_; std::unique_ptr filter_; diff --git a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc index f65637c7fcfb..7f064293f3e2 100644 --- a/test/extensions/filters/http/jwt_authn/filter_integration_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_integration_test.cc @@ -47,7 +47,8 @@ class HeaderToFilterStateFilterConfig : public Common::EmptyHttpFilterConfig { HeaderToFilterStateFilterConfig() : Common::EmptyHttpFilterConfig(HeaderToFilterStateFilterName) {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamDecoderFilter( std::make_shared("jwt_selector", "jwt_selector")); diff --git a/test/extensions/filters/http/jwt_authn/mock.h b/test/extensions/filters/http/jwt_authn/mock.h index a97342ae11b4..025351a35dc0 100644 --- a/test/extensions/filters/http/jwt_authn/mock.h +++ b/test/extensions/filters/http/jwt_authn/mock.h @@ -28,7 +28,7 @@ class MockAuthenticator : public Authenticator { SetPayloadCallback set_payload_cb, AuthenticatorCallback callback)); void verify(Http::HeaderMap& headers, std::vector&& tokens, - SetPayloadCallback set_payload_cb, AuthenticatorCallback callback) { + SetPayloadCallback set_payload_cb, AuthenticatorCallback callback) override { doVerify(headers, &tokens, std::move(set_payload_cb), std::move(callback)); } diff --git a/test/extensions/filters/http/lua/lua_filter_test.cc b/test/extensions/filters/http/lua/lua_filter_test.cc index d9be20d3f301..2d444fc6f61a 100644 --- a/test/extensions/filters/http/lua/lua_filter_test.cc +++ b/test/extensions/filters/http/lua/lua_filter_test.cc @@ -66,7 +66,7 @@ class LuaHttpFilterTest : public testing::Test { EXPECT_CALL(encoder_callbacks_, encodingBuffer()).Times(AtLeast(0)); } - ~LuaHttpFilterTest() { filter_->onDestroy(); } + ~LuaHttpFilterTest() override { filter_->onDestroy(); } void setup(const std::string& lua_code) { config_.reset(new FilterConfig(lua_code, tls_, cluster_manager_)); diff --git a/test/extensions/filters/http/rbac/rbac_filter_test.cc b/test/extensions/filters/http/rbac/rbac_filter_test.cc index 62967602dfc6..ecc0b1dcafac 100644 --- a/test/extensions/filters/http/rbac/rbac_filter_test.cc +++ b/test/extensions/filters/http/rbac/rbac_filter_test.cc @@ -47,7 +47,7 @@ class RoleBasedAccessControlFilterTest : public testing::Test { RoleBasedAccessControlFilterTest() : config_(setupConfig()), filter_(config_) {} - void SetUp() { + void SetUp() override { EXPECT_CALL(callbacks_, connection()).WillRepeatedly(Return(&connection_)); EXPECT_CALL(callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); filter_.setDecoderFilterCallbacks(callbacks_); diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc index f1aa3012c7bf..e5cb254902c8 100644 --- a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc +++ b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc @@ -33,7 +33,7 @@ class HttpInspectorTest : public testing::Test { HttpInspectorTest() : cfg_(std::make_shared(store_)), io_handle_(std::make_unique(42)) {} - ~HttpInspectorTest() { io_handle_->close(); } + ~HttpInspectorTest() override { io_handle_->close(); } void init() { filter_ = std::make_unique(cfg_); diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc index 9412e3a0caf6..63cff699cffe 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc @@ -34,7 +34,7 @@ class TlsInspectorTest : public testing::Test { TlsInspectorTest() : cfg_(std::make_shared(store_)), io_handle_(std::make_unique(42)) {} - ~TlsInspectorTest() { io_handle_->close(); } + ~TlsInspectorTest() override { io_handle_->close(); } void init() { filter_ = std::make_unique(cfg_); diff --git a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc index 7c5e09642488..5321d370a189 100644 --- a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc +++ b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc @@ -59,7 +59,7 @@ class ClientSslAuthFilterTest : public testing::Test { ClientSslAuthFilterTest() : request_(&cm_.async_client_), interval_timer_(new Event::MockTimer(&dispatcher_)), api_(Api::createApiForTest(stats_store_)) {} - ~ClientSslAuthFilterTest() { tls_.shutdownThread(); } + ~ClientSslAuthFilterTest() override { tls_.shutdownThread(); } void setup() { std::string yaml = R"EOF( diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index ecea5c887c22..fd7d9ec9b7e5 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -43,7 +43,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF return Common::Redis::DecoderPtr{decoder_}; } - ~RedisClientImplTest() { + ~RedisClientImplTest() override { client_.reset(); EXPECT_TRUE(TestUtility::gaugesZeroed(host_->cluster_.stats_store_.gauges())); diff --git a/test/extensions/filters/network/common/redis/mocks.h b/test/extensions/filters/network/common/redis/mocks.h index 829983abd53d..859ea03cf42b 100644 --- a/test/extensions/filters/network/common/redis/mocks.h +++ b/test/extensions/filters/network/common/redis/mocks.h @@ -27,7 +27,7 @@ void PrintTo(const RespValuePtr& value, std::ostream* os); class MockEncoder : public Common::Redis::Encoder { public: MockEncoder(); - ~MockEncoder(); + ~MockEncoder() override; MOCK_METHOD2(encode, void(const Common::Redis::RespValue& value, Buffer::Instance& out)); @@ -38,7 +38,7 @@ class MockEncoder : public Common::Redis::Encoder { class MockDecoder : public Common::Redis::Decoder { public: MockDecoder(); - ~MockDecoder(); + ~MockDecoder() override; MOCK_METHOD1(decode, void(Buffer::Instance& data)); }; @@ -48,7 +48,7 @@ namespace Client { class MockClient : public Client { public: MockClient(); - ~MockClient(); + ~MockClient() override; void raiseEvent(Network::ConnectionEvent event) { for (Network::ConnectionCallbacks* callbacks : callbacks_) { @@ -80,7 +80,7 @@ class MockClient : public Client { class MockPoolRequest : public PoolRequest { public: MockPoolRequest(); - ~MockPoolRequest(); + ~MockPoolRequest() override; MOCK_METHOD0(cancel, void()); }; @@ -88,7 +88,7 @@ class MockPoolRequest : public PoolRequest { class MockPoolCallbacks : public PoolCallbacks { public: MockPoolCallbacks(); - ~MockPoolCallbacks(); + ~MockPoolCallbacks() override; void onResponse(Common::Redis::RespValuePtr&& value) override { onResponse_(value); } diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index f099e1dae233..a8dc3e35d24c 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -89,7 +89,9 @@ class TestConfigImpl : public ConfigImpl { class ConnectionManagerTest : public testing::Test { public: ConnectionManagerTest() : stats_(DubboFilterStats::generateStats("test.", store_)) {} - ~ConnectionManagerTest() { filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); } + ~ConnectionManagerTest() override { + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); + } TimeSource& timeSystem() { return factory_context_.dispatcher().timeSource(); } diff --git a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc index ae9c291011c2..aade1c594766 100644 --- a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc @@ -60,7 +60,7 @@ class DubboDecoderStateMachineTest : public DecoderStateMachineTestBase, public class DubboDecoderTest : public testing::Test { public: DubboDecoderTest() = default; - virtual ~DubboDecoderTest() override = default; + ~DubboDecoderTest() override = default; NiceMock protocol_; NiceMock handler_; diff --git a/test/extensions/filters/network/dubbo_proxy/mocks.h b/test/extensions/filters/network/dubbo_proxy/mocks.h index 884ab0f821ee..02a5176f1485 100644 --- a/test/extensions/filters/network/dubbo_proxy/mocks.h +++ b/test/extensions/filters/network/dubbo_proxy/mocks.h @@ -47,7 +47,7 @@ class MockStreamHandler : public StreamHandler { class MockRequestDecoderCallbacks : public RequestDecoderCallbacks { public: MockRequestDecoderCallbacks(); - ~MockRequestDecoderCallbacks() = default; + ~MockRequestDecoderCallbacks() override = default; MOCK_METHOD0(newStream, StreamHandler&()); MOCK_METHOD1(onHeartbeat, void(MessageMetadataSharedPtr)); @@ -57,7 +57,7 @@ class MockRequestDecoderCallbacks : public RequestDecoderCallbacks { class MockResponseDecoderCallbacks : public ResponseDecoderCallbacks { public: MockResponseDecoderCallbacks(); - ~MockResponseDecoderCallbacks() = default; + ~MockResponseDecoderCallbacks() override = default; MOCK_METHOD0(newStream, StreamHandler&()); MOCK_METHOD1(onHeartbeat, void(MessageMetadataSharedPtr)); @@ -79,7 +79,7 @@ class MockActiveStream : public ActiveStream { class MockDecoderStateMachineDelegate : public DecoderStateMachine::Delegate { public: MockDecoderStateMachineDelegate() = default; - ~MockDecoderStateMachineDelegate() = default; + ~MockDecoderStateMachineDelegate() override = default; MOCK_METHOD2(newStream, ActiveStream*(MessageMetadataSharedPtr, ContextSharedPtr)); MOCK_METHOD1(onHeartbeat, void(MessageMetadataSharedPtr)); @@ -88,7 +88,7 @@ class MockDecoderStateMachineDelegate : public DecoderStateMachine::Delegate { class MockSerializer : public Serializer { public: MockSerializer(); - ~MockSerializer(); + ~MockSerializer() override; // DubboProxy::Serializer MOCK_CONST_METHOD0(name, const std::string&()); @@ -106,7 +106,7 @@ class MockSerializer : public Serializer { class MockProtocol : public Protocol { public: MockProtocol(); - ~MockProtocol(); + ~MockProtocol() override; MOCK_CONST_METHOD0(name, const std::string&()); MOCK_CONST_METHOD0(type, ProtocolType()); @@ -157,7 +157,7 @@ namespace DubboFilters { class MockFilterChainFactoryCallbacks : public FilterChainFactoryCallbacks { public: MockFilterChainFactoryCallbacks(); - ~MockFilterChainFactoryCallbacks(); + ~MockFilterChainFactoryCallbacks() override; MOCK_METHOD1(addDecoderFilter, void(DecoderFilterSharedPtr)); MOCK_METHOD1(addEncoderFilter, void(EncoderFilterSharedPtr)); @@ -167,7 +167,7 @@ class MockFilterChainFactoryCallbacks : public FilterChainFactoryCallbacks { class MockDecoderFilter : public DecoderFilter { public: MockDecoderFilter(); - ~MockDecoderFilter(); + ~MockDecoderFilter() override; // DubboProxy::DubboFilters::DecoderFilter MOCK_METHOD0(onDestroy, void()); @@ -178,7 +178,7 @@ class MockDecoderFilter : public DecoderFilter { class MockDecoderFilterCallbacks : public DecoderFilterCallbacks { public: MockDecoderFilterCallbacks(); - ~MockDecoderFilterCallbacks(); + ~MockDecoderFilterCallbacks() override; // DubboProxy::DubboFilters::DecoderFilterCallbacks MOCK_CONST_METHOD0(requestId, uint64_t()); @@ -206,7 +206,7 @@ class MockDecoderFilterCallbacks : public DecoderFilterCallbacks { class MockEncoderFilter : public EncoderFilter { public: MockEncoderFilter(); - ~MockEncoderFilter(); + ~MockEncoderFilter() override; // DubboProxy::DubboFilters::EncoderFilter MOCK_METHOD0(onDestroy, void()); @@ -217,7 +217,7 @@ class MockEncoderFilter : public EncoderFilter { class MockEncoderFilterCallbacks : public EncoderFilterCallbacks { public: MockEncoderFilterCallbacks(); - ~MockEncoderFilterCallbacks(); + ~MockEncoderFilterCallbacks() override; // DubboProxy::DubboFilters::MockEncoderFilterCallbacks MOCK_CONST_METHOD0(requestId, uint64_t()); @@ -242,7 +242,7 @@ class MockEncoderFilterCallbacks : public EncoderFilterCallbacks { class MockCodecFilter : public CodecFilter { public: MockCodecFilter(); - ~MockCodecFilter(); + ~MockCodecFilter() override; MOCK_METHOD0(onDestroy, void()); MOCK_METHOD1(setEncoderFilterCallbacks, void(EncoderFilterCallbacks& callbacks)); @@ -254,7 +254,7 @@ class MockCodecFilter : public CodecFilter { class MockDirectResponse : public DirectResponse { public: MockDirectResponse() = default; - ~MockDirectResponse() = default; + ~MockDirectResponse() override = default; MOCK_CONST_METHOD3(encode, DirectResponse::ResponseType(MessageMetadata&, Protocol&, Buffer::Instance&)); @@ -291,7 +291,7 @@ template class MockFactoryBase : public NamedDubboFilterConf class MockFilterConfigFactory : public MockFactoryBase { public: MockFilterConfigFactory(); - ~MockFilterConfigFactory(); + ~MockFilterConfigFactory() override; DubboFilters::FilterFactoryCb createFilterFactoryFromProtoTyped(const ProtobufWkt::Struct& proto_config, @@ -310,7 +310,7 @@ namespace Router { class MockRouteEntry : public RouteEntry { public: MockRouteEntry(); - ~MockRouteEntry(); + ~MockRouteEntry() override; // DubboProxy::Router::RouteEntry MOCK_CONST_METHOD0(clusterName, const std::string&()); @@ -322,7 +322,7 @@ class MockRouteEntry : public RouteEntry { class MockRoute : public Route { public: MockRoute(); - ~MockRoute(); + ~MockRoute() override; // DubboProxy::Router::Route MOCK_CONST_METHOD0(routeEntry, const RouteEntry*()); diff --git a/test/extensions/filters/network/ext_authz/ext_authz_test.cc b/test/extensions/filters/network/ext_authz/ext_authz_test.cc index 1e4d08186dce..8a344ed162f5 100644 --- a/test/extensions/filters/network/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/network/ext_authz/ext_authz_test.cc @@ -69,7 +69,7 @@ class ExtAuthzFilterTest : public testing::Test { return response; } - ~ExtAuthzFilterTest() { + ~ExtAuthzFilterTest() override { for (const Stats::GaugeSharedPtr& gauge : stats_store_.gauges()) { EXPECT_EQ(0U, gauge->value()); } diff --git a/test/extensions/filters/network/kafka/serialization_utilities.h b/test/extensions/filters/network/kafka/serialization_utilities.h index caadecb8b69d..8417afb6e61e 100644 --- a/test/extensions/filters/network/kafka/serialization_utilities.h +++ b/test/extensions/filters/network/kafka/serialization_utilities.h @@ -120,16 +120,14 @@ template class CapturingCall /** * Stores the message. */ - virtual void onMessage(Message message) override { captured_messages_.push_back(message); } + void onMessage(Message message) override { captured_messages_.push_back(message); } /** * Returns the stored messages. */ const std::vector& getCapturedMessages() const { return captured_messages_; } - virtual void onFailedParse(Failure failure_data) override { - parse_failures_.push_back(failure_data); - } + void onFailedParse(Failure failure_data) override { parse_failures_.push_back(failure_data); } const std::vector& getParseFailures() const { return parse_failures_; } diff --git a/test/extensions/filters/network/ratelimit/ratelimit_test.cc b/test/extensions/filters/network/ratelimit/ratelimit_test.cc index 1ff47f4dbe6c..0b639e1275d2 100644 --- a/test/extensions/filters/network/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/network/ratelimit/ratelimit_test.cc @@ -50,7 +50,7 @@ class RateLimitFilterTest : public testing::Test { filter_->onBelowWriteBufferLowWatermark(); } - ~RateLimitFilterTest() { + ~RateLimitFilterTest() override { for (const Stats::GaugeSharedPtr& gauge : stats_store_.gauges()) { EXPECT_EQ(0U, gauge->value()); } diff --git a/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc b/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc index b4be8f79419f..173e5efc8c2a 100644 --- a/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc +++ b/test/extensions/filters/network/redis_proxy/command_lookup_speed_test.cc @@ -24,7 +24,7 @@ namespace RedisProxy { class NoOpSplitCallbacks : public CommandSplitter::SplitCallbacks { public: NoOpSplitCallbacks() = default; - ~NoOpSplitCallbacks() = default; + ~NoOpSplitCallbacks() override = default; bool connectionAllowed() override { return true; } void onAuth(const std::string&) override {} diff --git a/test/extensions/filters/network/redis_proxy/mocks.h b/test/extensions/filters/network/redis_proxy/mocks.h index cafd3f31966d..7a169df1f767 100644 --- a/test/extensions/filters/network/redis_proxy/mocks.h +++ b/test/extensions/filters/network/redis_proxy/mocks.h @@ -22,7 +22,7 @@ namespace RedisProxy { class MockRouter : public Router { public: MockRouter(); - ~MockRouter(); + ~MockRouter() override; MOCK_METHOD1(upstreamPool, RouteSharedPtr(std::string& key)); }; @@ -30,7 +30,7 @@ class MockRouter : public Router { class MockRoute : public Route { public: MockRoute(ConnPool::InstanceSharedPtr); - ~MockRoute(); + ~MockRoute() override; MOCK_CONST_METHOD0(upstream, ConnPool::InstanceSharedPtr()); MOCK_CONST_METHOD0(mirrorPolicies, const MirrorPolicies&()); @@ -45,7 +45,7 @@ namespace ConnPool { class MockInstance : public Instance { public: MockInstance(); - ~MockInstance(); + ~MockInstance() override; MOCK_METHOD3(makeRequest, Common::Redis::Client::PoolRequest*( @@ -64,7 +64,7 @@ namespace CommandSplitter { class MockSplitRequest : public SplitRequest { public: MockSplitRequest(); - ~MockSplitRequest(); + ~MockSplitRequest() override; MOCK_METHOD0(cancel, void()); }; @@ -72,7 +72,7 @@ class MockSplitRequest : public SplitRequest { class MockSplitCallbacks : public SplitCallbacks { public: MockSplitCallbacks(); - ~MockSplitCallbacks(); + ~MockSplitCallbacks() override; MOCK_METHOD0(connectionAllowed, bool()); MOCK_METHOD1(onAuth, void(const std::string& password)); @@ -85,7 +85,7 @@ class MockSplitCallbacks : public SplitCallbacks { class MockInstance : public Instance { public: MockInstance(); - ~MockInstance(); + ~MockInstance() override; SplitRequestPtr makeRequest(Common::Redis::RespValuePtr&& request, SplitCallbacks& callbacks) override { diff --git a/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc b/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc index c29cf0a02220..791a2f9f4d59 100644 --- a/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc @@ -25,7 +25,7 @@ namespace { class MockBuffer : public Envoy::MockBuffer { public: MockBuffer() = default; - ~MockBuffer() = default; + ~MockBuffer() override = default; MOCK_CONST_METHOD0(length, uint64_t()); }; diff --git a/test/extensions/filters/network/thrift_proxy/mocks.h b/test/extensions/filters/network/thrift_proxy/mocks.h index 1d4f8631eba1..542cbca2f170 100644 --- a/test/extensions/filters/network/thrift_proxy/mocks.h +++ b/test/extensions/filters/network/thrift_proxy/mocks.h @@ -28,7 +28,7 @@ namespace ThriftProxy { class MockConfig : public Config { public: MockConfig(); - ~MockConfig(); + ~MockConfig() override; // ThriftProxy::Config MOCK_METHOD0(filterFactory, ThriftFilters::FilterChainFactory&()); @@ -40,7 +40,7 @@ class MockConfig : public Config { class MockTransport : public Transport { public: MockTransport(); - ~MockTransport(); + ~MockTransport() override; // ThriftProxy::Transport MOCK_CONST_METHOD0(name, const std::string&()); @@ -56,7 +56,7 @@ class MockTransport : public Transport { class MockProtocol : public Protocol { public: MockProtocol(); - ~MockProtocol(); + ~MockProtocol() override; // ThriftProxy::Protocol MOCK_CONST_METHOD0(name, const std::string&()); @@ -121,7 +121,7 @@ class MockProtocol : public Protocol { class MockDecoderCallbacks : public DecoderCallbacks { public: MockDecoderCallbacks(); - ~MockDecoderCallbacks(); + ~MockDecoderCallbacks() override; // ThriftProxy::DecoderCallbacks MOCK_METHOD0(newDecoderEventHandler, DecoderEventHandler&()); @@ -130,7 +130,7 @@ class MockDecoderCallbacks : public DecoderCallbacks { class MockDecoderEventHandler : public DecoderEventHandler { public: MockDecoderEventHandler(); - ~MockDecoderEventHandler(); + ~MockDecoderEventHandler() override; // ThriftProxy::DecoderEventHandler MOCK_METHOD1(transportBegin, FilterStatus(MessageMetadataSharedPtr metadata)); @@ -160,7 +160,7 @@ class MockDecoderEventHandler : public DecoderEventHandler { class MockDirectResponse : public DirectResponse { public: MockDirectResponse(); - ~MockDirectResponse(); + ~MockDirectResponse() override; // ThriftProxy::DirectResponse MOCK_CONST_METHOD3(encode, @@ -170,7 +170,7 @@ class MockDirectResponse : public DirectResponse { class MockThriftObject : public ThriftObject { public: MockThriftObject(); - ~MockThriftObject(); + ~MockThriftObject() override; MOCK_CONST_METHOD0(fields, ThriftFieldPtrList&()); MOCK_METHOD1(onData, bool(Buffer::Instance&)); @@ -185,7 +185,7 @@ namespace ThriftFilters { class MockFilterChainFactoryCallbacks : public FilterChainFactoryCallbacks { public: MockFilterChainFactoryCallbacks(); - ~MockFilterChainFactoryCallbacks(); + ~MockFilterChainFactoryCallbacks() override; MOCK_METHOD1(addDecoderFilter, void(DecoderFilterSharedPtr)); }; @@ -193,7 +193,7 @@ class MockFilterChainFactoryCallbacks : public FilterChainFactoryCallbacks { class MockDecoderFilter : public DecoderFilter { public: MockDecoderFilter(); - ~MockDecoderFilter(); + ~MockDecoderFilter() override; // ThriftProxy::ThriftFilters::DecoderFilter MOCK_METHOD0(onDestroy, void()); @@ -228,7 +228,7 @@ class MockDecoderFilter : public DecoderFilter { class MockDecoderFilterCallbacks : public DecoderFilterCallbacks { public: MockDecoderFilterCallbacks(); - ~MockDecoderFilterCallbacks(); + ~MockDecoderFilterCallbacks() override; // ThriftProxy::ThriftFilters::DecoderFilterCallbacks MOCK_CONST_METHOD0(streamId, uint64_t()); @@ -252,7 +252,7 @@ class MockDecoderFilterCallbacks : public DecoderFilterCallbacks { class MockFilterConfigFactory : public ThriftFilters::FactoryBase { public: MockFilterConfigFactory(); - ~MockFilterConfigFactory(); + ~MockFilterConfigFactory() override; ThriftFilters::FilterFactoryCb createFilterFactoryFromProtoTyped(const ProtobufWkt::Struct& proto_config, @@ -271,7 +271,7 @@ namespace Router { class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { public: MockRateLimitPolicyEntry(); - ~MockRateLimitPolicyEntry(); + ~MockRateLimitPolicyEntry() override; MOCK_CONST_METHOD0(stage, uint32_t()); MOCK_CONST_METHOD0(disableKey, const std::string&()); @@ -286,7 +286,7 @@ class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { class MockRateLimitPolicy : public RateLimitPolicy { public: MockRateLimitPolicy(); - ~MockRateLimitPolicy(); + ~MockRateLimitPolicy() override; MOCK_CONST_METHOD0(empty, bool()); MOCK_CONST_METHOD1( @@ -299,7 +299,7 @@ class MockRateLimitPolicy : public RateLimitPolicy { class MockRouteEntry : public RouteEntry { public: MockRouteEntry(); - ~MockRouteEntry(); + ~MockRouteEntry() override; // ThriftProxy::Router::RouteEntry MOCK_CONST_METHOD0(clusterName, const std::string&()); @@ -313,7 +313,7 @@ class MockRouteEntry : public RouteEntry { class MockRoute : public Route { public: MockRoute(); - ~MockRoute(); + ~MockRoute() override; // ThriftProxy::Router::Route MOCK_CONST_METHOD0(routeEntry, const RouteEntry*()); diff --git a/test/extensions/grpc_credentials/file_based_metadata/file_based_metadata_grpc_credentials_test.cc b/test/extensions/grpc_credentials/file_based_metadata/file_based_metadata_grpc_credentials_test.cc index ec1f153c8d4f..d683abfa4974 100644 --- a/test/extensions/grpc_credentials/file_based_metadata/file_based_metadata_grpc_credentials_test.cc +++ b/test/extensions/grpc_credentials/file_based_metadata/file_based_metadata_grpc_credentials_test.cc @@ -30,7 +30,7 @@ class GrpcFileBasedMetadataClientIntegrationTest : public GrpcSslClientIntegrati } } - virtual envoy::api::v2::core::GrpcService createGoogleGrpcConfig() override { + envoy::api::v2::core::GrpcService createGoogleGrpcConfig() override { auto config = GrpcClientIntegrationTest::createGoogleGrpcConfig(); auto* google_grpc = config.mutable_google_grpc(); google_grpc->set_credentials_factory_name(credentials_factory_name_); diff --git a/test/extensions/quic_listeners/quiche/platform/quic_mock_log_impl.h b/test/extensions/quic_listeners/quiche/platform/quic_mock_log_impl.h index b57b9fe48ffc..ea9326cc4eb2 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_mock_log_impl.h +++ b/test/extensions/quic_listeners/quiche/platform/quic_mock_log_impl.h @@ -21,7 +21,7 @@ class QuicEnvoyMockLog : public QuicLogSink { public: QuicEnvoyMockLog() : is_capturing_(false) {} - virtual ~QuicEnvoyMockLog() { + ~QuicEnvoyMockLog() override { if (is_capturing_) { StopCapturingLogs(); } diff --git a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc index 34e406378117..090d948bfa6f 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc @@ -85,7 +85,7 @@ class QuicPlatformTest : public testing::Test { GetLogger().set_level(ERROR); } - ~QuicPlatformTest() { + ~QuicPlatformTest() override { SetVerbosityLogThreshold(verbosity_log_threshold_); GetLogger().set_level(log_level_); } diff --git a/test/extensions/stats_sinks/metrics_service/grpc_metrics_service_impl_test.cc b/test/extensions/stats_sinks/metrics_service/grpc_metrics_service_impl_test.cc index 30fabc69b5f2..4ed180c0a05f 100644 --- a/test/extensions/stats_sinks/metrics_service/grpc_metrics_service_impl_test.cc +++ b/test/extensions/stats_sinks/metrics_service/grpc_metrics_service_impl_test.cc @@ -91,7 +91,7 @@ class TestGrpcMetricsStreamer : public GrpcMetricsStreamer { public: int metric_count; // GrpcMetricsStreamer - void send(envoy::service::metrics::v2::StreamMetricsMessage& message) { + void send(envoy::service::metrics::v2::StreamMetricsMessage& message) override { metric_count = message.envoy_metrics_size(); } }; diff --git a/test/extensions/tracers/zipkin/tracer_test.cc b/test/extensions/tracers/zipkin/tracer_test.cc index 0b1de42b7221..04fdbe3e07b4 100644 --- a/test/extensions/tracers/zipkin/tracer_test.cc +++ b/test/extensions/tracers/zipkin/tracer_test.cc @@ -28,7 +28,7 @@ namespace { class TestReporterImpl : public Reporter { public: TestReporterImpl(int value) : value_(value) {} - void reportSpan(const Span& span) { reported_spans_.push_back(span); } + void reportSpan(const Span& span) override { reported_spans_.push_back(span); } int getValue() { return value_; } std::vector& reportedSpans() { return reported_spans_; } diff --git a/test/integration/autonomous_upstream.h b/test/integration/autonomous_upstream.h index bdfc8a8eaa73..e749294d7426 100644 --- a/test/integration/autonomous_upstream.h +++ b/test/integration/autonomous_upstream.h @@ -22,7 +22,7 @@ class AutonomousStream : public FakeStream { AutonomousStream(FakeHttpConnection& parent, Http::StreamEncoder& encoder, AutonomousUpstream& upstream); - ~AutonomousStream(); + ~AutonomousStream() override; void setEndStream(bool set) override; @@ -55,7 +55,7 @@ class AutonomousUpstream : public FakeUpstream { AutonomousUpstream(uint32_t port, FakeHttpConnection::Type type, Network::Address::IpVersion version, Event::TestTimeSystem& time_system) : FakeUpstream(port, type, version, time_system) {} - ~AutonomousUpstream(); + ~AutonomousUpstream() override; bool createNetworkFilterChain(Network::Connection& connection, const std::vector& filter_factories) override; diff --git a/test/integration/fake_upstream.h b/test/integration/fake_upstream.h index 72b831ad4664..4338e4a1e998 100644 --- a/test/integration/fake_upstream.h +++ b/test/integration/fake_upstream.h @@ -531,7 +531,7 @@ class FakeUpstream : Logger::Loggable, FakeUpstream(Network::TransportSocketFactoryPtr&& transport_socket_factory, uint32_t port, FakeHttpConnection::Type type, Network::Address::IpVersion version, Event::TestTimeSystem& time_system); - ~FakeUpstream(); + ~FakeUpstream() override; FakeHttpConnection::Type httpType() { return http_type_; } diff --git a/test/integration/filters/add_trailers_filter.cc b/test/integration/filters/add_trailers_filter.cc index dd6c0f0fa0d2..cd698d71e736 100644 --- a/test/integration/filters/add_trailers_filter.cc +++ b/test/integration/filters/add_trailers_filter.cc @@ -12,7 +12,7 @@ namespace Envoy { // A test filter that inserts trailers at the end of encode/decode class AddTrailersStreamFilter : public Http::PassThroughFilter { public: - Http::FilterDataStatus decodeData(Buffer::Instance&, bool end_stream) { + Http::FilterDataStatus decodeData(Buffer::Instance&, bool end_stream) override { if (end_stream) { decoder_callbacks_->addDecodedTrailers().insertGrpcMessage().value(std::string("decode")); } @@ -20,7 +20,7 @@ class AddTrailersStreamFilter : public Http::PassThroughFilter { return Http::FilterDataStatus::Continue; } - Http::FilterDataStatus encodeData(Buffer::Instance&, bool end_stream) { + Http::FilterDataStatus encodeData(Buffer::Instance&, bool end_stream) override { if (end_stream) { encoder_callbacks_->addEncodedTrailers().insertGrpcMessage().value(std::string("encode")); } @@ -34,7 +34,8 @@ class AddTrailersStreamFilterConfig public: AddTrailersStreamFilterConfig() : EmptyHttpFilterConfig("add-trailers-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::AddTrailersStreamFilter>()); }; diff --git a/test/integration/filters/clear_route_cache_filter.cc b/test/integration/filters/clear_route_cache_filter.cc index 1e7f72cf6627..94f75d1beb61 100644 --- a/test/integration/filters/clear_route_cache_filter.cc +++ b/test/integration/filters/clear_route_cache_filter.cc @@ -22,7 +22,8 @@ class ClearRouteCacheFilterConfig : public Extensions::HttpFilters::Common::Empt public: ClearRouteCacheFilterConfig() : EmptyHttpFilterConfig("clear-route-cache") {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::ClearRouteCacheFilter>()); }; diff --git a/test/integration/filters/common.h b/test/integration/filters/common.h index 524fbce49272..247e756d5e45 100644 --- a/test/integration/filters/common.h +++ b/test/integration/filters/common.h @@ -15,7 +15,8 @@ class SimpleFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpFilt public: SimpleFilterConfig() : EmptyHttpFilterConfig(T::name) {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared()); }; diff --git a/test/integration/filters/modify_buffer_filter.cc b/test/integration/filters/modify_buffer_filter.cc index 82fc65d6ffe9..f96a258b09ce 100644 --- a/test/integration/filters/modify_buffer_filter.cc +++ b/test/integration/filters/modify_buffer_filter.cc @@ -13,7 +13,7 @@ namespace Envoy { // the content of the filter buffer. class ModifyBufferStreamFilter : public Http::PassThroughFilter { public: - Http::FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) { + Http::FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override { decoder_callbacks_->addDecodedData(data, true); if (end_stream) { @@ -27,7 +27,7 @@ class ModifyBufferStreamFilter : public Http::PassThroughFilter { return Http::FilterDataStatus::StopIterationNoBuffer; } - Http::FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) { + Http::FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override { encoder_callbacks_->addEncodedData(data, true); if (end_stream) { @@ -46,7 +46,8 @@ class ModifyBuffferFilterConfig : public Extensions::HttpFilters::Common::EmptyH public: ModifyBuffferFilterConfig() : EmptyHttpFilterConfig("modify-buffer-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::ModifyBufferStreamFilter>()); }; diff --git a/test/integration/filters/pause_filter.cc b/test/integration/filters/pause_filter.cc index 228dce27f725..8ed74d4198b8 100644 --- a/test/integration/filters/pause_filter.cc +++ b/test/integration/filters/pause_filter.cc @@ -63,7 +63,8 @@ class TestPauseFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpF public: TestPauseFilterConfig() : EmptyHttpFilterConfig("pause-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [&](Http::FilterChainFactoryCallbacks& callbacks) -> void { // GUARDED_BY insists the lock be held when the guarded variables are passed by reference. absl::WriterMutexLock m(&encode_lock_); diff --git a/test/integration/filters/random_pause_filter.cc b/test/integration/filters/random_pause_filter.cc index 15f3f71e160e..b709da758ce9 100644 --- a/test/integration/filters/random_pause_filter.cc +++ b/test/integration/filters/random_pause_filter.cc @@ -49,7 +49,8 @@ class RandomPauseFilterConfig : public Extensions::HttpFilters::Common::EmptyHtt public: RandomPauseFilterConfig() : EmptyHttpFilterConfig("random-pause-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [&](Http::FilterChainFactoryCallbacks& callbacks) -> void { absl::WriterMutexLock m(&rand_lock_); if (rng_ == nullptr) { diff --git a/test/integration/filters/request_metadata_filter.cc b/test/integration/filters/request_metadata_filter.cc index 19c6425bdc3a..b3823542213d 100644 --- a/test/integration/filters/request_metadata_filter.cc +++ b/test/integration/filters/request_metadata_filter.cc @@ -51,7 +51,8 @@ class AddRequestMetadataStreamFilterConfig : public Extensions::HttpFilters::Common::EmptyHttpFilterConfig { public: AddRequestMetadataStreamFilterConfig() : EmptyHttpFilterConfig("request-metadata-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::RequestMetadataStreamFilter>()); }; diff --git a/test/integration/filters/response_metadata_filter.cc b/test/integration/filters/response_metadata_filter.cc index 4d5f45f07a3c..bc3779d3692e 100644 --- a/test/integration/filters/response_metadata_filter.cc +++ b/test/integration/filters/response_metadata_filter.cc @@ -83,7 +83,8 @@ class AddMetadataStreamFilterConfig public: AddMetadataStreamFilterConfig() : EmptyHttpFilterConfig("response-metadata-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::ResponseMetadataStreamFilter>()); }; diff --git a/test/integration/filters/stop_iteration_and_continue_filter.cc b/test/integration/filters/stop_iteration_and_continue_filter.cc index b86cccbb1396..f90f9d483182 100644 --- a/test/integration/filters/stop_iteration_and_continue_filter.cc +++ b/test/integration/filters/stop_iteration_and_continue_filter.cc @@ -12,7 +12,7 @@ namespace Envoy { // A test filter that does StopIterationNoBuffer then continues after a 0ms alarm. class StopIterationAndContinueFilter : public Http::PassThroughFilter { public: - Http::FilterDataStatus decodeData(Buffer::Instance&, bool end_stream) { + Http::FilterDataStatus decodeData(Buffer::Instance&, bool end_stream) override { RELEASE_ASSERT(!end_stream_seen_, "end stream seen twice"); if (end_stream) { end_stream_seen_ = true; @@ -33,7 +33,8 @@ class StopIterationAndContinueFilterConfig StopIterationAndContinueFilterConfig() : EmptyHttpFilterConfig("stop-iteration-and-continue-filter") {} - Http::FilterFactoryCb createFilter(const std::string&, Server::Configuration::FactoryContext&) { + Http::FilterFactoryCb createFilter(const std::string&, + Server::Configuration::FactoryContext&) override { return [](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared<::Envoy::StopIterationAndContinueFilter>()); }; diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index 2df23c0462d0..bbeda044911e 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -98,7 +98,7 @@ class HttpIntegrationTest : public BaseIntegrationTest { const InstanceConstSharedPtrFn& upstream_address_fn, Network::Address::IpVersion version, const std::string& config = ConfigHelper::HTTP_PROXY_CONFIG); - virtual ~HttpIntegrationTest(); + ~HttpIntegrationTest() override; // Waits for the first access log entry. std::string waitForAccessLog(const std::string& filename); diff --git a/test/integration/server.h b/test/integration/server.h index 3cc7489cad55..55e3c11eab23 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -236,7 +236,7 @@ class IntegrationTestServer : public Logger::Loggable, absl::optional> process_object = absl::nullopt); // Note that the derived class is responsible for tearing down the server in its // destructor. - ~IntegrationTestServer(); + ~IntegrationTestServer() override; void waitUntilListenersReady(); diff --git a/test/mocks/access_log/mocks.h b/test/mocks/access_log/mocks.h index 562d817e896b..51748550b559 100644 --- a/test/mocks/access_log/mocks.h +++ b/test/mocks/access_log/mocks.h @@ -13,7 +13,7 @@ namespace AccessLog { class MockAccessLogFile : public AccessLogFile { public: MockAccessLogFile(); - ~MockAccessLogFile(); + ~MockAccessLogFile() override; // AccessLog::AccessLogFile MOCK_METHOD1(write, void(absl::string_view data)); @@ -24,7 +24,7 @@ class MockAccessLogFile : public AccessLogFile { class MockFilter : public Filter { public: MockFilter(); - ~MockFilter(); + ~MockFilter() override; // AccessLog::Filter MOCK_METHOD4(evaluate, @@ -36,7 +36,7 @@ class MockFilter : public Filter { class MockAccessLogManager : public AccessLogManager { public: MockAccessLogManager(); - ~MockAccessLogManager(); + ~MockAccessLogManager() override; // AccessLog::AccessLogManager MOCK_METHOD0(reopen, void()); @@ -48,7 +48,7 @@ class MockAccessLogManager : public AccessLogManager { class MockInstance : public Instance { public: MockInstance(); - ~MockInstance(); + ~MockInstance() override; // AccessLog::Instance MOCK_METHOD4(log, diff --git a/test/mocks/api/mocks.h b/test/mocks/api/mocks.h index 4bda9755288a..a85587a62683 100644 --- a/test/mocks/api/mocks.h +++ b/test/mocks/api/mocks.h @@ -26,7 +26,7 @@ namespace Api { class MockApi : public Api { public: MockApi(); - ~MockApi(); + ~MockApi() override; // Api::Api Event::DispatcherPtr allocateDispatcher() override; @@ -49,7 +49,7 @@ class MockApi : public Api { class MockOsSysCalls : public OsSysCallsImpl { public: MockOsSysCalls(); - ~MockOsSysCalls(); + ~MockOsSysCalls() override; // Api::OsSysCalls SysCallIntResult setsockopt(int sockfd, int level, int optname, const void* optval, diff --git a/test/mocks/buffer/mocks.h b/test/mocks/buffer/mocks.h index 6713259969f7..2f9eb963eca4 100644 --- a/test/mocks/buffer/mocks.h +++ b/test/mocks/buffer/mocks.h @@ -90,7 +90,7 @@ class MockWatermarkBuffer : public MockBufferBase { class MockBufferFactory : public Buffer::WatermarkFactory { public: MockBufferFactory(); - ~MockBufferFactory(); + ~MockBufferFactory() override; Buffer::InstancePtr create(std::function below_low, std::function above_high) override { diff --git a/test/mocks/common.h b/test/mocks/common.h index 77637b83ade2..2bc37836c50e 100644 --- a/test/mocks/common.h +++ b/test/mocks/common.h @@ -43,7 +43,7 @@ class ReadyWatcher { class MockTimeSystem : public Event::TestTimeSystem { public: MockTimeSystem(); - ~MockTimeSystem(); + ~MockTimeSystem() override; // TODO(#4160): Eliminate all uses of MockTimeSystem, replacing with SimulatedTimeSystem, // where timer callbacks are triggered by the advancement of time. This implementation diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 5cbd2f076f5f..ad190e23e939 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -28,7 +28,7 @@ namespace Event { class MockDispatcher : public Dispatcher { public: MockDispatcher(); - ~MockDispatcher(); + ~MockDispatcher() override; // Dispatcher TimeSource& timeSource() override { return time_system_; } @@ -125,7 +125,7 @@ class MockTimer : public Timer { public: MockTimer(); MockTimer(MockDispatcher* dispatcher); - ~MockTimer(); + ~MockTimer() override; void invokeCallback() { EXPECT_TRUE(enabled_); @@ -147,7 +147,7 @@ class MockTimer : public Timer { class MockSignalEvent : public SignalEvent { public: MockSignalEvent(MockDispatcher* dispatcher); - ~MockSignalEvent(); + ~MockSignalEvent() override; SignalCb callback_; }; @@ -155,7 +155,7 @@ class MockSignalEvent : public SignalEvent { class MockFileEvent : public FileEvent { public: MockFileEvent(); - ~MockFileEvent(); + ~MockFileEvent() override; MOCK_METHOD1(activate, void(uint32_t events)); MOCK_METHOD1(setEnabled, void(uint32_t events)); diff --git a/test/mocks/grpc/mocks.h b/test/mocks/grpc/mocks.h index f851f0e0176d..a2393b32444a 100644 --- a/test/mocks/grpc/mocks.h +++ b/test/mocks/grpc/mocks.h @@ -19,7 +19,7 @@ namespace Grpc { class MockAsyncRequest : public AsyncRequest { public: MockAsyncRequest(); - ~MockAsyncRequest(); + ~MockAsyncRequest() override; MOCK_METHOD0(cancel, void()); }; @@ -27,9 +27,9 @@ class MockAsyncRequest : public AsyncRequest { class MockAsyncStream : public RawAsyncStream { public: MockAsyncStream(); - ~MockAsyncStream(); + ~MockAsyncStream() override; - void sendMessageRaw(Buffer::InstancePtr&& request, bool end_stream) { + void sendMessageRaw(Buffer::InstancePtr&& request, bool end_stream) override { sendMessageRaw_(request, end_stream); } MOCK_METHOD2_T(sendMessageRaw_, void(Buffer::InstancePtr& request, bool end_stream)); @@ -85,7 +85,7 @@ class MockAsyncClient : public RawAsyncClient { class MockAsyncClientFactory : public AsyncClientFactory { public: MockAsyncClientFactory(); - ~MockAsyncClientFactory(); + ~MockAsyncClientFactory() override; MOCK_METHOD0(create, RawAsyncClientPtr()); }; @@ -93,7 +93,7 @@ class MockAsyncClientFactory : public AsyncClientFactory { class MockAsyncClientManager : public AsyncClientManager { public: MockAsyncClientManager(); - ~MockAsyncClientManager(); + ~MockAsyncClientManager() override; MOCK_METHOD3(factoryForGrpcService, AsyncClientFactoryPtr(const envoy::api::v2::core::GrpcService& grpc_service, diff --git a/test/mocks/http/conn_pool.h b/test/mocks/http/conn_pool.h index 18ea24063c56..60566c96b129 100644 --- a/test/mocks/http/conn_pool.h +++ b/test/mocks/http/conn_pool.h @@ -14,7 +14,7 @@ class MockCancellable : public Cancellable { public: MockCancellable(); - ~MockCancellable(); + ~MockCancellable() override; // Http::ConnectionPool::Cancellable MOCK_METHOD0(cancel, void()); @@ -23,7 +23,7 @@ class MockCancellable : public Cancellable { class MockInstance : public Instance { public: MockInstance(); - ~MockInstance(); + ~MockInstance() override; // Http::ConnectionPool::Instance MOCK_CONST_METHOD0(protocol, Http::Protocol()); diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index a3a3f53768c9..34cb97b465b9 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -41,7 +41,7 @@ namespace Http { class MockConnectionCallbacks : public virtual ConnectionCallbacks { public: MockConnectionCallbacks(); - ~MockConnectionCallbacks(); + ~MockConnectionCallbacks() override; // Http::ConnectionCallbacks MOCK_METHOD0(onGoAway, void()); @@ -51,7 +51,7 @@ class MockServerConnectionCallbacks : public ServerConnectionCallbacks, public MockConnectionCallbacks { public: MockServerConnectionCallbacks(); - ~MockServerConnectionCallbacks(); + ~MockServerConnectionCallbacks() override; // Http::ServerConnectionCallbacks MOCK_METHOD2(newStream, @@ -61,7 +61,7 @@ class MockServerConnectionCallbacks : public ServerConnectionCallbacks, class MockStreamCallbacks : public StreamCallbacks { public: MockStreamCallbacks(); - ~MockStreamCallbacks(); + ~MockStreamCallbacks() override; // Http::StreamCallbacks MOCK_METHOD2(onResetStream, void(StreamResetReason reason, absl::string_view)); @@ -72,7 +72,7 @@ class MockStreamCallbacks : public StreamCallbacks { class MockServerConnection : public ServerConnection { public: MockServerConnection(); - ~MockServerConnection(); + ~MockServerConnection() override; // Http::Connection MOCK_METHOD1(dispatch, void(Buffer::Instance& data)); @@ -89,7 +89,7 @@ class MockServerConnection : public ServerConnection { class MockClientConnection : public ClientConnection { public: MockClientConnection(); - ~MockClientConnection(); + ~MockClientConnection() override; // Http::Connection MOCK_METHOD1(dispatch, void(Buffer::Instance& data)); @@ -107,7 +107,7 @@ class MockClientConnection : public ClientConnection { class MockFilterChainFactory : public FilterChainFactory { public: MockFilterChainFactory(); - ~MockFilterChainFactory(); + ~MockFilterChainFactory() override; // Http::FilterChainFactory MOCK_METHOD1(createFilterChain, void(FilterChainFactoryCallbacks& callbacks)); @@ -128,7 +128,7 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, public MockStreamFilterCallbacksBase { public: MockStreamDecoderFilterCallbacks(); - ~MockStreamDecoderFilterCallbacks(); + ~MockStreamDecoderFilterCallbacks() override; // Http::StreamFilterCallbacks MOCK_METHOD0(connection, const Network::Connection*()); @@ -199,7 +199,7 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, public MockStreamFilterCallbacksBase { public: MockStreamEncoderFilterCallbacks(); - ~MockStreamEncoderFilterCallbacks(); + ~MockStreamEncoderFilterCallbacks() override; // Http::StreamFilterCallbacks MOCK_METHOD0(connection, const Network::Connection*()); @@ -233,7 +233,7 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, class MockStreamDecoderFilter : public StreamDecoderFilter { public: MockStreamDecoderFilter(); - ~MockStreamDecoderFilter(); + ~MockStreamDecoderFilter() override; // Http::StreamFilterBase MOCK_METHOD0(onDestroy, void()); @@ -252,7 +252,7 @@ class MockStreamDecoderFilter : public StreamDecoderFilter { class MockStreamEncoderFilter : public StreamEncoderFilter { public: MockStreamEncoderFilter(); - ~MockStreamEncoderFilter(); + ~MockStreamEncoderFilter() override; // Http::StreamFilterBase MOCK_METHOD0(onDestroy, void()); @@ -272,7 +272,7 @@ class MockStreamEncoderFilter : public StreamEncoderFilter { class MockStreamFilter : public StreamFilter { public: MockStreamFilter(); - ~MockStreamFilter(); + ~MockStreamFilter() override; // Http::StreamFilterBase MOCK_METHOD0(onDestroy, void()); @@ -299,7 +299,7 @@ class MockStreamFilter : public StreamFilter { class MockAsyncClient : public AsyncClient { public: MockAsyncClient(); - ~MockAsyncClient(); + ~MockAsyncClient() override; MOCK_METHOD0(onRequestDestroy, void()); @@ -321,7 +321,7 @@ class MockAsyncClient : public AsyncClient { class MockAsyncClientCallbacks : public AsyncClient::Callbacks { public: MockAsyncClientCallbacks(); - ~MockAsyncClientCallbacks(); + ~MockAsyncClientCallbacks() override; void onSuccess(MessagePtr&& response) override { onSuccess_(response.get()); } @@ -333,7 +333,7 @@ class MockAsyncClientCallbacks : public AsyncClient::Callbacks { class MockAsyncClientStreamCallbacks : public AsyncClient::StreamCallbacks { public: MockAsyncClientStreamCallbacks(); - ~MockAsyncClientStreamCallbacks(); + ~MockAsyncClientStreamCallbacks() override; void onHeaders(HeaderMapPtr&& headers, bool end_stream) override { onHeaders_(*headers, end_stream); @@ -349,7 +349,7 @@ class MockAsyncClientStreamCallbacks : public AsyncClient::StreamCallbacks { class MockAsyncClientRequest : public AsyncClient::Request { public: MockAsyncClientRequest(MockAsyncClient* client); - ~MockAsyncClientRequest(); + ~MockAsyncClientRequest() override; MOCK_METHOD0(cancel, void()); @@ -359,7 +359,7 @@ class MockAsyncClientRequest : public AsyncClient::Request { class MockAsyncClientStream : public AsyncClient::Stream { public: MockAsyncClientStream(); - ~MockAsyncClientStream(); + ~MockAsyncClientStream() override; MOCK_METHOD2(sendHeaders, void(HeaderMap& headers, bool end_stream)); MOCK_METHOD2(sendData, void(Buffer::Instance& data, bool end_stream)); @@ -370,7 +370,7 @@ class MockAsyncClientStream : public AsyncClient::Stream { class MockFilterChainFactoryCallbacks : public Http::FilterChainFactoryCallbacks { public: MockFilterChainFactoryCallbacks(); - ~MockFilterChainFactoryCallbacks(); + ~MockFilterChainFactoryCallbacks() override; MOCK_METHOD1(addStreamDecoderFilter, void(Http::StreamDecoderFilterSharedPtr filter)); MOCK_METHOD1(addStreamEncoderFilter, void(Http::StreamEncoderFilterSharedPtr filter)); diff --git a/test/mocks/http/stream_encoder.h b/test/mocks/http/stream_encoder.h index 5c2e067979e5..bc9138e37746 100644 --- a/test/mocks/http/stream_encoder.h +++ b/test/mocks/http/stream_encoder.h @@ -12,7 +12,7 @@ namespace Http { class MockStreamEncoder : public StreamEncoder { public: MockStreamEncoder(); - ~MockStreamEncoder(); + ~MockStreamEncoder() override; // Http::StreamEncoder MOCK_METHOD1(encode100ContinueHeaders, void(const HeaderMap& headers)); diff --git a/test/mocks/local_info/mocks.h b/test/mocks/local_info/mocks.h index 6c53a2c19a3f..655eaaf59030 100644 --- a/test/mocks/local_info/mocks.h +++ b/test/mocks/local_info/mocks.h @@ -12,7 +12,7 @@ namespace LocalInfo { class MockLocalInfo : public LocalInfo { public: MockLocalInfo(); - ~MockLocalInfo(); + ~MockLocalInfo() override; MOCK_CONST_METHOD0(address, Network::Address::InstanceConstSharedPtr()); MOCK_CONST_METHOD0(zoneName, const std::string&()); diff --git a/test/mocks/network/connection.h b/test/mocks/network/connection.h index 880802982a11..f660269de13a 100644 --- a/test/mocks/network/connection.h +++ b/test/mocks/network/connection.h @@ -17,7 +17,7 @@ namespace Network { class MockConnectionCallbacks : public ConnectionCallbacks { public: MockConnectionCallbacks(); - ~MockConnectionCallbacks(); + ~MockConnectionCallbacks() override; // Network::ConnectionCallbacks MOCK_METHOD1(onEvent, void(Network::ConnectionEvent event)); @@ -48,7 +48,7 @@ class MockConnectionBase { class MockConnection : public Connection, public MockConnectionBase { public: MockConnection(); - ~MockConnection(); + ~MockConnection() override; // Network::Connection MOCK_METHOD1(addConnectionCallbacks, void(ConnectionCallbacks& cb)); @@ -94,7 +94,7 @@ class MockConnection : public Connection, public MockConnectionBase { class MockClientConnection : public ClientConnection, public MockConnectionBase { public: MockClientConnection(); - ~MockClientConnection(); + ~MockClientConnection() override; // Network::Connection MOCK_METHOD1(addConnectionCallbacks, void(ConnectionCallbacks& cb)); @@ -143,7 +143,7 @@ class MockClientConnection : public ClientConnection, public MockConnectionBase class MockFilterManagerConnection : public FilterManagerConnection, public MockConnectionBase { public: MockFilterManagerConnection(); - ~MockFilterManagerConnection(); + ~MockFilterManagerConnection() override; // Network::Connection MOCK_METHOD1(addConnectionCallbacks, void(ConnectionCallbacks& cb)); diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 28e12f6f4d02..00f9b902386b 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -29,7 +29,7 @@ namespace Network { class MockActiveDnsQuery : public ActiveDnsQuery { public: MockActiveDnsQuery(); - ~MockActiveDnsQuery(); + ~MockActiveDnsQuery() override; // Network::ActiveDnsQuery MOCK_METHOD0(cancel, void()); @@ -38,7 +38,7 @@ class MockActiveDnsQuery : public ActiveDnsQuery { class MockDnsResolver : public DnsResolver { public: MockDnsResolver(); - ~MockDnsResolver(); + ~MockDnsResolver() override; // Network::DnsResolver MOCK_METHOD3(resolve, ActiveDnsQuery*(const std::string& dns_name, @@ -50,7 +50,7 @@ class MockDnsResolver : public DnsResolver { class MockAddressResolver : public Address::Resolver { public: MockAddressResolver(); - ~MockAddressResolver(); + ~MockAddressResolver() override; MOCK_METHOD1(resolve, Address::InstanceConstSharedPtr(const envoy::api::v2::core::SocketAddress&)); @@ -60,7 +60,7 @@ class MockAddressResolver : public Address::Resolver { class MockReadFilterCallbacks : public ReadFilterCallbacks { public: MockReadFilterCallbacks(); - ~MockReadFilterCallbacks(); + ~MockReadFilterCallbacks() override; MOCK_METHOD0(connection, Connection&()); MOCK_METHOD0(continueReading, void()); @@ -75,7 +75,7 @@ class MockReadFilterCallbacks : public ReadFilterCallbacks { class MockReadFilter : public ReadFilter { public: MockReadFilter(); - ~MockReadFilter(); + ~MockReadFilter() override; MOCK_METHOD2(onData, FilterStatus(Buffer::Instance& data, bool end_stream)); MOCK_METHOD0(onNewConnection, FilterStatus()); @@ -87,7 +87,7 @@ class MockReadFilter : public ReadFilter { class MockWriteFilterCallbacks : public WriteFilterCallbacks { public: MockWriteFilterCallbacks(); - ~MockWriteFilterCallbacks(); + ~MockWriteFilterCallbacks() override; MOCK_METHOD0(connection, Connection&()); MOCK_METHOD2(injectWriteDataToFilterChain, void(Buffer::Instance& data, bool end_stream)); @@ -98,7 +98,7 @@ class MockWriteFilterCallbacks : public WriteFilterCallbacks { class MockWriteFilter : public WriteFilter { public: MockWriteFilter(); - ~MockWriteFilter(); + ~MockWriteFilter() override; MOCK_METHOD2(onWrite, FilterStatus(Buffer::Instance& data, bool end_stream)); MOCK_METHOD1(initializeWriteFilterCallbacks, void(WriteFilterCallbacks& callbacks)); @@ -109,7 +109,7 @@ class MockWriteFilter : public WriteFilter { class MockFilter : public Filter { public: MockFilter(); - ~MockFilter(); + ~MockFilter() override; MOCK_METHOD2(onData, FilterStatus(Buffer::Instance& data, bool end_stream)); MOCK_METHOD0(onNewConnection, FilterStatus()); @@ -124,7 +124,7 @@ class MockFilter : public Filter { class MockListenerCallbacks : public ListenerCallbacks { public: MockListenerCallbacks(); - ~MockListenerCallbacks(); + ~MockListenerCallbacks() override; void onAccept(ConnectionSocketPtr&& socket, bool redirected) override { onAccept_(socket, redirected); @@ -138,7 +138,7 @@ class MockListenerCallbacks : public ListenerCallbacks { class MockUdpListenerCallbacks : public UdpListenerCallbacks { public: MockUdpListenerCallbacks(); - ~MockUdpListenerCallbacks(); + ~MockUdpListenerCallbacks() override; void onData(UdpRecvData& data) override { onData_(data); } @@ -158,7 +158,7 @@ class MockUdpListenerCallbacks : public UdpListenerCallbacks { class MockDrainDecision : public DrainDecision { public: MockDrainDecision(); - ~MockDrainDecision(); + ~MockDrainDecision() override; MOCK_CONST_METHOD0(drainClose, bool()); }; @@ -166,7 +166,7 @@ class MockDrainDecision : public DrainDecision { class MockListenerFilter : public ListenerFilter { public: MockListenerFilter(); - ~MockListenerFilter(); + ~MockListenerFilter() override; MOCK_METHOD1(onAccept, Network::FilterStatus(ListenerFilterCallbacks&)); }; @@ -174,7 +174,7 @@ class MockListenerFilter : public ListenerFilter { class MockListenerFilterManager : public ListenerFilterManager { public: MockListenerFilterManager(); - ~MockListenerFilterManager(); + ~MockListenerFilterManager() override; void addAcceptFilter(ListenerFilterPtr&& filter) override { addAcceptFilter_(filter); } @@ -184,7 +184,7 @@ class MockListenerFilterManager : public ListenerFilterManager { class MockFilterChain : public FilterChain { public: MockFilterChain(); - ~MockFilterChain(); + ~MockFilterChain() override; // Network::FilterChain MOCK_CONST_METHOD0(transportSocketFactory, const TransportSocketFactory&()); @@ -194,7 +194,7 @@ class MockFilterChain : public FilterChain { class MockFilterChainManager : public FilterChainManager { public: MockFilterChainManager(); - ~MockFilterChainManager(); + ~MockFilterChainManager() override; // Network::FilterChainManager MOCK_CONST_METHOD1(findFilterChain, const FilterChain*(const ConnectionSocket& socket)); @@ -203,7 +203,7 @@ class MockFilterChainManager : public FilterChainManager { class MockFilterChainFactory : public FilterChainFactory { public: MockFilterChainFactory(); - ~MockFilterChainFactory(); + ~MockFilterChainFactory() override; MOCK_METHOD2(createNetworkFilterChain, bool(Connection& connection, @@ -239,7 +239,7 @@ class MockListenSocket : public Socket { class MockSocketOption : public Socket::Option { public: MockSocketOption(); - ~MockSocketOption(); + ~MockSocketOption() override; MOCK_CONST_METHOD2(setOption, bool(Socket&, envoy::api::v2::core::SocketOption::SocketState state)); @@ -285,7 +285,7 @@ class MockConnectionSocket : public ConnectionSocket { class MockListenerFilterCallbacks : public ListenerFilterCallbacks { public: MockListenerFilterCallbacks(); - ~MockListenerFilterCallbacks(); + ~MockListenerFilterCallbacks() override; MOCK_METHOD0(socket, ConnectionSocket&()); MOCK_METHOD0(dispatcher, Event::Dispatcher&()); @@ -297,7 +297,7 @@ class MockListenerFilterCallbacks : public ListenerFilterCallbacks { class MockListenerConfig : public ListenerConfig { public: MockListenerConfig(); - ~MockListenerConfig(); + ~MockListenerConfig() override; MOCK_METHOD0(filterChainManager, FilterChainManager&()); MOCK_METHOD0(filterChainFactory, FilterChainFactory&()); @@ -320,7 +320,7 @@ class MockListenerConfig : public ListenerConfig { class MockListener : public Listener { public: MockListener(); - ~MockListener(); + ~MockListener() override; MOCK_METHOD0(onDestroy, void()); MOCK_METHOD0(enable, void()); @@ -330,7 +330,7 @@ class MockListener : public Listener { class MockConnectionHandler : public ConnectionHandler { public: MockConnectionHandler(); - ~MockConnectionHandler(); + ~MockConnectionHandler() override; MOCK_METHOD0(numConnections, uint64_t()); MOCK_METHOD1(addListener, void(ListenerConfig& config)); @@ -346,7 +346,7 @@ class MockConnectionHandler : public ConnectionHandler { class MockIp : public Address::Ip { public: MockIp(); - ~MockIp(); + ~MockIp() override; MOCK_CONST_METHOD0(addressAsString, const std::string&()); MOCK_CONST_METHOD0(isAnyAddress, bool()); @@ -361,7 +361,7 @@ class MockResolvedAddress : public Address::Instance { public: MockResolvedAddress(const std::string& logical, const std::string& physical) : logical_(logical), physical_(physical) {} - ~MockResolvedAddress(); + ~MockResolvedAddress() override; bool operator==(const Address::Instance& other) const override { return asString() == other.asString(); @@ -385,7 +385,7 @@ class MockResolvedAddress : public Address::Instance { class MockTransportSocket : public TransportSocket { public: MockTransportSocket(); - ~MockTransportSocket(); + ~MockTransportSocket() override; MOCK_METHOD1(setTransportSocketCallbacks, void(TransportSocketCallbacks& callbacks)); MOCK_CONST_METHOD0(protocol, std::string()); @@ -403,7 +403,7 @@ class MockTransportSocket : public TransportSocket { class MockTransportSocketFactory : public TransportSocketFactory { public: MockTransportSocketFactory(); - ~MockTransportSocketFactory(); + ~MockTransportSocketFactory() override; MOCK_CONST_METHOD0(implementsSecureTransport, bool()); MOCK_CONST_METHOD1(createTransportSocket, TransportSocketPtr(TransportSocketOptionsSharedPtr)); @@ -412,7 +412,7 @@ class MockTransportSocketFactory : public TransportSocketFactory { class MockTransportSocketCallbacks : public TransportSocketCallbacks { public: MockTransportSocketCallbacks(); - ~MockTransportSocketCallbacks(); + ~MockTransportSocketCallbacks() override; MOCK_METHOD0(ioHandle, IoHandle&()); MOCK_CONST_METHOD0(ioHandle, const IoHandle&()); @@ -427,7 +427,7 @@ class MockTransportSocketCallbacks : public TransportSocketCallbacks { class MockUdpListener : public UdpListener { public: MockUdpListener(); - ~MockUdpListener(); + ~MockUdpListener() override; MOCK_METHOD0(onDestroy, void()); MOCK_METHOD0(enable, void()); @@ -440,7 +440,7 @@ class MockUdpListener : public UdpListener { class MockUdpReadFilterCallbacks : public UdpReadFilterCallbacks { public: MockUdpReadFilterCallbacks(); - ~MockUdpReadFilterCallbacks(); + ~MockUdpReadFilterCallbacks() override; MOCK_METHOD0(udpListener, UdpListener&()); @@ -450,7 +450,7 @@ class MockUdpReadFilterCallbacks : public UdpReadFilterCallbacks { class MockUdpListenerReadFilter : public UdpListenerReadFilter { public: MockUdpListenerReadFilter(UdpReadFilterCallbacks& callbacks); - ~MockUdpListenerReadFilter(); + ~MockUdpListenerReadFilter() override; MOCK_METHOD1(onData, void(UdpRecvData&)); }; @@ -458,7 +458,7 @@ class MockUdpListenerReadFilter : public UdpListenerReadFilter { class MockUdpListenerFilterManager : public UdpListenerFilterManager { public: MockUdpListenerFilterManager(); - ~MockUdpListenerFilterManager(); + ~MockUdpListenerFilterManager() override; void addReadFilter(UdpListenerReadFilterPtr&& filter) override { addReadFilter_(filter); } diff --git a/test/mocks/protobuf/mocks.h b/test/mocks/protobuf/mocks.h index 48c975e31ac1..66e2f5d2b9c1 100644 --- a/test/mocks/protobuf/mocks.h +++ b/test/mocks/protobuf/mocks.h @@ -10,7 +10,7 @@ namespace ProtobufMessage { class MockValidationVisitor : public ValidationVisitor { public: MockValidationVisitor(); - ~MockValidationVisitor(); + ~MockValidationVisitor() override; MOCK_METHOD1(onUnknownField, void(absl::string_view)); }; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index f84aacda8ab8..aeda21400f98 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -34,7 +34,7 @@ namespace Router { class MockDirectResponseEntry : public DirectResponseEntry { public: MockDirectResponseEntry(); - ~MockDirectResponseEntry(); + ~MockDirectResponseEntry() override; // DirectResponseEntry MOCK_CONST_METHOD2(finalizeResponseHeaders, @@ -112,7 +112,7 @@ class TestRetryPolicy : public RetryPolicy { class MockRetryState : public RetryState { public: MockRetryState(); - ~MockRetryState(); + ~MockRetryState() override; void expectHeadersRetry(); void expectHedgedPerTryTimeoutRetry(); @@ -138,7 +138,7 @@ class MockRetryState : public RetryState { class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { public: MockRateLimitPolicyEntry(); - ~MockRateLimitPolicyEntry(); + ~MockRateLimitPolicyEntry() override; // Router::RateLimitPolicyEntry MOCK_CONST_METHOD0(stage, uint64_t()); @@ -156,7 +156,7 @@ class MockRateLimitPolicyEntry : public RateLimitPolicyEntry { class MockRateLimitPolicy : public RateLimitPolicy { public: MockRateLimitPolicy(); - ~MockRateLimitPolicy(); + ~MockRateLimitPolicy() override; // Router::RateLimitPolicy MOCK_CONST_METHOD1( @@ -182,7 +182,7 @@ class TestShadowPolicy : public ShadowPolicy { class MockShadowWriter : public ShadowWriter { public: MockShadowWriter(); - ~MockShadowWriter(); + ~MockShadowWriter() override; // Router::ShadowWriter void shadow(const std::string& cluster, Http::MessagePtr&& request, @@ -206,7 +206,7 @@ class TestVirtualCluster : public VirtualCluster { class MockVirtualHost : public VirtualHost { public: MockVirtualHost(); - ~MockVirtualHost(); + ~MockVirtualHost() override; // Router::VirtualHost MOCK_CONST_METHOD0(name, const std::string&()); @@ -233,7 +233,7 @@ class MockVirtualHost : public VirtualHost { class MockHashPolicy : public HashPolicy { public: MockHashPolicy(); - ~MockHashPolicy(); + ~MockHashPolicy() override; // Router::HashPolicy MOCK_CONST_METHOD3(generateHash, @@ -245,7 +245,7 @@ class MockHashPolicy : public HashPolicy { class MockMetadataMatchCriteria : public MetadataMatchCriteria { public: MockMetadataMatchCriteria(); - ~MockMetadataMatchCriteria(); + ~MockMetadataMatchCriteria() override; // Router::MetadataMatchCriteria MOCK_CONST_METHOD0(metadataMatchCriteria, @@ -256,7 +256,7 @@ class MockMetadataMatchCriteria : public MetadataMatchCriteria { class MockPathMatchCriterion : public PathMatchCriterion { public: MockPathMatchCriterion(); - ~MockPathMatchCriterion(); + ~MockPathMatchCriterion() override; // Router::PathMatchCriterion MOCK_CONST_METHOD0(matchType, PathMatchType()); @@ -269,7 +269,7 @@ class MockPathMatchCriterion : public PathMatchCriterion { class MockRouteEntry : public RouteEntry { public: MockRouteEntry(); - ~MockRouteEntry(); + ~MockRouteEntry() override; // Router::Config MOCK_CONST_METHOD0(clusterName, const std::string&()); @@ -326,7 +326,7 @@ class MockRouteEntry : public RouteEntry { class MockDecorator : public Decorator { public: MockDecorator(); - ~MockDecorator(); + ~MockDecorator() override; // Router::Decorator MOCK_CONST_METHOD0(getOperation, const std::string&()); @@ -338,7 +338,7 @@ class MockDecorator : public Decorator { class MockRouteTracing : public RouteTracing { public: MockRouteTracing(); - ~MockRouteTracing(); + ~MockRouteTracing() override; // Router::RouteTracing MOCK_CONST_METHOD0(getClientSampling, const envoy::type::FractionalPercent&()); @@ -349,7 +349,7 @@ class MockRouteTracing : public RouteTracing { class MockRoute : public Route { public: MockRoute(); - ~MockRoute(); + ~MockRoute() override; // Router::Route MOCK_CONST_METHOD0(directResponseEntry, const DirectResponseEntry*()); @@ -366,7 +366,7 @@ class MockRoute : public Route { class MockConfig : public Config { public: MockConfig(); - ~MockConfig(); + ~MockConfig() override; // Router::Config MOCK_CONST_METHOD2(route, RouteConstSharedPtr(const Http::HeaderMap&, uint64_t random_value)); @@ -382,7 +382,7 @@ class MockConfig : public Config { class MockRouteConfigProviderManager : public RouteConfigProviderManager { public: MockRouteConfigProviderManager(); - ~MockRouteConfigProviderManager(); + ~MockRouteConfigProviderManager() override; MOCK_METHOD3(createRdsRouteConfigProvider, RouteConfigProviderPtr( @@ -397,7 +397,7 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { class MockScopedConfig : public ScopedConfig { public: MockScopedConfig(); - ~MockScopedConfig(); + ~MockScopedConfig() override; MOCK_CONST_METHOD1(getRouteConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); }; diff --git a/test/mocks/runtime/mocks.h b/test/mocks/runtime/mocks.h index e67e34361958..3592a40200fa 100644 --- a/test/mocks/runtime/mocks.h +++ b/test/mocks/runtime/mocks.h @@ -15,7 +15,7 @@ namespace Runtime { class MockRandomGenerator : public RandomGenerator { public: MockRandomGenerator(); - ~MockRandomGenerator(); + ~MockRandomGenerator() override; MOCK_METHOD0(random, uint64_t()); MOCK_METHOD0(uuid, std::string()); @@ -60,7 +60,7 @@ class MockSnapshot : public Snapshot { class MockLoader : public Loader { public: MockLoader(); - ~MockLoader(); + ~MockLoader() override; MOCK_METHOD1(initialize, void(Upstream::ClusterManager& cm)); MOCK_METHOD0(snapshot, Snapshot&()); @@ -72,7 +72,7 @@ class MockLoader : public Loader { class MockOverrideLayer : public Snapshot::OverrideLayer { public: MockOverrideLayer(); - ~MockOverrideLayer(); + ~MockOverrideLayer() override; MOCK_CONST_METHOD0(name, const std::string&()); MOCK_CONST_METHOD0(values, const Snapshot::EntryMap&()); diff --git a/test/mocks/secret/mocks.h b/test/mocks/secret/mocks.h index 428f1ec28faa..2f3f2c5f073e 100644 --- a/test/mocks/secret/mocks.h +++ b/test/mocks/secret/mocks.h @@ -14,7 +14,7 @@ namespace Secret { class MockSecretManager : public SecretManager { public: MockSecretManager(); - ~MockSecretManager(); + ~MockSecretManager() override; MOCK_METHOD1(addStaticSecret, void(const envoy::api::v2::auth::Secret& secret)); MOCK_CONST_METHOD1(findStaticTlsCertificateProvider, @@ -42,7 +42,7 @@ class MockSecretManager : public SecretManager { class MockSecretCallbacks : public SecretCallbacks { public: MockSecretCallbacks(); - ~MockSecretCallbacks(); + ~MockSecretCallbacks() override; MOCK_METHOD0(onAddOrUpdateSecret, void()); }; diff --git a/test/mocks/ssl/mocks.h b/test/mocks/ssl/mocks.h index cd91a237327f..20621bd29ca4 100644 --- a/test/mocks/ssl/mocks.h +++ b/test/mocks/ssl/mocks.h @@ -20,7 +20,7 @@ namespace Ssl { class MockContextManager : public ContextManager { public: MockContextManager(); - ~MockContextManager(); + ~MockContextManager() override; MOCK_METHOD2(createSslClientContext, ClientContextSharedPtr(Stats::Scope& scope, const ClientContextConfig& config)); @@ -34,7 +34,7 @@ class MockContextManager : public ContextManager { class MockConnectionInfo : public ConnectionInfo { public: MockConnectionInfo(); - ~MockConnectionInfo(); + ~MockConnectionInfo() override; MOCK_CONST_METHOD0(peerCertificatePresented, bool()); MOCK_CONST_METHOD0(uriSanLocalCertificate, std::vector()); @@ -59,7 +59,7 @@ class MockConnectionInfo : public ConnectionInfo { class MockClientContext : public ClientContext { public: MockClientContext(); - ~MockClientContext(); + ~MockClientContext() override; MOCK_CONST_METHOD0(daysUntilFirstCertExpires, size_t()); MOCK_CONST_METHOD0(getCaCertInformation, CertificateDetailsPtr()); @@ -69,7 +69,7 @@ class MockClientContext : public ClientContext { class MockClientContextConfig : public ClientContextConfig { public: MockClientContextConfig(); - ~MockClientContextConfig(); + ~MockClientContextConfig() override; MOCK_CONST_METHOD0(alpnProtocols, const std::string&()); MOCK_CONST_METHOD0(cipherSuites, const std::string&()); @@ -91,7 +91,7 @@ class MockClientContextConfig : public ClientContextConfig { class MockServerContextConfig : public ServerContextConfig { public: MockServerContextConfig(); - ~MockServerContextConfig(); + ~MockServerContextConfig() override; MOCK_CONST_METHOD0(alpnProtocols, const std::string&()); MOCK_CONST_METHOD0(cipherSuites, const std::string&()); diff --git a/test/mocks/stream_info/mocks.h b/test/mocks/stream_info/mocks.h index 9332fec087ec..0fbd34222291 100644 --- a/test/mocks/stream_info/mocks.h +++ b/test/mocks/stream_info/mocks.h @@ -14,7 +14,7 @@ namespace StreamInfo { class MockStreamInfo : public StreamInfo { public: MockStreamInfo(); - ~MockStreamInfo(); + ~MockStreamInfo() override; // StreamInfo::StreamInfo MOCK_METHOD1(setResponseFlag, void(ResponseFlag response_flag)); diff --git a/test/mocks/tcp/mocks.h b/test/mocks/tcp/mocks.h index 3f864ec3296d..5ac8b8ee6f80 100644 --- a/test/mocks/tcp/mocks.h +++ b/test/mocks/tcp/mocks.h @@ -18,7 +18,7 @@ namespace ConnectionPool { class MockCancellable : public Cancellable { public: MockCancellable(); - ~MockCancellable(); + ~MockCancellable() override; // Tcp::ConnectionPool::Cancellable MOCK_METHOD1(cancel, void(CancelPolicy cancel_policy)); @@ -27,7 +27,7 @@ class MockCancellable : public Cancellable { class MockUpstreamCallbacks : public UpstreamCallbacks { public: MockUpstreamCallbacks(); - ~MockUpstreamCallbacks(); + ~MockUpstreamCallbacks() override; // Tcp::ConnectionPool::UpstreamCallbacks MOCK_METHOD2(onUpstreamData, void(Buffer::Instance& data, bool end_stream)); @@ -39,7 +39,7 @@ class MockUpstreamCallbacks : public UpstreamCallbacks { class MockConnectionData : public ConnectionData { public: MockConnectionData(); - ~MockConnectionData(); + ~MockConnectionData() override; // Tcp::ConnectionPool::ConnectionData MOCK_METHOD0(connection, Network::ClientConnection&()); @@ -57,7 +57,7 @@ class MockConnectionData : public ConnectionData { class MockInstance : public Instance { public: MockInstance(); - ~MockInstance(); + ~MockInstance() override; // Tcp::ConnectionPool::Instance MOCK_METHOD1(addDrainedCallback, void(DrainedCb cb)); diff --git a/test/mocks/thread_local/mocks.h b/test/mocks/thread_local/mocks.h index 24393722887a..3824deac4ac4 100644 --- a/test/mocks/thread_local/mocks.h +++ b/test/mocks/thread_local/mocks.h @@ -15,7 +15,7 @@ namespace ThreadLocal { class MockInstance : public Instance { public: MockInstance(); - ~MockInstance(); + ~MockInstance() override; MOCK_METHOD1(runOnAllThreads, void(Event::PostCb cb)); MOCK_METHOD2(runOnAllThreads, void(Event::PostCb cb, Event::PostCb main_callback)); @@ -48,7 +48,7 @@ class MockInstance : public Instance { parent_.data_.resize(index_ + 1); } - ~SlotImpl() { + ~SlotImpl() override { // Do not actually clear slot data during shutdown. This mimics the production code. if (!parent_.shutdown_) { EXPECT_LT(index_, parent_.data_.size()); diff --git a/test/mocks/tracing/mocks.h b/test/mocks/tracing/mocks.h index d62036fd8d07..22a694edfd9a 100644 --- a/test/mocks/tracing/mocks.h +++ b/test/mocks/tracing/mocks.h @@ -13,7 +13,7 @@ namespace Tracing { class MockConfig : public Config { public: MockConfig(); - ~MockConfig(); + ~MockConfig() override; MOCK_CONST_METHOD0(operationName, OperationName()); MOCK_CONST_METHOD0(requestHeadersForTags, const std::vector&()); @@ -27,7 +27,7 @@ class MockConfig : public Config { class MockSpan : public Span { public: MockSpan(); - ~MockSpan(); + ~MockSpan() override; MOCK_METHOD1(setOperation, void(absl::string_view operation)); MOCK_METHOD2(setTag, void(absl::string_view name, absl::string_view value)); @@ -48,7 +48,7 @@ class MockSpan : public Span { class MockHttpTracer : public HttpTracer { public: MockHttpTracer(); - ~MockHttpTracer(); + ~MockHttpTracer() override; SpanPtr startSpan(const Config& config, Http::HeaderMap& request_headers, const StreamInfo::StreamInfo& stream_info, @@ -64,7 +64,7 @@ class MockHttpTracer : public HttpTracer { class MockDriver : public Driver { public: MockDriver(); - ~MockDriver(); + ~MockDriver() override; SpanPtr startSpan(const Config& config, Http::HeaderMap& request_headers, const std::string& operation_name, SystemTime start_time, diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index 46b318c3f36d..56df9fa4dcca 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -26,7 +26,7 @@ namespace Upstream { class MockLoadBalancerSubsetInfo : public LoadBalancerSubsetInfo { public: MockLoadBalancerSubsetInfo(); - ~MockLoadBalancerSubsetInfo(); + ~MockLoadBalancerSubsetInfo() override; // Upstream::LoadBalancerSubsetInfo MOCK_CONST_METHOD0(isEnabled, bool()); @@ -61,7 +61,7 @@ class MockClusterTypedMetadata : public Config::TypedMetadataImpl()); MOCK_METHOD0(metadataMatchCriteria, Router::MetadataMatchCriteria*()); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index e9b86baf2be6..f194bffceca0 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -38,7 +38,7 @@ class MockHostSet : public HostSet { public: MockHostSet(uint32_t priority = 0, uint32_t overprovisioning_factor = kDefaultOverProvisioningFactor); - ~MockHostSet(); + ~MockHostSet() override; void runCallbacks(const HostVector added, const HostVector removed) { member_update_cb_helper_.runCallbacks(priority(), added, removed); @@ -92,7 +92,7 @@ class MockHostSet : public HostSet { class MockPrioritySet : public PrioritySet { public: MockPrioritySet(); - ~MockPrioritySet(); + ~MockPrioritySet() override; HostSet& getHostSet(uint32_t priority); void runUpdateCallbacks(uint32_t priority, const HostVector& hosts_added, @@ -125,10 +125,10 @@ class MockRetryPriority : public RetryPriority { const DegradedLoad& degraded_priority_load) : priority_load_({healthy_priority_load, degraded_priority_load}) {} MockRetryPriority(const MockRetryPriority& other) : priority_load_(other.priority_load_) {} - ~MockRetryPriority(); + ~MockRetryPriority() override; const HealthyAndDegradedLoad& determinePriorityLoad(const PrioritySet&, - const HealthyAndDegradedLoad&) { + const HealthyAndDegradedLoad&) override { return priority_load_; } @@ -158,7 +158,7 @@ class MockRetryPriorityFactory : public RetryPriorityFactory { class MockCluster : public Cluster { public: MockCluster(); - ~MockCluster(); + ~MockCluster() override; // Upstream::Cluster MOCK_METHOD0(healthChecker, HealthChecker*()); @@ -182,7 +182,7 @@ class MockCluster : public Cluster { class MockClusterRealPrioritySet : public MockCluster { public: MockClusterRealPrioritySet(); - ~MockClusterRealPrioritySet(); + ~MockClusterRealPrioritySet() override; // Upstream::Cluster PrioritySetImpl& prioritySet() override { return priority_set_; } @@ -195,7 +195,7 @@ class MockClusterRealPrioritySet : public MockCluster { class MockClusterMockPrioritySet : public MockCluster { public: MockClusterMockPrioritySet(); - ~MockClusterMockPrioritySet(); + ~MockClusterMockPrioritySet() override; // Upstream::Cluster MockPrioritySet& prioritySet() override { return priority_set_; } @@ -207,7 +207,7 @@ class MockClusterMockPrioritySet : public MockCluster { class MockLoadBalancer : public LoadBalancer { public: MockLoadBalancer(); - ~MockLoadBalancer(); + ~MockLoadBalancer() override; // Upstream::LoadBalancer MOCK_METHOD1(chooseHost, HostConstSharedPtr(LoadBalancerContext* context)); @@ -218,7 +218,7 @@ class MockLoadBalancer : public LoadBalancer { class MockThreadAwareLoadBalancer : public ThreadAwareLoadBalancer { public: MockThreadAwareLoadBalancer(); - ~MockThreadAwareLoadBalancer(); + ~MockThreadAwareLoadBalancer() override; // Upstream::ThreadAwareLoadBalancer MOCK_METHOD0(factory, LoadBalancerFactorySharedPtr()); @@ -228,7 +228,7 @@ class MockThreadAwareLoadBalancer : public ThreadAwareLoadBalancer { class MockThreadLocalCluster : public ThreadLocalCluster { public: MockThreadLocalCluster(); - ~MockThreadLocalCluster(); + ~MockThreadLocalCluster() override; // Upstream::ThreadLocalCluster MOCK_METHOD0(prioritySet, const PrioritySet&()); @@ -242,7 +242,7 @@ class MockThreadLocalCluster : public ThreadLocalCluster { class MockClusterManagerFactory : public ClusterManagerFactory { public: MockClusterManagerFactory(); - ~MockClusterManagerFactory(); + ~MockClusterManagerFactory() override; Secret::MockSecretManager& secretManager() override { return secret_manager_; }; @@ -275,14 +275,14 @@ class MockClusterManagerFactory : public ClusterManagerFactory { class MockClusterUpdateCallbacksHandle : public ClusterUpdateCallbacksHandle { public: MockClusterUpdateCallbacksHandle(); - ~MockClusterUpdateCallbacksHandle(); + ~MockClusterUpdateCallbacksHandle() override; }; class MockClusterManager : public ClusterManager { public: explicit MockClusterManager(TimeSource& time_source); MockClusterManager(); - ~MockClusterManager(); + ~MockClusterManager() override; ClusterUpdateCallbacksHandlePtr addThreadLocalClusterUpdateCallbacks(ClusterUpdateCallbacks& callbacks) override { @@ -344,7 +344,7 @@ class MockClusterManager : public ClusterManager { class MockHealthChecker : public HealthChecker { public: MockHealthChecker(); - ~MockHealthChecker(); + ~MockHealthChecker() override; MOCK_METHOD1(addHostCheckCompleteCb, void(HostStatusCb callback)); MOCK_METHOD0(start, void()); @@ -377,7 +377,7 @@ class MockHealthCheckEventLogger : public HealthCheckEventLogger { class MockCdsApi : public CdsApi { public: MockCdsApi(); - ~MockCdsApi(); + ~MockCdsApi() override; MOCK_METHOD0(initialize, void()); MOCK_METHOD1(setInitializedCb, void(std::function callback)); @@ -389,7 +389,7 @@ class MockCdsApi : public CdsApi { class MockClusterUpdateCallbacks : public ClusterUpdateCallbacks { public: MockClusterUpdateCallbacks(); - ~MockClusterUpdateCallbacks(); + ~MockClusterUpdateCallbacks() override; MOCK_METHOD1(onClusterAddOrUpdate, void(ThreadLocalCluster& cluster)); MOCK_METHOD1(onClusterRemoval, void(const std::string& cluster_name)); @@ -398,7 +398,7 @@ class MockClusterUpdateCallbacks : public ClusterUpdateCallbacks { class MockClusterInfoFactory : public ClusterInfoFactory, Logger::Loggable { public: MockClusterInfoFactory(); - ~MockClusterInfoFactory(); + ~MockClusterInfoFactory() override; MOCK_METHOD1(createClusterInfo, ClusterInfoConstSharedPtr(const CreateClusterInfoParams&)); }; @@ -406,7 +406,7 @@ class MockClusterInfoFactory : public ClusterInfoFactory, Logger::Loggable port_chains; for (int i = 0; i < input_size; i++) { diff --git a/test/server/guarddog_impl_test.cc b/test/server/guarddog_impl_test.cc index 77b3397ffd59..e4f5d35f8e2c 100644 --- a/test/server/guarddog_impl_test.cc +++ b/test/server/guarddog_impl_test.cc @@ -30,12 +30,12 @@ namespace { class DebugTestInterlock : public GuardDogImpl::TestInterlockHook { public: // GuardDogImpl::TestInterlockHook - virtual void signalFromImpl(MonotonicTime time) { + void signalFromImpl(MonotonicTime time) override { impl_reached_ = time; impl_.notifyAll(); } - virtual void waitFromTest(Thread::MutexBasicLockable& mutex, MonotonicTime time) + void waitFromTest(Thread::MutexBasicLockable& mutex, MonotonicTime time) override EXCLUSIVE_LOCKS_REQUIRED(mutex) { while (impl_reached_ < time) { impl_.wait(mutex); diff --git a/test/server/http/config_tracker_impl_test.cc b/test/server/http/config_tracker_impl_test.cc index d085e2c962af..60d4633f08a7 100644 --- a/test/server/http/config_tracker_impl_test.cc +++ b/test/server/http/config_tracker_impl_test.cc @@ -21,7 +21,7 @@ class ConfigTrackerImplTest : public testing::Test { ProtobufTypes::MessagePtr test_msg() { return std::make_unique(); } - virtual ~ConfigTrackerImplTest() = default; + ~ConfigTrackerImplTest() override = default; ConfigTrackerImpl tracker; const std::map& cbs_map; diff --git a/test/test_common/logging.h b/test/test_common/logging.h index c7e1fed70c9a..62fe9f6d7d53 100644 --- a/test/test_common/logging.h +++ b/test/test_common/logging.h @@ -49,7 +49,7 @@ class LogLevelSetter { class LogRecordingSink : public Logger::SinkDelegate { public: explicit LogRecordingSink(Logger::DelegatingLogSinkPtr log_sink); - virtual ~LogRecordingSink(); + ~LogRecordingSink() override; // Logger::SinkDelegate void log(absl::string_view msg) override; diff --git a/test/test_common/simulated_time_system.cc b/test/test_common/simulated_time_system.cc index 1e700f650832..c4801ae68bda 100644 --- a/test/test_common/simulated_time_system.cc +++ b/test/test_common/simulated_time_system.cc @@ -55,7 +55,7 @@ class SimulatedTimeSystemHelper::Alarm : public Timer { time_system_(time_system), index_(time_system.nextIndex()), armed_(false), pending_(false) { } - virtual ~Alarm(); + ~Alarm() override; // Timer void disableTimer() override; diff --git a/test/test_common/test_time_system.h b/test/test_common/test_time_system.h index bd3c184648ad..4d2ef8196642 100644 --- a/test/test_common/test_time_system.h +++ b/test/test_common/test_time_system.h @@ -15,7 +15,7 @@ class TestTimeSystem; // Adds sleep() and waitFor() interfaces to Event::TimeSystem. class TestTimeSystem : public Event::TimeSystem { public: - virtual ~TestTimeSystem() = default; + ~TestTimeSystem() override = default; /** * Advances time forward by the specified duration, running any timers From c3a7d7d05ad6c8f3f84545aa31d35d6fc7fac98f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Sat, 20 Jul 2019 15:26:19 -0400 Subject: [PATCH 212/542] http_inspector: const declarations (#7656) Risk Level: low Testing: existing Docs Changes: n/a Release Notes: n/a Signed-off-by: Raul Gutierrez Segales --- .../listener/http_inspector/http_inspector.cc | 11 +++++----- .../http_inspector_config_test.cc | 2 +- .../http_inspector/http_inspector_test.cc | 20 +++++++++---------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/extensions/filters/listener/http_inspector/http_inspector.cc b/source/extensions/filters/listener/http_inspector/http_inspector.cc index 332149a86813..7c768ca290df 100644 --- a/source/extensions/filters/listener/http_inspector/http_inspector.cc +++ b/source/extensions/filters/listener/http_inspector/http_inspector.cc @@ -30,9 +30,9 @@ Filter::Filter(const ConfigSharedPtr config) : config_(config) {} Network::FilterStatus Filter::onAccept(Network::ListenerFilterCallbacks& cb) { ENVOY_LOG(debug, "http inspector: new connection accepted"); - Network::ConnectionSocket& socket = cb.socket(); + const Network::ConnectionSocket& socket = cb.socket(); - absl::string_view transport_protocol = socket.detectedTransportProtocol(); + const absl::string_view transport_protocol = socket.detectedTransportProtocol(); if (!transport_protocol.empty() && transport_protocol != TransportSockets::TransportSocketNames::get().RawBuffer) { ENVOY_LOG(trace, "http inspector: cannot inspect http protocol with transport socket {}", @@ -56,8 +56,9 @@ Network::FilterStatus Filter::onAccept(Network::ListenerFilterCallbacks& cb) { void Filter::onRead() { auto& os_syscalls = Api::OsSysCallsSingleton::get(); + const Network::ConnectionSocket& socket = cb_->socket(); const Api::SysCallSizeResult result = - os_syscalls.recv(cb_->socket().ioHandle().fd(), buf_, Config::MAX_INSPECT_SIZE, MSG_PEEK); + os_syscalls.recv(socket.ioHandle().fd(), buf_, Config::MAX_INSPECT_SIZE, MSG_PEEK); ENVOY_LOG(trace, "http inspector: recv: {}", result.rc_); if (result.rc_ == -1 && result.errno_ == EAGAIN) { return; @@ -76,7 +77,7 @@ void Filter::parseHttpHeader(absl::string_view data) { protocol_ = "HTTP/2"; done(true); } else { - size_t pos = data.find_first_of("\r\n"); + const size_t pos = data.find_first_of("\r\n"); // Cannot find \r or \n if (pos == absl::string_view::npos) { @@ -84,7 +85,7 @@ void Filter::parseHttpHeader(absl::string_view data) { return done(false); } - absl::string_view request_line = data.substr(0, pos); + const absl::string_view request_line = data.substr(0, pos); std::vector fields = absl::StrSplit(request_line, absl::MaxSplits(' ', 4)); // Method SP Request-URI SP HTTP-Version diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_config_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_config_test.cc index 4e647709f7d8..3e8bc84d4b72 100644 --- a/test/extensions/filters/listener/http_inspector/http_inspector_config_test.cc +++ b/test/extensions/filters/listener/http_inspector/http_inspector_config_test.cc @@ -21,7 +21,7 @@ TEST(HttpInspectorConfigFactoryTest, TestCreateFactory) { EXPECT_EQ(factory->name(), ListenerFilters::ListenerFilterNames::get().HttpInspector); - std::string yaml = R"EOF( + const std::string yaml = R"EOF( {} )EOF"; diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc index e5cb254902c8..bda4ed6602ba 100644 --- a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc +++ b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc @@ -41,7 +41,7 @@ class HttpInspectorTest : public testing::Test { EXPECT_CALL(cb_, socket()).WillRepeatedly(ReturnRef(socket_)); EXPECT_CALL(socket_, detectedTransportProtocol()).WillRepeatedly(Return("raw_buffer")); EXPECT_CALL(cb_, dispatcher()).WillRepeatedly(ReturnRef(dispatcher_)); - EXPECT_CALL(socket_, ioHandle()).WillRepeatedly(ReturnRef(*io_handle_)); + EXPECT_CALL(testing::Const(socket_), ioHandle()).WillRepeatedly(ReturnRef(*io_handle_)); EXPECT_CALL(dispatcher_, createFileEvent_(_, _, Event::FileTriggerType::Edge, Event::FileReadyType::Read)) @@ -72,7 +72,7 @@ TEST_F(HttpInspectorTest, SkipHttpInspectForTLS) { TEST_F(HttpInspectorTest, InspectHttp10) { init(); - absl::string_view header = + const absl::string_view header = "GET /anything HTTP/1.0\r\nhost: google.com\r\nuser-agent: curl/7.64.0\r\naccept: " "*/*\r\nx-forwarded-proto: http\r\nx-request-id: " "a52df4a0-ed00-4a19-86a7-80e5049c6c84\r\nx-envoy-expected-rq-timeout-ms: " @@ -95,7 +95,7 @@ TEST_F(HttpInspectorTest, InspectHttp10) { TEST_F(HttpInspectorTest, InspectHttp11) { init(); - absl::string_view header = + const absl::string_view header = "GET /anything HTTP/1.1\r\nhost: google.com\r\nuser-agent: curl/7.64.0\r\naccept: " "*/*\r\nx-forwarded-proto: http\r\nx-request-id: " "a52df4a0-ed00-4a19-86a7-80e5049c6c84\r\nx-envoy-expected-rq-timeout-ms: " @@ -118,7 +118,7 @@ TEST_F(HttpInspectorTest, InspectHttp11) { TEST_F(HttpInspectorTest, InvalidHttpMethod) { init(); - absl::string_view header = "BAD /anything HTTP/1.1\r\n"; + const absl::string_view header = "BAD /anything HTTP/1.1\r\n"; EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { @@ -137,7 +137,7 @@ TEST_F(HttpInspectorTest, InvalidHttpMethod) { TEST_F(HttpInspectorTest, InvalidHttpRequestLine) { init(); - absl::string_view header = "BAD /anything HTTP/1.1"; + const absl::string_view header = "BAD /anything HTTP/1.1"; EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { @@ -154,7 +154,7 @@ TEST_F(HttpInspectorTest, InvalidHttpRequestLine) { TEST_F(HttpInspectorTest, UnsupportedHttpProtocol) { init(); - absl::string_view header = "GET /anything HTTP/0.9\r\n"; + const absl::string_view header = "GET /anything HTTP/0.9\r\n"; EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { @@ -171,7 +171,7 @@ TEST_F(HttpInspectorTest, UnsupportedHttpProtocol) { TEST_F(HttpInspectorTest, InvalidRequestLine) { init(); - absl::string_view header = "GET /anything HTTP/1.1 BadRequestLine\r\n"; + const absl::string_view header = "GET /anything HTTP/1.1 BadRequestLine\r\n"; EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) .WillOnce(Invoke([&header](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { @@ -189,7 +189,7 @@ TEST_F(HttpInspectorTest, InvalidRequestLine) { TEST_F(HttpInspectorTest, InspectHttp2) { init(); - std::string header = + const std::string header = "505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00000c04000000000000041000000000020000000000" "00040800000000000fff000100007d010500000001418aa0e41d139d09b8f0000f048860757a4ce6aa660582867a" "8825b650c3abb8d2e053032a2f2a408df2b4a7b3c0ec90b22d5d8749ff839d29af4089f2b585ed6950958d279a18" @@ -215,8 +215,8 @@ TEST_F(HttpInspectorTest, InspectHttp2) { TEST_F(HttpInspectorTest, InvalidConnectionPreface) { init(); - std::string header = "505249202a20485454502f322e300d0a"; - std::vector data = Hex::decode(header); + const std::string header = "505249202a20485454502f322e300d0a"; + const std::vector data = Hex::decode(header); EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) .WillOnce(Invoke([&data](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { From 7e7486588239f42b4be3a6383e3c0776d2d05bef Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Sun, 21 Jul 2019 01:05:19 +0530 Subject: [PATCH 213/542] config: change default initial fetch timeout to 15s (#7571) Signed-off-by: Rama Chavali --- api/envoy/api/v2/core/config_source.proto | 3 +-- .../intro/arch_overview/operations/init.rst | 18 +++++++++++------- docs/root/intro/version_history.rst | 1 + source/common/config/utility.cc | 2 +- .../config/subscription_factory_impl_test.cc | 6 +++--- test/common/config/utility_test.cc | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/api/envoy/api/v2/core/config_source.proto b/api/envoy/api/v2/core/config_source.proto index 8b6014dcbf9d..913f3876115c 100644 --- a/api/envoy/api/v2/core/config_source.proto +++ b/api/envoy/api/v2/core/config_source.proto @@ -112,13 +112,12 @@ message ConfigSource { AggregatedConfigSource ads = 3; } - // Optional initialization timeout. // When this timeout is specified, Envoy will wait no longer than the specified time for first // config response on this xDS subscription during the :ref:`initialization process // `. After reaching the timeout, Envoy will move to the next // initialization phase, even if the first config is not delivered yet. The timer is activated // when the xDS API subscription starts, and is disarmed on first config update or on error. 0 // means no timeout - Envoy will wait indefinitely for the first xDS config (unless another - // timeout applies). Default 0. + // timeout applies). The default is 15s. google.protobuf.Duration initial_fetch_timeout = 4; } diff --git a/docs/root/intro/arch_overview/operations/init.rst b/docs/root/intro/arch_overview/operations/init.rst index 2e05c5a75056..4ce245d78f51 100644 --- a/docs/root/intro/arch_overview/operations/init.rst +++ b/docs/root/intro/arch_overview/operations/init.rst @@ -10,20 +10,24 @@ accepting new connections. * During startup, the :ref:`cluster manager ` goes through a multi-phase initialization where it first initializes static/DNS clusters, then predefined :ref:`EDS ` clusters. Then it initializes - :ref:`CDS ` if applicable, waits for one response (or failure), + :ref:`CDS ` if applicable, waits for one response (or failure) + for a :ref:`bounded period of time `, and does the same primary/secondary initialization of CDS provided clusters. * If clusters use :ref:`active health checking `, Envoy also does a single active health check round. * Once cluster manager initialization is done, :ref:`RDS ` and - :ref:`LDS ` initialize (if applicable). The server - doesn't start accepting connections until there has been at least one response (or failure) for - LDS/RDS requests. -* If LDS itself returns a listener that needs an RDS response, Envoy further waits until an RDS + :ref:`LDS ` initialize (if applicable). The server waits + for a :ref:`bounded period of time ` + for at least one response (or failure) for LDS/RDS requests. After which, it starts accepting connections. +* If LDS itself returns a listener that needs an RDS response, Envoy further waits for + a :ref:`bounded period of time ` until an RDS response (or failure) is received. Note that this process takes place on every future listener addition via LDS and is known as :ref:`listener warming `. * After all of the previous steps have taken place, the listeners start accepting new connections. This flow ensures that during hot restart the new process is fully capable of accepting and processing new connections before the draining of the old process begins. -All mentioned "waiting for one response" periods can be limited by setting corresponding -:ref:`initial_fetch_timeout `. +A key design principle of initialization is that an Envoy is always guaranteed to initialize within +:ref:`initial_fetch_timeout `, +with a best effort made to obtain the complete set of xDS configuration within that subject to the +management server availability. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ab727239f444..e38946186a4e 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -5,6 +5,7 @@ Version history ================ * admin: added ability to configure listener :ref:`socket options `. * config: async data access for local and remote data source. +* config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * listeners: added :ref:`HTTP inspector listener filter `. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. diff --git a/source/common/config/utility.cc b/source/common/config/utility.cc index d22cc0a4cca6..84dedbcda38f 100644 --- a/source/common/config/utility.cc +++ b/source/common/config/utility.cc @@ -180,7 +180,7 @@ std::chrono::milliseconds Utility::apiConfigSourceRequestTimeout( std::chrono::milliseconds Utility::configSourceInitialFetchTimeout(const envoy::api::v2::core::ConfigSource& config_source) { return std::chrono::milliseconds( - PROTOBUF_GET_MS_OR_DEFAULT(config_source, initial_fetch_timeout, 0)); + PROTOBUF_GET_MS_OR_DEFAULT(config_source, initial_fetch_timeout, 15000)); } void Utility::translateRdsConfig( diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index 9c10b7dad282..c72b02d14608 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -226,7 +226,7 @@ TEST_F(SubscriptionFactoryTest, HttpSubscriptionCustomRequestTimeout) { EXPECT_CALL(cm_, clusters()).WillOnce(Return(cluster_map)); EXPECT_CALL(cluster, info()).Times(2); EXPECT_CALL(*cluster.info_, addedViaApi()); - EXPECT_CALL(dispatcher_, createTimer_(_)); + EXPECT_CALL(dispatcher_, createTimer_(_)).Times(2); EXPECT_CALL(cm_, httpAsyncClientForCluster("static_cluster")); EXPECT_CALL( cm_.async_client_, @@ -246,7 +246,7 @@ TEST_F(SubscriptionFactoryTest, HttpSubscription) { EXPECT_CALL(cm_, clusters()).WillOnce(Return(cluster_map)); EXPECT_CALL(cluster, info()).Times(2); EXPECT_CALL(*cluster.info_, addedViaApi()); - EXPECT_CALL(dispatcher_, createTimer_(_)); + EXPECT_CALL(dispatcher_, createTimer_(_)).Times(2); EXPECT_CALL(cm_, httpAsyncClientForCluster("static_cluster")); EXPECT_CALL(cm_.async_client_, send_(_, _, _)) .WillOnce(Invoke([this](Http::MessagePtr& request, Http::AsyncClient::Callbacks&, @@ -301,7 +301,7 @@ TEST_F(SubscriptionFactoryTest, GrpcSubscription) { return async_client_factory; })); EXPECT_CALL(random_, random()); - EXPECT_CALL(dispatcher_, createTimer_(_)); + EXPECT_CALL(dispatcher_, createTimer_(_)).Times(2); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); subscriptionFromConfigSource(config)->start({"static_cluster"}); } diff --git a/test/common/config/utility_test.cc b/test/common/config/utility_test.cc index 30d8fa8b3f3b..90573d4f7d9b 100644 --- a/test/common/config/utility_test.cc +++ b/test/common/config/utility_test.cc @@ -55,7 +55,7 @@ TEST(UtilityTest, ApiConfigSourceRequestTimeout) { TEST(UtilityTest, ConfigSourceDefaultInitFetchTimeout) { envoy::api::v2::core::ConfigSource config_source; - EXPECT_EQ(0, Utility::configSourceInitialFetchTimeout(config_source).count()); + EXPECT_EQ(15000, Utility::configSourceInitialFetchTimeout(config_source).count()); } TEST(UtilityTest, ConfigSourceInitFetchTimeout) { From 09139a01a964c3b414feefe49f16f7616a293516 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Sun, 21 Jul 2019 14:23:06 -0700 Subject: [PATCH 214/542] clang-tidy: fix existing errors on master branch (#7664) Description: Fix existing violations of `performance-inefficient-vector-operation` and `readability-redundant-control-flow`. Risk Level: Low Testing: existing Docs Changes: N/A Release Notes: N/A Signed-off-by: Derek Argueta --- test/server/filter_chain_benchmark_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/server/filter_chain_benchmark_test.cc b/test/server/filter_chain_benchmark_test.cc index 3cc2788df84f..0517635414c7 100644 --- a/test/server/filter_chain_benchmark_test.cc +++ b/test/server/filter_chain_benchmark_test.cc @@ -85,7 +85,7 @@ class MockConnectionSocket : public Network::ConnectionSocket { return Network::Address::SocketType::Stream; } void setLocalAddress(const Network::Address::InstanceConstSharedPtr&) override {} - void restoreLocalAddress(const Network::Address::InstanceConstSharedPtr&) override { return; } + void restoreLocalAddress(const Network::Address::InstanceConstSharedPtr&) override {} void setRemoteAddress(const Network::Address::InstanceConstSharedPtr&) override {} bool localAddressRestored() const override { return true; } void setDetectedTransportProtocol(absl::string_view) override {} @@ -152,6 +152,7 @@ class FilterChainBenchmarkFixture : public benchmark::Fixture { void SetUp(const ::benchmark::State& state) override { int64_t input_size = state.range(0); std::vector port_chains; + port_chains.reserve(input_size); for (int i = 0; i < input_size; i++) { port_chains.push_back(absl::StrCat(YamlSingleDstPortTop, 10000 + i, YamlSingleDstPortBottom)); } From 81654d18dfe211d739082c922776cf18bbf03275 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Sun, 21 Jul 2019 19:13:26 -0700 Subject: [PATCH 215/542] forward proxy: ip host headers (#7565) Support IP addresses in host headers. This completes the MVP of dynamic forward proxy. Fixes https://github.com/envoyproxy/envoy/issues/1606 Signed-off-by: Matt Klein --- docs/root/intro/version_history.rst | 2 + .../clusters/dynamic_forward_proxy/cluster.cc | 4 +- .../common/dynamic_forward_proxy/dns_cache.h | 5 + .../dynamic_forward_proxy/dns_cache_impl.cc | 42 ++++-- .../dynamic_forward_proxy/dns_cache_impl.h | 8 +- source/extensions/transport_sockets/tls/BUILD | 1 + .../transport_sockets/tls/context_impl.cc | 36 ++++- .../certs/upstreamlocalhostcert.cfg | 4 +- .../certs/upstreamlocalhostcert.pem | 36 ++--- .../certs/upstreamlocalhostcert_hash.h | 4 +- .../certs/upstreamlocalhostkey.pem | 50 +++---- .../dynamic_forward_proxy/cluster_test.cc | 1 + .../dns_cache_impl_test.cc | 127 ++++++++++++++++-- .../common/dynamic_forward_proxy/mocks.h | 1 + .../proxy_filter_integration_test.cc | 25 ++++ test/extensions/transport_sockets/tls/BUILD | 3 + .../transport_sockets/tls/ssl_socket_test.cc | 46 +++++++ 17 files changed, 316 insertions(+), 79 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index e38946186a4e..8442328f715e 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -8,6 +8,8 @@ Version history * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * listeners: added :ref:`HTTP inspector listener filter `. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. +* tls: added verification of IP address SAN fields in certificates against configured SANs in the + certificate validation context. 1.11.0 (July 11, 2019) ====================== diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc index 37bfa8f6f5b5..44efc1b24f31 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.cc +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.cc @@ -84,10 +84,10 @@ void Cluster::onDnsHostAddOrUpdate( // Create an override transport socket options that automatically provides both SNI as well as // SAN verification for the resolved host if the cluster has been configured with TLS. - // TODO(mattklein123): If the host is an IP address we should not set SNI. Network::TransportSocketOptionsSharedPtr transport_socket_options = std::make_shared( - host_info->resolvedHost(), std::vector{host_info->resolvedHost()}); + !host_info->isIpAddress() ? host_info->resolvedHost() : "", + std::vector{host_info->resolvedHost()}); const auto new_host_map = std::make_shared(*current_map); const auto emplaced = diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache.h b/source/extensions/common/dynamic_forward_proxy/dns_cache.h index 3a0fb4580ef3..b01cc478f660 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache.h @@ -29,6 +29,11 @@ class DnsHostInfo { */ virtual const std::string& resolvedHost() PURE; + /** + * Returns whether the original host is an IP address. + */ + virtual bool isIpAddress() PURE; + /** * Indicates that the host has been used and should not be purged depending on any configured * TTL policy diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc index 0e9a978ff40f..bd206039920d 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc @@ -79,34 +79,54 @@ void DnsCacheImpl::startCacheLoad(const std::string& host, uint16_t default_port return; } - // TODO(mattklein123): Figure out if we want to support addresses of the form :. This - // seems unlikely to be useful in TLS scenarios, but it is technically - // supported. We might want to block this form for now. - const auto colon_pos = host.find(':'); + // First try to see if there is a port included. This also checks to see that there is not a ']' + // as the last character which is indicative of an IPv6 address without a port. This is a best + // effort attempt. + const auto colon_pos = host.rfind(':'); absl::string_view host_to_resolve = host; - if (colon_pos != absl::string_view::npos) { + if (colon_pos != absl::string_view::npos && host_to_resolve.back() != ']') { const absl::string_view string_view_host = host; host_to_resolve = string_view_host.substr(0, colon_pos); const auto port_str = string_view_host.substr(colon_pos + 1); uint64_t port64; if (port_str.empty() || !absl::SimpleAtoi(port_str, &port64) || port64 > 65535) { // Just attempt to resolve whatever we were given. This will very likely fail. - // TODO(mattklein123): Should we actually fail here or do something different? host_to_resolve = host; } else { default_port = port64; } } + // Now see if this is an IP address. We need to know this because some things (such as setting + // SNI) are special cased if this is an IP address. Either way, we still go through the normal + // resolver flow. We could short-circuit the DNS resolver in this case, but the extra code to do + // so is not worth it since the DNS resolver should handle it for us. + bool is_ip_address = false; + try { + absl::string_view potential_ip_address = host_to_resolve; + // TODO(mattklein123): Optimally we would support bracket parsing in parseInternetAddress(), + // but we still need to trim the brackets to send the IPv6 address into the DNS resolver. For + // now, just do all the trimming here, but in the future we should consider whether we can + // have unified [] handling as low as possible in the stack. + if (potential_ip_address.front() == '[' && potential_ip_address.back() == ']') { + potential_ip_address.remove_prefix(1); + potential_ip_address.remove_suffix(1); + } + Network::Utility::parseInternetAddress(std::string(potential_ip_address)); + is_ip_address = true; + host_to_resolve = potential_ip_address; + } catch (const EnvoyException&) { + } + // TODO(mattklein123): Right now, the same host with different ports will become two // independent primary hosts with independent DNS resolutions. I'm not sure how much this will // matter, but we could consider collapsing these down and sharing the underlying DNS resolution. auto& primary_host = *primary_hosts_ // try_emplace() is used here for direct argument forwarding. - .try_emplace(host, - std::make_unique(*this, host_to_resolve, default_port, - [this, host]() { onReResolve(host); })) + .try_emplace(host, std::make_unique( + *this, host_to_resolve, default_port, is_ip_address, + [this, host]() { onReResolve(host); })) .first->second; startResolve(host, primary_host); } @@ -244,11 +264,11 @@ void DnsCacheImpl::ThreadLocalHostInfo::updateHostMap(const TlsHostMapSharedPtr& DnsCacheImpl::PrimaryHostInfo::PrimaryHostInfo(DnsCacheImpl& parent, absl::string_view host_to_resolve, uint16_t port, - const Event::TimerCb& timer_cb) + bool is_ip_address, const Event::TimerCb& timer_cb) : parent_(parent), port_(port), refresh_timer_(parent.main_thread_dispatcher_.createTimer(timer_cb)), host_info_(std::make_shared(parent.main_thread_dispatcher_.timeSource(), - host_to_resolve)) { + host_to_resolve, is_ip_address)) { parent_.stats_.host_added_.inc(); parent_.stats_.num_hosts_.inc(); } diff --git a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h index af3b81704b38..32c887f2284e 100644 --- a/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h +++ b/source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h @@ -71,18 +71,20 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable + #include #include #include @@ -13,6 +15,7 @@ #include "common/common/fmt.h" #include "common/common/hex.h" #include "common/common/utility.h" +#include "common/network/address_impl.h" #include "common/protobuf/utility.h" #include "extensions/transport_sockets/tls/utility.h" @@ -491,7 +494,8 @@ bool ContextImpl::verifySubjectAltName(X509* cert, return false; } for (const GENERAL_NAME* san : san_names.get()) { - if (san->type == GEN_DNS) { + switch (san->type) { + case GEN_DNS: { ASN1_STRING* str = san->d.dNSName; const char* dns_name = reinterpret_cast(ASN1_STRING_data(str)); for (auto& config_san : subject_alt_names) { @@ -499,7 +503,9 @@ bool ContextImpl::verifySubjectAltName(X509* cert, return true; } } - } else if (san->type == GEN_URI) { + break; + } + case GEN_URI: { ASN1_STRING* str = san->d.uniformResourceIdentifier; const char* uri = reinterpret_cast(ASN1_STRING_data(str)); for (auto& config_san : subject_alt_names) { @@ -507,6 +513,32 @@ bool ContextImpl::verifySubjectAltName(X509* cert, return true; } } + break; + } + case GEN_IPADD: { + if (san->d.ip->length == 4) { + sockaddr_in sin; + sin.sin_family = AF_INET; + memcpy(&sin.sin_addr, san->d.ip->data, sizeof(sin.sin_addr)); + Network::Address::Ipv4Instance addr(&sin); + for (auto& config_san : subject_alt_names) { + if (config_san == addr.ip()->addressAsString()) { + return true; + } + } + } else if (san->d.ip->length == 16) { + sockaddr_in6 sin6; + sin6.sin6_family = AF_INET6; + memcpy(&sin6.sin6_addr, san->d.ip->data, sizeof(sin6.sin6_addr)); + Network::Address::Ipv6Instance addr(sin6); + for (auto& config_san : subject_alt_names) { + if (config_san == addr.ip()->addressAsString()) { + return true; + } + } + } + break; + } } } return false; diff --git a/test/config/integration/certs/upstreamlocalhostcert.cfg b/test/config/integration/certs/upstreamlocalhostcert.cfg index fbea513733e9..68c182e2c9f0 100644 --- a/test/config/integration/certs/upstreamlocalhostcert.cfg +++ b/test/config/integration/certs/upstreamlocalhostcert.cfg @@ -34,5 +34,5 @@ authorityKeyIdentifier = keyid:always [alt_names] DNS.2 = localhost -IP.1 = 0.0.0.0 -IP.2 = :: +IP.1 = 127.0.0.1 +IP.2 = ::1 diff --git a/test/config/integration/certs/upstreamlocalhostcert.pem b/test/config/integration/certs/upstreamlocalhostcert.pem index e34ab833d86c..169d1c63e568 100644 --- a/test/config/integration/certs/upstreamlocalhostcert.pem +++ b/test/config/integration/certs/upstreamlocalhostcert.pem @@ -1,25 +1,25 @@ -----BEGIN CERTIFICATE----- -MIIEPTCCAyWgAwIBAgIUS0ht/ypqxlVqt86GiCya6cw/jJ0wDQYJKoZIhvcNAQEL +MIIEPTCCAyWgAwIBAgIUfoTig3pqtlASJyyhMZ2/x0Hg33UwDQYJKoZIhvcNAQEL BQAwfzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFjAUBgNVBAcM DVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsMEEx5ZnQgRW5n -aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMTkwNzA4MjE0 -NTU3WhcNMjEwNzA3MjE0NTU3WjCBgzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh +aW5lZXJpbmcxGTAXBgNVBAMMEFRlc3QgVXBzdHJlYW0gQ0EwHhcNMTkwNzEyMjI0 +MzQ1WhcNMjEwNzExMjI0MzQ1WjCBgzELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh bGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQx GTAXBgNVBAsMEEx5ZnQgRW5naW5lZXJpbmcxHTAbBgNVBAMMFFRlc3QgVXBzdHJl -YW0gU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAujOqV+UB -T8oKxxnIlgKMPn10hIZxOOEzA96CDMQtQ2+180HLfSTErLWzQFNeP6jRDcbXTN0w -tYJlIUmVJtPaj7Dh4VvpORhRwAPZt9bkHcKKFCIaGYj61YCv3YpNyBSfJ0vwgATD -Yn6I2R8nobMKau/hMk4SpPZ6Z3pwSEt0GHd9/cE7t1WvE4BhqIjznexeFO+YrgvF -2ea4j7u4hJxezZhzAqOUyqtlbfkHQwXXzg/93PxBY5Y1mUPszjY+doGhW3DfTI1O -qgU2OfAoFZ6SKtUphUG/gt5DKHvKeARCWMEaUXC9UkyzhSNIl7s8qnRzweZwyOIk -KClryNQCtTjHOwIDAQABo4GrMIGoMAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgXg +YW0gU2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApWnYvUff +dh+TcLEYxQiw+ZUaGfBedmVmaxOHAbsWwBcMcwt3ITAjRPLPFEUt/DUxgmXO80zo +6YOc9uUIUGU0vqIFTQP3JfS9kMevrvQIkZbsO2rtNMYQ+F7HOmGUS3RiKdNNbHnX +NKKPsHe/UFiFCBwxVCT+NSGI3yHZFUSFlvH9BEO+a1lx4pp2R7UJTLdBVaGx42t1 +pTTR2E2S6E1tKXhtS/qN4+X+dDaUFfi7mz+QiNFsYKu5QgO9f8ewfdOPj9MImZvG +4l37/eNtNjzwTMx8Ph4moTSo4yH9ZBGHffPDm4ljPXpLUfiVyIvzR6ygG468ODcP +umQjprHM3TTxlQIDAQABo4GrMIGoMAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgXg MB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAsBgNVHREEJTAjgglsb2Nh -bGhvc3SHBAAAAACHEAAAAAAAAAAAAAAAAAAAAAAwHQYDVR0OBBYEFPlBuX/WSFDb -+nUGMLTu7svrrtolMB8GA1UdIwQYMBaAFOLTMLryzNAcuxe3cKEhClkL7IduMA0G -CSqGSIb3DQEBCwUAA4IBAQCgknWXc/Iz/Av6inDxGOncsNlYehKU+UBoR69HlcUE -AEOW7nFaPey5zLL3dgTJd1nOe0u97yT5Hoy9b7O9z5cBWHkNYqFh2oEZKXeDtS81 -z/N4ZQuPxxAlS4d7krAsQNB2vjMFp81eGude680twbto6LKRg9iJMv+AeEJD9p0j -ubeZA20j8YV8Aijm/kNe82d+TQYULxQeLo5QM6VU0pK2VcCunHywcYFc/t99Ync6 -NaqGrxOu6Jfduzg0TsZsIX7GveYC4dmx3CK1qOSB7SE2SVQjAITZL7gIvbLQPEKu -XJrmpIIhgUw+AgPq7D8JEaLoYERCRQLWt4v/yVus2Tgd +bGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwHQYDVR0OBBYEFFhoq2tgE+IN +uLyrUa1YpgsOpkwTMB8GA1UdIwQYMBaAFOLTMLryzNAcuxe3cKEhClkL7IduMA0G +CSqGSIb3DQEBCwUAA4IBAQB6jYcrAtO+8rLWdBp2W2tj9lgTHE8+W9BzMONrFwVo +PNMPHKPl2XfqKbv64sW3Z9sIA7ThSUu0nAe8i3ddaL4xQvPnaOwTdSnhykyMg2Hp +dDVAT8nCQb7Q87cuiFKE7o87XaQeOS3hgKeQ4uKexA48MkKwNYTVRMb7iC64yZcg +DY5H0nxRISs6pIAD9SPvPOQv+KZ35/LDJy59Kvso1zPoM9e+CJcdHU3hKM3RBK5J +FOjDL8qFair7vyjO9DUPDgoAZJntCQYGC0ToYxTe8cQVJD+sW5gqqIBLRchMr3gL +ni7rAL+oUyufHIEu3upRId+FdVF/hQMCLF2xk54/KX0V -----END CERTIFICATE----- diff --git a/test/config/integration/certs/upstreamlocalhostcert_hash.h b/test/config/integration/certs/upstreamlocalhostcert_hash.h index 409ad0af1cd3..9f81e65fcd83 100644 --- a/test/config/integration/certs/upstreamlocalhostcert_hash.h +++ b/test/config/integration/certs/upstreamlocalhostcert_hash.h @@ -1,4 +1,4 @@ // NOLINT(namespace-envoy) constexpr char TEST_UPSTREAMLOCALHOST_CERT_HASH[] = - "5B:5C:02:47:DE:17:B7:1B:98:05:0A:DB:41:2C:F6:8F:65:E3:86:E6:03:B3:9A:EC:67:33:2E:39:1F:05:88:" - "B0"; + "44:A1:C4:AD:71:B5:AE:A0:A2:24:63:DA:C8:FF:0C:FC:26:6E:4B:D7:08:DE:64:60:34:A0:72:42:6E:18:BA:" + "87"; diff --git a/test/config/integration/certs/upstreamlocalhostkey.pem b/test/config/integration/certs/upstreamlocalhostkey.pem index 7bf369f08b6d..5331d810952c 100644 --- a/test/config/integration/certs/upstreamlocalhostkey.pem +++ b/test/config/integration/certs/upstreamlocalhostkey.pem @@ -1,27 +1,27 @@ -----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAujOqV+UBT8oKxxnIlgKMPn10hIZxOOEzA96CDMQtQ2+180HL -fSTErLWzQFNeP6jRDcbXTN0wtYJlIUmVJtPaj7Dh4VvpORhRwAPZt9bkHcKKFCIa -GYj61YCv3YpNyBSfJ0vwgATDYn6I2R8nobMKau/hMk4SpPZ6Z3pwSEt0GHd9/cE7 -t1WvE4BhqIjznexeFO+YrgvF2ea4j7u4hJxezZhzAqOUyqtlbfkHQwXXzg/93PxB -Y5Y1mUPszjY+doGhW3DfTI1OqgU2OfAoFZ6SKtUphUG/gt5DKHvKeARCWMEaUXC9 -UkyzhSNIl7s8qnRzweZwyOIkKClryNQCtTjHOwIDAQABAoIBABS0H/Gr9exgQ7iF -pmb/m4ZrPqRpqncvmxOIDx/KRFomNq34l96vUur9PRQe8PDVHYGRpWjXg037VLFR -1DLABaJKgaMkLBd8G8Lk6rVlQHIKqn24mPxT3cgVifhxI1rm6BdfeztQzETMWv0B -WM/C75qaV4jXY31SJqQQ2iE/uoXpuqHtBqxVE9TnBi0NvN2ZXlcxGgwnv7SYKrkE -4c9M5q0F5mJAAkHsrgyyQY18/op/vtQGbKvsdkuT9ihzruaeXxB6M04nevxdDn3r -dC5GUz05c+GCqCmMppU2gRPKYH0t/mvXfZhoGiOujfhTUbiIjQBle1kaxXmZl7Zr -Kz+lO+kCgYEA8IOstfsxJ6Zdf4Xvqi6MLr3MKZT3MmSpKT1dOixzYtD1IS1qc0hF -t4+SNhlP1ny5TlFF12356AC4TMM+kjlhCu9uAsvjqI5M/XSQ00SvybADbYoQc7jy -v9d69AEPOslmfQ7GYDAlFKT5NTK6tXEVpX1MIs6+LSl2pi4emtW7EzUCgYEAxjDG -Db/yEzanLyPC2Yfjk8KotnmprVZmXvmSWl0frOjWxbmYq2auVMNgBysY0+k8OHWk -MQjtjbVqA5b4Ze5Rm0Z7XlWieioxLt1naG3Gb0NAtaivmouaovlrIlZf7GtT4haC -Q4/GN9gGUxdjLLJRJ/WcdOG4X4gp7Opj19eazq8CgYAXVF5rZIs3El8dYIuH0W4N -lqF4Ixf7TmJOOsKRQwCKRESSzEn4FrmUfZusHbZt0rlSzHVe2S8VfwRhhcrK+j/c -hK8CHG7fybXUG/t0UsROZwFeHbdM0lLRowAtLPEiPajwVn+Nkv31y67UpzAPK4Hz -BH1fHvi5fr0gj3auhC7aRQKBgEEAAhTEXSp8BDzrp54ceUEe2KJwKHwXGCASDjPg -0uCsxLO4eR/N32MhaL8xHUVy+zMxMhZ67R5K32gp/XHAxbb9WLzJrS4P5G2QY7fW -OPyIvBJYLq+rFZ5Z2w858N/jG3HNHA/4eXQbP4fE5dvk58UJQrT6yrNaPxXakcBa -kAU1AoGBAJSSjgV9kR2tGzhBFqL0K11E5GP9uzO3sib3Zd7gtOJTO1cvJaLES3BV -Q6sxR9gPJ5cSpTXCNbaJwZ+jAsTfhJj8PE15am/JG7d7xZkflZIdfSf9/23g26w1 -dagOuDPRC2mcbjzprdPRLbNk3NfI/Llw+CboMP6R/smOYf/HRpS9 +MIIEowIBAAKCAQEApWnYvUffdh+TcLEYxQiw+ZUaGfBedmVmaxOHAbsWwBcMcwt3 +ITAjRPLPFEUt/DUxgmXO80zo6YOc9uUIUGU0vqIFTQP3JfS9kMevrvQIkZbsO2rt +NMYQ+F7HOmGUS3RiKdNNbHnXNKKPsHe/UFiFCBwxVCT+NSGI3yHZFUSFlvH9BEO+ +a1lx4pp2R7UJTLdBVaGx42t1pTTR2E2S6E1tKXhtS/qN4+X+dDaUFfi7mz+QiNFs +YKu5QgO9f8ewfdOPj9MImZvG4l37/eNtNjzwTMx8Ph4moTSo4yH9ZBGHffPDm4lj +PXpLUfiVyIvzR6ygG468ODcPumQjprHM3TTxlQIDAQABAoIBAH2pMnFg937qL/0N +XM7acm+4eLK561kwYST5GbgT5A2btOZ1EFRTGIgZmX1BrNSLqIfyRcyJYet8A7OA +fNdueypTNYmzeH8KNTSWrn1PgG7x45aj/X349g1pGxrb5GeKC8TQdGHzEa03zcb2 +wY0NIkrt9/9/durwBeXU9fB1NLNc++Gbrok7kvbDcU6jN8Fas1H86beqhjTohkcN +C0lXk+VLi6m/lBMMjMlezqvTevMdBAjp0LfjwKLMMQE2JhqiS3GxVXhn9N3v0PoP +wI3PuwiUmcIMSY5LvPWD0iy5e1yNFNxJRoTNnnpLnuUmymBSzEeZp4XPrtMefA84 +5Q5BOeECgYEA0yK96/qyTnYcvflqLYukz1YZT029haRR+Yd+TLegVgC+j34q1IuC +TDc2ypHbjwWzkzZYz/6VFMLDXLCPimVS9rpn8vaunZDCLH1xTJbFSTjHOcx83RMT +EBiNH07R33UfdRGJtL5xoLj1DI971wy6LR3yuCzu04VolCMu+PO3Zx0CgYEAyI/t +mF96dZhHB8DDkVe+MRVOuazfkq5JtoUlv74mtKAbx15Jk+iy7G37qHNLRHCGegYj +7PQDZhuGbrda3tce956d9SqpA6uZRqp/aTQBWTfHe6OanXvHPKG1q1hTmfSuWi7/ +Izjh1AGDIaPxtWgx5v7AU8mX4UifhlTvl9KwktkCgYEAug4rfv/0kN/UhDR+NJSS +L4OX2iKPmG0tL88OpVxLln4hbyGnbJVjxPYC+o9+A5LqpBeIPAIELb9TmSKd2z9e +1L1/TMPFLGScN8hzRyK1x8iZB34Dqm1cpxp7gdNbbqcviWJjDzujthZHG0J1xxQY +HBoAAfzWmN8/QQugIRHj1KECgYA1Uo7IxBm6yhGYbheQvNNEGXYkx2FpjgzrCdtP +by67NxYrm1XUjTmEwnj2ADEysPgP2TIT/YwpyYekR/tQ48DH9NPqKr1kzGqj7xCQ +19LD9aCDrquc0xvVcujp9UHE3Ni+AWCz7Jud0gkbGItav6kE0RYxMJfAvZ4sCMjq +hImNgQKBgBHs0Hhg+pie4VJfXh3sxb6Px+WV7UlTyNyTbDMRgq5RKxrisA4BpSax +CjvX/IpTdsP5pNCjAdRK3zs/XC10/wa4wHmk0cXHM9IAFgIncVOuHCkZQyzQ1JJA +cbG7OQFr/UiHJuafueERY2HukfPHfgIUVM3MsXXNR5zIypseBQLW -----END RSA PRIVATE KEY----- diff --git a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc index 6d8db1e762b7..7e5ef2ae8f74 100644 --- a/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc +++ b/test/extensions/clusters/dynamic_forward_proxy/cluster_test.cc @@ -67,6 +67,7 @@ class ClusterTest : public testing::Test, // Allow touch() to still be strict. EXPECT_CALL(*host_map_[host], address()).Times(AtLeast(0)); + EXPECT_CALL(*host_map_[host], isIpAddress()).Times(AtLeast(0)); EXPECT_CALL(*host_map_[host], resolvedHost()).Times(AtLeast(0)); } diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index 36ebfec8391c..7f69ca894a68 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -58,10 +58,22 @@ class DnsCacheImplTest : public testing::Test, public Event::TestUsingSimulatedT DnsCache::AddUpdateCallbacksHandlePtr update_callbacks_handle_; }; -MATCHER_P(SharedAddressEquals, expected, "") { - const bool equal = expected == arg->address()->asString(); +MATCHER_P3(DnsHostInfoEquals, address, resolved_host, is_ip_address, "") { + bool equal = address == arg->address()->asString(); if (!equal) { - *result_listener << fmt::format("'{}' != '{}'", expected, arg->address()->asString()); + *result_listener << fmt::format("address '{}' != '{}'", address, arg->address()->asString()); + return equal; + } + equal &= resolved_host == arg->resolvedHost(); + if (!equal) { + *result_listener << fmt::format("resolved_host '{}' != '{}'", resolved_host, + arg->resolvedHost()); + return equal; + } + equal &= is_ip_address == arg->isIpAddress(); + if (!equal) { + *result_listener << fmt::format("is_ip_address '{}' != '{}'", is_ip_address, + arg->isIpAddress()); } return equal; } @@ -84,7 +96,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); @@ -117,7 +129,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { // Address does change. EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.2:80"))); + onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.2"})); @@ -125,6 +137,92 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); } +// Ipv4 address. +TEST_F(DnsCacheImplTest, Ipv4Address) { + initialize(); + InSequence s; + + MockLoadDnsCacheEntryCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("127.0.0.1", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + auto result = dns_cache_->loadDnsCacheEntry("127.0.0.1", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); + + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("127.0.0.1", DnsHostInfoEquals("127.0.0.1:80", "127.0.0.1", true))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(TestUtility::makeDnsResponse({"127.0.0.1"})); +} + +// Ipv4 address with port. +TEST_F(DnsCacheImplTest, Ipv4AddressWithPort) { + initialize(); + InSequence s; + + MockLoadDnsCacheEntryCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("127.0.0.1", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + auto result = dns_cache_->loadDnsCacheEntry("127.0.0.1:10000", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("127.0.0.1:10000", + DnsHostInfoEquals("127.0.0.1:10000", "127.0.0.1", true))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(TestUtility::makeDnsResponse({"127.0.0.1"})); +} + +// Ipv6 address. +TEST_F(DnsCacheImplTest, Ipv6Address) { + initialize(); + InSequence s; + + MockLoadDnsCacheEntryCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("::1", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + auto result = dns_cache_->loadDnsCacheEntry("[::1]", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("[::1]", DnsHostInfoEquals("[::1]:80", "::1", true))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(TestUtility::makeDnsResponse({"::1"})); +} + +// Ipv6 address with port. +TEST_F(DnsCacheImplTest, Ipv6AddressWithPort) { + initialize(); + InSequence s; + + MockLoadDnsCacheEntryCallbacks callbacks; + Network::DnsResolver::ResolveCb resolve_cb; + Event::MockTimer* resolve_timer = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*resolver_, resolve("::1", _, _)) + .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); + auto result = dns_cache_->loadDnsCacheEntry("[::1]:10000", 80, callbacks); + EXPECT_EQ(DnsCache::LoadDnsCacheEntryStatus::Loading, result.status_); + EXPECT_NE(result.handle_, nullptr); + + EXPECT_CALL(update_callbacks_, + onDnsHostAddOrUpdate("[::1]:10000", DnsHostInfoEquals("[::1]:10000", "::1", true))); + EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + resolve_cb(TestUtility::makeDnsResponse({"::1"})); +} + // TTL purge test. TEST_F(DnsCacheImplTest, TTL) { initialize(); @@ -143,7 +241,7 @@ TEST_F(DnsCacheImplTest, TTL) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"}, std::chrono::seconds(0))); @@ -200,7 +298,7 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { EXPECT_NE(result.handle_, nullptr); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000))); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"}, std::chrono::seconds(0))); @@ -238,8 +336,9 @@ TEST_F(DnsCacheImplTest, InlineResolve) { callback(TestUtility::makeDnsResponse({"127.0.0.1"})); return nullptr; })); - EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("localhost", SharedAddressEquals("127.0.0.1:80"))); + EXPECT_CALL( + update_callbacks_, + onDnsHostAddOrUpdate("localhost", DnsHostInfoEquals("127.0.0.1:80", "localhost", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); post_cb(); @@ -286,7 +385,7 @@ TEST_F(DnsCacheImplTest, CancelResolve) { result.handle_.reset(); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); } @@ -310,7 +409,7 @@ TEST_F(DnsCacheImplTest, MultipleResolveSameHost) { EXPECT_NE(result2.handle_, nullptr); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks2, onLoadDnsCacheComplete()); EXPECT_CALL(callbacks1, onLoadDnsCacheComplete()); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); @@ -338,12 +437,12 @@ TEST_F(DnsCacheImplTest, MultipleResolveDifferentHost) { EXPECT_NE(result2.handle_, nullptr); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("bar.com", SharedAddressEquals("10.0.0.1:443"))); + onDnsHostAddOrUpdate("bar.com", DnsHostInfoEquals("10.0.0.1:443", "bar.com", false))); EXPECT_CALL(callbacks2, onLoadDnsCacheComplete()); resolve_cb2(TestUtility::makeDnsResponse({"10.0.0.1"})); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.2:80"))); + onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); EXPECT_CALL(callbacks1, onLoadDnsCacheComplete()); resolve_cb1(TestUtility::makeDnsResponse({"10.0.0.2"})); } @@ -362,7 +461,7 @@ TEST_F(DnsCacheImplTest, CacheHit) { EXPECT_NE(result.handle_, nullptr); EXPECT_CALL(update_callbacks_, - onDnsHostAddOrUpdate("foo.com", SharedAddressEquals("10.0.0.1:80"))); + onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); diff --git a/test/extensions/common/dynamic_forward_proxy/mocks.h b/test/extensions/common/dynamic_forward_proxy/mocks.h index 0ac9e476deac..7f7a57d22ac6 100644 --- a/test/extensions/common/dynamic_forward_proxy/mocks.h +++ b/test/extensions/common/dynamic_forward_proxy/mocks.h @@ -63,6 +63,7 @@ class MockDnsHostInfo : public DnsHostInfo { MOCK_METHOD0(address, Network::Address::InstanceConstSharedPtr()); MOCK_METHOD0(resolvedHost, const std::string&()); + MOCK_METHOD0(isIpAddress, bool()); MOCK_METHOD0(touch, void()); Network::Address::InstanceConstSharedPtr address_; diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index ddef72bc47fa..076fca7e3511 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -209,6 +209,31 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTls) { checkSimpleRequestSuccess(0, 0, response.get()); } +TEST_P(ProxyFilterIntegrationTest, UpstreamTlsWithIpHost) { + upstream_tls_ = true; + setup(); + codec_client_ = makeHttpConnection(lookupPort("http")); + const Http::TestHeaderMapImpl request_headers{ + {":method", "POST"}, + {":path", "/test/long/url"}, + {":scheme", "http"}, + {":authority", fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(GetParam()), + fake_upstreams_[0]->localAddress()->ip()->port())}}; + + auto response = codec_client_->makeHeaderOnlyRequest(request_headers); + waitForNextUpstreamRequest(); + + // No SNI for IP hosts. + const Extensions::TransportSockets::Tls::SslSocket* ssl_socket = + dynamic_cast( + fake_upstream_connection_->connection().ssl()); + EXPECT_STREQ(nullptr, SSL_get_servername(ssl_socket->rawSslForTest(), TLSEXT_NAMETYPE_host_name)); + + upstream_request_->encodeHeaders(default_response_headers_, true); + response->waitForEndStream(); + checkSimpleRequestSuccess(0, 0, response.get()); +} + // Verify that auto-SAN verification fails with an incorrect certificate. TEST_P(ProxyFilterIntegrationTest, UpstreamTlsInvalidSAN) { upstream_tls_ = true; diff --git a/test/extensions/transport_sockets/tls/BUILD b/test/extensions/transport_sockets/tls/BUILD index 0d5e5f67735b..15ac0e33560d 100644 --- a/test/extensions/transport_sockets/tls/BUILD +++ b/test/extensions/transport_sockets/tls/BUILD @@ -17,6 +17,9 @@ envoy_cc_test( ], data = [ "gen_unittest_certs.sh", + # TODO(mattklein123): We should consolidate all of our test certs in a single place as + # right now we have a bunch of duplication which is confusing. + "//test/config/integration/certs", "//test/extensions/transport_sockets/tls/test_data:certs", ], external_deps = ["ssl"], diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index f7395a7a518e..a1ef333b4bae 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -879,6 +879,52 @@ TEST_P(SslSocketTest, GetUriWithUriSan) { .setExpectedSerialNumber(TEST_SAN_URI_CERT_SERIAL)); } +// Verify that IP SANs work with an IPv4 address specified in the validation context. +TEST_P(SslSocketTest, Ipv4San) { + const std::string client_ctx_yaml = R"EOF( + common_tls_context: + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamcacert.pem" + verify_subject_alt_name: "127.0.0.1" +)EOF"; + + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamlocalhostcert.pem" + private_key: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamlocalhostkey.pem" +)EOF"; + + TestUtilOptions test_options(client_ctx_yaml, server_ctx_yaml, true, GetParam()); + testUtil(test_options); +} + +// Verify that IP SANs work with an IPv6 address specified in the validation context. +TEST_P(SslSocketTest, Ipv6San) { + const std::string client_ctx_yaml = R"EOF( + common_tls_context: + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamcacert.pem" + verify_subject_alt_name: "::1" +)EOF"; + + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamlocalhostcert.pem" + private_key: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamlocalhostkey.pem" +)EOF"; + + TestUtilOptions test_options(client_ctx_yaml, server_ctx_yaml, true, GetParam()); + testUtil(test_options); +} + TEST_P(SslSocketTest, GetNoUriWithDnsSan) { const std::string client_ctx_yaml = R"EOF( common_tls_context: From 78d35c9775a590d06bcf924fbaaacb29187f7183 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Mon, 22 Jul 2019 12:52:38 -0400 Subject: [PATCH 216/542] Add missing initialization of the Http1ServerConnectionImplTest.Http11AbsolutePathWithPort test (#7666) Signed-off-by: Yan Avlasov --- test/common/http/http1/codec_impl_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 1db1fee44f4e..a4ceb41c5836 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -241,6 +241,8 @@ TEST_F(Http1ServerConnectionImplTest, Http11AbsolutePath2) { } TEST_F(Http1ServerConnectionImplTest, Http11AbsolutePathWithPort) { + initialize(); + TestHeaderMapImpl expected_headers{ {":authority", "www.somewhere.com:4532"}, {":path", "/foo/bar"}, {":method", "GET"}}; Buffer::OwnedImpl buffer( From 712000a711ee2bf7d3065ab107ad6c7438c5bd7f Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 22 Jul 2019 16:44:22 -0400 Subject: [PATCH 217/542] runtime: changing snapshot access to be const (#7677) This is a precursor to #7601 just to land the API change more quickly and make sure it sticks. Risk Level: Low Testing: existing unit tests Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- include/envoy/runtime/runtime.h | 4 ++-- source/common/runtime/runtime_impl.cc | 2 +- source/common/runtime/runtime_impl.h | 2 +- test/common/runtime/runtime_impl_test.cc | 4 ++-- test/mocks/runtime/mocks.h | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/envoy/runtime/runtime.h b/include/envoy/runtime/runtime.h index b3e3ebeb468a..bd3e759ee175 100644 --- a/include/envoy/runtime/runtime.h +++ b/include/envoy/runtime/runtime.h @@ -220,11 +220,11 @@ class Loader { virtual void initialize(Upstream::ClusterManager& cm) PURE; /** - * @return Snapshot& the current snapshot. This reference is safe to use for the duration of + * @return const Snapshot& the current snapshot. This reference is safe to use for the duration of * the calling routine, but may be overwritten on a future event loop cycle so should be * fetched again when needed. */ - virtual Snapshot& snapshot() PURE; + virtual const Snapshot& snapshot() PURE; /** * Merge the given map of key-value pairs into the runtime's state. To remove a previous merge for diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index a1369e6ab141..ae565aecbde3 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -557,7 +557,7 @@ void LoaderImpl::loadNewSnapshot() { }); } -Snapshot& LoaderImpl::snapshot() { return tls_->getTyped(); } +const Snapshot& LoaderImpl::snapshot() { return tls_->getTyped(); } void LoaderImpl::mergeValues(const std::unordered_map& values) { if (admin_layer_ == nullptr) { diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index 855dc1ea370e..b9344d0cc972 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -243,7 +243,7 @@ class LoaderImpl : public Loader, Logger::Loggable { // Runtime::Loader void initialize(Upstream::ClusterManager& cm) override; - Snapshot& snapshot() override; + const Snapshot& snapshot() override; void mergeValues(const std::unordered_map& values) override; private: diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index cb73ae6e73bf..e5edecfc5b88 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -155,7 +155,7 @@ TEST_F(DiskLoaderImplTest, All) { EXPECT_EQ(123UL, loader_->snapshot().getInteger("file4", 1)); bool value; - SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); + const SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); // Validate that the layer name is set properly for static layers. EXPECT_EQ("base", snapshot->getLayers()[0]->name()); @@ -538,7 +538,7 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { // Boolean getting. bool value; - SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); + const SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); EXPECT_EQ(true, snapshot->getBoolean("file11", value)); EXPECT_EQ(true, value); diff --git a/test/mocks/runtime/mocks.h b/test/mocks/runtime/mocks.h index 3592a40200fa..6e4385b4a76a 100644 --- a/test/mocks/runtime/mocks.h +++ b/test/mocks/runtime/mocks.h @@ -63,7 +63,7 @@ class MockLoader : public Loader { ~MockLoader() override; MOCK_METHOD1(initialize, void(Upstream::ClusterManager& cm)); - MOCK_METHOD0(snapshot, Snapshot&()); + MOCK_METHOD0(snapshot, const Snapshot&()); MOCK_METHOD1(mergeValues, void(const std::unordered_map&)); testing::NiceMock snapshot_; From 8820a097985eeee3ef28fcd69853569f37644d3f Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Mon, 22 Jul 2019 14:18:48 -0700 Subject: [PATCH 218/542] fail envoy.deprecated_features.v1_filter_json_config by default (#7592) Signed-off-by: Matt Klein --- source/common/runtime/runtime_features.cc | 15 +- test/config/integration/server.yaml | 457 +++++++++--------- .../integration/server_unix_listener.yaml | 30 +- .../network/http_connection_manager/BUILD | 3 + .../http_connection_manager/config_test.cc | 71 +++ 5 files changed, 335 insertions(+), 241 deletions(-) diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 37eb7bb8cdc5..0b7c7fea2025 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -42,16 +42,19 @@ constexpr const char* disallowed_features[] = { // Acts as both a test entry for deprecated.proto and a marker for the Envoy // deprecation scripts. "envoy.deprecated_features.deprecated.proto:is_deprecated_fatal", - "envoy.deprecated_features.bootstrap.proto:runtime", - "envoy.deprecated_features.redis_proxy.proto:catch_all_cluster", - "envoy.deprecated_features.server_info.proto:max_stats", - "envoy.deprecated_features.redis_proxy.proto:cluster", - "envoy.deprecated_features.server_info.proto:max_obj_name_len", + // 1.10.0 "envoy.deprecated_features.config_source.proto:UNSUPPORTED_REST_LEGACY", "envoy.deprecated_features.ext_authz.proto:use_alpha", - "envoy.deprecated_features.route.proto:enabled", "envoy.deprecated_features.fault.proto:type", + "envoy.deprecated_features.route.proto:enabled", "envoy.deprecated_features.route.proto:runtime_key", + // 1.11.0 + "envoy.deprecated_features.bootstrap.proto:runtime", + "envoy.deprecated_features.redis_proxy.proto:catch_all_cluster", + "envoy.deprecated_features.redis_proxy.proto:cluster", + "envoy.deprecated_features.server_info.proto:max_obj_name_len", + "envoy.deprecated_features.server_info.proto:max_stats", + "envoy.deprecated_features.v1_filter_json_config", }; RuntimeFeatures::RuntimeFeatures() { diff --git a/test/config/integration/server.yaml b/test/config/integration/server.yaml index 91fdf5da5dfd..522f097d1d44 100644 --- a/test/config/integration/server.yaml +++ b/test/config/integration/server.yaml @@ -8,66 +8,74 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - value: - drain_timeout_ms: 5000 - route_config: - virtual_hosts: - - require_ssl: all - routes: - - cluster: cluster_1 - prefix: "/" - domains: - - www.redirect.com - name: redirect - - routes: - - prefix: "/" + drain_timeout_ms: 5000 + route_config: + virtual_hosts: + - require_ssl: all + routes: + - route: { cluster: cluster_1 } + match: { prefix: "/" } + domains: + - www.redirect.com + name: redirect + - routes: + - match: { prefix: "/" } + route: cluster: cluster_1 runtime: key: some_key default: 0 - - prefix: "/test/long/url" + - match: { prefix: "/test/long/url" } + route: rate_limits: - actions: - - type: destination_cluster + - destination_cluster: {} cluster: cluster_1 - - prefix: "/test/" - cluster: cluster_2 - - prefix: "/websocket/test" + - match: { prefix: "/test/" } + route: { cluster: cluster_2 } + - match: { prefix: "/websocket/test" } + route: prefix_rewrite: "/websocket" cluster: cluster_1 - domains: - - "*" - name: integration - codec_type: http1 - stat_prefix: router - filters: - - name: health_check - config: - endpoint: "/healthcheck" - pass_through_mode: false - - name: rate_limit - config: - domain: foo - - name: router - config: {} - access_log: - - format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + domains: + - "*" + name: integration + codec_type: http1 + stat_prefix: router + filters: + - name: health_check + config: + endpoint: "/healthcheck" + pass_through_mode: false + - name: rate_limit + config: + domain: foo + - name: router + config: {} + access_log: + - name: envoy.file_access_log + config: + format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' - path: "/dev/null" - filter: + path: /dev/null + filter: + or_filter: filters: - - type: status_code - op: ">=" - value: 500 - - type: duration - op: ">=" - value: 1000000 - type: logical_or - - path: "/dev/null" - deprecated_v1: true + - status_code_filter: + comparison: + op: GE + value: + default_value: 500 + runtime_key: access_log.access_error.status + - duration_filter: + comparison: + op: GE + value: + default_value: 1000 + runtime_key: access_log.access_error.duration - address: socket_address: address: {{ ip_loopback_address }} @@ -76,75 +84,83 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - value: - filters: - - name: health_check - config: - endpoint: "/healthcheck" - pass_through_mode: false - - name: rate_limit - config: - domain: foo - - name: router - config: {} - access_log: - - filter: - type: logical_or - filters: - - value: 500 - type: status_code - op: ">=" - - type: duration - op: ">=" - value: 1555500 + filters: + - name: health_check + config: + endpoint: "/healthcheck" + pass_through_mode: false + - name: rate_limit + config: + domain: foo + - name: router + config: {} + access_log: + - name: envoy.file_access_log + config: format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' - path: "/dev/null" - - path: "/dev/null" - drain_timeout_ms: 5000 - route_config: - virtual_hosts: - - routes: - - prefix: "/" - cluster: cluster_1 - domains: - - www.redirect.com - name: redirect - require_ssl: all - - routes: - - prefix: "/" + path: /dev/null + filter: + or_filter: + filters: + - status_code_filter: + comparison: + op: GE + value: + default_value: 500 + runtime_key: access_log.access_error.status + - duration_filter: + comparison: + op: GE + value: + default_value: 1000 + runtime_key: access_log.access_error.duration + drain_timeout_ms: 5000 + route_config: + virtual_hosts: + - routes: + - match: { prefix: "/" } + route: { cluster: cluster_1 } + domains: + - www.redirect.com + name: redirect + require_ssl: all + - routes: + - match: { prefix: "/" } + route: { cluster: cluster_1 } + domains: + - www.namewithport.com:1234 + name: redirect + require_ssl: all + - routes: + - route: cluster: cluster_1 - domains: - - www.namewithport.com:1234 - name: redirect - require_ssl: all - - routes: - - cluster: cluster_1 runtime: key: some_key default: 0 - prefix: "/" - - rate_limits: + match: { prefix: "/" } + - route: + rate_limits: - actions: - - type: destination_cluster + - destination_cluster: {} + cluster: cluster_1 + match: { prefix: "/test/long/url" } + - match: { prefix: "/test/" } + route: { cluster: cluster_2 } + - route: cluster: cluster_1 - prefix: "/test/long/url" - - prefix: "/test/" - cluster: cluster_2 - - cluster: cluster_1 - prefix: "/websocket/test" prefix_rewrite: "/websocket" - domains: - - "*" - name: integration - codec_type: http1 - stat_prefix: router - http1_settings: - allow_absolute_url: true - deprecated_v1: true + match: { prefix: "/websocket/test" } + domains: + - "*" + name: integration + codec_type: http1 + stat_prefix: router + http1_settings: + allow_absolute_url: true - address: socket_address: address: {{ ip_loopback_address }} @@ -153,21 +169,19 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - value: - route_config: - virtual_hosts: - - routes: - - cluster: cluster_3 - prefix: "/test/long/url" - domains: - - "*" - name: integration - filters: - - name: router - config: {} - codec_type: http1 - stat_prefix: router - deprecated_v1: true + route_config: + virtual_hosts: + - routes: + - route: { cluster: cluster_3 } + match: { prefix: "/test/long/url" } + domains: + - "*" + name: integration + filters: + - name: router + config: {} + codec_type: http1 + stat_prefix: router per_connection_buffer_limit_bytes: 1024 - address: socket_address: @@ -177,23 +191,21 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - value: - filters: - - name: http_dynamo_filter - config: {} - - name: router - config: {} - codec_type: http1 - stat_prefix: router - route_config: - virtual_hosts: - - routes: - - cluster: cluster_3 - prefix: "/dynamo/url" - domains: - - "*" - name: integration - deprecated_v1: true + filters: + - name: http_dynamo_filter + config: {} + - name: router + config: {} + codec_type: http1 + stat_prefix: router + route_config: + virtual_hosts: + - routes: + - route: { cluster: cluster_3 } + match: { prefix: "/dynamo/url" } + domains: + - "*" + name: integration per_connection_buffer_limit_bytes: 1024 - address: socket_address: @@ -203,23 +215,21 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - value: - route_config: - virtual_hosts: - - domains: - - "*" - name: integration - routes: - - prefix: "/test/long/url" - cluster: cluster_3 - filters: - - name: grpc_http1_bridge - config: {} - - name: router - config: {} - codec_type: http1 - stat_prefix: router - deprecated_v1: true + route_config: + virtual_hosts: + - domains: + - "*" + name: integration + routes: + - match: { prefix: "/test/long/url" } + route: { cluster: cluster_3 } + filters: + - name: grpc_http1_bridge + config: {} + - name: router + config: {} + codec_type: http1 + stat_prefix: router per_connection_buffer_limit_bytes: 1024 - address: socket_address: @@ -229,69 +239,77 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - value: - drain_timeout_ms: 5000 - route_config: - virtual_hosts: - - routes: - - cluster: cluster_1 - prefix: "/" - domains: - - www.redirect.com - name: redirect - require_ssl: all - - routes: - - cluster: cluster_1 + drain_timeout_ms: 5000 + route_config: + virtual_hosts: + - routes: + - route: { cluster: cluster_1 } + match: { prefix: "/" } + domains: + - www.redirect.com + name: redirect + require_ssl: all + - routes: + - route: + cluster: cluster_1 runtime: key: some_key default: 0 - prefix: "/" - - prefix: "/test/long/url" + match: { prefix: "/" } + - match: { prefix: "/test/long/url" } + route: rate_limits: - actions: - - type: destination_cluster + - destination_cluster: {} cluster: cluster_1 - - prefix: "/test/" - cluster: cluster_2 - - prefix: "/websocket/test" + - match: { prefix: "/test/" } + route: { cluster: cluster_2 } + - match: { prefix: "/websocket/test" } + route: prefix_rewrite: "/websocket" cluster: cluster_1 - domains: - - "*" - name: integration - codec_type: http1 - stat_prefix: router - filters: - - name: health_check - config: - endpoint: "/healthcheck" - pass_through_mode: false - - name: rate_limit - config: - domain: foo - - name: buffer - config: - max_request_bytes: 5242880 - - config: {} - name: router - access_log: - - filter: - filters: - - op: ">=" - value: 500 - type: status_code - - type: duration - op: ">=" - value: 1555500 - type: logical_or + domains: + - "*" + name: integration + codec_type: http1 + stat_prefix: router + filters: + - name: health_check + config: + endpoint: "/healthcheck" + pass_through_mode: false + - name: rate_limit + config: + domain: foo + - name: buffer + config: + max_request_bytes: 5242880 + - config: {} + name: router + access_log: + - name: envoy.file_access_log + config: format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' - path: "/dev/null" - - path: "/dev/null" - deprecated_v1: true + path: /dev/null + filter: + or_filter: + filters: + - status_code_filter: + comparison: + op: GE + value: + default_value: 500 + runtime_key: access_log.access_error.status + - duration_filter: + comparison: + op: GE + value: + default_value: 1000 + runtime_key: access_log.access_error.duration - address: socket_address: address: {{ ip_loopback_address }} @@ -300,17 +318,18 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - value: - filters: - - name: router - config: {} - codec_type: http1 - stat_prefix: rds_dummy - rds: - api_type: REST - route_config_name: foo - cluster: rds - deprecated_v1: true + filters: + - name: router + config: {} + codec_type: http1 + stat_prefix: rds_dummy + rds: + config_source: + api_config_source: + api_type: REST + cluster_names: [ rds ] + refresh_delay: 60s + route_config_name: foo - address: socket_address: address: {{ ip_loopback_address }} @@ -319,12 +338,12 @@ static_resources: - filters: - name: envoy.redis_proxy config: - value: - conn_pool: - op_timeout_ms: 400 - stat_prefix: redis - cluster_name: redis - deprecated_v1: true + settings: + op_timeout: 0.4s + stat_prefix: redis + prefix_routes: + catch_all_route: + cluster: redis clusters: - name: cds connect_timeout: 5s diff --git a/test/config/integration/server_unix_listener.yaml b/test/config/integration/server_unix_listener.yaml index f76f3bd126b4..0ba01442cb6d 100644 --- a/test/config/integration/server_unix_listener.yaml +++ b/test/config/integration/server_unix_listener.yaml @@ -7,22 +7,20 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - value: - filters: - - name: router - config: {} - codec_type: auto - stat_prefix: router - drain_timeout_ms: 5000 - route_config: - virtual_hosts: - - domains: - - "*" - name: vhost_0 - routes: - - prefix: "/" - cluster: cluster_0 - deprecated_v1: true + filters: + - name: router + config: {} + codec_type: auto + stat_prefix: router + drain_timeout_ms: 5000 + route_config: + virtual_hosts: + - domains: + - "*" + name: vhost_0 + routes: + - match: { prefix: "/" } + route: { cluster: cluster_0 } clusters: - name: cluster_0 connect_timeout: 5s diff --git a/test/extensions/filters/network/http_connection_manager/BUILD b/test/extensions/filters/network/http_connection_manager/BUILD index 71873e28e902..c2333de286e0 100644 --- a/test/extensions/filters/network/http_connection_manager/BUILD +++ b/test/extensions/filters/network/http_connection_manager/BUILD @@ -18,7 +18,10 @@ envoy_extension_cc_test( deps = [ "//source/common/buffer:buffer_lib", "//source/common/event:dispatcher_lib", + "//source/extensions/access_loggers/file:config", "//source/extensions/filters/http/dynamo:config", + "//source/extensions/filters/http/health_check:config", + "//source/extensions/filters/http/ratelimit:config", "//source/extensions/filters/http/router:config", "//source/extensions/filters/network/http_connection_manager:config", "//test/mocks/network:network_mocks", diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index c232ad8fd276..9afdaa5a6c2b 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -49,6 +49,77 @@ TEST_F(HttpConnectionManagerConfigTest, ValidateFail) { ProtoValidationException); } +// Verify that the v1 JSON config path still works. This will be deleted when v1 is fully removed. +TEST_F(HttpConnectionManagerConfigTest, V1Config) { + const std::string yaml_string = R"EOF( +drain_timeout_ms: 5000 +route_config: + virtual_hosts: + - require_ssl: all + routes: + - cluster: cluster_1 + prefix: "/" + domains: + - www.redirect.com + name: redirect + - routes: + - prefix: "/" + cluster: cluster_1 + runtime: + key: some_key + default: 0 + - prefix: "/test/long/url" + rate_limits: + - actions: + - type: destination_cluster + cluster: cluster_1 + - prefix: "/test/" + cluster: cluster_2 + - prefix: "/websocket/test" + prefix_rewrite: "/websocket" + cluster: cluster_1 + domains: + - "*" + name: integration +codec_type: http1 +stat_prefix: router +filters: +- name: health_check + config: + endpoint: "/healthcheck" + pass_through_mode: false +- name: rate_limit + config: + domain: foo +- name: router + config: {} +access_log: +- format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% + %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% + %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" + "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' + path: "/dev/null" + filter: + filters: + - type: status_code + op: ">=" + value: 500 + - type: duration + op: ">=" + value: 1000000 + type: logical_or +- path: "/dev/null" + )EOF"; + + ON_CALL(context_.runtime_loader_.snapshot_, + deprecatedFeatureEnabled("envoy.deprecated_features.v1_filter_json_config")) + .WillByDefault(Return(true)); + + HttpConnectionManagerFilterConfigFactory().createFilterFactory( + *Json::Factory::loadFromYamlString(yaml_string), context_); +} + TEST_F(HttpConnectionManagerConfigTest, InvalidFilterName) { const std::string yaml_string = R"EOF( codec_type: http1 From ae7a97901df2345a057f16ad2c40be9f93f51c42 Mon Sep 17 00:00:00 2001 From: Xuyang Tao Date: Mon, 22 Jul 2019 14:19:21 -0700 Subject: [PATCH 219/542] Auth code (#7679) * [Jwt Authn] Change 401 to 403 When Audience not Allowed When the Audience is not allowed, it should be unauthorized(403) to that api instead of being unauthenticated(401). Signed-off-by: Xuyang Tao * Add unit tets and remove duplicate contexts Signed-off-by: Xuyang Tao --- .../filters/http/jwt_authn/filter.cc | 3 +- .../filters/http/jwt_authn/filter_test.cc | 64 +++++++++++++++++-- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/source/extensions/filters/http/jwt_authn/filter.cc b/source/extensions/filters/http/jwt_authn/filter.cc index 02576892a517..3dd3d3ab8c79 100644 --- a/source/extensions/filters/http/jwt_authn/filter.cc +++ b/source/extensions/filters/http/jwt_authn/filter.cc @@ -64,7 +64,8 @@ void Filter::onComplete(const Status& status) { stats_.denied_.inc(); state_ = Responded; // verification failed - Http::Code code = Http::Code::Unauthorized; + Http::Code code = + status == Status::JwtAudienceNotAllowed ? Http::Code::Forbidden : Http::Code::Unauthorized; // return failure reason as message body decoder_callbacks_->sendLocalReply(code, ::google::jwt_verify::getStatusString(status), nullptr, absl::nullopt, RcDetails::get().JwtAuthnAccessDenied); diff --git a/test/extensions/filters/http/jwt_authn/filter_test.cc b/test/extensions/filters/http/jwt_authn/filter_test.cc index 3a65d74c3bcc..f0ed58f82de9 100644 --- a/test/extensions/filters/http/jwt_authn/filter_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_test.cc @@ -104,11 +104,13 @@ TEST_F(FilterTest, TestSetPayloadCall) { EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(headers)); } -// This test verifies Verifier::Callback is called inline with a failure status. +// This test verifies Verifier::Callback is called inline with a failure(401 Unauthorized) status. // All functions should return Continue except decodeHeaders(), it returns StopIteration. -TEST_F(FilterTest, InlineFailure) { +TEST_F(FilterTest, InlineUnauthorizedFailure) { setupMockConfig(); // A failed authentication completed inline: callback is called inside verify(). + + EXPECT_CALL(filter_callbacks_, sendLocalReply(Http::Code::Unauthorized, _, _, _, _)); EXPECT_CALL(*mock_verifier_, verify(_)).WillOnce(Invoke([](ContextSharedPtr context) { context->callback()->onComplete(Status::JwtBadFormat); })); @@ -123,6 +125,27 @@ TEST_F(FilterTest, InlineFailure) { EXPECT_EQ("jwt_authn_access_denied", filter_callbacks_.details_); } +// This test verifies Verifier::Callback is called inline with a failure(403 Forbidden) status. +// All functions should return Continue except decodeHeaders(), it returns StopIteration. +TEST_F(FilterTest, InlineForbiddenFailure) { + setupMockConfig(); + // A failed authentication completed inline: callback is called inside verify(). + + EXPECT_CALL(filter_callbacks_, sendLocalReply(Http::Code::Forbidden, _, _, _, _)); + EXPECT_CALL(*mock_verifier_, verify(_)).WillOnce(Invoke([](ContextSharedPtr context) { + context->callback()->onComplete(Status::JwtAudienceNotAllowed); + })); + + auto headers = Http::TestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(headers, false)); + EXPECT_EQ(1U, mock_config_->stats().denied_.value()); + + Buffer::OwnedImpl data(""); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(headers)); + EXPECT_EQ("jwt_authn_access_denied", filter_callbacks_.details_); +} + // This test verifies Verifier::Callback is called with OK status after verify(). TEST_F(FilterTest, OutBoundOK) { setupMockConfig(); @@ -140,7 +163,6 @@ TEST_F(FilterTest, OutBoundOK) { EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(headers)); // Callback is called now with OK status. - auto context = Verifier::createContext(headers, &verifier_callback_); m_cb->onComplete(Status::Ok); EXPECT_EQ(1U, mock_config_->stats().allowed_.value()); @@ -148,8 +170,9 @@ TEST_F(FilterTest, OutBoundOK) { EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(headers)); } -// This test verifies Verifier::Callback is called with a failure after verify(). -TEST_F(FilterTest, OutBoundFailure) { +// This test verifies Verifier::Callback is called with a failure(401 Unauthorized) after verify() +// returns any NonOK status except JwtAudienceNotAllowed. +TEST_F(FilterTest, OutBoundUnauthorizedFailure) { setupMockConfig(); Verifier::Callbacks* m_cb; // callback is saved, not called right @@ -164,8 +187,8 @@ TEST_F(FilterTest, OutBoundFailure) { EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(data, false)); EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(headers)); - auto context = Verifier::createContext(headers, &verifier_callback_); // Callback is called now with a failure status. + EXPECT_CALL(filter_callbacks_, sendLocalReply(Http::Code::Unauthorized, _, _, _, _)); m_cb->onComplete(Status::JwtBadFormat); EXPECT_EQ(1U, mock_config_->stats().denied_.value()); @@ -176,6 +199,35 @@ TEST_F(FilterTest, OutBoundFailure) { m_cb->onComplete(Status::JwtBadFormat); } +// This test verifies Verifier::Callback is called with a failure(403 Forbidden) after verify() +// returns JwtAudienceNotAllowed. +TEST_F(FilterTest, OutBoundForbiddenFailure) { + setupMockConfig(); + Verifier::Callbacks* m_cb; + // callback is saved, not called right + EXPECT_CALL(*mock_verifier_, verify(_)).WillOnce(Invoke([&m_cb](ContextSharedPtr context) { + m_cb = context->callback(); + })); + + auto headers = Http::TestHeaderMapImpl{}; + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_->decodeHeaders(headers, false)); + + Buffer::OwnedImpl data(""); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(data, false)); + EXPECT_EQ(Http::FilterTrailersStatus::StopIteration, filter_->decodeTrailers(headers)); + + // Callback is called now with a failure status. + EXPECT_CALL(filter_callbacks_, sendLocalReply(Http::Code::Forbidden, _, _, _, _)); + m_cb->onComplete(Status::JwtAudienceNotAllowed); + + EXPECT_EQ(1U, mock_config_->stats().denied_.value()); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(data, false)); + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(headers)); + + // Should be OK to call the onComplete() again. + m_cb->onComplete(Status::JwtAudienceNotAllowed); +} + // Test verifies that if no route matched requirement, then request is allowed. TEST_F(FilterTest, TestNoRouteMatched) { EXPECT_CALL(*mock_config_.get(), findVerifier(_, _)).WillOnce(Return(nullptr)); From 9c00735e68148b9100473eecce2ee536c3072c6b Mon Sep 17 00:00:00 2001 From: Cynthia Coan Date: Mon, 22 Jul 2019 17:07:56 -0600 Subject: [PATCH 220/542] bump protoc-gen-validate to latest (#7641) Description: this commit bumps protoc-gen-validate to the latest version. this should unblock `wrowe` in slack working on windows support. after this I believe we can also take use of the new address validation type to fix some unique error messages, but the first step is bumping it as a side note: - SocketState was using `.message.required` however it was not a message type. as far as I can tell this was a bug that PGV fixed. Risk Level: Low Testing: Ensure that envoy successfully builds. Docs Changes: None Release Notes: None Signed-off-by: Cynthia Coan --- api/bazel/repository_locations.bzl | 4 ++-- api/envoy/api/v2/core/base.proto | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 541f6c4a5d76..e2bd881f0388 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -7,8 +7,8 @@ GOGOPROTO_SHA256 = "99e423905ba8921e86817607a5294ffeedb66fdd4a85efce5eb2848f715f OPENCENSUS_PROTO_GIT_SHA = "d5d80953a8c2ff4633087af6933cd152678434bb" # Mar 18, 2019 OPENCENSUS_PROTO_SHA256 = "a4e87a1da21d1b3a16674332c3ee6e2689d52f3532e2ff8cb4a626c8bcdabcfc" -PGV_GIT_SHA = "26db5cb7c01a67c6a2e21a832106406185530b2f" -PGV_SHA256 = "6510cbcf69d99059c652ae2376f6240bc761d0b019cd962225f4f609be361e26" +PGV_GIT_SHA = "dd90514d96e35023ed20201f349542b0bc64461d" +PGV_SHA256 = "f7d67677fbec5b0f561f6ee6788223fb535d3864b53a28b6678e319f5d1f4f25" GOOGLEAPIS_GIT_SHA = "d642131a6e6582fc226caf9893cb7fe7885b3411" # May 23, 2018 GOOGLEAPIS_SHA = "16f5b2e8bf1e747a32f9a62e211f8f33c94645492e9bbd72458061d9a9de1f63" diff --git a/api/envoy/api/v2/core/base.proto b/api/envoy/api/v2/core/base.proto index 23704f765502..6375d87e6824 100644 --- a/api/envoy/api/v2/core/base.proto +++ b/api/envoy/api/v2/core/base.proto @@ -258,8 +258,7 @@ message SocketOption { } // The state in which the option will be applied. When used in BindConfig // STATE_PREBIND is currently the only valid value. - SocketState state = 6 - [(validate.rules).message.required = true, (validate.rules).enum.defined_only = true]; + SocketState state = 6 [(validate.rules).enum.defined_only = true]; } // Runtime derived FractionalPercent with defaults for when the numerator or denominator is not From d3248695218b06908e466d415b77517afe662f28 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Mon, 22 Jul 2019 16:08:41 -0700 Subject: [PATCH 221/542] clang-tidy: enable checks that don't have existing errors for performance-* (#7662) * clang-tidy: enable checks that don't have existing errors Signed-off-by: Derek Argueta --- .clang-tidy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.clang-tidy b/.clang-tidy index 3e39eb682ee8..b315a91c40d1 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -28,8 +28,11 @@ WarningsAsErrors: 'abseil-duration-*, modernize-use-using, performance-faster-string-find, performance-for-range-copy, + performance-inefficient-algorithm, performance-inefficient-vector-operation, performance-noexcept-move-constructor, + performance-move-constructor-init, + performance-type-promotion-in-math-fn, performance-unnecessary-copy-initialization, readability-braces-around-statements, readability-container-size-empty, From 53cfe69802242fa0f4ec0f3b4df93e9c56bbdb19 Mon Sep 17 00:00:00 2001 From: Mike Schore Date: Mon, 22 Jul 2019 16:28:36 -0700 Subject: [PATCH 222/542] dispatcher: surface isThreadSafe() check (#7637) Co-authored-by: Jose Ulises Nino Rivera Signed-off-by: Mike Schore --- include/envoy/event/dispatcher.h | 6 ++++ source/common/event/dispatcher_impl.h | 2 +- test/common/event/dispatcher_impl_test.cc | 34 +++++++++++++++++++++++ test/mocks/event/mocks.h | 1 + 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/include/envoy/event/dispatcher.h b/include/envoy/event/dispatcher.h index 4dbe2ab06e16..8c3a8b8a88b7 100644 --- a/include/envoy/event/dispatcher.h +++ b/include/envoy/event/dispatcher.h @@ -211,6 +211,12 @@ class Dispatcher { * @return The previously tracked object or nullptr if there was none. */ virtual const ScopeTrackedObject* setTrackedObject(const ScopeTrackedObject* object) PURE; + + /** + * Validates that an operation is thread-safe with respect to this dispatcher; i.e. that the + * current thread of execution is on the same thread upon which the dispatcher loop is running. + */ + virtual bool isThreadSafe() const PURE; }; using DispatcherPtr = std::unique_ptr; diff --git a/source/common/event/dispatcher_impl.h b/source/common/event/dispatcher_impl.h index 508e18e2a312..5cbccddb66e0 100644 --- a/source/common/event/dispatcher_impl.h +++ b/source/common/event/dispatcher_impl.h @@ -93,7 +93,7 @@ class DispatcherImpl : Logger::Loggable, // Validate that an operation is thread safe, i.e. it's invoked on the same thread that the // dispatcher run loop is executing on. We allow run_tid_ to be empty for tests where we don't // invoke run(). - bool isThreadSafe() const { + bool isThreadSafe() const override { return run_tid_.isEmpty() || run_tid_ == api_.threadFactory().currentThreadId(); } diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index b6fc1de1b124..c2b7a4407b7e 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -187,6 +187,40 @@ TEST_F(DispatcherImplTest, Timer) { } } +TEST_F(DispatcherImplTest, IsThreadSafe) { + dispatcher_->post([this]() { + { + Thread::LockGuard lock(mu_); + // Thread safe because it is called within the dispatcher thread's context. + EXPECT_TRUE(dispatcher_->isThreadSafe()); + work_finished_ = true; + } + cv_.notifyOne(); + }); + + Thread::LockGuard lock(mu_); + while (!work_finished_) { + cv_.wait(mu_); + } + // Not thread safe because it is not called within the dispatcher thread's context. + EXPECT_FALSE(dispatcher_->isThreadSafe()); +} + +class NotStartedDispatcherImplTest : public testing::Test { +protected: + NotStartedDispatcherImplTest() + : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher()) {} + + Api::ApiPtr api_; + DispatcherPtr dispatcher_; +}; + +TEST_F(NotStartedDispatcherImplTest, IsThreadSafe) { + // Thread safe because the dispatcher has not started. + // Therefore, no thread id has been assigned. + EXPECT_TRUE(dispatcher_->isThreadSafe()); +} + TEST(TimerImplTest, TimerEnabledDisabled) { Api::ApiPtr api = Api::createApiForTest(); DispatcherPtr dispatcher(api->allocateDispatcher()); diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index ad190e23e939..4ff1eda3140c 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -114,6 +114,7 @@ class MockDispatcher : public Dispatcher { MOCK_METHOD1(post, void(std::function callback)); MOCK_METHOD1(run, void(RunType type)); MOCK_METHOD1(setTrackedObject, const ScopeTrackedObject*(const ScopeTrackedObject* object)); + MOCK_CONST_METHOD0(isThreadSafe, bool()); Buffer::WatermarkFactory& getWatermarkFactory() override { return buffer_factory_; } GlobalTimeSystem time_system_; From 79bc784da99cd1fe934e21644058c54875ee9d94 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Mon, 22 Jul 2019 19:58:41 -0700 Subject: [PATCH 223/542] http health check: clear response headers on conn close (#7683) Satisfy an existing assert by clearing response headers when we timeout with a partial response. The code worked correctly in release builds but this will fix the assert from firing. Fixes https://github.com/envoyproxy/envoy/issues/6557 Signed-off-by: Matt Klein --- source/common/upstream/health_checker_impl.cc | 1 + .../upstream/health_checker_impl_test.cc | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/source/common/upstream/health_checker_impl.cc b/source/common/upstream/health_checker_impl.cc index 8806f613c8d5..9e7718b3e35f 100644 --- a/source/common/upstream/health_checker_impl.cc +++ b/source/common/upstream/health_checker_impl.cc @@ -187,6 +187,7 @@ void HttpHealthCheckerImpl::HttpActiveHealthCheckSession::onEvent(Network::Conne // For the raw disconnect event, we are either between intervals in which case we already have // a timer setup, or we did the close or got a reset, in which case we already setup a new // timer. There is nothing to do here other than blow away the client. + response_headers_.reset(); parent_.dispatcher_.deferredDelete(std::move(client_)); } } diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index 95054a43fcf8..6992e1ed0a8e 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -1484,6 +1484,42 @@ TEST_F(HttpHealthCheckerImplTest, Timeout) { Host::ActiveHealthFailureType::TIMEOUT); } +// Make sure that a timeout during a partial response works correctly. +TEST_F(HttpHealthCheckerImplTest, TimeoutThenSuccess) { + setupNoServiceValidationHC(); + + cluster_->prioritySet().getMockHostSet(0)->hosts_ = { + makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; + expectSessionCreate(); + expectStreamCreate(0); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + health_checker_->start(); + + // Do a response that is not complete but includes headers. + std::unique_ptr response_headers( + new Http::TestHeaderMapImpl{{":status", "200"}}); + test_sessions_[0]->stream_response_callbacks_->decodeHeaders(std::move(response_headers), false); + + EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); + EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); + EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); + test_sessions_[0]->timeout_timer_->callback_(); + EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); + + expectClientCreate(0); + expectStreamCreate(0); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + test_sessions_[0]->interval_timer_->callback_(); + + EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); + respond(0, "200", false, false, true); + EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); +} + TEST_F(HttpHealthCheckerImplTest, TimeoutThenRemoteClose) { setupNoServiceValidationHC(); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); From 96f7c64d9b999c15357c5741ab6fe40c07222edd Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 22 Jul 2019 20:14:21 -0700 Subject: [PATCH 224/542] fix static initialization fiasco (#7684) Signed-off-by: Lizan Zhou --- .../redis/redis_cluster_integration_test.cc | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/extensions/clusters/redis/redis_cluster_integration_test.cc b/test/extensions/clusters/redis/redis_cluster_integration_test.cc index 45924a3a218b..546fee9a81c4 100644 --- a/test/extensions/clusters/redis/redis_cluster_integration_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_integration_test.cc @@ -1,6 +1,8 @@ #include #include +#include "common/common/macros.h" + #include "extensions/filters/network/redis_proxy/command_splitter_impl.h" #include "test/integration/integration.h" @@ -14,7 +16,8 @@ namespace { // in the cluster. The load balancing policy must be set // to random for proper test operation. -const std::string CONFIG = R"EOF( +const std::string& testConfig() { + CONSTRUCT_ON_FIRST_USE(std::string, R"EOF( admin: access_log_path: /dev/null address: @@ -50,15 +53,18 @@ const std::string CONFIG = R"EOF( value: cluster_refresh_rate: 1s cluster_refresh_timeout: 4s -)EOF"; +)EOF"); +} // This is the basic redis_proxy configuration with an upstream // authentication password specified. -const std::string CONFIG_WITH_AUTH = CONFIG + R"EOF( +const std::string& testConfigWithAuth() { + CONSTRUCT_ON_FIRST_USE(std::string, testConfig() + R"EOF( extension_protocol_options: envoy.redis_proxy: { auth_password: { inline_string: somepassword }} -)EOF"; +)EOF"); +} // This function encodes commands as an array of bulkstrings as transmitted by Redis clients to // Redis servers, according to the Redis protocol. @@ -77,7 +83,7 @@ std::string makeBulkStringArray(std::vector&& command_strings) { class RedisClusterIntegrationTest : public testing::TestWithParam, public BaseIntegrationTest { public: - RedisClusterIntegrationTest(const std::string& config = CONFIG, int num_upstreams = 2) + RedisClusterIntegrationTest(const std::string& config = testConfig(), int num_upstreams = 2) : BaseIntegrationTest(GetParam(), config), num_upstreams_(num_upstreams), version_(GetParam()) {} @@ -265,7 +271,7 @@ class RedisClusterIntegrationTest : public testing::TestWithParam Date: Mon, 22 Jul 2019 20:14:48 -0700 Subject: [PATCH 225/542] bazelci: fix again (#7685) Signed-off-by: Lizan Zhou --- .bazelci/presubmit.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 6c5da1e60f0b..9382e025d83c 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -1,8 +1,11 @@ --- tasks: release: - platform: ubuntu1604 + platform: ubuntu1804 build_targets: - "//source/exe:envoy-static" test_targets: - "//test/..." + test_flags: + # Workaround for https://github.com/envoyproxy/envoy/issues/7647 + - "--deleted_packages=//test/extensions/tracers/dynamic_ot" From 8d6cd7e4eb0b2e680361e5494b10f83d6ad6b2b8 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Mon, 22 Jul 2019 20:35:42 -0700 Subject: [PATCH 226/542] v1 cleanup: delete several unused conversion paths (#7681) Signed-off-by: Matt Klein --- source/common/config/BUILD | 33 --- source/common/config/address_json.cc | 26 -- source/common/config/address_json.h | 11 - source/common/config/cds_json.cc | 223 ------------------ source/common/config/cds_json.h | 71 ------ source/common/config/tls_context_json.cc | 70 ------ source/common/config/tls_context_json.h | 37 --- source/common/upstream/BUILD | 3 - .../common/upstream/cluster_manager_impl.cc | 1 - source/common/upstream/upstream_impl.cc | 1 - source/extensions/transport_sockets/tls/BUILD | 1 - .../tls/context_config_impl.cc | 12 - .../tls/context_config_impl.h | 3 - test/common/config/BUILD | 2 - test/common/config/utility_test.cc | 31 --- test/common/upstream/BUILD | 2 - .../upstream/health_checker_impl_test.cc | 1 - .../upstream/logical_dns_cluster_test.cc | 105 +++------ .../upstream/original_dst_cluster_test.cc | 31 +-- test/common/upstream/upstream_impl_test.cc | 221 ++++++++--------- test/common/upstream/utility.h | 9 - .../tls/context_impl_test.cc | 4 +- test/integration/xfcc_integration_test.cc | 39 +-- test/server/http/admin_test.cc | 4 +- 24 files changed, 165 insertions(+), 776 deletions(-) delete mode 100644 source/common/config/cds_json.cc delete mode 100644 source/common/config/cds_json.h delete mode 100644 source/common/config/tls_context_json.cc delete mode 100644 source/common/config/tls_context_json.h diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 8ebd3aedae14..378512d4df5f 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -31,27 +31,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "cds_json_lib", - srcs = ["cds_json.cc"], - hdrs = ["cds_json.h"], - external_deps = ["abseil_optional"], - deps = [ - ":address_json_lib", - ":json_utility_lib", - ":protocol_json_lib", - ":tls_context_json_lib", - ":utility_lib", - "//include/envoy/json:json_object_interface", - "//include/envoy/upstream:cluster_manager_interface", - "//source/common/common:assert_lib", - "//source/common/json:config_schemas_lib", - "//source/common/network:utility_lib", - "@envoy_api//envoy/api/v2:cds_cc", - "@envoy_api//envoy/api/v2/cluster:circuit_breaker_cc", - ], -) - envoy_cc_library( name = "filesystem_subscription_lib", srcs = ["filesystem_subscription_impl.cc"], @@ -336,18 +315,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "tls_context_json_lib", - srcs = ["tls_context_json.cc"], - hdrs = ["tls_context_json.h"], - deps = [ - ":json_utility_lib", - "//include/envoy/json:json_object_interface", - "//source/common/common:utility_lib", - "@envoy_api//envoy/api/v2/auth:cert_cc", - ], -) - envoy_cc_library( name = "type_to_endpoint_lib", srcs = ["type_to_endpoint.cc"], diff --git a/source/common/config/address_json.cc b/source/common/config/address_json.cc index 7206e7a3d879..f309c0d121a7 100644 --- a/source/common/config/address_json.cc +++ b/source/common/config/address_json.cc @@ -7,32 +7,6 @@ namespace Envoy { namespace Config { -void AddressJson::translateAddress(const std::string& json_address, bool url, bool resolved, - envoy::api::v2::core::Address& address) { - if (resolved) { - Network::Address::InstanceConstSharedPtr instance = - url ? Network::Utility::resolveUrl(json_address) - : Network::Utility::parseInternetAddressAndPort(json_address); - if (instance->type() == Network::Address::Type::Ip) { - address.mutable_socket_address()->set_address(instance->ip()->addressAsString()); - address.mutable_socket_address()->set_port_value(instance->ip()->port()); - } else { - ASSERT(instance->type() == Network::Address::Type::Pipe); - address.mutable_pipe()->set_path(instance->asString()); - } - return; - } - - // We don't have v1 JSON with unresolved addresses in non-URL form. - ASSERT(url); - // Non-TCP scheme (e.g. Unix scheme) is not supported with unresolved address. - if (!Network::Utility::urlIsTcpScheme(json_address)) { - throw EnvoyException(fmt::format("unresolved URL must be TCP scheme, got: {}", json_address)); - } - address.mutable_socket_address()->set_address(Network::Utility::hostFromTcpUrl(json_address)); - address.mutable_socket_address()->set_port_value(Network::Utility::portFromTcpUrl(json_address)); -} - void AddressJson::translateCidrRangeList( const std::vector& json_ip_list, Protobuf::RepeatedPtrField& range_list) { diff --git a/source/common/config/address_json.h b/source/common/config/address_json.h index 9dd890480113..2e35b466ad06 100644 --- a/source/common/config/address_json.h +++ b/source/common/config/address_json.h @@ -10,17 +10,6 @@ namespace Config { class AddressJson { public: - /** - * Translate a v1 JSON address to v2 envoy::api::v2::core::Address. - * @param json_address source address. - * @param url is json_address a URL? E.g. tcp://:. If not, it is - * treated as :. - * @param resolved is json_address a concrete IP/pipe or unresolved hostname? - * @param address destination envoy::api::v2::core::Address. - */ - static void translateAddress(const std::string& json_address, bool url, bool resolved, - envoy::api::v2::core::Address& address); - /** * Translate a v1 JSON array of IP ranges to v2 * Protobuf::RepeatedPtrField. diff --git a/source/common/config/cds_json.cc b/source/common/config/cds_json.cc deleted file mode 100644 index 95fede2e33c4..000000000000 --- a/source/common/config/cds_json.cc +++ /dev/null @@ -1,223 +0,0 @@ -#include "common/config/cds_json.h" - -#include "common/common/assert.h" -#include "common/config/address_json.h" -#include "common/config/json_utility.h" -#include "common/config/protocol_json.h" -#include "common/config/tls_context_json.h" -#include "common/config/utility.h" -#include "common/json/config_schemas.h" - -namespace Envoy { -namespace Config { - -void CdsJson::translateRingHashLbConfig( - const Json::Object& json_ring_hash_lb_config, - envoy::api::v2::Cluster::RingHashLbConfig& ring_hash_lb_config) { - JSON_UTIL_SET_INTEGER(json_ring_hash_lb_config, ring_hash_lb_config, minimum_ring_size); -} - -void CdsJson::translateHealthCheck(const Json::Object& json_health_check, - envoy::api::v2::core::HealthCheck& health_check) { - json_health_check.validateSchema(Json::Schema::CLUSTER_HEALTH_CHECK_SCHEMA); - - JSON_UTIL_SET_DURATION(json_health_check, health_check, timeout); - JSON_UTIL_SET_DURATION(json_health_check, health_check, interval); - JSON_UTIL_SET_DURATION(json_health_check, health_check, interval_jitter); - JSON_UTIL_SET_INTEGER(json_health_check, health_check, unhealthy_threshold); - JSON_UTIL_SET_INTEGER(json_health_check, health_check, healthy_threshold); - JSON_UTIL_SET_BOOL(json_health_check, health_check, reuse_connection); - - const std::string health_check_type = json_health_check.getString("type"); - if (health_check_type == "http") { - health_check.mutable_http_health_check()->set_path(json_health_check.getString("path")); - if (json_health_check.hasObject("service_name")) { - health_check.mutable_http_health_check()->set_service_name( - json_health_check.getString("service_name")); - } - } else if (health_check_type == "tcp") { - auto* tcp_health_check = health_check.mutable_tcp_health_check(); - std::string send_text; - for (const Json::ObjectSharedPtr& entry : json_health_check.getObjectArray("send")) { - const std::string hex_string = entry->getString("binary"); - send_text += hex_string; - } - if (!send_text.empty()) { - tcp_health_check->mutable_send()->set_text(send_text); - } - for (const Json::ObjectSharedPtr& entry : json_health_check.getObjectArray("receive")) { - const std::string hex_string = entry->getString("binary"); - tcp_health_check->mutable_receive()->Add()->set_text(hex_string); - } - } else { - ASSERT(health_check_type == "redis"); - auto* redis_health_check = health_check.mutable_custom_health_check(); - redis_health_check->set_name("envoy.health_checkers.redis"); - if (json_health_check.hasObject("redis_key")) { - redis_health_check->mutable_config()->MergeFrom( - MessageUtil::keyValueStruct("key", json_health_check.getString("redis_key"))); - } - } -} - -void CdsJson::translateThresholds( - const Json::Object& json_thresholds, const envoy::api::v2::core::RoutingPriority& priority, - envoy::api::v2::cluster::CircuitBreakers::Thresholds& thresholds) { - thresholds.set_priority(priority); - JSON_UTIL_SET_INTEGER(json_thresholds, thresholds, max_connections); - JSON_UTIL_SET_INTEGER(json_thresholds, thresholds, max_pending_requests); - JSON_UTIL_SET_INTEGER(json_thresholds, thresholds, max_requests); - JSON_UTIL_SET_INTEGER(json_thresholds, thresholds, max_retries); -} - -void CdsJson::translateCircuitBreakers(const Json::Object& json_circuit_breakers, - envoy::api::v2::cluster::CircuitBreakers& circuit_breakers) { - translateThresholds(*json_circuit_breakers.getObject("default", true), - envoy::api::v2::core::RoutingPriority::DEFAULT, - *circuit_breakers.mutable_thresholds()->Add()); - translateThresholds(*json_circuit_breakers.getObject("high", true), - envoy::api::v2::core::RoutingPriority::HIGH, - *circuit_breakers.mutable_thresholds()->Add()); -} - -void CdsJson::translateOutlierDetection( - const Json::Object& json_outlier_detection, - envoy::api::v2::cluster::OutlierDetection& outlier_detection) { - JSON_UTIL_SET_DURATION(json_outlier_detection, outlier_detection, interval); - JSON_UTIL_SET_DURATION(json_outlier_detection, outlier_detection, base_ejection_time); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, consecutive_5xx); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, consecutive_gateway_failure); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, max_ejection_percent); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, enforcing_consecutive_5xx); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, - enforcing_consecutive_gateway_failure); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, enforcing_success_rate); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, success_rate_minimum_hosts); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, success_rate_request_volume); - JSON_UTIL_SET_INTEGER(json_outlier_detection, outlier_detection, success_rate_stdev_factor); -} - -void CdsJson::translateCluster(const Json::Object& json_cluster, - const absl::optional& eds_config, - envoy::api::v2::Cluster& cluster) { - json_cluster.validateSchema(Json::Schema::CLUSTER_SCHEMA); - - const std::string name = json_cluster.getString("name"); - cluster.set_name(name); - - const std::string string_type = json_cluster.getString("type"); - auto set_dns_hosts = [&json_cluster, &cluster] { - const auto hosts = json_cluster.getObjectArray("hosts"); - std::transform(hosts.cbegin(), hosts.cend(), - Protobuf::RepeatedPtrFieldBackInserter(cluster.mutable_hosts()), - [](const Json::ObjectSharedPtr& host) { - envoy::api::v2::core::Address address; - AddressJson::translateAddress(host->getString("url"), true, false, address); - return address; - }); - }; - if (string_type == "static") { - cluster.set_type(envoy::api::v2::Cluster::STATIC); - const auto hosts = json_cluster.getObjectArray("hosts"); - std::transform(hosts.cbegin(), hosts.cend(), - Protobuf::RepeatedPtrFieldBackInserter(cluster.mutable_hosts()), - [](const Json::ObjectSharedPtr& host) { - envoy::api::v2::core::Address address; - AddressJson::translateAddress(host->getString("url"), true, true, address); - return address; - }); - } else if (string_type == "strict_dns") { - cluster.set_type(envoy::api::v2::Cluster::STRICT_DNS); - set_dns_hosts(); - } else if (string_type == "logical_dns") { - cluster.set_type(envoy::api::v2::Cluster::LOGICAL_DNS); - set_dns_hosts(); - } else if (string_type == "original_dst") { - if (json_cluster.hasObject("hosts")) { - throw EnvoyException("original_dst clusters must have no hosts configured"); - } - cluster.set_type(envoy::api::v2::Cluster::ORIGINAL_DST); - } else { - ASSERT(string_type == "sds"); - if (!eds_config) { - throw EnvoyException("cannot create sds cluster with no sds config"); - } - cluster.set_type(envoy::api::v2::Cluster::EDS); - cluster.mutable_eds_cluster_config()->mutable_eds_config()->CopyFrom(eds_config.value()); - JSON_UTIL_SET_STRING(json_cluster, *cluster.mutable_eds_cluster_config(), service_name); - } - - JSON_UTIL_SET_DURATION(json_cluster, cluster, cleanup_interval); - JSON_UTIL_SET_DURATION(json_cluster, cluster, connect_timeout); - JSON_UTIL_SET_INTEGER(json_cluster, cluster, per_connection_buffer_limit_bytes); - - const std::string lb_type = json_cluster.getString("lb_type"); - if (lb_type == "round_robin") { - cluster.set_lb_policy(envoy::api::v2::Cluster::ROUND_ROBIN); - } else if (lb_type == "least_request") { - cluster.set_lb_policy(envoy::api::v2::Cluster::LEAST_REQUEST); - } else if (lb_type == "random") { - cluster.set_lb_policy(envoy::api::v2::Cluster::RANDOM); - } else if (lb_type == "original_dst_lb") { - cluster.set_lb_policy(envoy::api::v2::Cluster::ORIGINAL_DST_LB); - } else { - ASSERT(lb_type == "ring_hash"); - cluster.set_lb_policy(envoy::api::v2::Cluster::RING_HASH); - } - - if (json_cluster.hasObject("ring_hash_lb_config")) { - translateRingHashLbConfig(*json_cluster.getObject("ring_hash_lb_config"), - *cluster.mutable_ring_hash_lb_config()); - } - - if (json_cluster.hasObject("health_check")) { - translateHealthCheck(*json_cluster.getObject("health_check"), - *cluster.mutable_health_checks()->Add()); - } - - JSON_UTIL_SET_INTEGER(json_cluster, cluster, max_requests_per_connection); - if (json_cluster.hasObject("circuit_breakers")) { - translateCircuitBreakers(*json_cluster.getObject("circuit_breakers"), - *cluster.mutable_circuit_breakers()); - } - - if (json_cluster.hasObject("ssl_context")) { - TlsContextJson::translateUpstreamTlsContext(*json_cluster.getObject("ssl_context"), - *cluster.mutable_tls_context()); - } - - if (json_cluster.getString("features", "") == "http2" || - json_cluster.hasObject("http2_settings")) { - ProtocolJson::translateHttp2ProtocolOptions(*json_cluster.getObject("http2_settings", true), - *cluster.mutable_http2_protocol_options()); - } - - JSON_UTIL_SET_DURATION(json_cluster, cluster, dns_refresh_rate); - const std::string dns_lookup_family = json_cluster.getString("dns_lookup_family", "v4_only"); - if (dns_lookup_family == "auto") { - cluster.set_dns_lookup_family(envoy::api::v2::Cluster::AUTO); - } else if (dns_lookup_family == "v6_only") { - cluster.set_dns_lookup_family(envoy::api::v2::Cluster::V6_ONLY); - } else { - ASSERT(dns_lookup_family == "v4_only"); - cluster.set_dns_lookup_family(envoy::api::v2::Cluster::V4_ONLY); - } - if (json_cluster.hasObject("dns_resolvers")) { - const auto dns_resolvers = json_cluster.getStringArray("dns_resolvers"); - std::transform(dns_resolvers.cbegin(), dns_resolvers.cend(), - Protobuf::RepeatedPtrFieldBackInserter(cluster.mutable_dns_resolvers()), - [](const std::string& json_address) { - envoy::api::v2::core::Address address; - AddressJson::translateAddress(json_address, false, true, address); - return address; - }); - } - - if (json_cluster.hasObject("outlier_detection")) { - translateOutlierDetection(*json_cluster.getObject("outlier_detection"), - *cluster.mutable_outlier_detection()); - } -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/cds_json.h b/source/common/config/cds_json.h deleted file mode 100644 index f2995074f79a..000000000000 --- a/source/common/config/cds_json.h +++ /dev/null @@ -1,71 +0,0 @@ -#pragma once - -#include "envoy/api/v2/cds.pb.h" -#include "envoy/api/v2/cluster/circuit_breaker.pb.h" -#include "envoy/json/json_object.h" -#include "envoy/upstream/cluster_manager.h" - -#include "absl/types/optional.h" - -namespace Envoy { -namespace Config { - -class CdsJson { -public: - /** - * Translate a v1 JSON ring hash config to envoy::api::v2::Cluster::RingHashLbConfig. - * @param json_ring_hash_lb_config source v1 JSON ring hash config object. - * @param ring_hash_lb_config destination v2 envoy::api::v2::Cluster::RingHashLbConfig. - */ - static void - translateRingHashLbConfig(const Json::Object& json_ring_hash_lb_config, - envoy::api::v2::Cluster::RingHashLbConfig& ring_hash_lb_config); - - /** - * Translate a v1 JSON health check object to v2 envoy::api::v2::core::HealthCheck. - * @param json_health_check source v1 JSON health check object. - * @param health_check destination v2 envoy::api::v2::core::HealthCheck. - */ - static void translateHealthCheck(const Json::Object& json_health_check, - envoy::api::v2::core::HealthCheck& health_check); - - /** - * Translate a v1 JSON thresholds object to v2 envoy::api::v2::Thresholds. - * @param json_thresholds source v1 JSON thresholds object. - * @param priority priority for thresholds. - * @param thresholds destination v2 envoy::api::v2::Thresholds. - */ - static void translateThresholds(const Json::Object& json_thresholds, - const envoy::api::v2::core::RoutingPriority& priority, - envoy::api::v2::cluster::CircuitBreakers::Thresholds& thresholds); - - /** - * Translate a v1 JSON circuit breakers object to v2 envoy::api::v2::cluster::CircuitBreakers. - * @param json_circuit_breakers source v1 JSON circuit breakers object. - * @param circuit_breakers destination v2 envoy::api::v2::cluster::CircuitBreakers. - */ - static void translateCircuitBreakers(const Json::Object& json_circuit_breakers, - envoy::api::v2::cluster::CircuitBreakers& circuit_breakers); - - /** - * Translate a v1 JSON outlier detection object to v2 envoy::api::v2::OutlierDetection. - * @param json_outlier_detection source v1 JSON outlier detection object. - * @param outlier_detection destination v2 envoy::api::v2::OutlierDetection. - */ - static void - translateOutlierDetection(const Json::Object& json_outlier_detection, - envoy::api::v2::cluster::OutlierDetection& outlier_detection); - - /** - * Translate a v1 JSON Cluster to v2 envoy::api::v2::Cluster. - * @param json_cluster source v1 JSON Cluster object. - * @param eds_config SDS config if 'sds' discovery type. - * @param cluster destination v2 envoy::api::v2::Cluster. - */ - static void translateCluster(const Json::Object& json_cluster, - const absl::optional& eds_config, - envoy::api::v2::Cluster& cluster); -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/tls_context_json.cc b/source/common/config/tls_context_json.cc deleted file mode 100644 index fece59b47676..000000000000 --- a/source/common/config/tls_context_json.cc +++ /dev/null @@ -1,70 +0,0 @@ -#include "common/config/tls_context_json.h" - -#include "envoy/api/v2/auth/cert.pb.validate.h" - -#include "common/common/utility.h" -#include "common/config/json_utility.h" -#include "common/protobuf/utility.h" - -namespace Envoy { -namespace Config { - -void TlsContextJson::translateUpstreamTlsContext( - const Json::Object& json_tls_context, - envoy::api::v2::auth::UpstreamTlsContext& upstream_tls_context) { - translateCommonTlsContext(json_tls_context, *upstream_tls_context.mutable_common_tls_context()); - upstream_tls_context.set_sni(json_tls_context.getString("sni", "")); - MessageUtil::validate(upstream_tls_context); -} - -void TlsContextJson::translateCommonTlsContext( - const Json::Object& json_tls_context, - envoy::api::v2::auth::CommonTlsContext& common_tls_context) { - const std::string alpn_protocols_str{json_tls_context.getString("alpn_protocols", "")}; - for (auto alpn_protocol : StringUtil::splitToken(alpn_protocols_str, ",")) { - common_tls_context.add_alpn_protocols(std::string{alpn_protocol}); - } - - translateTlsCertificate(json_tls_context, *common_tls_context.mutable_tls_certificates()->Add()); - - auto* validation_context = common_tls_context.mutable_validation_context(); - if (json_tls_context.hasObject("ca_cert_file")) { - validation_context->mutable_trusted_ca()->set_filename( - json_tls_context.getString("ca_cert_file", "")); - } - if (json_tls_context.hasObject("crl_file")) { - validation_context->mutable_crl()->set_filename(json_tls_context.getString("crl_file", "")); - } - if (json_tls_context.hasObject("verify_certificate_hash")) { - validation_context->add_verify_certificate_hash( - json_tls_context.getString("verify_certificate_hash")); - } - for (const auto& san : json_tls_context.getStringArray("verify_subject_alt_name", true)) { - validation_context->add_verify_subject_alt_name(san); - } - - const std::string cipher_suites_str{json_tls_context.getString("cipher_suites", "")}; - for (auto cipher_suite : StringUtil::splitToken(cipher_suites_str, ":")) { - common_tls_context.mutable_tls_params()->add_cipher_suites(std::string{cipher_suite}); - } - - const std::string ecdh_curves_str{json_tls_context.getString("ecdh_curves", "")}; - for (auto ecdh_curve : StringUtil::splitToken(ecdh_curves_str, ":")) { - common_tls_context.mutable_tls_params()->add_ecdh_curves(std::string{ecdh_curve}); - } -} - -void TlsContextJson::translateTlsCertificate( - const Json::Object& json_tls_context, envoy::api::v2::auth::TlsCertificate& tls_certificate) { - if (json_tls_context.hasObject("cert_chain_file")) { - tls_certificate.mutable_certificate_chain()->set_filename( - json_tls_context.getString("cert_chain_file", "")); - } - if (json_tls_context.hasObject("private_key_file")) { - tls_certificate.mutable_private_key()->set_filename( - json_tls_context.getString("private_key_file", "")); - } -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/tls_context_json.h b/source/common/config/tls_context_json.h deleted file mode 100644 index 1eb9cbc1139d..000000000000 --- a/source/common/config/tls_context_json.h +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include "envoy/api/v2/auth/cert.pb.h" -#include "envoy/json/json_object.h" - -namespace Envoy { -namespace Config { - -class TlsContextJson { -public: - /** - * Translate a v1 JSON TLS context to v2 envoy::api::v2::auth::UpstreamTlsContext. - * @param json_tls_context source v1 JSON TLS context object. - * @param upstream_tls_context destination v2 envoy::api::v2::Cluster. - */ - static void - translateUpstreamTlsContext(const Json::Object& json_tls_context, - envoy::api::v2::auth::UpstreamTlsContext& upstream_tls_context); - /** - * Translate a v1 JSON TLS context to v2 envoy::api::v2::auth::CommonTlsContext. - * @param json_tls_context source v1 JSON TLS context object. - * @param common_tls_context destination v2 envoy::api::v2::Cluster. - */ - static void translateCommonTlsContext(const Json::Object& json_tls_context, - envoy::api::v2::auth::CommonTlsContext& common_tls_context); - - /** - * Translate a v1 JSON TLS context to v2 envoy::api::v2::auth::TlsCertificate. - * @param json_tls_context source v1 JSON TLS context object. - * @param common_tls_context destination v2 envoy::api::v2::auth::TlsCertificate. - */ - static void translateTlsCertificate(const Json::Object& json_tls_context, - envoy::api::v2::auth::TlsCertificate& tls_certificate); -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 5177da76ddb4..10e0e60f814f 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -48,7 +48,6 @@ envoy_cc_library( "//source/common/common:cleanup_lib", "//source/common/common:enum_to_int", "//source/common/common:utility_lib", - "//source/common/config:cds_json_lib", "//source/common/config:grpc_mux_lib", "//source/common/config:subscription_factory_lib", "//source/common/config:utility_lib", @@ -381,7 +380,6 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:utility_lib", "//source/common/config:protocol_json_lib", - "//source/common/config:tls_context_json_lib", "//source/common/http:utility_lib", "//source/common/network:address_lib", "//source/common/network:resolver_lib", @@ -465,7 +463,6 @@ envoy_cc_library( "//source/common/common:enum_to_int", "//source/common/common:utility_lib", "//source/common/config:protocol_json_lib", - "//source/common/config:tls_context_json_lib", "//source/common/http:utility_lib", "//source/common/network:address_lib", "//source/common/network:resolver_lib", diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index d4a666f97af4..4c29bf9a70a6 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -18,7 +18,6 @@ #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/utility.h" -#include "common/config/cds_json.h" #include "common/config/resources.h" #include "common/config/utility.h" #include "common/grpc/async_client_manager_impl.h" diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index a8f3d695ece2..69160fa95a99 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -23,7 +23,6 @@ #include "common/common/fmt.h" #include "common/common/utility.h" #include "common/config/protocol_json.h" -#include "common/config/tls_context_json.h" #include "common/config/utility.h" #include "common/http/utility.h" #include "common/network/address_impl.h" diff --git a/source/extensions/transport_sockets/tls/BUILD b/source/extensions/transport_sockets/tls/BUILD index f58eaaefa356..2d9526cfcd7b 100644 --- a/source/extensions/transport_sockets/tls/BUILD +++ b/source/extensions/transport_sockets/tls/BUILD @@ -62,7 +62,6 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:empty_string", "//source/common/config:datasource_lib", - "//source/common/config:tls_context_json_lib", "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", "//source/common/secret:sds_api_lib", diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc index 3b8f32d27a0e..424e08ae2d59 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.cc +++ b/source/extensions/transport_sockets/tls/context_config_impl.cc @@ -6,7 +6,6 @@ #include "common/common/assert.h" #include "common/common/empty_string.h" #include "common/config/datasource.h" -#include "common/config/tls_context_json.h" #include "common/protobuf/utility.h" #include "common/secret/sds_api.h" #include "common/ssl/certificate_validation_context_config_impl.h" @@ -288,17 +287,6 @@ ClientContextConfigImpl::ClientContextConfigImpl( } } -ClientContextConfigImpl::ClientContextConfigImpl( - const Json::Object& config, - Server::Configuration::TransportSocketFactoryContext& factory_context) - : ClientContextConfigImpl( - [&config] { - envoy::api::v2::auth::UpstreamTlsContext upstream_tls_context; - Config::TlsContextJson::translateUpstreamTlsContext(config, upstream_tls_context); - return upstream_tls_context; - }(), - factory_context) {} - const unsigned ServerContextConfigImpl::DEFAULT_MIN_VERSION = TLS1_VERSION; const unsigned ServerContextConfigImpl::DEFAULT_MAX_VERSION = #ifndef BORINGSSL_FIPS diff --git a/source/extensions/transport_sockets/tls/context_config_impl.h b/source/extensions/transport_sockets/tls/context_config_impl.h index e1b0024ce906..fa48b9815f08 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.h +++ b/source/extensions/transport_sockets/tls/context_config_impl.h @@ -103,9 +103,6 @@ class ClientContextConfigImpl : public ContextConfigImpl, public Envoy::Ssl::Cli const envoy::api::v2::auth::UpstreamTlsContext& config, Server::Configuration::TransportSocketFactoryContext& secret_provider_context) : ClientContextConfigImpl(config, "", secret_provider_context) {} - ClientContextConfigImpl( - const Json::Object& config, - Server::Configuration::TransportSocketFactoryContext& secret_provider_context); // Ssl::ClientContextConfig const std::string& serverNameIndication() const override { return server_name_indication_; } diff --git a/test/common/config/BUILD b/test/common/config/BUILD index ebc6d79e5330..509d423919a9 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -260,8 +260,6 @@ envoy_cc_test( name = "utility_test", srcs = ["utility_test.cc"], deps = [ - "//source/common/config:cds_json_lib", - "//source/common/config:rds_json_lib", "//source/common/config:utility_lib", "//source/common/config:well_known_names", "//source/common/stats:stats_lib", diff --git a/test/common/config/utility_test.cc b/test/common/config/utility_test.cc index 90573d4f7d9b..dea04ddb1fd6 100644 --- a/test/common/config/utility_test.cc +++ b/test/common/config/utility_test.cc @@ -2,8 +2,6 @@ #include "envoy/common/exception.h" #include "common/common/fmt.h" -#include "common/config/cds_json.h" -#include "common/config/rds_json.h" #include "common/config/utility.h" #include "common/config/well_known_names.h" #include "common/protobuf/protobuf.h" @@ -102,35 +100,6 @@ TEST(UtilityTest, createTagProducer) { ASSERT_EQ(tags.size(), 1); } -TEST(UtilityTest, UnixClusterDns) { - - std::string cluster_type; - cluster_type = "strict_dns"; - std::string json = - R"EOF({ "name": "test", "type": ")EOF" + cluster_type + - R"EOF(", "lb_type": "random", "connect_timeout_ms" : 1, "hosts": [{"url": "unix:///test.sock"}]})EOF"; - auto json_object_ptr = Json::Factory::loadFromString(json); - envoy::api::v2::Cluster cluster; - envoy::api::v2::core::ConfigSource eds_config; - EXPECT_THROW_WITH_MESSAGE( - Config::CdsJson::translateCluster(*json_object_ptr, eds_config, cluster), EnvoyException, - "unresolved URL must be TCP scheme, got: unix:///test.sock"); -} - -TEST(UtilityTest, UnixClusterStatic) { - - std::string cluster_type; - cluster_type = "static"; - std::string json = - R"EOF({ "name": "test", "type": ")EOF" + cluster_type + - R"EOF(", "lb_type": "random", "connect_timeout_ms" : 1, "hosts": [{"url": "unix:///test.sock"}]})EOF"; - auto json_object_ptr = Json::Factory::loadFromString(json); - envoy::api::v2::Cluster cluster; - envoy::api::v2::core::ConfigSource eds_config; - Config::CdsJson::translateCluster(*json_object_ptr, eds_config, cluster); - EXPECT_EQ("/test.sock", cluster.hosts(0).pipe().path()); -} - TEST(UtilityTest, CheckFilesystemSubscriptionBackingPath) { Api::ApiPtr api = Api::createApiForTest(); diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 7bbf8314c71e..59956d5ca056 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -109,7 +109,6 @@ envoy_cc_test( deps = [ ":utility_lib", "//source/common/buffer:buffer_lib", - "//source/common/config:cds_json_lib", "//source/common/event:dispatcher_lib", "//source/common/http:headers_lib", "//source/common/json:json_loader_lib", @@ -396,7 +395,6 @@ envoy_cc_test_library( hdrs = ["utility.h"], deps = [ "//include/envoy/stats:stats_interface", - "//source/common/config:cds_json_lib", "//source/common/json:json_loader_lib", "//source/common/network:utility_lib", "//source/common/stats:stats_lib", diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index 6992e1ed0a8e..f48d8d2286ca 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -7,7 +7,6 @@ #include "common/buffer/buffer_impl.h" #include "common/buffer/zero_copy_input_stream_impl.h" -#include "common/config/cds_json.h" #include "common/grpc/common.h" #include "common/http/headers.h" #include "common/json/json_loader.h" diff --git a/test/common/upstream/logical_dns_cluster_test.cc b/test/common/upstream/logical_dns_cluster_test.cc index 03170a23af5a..4433db2931f1 100644 --- a/test/common/upstream/logical_dns_cluster_test.cc +++ b/test/common/upstream/logical_dns_cluster_test.cc @@ -35,31 +35,10 @@ namespace Envoy { namespace Upstream { namespace { -enum class ConfigType { V2_YAML, V1_JSON }; - class LogicalDnsClusterTest : public testing::Test { protected: LogicalDnsClusterTest() : api_(Api::createApiForTest(stats_store_)) {} - void setupFromV1Json(const std::string& json) { - resolve_timer_ = new Event::MockTimer(&dispatcher_); - NiceMock cm; - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); - Envoy::Stats::ScopePtr scope = stats_store_.createScope(fmt::format( - "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() - : cluster_config.alt_stat_name())); - Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( - admin_, ssl_context_manager_, *scope, cm, local_info_, dispatcher_, random_, stats_store_, - singleton_manager_, tls_, validation_visitor_, *api_); - cluster_.reset(new LogicalDnsCluster(cluster_config, runtime_, dns_resolver_, factory_context, - std::move(scope), false)); - cluster_->prioritySet().addPriorityUpdateCb( - [&](uint32_t, const HostVector&, const HostVector&) -> void { - membership_updated_.ready(); - }); - cluster_->initialize([&]() -> void { initialized_.ready(); }); - } - void setupFromV2Yaml(const std::string& yaml) { resolve_timer_ = new Event::MockTimer(&dispatcher_); NiceMock cm; @@ -90,14 +69,9 @@ class LogicalDnsClusterTest : public testing::Test { } void testBasicSetup(const std::string& config, const std::string& expected_address, - uint32_t expected_port, uint32_t expected_hc_port, - ConfigType config_type = ConfigType::V2_YAML) { + uint32_t expected_port, uint32_t expected_hc_port) { expectResolve(Network::DnsLookupFamily::V4Only, expected_address); - if (config_type == ConfigType::V1_JSON) { - setupFromV1Json(config); - } else { - setupFromV2Yaml(config); - } + setupFromV2Yaml(config); EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); @@ -224,28 +198,31 @@ using LogicalDnsConfigTuple = std::vector generateLogicalDnsParams() { std::vector dns_config; { - std::string family_json(""); - Network::DnsLookupFamily family(Network::DnsLookupFamily::V4Only); + std::string family_yaml(""); + Network::DnsLookupFamily family(Network::DnsLookupFamily::Auto); std::list dns_response{"127.0.0.1", "127.0.0.2"}; - dns_config.push_back(std::make_tuple(family_json, family, dns_response)); + dns_config.push_back(std::make_tuple(family_yaml, family, dns_response)); } { - std::string family_json(R"EOF("dns_lookup_family": "v4_only",)EOF"); + std::string family_yaml(R"EOF(dns_lookup_family: v4_only + )EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::V4Only); std::list dns_response{"127.0.0.1", "127.0.0.2"}; - dns_config.push_back(std::make_tuple(family_json, family, dns_response)); + dns_config.push_back(std::make_tuple(family_yaml, family, dns_response)); } { - std::string family_json(R"EOF("dns_lookup_family": "v6_only",)EOF"); + std::string family_yaml(R"EOF(dns_lookup_family: v6_only + )EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::V6Only); std::list dns_response{"::1", "::2"}; - dns_config.push_back(std::make_tuple(family_json, family, dns_response)); + dns_config.push_back(std::make_tuple(family_yaml, family, dns_response)); } { - std::string family_json(R"EOF("dns_lookup_family": "auto",)EOF"); + std::string family_yaml(R"EOF(dns_lookup_family: auto + )EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::Auto); std::list dns_response{"::1"}; - dns_config.push_back(std::make_tuple(family_json, family, dns_response)); + dns_config.push_back(std::make_tuple(family_yaml, family, dns_response)); } return dns_config; } @@ -260,16 +237,17 @@ INSTANTIATE_TEST_SUITE_P(DnsParam, LogicalDnsParamTest, // constructor, we have the expected host state and initialization callback // invocation. TEST_P(LogicalDnsParamTest, ImmediateResolve) { - const std::string json = R"EOF( - { - "name": "name", - "connect_timeout_ms": 250, - "type": "logical_dns", - "lb_type": "round_robin", + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: logical_dns + lb_policy: round_robin )EOF" + std::get<0>(GetParam()) + R"EOF( - "hosts": [{"url": "tcp://foo.bar.com:443"}] - } + hosts: + - socket_address: + address: foo.bar.com + port_value: 443 )EOF"; EXPECT_CALL(membership_updated_, ready()); @@ -281,7 +259,7 @@ TEST_P(LogicalDnsParamTest, ImmediateResolve) { cb(TestUtility::makeDnsResponse(std::get<2>(GetParam()))); return nullptr; })); - setupFromV1Json(json); + setupFromV2Yaml(yaml); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); EXPECT_EQ("foo.bar.com", @@ -291,17 +269,22 @@ TEST_P(LogicalDnsParamTest, ImmediateResolve) { } TEST_F(LogicalDnsClusterTest, BadConfig) { - const std::string multiple_hosts_json = R"EOF( - { - "name": "name", - "connect_timeout_ms": 250, - "type": "logical_dns", - "lb_type": "round_robin", - "hosts": [{"url": "tcp://foo.bar.com:443"}, {"url": "tcp://foo2.bar.com:443"}] - } + const std::string multiple_hosts_yaml = R"EOF( + name: name + type: LOGICAL_DNS + dns_refresh_rate: 4s + connect_timeout: 0.25s + lb_policy: ROUND_ROBIN + hosts: + - socket_address: + address: foo.bar.com + port_value: 443 + - socket_address: + address: foo2.bar.com + port_value: 443 )EOF"; - EXPECT_THROW_WITH_MESSAGE(setupFromV1Json(multiple_hosts_json), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(setupFromV2Yaml(multiple_hosts_yaml), EnvoyException, "LOGICAL_DNS clusters must have a single host"); const std::string multiple_lb_endpoints_yaml = R"EOF( @@ -394,17 +377,6 @@ TEST_F(LogicalDnsClusterTest, BadConfig) { } TEST_F(LogicalDnsClusterTest, Basic) { - const std::string json = R"EOF( - { - "name": "name", - "connect_timeout_ms": 250, - "type": "logical_dns", - "lb_type": "round_robin", - "hosts": [{"url": "tcp://foo.bar.com:443"}], - "dns_refresh_rate_ms": 4000 - } - )EOF"; - const std::string basic_yaml_hosts = R"EOF( name: name type: LOGICAL_DNS @@ -442,7 +414,6 @@ TEST_F(LogicalDnsClusterTest, Basic) { port_value: 8000 )EOF"; - testBasicSetup(json, "foo.bar.com", 443, 443, ConfigType::V1_JSON); testBasicSetup(basic_yaml_hosts, "foo.bar.com", 443, 443); // Expect to override the health check address port value. testBasicSetup(basic_yaml_load_assignment, "foo.bar.com", 443, 8000); diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index e3c5638dfef2..ba5e955b6fa6 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -104,33 +104,16 @@ class OriginalDstClusterTest : public testing::Test { Api::ApiPtr api_; }; -TEST(OriginalDstClusterConfigTest, BadConfig) { - std::string json = R"EOF( - { - "name": "name", - "connect_timeout_ms": 250, - "type": "original_dst", - "lb_type": "original_dst_lb", - "hosts": [{"url": "tcp://foo.bar.com:443"}] - } - )EOF"; // Help Emacs balance quotation marks: " - - EXPECT_THROW_WITH_MESSAGE(parseClusterFromJson(json), EnvoyException, - "original_dst clusters must have no hosts configured"); -} - TEST(OriginalDstClusterConfigTest, GoodConfig) { - std::string json = R"EOF( - { - "name": "name", - "connect_timeout_ms": 250, - "type": "original_dst", - "lb_type": "original_dst_lb", - "cleanup_interval_ms": 1000 - } + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: original_dst + lb_policy: original_dst_lb + cleanup_interval: 1s )EOF"; // Help Emacs balance quotation marks: " - EXPECT_TRUE(parseClusterFromJson(json).has_cleanup_interval()); + EXPECT_TRUE(parseClusterFromV2Yaml(yaml).has_cleanup_interval()); } TEST_F(OriginalDstClusterTest, BadConfigWithLoadAssignment) { diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 688e53b06e7b..d221c6d838ff 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -113,28 +113,31 @@ using StrictDnsConfigTuple = std::vector generateStrictDnsParams() { std::vector dns_config; { - std::string family_json(""); - Network::DnsLookupFamily family(Network::DnsLookupFamily::V4Only); + std::string family_yaml(""); + Network::DnsLookupFamily family(Network::DnsLookupFamily::Auto); std::list dns_response{"127.0.0.1", "127.0.0.2"}; - dns_config.push_back(std::make_tuple(family_json, family, dns_response)); + dns_config.push_back(std::make_tuple(family_yaml, family, dns_response)); } { - std::string family_json(R"EOF("dns_lookup_family": "v4_only",)EOF"); + std::string family_yaml(R"EOF(dns_lookup_family: v4_only + )EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::V4Only); std::list dns_response{"127.0.0.1", "127.0.0.2"}; - dns_config.push_back(std::make_tuple(family_json, family, dns_response)); + dns_config.push_back(std::make_tuple(family_yaml, family, dns_response)); } { - std::string family_json(R"EOF("dns_lookup_family": "v6_only",)EOF"); + std::string family_yaml(R"EOF(dns_lookup_family: v6_only + )EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::V6Only); std::list dns_response{"::1", "::2"}; - dns_config.push_back(std::make_tuple(family_json, family, dns_response)); + dns_config.push_back(std::make_tuple(family_yaml, family, dns_response)); } { - std::string family_json(R"EOF("dns_lookup_family": "auto",)EOF"); + std::string family_yaml(R"EOF(dns_lookup_family: auto + )EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::Auto); std::list dns_response{"127.0.0.1", "127.0.0.2"}; - dns_config.push_back(std::make_tuple(family_json, family, dns_response)); + dns_config.push_back(std::make_tuple(family_yaml, family, dns_response)); } return dns_config; } @@ -148,16 +151,17 @@ INSTANTIATE_TEST_SUITE_P(DnsParam, StrictDnsParamTest, TEST_P(StrictDnsParamTest, ImmediateResolve) { auto dns_resolver = std::make_shared>(); ReadyWatcher initialized; - const std::string json = R"EOF( - { - "name": "name", - "connect_timeout_ms": 250, - "type": "strict_dns", - )EOF" + std::get<0>(GetParam()) + + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: strict_dns + )EOF" + std::get<0>(GetParam()) + R"EOF( - "lb_type": "round_robin", - "hosts": [{"url": "tcp://foo.bar.com:443"}] - } + lb_policy: round_robin + hosts: + - socket_address: + address: foo.bar.com + port_value: 443 )EOF"; EXPECT_CALL(initialized, ready()); EXPECT_CALL(*dns_resolver, resolve("foo.bar.com", std::get<1>(GetParam()), _)) @@ -166,7 +170,7 @@ TEST_P(StrictDnsParamTest, ImmediateResolve) { cb(TestUtility::makeDnsResponse(std::get<2>(GetParam()))); return nullptr; })); - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() : cluster_config.alt_stat_name())); @@ -228,38 +232,33 @@ TEST_F(StrictDnsClusterImplTest, Basic) { ResolverData resolver2(*dns_resolver_, dispatcher_); ResolverData resolver1(*dns_resolver_, dispatcher_); - const std::string json = R"EOF( - { - "name": "name", - "connect_timeout_ms": 250, - "type": "strict_dns", - "dns_refresh_rate_ms": 4000, - "lb_type": "round_robin", - "circuit_breakers": { - "default": { - "max_connections": 43, - "max_pending_requests": 57, - "max_requests": 50, - "max_retries": 10 - }, - "high": { - "max_connections": 1, - "max_pending_requests": 2, - "max_requests": 3, - "max_retries": 4 - } - }, - "max_requests_per_connection": 3, - "http2_settings": { - "hpack_table_size": 0 - }, - "hosts": [{"url": "tcp://localhost1:11001"}, - {"url": "tcp://localhost2:11002"}] - - } + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: strict_dns + dns_refresh_rate: 4s + lb_policy: round_robin + circuit_breakers: + thresholds: + - priority: DEFAULT + max_connections: 43 + max_pending_requests: 57 + max_requests: 50 + max_retries: 10 + - priority: HIGH + max_connections: 1 + max_pending_requests: 2 + max_requests: 3 + max_retries: 4 + max_requests_per_connection: 3 + http2_protocol_options: + hpack_table_size: 0 + hosts: + - { socket_address: { address: localhost1, port_value: 11001 }} + - { socket_address: { address: localhost2, port_value: 11002 }} )EOF"; - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() : cluster_config.alt_stat_name())); @@ -1050,32 +1049,6 @@ TEST_F(StaticClusterImplTest, InitialHosts) { EXPECT_FALSE(cluster.info()->addedViaApi()); } -TEST_F(StaticClusterImplTest, EmptyHostname) { - const std::string json = R"EOF( - { - "name": "staticcluster", - "connect_timeout_ms": 250, - "type": "static", - "lb_type": "random", - "hosts": [{"url": "tcp://10.0.0.1:11001"}] - } - )EOF"; - - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); - Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( - "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() - : cluster_config.alt_stat_name())); - Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( - admin_, ssl_context_manager_, *scope, cm_, local_info_, dispatcher_, random_, stats_, - singleton_manager_, tls_, validation_visitor_, *api_); - StaticClusterImpl cluster(cluster_config, runtime_, factory_context, std::move(scope), false); - cluster.initialize([] {}); - - EXPECT_EQ(1UL, cluster.prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); - EXPECT_EQ("", cluster.prioritySet().hostSetsPerPriority()[0]->hosts()[0]->hostname()); - EXPECT_FALSE(cluster.info()->addedViaApi()); -} - TEST_F(StaticClusterImplTest, LoadAssignmentEmptyHostname) { const std::string yaml = R"EOF( name: staticcluster @@ -1278,17 +1251,15 @@ TEST_F(StaticClusterImplTest, AltStatName) { } TEST_F(StaticClusterImplTest, RingHash) { - const std::string json = R"EOF( - { - "name": "staticcluster", - "connect_timeout_ms": 250, - "type": "static", - "lb_type": "ring_hash", - "hosts": [{"url": "tcp://10.0.0.1:11001"}] - } + const std::string yaml = R"EOF( + name: staticcluster + connect_timeout: 0.25s + type: static + lb_policy: ring_hash + hosts: [{ socket_address: { address: 10.0.0.1, port_value: 11001 }}] )EOF"; - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() : cluster_config.alt_stat_name())); @@ -1304,18 +1275,17 @@ TEST_F(StaticClusterImplTest, RingHash) { } TEST_F(StaticClusterImplTest, OutlierDetector) { - const std::string json = R"EOF( - { - "name": "addressportconfig", - "connect_timeout_ms": 250, - "type": "static", - "lb_type": "random", - "hosts": [{"url": "tcp://10.0.0.1:11001"}, - {"url": "tcp://10.0.0.2:11002"}] - } + const std::string yaml = R"EOF( + name: addressportconfig + connect_timeout: 0.25s + type: static + lb_policy: random + hosts: + - { socket_address: { address: 10.0.0.1, port_value: 11001 }} + - { socket_address: { address: 10.0.0.1, port_value: 11002 }} )EOF"; - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() : cluster_config.alt_stat_name())); @@ -1353,18 +1323,17 @@ TEST_F(StaticClusterImplTest, OutlierDetector) { } TEST_F(StaticClusterImplTest, HealthyStat) { - const std::string json = R"EOF( - { - "name": "addressportconfig", - "connect_timeout_ms": 250, - "type": "static", - "lb_type": "random", - "hosts": [{"url": "tcp://10.0.0.1:11001"}, - {"url": "tcp://10.0.0.2:11002"}] - } + const std::string yaml = R"EOF( + name: addressportconfig + connect_timeout: 0.25s + type: static + lb_policy: random + hosts: + - { socket_address: { address: 10.0.0.1, port_value: 11001 }} + - { socket_address: { address: 10.0.0.1, port_value: 11002 }} )EOF"; - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() : cluster_config.alt_stat_name())); @@ -1485,18 +1454,17 @@ TEST_F(StaticClusterImplTest, HealthyStat) { } TEST_F(StaticClusterImplTest, UrlConfig) { - const std::string json = R"EOF( - { - "name": "addressportconfig", - "connect_timeout_ms": 250, - "type": "static", - "lb_type": "random", - "hosts": [{"url": "tcp://10.0.0.1:11001"}, - {"url": "tcp://10.0.0.2:11002"}] - } + const std::string yaml = R"EOF( + name: addressportconfig + connect_timeout: 0.25s + type: static + lb_policy: random + hosts: + - { socket_address: { address: 10.0.0.1, port_value: 11001 }} + - { socket_address: { address: 10.0.0.2, port_value: 11002 }} )EOF"; - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() : cluster_config.alt_stat_name())); @@ -1530,20 +1498,19 @@ TEST_F(StaticClusterImplTest, UrlConfig) { } TEST_F(StaticClusterImplTest, UnsupportedLBType) { - const std::string json = R"EOF( - { - "name": "addressportconfig", - "connect_timeout_ms": 250, - "type": "static", - "lb_type": "fakelbtype", - "hosts": [{"url": "tcp://192.168.1.1:22"}, - {"url": "tcp://192.168.1.2:44"}] - } + const std::string yaml = R"EOF( + name: addressportconfig + connect_timeout: 0.25s + type: static + lb_policy: fakelbtype + hosts: + - { socket_address: { address: 192.168.1.1, port_value: 22 }} + - { socket_address: { address: 192.168.1.2, port_value: 44 }} )EOF"; EXPECT_THROW_WITH_MESSAGE( { - envoy::api::v2::Cluster cluster_config = parseClusterFromJson(json); + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format("cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() @@ -1555,10 +1522,8 @@ TEST_F(StaticClusterImplTest, UnsupportedLBType) { false); }, EnvoyException, - "JSON at lines 2-9 does not conform to schema.\n" - " Invalid schema: #/properties/lb_type\n" - " Schema violation: enum\n" - " Offending document key: #/lb_type"); + "Protobuf message (type envoy.api.v2.Cluster reason INVALID_ARGUMENT:(lb_policy): invalid " + "value \"fakelbtype\" for type TYPE_ENUM) has unknown fields"); } TEST_F(StaticClusterImplTest, MalformedHostIP) { diff --git a/test/common/upstream/utility.h b/test/common/upstream/utility.h index 6a541d08d6bd..b5da2071d09b 100644 --- a/test/common/upstream/utility.h +++ b/test/common/upstream/utility.h @@ -3,7 +3,6 @@ #include "envoy/upstream/upstream.h" #include "common/common/utility.h" -#include "common/config/cds_json.h" #include "common/json/json_loader.h" #include "common/network/utility.h" #include "common/upstream/upstream_impl.h" @@ -43,14 +42,6 @@ parseBootstrapFromV2Json(const std::string& json_string) { return bootstrap; } -inline envoy::api::v2::Cluster parseClusterFromJson(const std::string& json_string) { - envoy::api::v2::Cluster cluster; - auto json_object_ptr = Json::Factory::loadFromString(json_string); - Config::CdsJson::translateCluster(*json_object_ptr, - absl::optional(), cluster); - return cluster; -} - inline envoy::api::v2::Cluster parseClusterFromV2Json(const std::string& json_string) { envoy::api::v2::Cluster cluster; TestUtility::loadFromJson(json_string, cluster); diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index f987590cfae2..173ac795d476 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -286,8 +286,8 @@ TEST_F(SslContextImplTest, TestGetCertInformationWithExpiration) { } TEST_F(SslContextImplTest, TestNoCert) { - Json::ObjectSharedPtr loader = TestEnvironment::jsonLoadFromString("{}"); - ClientContextConfigImpl cfg(*loader, factory_context_); + envoy::api::v2::auth::UpstreamTlsContext config; + ClientContextConfigImpl cfg(config, factory_context_); Envoy::Ssl::ClientContextSharedPtr context(manager_.createSslClientContext(store_, cfg)); EXPECT_EQ(nullptr, context->getCaCertInformation()); EXPECT_TRUE(context->getCertChainInformation().empty()); diff --git a/test/integration/xfcc_integration_test.cc b/test/integration/xfcc_integration_test.cc index 2e9a5f12d8a6..4e2bed984cfb 100644 --- a/test/integration/xfcc_integration_test.cc +++ b/test/integration/xfcc_integration_test.cc @@ -39,30 +39,37 @@ void XfccIntegrationTest::TearDown() { } Network::TransportSocketFactoryPtr XfccIntegrationTest::createClientSslContext(bool mtls) { - std::string json_tls = R"EOF( -{ - "ca_cert_file": "{{ test_rundir }}/test/config/integration/certs/cacert.pem", - "verify_subject_alt_name": [ "spiffe://lyft.com/backend-team", "lyft.com", "www.lyft.com" ] -} + const std::string yaml_tls = R"EOF( +common_tls_context: + validation_context: + trusted_ca: + filename: {{ test_rundir }}/test/config/integration/certs/cacert.pem + verify_subject_alt_name: [ spiffe://lyft.com/backend-team, lyft.com, www.lyft.com ] )EOF"; - std::string json_mtls = R"EOF( -{ - "ca_cert_file": "{{ test_rundir }}/test/config/integration/certs/cacert.pem", - "cert_chain_file": "{{ test_rundir }}/test/config/integration/certs/clientcert.pem", - "private_key_file": "{{ test_rundir }}/test/config/integration/certs/clientkey.pem", - "verify_subject_alt_name": [ "spiffe://lyft.com/backend-team", "lyft.com", "www.lyft.com" ] -} + + const std::string yaml_mtls = R"EOF( +common_tls_context: + validation_context: + trusted_ca: + filename: {{ test_rundir }}/test/config/integration/certs/cacert.pem + verify_subject_alt_name: [ spiffe://lyft.com/backend-team, lyft.com, www.lyft.com ] + tls_certificates: + certificate_chain: + filename: {{ test_rundir }}/test/config/integration/certs/clientcert.pem + private_key: + filename: {{ test_rundir }}/test/config/integration/certs/clientkey.pem )EOF"; std::string target; if (mtls) { - target = json_mtls; + target = yaml_mtls; } else { - target = json_tls; + target = yaml_tls; } - Json::ObjectSharedPtr loader = TestEnvironment::jsonLoadFromString(target); + envoy::api::v2::auth::UpstreamTlsContext config; + TestUtility::loadFromYaml(TestEnvironment::substitute(target), config); auto cfg = std::make_unique( - *loader, factory_context_); + config, factory_context_); static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); return Network::TransportSocketFactoryPtr{ new Extensions::TransportSockets::Tls::ClientSslSocketFactory( diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index c70a39ad2d18..481a1bbfd533 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -902,8 +902,8 @@ TEST_P(AdminInstanceTest, ContextThatReturnsNullCertDetails) { // Setup a context that returns null cert details. testing::NiceMock factory_context; - Json::ObjectSharedPtr loader = TestEnvironment::jsonLoadFromString("{}"); - Extensions::TransportSockets::Tls::ClientContextConfigImpl cfg(*loader, factory_context); + envoy::api::v2::auth::UpstreamTlsContext config; + Extensions::TransportSockets::Tls::ClientContextConfigImpl cfg(config, factory_context); Stats::IsolatedStoreImpl store; Envoy::Ssl::ClientContextSharedPtr client_ctx( server_.sslContextManager().createSslClientContext(store, cfg)); From 4405d68462fd2643b75ac2b654f1b7e39253ec91 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 23 Jul 2019 08:19:16 +0200 Subject: [PATCH 227/542] stats: use counterFromStatName for ratelimit filters (#7573) * stats: use counterFromStatName for ratelimit filters and add format checks #7573 Signed-off-by: Joshua Marantz --- .../extensions/filters/common/ratelimit/BUILD | 9 ++++ .../filters/common/ratelimit/stat_names.h | 30 +++++++++++++ .../extensions/filters/http/ratelimit/BUILD | 1 + .../filters/http/ratelimit/ratelimit.cc | 9 ++-- .../filters/http/ratelimit/ratelimit.h | 5 ++- .../thrift_proxy/filters/ratelimit/BUILD | 1 + .../filters/ratelimit/ratelimit.cc | 9 ++-- .../filters/ratelimit/ratelimit.h | 9 +++- .../filters/http/ratelimit/ratelimit_test.cc | 4 +- .../filters/ratelimit/ratelimit_test.cc | 2 +- tools/check_format.py | 42 ++++++++++++++++++- tools/check_format_test_helper.py | 19 ++++++--- tools/format_python_tools.sh | 4 +- .../check_format/counter_from_string.cc | 7 ++++ .../check_format/gauge_from_string.cc | 7 ++++ .../check_format/histogram_from_string.cc | 7 ++++ 16 files changed, 143 insertions(+), 22 deletions(-) create mode 100644 source/extensions/filters/common/ratelimit/stat_names.h create mode 100644 tools/testdata/check_format/counter_from_string.cc create mode 100644 tools/testdata/check_format/gauge_from_string.cc create mode 100644 tools/testdata/check_format/histogram_from_string.cc diff --git a/source/extensions/filters/common/ratelimit/BUILD b/source/extensions/filters/common/ratelimit/BUILD index 6b0c197a810d..9d39e1980093 100644 --- a/source/extensions/filters/common/ratelimit/BUILD +++ b/source/extensions/filters/common/ratelimit/BUILD @@ -39,6 +39,15 @@ envoy_cc_library( "//include/envoy/ratelimit:ratelimit_interface", "//include/envoy/singleton:manager_interface", "//include/envoy/tracing:http_tracer_interface", + "//source/common/stats:symbol_table_lib", "@envoy_api//envoy/config/ratelimit/v2:rls_cc", ], ) + +envoy_cc_library( + name = "stat_names_lib", + hdrs = ["stat_names.h"], + deps = [ + "//source/common/stats:symbol_table_lib", + ], +) diff --git a/source/extensions/filters/common/ratelimit/stat_names.h b/source/extensions/filters/common/ratelimit/stat_names.h new file mode 100644 index 000000000000..33dd13a188b3 --- /dev/null +++ b/source/extensions/filters/common/ratelimit/stat_names.h @@ -0,0 +1,30 @@ +#pragma once + +#include "common/stats/symbol_table_impl.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace RateLimit { + +// Captures a set of stat-names needed for recording during rate-limit +// filters. These should generally be initialized once per process, and +// not per-request, to avoid lock contention. +struct StatNames { + explicit StatNames(Stats::SymbolTable& symbol_table) + : pool_(symbol_table), ok_(pool_.add("ratelimit.ok")), error_(pool_.add("ratelimit.error")), + failure_mode_allowed_(pool_.add("ratelimit.failure_mode_allowed")), + over_limit_(pool_.add("ratelimit.over_limit")) {} + Stats::StatNamePool pool_; + Stats::StatName ok_; + Stats::StatName error_; + Stats::StatName failure_mode_allowed_; + Stats::StatName over_limit_; +}; + +} // namespace RateLimit +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/ratelimit/BUILD b/source/extensions/filters/http/ratelimit/BUILD index 45e8abef7c78..22b8e4bba0b6 100644 --- a/source/extensions/filters/http/ratelimit/BUILD +++ b/source/extensions/filters/http/ratelimit/BUILD @@ -24,6 +24,7 @@ envoy_cc_library( "//source/common/http:codes_lib", "//source/common/router:config_lib", "//source/extensions/filters/common/ratelimit:ratelimit_client_interface", + "//source/extensions/filters/common/ratelimit:stat_names_lib", "@envoy_api//envoy/config/filter/http/rate_limit/v2:rate_limit_cc", ], ) diff --git a/source/extensions/filters/http/ratelimit/ratelimit.cc b/source/extensions/filters/http/ratelimit/ratelimit.cc index 2a5761c6b612..6bd65be54811 100644 --- a/source/extensions/filters/http/ratelimit/ratelimit.cc +++ b/source/extensions/filters/http/ratelimit/ratelimit.cc @@ -130,16 +130,17 @@ void Filter::complete(Filters::Common::RateLimit::LimitStatus status, state_ = State::Complete; headers_to_add_ = std::move(headers); Stats::StatName empty_stat_name; + Filters::Common::RateLimit::StatNames& stat_names = config_->statNames(); switch (status) { case Filters::Common::RateLimit::LimitStatus::OK: - cluster_->statsScope().counter("ratelimit.ok").inc(); + cluster_->statsScope().counterFromStatName(stat_names.ok_).inc(); break; case Filters::Common::RateLimit::LimitStatus::Error: - cluster_->statsScope().counter("ratelimit.error").inc(); + cluster_->statsScope().counterFromStatName(stat_names.error_).inc(); break; case Filters::Common::RateLimit::LimitStatus::OverLimit: - cluster_->statsScope().counter("ratelimit.over_limit").inc(); + cluster_->statsScope().counterFromStatName(stat_names.over_limit_).inc(); Http::CodeStats::ResponseStatInfo info{config_->scope(), cluster_->statsScope(), empty_stat_name, @@ -165,7 +166,7 @@ void Filter::complete(Filters::Common::RateLimit::LimitStatus status, callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::RateLimited); } else if (status == Filters::Common::RateLimit::LimitStatus::Error) { if (config_->failureModeAllow()) { - cluster_->statsScope().counter("ratelimit.failure_mode_allowed").inc(); + cluster_->statsScope().counterFromStatName(stat_names.failure_mode_allowed_).inc(); if (!initiating_call_) { callbacks_->continueDecoding(); } diff --git a/source/extensions/filters/http/ratelimit/ratelimit.h b/source/extensions/filters/http/ratelimit/ratelimit.h index 1103f9e9763b..dfda9883ee2d 100644 --- a/source/extensions/filters/http/ratelimit/ratelimit.h +++ b/source/extensions/filters/http/ratelimit/ratelimit.h @@ -18,6 +18,7 @@ #include "common/http/header_map_impl.h" #include "extensions/filters/common/ratelimit/ratelimit.h" +#include "extensions/filters/common/ratelimit/stat_names.h" namespace Envoy { namespace Extensions { @@ -46,7 +47,7 @@ class FilterConfig { config.rate_limited_as_resource_exhausted() ? absl::make_optional(Grpc::Status::GrpcStatus::ResourceExhausted) : absl::nullopt), - http_context_(http_context) {} + http_context_(http_context), stat_names_(scope.symbolTable()) {} const std::string& domain() const { return domain_; } const LocalInfo::LocalInfo& localInfo() const { return local_info_; } uint64_t stage() const { return stage_; } @@ -58,6 +59,7 @@ class FilterConfig { return rate_limited_grpc_status_; } Http::Context& httpContext() { return http_context_; } + Filters::Common::RateLimit::StatNames& statNames() { return stat_names_; } private: static FilterRequestType stringToType(const std::string& request_type) { @@ -80,6 +82,7 @@ class FilterConfig { const bool failure_mode_deny_; const absl::optional rate_limited_grpc_status_; Http::Context& http_context_; + Filters::Common::RateLimit::StatNames stat_names_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/BUILD b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/BUILD index 8796cbcc7400..1d6128fda8fb 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/BUILD +++ b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/BUILD @@ -18,6 +18,7 @@ envoy_cc_library( "//source/common/tracing:http_tracer_lib", "//source/extensions/filters/common/ratelimit:ratelimit_client_interface", "//source/extensions/filters/common/ratelimit:ratelimit_lib", + "//source/extensions/filters/common/ratelimit:stat_names_lib", "//source/extensions/filters/network/thrift_proxy:app_exception_lib", "//source/extensions/filters/network/thrift_proxy/filters:filter_interface", "//source/extensions/filters/network/thrift_proxy/router:router_ratelimit_interface", diff --git a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.cc b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.cc index 6131bb77b8b9..d4e5dc70f5d0 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.cc +++ b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.cc @@ -65,13 +65,14 @@ void Filter::complete(Filters::Common::RateLimit::LimitStatus status, UNREFERENCED_PARAMETER(headers); state_ = State::Complete; + Filters::Common::RateLimit::StatNames& stat_names = config_->statNames(); switch (status) { case Filters::Common::RateLimit::LimitStatus::OK: - cluster_->statsScope().counter("ratelimit.ok").inc(); + cluster_->statsScope().counterFromStatName(stat_names.ok_).inc(); break; case Filters::Common::RateLimit::LimitStatus::Error: - cluster_->statsScope().counter("ratelimit.error").inc(); + cluster_->statsScope().counterFromStatName(stat_names.error_).inc(); if (!config_->failureModeAllow()) { state_ = State::Responded; callbacks_->sendLocalReply( @@ -80,10 +81,10 @@ void Filter::complete(Filters::Common::RateLimit::LimitStatus status, callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::RateLimitServiceError); return; } - cluster_->statsScope().counter("ratelimit.failure_mode_allowed").inc(); + cluster_->statsScope().counterFromStatName(stat_names.failure_mode_allowed_).inc(); break; case Filters::Common::RateLimit::LimitStatus::OverLimit: - cluster_->statsScope().counter("ratelimit.over_limit").inc(); + cluster_->statsScope().counterFromStatName(stat_names.over_limit_).inc(); if (config_->runtime().snapshot().featureEnabled("ratelimit.thrift_filter_enforcing", 100)) { state_ = State::Responded; callbacks_->sendLocalReply( diff --git a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.h b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.h index eb9e60dbb5f0..2b68090bc8c9 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.h +++ b/source/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit.h @@ -9,7 +9,10 @@ #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" +#include "common/stats/symbol_table_impl.h" + #include "extensions/filters/common/ratelimit/ratelimit.h" +#include "extensions/filters/common/ratelimit/stat_names.h" #include "extensions/filters/network/thrift_proxy/filters/filter.h" namespace Envoy { @@ -28,7 +31,8 @@ class Config { const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, Runtime::Loader& runtime, Upstream::ClusterManager& cm) : domain_(config.domain()), stage_(config.stage()), local_info_(local_info), scope_(scope), - runtime_(runtime), cm_(cm), failure_mode_deny_(config.failure_mode_deny()) {} + runtime_(runtime), cm_(cm), failure_mode_deny_(config.failure_mode_deny()), + stat_names_(scope_.symbolTable()) {} const std::string& domain() const { return domain_; } const LocalInfo::LocalInfo& localInfo() const { return local_info_; } @@ -36,8 +40,8 @@ class Config { Stats::Scope& scope() { return scope_; } Runtime::Loader& runtime() { return runtime_; } Upstream::ClusterManager& cm() { return cm_; } - bool failureModeAllow() const { return !failure_mode_deny_; }; + Filters::Common::RateLimit::StatNames& statNames() { return stat_names_; } private: const std::string domain_; @@ -47,6 +51,7 @@ class Config { Runtime::Loader& runtime_; Upstream::ClusterManager& cm_; const bool failure_mode_deny_; + Filters::Common::RateLimit::StatNames stat_names_; }; using ConfigSharedPtr = std::shared_ptr; diff --git a/test/extensions/filters/http/ratelimit/ratelimit_test.cc b/test/extensions/filters/http/ratelimit/ratelimit_test.cc index c956709a2a54..4e030d64b8e7 100644 --- a/test/extensions/filters/http/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/http/ratelimit/ratelimit_test.cc @@ -76,9 +76,7 @@ class HttpRateLimitFilterTest : public testing::Test { domain: foo )EOF"; - FilterConfigSharedPtr config_; Filters::Common::RateLimit::MockClient* client_; - std::unique_ptr filter_; NiceMock filter_callbacks_; Filters::Common::RateLimit::RequestCallbacks* request_callbacks_{}; Http::TestHeaderMapImpl request_headers_; @@ -86,6 +84,8 @@ class HttpRateLimitFilterTest : public testing::Test { Buffer::OwnedImpl data_; Buffer::OwnedImpl response_data_; NiceMock stats_store_; + FilterConfigSharedPtr config_; + std::unique_ptr filter_; NiceMock runtime_; NiceMock route_rate_limit_; NiceMock vh_rate_limit_; diff --git a/test/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit_test.cc b/test/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit_test.cc index f8f3d4dd812b..01877c365c4e 100644 --- a/test/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit_test.cc @@ -75,6 +75,7 @@ class ThriftRateLimitFilterTest : public testing::Test { domain: foo )EOF"; + Stats::IsolatedStoreImpl stats_store_; ConfigSharedPtr config_; Filters::Common::RateLimit::MockClient* client_; std::unique_ptr filter_; @@ -84,7 +85,6 @@ class ThriftRateLimitFilterTest : public testing::Test { Http::TestHeaderMapImpl response_headers_; Buffer::OwnedImpl data_; Buffer::OwnedImpl response_data_; - Stats::IsolatedStoreImpl stats_store_; NiceMock runtime_; NiceMock cm_; NiceMock route_rate_limit_; diff --git a/tools/check_format.py b/tools/check_format.py index 4e9db37bf505..8a9bece5c4c0 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -42,6 +42,24 @@ "./test/test_common/utility.cc", "./test/test_common/utility.h", "./test/integration/integration.h") +# Files matching these directories can use stats by string for now. These should +# be eliminated but for now we don't want to grow this work. The goal for this +# whitelist is to eliminate it by making code transformations similar to +# https://github.com/envoyproxy/envoy/pull/7573 and others. +# +# TODO(#4196): Eliminate this list completely and then merge #4980. +STAT_FROM_STRING_WHITELIST = ("./source/common/memory/heap_shrinker.cc", + "./source/extensions/filters/http/dynamo/dynamo_filter.cc", + "./source/extensions/filters/http/ext_authz/ext_authz.cc", + "./source/extensions/filters/http/fault/fault_filter.cc", + "./source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc", + "./source/extensions/filters/network/mongo_proxy/proxy.cc", + "./source/extensions/filters/network/zookeeper_proxy/filter.cc", + "./source/extensions/stat_sinks/common/statsd/statsd.cc", + "./source/extensions/transport_sockets/tls/context_impl.cc", + "./source/server/guarddog_impl.cc", + "./source/server/overload_manager_impl.cc") + # Files in these paths can use MessageLite::SerializeAsString SERIALIZE_AS_STRING_WHITELIST = ("./test/common/protobuf/utility_test.cc", "./test/common/grpc/codec_test.cc") @@ -316,6 +334,10 @@ def whitelistedForJsonStringToMessage(file_path): return file_path in JSON_STRING_TO_MESSAGE_WHITELIST +def whitelistedForStatFromString(file_path): + return file_path in STAT_FROM_STRING_WHITELIST + + def findSubstringAndReturnError(pattern, file_path, error_message): with open(file_path) as f: text = f.read() @@ -456,11 +478,24 @@ def hasCondVarWaitFor(line): return True +# Determines whether the filename is either in the specified subdirectory, or +# at the top level. We consider files in the top level for the benefit of +# the check_format testcases in tools/testdata/check_format. +def isInSubdir(filename, *subdirs): + # Skip this check for check_format's unit-tests. + if filename.count("/") <= 1: + return True + for subdir in subdirs: + if filename.startswith('./' + subdir + '/'): + return True + return False + + def checkSourceLine(line, file_path, reportError): # Check fixable errors. These may have been fixed already. if line.find(". ") != -1: reportError("over-enthusiastic spaces") - if ('source' in file_path or 'include' in file_path) and X_ENVOY_USED_DIRECTLY_REGEX.match(line): + if isInSubdir(file_path, 'source', 'include') and X_ENVOY_USED_DIRECTLY_REGEX.match(line): reportError( "Please do not use the raw literal x-envoy in source code. See Envoy::Http::PrefixValue.") if hasInvalidAngleBracketDirectory(line): @@ -546,6 +581,11 @@ def checkSourceLine(line, file_path, reportError): # behavior. reportError("Don't use Protobuf::util::JsonStringToMessage, use TestUtility::loadFromJson.") + if isInSubdir(file_path, 'source') and file_path.endswith('.cc') and \ + not whitelistedForStatFromString(file_path) and \ + ('.counter(' in line or '.gauge(' in line or '.histogram(' in line): + reportError("Don't lookup stats by name at runtime; used StatName saved during construction") + def checkBuildLine(line, file_path, reportError): if "@bazel_tools" in line and not (isSkylarkFile(file_path) or file_path.startswith("./bazel/")): diff --git a/tools/check_format_test_helper.py b/tools/check_format_test_helper.py index 646512239866..2641112ea722 100755 --- a/tools/check_format_test_helper.py +++ b/tools/check_format_test_helper.py @@ -102,26 +102,26 @@ def emitStdoutAsError(stdout): logging.error("\n".join(stdout)) -def expectError(status, stdout, expected_substring): +def expectError(filename, status, stdout, expected_substring): if status == 0: - logging.error("Expected failure `%s`, but succeeded" % expected_substring) + logging.error("%s: Expected failure `%s`, but succeeded" % (filename, expected_substring)) return 1 for line in stdout: if expected_substring in line: return 0 - logging.error("Could not find '%s' in:\n" % expected_substring) + logging.error("%s: Could not find '%s' in:\n" % (filename, expected_substring)) emitStdoutAsError(stdout) return 1 def fixFileExpectingFailure(filename, expected_substring): command, infile, outfile, status, stdout = fixFileHelper(filename) - return expectError(status, stdout, expected_substring) + return expectError(filename, status, stdout, expected_substring) def checkFileExpectingError(filename, expected_substring): command, status, stdout = runCheckFormat("check", getInputFile(filename)) - return expectError(status, stdout, expected_substring) + return expectError(filename, status, stdout, expected_substring) def checkAndFixError(filename, expected_substring): @@ -208,6 +208,15 @@ def checkFileExpectingOK(filename): "version_history.rst", "Version history line malformed. Does not match VERSION_HISTORY_NEW_LINE_REGEX in " "check_format.py") + errors += checkUnfixableError( + "counter_from_string.cc", + "Don't lookup stats by name at runtime; used StatName saved during construction") + errors += checkUnfixableError( + "gauge_from_string.cc", + "Don't lookup stats by name at runtime; used StatName saved during construction") + errors += checkUnfixableError( + "histogram_from_string.cc", + "Don't lookup stats by name at runtime; used StatName saved during construction") errors += fixFileExpectingFailure( "api/missing_package.proto", diff --git a/tools/format_python_tools.sh b/tools/format_python_tools.sh index 91b34feb2122..6749ade47071 100755 --- a/tools/format_python_tools.sh +++ b/tools/format_python_tools.sh @@ -9,8 +9,8 @@ cd "$SCRIPTPATH" source_venv "$VENV_DIR" echo "Installing requirements..." -pip3 install --upgrade pip -pip3 install -r requirements.txt +pip3 -q install --upgrade pip +pip3 -q install -r requirements.txt echo "Running Python format check..." python3 format_python_tools.py $1 diff --git a/tools/testdata/check_format/counter_from_string.cc b/tools/testdata/check_format/counter_from_string.cc new file mode 100644 index 000000000000..8c89250fefe9 --- /dev/null +++ b/tools/testdata/check_format/counter_from_string.cc @@ -0,0 +1,7 @@ +namespace Envoy { + +void init(Stats::Scope& scope) { + scope.counter("hello"); +} + +} // namespace Envoy diff --git a/tools/testdata/check_format/gauge_from_string.cc b/tools/testdata/check_format/gauge_from_string.cc new file mode 100644 index 000000000000..06dbd01d2ea3 --- /dev/null +++ b/tools/testdata/check_format/gauge_from_string.cc @@ -0,0 +1,7 @@ +namespace Envoy { + +void init(Stats::Scope& scope) { + scope.gauge("hello"); +} + +} // namespace Envoy diff --git a/tools/testdata/check_format/histogram_from_string.cc b/tools/testdata/check_format/histogram_from_string.cc new file mode 100644 index 000000000000..3c16b433a1a9 --- /dev/null +++ b/tools/testdata/check_format/histogram_from_string.cc @@ -0,0 +1,7 @@ +namespace Envoy { + +void init(Stats::Scope& scope) { + scope.histogram("hello"); +} + +} // namespace Envoy From e20113ab2a4d7450d6c32c91094c9987b4200e5d Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 23 Jul 2019 09:10:36 -0400 Subject: [PATCH 228/542] http: tracking object scope on the encode path (#7603) Tracking the active stream on the encode path, for crash logging. Risk Level: Medium (touching the router) Testing: new unit tests Docs Changes: n/a Release Notes: n/a #7300 Signed-off-by: Alyssa Wilk --- include/envoy/http/BUILD | 1 + include/envoy/http/filter.h | 6 ++++++ source/common/http/async_client_impl.h | 12 +++++++++++- source/common/http/conn_manager_impl.h | 1 + source/common/router/BUILD | 1 + source/common/router/router.cc | 11 +++++++++++ test/common/http/async_client_impl_test.cc | 15 +++++++++++++++ test/common/router/router_test.cc | 7 +++++++ test/common/router/router_upstream_log_test.cc | 1 + test/mocks/common.h | 5 +++++ test/mocks/http/mocks.cc | 2 ++ test/mocks/http/mocks.h | 4 ++++ 12 files changed, 65 insertions(+), 1 deletion(-) diff --git a/include/envoy/http/BUILD b/include/envoy/http/BUILD index 4f0355015ce3..ce03a17f9624 100644 --- a/include/envoy/http/BUILD +++ b/include/envoy/http/BUILD @@ -62,6 +62,7 @@ envoy_cc_library( ":codec_interface", ":header_map_interface", "//include/envoy/access_log:access_log_interface", + "//include/envoy/common:scope_tracker_interface", "//include/envoy/event:dispatcher_interface", "//include/envoy/grpc:status", "//include/envoy/router:router_interface", diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index e34aeec39a94..609205443d25 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -6,6 +6,7 @@ #include #include "envoy/access_log/access_log.h" +#include "envoy/common/scope_tracker.h" #include "envoy/event/dispatcher.h" #include "envoy/grpc/status.h" #include "envoy/http/codec.h" @@ -185,6 +186,11 @@ class StreamFilterCallbacks { * @return tracing configuration. */ virtual const Tracing::Config& tracingConfig() PURE; + + /** + * @return the ScopeTrackedObject for this stream. + */ + virtual const ScopeTrackedObject& scope() PURE; }; /** diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index e0a079dddf73..b5374c455f1a 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -9,6 +9,7 @@ #include #include +#include "envoy/common/scope_tracker.h" #include "envoy/config/typed_metadata.h" #include "envoy/event/dispatcher.h" #include "envoy/http/async_client.h" @@ -73,7 +74,8 @@ class AsyncStreamImpl : public AsyncClient::Stream, public StreamDecoderFilterCallbacks, public Event::DeferredDeletable, Logger::Loggable, - LinkedObject { + LinkedObject, + public ScopeTrackedObject { public: AsyncStreamImpl(AsyncClientImpl& parent, AsyncClient::StreamCallbacks& callbacks, const AsyncClient::StreamOptions& options); @@ -340,9 +342,17 @@ class AsyncStreamImpl : public AsyncClient::Stream, void setDecoderBufferLimit(uint32_t) override {} uint32_t decoderBufferLimit() override { return 0; } bool recreateStream() override { return false; } + const ScopeTrackedObject& scope() override { return *this; } void addUpstreamSocketOptions(const Network::Socket::OptionsSharedPtr&) override {} Network::Socket::OptionsSharedPtr getUpstreamSocketOptions() const override { return {}; } + // ScopeTrackedObject + void dumpState(std::ostream& os, int indent_level) const override { + const char* spaces = spacesForLevel(indent_level); + os << spaces << "AsyncClient " << this << DUMP_MEMBER(stream_id_) << "\n"; + DUMP_DETAILS(&stream_info_); + } + AsyncClient::StreamCallbacks& stream_callbacks_; const uint64_t stream_id_; Router::ProdFilter router_; diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index e2a790b28683..7b3a650d26dd 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -144,6 +144,7 @@ class ConnectionManagerImpl : Logger::Loggable, StreamInfo::StreamInfo& streamInfo() override; Tracing::Span& activeSpan() override; Tracing::Config& tracingConfig() override; + const ScopeTrackedObject& scope() override { return parent_; } // Functions to set or get iteration state. bool canIterate() { return iteration_state_ == IterationState::Continue; } diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 445cc2232673..6e07d093018a 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -240,6 +240,7 @@ envoy_cc_library( "//source/common/common:hex_lib", "//source/common/common:linked_object", "//source/common/common:minimal_logger_lib", + "//source/common/common:scope_tracker", "//source/common/common:utility_lib", "//source/common/grpc:common_lib", "//source/common/http:codes_lib", diff --git a/source/common/router/router.cc b/source/common/router/router.cc index d01fae5bf5a7..c7ff569b06ca 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -16,6 +16,7 @@ #include "common/common/assert.h" #include "common/common/empty_string.h" #include "common/common/enum_to_int.h" +#include "common/common/scope_tracker.h" #include "common/common/utility.h" #include "common/grpc/common.h" #include "common/http/codes.h" @@ -1339,11 +1340,15 @@ Filter::UpstreamRequest::~UpstreamRequest() { } void Filter::UpstreamRequest::decode100ContinueHeaders(Http::HeaderMapPtr&& headers) { + ScopeTrackerScopeState scope(&parent_.callbacks_->scope(), parent_.callbacks_->dispatcher()); + ASSERT(100 == Http::Utility::getResponseStatus(*headers)); parent_.onUpstream100ContinueHeaders(std::move(headers), *this); } void Filter::UpstreamRequest::decodeHeaders(Http::HeaderMapPtr&& headers, bool end_stream) { + ScopeTrackerScopeState scope(&parent_.callbacks_->scope(), parent_.callbacks_->dispatcher()); + // TODO(rodaine): This is actually measuring after the headers are parsed and not the first byte. upstream_timing_.onFirstUpstreamRxByteReceived(parent_.callbacks_->dispatcher().timeSource()); maybeEndDecode(end_stream); @@ -1358,12 +1363,16 @@ void Filter::UpstreamRequest::decodeHeaders(Http::HeaderMapPtr&& headers, bool e } void Filter::UpstreamRequest::decodeData(Buffer::Instance& data, bool end_stream) { + ScopeTrackerScopeState scope(&parent_.callbacks_->scope(), parent_.callbacks_->dispatcher()); + maybeEndDecode(end_stream); stream_info_.addBytesReceived(data.length()); parent_.onUpstreamData(data, *this, end_stream); } void Filter::UpstreamRequest::decodeTrailers(Http::HeaderMapPtr&& trailers) { + ScopeTrackerScopeState scope(&parent_.callbacks_->scope(), parent_.callbacks_->dispatcher()); + maybeEndDecode(true); if (!parent_.config_.upstream_logs_.empty()) { upstream_trailers_ = std::make_unique(*trailers); @@ -1452,6 +1461,8 @@ void Filter::UpstreamRequest::encodeMetadata(Http::MetadataMapPtr&& metadata_map void Filter::UpstreamRequest::onResetStream(Http::StreamResetReason reason, absl::string_view transport_failure_reason) { + ScopeTrackerScopeState scope(&parent_.callbacks_->scope(), parent_.callbacks_->dispatcher()); + clearRequestEncoder(); awaiting_headers_ = false; if (!calling_encode_headers_) { diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index 60fdcf67b4d3..9a58c3f4f83d 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -908,6 +908,21 @@ TEST_F(AsyncClientImplTest, RdsGettersTest) { EXPECT_CALL(stream_callbacks_, onReset()); } +TEST_F(AsyncClientImplTest, DumpState) { + TestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + AsyncClient::Stream* stream = client_.start(stream_callbacks_, AsyncClient::StreamOptions()); + Http::StreamDecoderFilterCallbacks* filter_callbacks = + static_cast(stream); + + std::stringstream out; + filter_callbacks->scope().dumpState(out); + std::string state = out.str(); + EXPECT_THAT(state, testing::HasSubstr("protocol_: 1")); + + EXPECT_CALL(stream_callbacks_, onReset()); +} + } // namespace // Must not be in anonymous namespace for friend to work. diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index aa29d83fd57e..09e769996a03 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -33,6 +33,7 @@ #include "gtest/gtest.h" using testing::_; +using testing::AnyNumber; using testing::AssertionFailure; using testing::AssertionResult; using testing::AssertionSuccess; @@ -98,6 +99,9 @@ class RouterTestBase : public testing::Test { // Make the "system time" non-zero, because 0 is considered invalid by DateUtil. test_time_.setMonotonicTime(std::chrono::milliseconds(50)); + + // Allow any number of setTrackedObject calls for the dispatcher strict mock. + EXPECT_CALL(callbacks_.dispatcher_, setTrackedObject(_)).Times(AnyNumber()); } void expectResponseTimerCreate() { @@ -1292,10 +1296,13 @@ TEST_F(RouterTest, GrpcOk) { HttpTestUtility::addDefaultHeaders(headers); router_.decodeHeaders(headers, true); + EXPECT_CALL(callbacks_.dispatcher_, setTrackedObject(_)).Times(2); Http::HeaderMapPtr response_headers(new Http::TestHeaderMapImpl{{":status", "200"}}); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(200)); response_decoder->decodeHeaders(std::move(response_headers), false); EXPECT_TRUE(verifyHostUpstreamStats(0, 0)); + + EXPECT_CALL(callbacks_.dispatcher_, setTrackedObject(_)).Times(2); Http::HeaderMapPtr response_trailers(new Http::TestHeaderMapImpl{{"grpc-status", "0"}}); response_decoder->decodeTrailers(std::move(response_trailers)); EXPECT_TRUE(verifyHostUpstreamStats(1, 0)); diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index 17e9a4c978d1..ce4574fe9358 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -91,6 +91,7 @@ class RouterUpstreamLogTest : public testing::Test { router_proto)); router_.reset(new TestFilter(*config_)); router_->setDecoderFilterCallbacks(callbacks_); + EXPECT_CALL(callbacks_.dispatcher_, setTrackedObject(_)).Times(testing::AnyNumber()); upstream_locality_.set_zone("to_az"); diff --git a/test/mocks/common.h b/test/mocks/common.h index 2bc37836c50e..22f9fcafa422 100644 --- a/test/mocks/common.h +++ b/test/mocks/common.h @@ -2,6 +2,7 @@ #include +#include "envoy/common/scope_tracker.h" #include "envoy/common/time.h" #include "envoy/common/token_bucket.h" #include "envoy/event/timer.h" @@ -86,4 +87,8 @@ inline bool operator==(const StringViewSaver& saver, const char* str) { return saver.value() == str; } +class MockScopedTrackedObject : public ScopeTrackedObject { + MOCK_CONST_METHOD2(dumpState, void(std::ostream&, int)); +}; + } // namespace Envoy diff --git a/test/mocks/http/mocks.cc b/test/mocks/http/mocks.cc index c30c9bc400ea..0f1161c5725e 100644 --- a/test/mocks/http/mocks.cc +++ b/test/mocks/http/mocks.cc @@ -65,6 +65,7 @@ MockStreamDecoderFilterCallbacks::MockStreamDecoderFilterCallbacks() { ON_CALL(*this, activeSpan()).WillByDefault(ReturnRef(active_span_)); ON_CALL(*this, tracingConfig()).WillByDefault(ReturnRef(tracing_config_)); + ON_CALL(*this, scope()).WillByDefault(ReturnRef(scope_)); ON_CALL(*this, sendLocalReply(_, _, _, _, _)) .WillByDefault(Invoke([this](Code code, absl::string_view body, std::function modify_headers, @@ -97,6 +98,7 @@ MockStreamEncoderFilterCallbacks::MockStreamEncoderFilterCallbacks() { ON_CALL(*this, encodingBuffer()).WillByDefault(Invoke(&buffer_, &Buffer::InstancePtr::get)); ON_CALL(*this, activeSpan()).WillByDefault(ReturnRef(active_span_)); ON_CALL(*this, tracingConfig()).WillByDefault(ReturnRef(tracing_config_)); + ON_CALL(*this, scope()).WillByDefault(ReturnRef(scope_)); } MockStreamEncoderFilterCallbacks::~MockStreamEncoderFilterCallbacks() = default; diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 34cb97b465b9..13f5d6717a03 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -141,6 +141,7 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, MOCK_METHOD0(streamInfo, StreamInfo::StreamInfo&()); MOCK_METHOD0(activeSpan, Tracing::Span&()); MOCK_METHOD0(tracingConfig, Tracing::Config&()); + MOCK_METHOD0(scope, const ScopeTrackedObject&()); MOCK_METHOD0(onDecoderFilterAboveWriteBufferHighWatermark, void()); MOCK_METHOD0(onDecoderFilterBelowWriteBufferLowWatermark, void()); MOCK_METHOD1(addDownstreamWatermarkCallbacks, void(DownstreamWatermarkCallbacks&)); @@ -189,6 +190,7 @@ class MockStreamDecoderFilterCallbacks : public StreamDecoderFilterCallbacks, std::list callbacks_{}; testing::NiceMock active_span_; testing::NiceMock tracing_config_; + testing::NiceMock scope_; std::string details_; bool is_grpc_request_{}; bool is_head_request_{false}; @@ -212,6 +214,7 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, MOCK_METHOD0(streamInfo, StreamInfo::StreamInfo&()); MOCK_METHOD0(activeSpan, Tracing::Span&()); MOCK_METHOD0(tracingConfig, Tracing::Config&()); + MOCK_METHOD0(scope, const ScopeTrackedObject&()); MOCK_METHOD0(onEncoderFilterAboveWriteBufferHighWatermark, void()); MOCK_METHOD0(onEncoderFilterBelowWriteBufferLowWatermark, void()); MOCK_METHOD1(setEncoderBufferLimit, void(uint32_t)); @@ -228,6 +231,7 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, Buffer::InstancePtr buffer_; testing::NiceMock active_span_; testing::NiceMock tracing_config_; + testing::NiceMock scope_; }; class MockStreamDecoderFilter : public StreamDecoderFilter { From 0979ccb9fa9a0e3bba11cf252968c58d30c7ecb0 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Tue, 23 Jul 2019 08:42:36 -0700 Subject: [PATCH 229/542] admin: call listen in tests to verify behavior of SO_REUSEPORT (#7655) Signed-off-by: Ruslan Nigmatullin --- test/server/server_test.cc | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 7c754040455a..16a063ea8e05 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -568,17 +568,37 @@ TEST_P(ServerInstanceImplTest, BootstrapNodeWithoutAccessLog) { "An admin access log path is required for a listening server."); } +namespace { +void bindAndListenTcpSocket(const Network::Address::InstanceConstSharedPtr& address, + const Network::Socket::OptionsSharedPtr& options) { + auto socket = std::make_unique(address, options, true); + // Some kernels erroneously allow `bind` without SO_REUSEPORT for addresses + // with some other socket already listening on it, see #7636. + if (::listen(socket->ioHandle().fd(), 1) != 0) { + // Mimic bind exception for the test simplicity. + throw Network::SocketBindException(fmt::format("cannot listen: {}", strerror(errno)), errno); + } +} +} // namespace + // Test that `socket_options` field in an Admin proto is honored. TEST_P(ServerInstanceImplTest, BootstrapNodeWithSocketOptions) { + // Start Envoy instance with admin port with SO_REUSEPORT option. ASSERT_NO_THROW(initialize("test/server/node_bootstrap_with_admin_socket_options.yaml")); const auto address = server_->admin().socket().localAddress(); - EXPECT_THAT_THROWS_MESSAGE(std::make_unique(address, nullptr, true), - EnvoyException, HasSubstr("Address already in use")); + + // First attempt to bind and listen socket should fail due to the lack of SO_REUSEPORT socket + // options. + EXPECT_THAT_THROWS_MESSAGE(bindAndListenTcpSocket(address, nullptr), EnvoyException, + HasSubstr(strerror(EADDRINUSE))); + + // Second attempt should succeed as kernel allows multiple sockets to listen the same address iff + // both of them use SO_REUSEPORT socket option. auto options = std::make_shared(); options->emplace_back(std::make_shared( envoy::api::v2::core::SocketOption::STATE_PREBIND, ENVOY_MAKE_SOCKET_OPTION_NAME(SOL_SOCKET, SO_REUSEPORT), 1)); - EXPECT_NO_THROW(std::make_unique(address, options, true)); + EXPECT_NO_THROW(bindAndListenTcpSocket(address, options)); } // Empty bootstrap succeeds. From defd9333f170e8ae864154c5dc3f658d057a6829 Mon Sep 17 00:00:00 2001 From: ahedberg Date: Tue, 23 Jul 2019 14:04:37 -0400 Subject: [PATCH 230/542] Fix msan use-of-unitialized-value (#7697) Signed-off-by: Ashley Hedberg --- source/extensions/transport_sockets/tls/context_impl.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index ac4aaadac862..03d247c489cc 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -518,6 +518,7 @@ bool ContextImpl::verifySubjectAltName(X509* cert, case GEN_IPADD: { if (san->d.ip->length == 4) { sockaddr_in sin; + sin.sin_port = 0; sin.sin_family = AF_INET; memcpy(&sin.sin_addr, san->d.ip->data, sizeof(sin.sin_addr)); Network::Address::Ipv4Instance addr(&sin); @@ -528,6 +529,7 @@ bool ContextImpl::verifySubjectAltName(X509* cert, } } else if (san->d.ip->length == 16) { sockaddr_in6 sin6; + sin6.sin6_port = 0; sin6.sin6_family = AF_INET6; memcpy(&sin6.sin6_addr, san->d.ip->data, sizeof(sin6.sin6_addr)); Network::Address::Ipv6Instance addr(sin6); From ec051ad7f2fb23161e72e5a536f42b8d0aa31478 Mon Sep 17 00:00:00 2001 From: "Evan J. Ercolano" Date: Tue, 23 Jul 2019 15:27:34 -0400 Subject: [PATCH 231/542] docs: add more info about the admin logging endpoint (#7678) Signed-off-by: evanercolano --- docs/root/operations/admin.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index 62e4a5e67993..f0d1f1d73bc2 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -179,8 +179,15 @@ modify different aspects of the server: .. http:post:: /logging - Enable/disable different logging levels on different subcomponents. Generally only used during - development. + Enable/disable different logging levels on a particular logger or all loggers. + + - To change the logging level across all loggers, set the query parameter as level=. + - To change a particular logger's level, set the query parameter like so, =. + - To list the loggers, send a POST request to the /logging endpoint without a query parameter. + + .. note:: + + Generally only used during development. .. http:post:: /memory From a48143e13154b3c96507f2635339be1a4fb280ed Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Tue, 23 Jul 2019 14:48:30 -0700 Subject: [PATCH 232/542] http inspector: multiple reads for http inspector (#7673) Description: multiple reads when inspecting http protocol Risk Level: Low Testing: Unit test Docs Changes: N/A Release Notes: N/A Signed-off-by: Yan Xue --- .../filters/listener/http_inspector/BUILD | 5 +- .../listener/http_inspector/http_inspector.cc | 99 +++++++--- .../listener/http_inspector/http_inspector.h | 1 + .../http_inspector/http_protocol_header.h | 65 +++++++ .../http_inspector/http_inspector_test.cc | 169 ++++++++++++++++-- 5 files changed, 295 insertions(+), 44 deletions(-) create mode 100644 source/extensions/filters/listener/http_inspector/http_protocol_header.h diff --git a/source/extensions/filters/listener/http_inspector/BUILD b/source/extensions/filters/listener/http_inspector/BUILD index 1d5b13218f00..e9d3307bc848 100644 --- a/source/extensions/filters/listener/http_inspector/BUILD +++ b/source/extensions/filters/listener/http_inspector/BUILD @@ -13,7 +13,10 @@ envoy_package() envoy_cc_library( name = "http_inspector_lib", srcs = ["http_inspector.cc"], - hdrs = ["http_inspector.h"], + hdrs = [ + "http_inspector.h", + "http_protocol_header.h", + ], deps = [ "//include/envoy/event:dispatcher_interface", "//include/envoy/event:timer_interface", diff --git a/source/extensions/filters/listener/http_inspector/http_inspector.cc b/source/extensions/filters/listener/http_inspector/http_inspector.cc index 7c768ca290df..4a31065b71f8 100644 --- a/source/extensions/filters/listener/http_inspector/http_inspector.cc +++ b/source/extensions/filters/listener/http_inspector/http_inspector.cc @@ -9,6 +9,7 @@ #include "common/common/macros.h" #include "common/http/headers.h" +#include "extensions/filters/listener/http_inspector/http_protocol_header.h" #include "extensions/transport_sockets/well_known_names.h" #include "absl/strings/match.h" @@ -64,46 +65,46 @@ void Filter::onRead() { return; } else if (result.rc_ < 0) { config_->stats().read_error_.inc(); - done(false); - return; + return done(false); } parseHttpHeader(absl::string_view(reinterpret_cast(buf_), result.rc_)); } void Filter::parseHttpHeader(absl::string_view data) { - if (absl::StartsWith(data, Filter::HTTP2_CONNECTION_PREFACE)) { + const size_t len = std::min(data.length(), Filter::HTTP2_CONNECTION_PREFACE.length()); + if (Filter::HTTP2_CONNECTION_PREFACE.compare(0, len, data, 0, len) == 0) { + if (data.length() < Filter::HTTP2_CONNECTION_PREFACE.length()) { + return; + } ENVOY_LOG(trace, "http inspector: http2 connection preface found"); protocol_ = "HTTP/2"; done(true); } else { const size_t pos = data.find_first_of("\r\n"); - - // Cannot find \r or \n - if (pos == absl::string_view::npos) { - ENVOY_LOG(trace, "http inspector: no request line detected"); - return done(false); - } - - const absl::string_view request_line = data.substr(0, pos); - std::vector fields = absl::StrSplit(request_line, absl::MaxSplits(' ', 4)); - - // Method SP Request-URI SP HTTP-Version - if (fields.size() != 3) { - ENVOY_LOG(trace, "http inspector: invalid http1x request line"); - return done(false); - } - - if (httpProtocols().count(fields[2]) == 0) { - ENVOY_LOG(trace, "http inspector: protocol: {} not valid", fields[2]); - return done(false); + if (pos != absl::string_view::npos) { + const absl::string_view request_line = data.substr(0, pos); + const std::vector fields = + absl::StrSplit(request_line, absl::MaxSplits(' ', 4)); + + // Method SP Request-URI SP HTTP-Version + if (fields.size() != 3) { + ENVOY_LOG(trace, "http inspector: invalid http1x request line"); + return done(false); + } + + if (http1xMethods().count(fields[0]) == 0 || httpProtocols().count(fields[2]) == 0) { + ENVOY_LOG(trace, "http inspector: method: {} or protocol: {} not valid", fields[0], + fields[2]); + return done(false); + } + + ENVOY_LOG(trace, "http inspector: method: {}, request uri: {}, protocol: {}", fields[0], + fields[1], fields[2]); + + protocol_ = fields[2]; + return done(true); } - - ENVOY_LOG(trace, "http inspector: method: {}, request uri: {}, protocol: {}", fields[0], - fields[1], fields[2]); - - protocol_ = fields[2]; - done(true); } } @@ -140,6 +141,48 @@ const absl::flat_hash_set& Filter::httpProtocols() const { Http::Headers::get().ProtocolStrings.Http11String); } +const absl::flat_hash_set& Filter::http1xMethods() const { + CONSTRUCT_ON_FIRST_USE(absl::flat_hash_set, + {HttpInspector::ExtendedHeader::get().MethodValues.Acl, + HttpInspector::ExtendedHeader::get().MethodValues.Baseline_Control, + HttpInspector::ExtendedHeader::get().MethodValues.Bind, + HttpInspector::ExtendedHeader::get().MethodValues.Checkin, + HttpInspector::ExtendedHeader::get().MethodValues.Checkout, + HttpInspector::ExtendedHeader::get().MethodValues.Connect, + HttpInspector::ExtendedHeader::get().MethodValues.Copy, + HttpInspector::ExtendedHeader::get().MethodValues.Delete, + HttpInspector::ExtendedHeader::get().MethodValues.Get, + HttpInspector::ExtendedHeader::get().MethodValues.Head, + HttpInspector::ExtendedHeader::get().MethodValues.Label, + HttpInspector::ExtendedHeader::get().MethodValues.Link, + HttpInspector::ExtendedHeader::get().MethodValues.Lock, + HttpInspector::ExtendedHeader::get().MethodValues.Merge, + HttpInspector::ExtendedHeader::get().MethodValues.Mkactivity, + HttpInspector::ExtendedHeader::get().MethodValues.Mkcalendar, + HttpInspector::ExtendedHeader::get().MethodValues.Mkcol, + HttpInspector::ExtendedHeader::get().MethodValues.Mkredirectref, + HttpInspector::ExtendedHeader::get().MethodValues.Mkworkspace, + HttpInspector::ExtendedHeader::get().MethodValues.Move, + HttpInspector::ExtendedHeader::get().MethodValues.Options, + HttpInspector::ExtendedHeader::get().MethodValues.Orderpatch, + HttpInspector::ExtendedHeader::get().MethodValues.Patch, + HttpInspector::ExtendedHeader::get().MethodValues.Post, + HttpInspector::ExtendedHeader::get().MethodValues.Proppatch, + HttpInspector::ExtendedHeader::get().MethodValues.Purge, + HttpInspector::ExtendedHeader::get().MethodValues.Put, + HttpInspector::ExtendedHeader::get().MethodValues.Rebind, + HttpInspector::ExtendedHeader::get().MethodValues.Report, + HttpInspector::ExtendedHeader::get().MethodValues.Search, + HttpInspector::ExtendedHeader::get().MethodValues.Trace, + HttpInspector::ExtendedHeader::get().MethodValues.Unbind, + HttpInspector::ExtendedHeader::get().MethodValues.Uncheckout, + HttpInspector::ExtendedHeader::get().MethodValues.Unlink, + HttpInspector::ExtendedHeader::get().MethodValues.Unlock, + HttpInspector::ExtendedHeader::get().MethodValues.Update, + HttpInspector::ExtendedHeader::get().MethodValues.Updateredirectref, + HttpInspector::ExtendedHeader::get().MethodValues.Version_Control}); +} + } // namespace HttpInspector } // namespace ListenerFilters } // namespace Extensions diff --git a/source/extensions/filters/listener/http_inspector/http_inspector.h b/source/extensions/filters/listener/http_inspector/http_inspector.h index cb284910a0d1..affd419ca6e2 100644 --- a/source/extensions/filters/listener/http_inspector/http_inspector.h +++ b/source/extensions/filters/listener/http_inspector/http_inspector.h @@ -67,6 +67,7 @@ class Filter : public Network::ListenerFilter, Logger::Loggable& httpProtocols() const; + const absl::flat_hash_set& http1xMethods() const; ConfigSharedPtr config_; Network::ListenerFilterCallbacks* cb_{nullptr}; diff --git a/source/extensions/filters/listener/http_inspector/http_protocol_header.h b/source/extensions/filters/listener/http_inspector/http_protocol_header.h new file mode 100644 index 000000000000..ddf2ff517f57 --- /dev/null +++ b/source/extensions/filters/listener/http_inspector/http_protocol_header.h @@ -0,0 +1,65 @@ +#pragma once + +#include + +#include "common/singleton/const_singleton.h" + +namespace Envoy { +namespace Extensions { +namespace ListenerFilters { +namespace HttpInspector { + +/** + * Class including extended methods. + */ +class ExtendedHeaderValues { +public: + struct { + const std::string Acl{"ACL"}; + const std::string Baseline_Control{"BASELINE-CONTROL"}; + const std::string Bind{"BIND"}; + const std::string Checkin{"CHECKIN"}; + const std::string Checkout{"CHECKOUT"}; + const std::string Connect{"CONNECT"}; + const std::string Copy{"COPY"}; + const std::string Delete{"DELETE"}; + const std::string Get{"GET"}; + const std::string Head{"HEAD"}; + const std::string Label{"LABEL"}; + const std::string Link{"LINK"}; + const std::string Lock{"LOCK"}; + const std::string Merge{"Merge"}; + const std::string Mkactivity{"MKACTIVITY"}; + const std::string Mkcalendar{"MKCALENDAR"}; + const std::string Mkcol{"MKCOL"}; + const std::string Mkredirectref{"MKREDIRECTREF"}; + const std::string Mkworkspace{"MKWORKSPACE"}; + const std::string Move{"MOVE"}; + const std::string Options{"OPTIONS"}; + const std::string Orderpatch{"ORDERPATCH"}; + const std::string Patch{"PATCH"}; + const std::string Post{"POST"}; + const std::string Pri{"PRI"}; + const std::string Proppatch{"PROPPATCH"}; + const std::string Purge{"PURGE"}; + const std::string Put{"PUT"}; + const std::string Rebind{"REBIND"}; + const std::string Report{"REPORT"}; + const std::string Search{"SEARCH"}; + const std::string Trace{"TRACE"}; + const std::string Unbind{"UNBIND"}; + const std::string Uncheckout{"UNCHECKOUT"}; + const std::string Unlink{"UNLINK"}; + const std::string Unlock{"UNLOCK"}; + const std::string Update{"UPDATE"}; + const std::string Updateredirectref{"UPDATEREDIRECTREF"}; + const std::string Version_Control{"VERSION-CONTROL"}; + } MethodValues; +}; + +using ExtendedHeader = ConstSingleton; + +} // namespace HttpInspector +} // namespace ListenerFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc index bda4ed6602ba..afca8e66e9f7 100644 --- a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc +++ b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc @@ -127,12 +127,10 @@ TEST_F(HttpInspectorTest, InvalidHttpMethod) { return Api::SysCallSizeResult{ssize_t(header.size()), 0}; })); - const std::vector alpn_protos{absl::string_view("http/1.1")}; - - EXPECT_CALL(socket_, setRequestedApplicationProtocols(alpn_protos)); + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); EXPECT_CALL(cb_, continueFilterChain(true)); file_event_callback_(Event::FileReadyType::Read); - EXPECT_EQ(1, cfg_->stats().http11_found_.value()); + EXPECT_EQ(0, cfg_->stats().http11_found_.value()); } TEST_F(HttpInspectorTest, InvalidHttpRequestLine) { @@ -147,9 +145,8 @@ TEST_F(HttpInspectorTest, InvalidHttpRequestLine) { })); EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); - EXPECT_CALL(cb_, continueFilterChain(true)); + EXPECT_CALL(cb_, continueFilterChain(_)).Times(0); file_event_callback_(Event::FileReadyType::Read); - EXPECT_EQ(1, cfg_->stats().http_not_found_.value()); } TEST_F(HttpInspectorTest, UnsupportedHttpProtocol) { @@ -226,9 +223,9 @@ TEST_F(HttpInspectorTest, InvalidConnectionPreface) { })); EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); - EXPECT_CALL(cb_, continueFilterChain(true)); + EXPECT_CALL(cb_, continueFilterChain(true)).Times(0); file_event_callback_(Event::FileReadyType::Read); - EXPECT_EQ(1, cfg_->stats().http_not_found_.value()); + EXPECT_EQ(0, cfg_->stats().http_not_found_.value()); } TEST_F(HttpInspectorTest, ReadError) { @@ -242,7 +239,7 @@ TEST_F(HttpInspectorTest, ReadError) { EXPECT_EQ(1, cfg_->stats().read_error_.value()); } -TEST_F(HttpInspectorTest, MultipleReads) { +TEST_F(HttpInspectorTest, MultipleReadsHttp2) { init(); const std::vector alpn_protos = {absl::string_view("h2")}; @@ -260,12 +257,16 @@ TEST_F(HttpInspectorTest, MultipleReads) { EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)).WillOnce(InvokeWithoutArgs([]() { return Api::SysCallSizeResult{ssize_t(-1), EAGAIN}; })); - EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) - .WillOnce(Invoke([&data](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { - ASSERT(length >= data.size()); - memcpy(buffer, data.data(), data.size()); - return Api::SysCallSizeResult{ssize_t(data.size()), 0}; - })); + + for (size_t i = 1; i <= 24; i++) { + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce( + Invoke([&data, i](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= data.size()); + memcpy(buffer, data.data(), data.size()); + return Api::SysCallSizeResult{ssize_t(i), 0}; + })); + } } bool got_continue = false; @@ -279,6 +280,144 @@ TEST_F(HttpInspectorTest, MultipleReads) { EXPECT_EQ(1, cfg_->stats().http2_found_.value()); } +TEST_F(HttpInspectorTest, MultipleReadsHttp2BadPreface) { + + init(); + const std::string header = "505249202a20485454502f322e300d0a0d0c"; + const std::vector data = Hex::decode(header); + { + InSequence s; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)).WillOnce(InvokeWithoutArgs([]() { + return Api::SysCallSizeResult{ssize_t(-1), EAGAIN}; + })); + + for (size_t i = 1; i <= data.size(); i++) { + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce( + Invoke([&data, i](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= data.size()); + memcpy(buffer, data.data(), data.size()); + return Api::SysCallSizeResult{ssize_t(i), 0}; + })); + } + } + + bool got_continue = false; + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); + EXPECT_CALL(cb_, continueFilterChain(true)).WillOnce(InvokeWithoutArgs([&got_continue]() { + got_continue = true; + })); + while (!got_continue) { + file_event_callback_(Event::FileReadyType::Read); + } + EXPECT_EQ(1, cfg_->stats().http_not_found_.value()); +} + +TEST_F(HttpInspectorTest, MultipleReadsHttp1) { + + init(); + + const absl::string_view data = "GET /anything HTTP/1.0\r"; + { + InSequence s; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)).WillOnce(InvokeWithoutArgs([]() { + return Api::SysCallSizeResult{ssize_t(-1), EAGAIN}; + })); + + for (size_t i = 1; i <= data.length(); i++) { + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce( + Invoke([&data, i](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= data.size()); + memcpy(buffer, data.data(), data.size()); + return Api::SysCallSizeResult{ssize_t(i), 0}; + })); + } + } + + bool got_continue = false; + const std::vector alpn_protos = {absl::string_view("http/1.0")}; + EXPECT_CALL(socket_, setRequestedApplicationProtocols(alpn_protos)); + EXPECT_CALL(cb_, continueFilterChain(true)).WillOnce(InvokeWithoutArgs([&got_continue]() { + got_continue = true; + })); + while (!got_continue) { + file_event_callback_(Event::FileReadyType::Read); + } + EXPECT_EQ(1, cfg_->stats().http10_found_.value()); +} + +TEST_F(HttpInspectorTest, MultipleReadsHttp1IncompleteHeader) { + + init(); + + const absl::string_view data = "GE"; + bool end_stream = false; + + { + InSequence s; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)).WillOnce(InvokeWithoutArgs([]() { + return Api::SysCallSizeResult{ssize_t(-1), EAGAIN}; + })); + + for (size_t i = 1; i <= data.length(); i++) { + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke([&data, &end_stream, i](int, void* buffer, size_t length, + int) -> Api::SysCallSizeResult { + ASSERT(length >= data.size()); + memcpy(buffer, data.data(), data.size()); + if (i == data.length()) { + end_stream = true; + } + + return Api::SysCallSizeResult{ssize_t(i), 0}; + })); + } + } + + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); + EXPECT_EQ(0, cfg_->stats().http_not_found_.value()); + while (!end_stream) { + file_event_callback_(Event::FileReadyType::Read); + } +} +TEST_F(HttpInspectorTest, MultipleReadsHttp1BadProtocol) { + + init(); + + const absl::string_view data = "GET /index HTT\r"; + { + InSequence s; + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)).WillOnce(InvokeWithoutArgs([]() { + return Api::SysCallSizeResult{ssize_t(-1), EAGAIN}; + })); + + for (size_t i = 1; i <= data.length(); i++) { + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce( + Invoke([&data, i](int, void* buffer, size_t length, int) -> Api::SysCallSizeResult { + ASSERT(length >= data.size()); + memcpy(buffer, data.data(), data.size()); + return Api::SysCallSizeResult{ssize_t(i), 0}; + })); + } + } + + bool got_continue = false; + EXPECT_CALL(socket_, setRequestedApplicationProtocols(_)).Times(0); + EXPECT_CALL(cb_, continueFilterChain(true)).WillOnce(InvokeWithoutArgs([&got_continue]() { + got_continue = true; + })); + while (!got_continue) { + file_event_callback_(Event::FileReadyType::Read); + } + EXPECT_EQ(1, cfg_->stats().http_not_found_.value()); +} + } // namespace } // namespace HttpInspector } // namespace ListenerFilters From e97e42f046ff54fff213c9d800eade7bc42046e5 Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Tue, 23 Jul 2019 15:01:35 -0700 Subject: [PATCH 233/542] ci: fix 'six' Python library download URL (#7682) Signed-off-by: Fred Douglas --- bazel/repository_locations.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index dece1e411e59..a9db5ba2a0ac 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -230,7 +230,7 @@ REPOSITORY_LOCATIONS = dict( ), six_archive = dict( sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", - urls = ["https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55"], + urls = ["https://files.pythonhosted.org/packages/b3/b2/238e2590826bfdd113244a40d9d3eb26918bd798fc187e2360a8367068db/six-1.10.0.tar.gz"], ), io_opencensus_cpp = dict( sha256 = "d6d68704c419a9e892bd1f942e09509ebc5a318499a1abcf2c09734e5dc56e19", From 87a6cdc5eb23db40b8e3782b806bcb086f231f1a Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 24 Jul 2019 13:21:15 -0400 Subject: [PATCH 234/542] test: cleaning out unused test config (#7696) #7592 was made more annoying by the config creep which happened when the server.[json/yaml] files required custom config per feature under test. Clawing back that creep and removing unused files (assuming this passes the coverage build, my only concern) Risk Level: low (test only) Testing: n/a Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- test/config/integration/BUILD | 2 - test/config/integration/server.json | 526 ------------------------ test/config/integration/server.yaml | 259 ------------ test/config/integration/server_ads.yaml | 25 -- test/server/BUILD | 2 - test/server/server_test.cc | 5 +- 6 files changed, 1 insertion(+), 818 deletions(-) delete mode 100644 test/config/integration/server.json delete mode 100644 test/config/integration/server_ads.yaml diff --git a/test/config/integration/BUILD b/test/config/integration/BUILD index ec19f4f4350b..d28d2d8011fa 100644 --- a/test/config/integration/BUILD +++ b/test/config/integration/BUILD @@ -9,8 +9,6 @@ envoy_package() exports_files([ "server.yaml", - "server.json", - "server_ads.yaml", "server_unix_listener.yaml", ]) diff --git a/test/config/integration/server.json b/test/config/integration/server.json deleted file mode 100644 index 6dd4cc49fd0d..000000000000 --- a/test/config/integration/server.json +++ /dev/null @@ -1,526 +0,0 @@ -{ - "listeners": [ - { - "address": "tcp://{{ ip_loopback_address }}:0", - "filters": [ - { - "name": "http_connection_manager", - "config": { - "codec_type": "http1", - "drain_timeout_ms": 5000, - "access_log": [ - { - "path": "/dev/null", - "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%REQUEST_DURATION%\" \"%RESPONSE_DURATION%\"\n", - "filter" : { - "type": "logical_or", - "filters": [ - { - "type": "status_code", - "op": ">=", - "value": 500 - }, - { - "type": "duration", - "op": ">=", - "value": 1000000 - } - ] - } - }, - { - "path": "/dev/null" - }], - "stat_prefix": "router", - "route_config": - { - "virtual_hosts": [ - { - "name": "redirect", - "domains": [ "www.redirect.com" ], - "require_ssl": "all", - "routes": [ - { - "prefix": "/", - "cluster": "cluster_1" - } - ] - }, - { - "name": "integration", - "domains": [ "*" ], - "routes": [ - { - "prefix": "/", - "cluster": "cluster_1", - "runtime": { - "key": "some_key", - "default": 0 - } - }, - { - "prefix": "/test/long/url", - "cluster": "cluster_1", - "rate_limits": [ - { - "actions": [ - {"type": "destination_cluster"} - ] - } - ] - }, - { - "prefix": "/test/", - "cluster": "cluster_2" - }, - { - "prefix": "/websocket/test", - "prefix_rewrite": "/websocket", - "cluster": "cluster_1" - } - ] - } - ] - }, - "filters": [ - { "name": "health_check", - "config": { - "pass_through_mode": false, "endpoint": "/healthcheck" - } - }, - { "type": "decoder", "name": "rate_limit", - "config": { - "domain": "foo" - } - }, - { "type": "decoder", "name": "router", "config": {} } - ] - } - }] - }, - { - "address": "tcp://{{ ip_loopback_address }}:0", - "filters": [ - { - "name": "http_connection_manager", - "config": { - "codec_type": "http1", - "http1_settings": { - "allow_absolute_url": true - }, - "drain_timeout_ms": 5000, - "access_log": [ - { - "path": "/dev/null", - "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%REQUEST_DURATION%\" \"%RESPONSE_DURATION%\"\n", - "filter" : { - "type": "logical_or", - "filters": [ - { - "type": "status_code", - "op": ">=", - "value": 500 - }, - { - "type": "duration", - "op": ">=", - "value": 1000000 - } - ] - } - }, - { - "path": "/dev/null" - }], - "stat_prefix": "router", - "route_config": - { - "virtual_hosts": [ - { - "name": "redirect", - "domains": [ "www.redirect.com" ], - "require_ssl": "all", - "routes": [ - { - "prefix": "/", - "cluster": "cluster_1" - } - ] - }, - { - "name": "redirect", - "domains": [ "www.namewithport.com:1234" ], - "require_ssl": "all", - "routes": [ - { - "prefix": "/", - "cluster": "cluster_1" - } - ] - }, - { - "name": "integration", - "domains": [ "*" ], - "routes": [ - { - "prefix": "/", - "cluster": "cluster_1", - "runtime": { - "key": "some_key", - "default": 0 - } - }, - { - "prefix": "/test/long/url", - "cluster": "cluster_1", - "rate_limits": [ - { - "actions": [ - {"type": "destination_cluster"} - ] - } - ] - }, - { - "prefix": "/test/", - "cluster": "cluster_2" - }, - { - "prefix": "/websocket/test", - "prefix_rewrite": "/websocket", - "cluster": "cluster_1" - } - ] - } - ] - }, - "filters": [ - { "name": "health_check", - "config": { - "pass_through_mode": false, "endpoint": "/healthcheck" - } - }, - { "type": "decoder", "name": "rate_limit", - "config": { - "domain": "foo" - } - }, - { "type": "decoder", "name": "router", "config": {} } - ] - } - }] - }, - { - "address": "tcp://{{ ip_loopback_address }}:0", - "per_connection_buffer_limit_bytes": 1024, - "filters": [ - { - "name": "http_connection_manager", - "config": { - "codec_type": "http1", - "stat_prefix": "router", - "route_config": - { - "virtual_hosts": [ - { - "name": "integration", - "domains": [ "*" ], - "routes": [ - { - "prefix": "/test/long/url", - "cluster": "cluster_3" - } - ] - } - ] - }, - "filters": [ - { "type": "decoder", "name": "router", "config": {} } - ] - } - }] - }, - { - "address": "tcp://{{ ip_loopback_address }}:0", - "per_connection_buffer_limit_bytes": 1024, - "filters": [ - { - "name": "http_connection_manager", - "config": { - "codec_type": "http1", - "stat_prefix": "router", - "route_config": - { - "virtual_hosts": [ - { - "name": "integration", - "domains": [ "*" ], - "routes": [ - { - "prefix": "/dynamo/url", - "cluster": "cluster_3" - } - ] - } - ] - }, - "filters": [ - { "name": "http_dynamo_filter", "config": {} }, - { "type": "decoder", "name": "router", "config": {} } - ] - } - }] - }, - { - "address": "tcp://{{ ip_loopback_address }}:0", - "per_connection_buffer_limit_bytes": 1024, - "filters": [ - { - "name": "http_connection_manager", - "config": { - "codec_type": "http1", - "stat_prefix": "router", - "route_config": - { - "virtual_hosts": [ - { - "name": "integration", - "domains": [ "*" ], - "routes": [ - { - "prefix": "/test/long/url", - "cluster": "cluster_3" - } - ] - } - ] - }, - "filters": [ - { "type": "both", "name": "grpc_http1_bridge", "config": {} }, - { "type": "decoder", "name": "router", "config": {} } - ] - } - }] - }, - { - "address": "tcp://{{ ip_loopback_address }}:0", - "filters": [ - { - "name": "http_connection_manager", - "config": { - "codec_type": "http1", - "drain_timeout_ms": 5000, - "access_log": [ - { - "path": "/dev/null", - "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%REQUEST_DURATION%\" \"%RESPONSE_DURATION%\"\n", - "filter" : { - "type": "logical_or", - "filters": [ - { - "type": "status_code", - "op": ">=", - "value": 500 - }, - { - "type": "duration", - "op": ">=", - "value": 1000000 - } - ] - } - }, - { - "path": "/dev/null" - }], - "stat_prefix": "router", - "route_config": - { - "virtual_hosts": [ - { - "name": "redirect", - "domains": [ "www.redirect.com" ], - "require_ssl": "all", - "routes": [ - { - "prefix": "/", - "cluster": "cluster_1" - } - ] - }, - { - "name": "integration", - "domains": [ "*" ], - "routes": [ - { - "prefix": "/", - "cluster": "cluster_1", - "runtime": { - "key": "some_key", - "default": 0 - } - }, - { - "prefix": "/test/long/url", - "cluster": "cluster_1", - "rate_limits": [ - { - "actions": [ - {"type": "destination_cluster"} - ] - } - ] - }, - { - "prefix": "/test/", - "cluster": "cluster_2" - }, - { - "prefix": "/websocket/test", - "prefix_rewrite": "/websocket", - "cluster": "cluster_1" - } - ] - } - ] - }, - "filters": [ - { "name": "health_check", - "config": { - "pass_through_mode": false, "endpoint": "/healthcheck" - } - }, - { "type": "decoder", "name": "rate_limit", - "config": { - "domain": "foo" - } - }, - { "type": "decoder", "name": "buffer", - "config": { - "max_request_bytes": 5242880 - } - }, - { "type": "decoder", "name": "router", "config": {} } - ] - } - }] - }, - { - "address": "tcp://{{ ip_loopback_address }}:0", - "filters": [ - { - "name": "http_connection_manager", - "config": { - "codec_type": "http1", - "stat_prefix": "rds_dummy", - "rds": { - "cluster": "rds", - "route_config_name": "foo" - }, - "filters": [ - { "type": "decoder", "name": "router", "config": {} } - ] - } - }] - }, - { - "address": "tcp://{{ ip_loopback_address }}:0", - "filters": [ - { - "name": "redis_proxy", - "config": { - "cluster_name": "redis", - "stat_prefix": "redis", - "conn_pool": { - "op_timeout_ms": 400 - } - } - }] - }], - - "admin": { "access_log_path": "/dev/null", - "profile_path": "{{ test_tmpdir }}/envoy.prof", - "address": "tcp://{{ ip_loopback_address }}:0" }, - "flags_path": "/invalid_flags", - "statsd_udp_ip_address": "{{ ip_loopback_address }}:8125", - "statsd_tcp_cluster_name": "statsd", - - "lds": { - "api_type": "REST", - "cluster": "lds" - }, - - "runtime": { - "symlink_root": "{{ test_tmpdir }}/test/common/runtime/test_data/current", - "subdirectory": "envoy", - "override_subdirectory": "envoy_override" - }, - - "cluster_manager": { - "cds": { - "api_type": "REST", - "cluster": { - "name": "cds", - "connect_timeout_ms": 5000, - "type": "static", - "lb_type": "round_robin", - "hosts": [{"url": "tcp://{{ ip_loopback_address }}:4"}] - } - }, - "clusters": [ - { - "name": "rds", - "connect_timeout_ms": 5000, - "type": "static", - "lb_type": "round_robin", - "hosts": [{"url": "tcp://{{ ip_loopback_address }}:4"}] - }, - { - "name": "lds", - "connect_timeout_ms": 5000, - "type": "static", - "lb_type": "round_robin", - "hosts": [{"url": "tcp://{{ ip_loopback_address }}:4"}] - }, - { - "name": "cluster_1", - "connect_timeout_ms": 5000, - "type": "static", - "lb_type": "round_robin", - "hosts": [{"url": "tcp://{{ ip_loopback_address }}:{{ upstream_0 }}"}] - }, - { - "name": "cluster_2", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "dns_lookup_family": "{{ dns_lookup_family }}", - "hosts": [{"url": "tcp://localhost:{{ upstream_1 }}"}] - }, - { - "name": "cluster_3", - "per_connection_buffer_limit_bytes": 1024, - "connect_timeout_ms": 5000, - "type": "static", - "lb_type": "round_robin", - "hosts": [{"url": "tcp://{{ ip_loopback_address }}:{{ upstream_0 }}"}] - }, - { - "name": "statsd", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "dns_lookup_family": "{{ dns_lookup_family }}", - "hosts": [{"url": "tcp://localhost:4"}] - }, - { - "name": "redis", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "ring_hash", - "dns_lookup_family": "{{ dns_lookup_family }}", - "hosts": [{"url": "tcp://localhost:4"}], - "outlier_detection": {} - }] - } -} diff --git a/test/config/integration/server.yaml b/test/config/integration/server.yaml index 522f097d1d44..455a17bc0592 100644 --- a/test/config/integration/server.yaml +++ b/test/config/integration/server.yaml @@ -55,11 +55,6 @@ static_resources: access_log: - name: envoy.file_access_log config: - format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% - %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% - %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" - "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" - "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' path: /dev/null filter: or_filter: @@ -76,260 +71,6 @@ static_resources: value: default_value: 1000 runtime_key: access_log.access_error.duration - - address: - socket_address: - address: {{ ip_loopback_address }} - port_value: 0 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - filters: - - name: health_check - config: - endpoint: "/healthcheck" - pass_through_mode: false - - name: rate_limit - config: - domain: foo - - name: router - config: {} - access_log: - - name: envoy.file_access_log - config: - format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% - %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% - %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" - "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" - "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' - path: /dev/null - filter: - or_filter: - filters: - - status_code_filter: - comparison: - op: GE - value: - default_value: 500 - runtime_key: access_log.access_error.status - - duration_filter: - comparison: - op: GE - value: - default_value: 1000 - runtime_key: access_log.access_error.duration - drain_timeout_ms: 5000 - route_config: - virtual_hosts: - - routes: - - match: { prefix: "/" } - route: { cluster: cluster_1 } - domains: - - www.redirect.com - name: redirect - require_ssl: all - - routes: - - match: { prefix: "/" } - route: { cluster: cluster_1 } - domains: - - www.namewithport.com:1234 - name: redirect - require_ssl: all - - routes: - - route: - cluster: cluster_1 - runtime: - key: some_key - default: 0 - match: { prefix: "/" } - - route: - rate_limits: - - actions: - - destination_cluster: {} - cluster: cluster_1 - match: { prefix: "/test/long/url" } - - match: { prefix: "/test/" } - route: { cluster: cluster_2 } - - route: - cluster: cluster_1 - prefix_rewrite: "/websocket" - match: { prefix: "/websocket/test" } - domains: - - "*" - name: integration - codec_type: http1 - stat_prefix: router - http1_settings: - allow_absolute_url: true - - address: - socket_address: - address: {{ ip_loopback_address }} - port_value: 0 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - route_config: - virtual_hosts: - - routes: - - route: { cluster: cluster_3 } - match: { prefix: "/test/long/url" } - domains: - - "*" - name: integration - filters: - - name: router - config: {} - codec_type: http1 - stat_prefix: router - per_connection_buffer_limit_bytes: 1024 - - address: - socket_address: - address: {{ ip_loopback_address }} - port_value: 0 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - filters: - - name: http_dynamo_filter - config: {} - - name: router - config: {} - codec_type: http1 - stat_prefix: router - route_config: - virtual_hosts: - - routes: - - route: { cluster: cluster_3 } - match: { prefix: "/dynamo/url" } - domains: - - "*" - name: integration - per_connection_buffer_limit_bytes: 1024 - - address: - socket_address: - address: {{ ip_loopback_address }} - port_value: 0 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - route_config: - virtual_hosts: - - domains: - - "*" - name: integration - routes: - - match: { prefix: "/test/long/url" } - route: { cluster: cluster_3 } - filters: - - name: grpc_http1_bridge - config: {} - - name: router - config: {} - codec_type: http1 - stat_prefix: router - per_connection_buffer_limit_bytes: 1024 - - address: - socket_address: - address: {{ ip_loopback_address }} - port_value: 0 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - drain_timeout_ms: 5000 - route_config: - virtual_hosts: - - routes: - - route: { cluster: cluster_1 } - match: { prefix: "/" } - domains: - - www.redirect.com - name: redirect - require_ssl: all - - routes: - - route: - cluster: cluster_1 - runtime: - key: some_key - default: 0 - match: { prefix: "/" } - - match: { prefix: "/test/long/url" } - route: - rate_limits: - - actions: - - destination_cluster: {} - cluster: cluster_1 - - match: { prefix: "/test/" } - route: { cluster: cluster_2 } - - match: { prefix: "/websocket/test" } - route: - prefix_rewrite: "/websocket" - cluster: cluster_1 - domains: - - "*" - name: integration - codec_type: http1 - stat_prefix: router - filters: - - name: health_check - config: - endpoint: "/healthcheck" - pass_through_mode: false - - name: rate_limit - config: - domain: foo - - name: buffer - config: - max_request_bytes: 5242880 - - config: {} - name: router - access_log: - - name: envoy.file_access_log - config: - format: '[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% - %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% - %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" - "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" - "%REQUEST_DURATION%" "%RESPONSE_DURATION%"' - path: /dev/null - filter: - or_filter: - filters: - - status_code_filter: - comparison: - op: GE - value: - default_value: 500 - runtime_key: access_log.access_error.status - - duration_filter: - comparison: - op: GE - value: - default_value: 1000 - runtime_key: access_log.access_error.duration - - address: - socket_address: - address: {{ ip_loopback_address }} - port_value: 0 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - filters: - - name: router - config: {} - codec_type: http1 - stat_prefix: rds_dummy - rds: - config_source: - api_config_source: - api_type: REST - cluster_names: [ rds ] - refresh_delay: 60s - route_config_name: foo - address: socket_address: address: {{ ip_loopback_address }} diff --git a/test/config/integration/server_ads.yaml b/test/config/integration/server_ads.yaml deleted file mode 100644 index 3400df7e47e5..000000000000 --- a/test/config/integration/server_ads.yaml +++ /dev/null @@ -1,25 +0,0 @@ -dynamic_resources: - lds_config: {ads: {}} - cds_config: {ads: {}} - ads_config: - api_type: GRPC - grpc_services: - envoy_grpc: - cluster_name: ads_cluster -static_resources: - clusters: - - name: ads_cluster - connect_timeout: { seconds: 5 } - type: STATIC - hosts: - - socket_address: - address: {{ ntop_ip_loopback_address }} - port_value: {{ ads_upstream }} - lb_policy: ROUND_ROBIN - http2_protocol_options: {} -admin: - access_log_path: /dev/null - address: - socket_address: - address: {{ ntop_ip_loopback_address }} - port_value: 0 diff --git a/test/server/BUILD b/test/server/BUILD index 490eccbac91d..c0fb46d6a3f6 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -260,8 +260,6 @@ envoy_cc_test( ":runtime_test_data", ":stats_sink_bootstrap.yaml", ":zipkin_tracing.yaml", - "//test/config/integration:server.json", - "//test/config/integration:server_config_files", ], deps = [ "//source/common/common:version_lib", diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 16a063ea8e05..6ed2242cbc2e 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -172,10 +172,7 @@ class ServerInstanceImplTest : public testing::TestWithParam Date: Wed, 24 Jul 2019 23:59:56 +0530 Subject: [PATCH 235/542] router:add retry stat for completed requests (#7671) Signed-off-by: Rama Chavali --- docs/root/configuration/http_filters/router_filter.rst | 3 ++- docs/root/intro/version_history.rst | 3 ++- source/common/router/router.cc | 1 + source/common/router/router.h | 3 ++- test/common/router/router_test.cc | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/root/configuration/http_filters/router_filter.rst b/docs/root/configuration/http_filters/router_filter.rst index c235c9faecae..e676e267aa4c 100644 --- a/docs/root/configuration/http_filters/router_filter.rst +++ b/docs/root/configuration/http_filters/router_filter.rst @@ -351,7 +351,8 @@ owning HTTP connection manager. rq_redirect, Counter, Total requests that resulted in a redirect response rq_direct_response, Counter, Total requests that resulted in a direct response rq_total, Counter, Total routed requests - rq_reset_after_downstream_response_started, Counter, Total requests that were reset after downstream response had started. + rq_reset_after_downstream_response_started, Counter, Total requests that were reset after downstream response had started + rq_retry_skipped_request_not_complete, Counter, Total retries that were skipped as the request is not yet complete Virtual cluster statistics are output in the *vhost..vcluster..* namespace and include the following diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 8442328f715e..ccdf5f541bf7 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -6,8 +6,9 @@ Version history * admin: added ability to configure listener :ref:`socket options `. * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. -* listeners: added :ref:`HTTP inspector listener filter `. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. +* listeners: added :ref:`HTTP inspector listener filter `. +* router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. diff --git a/source/common/router/router.cc b/source/common/router/router.cc index c7ff569b06ca..5ad636479afc 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -1217,6 +1217,7 @@ bool Filter::setupRetry() { // this filter which will make this a non-issue. The implementation of supporting retry in cases // where the request is not complete is more complicated so we will start with this for now. if (!downstream_end_stream_) { + config_.stats_.rq_retry_skipped_request_not_complete_.inc(); return false; } pending_retries_++; diff --git a/source/common/router/router.h b/source/common/router/router.h index 65d6d9a9f824..fac9bbb07ee0 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -43,7 +43,8 @@ namespace Router { COUNTER(rq_redirect) \ COUNTER(rq_direct_response) \ COUNTER(rq_total) \ - COUNTER(rq_reset_after_downstream_response_started) + COUNTER(rq_reset_after_downstream_response_started) \ + COUNTER(rq_retry_skipped_request_not_complete) // clang-format on /** diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 09e769996a03..b4d39e204140 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -1934,6 +1934,7 @@ TEST_F(RouterTest, RetryRequestNotComplete) { putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); + EXPECT_EQ(1UL, stats_store_.counter("test.rq_retry_skipped_request_not_complete").value()); } // Two requests are sent (slow request + hedged retry) and then global timeout From a384ff11cd50e47045c3b91f2cddf974eb96f6cd Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Thu, 25 Jul 2019 00:03:01 +0100 Subject: [PATCH 236/542] stats: convert Dynamo stats to use StatName interface (#7634) * stats: convert Dynamo stats to use StatName interface #7634 Signed-off-by: Joshua Marantz --- source/common/memory/BUILD | 1 + source/common/memory/heap_shrinker.cc | 5 +- source/extensions/filters/http/dynamo/BUILD | 23 ++-- .../extensions/filters/http/dynamo/config.cc | 8 +- .../filters/http/dynamo/dynamo_filter.cc | 69 +++++------ .../filters/http/dynamo/dynamo_filter.h | 10 +- .../http/dynamo/dynamo_request_parser.cc | 18 +++ .../http/dynamo/dynamo_request_parser.h | 11 ++ .../filters/http/dynamo/dynamo_stats.cc | 109 ++++++++++++++++++ .../filters/http/dynamo/dynamo_stats.h | 73 ++++++++++++ .../filters/http/dynamo/dynamo_utility.cc | 27 ----- .../filters/http/dynamo/dynamo_utility.h | 32 ----- test/extensions/filters/http/dynamo/BUILD | 6 +- .../filters/http/dynamo/dynamo_filter_test.cc | 5 +- ...o_utility_test.cc => dynamo_stats_test.cc} | 27 +++-- tools/check_format.py | 6 +- tools/check_format_test_helper.py | 6 +- 17 files changed, 298 insertions(+), 138 deletions(-) create mode 100644 source/extensions/filters/http/dynamo/dynamo_stats.cc create mode 100644 source/extensions/filters/http/dynamo/dynamo_stats.h delete mode 100644 source/extensions/filters/http/dynamo/dynamo_utility.cc delete mode 100644 source/extensions/filters/http/dynamo/dynamo_utility.h rename test/extensions/filters/http/dynamo/{dynamo_utility_test.cc => dynamo_stats_test.cc} (63%) diff --git a/source/common/memory/BUILD b/source/common/memory/BUILD index 2b22e25f9f28..83a46fa4fcc2 100644 --- a/source/common/memory/BUILD +++ b/source/common/memory/BUILD @@ -34,5 +34,6 @@ envoy_cc_library( "//include/envoy/event:dispatcher_interface", "//include/envoy/server:overload_manager_interface", "//include/envoy/stats:stats_interface", + "//source/common/stats:symbol_table_lib", ], ) diff --git a/source/common/memory/heap_shrinker.cc b/source/common/memory/heap_shrinker.cc index 2582cfe76c84..da0413bbfce3 100644 --- a/source/common/memory/heap_shrinker.cc +++ b/source/common/memory/heap_shrinker.cc @@ -1,6 +1,7 @@ #include "common/memory/heap_shrinker.h" #include "common/memory/utils.h" +#include "common/stats/symbol_table_impl.h" #include "absl/strings/str_cat.h" @@ -18,7 +19,9 @@ HeapShrinker::HeapShrinker(Event::Dispatcher& dispatcher, Server::OverloadManage [this](Server::OverloadActionState state) { active_ = (state == Server::OverloadActionState::Active); })) { - shrink_counter_ = &stats.counter(absl::StrCat("overload.", action_name, ".shrink_count")); + Envoy::Stats::StatNameManagedStorage stat_name( + absl::StrCat("overload.", action_name, ".shrink_count"), stats.symbolTable()); + shrink_counter_ = &stats.counterFromStatName(stat_name.statName()); timer_ = dispatcher.createTimer([this] { shrinkHeap(); timer_->enableTimer(kTimerInterval); diff --git a/source/extensions/filters/http/dynamo/BUILD b/source/extensions/filters/http/dynamo/BUILD index e1ba97728475..048be9323395 100644 --- a/source/extensions/filters/http/dynamo/BUILD +++ b/source/extensions/filters/http/dynamo/BUILD @@ -17,7 +17,7 @@ envoy_cc_library( hdrs = ["dynamo_filter.h"], deps = [ ":dynamo_request_parser_lib", - ":dynamo_utility_lib", + ":dynamo_stats_lib", "//include/envoy/http:filter_interface", "//include/envoy/runtime:runtime_interface", "//source/common/buffer:buffer_lib", @@ -37,16 +37,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "dynamo_utility_lib", - srcs = ["dynamo_utility.cc"], - hdrs = ["dynamo_utility.h"], - deps = [ - "//include/envoy/stats:stats_interface", - "//source/common/stats:stats_lib", - ], -) - envoy_cc_library( name = "config", srcs = ["config.cc"], @@ -59,3 +49,14 @@ envoy_cc_library( "//source/extensions/filters/http/common:empty_http_filter_config_lib", ], ) + +envoy_cc_library( + name = "dynamo_stats_lib", + srcs = ["dynamo_stats.cc"], + hdrs = ["dynamo_stats.h"], + deps = [ + ":dynamo_request_parser_lib", + "//include/envoy/stats:stats_interface", + "//source/common/stats:symbol_table_lib", + ], +) diff --git a/source/extensions/filters/http/dynamo/config.cc b/source/extensions/filters/http/dynamo/config.cc index 7355e46760f6..77dde48be63c 100644 --- a/source/extensions/filters/http/dynamo/config.cc +++ b/source/extensions/filters/http/dynamo/config.cc @@ -5,6 +5,7 @@ #include "envoy/registry/registry.h" #include "extensions/filters/http/dynamo/dynamo_filter.h" +#include "extensions/filters/http/dynamo/dynamo_stats.h" namespace Envoy { namespace Extensions { @@ -14,9 +15,10 @@ namespace Dynamo { Http::FilterFactoryCb DynamoFilterConfig::createFilter(const std::string& stat_prefix, Server::Configuration::FactoryContext& context) { - return [&context, stat_prefix](Http::FilterChainFactoryCallbacks& callbacks) -> void { - callbacks.addStreamFilter(Http::StreamFilterSharedPtr{new Dynamo::DynamoFilter( - context.runtime(), stat_prefix, context.scope(), context.dispatcher().timeSource())}); + auto stats = std::make_shared(context.scope(), stat_prefix); + return [&context, stat_prefix, stats](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter(std::make_shared( + context.runtime(), stats, context.dispatcher().timeSource())); }; } diff --git a/source/extensions/filters/http/dynamo/dynamo_filter.cc b/source/extensions/filters/http/dynamo/dynamo_filter.cc index cb7ee4c35702..c9072aaf5b1a 100644 --- a/source/extensions/filters/http/dynamo/dynamo_filter.cc +++ b/source/extensions/filters/http/dynamo/dynamo_filter.cc @@ -15,7 +15,7 @@ #include "common/json/json_loader.h" #include "extensions/filters/http/dynamo/dynamo_request_parser.h" -#include "extensions/filters/http/dynamo/dynamo_utility.h" +#include "extensions/filters/http/dynamo/dynamo_stats.h" namespace Envoy { namespace Extensions { @@ -62,7 +62,7 @@ void DynamoFilter::onDecodeComplete(const Buffer::Instance& data) { table_descriptor_ = RequestParser::parseTable(operation_, *json_body); } catch (const Json::Exception& jsonEx) { // Body parsing failed. This should not happen, just put a stat for that. - scope_.counter(fmt::format("{}invalid_req_body", stat_prefix_)).inc(); + stats_->counter({stats_->invalid_req_body_}).inc(); } } } @@ -89,7 +89,7 @@ void DynamoFilter::onEncodeComplete(const Buffer::Instance& data) { } } catch (const Json::Exception&) { // Body parsing failed. This should not happen, just put a stat for that. - scope_.counter(fmt::format("{}invalid_resp_body", stat_prefix_)).inc(); + stats_->counter({stats_->invalid_resp_body_}).inc(); } } } @@ -158,15 +158,15 @@ void DynamoFilter::chargeBasicStats(uint64_t status) { if (!operation_.empty()) { chargeStatsPerEntity(operation_, "operation", status); } else { - scope_.counter(fmt::format("{}operation_missing", stat_prefix_)).inc(); + stats_->counter({stats_->operation_missing_}).inc(); } if (!table_descriptor_.table_name.empty()) { chargeStatsPerEntity(table_descriptor_.table_name, "table", status); } else if (table_descriptor_.is_single_table) { - scope_.counter(fmt::format("{}table_missing", stat_prefix_)).inc(); + stats_->counter({stats_->table_missing_}).inc(); } else { - scope_.counter(fmt::format("{}multiple_tables", stat_prefix_)).inc(); + stats_->counter({stats_->multiple_tables_}).inc(); } } @@ -175,29 +175,23 @@ void DynamoFilter::chargeStatsPerEntity(const std::string& entity, const std::st std::chrono::milliseconds latency = std::chrono::duration_cast( time_source_.monotonicTime() - start_decode_); - std::string group_string = - Http::CodeUtility::groupStringForResponseCode(static_cast(status)); + size_t group_index = DynamoStats::groupIndex(status); + const Stats::StatName entity_type_name = stats_->getStatName(entity_type); + const Stats::StatName entity_name = stats_->getStatName(entity); + const Stats::StatName total_name = + stats_->getStatName(absl::StrCat("upstream_rq_total_", status)); + const Stats::StatName time_name = stats_->getStatName(absl::StrCat("upstream_rq_time_", status)); - scope_.counter(fmt::format("{}{}.{}.upstream_rq_total", stat_prefix_, entity_type, entity)).inc(); - scope_ - .counter(fmt::format("{}{}.{}.upstream_rq_total_{}", stat_prefix_, entity_type, entity, - group_string)) - .inc(); - scope_ - .counter(fmt::format("{}{}.{}.upstream_rq_total_{}", stat_prefix_, entity_type, entity, - std::to_string(status))) - .inc(); + stats_->counter({entity_type_name, entity_name, stats_->upstream_rq_total_}).inc(); + const Stats::StatName total_group = stats_->upstream_rq_total_groups_[group_index]; + stats_->counter({entity_type_name, entity_name, total_group}).inc(); + stats_->counter({entity_type_name, entity_name, total_name}).inc(); - scope_.histogram(fmt::format("{}{}.{}.upstream_rq_time", stat_prefix_, entity_type, entity)) - .recordValue(latency.count()); - scope_ - .histogram(fmt::format("{}{}.{}.upstream_rq_time_{}", stat_prefix_, entity_type, entity, - group_string)) - .recordValue(latency.count()); - scope_ - .histogram(fmt::format("{}{}.{}.upstream_rq_time_{}", stat_prefix_, entity_type, entity, - std::to_string(status))) + stats_->histogram({entity_type_name, entity_name, stats_->upstream_rq_time_}) .recordValue(latency.count()); + const Stats::StatName time_group = stats_->upstream_rq_time_groups_[group_index]; + stats_->histogram({entity_type_name, entity_name, time_group}).recordValue(latency.count()); + stats_->histogram({entity_type_name, entity_name, time_name}).recordValue(latency.count()); } void DynamoFilter::chargeUnProcessedKeysStats(const Json::Object& json_body) { @@ -205,9 +199,9 @@ void DynamoFilter::chargeUnProcessedKeysStats(const Json::Object& json_body) { // complete apart of the batch operation. Only the table names will be logged for errors. std::vector unprocessed_tables = RequestParser::parseBatchUnProcessedKeys(json_body); for (const std::string& unprocessed_table : unprocessed_tables) { - scope_ - .counter( - fmt::format("{}error.{}.BatchFailureUnprocessedKeys", stat_prefix_, unprocessed_table)) + stats_ + ->counter({stats_->error_, stats_->getStatName(unprocessed_table), + stats_->batch_failure_unprocessed_keys_}) .inc(); } } @@ -217,15 +211,15 @@ void DynamoFilter::chargeFailureSpecificStats(const Json::Object& json_body) { if (!error_type.empty()) { if (table_descriptor_.table_name.empty()) { - scope_.counter(fmt::format("{}error.no_table.{}", stat_prefix_, error_type)).inc(); + stats_->counter({stats_->error_, stats_->no_table_, stats_->getStatName(error_type)}).inc(); } else { - scope_ - .counter( - fmt::format("{}error.{}.{}", stat_prefix_, table_descriptor_.table_name, error_type)) + stats_ + ->counter({stats_->error_, stats_->getStatName(table_descriptor_.table_name), + stats_->getStatName(error_type)}) .inc(); } } else { - scope_.counter(fmt::format("{}empty_response_body", stat_prefix_)).inc(); + stats_->counter({stats_->empty_response_body_}).inc(); } } @@ -237,9 +231,10 @@ void DynamoFilter::chargeTablePartitionIdStats(const Json::Object& json_body) { std::vector partitions = RequestParser::parsePartitions(json_body); for (const RequestParser::PartitionDescriptor& partition : partitions) { - std::string scope_string = Utility::buildPartitionStatString( - stat_prefix_, table_descriptor_.table_name, operation_, partition.partition_id_); - scope_.counter(scope_string).add(partition.capacity_); + stats_ + ->buildPartitionStatCounter(table_descriptor_.table_name, operation_, + partition.partition_id_) + .add(partition.capacity_); } } diff --git a/source/extensions/filters/http/dynamo/dynamo_filter.h b/source/extensions/filters/http/dynamo/dynamo_filter.h index 3de6e378db1f..8702444a1150 100644 --- a/source/extensions/filters/http/dynamo/dynamo_filter.h +++ b/source/extensions/filters/http/dynamo/dynamo_filter.h @@ -10,6 +10,7 @@ #include "common/json/json_loader.h" #include "extensions/filters/http/dynamo/dynamo_request_parser.h" +#include "extensions/filters/http/dynamo/dynamo_stats.h" namespace Envoy { namespace Extensions { @@ -24,10 +25,8 @@ namespace Dynamo { */ class DynamoFilter : public Http::StreamFilter { public: - DynamoFilter(Runtime::Loader& runtime, const std::string& stat_prefix, Stats::Scope& scope, - TimeSource& time_system) - : runtime_(runtime), stat_prefix_(stat_prefix + "dynamodb."), scope_(scope), - time_source_(time_system) { + DynamoFilter(Runtime::Loader& runtime, const DynamoStatsSharedPtr& stats, TimeSource& time_source) + : runtime_(runtime), stats_(stats), time_source_(time_source) { enabled_ = runtime_.snapshot().featureEnabled("dynamodb.filter_enabled", 100); } @@ -68,8 +67,7 @@ class DynamoFilter : public Http::StreamFilter { void chargeTablePartitionIdStats(const Json::Object& json_body); Runtime::Loader& runtime_; - std::string stat_prefix_; - Stats::Scope& scope_; + const DynamoStatsSharedPtr stats_; bool enabled_{}; std::string operation_{}; diff --git a/source/extensions/filters/http/dynamo/dynamo_request_parser.cc b/source/extensions/filters/http/dynamo/dynamo_request_parser.cc index 3ec282c78428..357357f067eb 100644 --- a/source/extensions/filters/http/dynamo/dynamo_request_parser.cc +++ b/source/extensions/filters/http/dynamo/dynamo_request_parser.cc @@ -186,6 +186,24 @@ RequestParser::parsePartitions(const Json::Object& json_data) { return partition_descriptors; } +void RequestParser::forEachStatString(const StringFn& fn) { + for (const std::string& str : SINGLE_TABLE_OPERATIONS) { + fn(str); + } + for (const std::string& str : SUPPORTED_ERROR_TYPES) { + fn(str); + } + for (const std::string& str : BATCH_OPERATIONS) { + fn(str); + } + for (const std::string& str : TRANSACT_OPERATIONS) { + fn(str); + } + for (const std::string& str : TRANSACT_ITEM_OPERATIONS) { + fn(str); + } +} + } // namespace Dynamo } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/dynamo/dynamo_request_parser.h b/source/extensions/filters/http/dynamo/dynamo_request_parser.h index c7bd669b0bec..8c63b32fd641 100644 --- a/source/extensions/filters/http/dynamo/dynamo_request_parser.h +++ b/source/extensions/filters/http/dynamo/dynamo_request_parser.h @@ -97,6 +97,17 @@ class RequestParser { */ static std::vector parsePartitions(const Json::Object& json_data); + using StringFn = std::function; + + /** + * Calls a function for every string that is likely to be included as a token + * in a stat. This is not functionally necessary, but can reduce potentially + * contented access to create entries in the symbol table in the hot path. + * + * @param fn the function to call for every potential stat name. + */ + static void forEachStatString(const StringFn& fn); + private: static const Http::LowerCaseString X_AMZ_TARGET; static const std::vector SINGLE_TABLE_OPERATIONS; diff --git a/source/extensions/filters/http/dynamo/dynamo_stats.cc b/source/extensions/filters/http/dynamo/dynamo_stats.cc new file mode 100644 index 000000000000..51e12d96d452 --- /dev/null +++ b/source/extensions/filters/http/dynamo/dynamo_stats.cc @@ -0,0 +1,109 @@ +#include "extensions/filters/http/dynamo/dynamo_stats.h" + +#include +#include +#include + +#include "envoy/stats/scope.h" + +#include "common/stats/symbol_table_impl.h" + +#include "extensions/filters/http/dynamo/dynamo_request_parser.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Dynamo { + +DynamoStats::DynamoStats(Stats::Scope& scope, const std::string& prefix) + : scope_(scope), pool_(scope.symbolTable()), prefix_(pool_.add(prefix + "dynamodb")), + batch_failure_unprocessed_keys_(pool_.add("BatchFailureUnprocessedKeys")), + capacity_(pool_.add("capacity")), empty_response_body_(pool_.add("empty_response_body")), + error_(pool_.add("error")), invalid_req_body_(pool_.add("invalid_req_body")), + invalid_resp_body_(pool_.add("invalid_resp_body")), + multiple_tables_(pool_.add("multiple_tables")), no_table_(pool_.add("no_table")), + operation_missing_(pool_.add("operation_missing")), table_(pool_.add("table")), + table_missing_(pool_.add("table_missing")), upstream_rq_time_(pool_.add("upstream_rq_time")), + upstream_rq_total_(pool_.add("upstream_rq_total")) { + upstream_rq_total_groups_[0] = pool_.add("upstream_rq_total_unknown"); + upstream_rq_time_groups_[0] = pool_.add("upstream_rq_time_unknown"); + for (size_t i = 1; i < DynamoStats::NumGroupEntries; ++i) { + upstream_rq_total_groups_[i] = pool_.add(fmt::format("upstream_rq_total_{}xx", i)); + upstream_rq_time_groups_[i] = pool_.add(fmt::format("upstream_rq_time_{}xx", i)); + } + RequestParser::forEachStatString([this](const std::string& str) { + // Thread annotation does not realize this function is only called from the + // constructor, so we need to lock the mutex. It's easier to just do that + // and it's no real penalty. + absl::MutexLock lock(&mutex_); + builtin_stat_names_[str] = pool_.add(str); + }); + builtin_stat_names_[""] = Stats::StatName(); +} + +Stats::SymbolTable::StoragePtr DynamoStats::addPrefix(const std::vector& names) { + std::vector names_with_prefix{prefix_}; + names_with_prefix.reserve(names.end() - names.begin()); + names_with_prefix.insert(names_with_prefix.end(), names.begin(), names.end()); + return scope_.symbolTable().join(names_with_prefix); +} + +Stats::Counter& DynamoStats::counter(const std::vector& names) { + const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(names); + return scope_.counterFromStatName(Stats::StatName(stat_name_storage.get())); +} + +Stats::Histogram& DynamoStats::histogram(const std::vector& names) { + const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(names); + return scope_.histogramFromStatName(Stats::StatName(stat_name_storage.get())); +} + +Stats::Counter& DynamoStats::buildPartitionStatCounter(const std::string& table_name, + const std::string& operation, + const std::string& partition_id) { + // Use the last 7 characters of the partition id. + absl::string_view id_last_7 = absl::string_view(partition_id).substr(partition_id.size() - 7); + const Stats::SymbolTable::StoragePtr stat_name_storage = + addPrefix({table_, getStatName(table_name), capacity_, getStatName(operation), + getStatName(absl::StrCat("__partition_id=", id_last_7))}); + return scope_.counterFromStatName(Stats::StatName(stat_name_storage.get())); +} + +size_t DynamoStats::groupIndex(uint64_t status) { + size_t index = status / 100; + if (index >= NumGroupEntries) { + index = 0; // status-code 600 or higher is unknown. + } + return index; +} + +Stats::StatName DynamoStats::getStatName(const std::string& token) { + // The Dynamo system has a few well-known tokens that we have stored during + // construction, and we can access StatNames for those without a lock. Note + // that we only mutate builtin_stat_names_ during construction. + auto iter = builtin_stat_names_.find(token); + if (iter != builtin_stat_names_.end()) { + return iter->second; + } + + // However, some of the tokens come from json data received during requests, + // so we need to store these in a mutex-protected map. Once we hold the mutex, + // we can check dynamic_stat_names_. If it's missing we'll have to add it + // to the symbol table, whose mutex is more likely to be contended, and then + // store it in dynamic_stat_names. + // + // TODO(jmarantz): Potential perf issue here with contention, both on this + // mutex and also the SymbolTable mutex which must be taken during + // StatNamePool::add(). + absl::MutexLock lock(&mutex_); + Stats::StatName& stat_name = dynamic_stat_names_[token]; + if (stat_name.empty()) { // Note that builtin_stat_names_ already has one for "". + stat_name = pool_.add(token); + } + return stat_name; +} + +} // namespace Dynamo +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/dynamo/dynamo_stats.h b/source/extensions/filters/http/dynamo/dynamo_stats.h new file mode 100644 index 000000000000..57671514e581 --- /dev/null +++ b/source/extensions/filters/http/dynamo/dynamo_stats.h @@ -0,0 +1,73 @@ +#pragma once + +#include +#include + +#include "envoy/stats/scope.h" + +#include "common/stats/symbol_table_impl.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace Dynamo { + +class DynamoStats { +public: + DynamoStats(Stats::Scope& scope, const std::string& prefix); + + Stats::Counter& counter(const std::vector& names); + Stats::Histogram& histogram(const std::vector& names); + + /** + * Creates the partition id stats string. The stats format is + * "table..capacity..__partition_id=". + * Partition ids and dynamodb table names can be long. To satisfy the string + * length, we truncate, taking only the last 7 characters of the partition id. + */ + Stats::Counter& buildPartitionStatCounter(const std::string& table_name, + const std::string& operation, + const std::string& partition_id); + + static size_t groupIndex(uint64_t status); + + Stats::StatName getStatName(const std::string& str); + +private: + Stats::SymbolTable::StoragePtr addPrefix(const std::vector& names); + + Stats::Scope& scope_; + Stats::StatNamePool pool_ GUARDED_BY(mutex_); + const Stats::StatName prefix_; + absl::Mutex mutex_; + using StringStatNameMap = absl::flat_hash_map; + StringStatNameMap builtin_stat_names_; + StringStatNameMap dynamic_stat_names_ GUARDED_BY(mutex_); + +public: + const Stats::StatName batch_failure_unprocessed_keys_; + const Stats::StatName capacity_; + const Stats::StatName empty_response_body_; + const Stats::StatName error_; + const Stats::StatName invalid_req_body_; + const Stats::StatName invalid_resp_body_; + const Stats::StatName multiple_tables_; + const Stats::StatName no_table_; + const Stats::StatName operation_missing_; + const Stats::StatName table_; + const Stats::StatName table_missing_; + const Stats::StatName upstream_rq_time_; + const Stats::StatName upstream_rq_total_; + const Stats::StatName upstream_rq_unknown_; + + // Keep group codes for HTTP status codes through the 500s. + static constexpr size_t NumGroupEntries = 6; + Stats::StatName upstream_rq_total_groups_[NumGroupEntries]; + Stats::StatName upstream_rq_time_groups_[NumGroupEntries]; +}; +using DynamoStatsSharedPtr = std::shared_ptr; + +} // namespace Dynamo +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/dynamo/dynamo_utility.cc b/source/extensions/filters/http/dynamo/dynamo_utility.cc deleted file mode 100644 index a71408439fc3..000000000000 --- a/source/extensions/filters/http/dynamo/dynamo_utility.cc +++ /dev/null @@ -1,27 +0,0 @@ -#include "extensions/filters/http/dynamo/dynamo_utility.h" - -#include - -#include "common/common/fmt.h" - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace Dynamo { - -std::string Utility::buildPartitionStatString(const std::string& stat_prefix, - const std::string& table_name, - const std::string& operation, - const std::string& partition_id) { - // Use the last 7 characters of the partition id. - std::string stats_partition_postfix = - fmt::format(".capacity.{}.__partition_id={}", operation, - partition_id.substr(partition_id.size() - 7, partition_id.size())); - std::string stats_table_prefix = fmt::format("{}table.{}", stat_prefix, table_name); - return fmt::format("{}{}", stats_table_prefix, stats_partition_postfix); -} - -} // namespace Dynamo -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/filters/http/dynamo/dynamo_utility.h b/source/extensions/filters/http/dynamo/dynamo_utility.h deleted file mode 100644 index 87b4bd0bc119..000000000000 --- a/source/extensions/filters/http/dynamo/dynamo_utility.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include - -namespace Envoy { -namespace Extensions { -namespace HttpFilters { -namespace Dynamo { - -class Utility { -public: - /** - * Creates the partition id stats string. - * The stats format is - * "table..capacity..__partition_id=". - * Partition ids and dynamodb table names can be long. To satisfy the string length, - * we truncate in two ways: - * 1. We only take the last 7 characters of the partition id. - * 2. If the stats string with is longer than the stats MAX_NAME_SIZE, we will - * truncate the table name to - * fit the size requirements. - */ - static std::string buildPartitionStatString(const std::string& stat_prefix, - const std::string& table_name, - const std::string& operation, - const std::string& partition_id); -}; - -} // namespace Dynamo -} // namespace HttpFilters -} // namespace Extensions -} // namespace Envoy diff --git a/test/extensions/filters/http/dynamo/BUILD b/test/extensions/filters/http/dynamo/BUILD index 01797af74456..9d87a0531b40 100644 --- a/test/extensions/filters/http/dynamo/BUILD +++ b/test/extensions/filters/http/dynamo/BUILD @@ -40,12 +40,12 @@ envoy_extension_cc_test( ) envoy_extension_cc_test( - name = "dynamo_utility_test", - srcs = ["dynamo_utility_test.cc"], + name = "dynamo_stats_test", + srcs = ["dynamo_stats_test.cc"], extension_name = "envoy.filters.http.dynamo", deps = [ "//source/common/stats:stats_lib", - "//source/extensions/filters/http/dynamo:dynamo_utility_lib", + "//source/extensions/filters/http/dynamo:dynamo_stats_lib", "//test/mocks/stats:stats_mocks", ], ) diff --git a/test/extensions/filters/http/dynamo/dynamo_filter_test.cc b/test/extensions/filters/http/dynamo/dynamo_filter_test.cc index a21e4734963f..eb93be0c670c 100644 --- a/test/extensions/filters/http/dynamo/dynamo_filter_test.cc +++ b/test/extensions/filters/http/dynamo/dynamo_filter_test.cc @@ -34,7 +34,8 @@ class DynamoFilterTest : public testing::Test { .WillByDefault(Return(enabled)); EXPECT_CALL(loader_.snapshot_, featureEnabled("dynamodb.filter_enabled", 100)); - filter_ = std::make_unique(loader_, stat_prefix_, stats_, + auto stats = std::make_shared(stats_, "prefix."); + filter_ = std::make_unique(loader_, stats, decoder_callbacks_.dispatcher().timeSource()); filter_->setDecoderFilterCallbacks(decoder_callbacks_); @@ -43,10 +44,10 @@ class DynamoFilterTest : public testing::Test { ~DynamoFilterTest() override { filter_->onDestroy(); } + NiceMock stats_; std::unique_ptr filter_; NiceMock loader_; std::string stat_prefix_{"prefix."}; - NiceMock stats_; NiceMock decoder_callbacks_; NiceMock encoder_callbacks_; }; diff --git a/test/extensions/filters/http/dynamo/dynamo_utility_test.cc b/test/extensions/filters/http/dynamo/dynamo_stats_test.cc similarity index 63% rename from test/extensions/filters/http/dynamo/dynamo_utility_test.cc rename to test/extensions/filters/http/dynamo/dynamo_stats_test.cc index b00fd773e036..0458c5dc3879 100644 --- a/test/extensions/filters/http/dynamo/dynamo_utility_test.cc +++ b/test/extensions/filters/http/dynamo/dynamo_stats_test.cc @@ -1,6 +1,6 @@ #include -#include "extensions/filters/http/dynamo/dynamo_utility.h" +#include "extensions/filters/http/dynamo/dynamo_stats.h" #include "test/mocks/stats/mocks.h" @@ -17,40 +17,49 @@ namespace HttpFilters { namespace Dynamo { namespace { -TEST(DynamoUtility, PartitionIdStatString) { +TEST(DynamoStats, PartitionIdStatString) { + Stats::IsolatedStoreImpl store; + auto build_partition_string = + [&store](const std::string& stat_prefix, const std::string& table_name, + const std::string& operation, const std::string& partition_id) -> std::string { + DynamoStats stats(store, stat_prefix); + Stats::Counter& counter = stats.buildPartitionStatCounter(table_name, operation, partition_id); + return counter.name(); + }; + { - std::string stat_prefix = "stat.prefix."; + std::string stats_prefix = "prefix."; std::string table_name = "locations"; std::string operation = "GetItem"; std::string partition_id = "6235c781-1d0d-47a3-a4ea-eec04c5883ca"; std::string partition_stat_string = - Utility::buildPartitionStatString(stat_prefix, table_name, operation, partition_id); + build_partition_string(stats_prefix, table_name, operation, partition_id); std::string expected_stat_string = - "stat.prefix.table.locations.capacity.GetItem.__partition_id=c5883ca"; + "prefix.dynamodb.table.locations.capacity.GetItem.__partition_id=c5883ca"; EXPECT_EQ(expected_stat_string, partition_stat_string); } { - std::string stat_prefix = "http.egress_dynamodb_iad.dynamodb."; + std::string stats_prefix = "http.egress_dynamodb_iad."; std::string table_name = "locations-sandbox-partition-test-iad-mytest-really-long-name"; std::string operation = "GetItem"; std::string partition_id = "6235c781-1d0d-47a3-a4ea-eec04c5883ca"; std::string partition_stat_string = - Utility::buildPartitionStatString(stat_prefix, table_name, operation, partition_id); + build_partition_string(stats_prefix, table_name, operation, partition_id); std::string expected_stat_string = "http.egress_dynamodb_iad.dynamodb.table.locations-sandbox-partition-test-iad-mytest-" "really-long-name.capacity.GetItem.__partition_id=c5883ca"; EXPECT_EQ(expected_stat_string, partition_stat_string); } { - std::string stat_prefix = "http.egress_dynamodb_iad.dynamodb."; + std::string stats_prefix = "http.egress_dynamodb_iad."; std::string table_name = "locations-sandbox-partition-test-iad-mytest-rea"; std::string operation = "GetItem"; std::string partition_id = "6235c781-1d0d-47a3-a4ea-eec04c5883ca"; std::string partition_stat_string = - Utility::buildPartitionStatString(stat_prefix, table_name, operation, partition_id); + build_partition_string(stats_prefix, table_name, operation, partition_id); std::string expected_stat_string = "http.egress_dynamodb_iad.dynamodb.table.locations-sandbox-" "partition-test-iad-mytest-rea.capacity.GetItem.__partition_" "id=c5883ca"; diff --git a/tools/check_format.py b/tools/check_format.py index 8a9bece5c4c0..8f5759c5aa96 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -48,9 +48,7 @@ # https://github.com/envoyproxy/envoy/pull/7573 and others. # # TODO(#4196): Eliminate this list completely and then merge #4980. -STAT_FROM_STRING_WHITELIST = ("./source/common/memory/heap_shrinker.cc", - "./source/extensions/filters/http/dynamo/dynamo_filter.cc", - "./source/extensions/filters/http/ext_authz/ext_authz.cc", +STAT_FROM_STRING_WHITELIST = ("./source/extensions/filters/http/ext_authz/ext_authz.cc", "./source/extensions/filters/http/fault/fault_filter.cc", "./source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc", "./source/extensions/filters/network/mongo_proxy/proxy.cc", @@ -584,7 +582,7 @@ def checkSourceLine(line, file_path, reportError): if isInSubdir(file_path, 'source') and file_path.endswith('.cc') and \ not whitelistedForStatFromString(file_path) and \ ('.counter(' in line or '.gauge(' in line or '.histogram(' in line): - reportError("Don't lookup stats by name at runtime; used StatName saved during construction") + reportError("Don't lookup stats by name at runtime; use StatName saved during construction") def checkBuildLine(line, file_path, reportError): diff --git a/tools/check_format_test_helper.py b/tools/check_format_test_helper.py index 2641112ea722..7309a4bcaff1 100755 --- a/tools/check_format_test_helper.py +++ b/tools/check_format_test_helper.py @@ -210,13 +210,13 @@ def checkFileExpectingOK(filename): "check_format.py") errors += checkUnfixableError( "counter_from_string.cc", - "Don't lookup stats by name at runtime; used StatName saved during construction") + "Don't lookup stats by name at runtime; use StatName saved during construction") errors += checkUnfixableError( "gauge_from_string.cc", - "Don't lookup stats by name at runtime; used StatName saved during construction") + "Don't lookup stats by name at runtime; use StatName saved during construction") errors += checkUnfixableError( "histogram_from_string.cc", - "Don't lookup stats by name at runtime; used StatName saved during construction") + "Don't lookup stats by name at runtime; use StatName saved during construction") errors += fixFileExpectingFailure( "api/missing_package.proto", From ae6bc0185c235a68d5c3845fd54daa270f210685 Mon Sep 17 00:00:00 2001 From: Jianfei Hu Date: Wed, 24 Jul 2019 16:10:48 -0700 Subject: [PATCH 237/542] Config Dump for Secret Discovery Service. (#7365) Description: SDSConfig Dump Risk Level: Low Testing: Unit Test Docs Changes: N/A Release Notes: Added Fixes #7111 Signed-off-by: Jianfei Hu --- api/envoy/admin/v2alpha/BUILD | 1 + api/envoy/admin/v2alpha/config_dump.proto | 48 ++- docs/root/intro/version_history.rst | 1 + source/common/secret/BUILD | 1 + source/common/secret/sds_api.cc | 13 +- source/common/secret/sds_api.h | 25 +- source/common/secret/secret_manager_impl.cc | 99 +++++ source/common/secret/secret_manager_impl.h | 16 + source/server/config_validation/server.cc | 2 +- source/server/server.cc | 10 +- test/common/secret/BUILD | 1 + test/common/secret/sds_api_test.cc | 42 +- .../common/secret/secret_manager_impl_test.cc | 378 +++++++++++++++++- test/config/utility.cc | 10 + .../tls/context_impl_test.cc | 8 + .../transport_sockets/tls/ssl_socket_test.cc | 4 + test/integration/integration_admin_test.cc | 9 +- test/mocks/server/mocks.cc | 6 +- test/mocks/server/mocks.h | 5 +- 19 files changed, 629 insertions(+), 50 deletions(-) diff --git a/api/envoy/admin/v2alpha/BUILD b/api/envoy/admin/v2alpha/BUILD index 0cbbb7f0d389..aa35aa7c8d7d 100644 --- a/api/envoy/admin/v2alpha/BUILD +++ b/api/envoy/admin/v2alpha/BUILD @@ -11,6 +11,7 @@ api_proto_library_internal( "//envoy/api/v2:lds", "//envoy/api/v2:rds", "//envoy/api/v2:srds", + "//envoy/api/v2/auth:cert", "//envoy/config/bootstrap/v2:bootstrap", ], ) diff --git a/api/envoy/admin/v2alpha/config_dump.proto b/api/envoy/admin/v2alpha/config_dump.proto index ba5b36df36c7..8ff5ba8fa102 100644 --- a/api/envoy/admin/v2alpha/config_dump.proto +++ b/api/envoy/admin/v2alpha/config_dump.proto @@ -6,6 +6,7 @@ option java_outer_classname = "ConfigDumpProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.admin.v2alpha"; +import "envoy/api/v2/auth/cert.proto"; import "envoy/api/v2/cds.proto"; import "envoy/api/v2/lds.proto"; import "envoy/api/v2/rds.proto"; @@ -55,7 +56,7 @@ message ListenersConfigDump { // will be "". string version_info = 1; - // Describes a statically loaded cluster. + // Describes a statically loaded listener. message StaticListener { // The listener config. envoy.api.v2.Listener listener = 1; @@ -219,3 +220,48 @@ message ScopedRoutesConfigDump { repeated DynamicScopedRouteConfigs dynamic_scoped_route_configs = 2 [(gogoproto.nullable) = false]; } + +// Envoys SDS implementation fills this message with all secrets fetched dynamically via SDS. +message SecretsConfigDump { + // DynamicSecret contains secret information fetched via SDS. + message DynamicSecret { + // The name assigned to the secret. + string name = 1; + + // This is the per-resource version information. + string version_info = 2; + + // The timestamp when the secret was last updated. + google.protobuf.Timestamp last_updated = 3; + + // The actual secret information. + // Security sensitive information is redacted (replaced with "[redacted]") for + // private keys and passwords in TLS certificates. + envoy.api.v2.auth.Secret secret = 4; + } + + // StaticSecret specifies statically loaded secret in bootstrap. + message StaticSecret { + // The name assigned to the secret. + string name = 1; + + // The timestamp when the secret was last updated. + google.protobuf.Timestamp last_updated = 2; + + // The actual secret information. + // Security sensitive information is redacted (replaced with "[redacted]") for + // private keys and passwords in TLS certificates. + envoy.api.v2.auth.Secret secret = 3; + } + + // The statically loaded secrets. + repeated StaticSecret static_secrets = 1; + + // The dynamically loaded active secrets. These are secrets that are available to service + // clusters or listeners. + repeated DynamicSecret dynamic_active_secrets = 2; + + // The dynamically loaded warming secrets. These are secrets that are currently undergoing + // warming in preparation to service clusters or listeners. + repeated DynamicSecret dynamic_warming_secrets = 3; +} diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ccdf5f541bf7..5ff2657e1949 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -4,6 +4,7 @@ Version history 1.12.0 (pending) ================ * admin: added ability to configure listener :ref:`socket options `. +* admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. diff --git a/source/common/secret/BUILD b/source/common/secret/BUILD index ca30d94ddfb3..b584cd57c7da 100644 --- a/source/common/secret/BUILD +++ b/source/common/secret/BUILD @@ -19,6 +19,7 @@ envoy_cc_library( "//include/envoy/server:transport_socket_config_interface", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", + "@envoy_api//envoy/admin/v2alpha:config_dump_cc", "@envoy_api//envoy/api/v2/auth:cert_cc", ], ) diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index aa4d4442747c..7626433b9a25 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -11,13 +11,15 @@ namespace Envoy { namespace Secret { SdsApi::SdsApi(envoy::api::v2::core::ConfigSource sds_config, absl::string_view sds_config_name, - Config::SubscriptionFactory& subscription_factory, + Config::SubscriptionFactory& subscription_factory, TimeSource& time_source, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::Store& stats, Init::Manager& init_manager, std::function destructor_cb) : init_target_(fmt::format("SdsApi {}", sds_config_name), [this] { initialize(); }), stats_(stats), sds_config_(std::move(sds_config)), sds_config_name_(sds_config_name), secret_hash_(0), clean_up_(std::move(destructor_cb)), validation_visitor_(validation_visitor), - subscription_factory_(subscription_factory) { + subscription_factory_(subscription_factory), + time_source_(time_source), secret_data_{sds_config_name_, "uninitialized", + time_source_.systemTime()} { // TODO(JimmyCYJ): Implement chained_init_manager, so that multiple init_manager // can be chained together to behave as one init_manager. In that way, we let // two listeners which share same SdsApi to register at separate init managers, and @@ -26,7 +28,7 @@ SdsApi::SdsApi(envoy::api::v2::core::ConfigSource sds_config, absl::string_view } void SdsApi::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, - const std::string&) { + const std::string& version_info) { validateUpdateSize(resources.size()); auto secret = MessageUtil::anyConvert(resources[0], validation_visitor_); @@ -44,7 +46,8 @@ void SdsApi::onConfigUpdate(const Protobuf::RepeatedPtrField& setSecret(secret); update_callback_manager_.runCallbacks(); } - + secret_data_.last_updated_ = time_source_.systemTime(); + secret_data_.version_info_ = version_info; init_target_.ready(); } @@ -79,5 +82,7 @@ void SdsApi::initialize() { subscription_->start({sds_config_name_}); } +SdsApi::SecretData SdsApi::secretData() { return secret_data_; } + } // namespace Secret } // namespace Envoy diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index e05e3e5c0302..a99fafb6c812 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -32,11 +32,19 @@ namespace Secret { */ class SdsApi : public Config::SubscriptionCallbacks { public: + struct SecretData { + const std::string resource_name_; + std::string version_info_; + SystemTime last_updated_; + }; + SdsApi(envoy::api::v2::core::ConfigSource sds_config, absl::string_view sds_config_name, - Config::SubscriptionFactory& subscription_factory, + Config::SubscriptionFactory& subscription_factory, TimeSource& time_source, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::Store& stats, Init::Manager& init_manager, std::function destructor_cb); + SecretData secretData(); + protected: // Creates new secrets. virtual void setSecret(const envoy::api::v2::auth::Secret&) PURE; @@ -68,6 +76,8 @@ class SdsApi : public Config::SubscriptionCallbacks { Cleanup clean_up_; ProtobufMessage::ValidationVisitor& validation_visitor_; Config::SubscriptionFactory& subscription_factory_; + TimeSource& time_source_; + SecretData secret_data_; }; class TlsCertificateSdsApi; @@ -90,17 +100,18 @@ class TlsCertificateSdsApi : public SdsApi, public TlsCertificateConfigProvider Config::Utility::checkLocalInfo("TlsCertificateSdsApi", secret_provider_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), + secret_provider_context.dispatcher().timeSource(), secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), *secret_provider_context.initManager(), destructor_cb); } TlsCertificateSdsApi(const envoy::api::v2::core::ConfigSource& sds_config, const std::string& sds_config_name, - Config::SubscriptionFactory& subscription_factory, + Config::SubscriptionFactory& subscription_factory, TimeSource& time_source, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::Store& stats, Init::Manager& init_manager, std::function destructor_cb) - : SdsApi(sds_config, sds_config_name, subscription_factory, validation_visitor, stats, - init_manager, std::move(destructor_cb)) {} + : SdsApi(sds_config, sds_config_name, subscription_factory, time_source, validation_visitor, + stats, init_manager, std::move(destructor_cb)) {} // SecretProvider const envoy::api::v2::auth::TlsCertificate* secret() const override { @@ -138,17 +149,19 @@ class CertificateValidationContextSdsApi : public SdsApi, secret_provider_context.localInfo()); return std::make_shared( sds_config, sds_config_name, secret_provider_context.clusterManager().subscriptionFactory(), + secret_provider_context.dispatcher().timeSource(), secret_provider_context.messageValidationVisitor(), secret_provider_context.stats(), *secret_provider_context.initManager(), destructor_cb); } CertificateValidationContextSdsApi(const envoy::api::v2::core::ConfigSource& sds_config, const std::string& sds_config_name, Config::SubscriptionFactory& subscription_factory, + TimeSource& time_source, ProtobufMessage::ValidationVisitor& validation_visitor, Stats::Store& stats, Init::Manager& init_manager, std::function destructor_cb) - : SdsApi(sds_config, sds_config_name, subscription_factory, validation_visitor, stats, - init_manager, std::move(destructor_cb)) {} + : SdsApi(sds_config, sds_config_name, subscription_factory, time_source, validation_visitor, + stats, init_manager, std::move(destructor_cb)) {} // SecretProvider const envoy::api::v2::auth::CertificateValidationContext* secret() const override { diff --git a/source/common/secret/secret_manager_impl.cc b/source/common/secret/secret_manager_impl.cc index aed0c374255c..411f71ae6d0b 100644 --- a/source/common/secret/secret_manager_impl.cc +++ b/source/common/secret/secret_manager_impl.cc @@ -1,8 +1,10 @@ #include "common/secret/secret_manager_impl.h" +#include "envoy/admin/v2alpha/config_dump.pb.h" #include "envoy/common/exception.h" #include "common/common/assert.h" +#include "common/common/logger.h" #include "common/secret/sds_api.h" #include "common/secret/secret_provider_impl.h" #include "common/ssl/certificate_validation_context_config_impl.h" @@ -11,6 +13,9 @@ namespace Envoy { namespace Secret { +SecretManagerImpl::SecretManagerImpl(Server::ConfigTracker& config_tracker) + : config_tracker_entry_(config_tracker.add("secrets", [this] { return dumpSecretConfigs(); })) { +} void SecretManagerImpl::addStaticSecret(const envoy::api::v2::auth::Secret& secret) { switch (secret.type_case()) { case envoy::api::v2::auth::Secret::TypeCase::kTlsCertificate: { @@ -79,5 +84,99 @@ SecretManagerImpl::findOrCreateCertificateValidationContextProvider( secret_provider_context); } +// We clear private key and password to avoid information leaking. +// TODO(incfly): switch to more generic scrubbing mechanism once +// https://github.com/envoyproxy/envoy/issues/4757 is resolved. +void redactSecret(::envoy::api::v2::auth::Secret* secret) { + if (secret && secret->type_case() == envoy::api::v2::auth::Secret::TypeCase::kTlsCertificate) { + auto tls_certificate = secret->mutable_tls_certificate(); + if (tls_certificate->has_private_key() && tls_certificate->private_key().specifier_case() != + envoy::api::v2::core::DataSource::kFilename) { + tls_certificate->mutable_private_key()->set_inline_string("[redacted]"); + } + if (tls_certificate->has_password() && tls_certificate->password().specifier_case() != + envoy::api::v2::core::DataSource::kFilename) { + tls_certificate->mutable_password()->set_inline_string("[redacted]"); + } + } +} + +ProtobufTypes::MessagePtr SecretManagerImpl::dumpSecretConfigs() { + auto config_dump = std::make_unique(); + // Handle static tls key/cert providers. + for (const auto& cert_iter : static_tls_certificate_providers_) { + const auto& tls_cert = cert_iter.second; + auto static_secret = config_dump->mutable_static_secrets()->Add(); + static_secret->set_name(cert_iter.first); + ASSERT(tls_cert != nullptr); + auto dump_secret = static_secret->mutable_secret(); + dump_secret->set_name(cert_iter.first); + dump_secret->mutable_tls_certificate()->MergeFrom(*tls_cert->secret()); + redactSecret(dump_secret); + } + + // Handle static certificate validation context providers. + for (const auto& context_iter : static_certificate_validation_context_providers_) { + const auto& validation_context = context_iter.second; + auto static_secret = config_dump->mutable_static_secrets()->Add(); + static_secret->set_name(context_iter.first); + ASSERT(validation_context != nullptr); + auto dump_secret = static_secret->mutable_secret(); + dump_secret->set_name(context_iter.first); + dump_secret->mutable_validation_context()->MergeFrom(*validation_context->secret()); + } + + // Handle dynamic tls_certificate providers. + const auto providers = certificate_providers_.allSecretProviders(); + for (const auto& cert_secrets : providers) { + const auto& secret_data = cert_secrets->secretData(); + const auto& tls_cert = cert_secrets->secret(); + ::envoy::admin::v2alpha::SecretsConfigDump_DynamicSecret* dump_secret; + const bool secret_ready = tls_cert != nullptr; + if (secret_ready) { + dump_secret = config_dump->mutable_dynamic_active_secrets()->Add(); + } else { + dump_secret = config_dump->mutable_dynamic_warming_secrets()->Add(); + } + dump_secret->set_name(secret_data.resource_name_); + auto secret = dump_secret->mutable_secret(); + secret->set_name(secret_data.resource_name_); + ProtobufWkt::Timestamp last_updated_ts; + TimestampUtil::systemClockToTimestamp(secret_data.last_updated_, last_updated_ts); + dump_secret->set_version_info(secret_data.version_info_); + *dump_secret->mutable_last_updated() = last_updated_ts; + secret->set_name(secret_data.resource_name_); + if (secret_ready) { + secret->mutable_tls_certificate()->MergeFrom(*tls_cert); + } + redactSecret(secret); + } + + // Handling dynamic cert validation context providers. + const auto context_secret_provider = validation_context_providers_.allSecretProviders(); + for (const auto& validation_context_secret : context_secret_provider) { + const auto& secret_data = validation_context_secret->secretData(); + const auto& validation_context = validation_context_secret->secret(); + ::envoy::admin::v2alpha::SecretsConfigDump_DynamicSecret* dump_secret; + const bool secret_ready = validation_context != nullptr; + if (secret_ready) { + dump_secret = config_dump->mutable_dynamic_active_secrets()->Add(); + } else { + dump_secret = config_dump->mutable_dynamic_warming_secrets()->Add(); + } + dump_secret->set_name(secret_data.resource_name_); + auto secret = dump_secret->mutable_secret(); + secret->set_name(secret_data.resource_name_); + ProtobufWkt::Timestamp last_updated_ts; + TimestampUtil::systemClockToTimestamp(secret_data.last_updated_, last_updated_ts); + dump_secret->set_version_info(secret_data.version_info_); + *dump_secret->mutable_last_updated() = last_updated_ts; + if (secret_ready) { + secret->mutable_validation_context()->MergeFrom(*validation_context); + } + } + return config_dump; +} + } // namespace Secret } // namespace Envoy diff --git a/source/common/secret/secret_manager_impl.h b/source/common/secret/secret_manager_impl.h index aa7ee6e4b215..7e3da018ecc8 100644 --- a/source/common/secret/secret_manager_impl.h +++ b/source/common/secret/secret_manager_impl.h @@ -16,6 +16,7 @@ namespace Secret { class SecretManagerImpl : public SecretManager { public: + SecretManagerImpl(Server::ConfigTracker& config_tracker); void addStaticSecret(const envoy::api::v2::auth::Secret& secret) override; TlsCertificateConfigProviderSharedPtr @@ -42,6 +43,8 @@ class SecretManagerImpl : public SecretManager { Server::Configuration::TransportSocketFactoryContext& secret_provider_context) override; private: + ProtobufTypes::MessagePtr dumpSecretConfigs(); + template class DynamicSecretProviders : public Logger::Loggable { public: @@ -68,6 +71,17 @@ class SecretManagerImpl : public SecretManager { return secret_provider; } + std::vector> allSecretProviders() { + std::vector> providers; + for (const auto& secret_entry : dynamic_secret_providers_) { + std::shared_ptr secret_provider = secret_entry.second.lock(); + if (secret_provider) { + providers.push_back(std::move(secret_provider)); + } + } + return providers; + } + private: // Removes dynamic secret provider which has been deleted. void removeDynamicSecretProvider(const std::string& map_key) { @@ -91,6 +105,8 @@ class SecretManagerImpl : public SecretManager { // map hash code of SDS config source and SdsApi object. DynamicSecretProviders certificate_providers_; DynamicSecretProviders validation_context_providers_; + + Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; }; } // namespace Secret diff --git a/source/server/config_validation/server.cc b/source/server/config_validation/server.cc index 3b225808daff..9e32616d7124 100644 --- a/source/server/config_validation/server.cc +++ b/source/server/config_validation/server.cc @@ -92,7 +92,7 @@ void ValidationInstance::initialize(const Options& options, listener_manager_ = std::make_unique(*this, *this, *this, false); thread_local_.registerThread(*dispatcher_, true); runtime_loader_ = component_factory.createRuntime(*this, initial_config); - secret_manager_ = std::make_unique(); + secret_manager_ = std::make_unique(admin().getConfigTracker()); ssl_context_manager_ = createContextManager(Ssl::ContextManagerFactory::name(), api_->timeSource()); cluster_manager_factory_ = std::make_unique( diff --git a/source/server/server.cc b/source/server/server.cc index f595909746c0..e3432967fa0b 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -56,10 +56,10 @@ InstanceImpl::InstanceImpl(const Options& options, Event::TimeSystem& time_syste ThreadLocal::Instance& tls, Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system, std::unique_ptr process_context) - : secret_manager_(std::make_unique()), workers_started_(false), - shutdown_(false), options_(options), time_source_(time_system), restarter_(restarter), - start_time_(time(nullptr)), original_start_time_(start_time_), stats_store_(store), - thread_local_(tls), api_(new Api::Impl(thread_factory, store, time_system, file_system)), + : workers_started_(false), shutdown_(false), options_(options), time_source_(time_system), + restarter_(restarter), start_time_(time(nullptr)), original_start_time_(start_time_), + stats_store_(store), thread_local_(tls), + api_(new Api::Impl(thread_factory, store, time_system, file_system)), dispatcher_(api_->allocateDispatcher()), singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), handler_(new ConnectionHandlerImpl(ENVOY_LOGGER(), *dispatcher_)), @@ -334,6 +334,8 @@ void InstanceImpl::initialize(const Options& options, loadServerFlags(initial_config.flagsPath()); + secret_manager_ = std::make_unique(admin_->getConfigTracker()); + // Initialize the overload manager early so other modules can register for actions. overload_manager_ = std::make_unique( *dispatcher_, stats_store_, thread_local_, bootstrap_.overload_manager(), diff --git a/test/common/secret/BUILD b/test/common/secret/BUILD index 2a354658e4d3..19712797f54a 100644 --- a/test/common/secret/BUILD +++ b/test/common/secret/BUILD @@ -22,6 +22,7 @@ envoy_cc_test( "//test/mocks/server:server_mocks", "//test/test_common:environment_lib", "//test/test_common:registry_lib", + "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index 5db20fa89e3b..780211864399 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -42,6 +42,7 @@ class SdsApiTest : public testing::Test { NiceMock subscription_factory_; NiceMock init_manager_; NiceMock init_watcher_; + Event::GlobalTimeSystem time_system_; Init::TargetHandlePtr init_target_handle_; }; @@ -52,8 +53,8 @@ TEST_F(SdsApiTest, BasicTest) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; - TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, validation_visitor_, - server.stats(), init_manager_, []() {}); + TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, time_system_, + validation_visitor_, server.stats(), init_manager_, []() {}); initialize(); } @@ -62,8 +63,8 @@ TEST_F(SdsApiTest, BasicTest) { TEST_F(SdsApiTest, DynamicTlsCertificateUpdateSuccess) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; - TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, validation_visitor_, - server.stats(), init_manager_, []() {}); + TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, time_system_, + validation_visitor_, server.stats(), init_manager_, []() {}); initialize(); NiceMock secret_callback; auto handle = @@ -104,9 +105,9 @@ class PartialMockSds : public SdsApi { public: PartialMockSds(NiceMock& server, NiceMock& init_manager, envoy::api::v2::core::ConfigSource& config_source, - Config::SubscriptionFactory& subscription_factory) - : SdsApi(config_source, "abc.com", subscription_factory, validation_visitor_, server.stats(), - init_manager, []() {}) {} + Config::SubscriptionFactory& subscription_factory, TimeSource& time_source) + : SdsApi(config_source, "abc.com", subscription_factory, time_source, validation_visitor_, + server.stats(), init_manager, []() {}) {} MOCK_METHOD2(onConfigUpdate, void(const Protobuf::RepeatedPtrField&, const std::string&)); @@ -137,7 +138,8 @@ TEST_F(SdsApiTest, Delta) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; - PartialMockSds sds(server, init_manager_, config_source, subscription_factory_); + Event::GlobalTimeSystem time_system; + PartialMockSds sds(server, init_manager_, config_source, subscription_factory_, time_system); initialize(); EXPECT_CALL(sds, onConfigUpdate(RepeatedProtoEq(for_matching), "version1")); subscription_factory_.callbacks_->onConfigUpdate(resources, {}, "ignored"); @@ -155,8 +157,8 @@ TEST_F(SdsApiTest, Delta) { TEST_F(SdsApiTest, DeltaUpdateSuccess) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; - TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, validation_visitor_, - server.stats(), init_manager_, []() {}); + TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, time_system_, + validation_visitor_, server.stats(), init_manager_, []() {}); NiceMock secret_callback; auto handle = @@ -200,8 +202,8 @@ TEST_F(SdsApiTest, DynamicCertificateValidationContextUpdateSuccess) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; CertificateValidationContextSdsApi sds_api(config_source, "abc.com", subscription_factory_, - validation_visitor_, server.stats(), init_manager_, - []() {}); + time_system_, validation_visitor_, server.stats(), + init_manager_, []() {}); NiceMock secret_callback; auto handle = @@ -252,8 +254,8 @@ TEST_F(SdsApiTest, DefaultCertificateValidationContextTest) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; CertificateValidationContextSdsApi sds_api(config_source, "abc.com", subscription_factory_, - validation_visitor_, server.stats(), init_manager_, - []() {}); + time_system_, validation_visitor_, server.stats(), + init_manager_, []() {}); NiceMock secret_callback; auto handle = @@ -321,8 +323,8 @@ TEST_F(SdsApiTest, DefaultCertificateValidationContextTest) { TEST_F(SdsApiTest, EmptyResource) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; - TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, validation_visitor_, - server.stats(), init_manager_, []() {}); + TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, time_system_, + validation_visitor_, server.stats(), init_manager_, []() {}); Protobuf::RepeatedPtrField secret_resources; @@ -336,8 +338,8 @@ TEST_F(SdsApiTest, EmptyResource) { TEST_F(SdsApiTest, SecretUpdateWrongSize) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; - TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, validation_visitor_, - server.stats(), init_manager_, []() {}); + TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, time_system_, + validation_visitor_, server.stats(), init_manager_, []() {}); std::string yaml = R"EOF( @@ -365,8 +367,8 @@ TEST_F(SdsApiTest, SecretUpdateWrongSize) { TEST_F(SdsApiTest, SecretUpdateWrongSecretName) { NiceMock server; envoy::api::v2::core::ConfigSource config_source; - TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, validation_visitor_, - server.stats(), init_manager_, []() {}); + TlsCertificateSdsApi sds_api(config_source, "abc.com", subscription_factory_, time_system_, + validation_visitor_, server.stats(), init_manager_, []() {}); std::string yaml = R"EOF( diff --git a/test/common/secret/secret_manager_impl_test.cc b/test/common/secret/secret_manager_impl_test.cc index 95daa82d0d78..b3a6105fb11f 100644 --- a/test/common/secret/secret_manager_impl_test.cc +++ b/test/common/secret/secret_manager_impl_test.cc @@ -1,15 +1,19 @@ #include +#include "envoy/admin/v2alpha/config_dump.pb.h" #include "envoy/api/v2/auth/cert.pb.h" #include "envoy/common/exception.h" +#include "common/common/logger.h" #include "common/secret/sds_api.h" #include "common/secret/secret_manager_impl.h" #include "common/ssl/certificate_validation_context_config_impl.h" #include "common/ssl/tls_certificate_config_impl.h" +#include "test/mocks/event/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/environment.h" +#include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -22,11 +26,24 @@ namespace Envoy { namespace Secret { namespace { -class SecretManagerImplTest : public testing::Test { +class SecretManagerImplTest : public testing::Test, public Logger::Loggable { protected: SecretManagerImplTest() : api_(Api::createApiForTest()) {} + void checkConfigDump(const std::string& expected_dump_yaml) { + auto message_ptr = config_tracker_.config_tracker_callbacks_["secrets"](); + const auto& secrets_config_dump = + dynamic_cast(*message_ptr); + envoy::admin::v2alpha::SecretsConfigDump expected_secrets_config_dump; + TestUtility::loadFromYaml(expected_dump_yaml, expected_secrets_config_dump); + EXPECT_EQ(expected_secrets_config_dump.DebugString(), secrets_config_dump.DebugString()); + } + + void setupSecretProviderContext() {} + Api::ApiPtr api_; + testing::NiceMock config_tracker_; + Event::SimulatedTimeSystem time_system_; }; // Validate that secret manager adds static TLS certificate secret successfully. @@ -42,7 +59,7 @@ name: "abc.com" filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem" )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); - std::unique_ptr secret_manager(new SecretManagerImpl()); + std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); secret_manager->addStaticSecret(secret_config); ASSERT_EQ(secret_manager->findStaticTlsCertificateProvider("undefined"), nullptr); @@ -75,7 +92,7 @@ TEST_F(SecretManagerImplTest, DuplicateStaticTlsCertificateSecret) { filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem" )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); - std::unique_ptr secret_manager(new SecretManagerImpl()); + std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); secret_manager->addStaticSecret(secret_config); ASSERT_NE(secret_manager->findStaticTlsCertificateProvider("abc.com"), nullptr); @@ -94,7 +111,7 @@ TEST_F(SecretManagerImplTest, CertificateValidationContextSecretLoadSuccess) { allow_expired_certificate: true )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); - std::unique_ptr secret_manager(new SecretManagerImpl()); + std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); secret_manager->addStaticSecret(secret_config); ASSERT_EQ(secret_manager->findStaticCertificateValidationContextProvider("undefined"), nullptr); @@ -119,7 +136,7 @@ TEST_F(SecretManagerImplTest, DuplicateStaticCertificateValidationContextSecret) allow_expired_certificate: true )EOF"; TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); - std::unique_ptr secret_manager(new SecretManagerImpl()); + std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); secret_manager->addStaticSecret(secret_config); ASSERT_NE(secret_manager->findStaticCertificateValidationContextProvider("abc.com"), nullptr); @@ -142,7 +159,7 @@ name: "abc.com" TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), secret_config); - std::unique_ptr secret_manager(new SecretManagerImpl()); + std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); EXPECT_THROW_WITH_MESSAGE(secret_manager->addStaticSecret(secret_config), EnvoyException, "Secret type not implemented"); @@ -150,7 +167,7 @@ name: "abc.com" TEST_F(SecretManagerImplTest, SdsDynamicSecretUpdateSuccess) { Server::MockInstance server; - std::unique_ptr secret_manager(std::make_unique()); + std::unique_ptr secret_manager(new SecretManagerImpl(config_tracker_)); NiceMock secret_context; @@ -168,6 +185,7 @@ TEST_F(SecretManagerImplTest, SdsDynamicSecretUpdateSuccess) { })); EXPECT_CALL(secret_context, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(secret_context, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(secret_context, dispatcher()).WillRepeatedly(ReturnRef(dispatcher)); EXPECT_CALL(secret_context, localInfo()).WillOnce(ReturnRef(local_info)); auto secret_provider = @@ -199,6 +217,352 @@ name: "abc.com" tls_config.privateKey()); } +TEST_F(SecretManagerImplTest, ConfigDumpHandler) { + Server::MockInstance server; + auto secret_manager = std::make_unique(config_tracker_); + time_system_.setSystemTime(std::chrono::milliseconds(1234567891234)); + + NiceMock secret_context; + + envoy::api::v2::core::ConfigSource config_source; + NiceMock local_info; + NiceMock dispatcher; + NiceMock random; + Stats::IsolatedStoreImpl stats; + NiceMock init_manager; + NiceMock init_watcher; + Init::TargetHandlePtr init_target_handle; + EXPECT_CALL(init_manager, add(_)) + .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { + init_target_handle = target.createHandle("test"); + })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(secret_context, dispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + + auto secret_provider = + secret_manager->findOrCreateTlsCertificateProvider(config_source, "abc.com", secret_context); + const std::string yaml = + R"EOF( +name: "abc.com" +tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "DUMMY_INLINE_BYTES_FOR_PRIVATE_KEY" + password: + inline_string: "DUMMY_PASSWORD" +)EOF"; + envoy::api::v2::auth::Secret typed_secret; + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml), typed_secret); + Protobuf::RepeatedPtrField secret_resources; + secret_resources.Add()->PackFrom(typed_secret); + init_target_handle->initialize(init_watcher); + secret_context.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(secret_resources, + "keycert-v1"); + Ssl::TlsCertificateConfigImpl tls_config(*secret_provider->secret(), *api_); + EXPECT_EQ("DUMMY_INLINE_BYTES_FOR_CERT_CHAIN", tls_config.certificateChain()); + EXPECT_EQ("DUMMY_INLINE_BYTES_FOR_PRIVATE_KEY", tls_config.privateKey()); + EXPECT_EQ("DUMMY_PASSWORD", tls_config.password()); + + // Private key and password are removed. + const std::string expected_secrets_config_dump = R"EOF( +dynamic_active_secrets: +- name: "abc.com" + version_info: "keycert-v1" + last_updated: + seconds: 1234567891 + nanos: 234000000 + secret: + name: "abc.com" + tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "[redacted]" + password: + inline_string: "[redacted]" +)EOF"; + checkConfigDump(expected_secrets_config_dump); + + // Add a dynamic tls validation context provider. + time_system_.setSystemTime(std::chrono::milliseconds(1234567899000)); + auto context_secret_provider = secret_manager->findOrCreateCertificateValidationContextProvider( + config_source, "abc.com.validation", secret_context); + const std::string validation_yaml = R"EOF( +name: "abc.com.validation" +validation_context: + trusted_ca: + inline_string: "DUMMY_INLINE_STRING_TRUSTED_CA" +)EOF"; + TestUtility::loadFromYaml(TestEnvironment::substitute(validation_yaml), typed_secret); + secret_resources.Clear(); + secret_resources.Add()->PackFrom(typed_secret); + + init_target_handle->initialize(init_watcher); + secret_context.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate( + secret_resources, "validation-context-v1"); + Ssl::CertificateValidationContextConfigImpl cert_validation_context( + *context_secret_provider->secret(), *api_); + EXPECT_EQ("DUMMY_INLINE_STRING_TRUSTED_CA", cert_validation_context.caCert()); + const std::string updated_config_dump = R"EOF( +dynamic_active_secrets: +- name: "abc.com" + version_info: "keycert-v1" + last_updated: + seconds: 1234567891 + nanos: 234000000 + secret: + name: "abc.com" + tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "[redacted]" + password: + inline_string: "[redacted]" +- name: "abc.com.validation" + version_info: "validation-context-v1" + last_updated: + seconds: 1234567899 + secret: + name: "abc.com.validation" + validation_context: + trusted_ca: + inline_string: "DUMMY_INLINE_STRING_TRUSTED_CA" +)EOF"; + checkConfigDump(updated_config_dump); +} + +TEST_F(SecretManagerImplTest, ConfigDumpHandlerWarmingSecrets) { + Server::MockInstance server; + auto secret_manager = std::make_unique(config_tracker_); + time_system_.setSystemTime(std::chrono::milliseconds(1234567891234)); + + NiceMock secret_context; + + envoy::api::v2::core::ConfigSource config_source; + NiceMock local_info; + NiceMock dispatcher; + NiceMock random; + Stats::IsolatedStoreImpl stats; + NiceMock init_manager; + NiceMock init_watcher; + Init::TargetHandlePtr init_target_handle; + EXPECT_CALL(init_manager, add(_)) + .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { + init_target_handle = target.createHandle("test"); + })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(secret_context, dispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + + auto secret_provider = + secret_manager->findOrCreateTlsCertificateProvider(config_source, "abc.com", secret_context); + const std::string expected_secrets_config_dump = R"EOF( +dynamic_warming_secrets: +- name: "abc.com" + version_info: "uninitialized" + last_updated: + seconds: 1234567891 + nanos: 234000000 + secret: + name: "abc.com" + )EOF"; + checkConfigDump(expected_secrets_config_dump); + + time_system_.setSystemTime(std::chrono::milliseconds(1234567899000)); + auto context_secret_provider = secret_manager->findOrCreateCertificateValidationContextProvider( + config_source, "abc.com.validation", secret_context); + init_target_handle->initialize(init_watcher); + const std::string updated_config_dump = R"EOF( +dynamic_warming_secrets: +- name: "abc.com" + version_info: "uninitialized" + last_updated: + seconds: 1234567891 + nanos: 234000000 + secret: + name: "abc.com" +- name: "abc.com.validation" + version_info: "uninitialized" + last_updated: + seconds: 1234567899 + secret: + name: "abc.com.validation" +)EOF"; + checkConfigDump(updated_config_dump); +} + +TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticSecrets) { + Server::MockInstance server; + auto secret_manager = std::make_unique(config_tracker_); + time_system_.setSystemTime(std::chrono::milliseconds(1234567891234)); + + NiceMock secret_context; + + envoy::api::v2::core::ConfigSource config_source; + NiceMock local_info; + NiceMock dispatcher; + NiceMock random; + Stats::IsolatedStoreImpl stats; + NiceMock init_manager; + NiceMock init_watcher; + Init::TargetHandlePtr init_target_handle; + EXPECT_CALL(init_manager, add(_)) + .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { + init_target_handle = target.createHandle("test"); + })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(secret_context, dispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + + const std::string tls_certificate = + R"EOF( +name: "abc.com" +tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "DUMMY_INLINE_BYTES_FOR_PRIVATE_KEY" + password: + inline_string: "DUMMY_PASSWORD" +)EOF"; + envoy::api::v2::auth::Secret tls_cert_secret; + TestUtility::loadFromYaml(TestEnvironment::substitute(tls_certificate), tls_cert_secret); + secret_manager->addStaticSecret(tls_cert_secret); + TestUtility::loadFromYaml(TestEnvironment::substitute(R"EOF( +name: "abc.com.nopassword" +tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "DUMMY_INLINE_BYTES_FOR_PRIVATE_KEY" +)EOF"), + tls_cert_secret); + secret_manager->addStaticSecret(tls_cert_secret); + const std::string expected_config_dump = R"EOF( +static_secrets: +- name: "abc.com.nopassword" + secret: + name: "abc.com.nopassword" + tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "[redacted]" +- name: "abc.com" + secret: + name: "abc.com" + tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "[redacted]" + password: + inline_string: "[redacted]" +)EOF"; + checkConfigDump(expected_config_dump); +} + +TEST_F(SecretManagerImplTest, ConfigDumpNotRedactFilenamePrivateKey) { + Server::MockInstance server; + auto secret_manager = std::make_unique(config_tracker_); + time_system_.setSystemTime(std::chrono::milliseconds(1234567891234)); + NiceMock secret_context; + envoy::api::v2::core::ConfigSource config_source; + NiceMock local_info; + NiceMock dispatcher; + NiceMock random; + Stats::IsolatedStoreImpl stats; + NiceMock init_manager; + NiceMock init_watcher; + Init::TargetHandlePtr init_target_handle; + EXPECT_CALL(init_manager, add(_)) + .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { + init_target_handle = target.createHandle("test"); + })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(secret_context, dispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + + const std::string tls_certificate = R"EOF( +name: "abc.com" +tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "DUMMY_INLINE_BYTES_FOR_PRIVATE_KEY" + password: + filename: "/etc/certs/password" +)EOF"; + envoy::api::v2::auth::Secret tls_cert_secret; + TestUtility::loadFromYaml(TestEnvironment::substitute(tls_certificate), tls_cert_secret); + secret_manager->addStaticSecret(tls_cert_secret); + const std::string expected_config_dump = R"EOF( +static_secrets: +- name: "abc.com" + secret: + name: "abc.com" + tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES_FOR_CERT_CHAIN" + private_key: + inline_string: "[redacted]" + password: + filename: "/etc/certs/password" +)EOF"; + checkConfigDump(expected_config_dump); +} + +TEST_F(SecretManagerImplTest, ConfigDumpHandlerStaticValidationContext) { + Server::MockInstance server; + auto secret_manager = std::make_unique(config_tracker_); + time_system_.setSystemTime(std::chrono::milliseconds(1234567891234)); + NiceMock secret_context; + envoy::api::v2::core::ConfigSource config_source; + NiceMock local_info; + NiceMock dispatcher; + NiceMock random; + Stats::IsolatedStoreImpl stats; + NiceMock init_manager; + NiceMock init_watcher; + Init::TargetHandlePtr init_target_handle; + EXPECT_CALL(init_manager, add(_)) + .WillRepeatedly(Invoke([&init_target_handle](const Init::Target& target) { + init_target_handle = target.createHandle("test"); + })); + EXPECT_CALL(secret_context, stats()).WillRepeatedly(ReturnRef(stats)); + EXPECT_CALL(secret_context, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(secret_context, dispatcher()).WillRepeatedly(ReturnRef(dispatcher)); + EXPECT_CALL(secret_context, localInfo()).WillRepeatedly(ReturnRef(local_info)); + + const std::string validation_context = + R"EOF( +name: "abc.com.validation" +validation_context: + trusted_ca: + inline_string: "DUMMY_INLINE_STRING_TRUSTED_CA" +)EOF"; + envoy::api::v2::auth::Secret validation_secret; + TestUtility::loadFromYaml(TestEnvironment::substitute(validation_context), validation_secret); + secret_manager->addStaticSecret(validation_secret); + const std::string expected_config_dump = R"EOF( +static_secrets: +- name: "abc.com.validation" + secret: + name: "abc.com.validation" + validation_context: + trusted_ca: + inline_string: "DUMMY_INLINE_STRING_TRUSTED_CA" +)EOF"; + checkConfigDump(expected_config_dump); +} + } // namespace } // namespace Secret } // namespace Envoy diff --git a/test/config/utility.cc b/test/config/utility.cc index b49638a6ca01..cecf66ca568c 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -31,6 +31,15 @@ const std::string ConfigHelper::BASE_CONFIG = R"EOF( lds_config: path: /dev/null static_resources: + secrets: + - name: "secret_static_0" + tls_certificate: + certificate_chain: + inline_string: "DUMMY_INLINE_BYTES" + private_key: + inline_string: "DUMMY_INLINE_BYTES" + password: + inline_string: "DUMMY_INLINE_BYTES" clusters: name: cluster_0 hosts: @@ -703,4 +712,5 @@ void EdsHelper::setEdsAndWait( RELEASE_ASSERT( update_successes_ == server_stats.counter("cluster.cluster_0.update_success")->value(), ""); } + } // namespace Envoy diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index 173ac795d476..ef82b26fff43 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -748,9 +748,11 @@ TEST_F(ClientContextConfigImplTest, SecretNotReady) { NiceMock local_info; Stats::IsolatedStoreImpl stats; NiceMock init_manager; + NiceMock dispatcher; EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(factory_context_, dispatcher()).WillOnce(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); @@ -778,9 +780,11 @@ TEST_F(ClientContextConfigImplTest, ValidationContextNotReady) { NiceMock local_info; Stats::IsolatedStoreImpl stats; NiceMock init_manager; + NiceMock dispatcher; EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(factory_context_, dispatcher()).WillOnce(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_validation_context_sds_secret_config(); sds_secret_configs->set_name("abc.com"); @@ -1079,9 +1083,11 @@ TEST_F(ServerContextConfigImplTest, SecretNotReady) { NiceMock local_info; Stats::IsolatedStoreImpl stats; NiceMock init_manager; + NiceMock dispatcher; EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(factory_context_, dispatcher()).WillOnce(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_tls_certificate_sds_secret_configs()->Add(); sds_secret_configs->set_name("abc.com"); @@ -1109,9 +1115,11 @@ TEST_F(ServerContextConfigImplTest, ValidationContextNotReady) { NiceMock local_info; Stats::IsolatedStoreImpl stats; NiceMock init_manager; + NiceMock dispatcher; EXPECT_CALL(factory_context_, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context_, stats()).WillOnce(ReturnRef(stats)); EXPECT_CALL(factory_context_, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(factory_context_, dispatcher()).WillOnce(ReturnRef(dispatcher)); auto sds_secret_configs = tls_context.mutable_common_tls_context()->mutable_validation_context_sds_secret_config(); sds_secret_configs->set_name("abc.com"); diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index a1ef333b4bae..07daedca6ca0 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -3756,6 +3756,8 @@ TEST_P(SslSocketTest, DownstreamNotReadySslSocket) { NiceMock local_info; testing::NiceMock factory_context; NiceMock init_manager; + NiceMock dispatcher; + EXPECT_CALL(factory_context, dispatcher()).WillOnce(ReturnRef(dispatcher)); EXPECT_CALL(factory_context, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context, stats()).WillOnce(ReturnRef(stats_store)); EXPECT_CALL(factory_context, initManager()).WillRepeatedly(Return(&init_manager)); @@ -3790,9 +3792,11 @@ TEST_P(SslSocketTest, UpstreamNotReadySslSocket) { NiceMock local_info; testing::NiceMock factory_context; NiceMock init_manager; + NiceMock dispatcher; EXPECT_CALL(factory_context, localInfo()).WillOnce(ReturnRef(local_info)); EXPECT_CALL(factory_context, stats()).WillOnce(ReturnRef(stats_store)); EXPECT_CALL(factory_context, initManager()).WillRepeatedly(Return(&init_manager)); + EXPECT_CALL(factory_context, dispatcher()).WillOnce(ReturnRef(dispatcher)); envoy::api::v2::auth::UpstreamTlsContext tls_context; auto sds_secret_configs = diff --git a/test/integration/integration_admin_test.cc b/test/integration/integration_admin_test.cc index 137e972e0055..2af1c1f069df 100644 --- a/test/integration/integration_admin_test.cc +++ b/test/integration/integration_admin_test.cc @@ -393,7 +393,8 @@ TEST_P(IntegrationAdminTest, Admin) { "type.googleapis.com/envoy.admin.v2alpha.ClustersConfigDump", "type.googleapis.com/envoy.admin.v2alpha.ListenersConfigDump", "type.googleapis.com/envoy.admin.v2alpha.ScopedRoutesConfigDump", - "type.googleapis.com/envoy.admin.v2alpha.RoutesConfigDump"}; + "type.googleapis.com/envoy.admin.v2alpha.RoutesConfigDump", + "type.googleapis.com/envoy.admin.v2alpha.SecretsConfigDump"}; for (const Json::ObjectSharedPtr& obj_ptr : json->getObjectArray("configs")) { EXPECT_TRUE(expected_types[index].compare(obj_ptr->getString("@type")) == 0); @@ -403,12 +404,16 @@ TEST_P(IntegrationAdminTest, Admin) { // Validate we can parse as proto. envoy::admin::v2alpha::ConfigDump config_dump; TestUtility::loadFromJson(response->body(), config_dump); - EXPECT_EQ(5, config_dump.configs_size()); + EXPECT_EQ(6, config_dump.configs_size()); // .. and that we can unpack one of the entries. envoy::admin::v2alpha::RoutesConfigDump route_config_dump; config_dump.configs(4).UnpackTo(&route_config_dump); EXPECT_EQ("route_config_0", route_config_dump.static_route_configs(0).route_config().name()); + + envoy::admin::v2alpha::SecretsConfigDump secret_config_dump; + config_dump.configs(5).UnpackTo(&secret_config_dump); + EXPECT_EQ("secret_static_0", secret_config_dump.static_secrets(0).name()); } TEST_P(IntegrationAdminTest, AdminOnDestroyCallbacks) { diff --git a/test/mocks/server/mocks.cc b/test/mocks/server/mocks.cc index bfaa237f50d5..28634190fca6 100644 --- a/test/mocks/server/mocks.cc +++ b/test/mocks/server/mocks.cc @@ -124,8 +124,8 @@ MockWorker::MockWorker() { MockWorker::~MockWorker() = default; MockInstance::MockInstance() - : secret_manager_(new Secret::SecretManagerImpl()), cluster_manager_(timeSource()), - ssl_context_manager_(timeSource()), + : secret_manager_(std::make_unique(admin_.getConfigTracker())), + cluster_manager_(timeSource()), ssl_context_manager_(timeSource()), singleton_manager_(new Singleton::ManagerImpl(Thread::threadFactoryForTest())), grpc_context_(stats_store_.symbolTable()), http_context_(stats_store_.symbolTable()) { ON_CALL(*this, threadLocal()).WillByDefault(ReturnRef(thread_local_)); @@ -197,7 +197,7 @@ MockFactoryContext::MockFactoryContext() MockFactoryContext::~MockFactoryContext() = default; MockTransportSocketFactoryContext::MockTransportSocketFactoryContext() - : secret_manager_(new Secret::SecretManagerImpl()) { + : secret_manager_(std::make_unique(config_tracker_)) { ON_CALL(*this, clusterManager()).WillByDefault(ReturnRef(cluster_manager_)); ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); ON_CALL(*this, messageValidationVisitor()) diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 707fc9ebdf0c..d900080140c6 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -381,7 +381,6 @@ class MockInstance : public Instance { TimeSource& timeSource() override { return time_system_; } - std::unique_ptr secret_manager_; testing::NiceMock thread_local_; NiceMock stats_store_; std::shared_ptr> dns_resolver_{ @@ -389,6 +388,7 @@ class MockInstance : public Instance { testing::NiceMock api_; testing::NiceMock admin_; Event::GlobalTimeSystem time_system_; + std::unique_ptr secret_manager_; testing::NiceMock cluster_manager_; Thread::MutexBasicLockable access_log_lock_; testing::NiceMock runtime_loader_; @@ -508,8 +508,9 @@ class MockTransportSocketFactoryContext : public TransportSocketFactoryContext { MOCK_METHOD0(api, Api::Api&()); testing::NiceMock cluster_manager_; - std::unique_ptr secret_manager_; testing::NiceMock api_; + testing::NiceMock config_tracker_; + std::unique_ptr secret_manager_; }; class MockListenerFactoryContext : public MockFactoryContext, public ListenerFactoryContext { From 95e49d6f76fde00ac67dc34b731e22ce133bf71d Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Thu, 25 Jul 2019 09:11:39 -0700 Subject: [PATCH 238/542] get EXPECT_ out of helper for better failure visibility (#7710) Signed-off-by: Fred Douglas --- .../filesystem_subscription_impl_test.cc | 8 ++-- .../filesystem_subscription_test_harness.h | 8 ++-- .../config/grpc_subscription_impl_test.cc | 24 +++++----- .../config/http_subscription_impl_test.cc | 20 ++++---- test/common/config/subscription_impl_test.cc | 46 +++++++++---------- .../common/config/subscription_test_harness.h | 27 ++++++++--- 6 files changed, 73 insertions(+), 60 deletions(-) diff --git a/test/common/config/filesystem_subscription_impl_test.cc b/test/common/config/filesystem_subscription_impl_test.cc index 51efbd9a9e00..36c9815d1bf0 100644 --- a/test/common/config/filesystem_subscription_impl_test.cc +++ b/test/common/config/filesystem_subscription_impl_test.cc @@ -18,19 +18,19 @@ class FilesystemSubscriptionImplTest : public testing::Test, // Validate that the client can recover from bad JSON responses. TEST_F(FilesystemSubscriptionImplTest, BadJsonRecovery) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); updateFile(";!@#badjso n"); - verifyStats(2, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - verifyStats(3, 1, 0, 1, 7148434200721666028); + EXPECT_TRUE(statsAre(3, 1, 0, 1, 7148434200721666028)); } // Validate that a file that is initially available results in a successful update. TEST_F(FilesystemSubscriptionImplTest, InitialFile) { updateFile("{\"versionInfo\": \"0\", \"resources\": []}", false); startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 1, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(1, 1, 0, 0, 7148434200721666028)); } // Validate that if we fail to set a watch, we get a sensible warning. diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 28dea555e0d5..b34028e40b53 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -86,11 +86,11 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { updateFile(file_json); } - void verifyStats(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, - uint64_t version) override { + AssertionResult statsAre(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, + uint64_t version) override { // The first attempt always fail unless there was a file there to begin with. - SubscriptionTestHarness::verifyStats(attempt, success, rejected, - failure + (file_at_start_ ? 0 : 1), version); + return SubscriptionTestHarness::statsAre(attempt, success, rejected, + failure + (file_at_start_ ? 0 : 1), version); } void expectConfigUpdateFailed() override { diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index ed62b3b61599..35991e88717d 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -19,7 +19,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { EXPECT_CALL(random_, random()); EXPECT_CALL(*timer_, enableTimer(_)); subscription_->start({"cluster0", "cluster1"}); - verifyStats(2, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); // Ensure this doesn't cause an issue by sending a request, since we don't // have a gRPC stream. subscription_->updateResources({"cluster2"}); @@ -29,27 +29,27 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { expectSendMessage({"cluster2"}, ""); timer_cb_(); - verifyStats(3, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(3, 0, 0, 1, 0)); verifyControlPlaneStats(1); } // Validate that the client can recover from a remote stream closure via retry. TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); EXPECT_CALL(*timer_, enableTimer(_)); EXPECT_CALL(random_, random()); subscription_->grpcMux().grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); - verifyStats(2, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); verifyControlPlaneStats(0); // Retry and succeed. EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage({"cluster0", "cluster1"}, ""); timer_cb_(); - verifyStats(2, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); } // Validate that When the management server gets multiple requests for the same version, it can @@ -57,21 +57,21 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { TEST_F(GrpcSubscriptionImplTest, RepeatedNonce) { InSequence s; startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); // First with the initial, empty version update to "0". updateResources({"cluster2"}); - verifyStats(2, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(2, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster2"}, "0", false); - verifyStats(3, 0, 1, 0, 0); + EXPECT_TRUE(statsAre(3, 0, 1, 0, 0)); deliverConfigUpdate({"cluster0", "cluster2"}, "0", true); - verifyStats(4, 1, 1, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(4, 1, 1, 0, 7148434200721666028)); // Now with version "0" update to "1". updateResources({"cluster3"}); - verifyStats(5, 1, 1, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(5, 1, 1, 0, 7148434200721666028)); deliverConfigUpdate({"cluster3"}, "1", false); - verifyStats(6, 1, 2, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(6, 1, 2, 0, 7148434200721666028)); deliverConfigUpdate({"cluster3"}, "1", true); - verifyStats(7, 2, 2, 0, 13237225503670494420U); + EXPECT_TRUE(statsAre(7, 2, 2, 0, 13237225503670494420U)); } } // namespace diff --git a/test/common/config/http_subscription_impl_test.cc b/test/common/config/http_subscription_impl_test.cc index a8aaed17f7a1..6cc81becb889 100644 --- a/test/common/config/http_subscription_impl_test.cc +++ b/test/common/config/http_subscription_impl_test.cc @@ -17,11 +17,11 @@ TEST_F(HttpSubscriptionImplTest, OnRequestReset) { EXPECT_CALL(*timer_, enableTimer(_)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); http_callbacks_->onFailure(Http::AsyncClient::FailureReason::Reset); - verifyStats(1, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 1, 0)); timerTick(); - verifyStats(2, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - verifyStats(3, 1, 0, 1, 7148434200721666028); + EXPECT_TRUE(statsAre(3, 1, 0, 1, 7148434200721666028)); } // Validate that the client can recover from bad JSON responses. @@ -34,28 +34,28 @@ TEST_F(HttpSubscriptionImplTest, BadJsonRecovery) { EXPECT_CALL(*timer_, enableTimer(_)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); http_callbacks_->onSuccess(std::move(message)); - verifyStats(1, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 1, 0)); request_in_progress_ = false; timerTick(); - verifyStats(2, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - verifyStats(3, 1, 0, 1, 7148434200721666028); + EXPECT_TRUE(statsAre(3, 1, 0, 1, 7148434200721666028)); } TEST_F(HttpSubscriptionImplTest, ConfigNotModified) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); timerTick(); - verifyStats(2, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(2, 0, 0, 0, 0)); // accept and modify. deliverConfigUpdate({"cluster0", "cluster1"}, "0", true, true, "200"); - verifyStats(3, 1, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(3, 1, 0, 0, 7148434200721666028)); // accept and does not modify. deliverConfigUpdate({"cluster0", "cluster1"}, "0", true, false, "304"); - verifyStats(4, 1, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(4, 1, 0, 0, 7148434200721666028)); } } // namespace diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index 67a9566619c2..094c2dbf76b2 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -51,9 +51,9 @@ class SubscriptionImplTest : public testing::TestWithParam { test_harness_->expectSendMessage(cluster_names, version); } - void verifyStats(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, - uint64_t version) { - test_harness_->verifyStats(attempt, success, rejected, failure, version); + AssertionResult statsAre(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, + uint64_t version) { + return test_harness_->statsAre(attempt, success, rejected, failure, version); } void deliverConfigUpdate(const std::vector cluster_names, const std::string& version, @@ -88,57 +88,57 @@ INSTANTIATE_TEST_SUITE_P(SubscriptionImplTest, SubscriptionImplInitFetchTimeoutT // Validate basic request-response succeeds. TEST_P(SubscriptionImplTest, InitialRequestResponse) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - verifyStats(2, 1, 0, 0, 7148434200721666028); + statsAre(2, 1, 0, 0, 7148434200721666028); } // Validate that multiple streamed updates succeed. TEST_P(SubscriptionImplTest, ResponseStream) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - verifyStats(2, 1, 0, 0, 7148434200721666028); + statsAre(2, 1, 0, 0, 7148434200721666028); deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); - verifyStats(3, 2, 0, 0, 13237225503670494420U); + statsAre(3, 2, 0, 0, 13237225503670494420U); } // Validate that the client can reject a config. TEST_P(SubscriptionImplTest, RejectConfig) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - verifyStats(2, 0, 1, 0, 0); + statsAre(2, 0, 1, 0, 0); } // Validate that the client can reject a config and accept the same config later. TEST_P(SubscriptionImplTest, RejectAcceptConfig) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - verifyStats(2, 0, 1, 0, 0); + statsAre(2, 0, 1, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - verifyStats(3, 1, 1, 0, 7148434200721666028); + statsAre(3, 1, 1, 0, 7148434200721666028); } // Validate that the client can reject a config and accept another config later. TEST_P(SubscriptionImplTest, RejectAcceptNextConfig) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - verifyStats(2, 0, 1, 0, 0); + statsAre(2, 0, 1, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); - verifyStats(3, 1, 1, 0, 13237225503670494420U); + statsAre(3, 1, 1, 0, 13237225503670494420U); } // Validate that stream updates send a message with the updated resources. TEST_P(SubscriptionImplTest, UpdateResources) { startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - verifyStats(2, 1, 0, 0, 7148434200721666028); + statsAre(2, 1, 0, 0, 7148434200721666028); updateResources({"cluster2"}); - verifyStats(3, 1, 0, 0, 7148434200721666028); + statsAre(3, 1, 0, 0, 7148434200721666028); } // Validate that initial fetch timer is created and calls callback on timeout @@ -146,10 +146,10 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); expectConfigUpdateFailed(); callInitFetchTimeoutCb(); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); } // Validate that initial fetch timer is disabled on config update @@ -157,7 +157,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); expectDisableInitFetchTimeoutTimer(); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); } @@ -167,7 +167,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnFail) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - verifyStats(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0); expectDisableInitFetchTimeoutTimer(); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); } diff --git a/test/common/config/subscription_test_harness.h b/test/common/config/subscription_test_harness.h index 5e12ab8c4180..510d550ff9be 100644 --- a/test/common/config/subscription_test_harness.h +++ b/test/common/config/subscription_test_harness.h @@ -49,15 +49,28 @@ class SubscriptionTestHarness { virtual void deliverConfigUpdate(const std::vector& cluster_names, const std::string& version, bool accept) PURE; - virtual void verifyStats(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, - uint64_t version) { + virtual testing::AssertionResult statsAre(uint32_t attempt, uint32_t success, uint32_t rejected, + uint32_t failure, uint64_t version) { // TODO(fredlas) rework update_success_ to make sense across all xDS carriers. Its value in - // verifyStats() calls in many tests will probably have to be changed. + // statsAre() calls in many tests will probably have to be changed. UNREFERENCED_PARAMETER(attempt); - EXPECT_EQ(success, stats_.update_success_.value()); - EXPECT_EQ(rejected, stats_.update_rejected_.value()); - EXPECT_EQ(failure, stats_.update_failure_.value()); - EXPECT_EQ(version, stats_.version_.value()); + if (success != stats_.update_success_.value()) { + return testing::AssertionFailure() << "update_success: expected " << success << ", got " + << stats_.update_success_.value(); + } + if (rejected != stats_.update_rejected_.value()) { + return testing::AssertionFailure() << "update_rejected: expected " << rejected << ", got " + << stats_.update_rejected_.value(); + } + if (failure != stats_.update_failure_.value()) { + return testing::AssertionFailure() << "update_failure: expected " << failure << ", got " + << stats_.update_failure_.value(); + } + if (version != stats_.version_.value()) { + return testing::AssertionFailure() + << "version: expected " << version << ", got " << stats_.version_.value(); + } + return testing::AssertionSuccess(); } virtual void verifyControlPlaneStats(uint32_t connected_state) { From e61681d2268a6bfbc8bfc17509aa5bb45f8a59d1 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 25 Jul 2019 14:43:50 -0400 Subject: [PATCH 239/542] runtime: making runtime accessible from non-worker threads (#7695) Making runtime accessible for non-worker threads, and using the new accessor for runtime features. This allows the work done in #7601, moving the strict HTTP checks out of the HCM and into the codec, where the integration tests use them from client/server threads, and other downstream Envoys might use them from non-worker threads as well. Risk Level: High (affects runtime access for all runtime features) Testing: new unit tests, integration tests use in #7601 Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- include/envoy/runtime/runtime.h | 8 +++- include/envoy/thread_local/thread_local.h | 10 +++++ source/common/runtime/runtime_impl.cc | 30 +++++++++++--- source/common/runtime/runtime_impl.h | 4 ++ .../common/thread_local/thread_local_impl.cc | 6 ++- .../common/thread_local/thread_local_impl.h | 1 + test/common/runtime/runtime_impl_test.cc | 40 +++++++++++++++++++ test/mocks/runtime/mocks.h | 1 + test/mocks/thread_local/mocks.h | 2 + 9 files changed, 95 insertions(+), 7 deletions(-) diff --git a/include/envoy/runtime/runtime.h b/include/envoy/runtime/runtime.h index bd3e759ee175..f850ec18fde4 100644 --- a/include/envoy/runtime/runtime.h +++ b/include/envoy/runtime/runtime.h @@ -222,10 +222,16 @@ class Loader { /** * @return const Snapshot& the current snapshot. This reference is safe to use for the duration of * the calling routine, but may be overwritten on a future event loop cycle so should be - * fetched again when needed. + * fetched again when needed. This may only be called from worker threads. */ virtual const Snapshot& snapshot() PURE; + /** + * @return shared_ptr the current snapshot. This function may safely be called + * from non-worker theads. + */ + virtual std::shared_ptr threadsafeSnapshot() PURE; + /** * Merge the given map of key-value pairs into the runtime's state. To remove a previous merge for * a key, use an empty string as the value. diff --git a/include/envoy/thread_local/thread_local.h b/include/envoy/thread_local/thread_local.h index 14fac5cd5e22..6f082a4607f5 100644 --- a/include/envoy/thread_local/thread_local.h +++ b/include/envoy/thread_local/thread_local.h @@ -28,6 +28,16 @@ class Slot { public: virtual ~Slot() = default; + /** + * Returns if there is thread local data for this thread. + * + * This should return true for Envoy worker threads and false for threads which do not have thread + * local storage allocated. + * + * @return true if registerThread has been called for this thread, false otherwise. + */ + virtual bool currentThreadRegistered() PURE; + /** * @return ThreadLocalObjectSharedPtr a thread local object stored in the slot. */ diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index ae565aecbde3..6ed82c39a37d 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -27,7 +27,8 @@ namespace Runtime { bool runtimeFeatureEnabled(absl::string_view feature) { ASSERT(absl::StartsWith(feature, "envoy.reloadable_features")); if (Runtime::LoaderSingleton::getExisting()) { - return Runtime::LoaderSingleton::getExisting()->snapshot().runtimeFeatureEnabled(feature); + return Runtime::LoaderSingleton::getExisting()->threadsafeSnapshot()->runtimeFeatureEnabled( + feature); } ENVOY_LOG_TO_LOGGER(Envoy::Logger::Registry::getLog(Envoy::Logger::Id::runtime), warn, "Unable to use runtime singleton for feature {}", feature); @@ -551,13 +552,32 @@ void RtdsSubscription::validateUpdateSize(uint32_t num_resources) { } void LoaderImpl::loadNewSnapshot() { - ThreadLocal::ThreadLocalObjectSharedPtr ptr = createNewSnapshot(); - tls_->set([ptr = std::move(ptr)](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return ptr; + std::shared_ptr ptr = createNewSnapshot(); + tls_->set([ptr](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::static_pointer_cast(ptr); }); + + { + absl::MutexLock lock(&snapshot_mutex_); + thread_safe_snapshot_ = ptr; + } +} + +const Snapshot& LoaderImpl::snapshot() { + ASSERT(tls_->currentThreadRegistered(), "snapshot can only be called from a worker thread"); + return tls_->getTyped(); } -const Snapshot& LoaderImpl::snapshot() { return tls_->getTyped(); } +std::shared_ptr LoaderImpl::threadsafeSnapshot() { + if (tls_->currentThreadRegistered()) { + return std::dynamic_pointer_cast(tls_->get()); + } + + { + absl::ReaderMutexLock lock(&snapshot_mutex_); + return thread_safe_snapshot_; + } +} void LoaderImpl::mergeValues(const std::unordered_map& values) { if (admin_layer_ == nullptr) { diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index b9344d0cc972..a41260eca02f 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -244,6 +244,7 @@ class LoaderImpl : public Loader, Logger::Loggable { // Runtime::Loader void initialize(Upstream::ClusterManager& cm) override; const Snapshot& snapshot() override; + std::shared_ptr threadsafeSnapshot() override; void mergeValues(const std::unordered_map& values) override; private: @@ -265,6 +266,9 @@ class LoaderImpl : public Loader, Logger::Loggable { Api::Api& api_; std::vector subscriptions_; Upstream::ClusterManager* cm_{}; + + absl::Mutex snapshot_mutex_; + std::shared_ptr thread_safe_snapshot_ GUARDED_BY(snapshot_mutex_); }; } // namespace Runtime diff --git a/source/common/thread_local/thread_local_impl.cc b/source/common/thread_local/thread_local_impl.cc index 6884aae42d9a..4e8c32fed776 100644 --- a/source/common/thread_local/thread_local_impl.cc +++ b/source/common/thread_local/thread_local_impl.cc @@ -37,8 +37,12 @@ SlotPtr InstanceImpl::allocateSlot() { return slot; } +bool InstanceImpl::SlotImpl::currentThreadRegistered() { + return thread_local_data_.data_.size() > index_; +} + ThreadLocalObjectSharedPtr InstanceImpl::SlotImpl::get() { - ASSERT(thread_local_data_.data_.size() > index_); + ASSERT(currentThreadRegistered()); return thread_local_data_.data_[index_]; } diff --git a/source/common/thread_local/thread_local_impl.h b/source/common/thread_local/thread_local_impl.h index 25e346809f5a..3e8e39c8fa89 100644 --- a/source/common/thread_local/thread_local_impl.h +++ b/source/common/thread_local/thread_local_impl.h @@ -34,6 +34,7 @@ class InstanceImpl : Logger::Loggable, public Instance { // ThreadLocal::Slot ThreadLocalObjectSharedPtr get() override; + bool currentThreadRegistered() override; void runOnAllThreads(Event::PostCb cb) override { parent_.runOnAllThreads(cb); } void runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback) override { parent_.runOnAllThreads(cb, main_callback); diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index e5edecfc5b88..3d39c6b8e9a7 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -610,6 +610,46 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); } +TEST_F(StaticLoaderImplTest, RuntimeFromNonWorkerThreads) { + // Force the thread to be considered a non-worker thread. + tls_.registered_ = false; + setup(); + + // Set up foo -> bar + loader_->mergeValues({{"foo", "bar"}}); + EXPECT_EQ("bar", loader_->threadsafeSnapshot()->get("foo")); + const Snapshot* original_snapshot_pointer = loader_->threadsafeSnapshot().get(); + + // Now set up a test thread which verifies foo -> bar + // + // Then change foo and make sure the test thread picks up the change. + Thread::MutexBasicLockable mutex; + Thread::CondVar foo_read; + Thread::CondVar foo_changed; + const Snapshot* original_thread_snapshot_pointer = nullptr; + auto thread = Thread::threadFactoryForTest().createThread([&]() { + Thread::LockGuard lock(mutex); + EXPECT_EQ("bar", loader_->threadsafeSnapshot()->get("foo")); + original_thread_snapshot_pointer = loader_->threadsafeSnapshot().get(); + EXPECT_EQ(original_thread_snapshot_pointer, loader_->threadsafeSnapshot().get()); + foo_read.notifyOne(); + + foo_changed.wait(mutex); + EXPECT_EQ("eep", loader_->threadsafeSnapshot()->get("foo")); + }); + + { + Thread::LockGuard lock(mutex); + foo_read.wait(mutex); + loader_->mergeValues({{"foo", "eep"}}); + foo_changed.notifyOne(); + EXPECT_EQ("eep", loader_->threadsafeSnapshot()->get("foo")); + } + + thread->join(); + EXPECT_EQ(original_thread_snapshot_pointer, original_snapshot_pointer); +} + class DiskLayerTest : public testing::Test { protected: DiskLayerTest() : api_(Api::createApiForTest()) {} diff --git a/test/mocks/runtime/mocks.h b/test/mocks/runtime/mocks.h index 6e4385b4a76a..388d789a8940 100644 --- a/test/mocks/runtime/mocks.h +++ b/test/mocks/runtime/mocks.h @@ -64,6 +64,7 @@ class MockLoader : public Loader { MOCK_METHOD1(initialize, void(Upstream::ClusterManager& cm)); MOCK_METHOD0(snapshot, const Snapshot&()); + MOCK_METHOD0(threadsafeSnapshot, std::shared_ptr()); MOCK_METHOD1(mergeValues, void(const std::unordered_map&)); testing::NiceMock snapshot_; diff --git a/test/mocks/thread_local/mocks.h b/test/mocks/thread_local/mocks.h index 3824deac4ac4..3d7a43efaef8 100644 --- a/test/mocks/thread_local/mocks.h +++ b/test/mocks/thread_local/mocks.h @@ -58,6 +58,7 @@ class MockInstance : public Instance { // ThreadLocal::Slot ThreadLocalObjectSharedPtr get() override { return parent_.data_[index_]; } + bool currentThreadRegistered() override { return parent_.registered_; } void runOnAllThreads(Event::PostCb cb) override { parent_.runOnAllThreads(cb); } void runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback) override { parent_.runOnAllThreads(cb, main_callback); @@ -72,6 +73,7 @@ class MockInstance : public Instance { testing::NiceMock dispatcher_; std::vector data_; bool shutdown_{}; + bool registered_{true}; }; } // namespace ThreadLocal From ace23e9de99d023cb797405ddb02dc28df2d916a Mon Sep 17 00:00:00 2001 From: Jyoti Mahapatra <49211422+jyotimahapatra@users.noreply.github.com> Date: Thu, 25 Jul 2019 13:28:59 -0700 Subject: [PATCH 240/542] envoy.fault filter listener based fault injection (#7628) Signed-off-by: Jyoti Mahapatra --- .../config/filter/http/fault/v2/fault.proto | 24 ++++++++++++ docs/root/intro/version_history.rst | 1 + source/common/protobuf/utility.h | 4 ++ .../filters/http/fault/fault_filter.cc | 33 +++++++++++----- .../filters/http/fault/fault_filter.h | 38 +++++++++++++------ .../filters/http/fault/fault_filter_test.cc | 36 ++++++++++++++++++ 6 files changed, 114 insertions(+), 22 deletions(-) diff --git a/api/envoy/config/filter/http/fault/v2/fault.proto b/api/envoy/config/filter/http/fault/v2/fault.proto index bc491580bb15..51ee24ac91a8 100644 --- a/api/envoy/config/filter/http/fault/v2/fault.proto +++ b/api/envoy/config/filter/http/fault/v2/fault.proto @@ -88,4 +88,28 @@ message HTTPFault { // This is a per-stream limit versus a connection level limit. This means that concurrent streams // will each get an independent limit. filter.fault.v2.FaultRateLimit response_rate_limit = 7; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.delay.fixed_delay_percent + string delay_percent_runtime = 8; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.abort.abort_percent + string abort_percent_runtime = 9; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.delay.fixed_duration_ms + string delay_duration_runtime = 10; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.abort.http_status + string abort_http_status_runtime = 11; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.max_active_faults + string max_active_faults_runtime = 12; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.rate_limit.response_percent + string response_rate_limit_percent_runtime = 13; } diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 5ff2657e1949..47e81f6d9118 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -7,6 +7,7 @@ Version history * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. +* fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. * listeners: added :ref:`HTTP inspector listener filter `. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index f18683d8bb36..a05587338e26 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -32,6 +32,10 @@ ((message).has_##field_name() ? DurationUtil::durationToMilliseconds((message).field_name()) \ : (default_value)) +// Obtain the string value if the field is set. Otherwise, return the default value. +#define PROTOBUF_GET_STRING_OR_DEFAULT(message, field_name, default_value) \ + (!(message).field_name().empty() ? (message).field_name() : (default_value)) + // Obtain the milliseconds value of a google.protobuf.Duration field if set. Otherwise, return // absl::nullopt. #define PROTOBUF_GET_OPTIONAL_MS(message, field_name) \ diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 4c7a73c3f43d..7d914f553c2c 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -32,7 +32,20 @@ struct RcDetailsValues { }; using RcDetails = ConstSingleton; -FaultSettings::FaultSettings(const envoy::config::filter::http::fault::v2::HTTPFault& fault) { +FaultSettings::FaultSettings(const envoy::config::filter::http::fault::v2::HTTPFault& fault) + : delay_percent_runtime_(PROTOBUF_GET_STRING_OR_DEFAULT(fault, delay_percent_runtime, + RuntimeKeys::get().DelayPercentKey)), + abort_percent_runtime_(PROTOBUF_GET_STRING_OR_DEFAULT(fault, abort_percent_runtime, + RuntimeKeys::get().AbortPercentKey)), + delay_duration_runtime_(PROTOBUF_GET_STRING_OR_DEFAULT(fault, delay_duration_runtime, + RuntimeKeys::get().DelayDurationKey)), + abort_http_status_runtime_(PROTOBUF_GET_STRING_OR_DEFAULT( + fault, abort_http_status_runtime, RuntimeKeys::get().AbortHttpStatusKey)), + max_active_faults_runtime_(PROTOBUF_GET_STRING_OR_DEFAULT( + fault, max_active_faults_runtime, RuntimeKeys::get().MaxActiveFaultsKey)), + response_rate_limit_percent_runtime_( + PROTOBUF_GET_STRING_OR_DEFAULT(fault, response_rate_limit_percent_runtime, + RuntimeKeys::get().ResponseRateLimitPercentKey)) { if (fault.has_abort()) { const auto& abort = fault.abort(); abort_percentage_ = abort.percentage(); @@ -162,7 +175,7 @@ void FaultFilter::maybeSetupResponseRateLimit(const Http::HeaderMap& request_hea // TODO(mattklein123): Allow runtime override via downstream cluster similar to the other keys. if (!config_->runtime().snapshot().featureEnabled( - RuntimeKeys::get().ResponseRateLimitPercentKey, + fault_settings_->responseRateLimitPercentRuntime(), fault_settings_->responseRateLimit()->percentage())) { return; } @@ -184,9 +197,9 @@ void FaultFilter::maybeSetupResponseRateLimit(const Http::HeaderMap& request_hea bool FaultFilter::faultOverflow() { const uint64_t max_faults = config_->runtime().snapshot().getInteger( - RuntimeKeys::get().MaxActiveFaultsKey, fault_settings_->maxActiveFaults().has_value() - ? fault_settings_->maxActiveFaults().value() - : std::numeric_limits::max()); + fault_settings_->maxActiveFaultsRuntime(), fault_settings_->maxActiveFaults().has_value() + ? fault_settings_->maxActiveFaults().value() + : std::numeric_limits::max()); // Note: Since we don't compare/swap here this is a fuzzy limit which is similar to how the // other circuit breakers work. if (config_->stats().active_faults_.value() >= max_faults) { @@ -203,7 +216,7 @@ bool FaultFilter::isDelayEnabled() { } bool enabled = config_->runtime().snapshot().featureEnabled( - RuntimeKeys::get().DelayPercentKey, fault_settings_->requestDelay()->percentage()); + fault_settings_->delayPercentRuntime(), fault_settings_->requestDelay()->percentage()); if (!downstream_cluster_delay_percent_key_.empty()) { enabled |= config_->runtime().snapshot().featureEnabled( downstream_cluster_delay_percent_key_, fault_settings_->requestDelay()->percentage()); @@ -212,8 +225,8 @@ bool FaultFilter::isDelayEnabled() { } bool FaultFilter::isAbortEnabled() { - bool enabled = config_->runtime().snapshot().featureEnabled(RuntimeKeys::get().AbortPercentKey, - fault_settings_->abortPercentage()); + bool enabled = config_->runtime().snapshot().featureEnabled( + fault_settings_->abortPercentRuntime(), fault_settings_->abortPercentage()); if (!downstream_cluster_abort_percent_key_.empty()) { enabled |= config_->runtime().snapshot().featureEnabled(downstream_cluster_abort_percent_key_, fault_settings_->abortPercentage()); @@ -239,7 +252,7 @@ FaultFilter::delayDuration(const Http::HeaderMap& request_headers) { std::chrono::milliseconds duration = std::chrono::milliseconds(config_->runtime().snapshot().getInteger( - RuntimeKeys::get().DelayDurationKey, config_duration.value().count())); + fault_settings_->delayDurationRuntime(), config_duration.value().count())); if (!downstream_cluster_delay_duration_key_.empty()) { duration = std::chrono::milliseconds(config_->runtime().snapshot().getInteger( downstream_cluster_delay_duration_key_, duration.count())); @@ -256,7 +269,7 @@ FaultFilter::delayDuration(const Http::HeaderMap& request_headers) { uint64_t FaultFilter::abortHttpStatus() { // TODO(mattklein123): check http status codes obtained from runtime. uint64_t http_status = config_->runtime().snapshot().getInteger( - RuntimeKeys::get().AbortHttpStatusKey, fault_settings_->abortCode()); + fault_settings_->abortHttpStatusRuntime(), fault_settings_->abortCode()); if (!downstream_cluster_abort_http_status_key_.empty()) { http_status = config_->runtime().snapshot().getInteger( diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index 2a592c98a0f7..824fecd64ea1 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -62,8 +62,28 @@ class FaultSettings : public Router::RouteSpecificFilterConfig { const Filters::Common::Fault::FaultRateLimitConfig* responseRateLimit() const { return response_rate_limit_.get(); } + const std::string& abortPercentRuntime() const { return abort_percent_runtime_; } + const std::string& delayPercentRuntime() const { return delay_percent_runtime_; } + const std::string& abortHttpStatusRuntime() const { return abort_http_status_runtime_; } + const std::string& delayDurationRuntime() const { return delay_duration_runtime_; } + const std::string& maxActiveFaultsRuntime() const { return max_active_faults_runtime_; } + const std::string& responseRateLimitPercentRuntime() const { + return response_rate_limit_percent_runtime_; + } private: + class RuntimeKeyValues { + public: + const std::string DelayPercentKey = "fault.http.delay.fixed_delay_percent"; + const std::string AbortPercentKey = "fault.http.abort.abort_percent"; + const std::string DelayDurationKey = "fault.http.delay.fixed_duration_ms"; + const std::string AbortHttpStatusKey = "fault.http.abort.http_status"; + const std::string MaxActiveFaultsKey = "fault.http.max_active_faults"; + const std::string ResponseRateLimitPercentKey = "fault.http.rate_limit.response_percent"; + }; + + using RuntimeKeys = ConstSingleton; + envoy::type::FractionalPercent abort_percentage_; uint64_t http_status_{}; // HTTP or gRPC return codes Filters::Common::Fault::FaultDelayConfigPtr request_delay_config_; @@ -72,6 +92,12 @@ class FaultSettings : public Router::RouteSpecificFilterConfig { absl::flat_hash_set downstream_nodes_{}; // Inject failures for specific downstream absl::optional max_active_faults_; Filters::Common::Fault::FaultRateLimitConfigPtr response_rate_limit_; + const std::string delay_percent_runtime_; + const std::string abort_percent_runtime_; + const std::string delay_duration_runtime_; + const std::string abort_http_status_runtime_; + const std::string max_active_faults_runtime_; + const std::string response_rate_limit_percent_runtime_; }; /** @@ -198,18 +224,6 @@ class FaultFilter : public Http::StreamFilter, Logger::Loggable; - bool faultOverflow(); void recordAbortsInjectedStats(); void recordDelaysInjectedStats(); diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index f6181e467675..e6cbf4ace6b9 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -1113,6 +1113,42 @@ TEST_F(FaultFilterRateLimitTest, ResponseRateLimitEnabled) { EXPECT_EQ(0UL, config_->stats().active_faults_.value()); } +class FaultFilterSettingsTest : public FaultFilterTest {}; + +TEST_F(FaultFilterSettingsTest, CheckDefaultRuntimeKeys) { + envoy::config::filter::http::fault::v2::HTTPFault fault; + + Fault::FaultSettings settings(fault); + + EXPECT_EQ("fault.http.delay.fixed_delay_percent", settings.delayPercentRuntime()); + EXPECT_EQ("fault.http.abort.abort_percent", settings.abortPercentRuntime()); + EXPECT_EQ("fault.http.delay.fixed_duration_ms", settings.delayDurationRuntime()); + EXPECT_EQ("fault.http.abort.http_status", settings.abortHttpStatusRuntime()); + EXPECT_EQ("fault.http.max_active_faults", settings.maxActiveFaultsRuntime()); + EXPECT_EQ("fault.http.rate_limit.response_percent", settings.responseRateLimitPercentRuntime()); +} + +TEST_F(FaultFilterSettingsTest, CheckOverrideRuntimeKeys) { + envoy::config::filter::http::fault::v2::HTTPFault fault; + fault.set_abort_percent_runtime(std::string("fault.abort_percent_runtime")); + fault.set_delay_percent_runtime(std::string("fault.delay_percent_runtime")); + fault.set_abort_http_status_runtime(std::string("fault.abort_http_status_runtime")); + fault.set_delay_duration_runtime(std::string("fault.delay_duration_runtime")); + fault.set_max_active_faults_runtime(std::string("fault.max_active_faults_runtime")); + fault.set_response_rate_limit_percent_runtime( + std::string("fault.response_rate_limit_percent_runtime")); + + Fault::FaultSettings settings(fault); + + EXPECT_EQ("fault.delay_percent_runtime", settings.delayPercentRuntime()); + EXPECT_EQ("fault.abort_percent_runtime", settings.abortPercentRuntime()); + EXPECT_EQ("fault.delay_duration_runtime", settings.delayDurationRuntime()); + EXPECT_EQ("fault.abort_http_status_runtime", settings.abortHttpStatusRuntime()); + EXPECT_EQ("fault.max_active_faults_runtime", settings.maxActiveFaultsRuntime()); + EXPECT_EQ("fault.response_rate_limit_percent_runtime", + settings.responseRateLimitPercentRuntime()); +} + } // namespace } // namespace Fault } // namespace HttpFilters From 6af53be976c79a2b51a3f55825b722b58686c8a0 Mon Sep 17 00:00:00 2001 From: Neri Marschik Date: Fri, 26 Jul 2019 10:30:22 +0900 Subject: [PATCH 241/542] grpc-json: add support for ignoring unknown query parameters (#7691) * grpc-json: add support for ignoring unknown query parameters Current behavior is not to transcode a request that contains a query parameter that cannot be mapped. In cases where there is a specific set of parameters to ignore one can use ignored_query_parameters. If there is a proxy or client adding query parameters that one cannot control this won't work and the transcoder becomes useless. ignore_unknown_query_parameters solves this problem. Risk Level: low to medium Testing: Added a unit test for the transcoder. Tested manually, locally. Docs Changes: Added field description in api/envoy/config/filter/http/transcoder/v2/transcoder.proto Release Notes: Signed-off-by: Neri Marschik --- .../http/transcoder/v2/transcoder.proto | 6 ++++ docs/root/intro/version_history.rst | 3 +- .../json_transcoder_filter.cc | 4 +++ .../json_transcoder_filter.h | 1 + .../grpc_json_transcoder_integration_test.cc | 34 ++++++++++++++++++- .../json_transcoder_filter_test.cc | 18 ++++++++++ 6 files changed, 64 insertions(+), 2 deletions(-) diff --git a/api/envoy/config/filter/http/transcoder/v2/transcoder.proto b/api/envoy/config/filter/http/transcoder/v2/transcoder.proto index a7f092d34655..14f54124508d 100644 --- a/api/envoy/config/filter/http/transcoder/v2/transcoder.proto +++ b/api/envoy/config/filter/http/transcoder/v2/transcoder.proto @@ -114,4 +114,10 @@ message GrpcJsonTranscoder { // The client could ``post`` a json body ``{"shelf": 1234}`` with the path of // ``/bookstore.Bookstore/GetShelfRequest`` to call ``GetShelfRequest``. bool auto_mapping = 7; + + // Whether to ignore query parameters that cannot be mapped to a corresponding + // protobuf field. Use this if you cannot control the query parameters and do + // not know them beforehand. Otherwise use ``ignored_query_parameters``. + // Defaults to false. + bool ignore_unknown_query_parameters = 8; } diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 47e81f6d9118..db59eeb09e7d 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -4,10 +4,11 @@ Version history 1.12.0 (pending) ================ * admin: added ability to configure listener :ref:`socket options `. -* admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. +* admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. +* grpc-json: added support for :ref:`ignoring unknown query parameters`. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. * listeners: added :ref:`HTTP inspector listener filter `. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc index 7b00de163760..fb26f1f39c0d 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc @@ -161,6 +161,7 @@ JsonTranscoderConfig::JsonTranscoderConfig( print_options_.preserve_proto_field_names = print_config.preserve_proto_field_names(); match_incoming_request_route_ = proto_config.match_incoming_request_route(); + ignore_unknown_query_parameters_ = proto_config.ignore_unknown_query_parameters(); } bool JsonTranscoderConfig::matchIncomingRequestInfo() const { @@ -203,6 +204,9 @@ ProtobufUtil::Status JsonTranscoderConfig::createTranscoder( status = type_helper_->ResolveFieldPath(*request_info.message_type, binding.field_path, &resolved_binding.field_path); if (!status.ok()) { + if (ignore_unknown_query_parameters_) { + continue; + } return status; } diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h index 0b7f3b7ff301..21976ef9b760 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.h @@ -89,6 +89,7 @@ class JsonTranscoderConfig : public Logger::Loggable { Protobuf::util::JsonPrintOptions print_options_; bool match_incoming_request_route_{false}; + bool ignore_unknown_query_parameters_{false}; }; using JsonTranscoderConfigSharedPtr = std::shared_ptr; diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index d4672646d22f..89459dd4f341 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -41,7 +41,6 @@ class GrpcJsonTranscoderIntegrationTest )EOF"; config_helper_.addFilter( fmt::format(filter, TestEnvironment::runfilesPath("/test/proto/bookstore.descriptor"))); - HttpIntegrationTest::initialize(); } /** @@ -161,6 +160,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, GrpcJsonTranscoderIntegrationTest, TestUtility::ipTestParamsToString); TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryPost) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{{":method", "POST"}, {":path", "/shelf"}, @@ -176,6 +176,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryPost) { } TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGet) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{{":method", "GET"}, {":path", "/shelves"}, {":authority", "host"}}, "", {""}, {R"(shelves { id: 20 theme: "Children" } @@ -189,6 +190,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGet) { } TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetHttpBody) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{{":method", "GET"}, {":path", "/index"}, {":authority", "host"}}, "", {""}, {R"(content_type: "text/html" data: "

Hello!

" )"}, Status(), @@ -200,6 +202,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetHttpBody) { } TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetError) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{ {":method", "GET"}, {":path", "/shelves/100?"}, {":authority", "host"}}, @@ -209,7 +212,30 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetError) { ""); } +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGetError1) { + const std::string filter = + R"EOF( + name: envoy.grpc_json_transcoder + config: + proto_descriptor : "{}" + services : "bookstore.Bookstore" + ignore_unknown_query_parameters : true + )EOF"; + config_helper_.addFilter( + fmt::format(filter, TestEnvironment::runfilesPath("/test/proto/bookstore.descriptor"))); + HttpIntegrationTest::initialize(); + testTranscoding( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/shelves/100?unknown=1&shelf=9999"}, + {":authority", "host"}}, + "", {"shelf: 9999"}, {}, Status(Code::NOT_FOUND, "Shelf 9999 Not Found"), + Http::TestHeaderMapImpl{ + {":status", "404"}, {"grpc-status", "5"}, {"grpc-message", "Shelf 9999 Not Found"}}, + ""); +} + TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryDelete) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{ {":method", "DELETE"}, {":path", "/shelves/456/books/123"}, {":authority", "host"}}, @@ -222,6 +248,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryDelete) { } TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryPatch) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{ {":method", "PATCH"}, {":path", "/shelves/456/books/123"}, {":authority", "host"}}, @@ -236,6 +263,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryPatch) { } TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryCustom) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{ {":method", "OPTIONS"}, {":path", "/shelves/456"}, {":authority", "host"}}, @@ -248,6 +276,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryCustom) { } TEST_P(GrpcJsonTranscoderIntegrationTest, BindingAndBody) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{ {":method", "PUT"}, {":path", "/shelves/1/books"}, {":authority", "host"}}, @@ -259,6 +288,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, BindingAndBody) { } TEST_P(GrpcJsonTranscoderIntegrationTest, ServerStreamingGet) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{ {":method", "GET"}, {":path", "/shelves/1/books"}, {":authority", "host"}}, @@ -271,6 +301,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, ServerStreamingGet) { } TEST_P(GrpcJsonTranscoderIntegrationTest, StreamingPost) { + HttpIntegrationTest::initialize(); testTranscoding( Http::TestHeaderMapImpl{ {":method", "POST"}, {":path", "/bulk/shelves"}, {":authority", "host"}}, @@ -300,6 +331,7 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, StreamingPost) { } TEST_P(GrpcJsonTranscoderIntegrationTest, InvalidJson) { + HttpIntegrationTest::initialize(); // Usually the response would be // "Unexpected token.\n" // "INVALID_JSON\n" diff --git a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc index 40181c29e9b7..673a13e0a80c 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc @@ -249,6 +249,24 @@ TEST_F(GrpcJsonTranscoderConfigTest, InvalidQueryParameter) { EXPECT_FALSE(transcoder); } +TEST_F(GrpcJsonTranscoderConfigTest, UnknownQueryParameterIsIgnored) { + auto proto_config = getProtoConfig( + TestEnvironment::runfilesPath("test/proto/bookstore.descriptor"), "bookstore.Bookstore"); + proto_config.set_ignore_unknown_query_parameters(true); + JsonTranscoderConfig config(proto_config, *api_); + + Http::TestHeaderMapImpl headers{{":method", "GET"}, {":path", "/shelves?foo=bar"}}; + + TranscoderInputStreamImpl request_in, response_in; + std::unique_ptr transcoder; + const MethodDescriptor* method_descriptor; + auto status = + config.createTranscoder(headers, request_in, response_in, transcoder, method_descriptor); + + EXPECT_TRUE(status.ok()); + EXPECT_TRUE(transcoder); +} + TEST_F(GrpcJsonTranscoderConfigTest, IgnoredQueryParameter) { std::vector ignored_query_parameters = {"key"}; JsonTranscoderConfig config( From 99af2e332999eb7d05f5ef72d1a9f8988962785b Mon Sep 17 00:00:00 2001 From: Qiang Zheng Date: Fri, 26 Jul 2019 09:31:50 +0800 Subject: [PATCH 242/542] add support for mips64 (#7694) Signed-off-by: zqzzq --- bazel/BUILD | 6 ++++++ source/common/api/BUILD | 2 ++ source/server/BUILD | 2 ++ 3 files changed, 10 insertions(+) diff --git a/bazel/BUILD b/bazel/BUILD index d78d94046059..d449250d2390 100755 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -214,6 +214,11 @@ config_setting( values = {"cpu": "ppc"}, ) +config_setting( + name = "linux_mips64", + values = {"cpu": "mips64"}, +) + config_setting( name = "windows_x86_64", values = {"cpu": "x64_windows"}, @@ -292,6 +297,7 @@ alias( ":linux_x86_64": ":linux_x86_64", ":linux_aarch64": ":linux_aarch64", ":linux_ppc": ":linux_ppc", + ":linux_mips64": ":linux_mips64", # If we're not on an linux platform return a value that will never match in the select() statement calling this # since it would have already been matched above. "//conditions:default": ":linux_x86_64", diff --git a/source/common/api/BUILD b/source/common/api/BUILD index 8c7909899c6c..b8b60aab6134 100644 --- a/source/common/api/BUILD +++ b/source/common/api/BUILD @@ -26,12 +26,14 @@ envoy_cc_library( "//bazel:linux_x86_64": ["os_sys_calls_impl_linux.cc"], "//bazel:linux_aarch64": ["os_sys_calls_impl_linux.cc"], "//bazel:linux_ppc": ["os_sys_calls_impl_linux.cc"], + "//bazel:linux_mips64": ["os_sys_calls_impl_linux.cc"], "//conditions:default": [], }) + envoy_select_hot_restart(["os_sys_calls_impl_hot_restart.cc"]), hdrs = ["os_sys_calls_impl.h"] + select({ "//bazel:linux_x86_64": ["os_sys_calls_impl_linux.h"], "//bazel:linux_aarch64": ["os_sys_calls_impl_linux.h"], "//bazel:linux_ppc": ["os_sys_calls_impl_linux.h"], + "//bazel:linux_mips64": ["os_sys_calls_impl_linux.h"], "//conditions:default": [], }) + envoy_select_hot_restart(["os_sys_calls_impl_hot_restart.h"]), deps = [ diff --git a/source/server/BUILD b/source/server/BUILD index 7cb1a512905e..f73da48b261f 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -187,6 +187,7 @@ envoy_cc_library( "//bazel:linux_x86_64": ["options_impl_platform_linux.cc"], "//bazel:linux_aarch64": ["options_impl_platform_linux.cc"], "//bazel:linux_ppc": ["options_impl_platform_linux.cc"], + "//bazel:linux_mips64": ["options_impl_platform_linux.cc"], "//conditions:default": ["options_impl_platform_default.cc"], }), hdrs = [ @@ -196,6 +197,7 @@ envoy_cc_library( "//bazel:linux_x86_64": ["options_impl_platform_linux.h"], "//bazel:linux_aarch64": ["options_impl_platform_linux.h"], "//bazel:linux_ppc": ["options_impl_platform_linux.h"], + "//bazel:linux_mips64": ["options_impl_platform_linux.h"], "//conditions:default": [], }), # TCLAP command line parser needs this to support int64_t/uint64_t in several build environments. From 88d118284fe494e1462a0691dbbf552abeeb2992 Mon Sep 17 00:00:00 2001 From: scw00 <616955249@qq.com> Date: Fri, 26 Jul 2019 09:32:29 +0800 Subject: [PATCH 243/542] Disable clang_tidy for win32 implementation (#7687) Signed-off-by: cwsong --- ci/run_clang_tidy.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ci/run_clang_tidy.sh b/ci/run_clang_tidy.sh index dc0e8aead732..9ab612b48d03 100755 --- a/ci/run_clang_tidy.sh +++ b/ci/run_clang_tidy.sh @@ -26,6 +26,13 @@ echo "build ${BAZEL_BUILD_OPTIONS}" >> .bazelrc # by clang-tidy "${ENVOY_SRCDIR}/tools/gen_compilation_database.py" --run_bazel_build --include_headers +# Do not run clang-tidy against win32 impl +# TODO(scw00): We should run clang-tidy against win32 impl. But currently we only have +# linux ci box. +function exclude_win32_impl() { + grep -v source/common/filesystem/win32/ | grep -v source/common/common/win32 | grep -v source/exe/win32 +} + # Do not run incremental clang-tidy on check_format testdata files. function exclude_testdata() { grep -v tools/testdata/check_format/ @@ -38,7 +45,7 @@ function exclude_chromium_url() { } function filter_excludes() { - exclude_testdata | exclude_chromium_url + exclude_testdata | exclude_chromium_url | exclude_win32_impl } if [[ "${RUN_FULL_CLANG_TIDY}" == 1 ]]; then From ef054f08695b8c883c94674904ad27210aa9ba38 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 25 Jul 2019 18:46:01 -0700 Subject: [PATCH 244/542] tracing: update opencensus and googleapis, use SetName for operation name (#7622) Description: Un-pin opencensus and googleapis to use master versions Use SetName span method to set route operation names (aligning with other tracers). Risk Level: low Testing: Unit tests Docs Changes: None Release Notes: None Signed-off-by: Kuat Yessenov --- WORKSPACE | 20 +- api/bazel/api_build_system.bzl | 45 +++-- api/bazel/repositories.bzl | 176 +----------------- api/bazel/repository_locations.bzl | 6 +- bazel/api_binding.bzl | 36 ++++ bazel/api_repositories.bzl | 35 +--- bazel/dependency_imports.bzl | 12 ++ bazel/envoy_build_system.bzl | 9 +- bazel/envoy_library.bzl | 4 +- .../foreign_cc/com_lightstep_tracer_cpp.patch | 2 +- bazel/foreign_cc/io_opencensus_cpp.patch | 11 -- bazel/repositories.bzl | 18 +- bazel/repository_locations.bzl | 6 +- ci/WORKSPACE.filter.example | 19 +- .../opencensus/opencensus_tracer_impl.cc | 4 +- .../tracers/opencensus/tracer_test.cc | 11 +- 16 files changed, 131 insertions(+), 283 deletions(-) create mode 100644 bazel/api_binding.bzl create mode 100644 bazel/dependency_imports.bzl delete mode 100644 bazel/foreign_cc/io_opencensus_cpp.patch diff --git a/WORKSPACE b/WORKSPACE index b10f47cf7026..ef120bc53d4f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,23 +1,17 @@ workspace(name = "envoy") +load("//bazel:api_binding.bzl", "envoy_api_binding") + +envoy_api_binding() + load("//bazel:api_repositories.bzl", "envoy_api_dependencies") envoy_api_dependencies() -load("//bazel:repositories.bzl", "GO_VERSION", "envoy_dependencies") +load("//bazel:repositories.bzl", "envoy_dependencies") envoy_dependencies() -load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") - -rules_foreign_cc_dependencies() - -load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") - -go_rules_dependencies() - -go_register_toolchains(go_version = GO_VERSION) - -load("@envoy//bazel/toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config") +load("//bazel:dependency_imports.bzl", "envoy_dependency_imports") -rbe_toolchains_config() +envoy_dependency_imports() diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index 192fdfaa8221..9eb8e434a531 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -1,4 +1,4 @@ -load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") +load("@com_google_protobuf//:protobuf.bzl", _py_proto_library = "py_proto_library") load("@com_envoyproxy_protoc_gen_validate//bazel:pgv_proto_library.bzl", "pgv_cc_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_grpc_library", "go_proto_library") load("@io_bazel_rules_go//go:def.bzl", "go_test") @@ -24,21 +24,42 @@ def _LibrarySuffix(library_name, suffix): # https://github.com/bazelbuild/bazel/issues/3935 and/or # https://github.com/bazelbuild/bazel/issues/2626 are resolved. def api_py_proto_library(name, srcs = [], deps = [], has_services = 0): - py_proto_library( + _py_proto_library( name = _Suffix(name, _PY_SUFFIX), srcs = srcs, default_runtime = "@com_google_protobuf//:protobuf_python", protoc = "@com_google_protobuf//:protoc", deps = [_LibrarySuffix(d, _PY_SUFFIX) for d in deps] + [ "@com_envoyproxy_protoc_gen_validate//validate:validate_py", - "@googleapis//:api_httpbody_protos_py", - "@googleapis//:http_api_protos_py", - "@googleapis//:rpc_status_protos_py", + "@com_google_googleapis//google/rpc:status_py_proto", + "@com_google_googleapis//google/api:annotations_py_proto", + "@com_google_googleapis//google/api:http_py_proto", + "@com_google_googleapis//google/api:httpbody_py_proto", "@com_github_gogo_protobuf//:gogo_proto_py", ], visibility = ["//visibility:public"], ) +# This defines googleapis py_proto_library. The repository does not provide its definition and requires +# overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details). +def py_proto_library(name, deps = []): + srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps] + proto_deps = [] + + # py_proto_library in googleapis specifies *_proto rules in dependencies. + # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved. + # As a workaround, manually specify the proto dependencies for the imported python rules. + if name == "annotations_py_proto": + proto_deps = proto_deps + [":http_py_proto"] + _py_proto_library( + name = name, + srcs = srcs, + default_runtime = "@com_google_protobuf//:protobuf_python", + protoc = "@com_google_protobuf//:protoc", + deps = proto_deps + ["@com_google_protobuf//:protobuf_python"], + visibility = ["//visibility:public"], + ) + def api_go_proto_library(name, proto, deps = []): go_proto_library( name = _Suffix(name, _GO_PROTO_SUFFIX), @@ -53,7 +74,7 @@ def api_go_proto_library(name, proto, deps = []): "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", "@io_bazel_rules_go//proto/wkt:wrappers_go_proto", "@com_envoyproxy_protoc_gen_validate//validate:go_default_library", - "@googleapis//:rpc_status_go_proto", + "@com_google_googleapis//google/rpc:status_go_proto", ], ) @@ -70,7 +91,7 @@ def api_go_grpc_library(name, proto, deps = []): "@io_bazel_rules_go//proto/wkt:struct_go_proto", "@io_bazel_rules_go//proto/wkt:wrappers_go_proto", "@com_envoyproxy_protoc_gen_validate//validate:go_default_library", - "@googleapis//:http_api_go_proto", + "@com_google_googleapis//google/api:annotations_go_proto", ], ) @@ -109,8 +130,9 @@ def api_proto_library( "@com_google_protobuf//:struct_proto", "@com_google_protobuf//:timestamp_proto", "@com_google_protobuf//:wrappers_proto", - "@googleapis//:http_api_protos_proto", - "@googleapis//:rpc_status_protos_lib", + "@com_google_googleapis//google/api:http_proto", + "@com_google_googleapis//google/api:annotations_proto", + "@com_google_googleapis//google/rpc:status_proto", "@com_github_gogo_protobuf//:gogo_proto", "@com_envoyproxy_protoc_gen_validate//validate:validate_proto", ], @@ -121,8 +143,9 @@ def api_proto_library( linkstatic = linkstatic, cc_deps = [_LibrarySuffix(d, _CC_SUFFIX) for d in deps] + external_cc_proto_deps + [ "@com_github_gogo_protobuf//:gogo_proto_cc", - "@googleapis//:http_api_protos", - "@googleapis//:rpc_status_protos", + "@com_google_googleapis//google/api:http_cc_proto", + "@com_google_googleapis//google/api:annotations_cc_proto", + "@com_google_googleapis//google/rpc:status_cc_proto", ], deps = [":" + name], visibility = ["//visibility:public"], diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index 67d38c4fb57a..3185d0f74072 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -12,9 +12,8 @@ def api_dependencies(): locations = REPOSITORY_LOCATIONS, ) envoy_http_archive( - name = "googleapis", + name = "com_google_googleapis", locations = REPOSITORY_LOCATIONS, - build_file_content = GOOGLEAPIS_BUILD_CONTENT, ) envoy_http_archive( name = "com_github_gogo_protobuf", @@ -36,179 +35,6 @@ def api_dependencies(): build_file_content = KAFKASOURCE_BUILD_CONTENT, ) -GOOGLEAPIS_BUILD_CONTENT = """ -load("@com_google_protobuf//:protobuf.bzl", "py_proto_library") -load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") -load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") - -filegroup( - name = "api_httpbody_protos_src", - srcs = [ - "google/api/httpbody.proto", - ], - visibility = ["//visibility:public"], -) - -proto_library( - name = "api_httpbody_protos_proto", - srcs = [":api_httpbody_protos_src"], - deps = ["@com_google_protobuf//:descriptor_proto"], - visibility = ["//visibility:public"], -) - -cc_proto_library( - name = "api_httpbody_protos", - deps = [":api_httpbody_protos_proto"], - visibility = ["//visibility:public"], -) - -py_proto_library( - name = "api_httpbody_protos_py", - srcs = [ - "google/api/httpbody.proto", - ], - include = ".", - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - visibility = ["//visibility:public"], - deps = ["@com_google_protobuf//:protobuf_python"], -) - -go_proto_library( - name = "api_httpbody_go_proto", - importpath = "google.golang.org/genproto/googleapis/api/httpbody", - proto = ":api_httpbody_protos_proto", - visibility = ["//visibility:public"], - deps = [ - ":descriptor_go_proto", - ], -) - -filegroup( - name = "http_api_protos_src", - srcs = [ - "google/api/annotations.proto", - "google/api/http.proto", - ], - visibility = ["//visibility:public"], -) - -go_proto_library( - name = "descriptor_go_proto", - importpath = "github.com/golang/protobuf/protoc-gen-go/descriptor", - proto = "@com_google_protobuf//:descriptor_proto", - visibility = ["//visibility:public"], -) - -proto_library( - name = "http_api_protos_proto", - srcs = [":http_api_protos_src"], - deps = ["@com_google_protobuf//:descriptor_proto"], - visibility = ["//visibility:public"], -) - -cc_proto_library( - name = "http_api_protos", - deps = [":http_api_protos_proto"], - visibility = ["//visibility:public"], -) - -py_proto_library( - name = "http_api_protos_py", - srcs = [ - "google/api/annotations.proto", - "google/api/http.proto", - ], - include = ".", - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - visibility = ["//visibility:public"], - deps = ["@com_google_protobuf//:protobuf_python"], -) - -go_proto_library( - name = "http_api_go_proto", - importpath = "google.golang.org/genproto/googleapis/api/annotations", - proto = ":http_api_protos_proto", - visibility = ["//visibility:public"], - deps = [ - ":descriptor_go_proto", - ], -) - -filegroup( - name = "rpc_status_protos_src", - srcs = [ - "google/rpc/status.proto", - ], - visibility = ["//visibility:public"], -) - -proto_library( - name = "rpc_status_protos_lib", - srcs = [":rpc_status_protos_src"], - deps = ["@com_google_protobuf//:any_proto"], - visibility = ["//visibility:public"], -) - -cc_proto_library( - name = "rpc_status_protos", - deps = [":rpc_status_protos_lib"], - visibility = ["//visibility:public"], -) - -go_proto_library( - name = "rpc_status_go_proto", - importpath = "google.golang.org/genproto/googleapis/rpc/status", - proto = ":rpc_status_protos_lib", - visibility = ["//visibility:public"], - deps = [ - "@io_bazel_rules_go//proto/wkt:any_go_proto", - ], -) - -py_proto_library( - name = "rpc_status_protos_py", - srcs = [ - "google/rpc/status.proto", - ], - include = ".", - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - visibility = ["//visibility:public"], - deps = ["@com_google_protobuf//:protobuf_python"], -) - -proto_library( - name = "tracing_proto_proto", - srcs = [ - "google/devtools/cloudtrace/v2/trace.proto", - "google/devtools/cloudtrace/v2/tracing.proto", - ], - deps = [ - ":http_api_protos_proto", - ":rpc_status_protos_lib", - "@com_google_protobuf//:timestamp_proto", - "@com_google_protobuf//:wrappers_proto", - "@com_google_protobuf//:empty_proto", - ], -) - -cc_proto_library( - name = "tracing_proto_cc", - deps = [":tracing_proto_proto"], -) - -cc_grpc_library( - name = "tracing_proto", - srcs = [":tracing_proto_proto"], - deps = [":tracing_proto_cc"], - grpc_only = True, - visibility = ["@io_opencensus_cpp//opencensus:__subpackages__"], -) - -""" - GOGOPROTO_BUILD_CONTENT = """ load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index e2bd881f0388..dd661b7d7886 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -10,8 +10,8 @@ OPENCENSUS_PROTO_SHA256 = "a4e87a1da21d1b3a16674332c3ee6e2689d52f3532e2ff8cb4a62 PGV_GIT_SHA = "dd90514d96e35023ed20201f349542b0bc64461d" PGV_SHA256 = "f7d67677fbec5b0f561f6ee6788223fb535d3864b53a28b6678e319f5d1f4f25" -GOOGLEAPIS_GIT_SHA = "d642131a6e6582fc226caf9893cb7fe7885b3411" # May 23, 2018 -GOOGLEAPIS_SHA = "16f5b2e8bf1e747a32f9a62e211f8f33c94645492e9bbd72458061d9a9de1f63" +GOOGLEAPIS_GIT_SHA = "be480e391cc88a75cf2a81960ef79c80d5012068" # Jul 24, 2019 +GOOGLEAPIS_SHA = "c1969e5b72eab6d9b6cfcff748e45ba57294aeea1d96fd04cd081995de0605c2" PROMETHEUS_GIT_SHA = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" # Nov 17, 2017 PROMETHEUS_SHA = "783bdaf8ee0464b35ec0c8704871e1e72afa0005c3f3587f65d9d6694bf3911b" @@ -28,7 +28,7 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "protoc-gen-validate-" + PGV_GIT_SHA, urls = ["https://github.com/envoyproxy/protoc-gen-validate/archive/" + PGV_GIT_SHA + ".tar.gz"], ), - googleapis = dict( + com_google_googleapis = dict( # TODO(dio): Consider writing a Skylark macro for importing Google API proto. sha256 = GOOGLEAPIS_SHA, strip_prefix = "googleapis-" + GOOGLEAPIS_GIT_SHA, diff --git a/bazel/api_binding.bzl b/bazel/api_binding.bzl new file mode 100644 index 000000000000..bf4f304687c1 --- /dev/null +++ b/bazel/api_binding.bzl @@ -0,0 +1,36 @@ +def _default_envoy_api_impl(ctx): + ctx.file("WORKSPACE", "") + ctx.file("BUILD.bazel", "") + api_dirs = [ + "bazel", + "docs", + "envoy", + "examples", + "test", + "tools", + ] + for d in api_dirs: + ctx.symlink(ctx.path(ctx.attr.api).dirname.get_child(d), d) + +_default_envoy_api = repository_rule( + implementation = _default_envoy_api_impl, + attrs = { + "api": attr.label(default = "@envoy//api:BUILD"), + }, +) + +def envoy_api_binding(): + # Treat the data plane API as an external repo, this simplifies exporting the API to + # https://github.com/envoyproxy/data-plane-api. + if "envoy_api" not in native.existing_rules().keys(): + _default_envoy_api(name = "envoy_api") + + # TODO(https://github.com/envoyproxy/envoy/issues/7719) need to remove both bindings and use canonical rules + native.bind( + name = "api_httpbody_protos", + actual = "@com_google_googleapis//google/api:httpbody_cc_proto", + ) + native.bind( + name = "http_api_protos", + actual = "@com_google_googleapis//google/api:annotations_cc_proto", + ) diff --git a/bazel/api_repositories.bzl b/bazel/api_repositories.bzl index 016fb16c8a2e..d4d2f1abd51d 100644 --- a/bazel/api_repositories.bzl +++ b/bazel/api_repositories.bzl @@ -1,35 +1,4 @@ -def _default_envoy_api_impl(ctx): - ctx.file("WORKSPACE", "") - ctx.file("BUILD.bazel", "") - api_dirs = [ - "bazel", - "docs", - "envoy", - "examples", - "test", - "tools", - ] - for d in api_dirs: - ctx.symlink(ctx.path(ctx.attr.api).dirname.get_child(d), d) - -_default_envoy_api = repository_rule( - implementation = _default_envoy_api_impl, - attrs = { - "api": attr.label(default = "@envoy//api:BUILD"), - }, -) +load("@envoy_api//bazel:repositories.bzl", "api_dependencies") def envoy_api_dependencies(): - # Treat the data plane API as an external repo, this simplifies exporting the API to - # https://github.com/envoyproxy/data-plane-api. - if "envoy_api" not in native.existing_rules().keys(): - _default_envoy_api(name = "envoy_api") - - native.bind( - name = "api_httpbody_protos", - actual = "@googleapis//:api_httpbody_protos", - ) - native.bind( - name = "http_api_protos", - actual = "@googleapis//:http_api_protos", - ) + api_dependencies() diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl new file mode 100644 index 000000000000..3915eb1b553f --- /dev/null +++ b/bazel/dependency_imports.bzl @@ -0,0 +1,12 @@ +load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") +load("@envoy//bazel/toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config") + +# go version for rules_go +GO_VERSION = "1.12.5" + +def envoy_dependency_imports(go_version = GO_VERSION): + rules_foreign_cc_dependencies() + go_rules_dependencies() + go_register_toolchains(go_version) + rbe_toolchains_config() diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index 1a7ed4e98d57..4dc11e496c20 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -129,12 +129,13 @@ def envoy_proto_descriptor(name, out, srcs = [], external_deps = []): include_paths = [".", native.package_name()] if "api_httpbody_protos" in external_deps: - srcs.append("@googleapis//:api_httpbody_protos_src") - include_paths.append("external/googleapis") + srcs.append("@com_google_googleapis//google/api:httpbody.proto") + include_paths.append("external/com_google_googleapis") if "http_api_protos" in external_deps: - srcs.append("@googleapis//:http_api_protos_src") - include_paths.append("external/googleapis") + srcs.append("@com_google_googleapis//google/api:annotations.proto") + srcs.append("@com_google_googleapis//google/api:http.proto") + include_paths.append("external/com_google_googleapis") if "well_known_protos" in external_deps: srcs.append("@com_google_protobuf//:well_known_protos") diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index 33e9e93fc049..b0f54ea64eb4 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -113,8 +113,8 @@ def envoy_proto_library(name, external_deps = [], **kwargs): external_proto_deps = [] external_cc_proto_deps = [] if "api_httpbody_protos" in external_deps: - external_cc_proto_deps.append("@googleapis//:api_httpbody_protos") - external_proto_deps.append("@googleapis//:api_httpbody_protos_proto") + external_cc_proto_deps.append("@com_google_googleapis//google/api:httpbody_cc_proto") + external_proto_deps.append("@com_google_googleapis//google/api:httpbody_proto") api_proto_library( name, external_cc_proto_deps = external_cc_proto_deps, diff --git a/bazel/foreign_cc/com_lightstep_tracer_cpp.patch b/bazel/foreign_cc/com_lightstep_tracer_cpp.patch index c0a4055ae483..88e1a81ad8b2 100644 --- a/bazel/foreign_cc/com_lightstep_tracer_cpp.patch +++ b/bazel/foreign_cc/com_lightstep_tracer_cpp.patch @@ -5,7 +5,7 @@ srcs = ["collector.proto"], deps = [ - "@lightstep_vendored_googleapis//:googleapis_proto", -+ "@googleapis//:http_api_protos_proto", ++ "@com_google_googleapis//google/api:annotations_proto", "@com_google_protobuf//:timestamp_proto", ], visibility = ["//visibility:public"], diff --git a/bazel/foreign_cc/io_opencensus_cpp.patch b/bazel/foreign_cc/io_opencensus_cpp.patch deleted file mode 100644 index f814d6583d81..000000000000 --- a/bazel/foreign_cc/io_opencensus_cpp.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- BUILD 2019-06-03 18:30:58.511651926 -0700 -+++ opencensus/exporters/trace/stackdriver/BUILD 2019-06-03 18:32:38.107571186 -0700 -@@ -28,7 +28,7 @@ - copts = DEFAULT_COPTS, - visibility = ["//visibility:public"], - deps = [ -- "//google/devtools/cloudtrace/v2:tracing_proto", -+ "@googleapis//:tracing_proto", - "//opencensus/common:version", - "//opencensus/common/internal/grpc:status", - "//opencensus/common/internal/grpc:with_user_agent", diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index fe86b72263d8..9cb20ae5f0a3 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -2,7 +2,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load(":genrule_repository.bzl", "genrule_repository") load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive") load(":repository_locations.bzl", "REPOSITORY_LOCATIONS") -load("@envoy_api//bazel:repositories.bzl", "api_dependencies") +load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") # dict of {build recipe name: longform extension name,} PPC_SKIP_TARGETS = {"luajit": "envoy.filters.http.lua"} @@ -15,9 +15,6 @@ NOBORINGSSL_SKIP_TARGETS = { "tls_inspector": "envoy.filters.listener.tls_inspector", } -# go version for rules_go -GO_VERSION = "1.12.5" - # Make all contents of an external repository accessible under a filegroup. Used for external HTTP # archives, e.g. cares. BUILD_ALL_CONTENT = """filegroup(name = "all", srcs = glob(["**"]), visibility = ["//visibility:public"])""" @@ -161,7 +158,16 @@ def envoy_dependencies(skip_targets = []): _python_deps() _cc_deps() _go_deps(skip_targets) - api_dependencies() + + switched_rules_by_language( + name = "com_google_googleapis_imports", + cc = True, + go = True, + grpc = True, + rules_override = { + "py_proto_library": "@envoy_api//bazel:api_build_system.bzl", + }, + ) def _boringssl(): _repository_impl("boringssl") @@ -516,8 +522,6 @@ def _io_opencensus_cpp(): location = REPOSITORY_LOCATIONS["io_opencensus_cpp"] http_archive( name = "io_opencensus_cpp", - patch_args = ["-p0"], - patches = ["@envoy//bazel/foreign_cc:io_opencensus_cpp.patch"], **location ) native.bind( diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index a9db5ba2a0ac..1ff6299a65ba 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -233,9 +233,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://files.pythonhosted.org/packages/b3/b2/238e2590826bfdd113244a40d9d3eb26918bd798fc187e2360a8367068db/six-1.10.0.tar.gz"], ), io_opencensus_cpp = dict( - sha256 = "d6d68704c419a9e892bd1f942e09509ebc5a318499a1abcf2c09734e5dc56e19", - strip_prefix = "opencensus-cpp-1145dd77ffb7a2845c71c8e6ca188ef55e4ff607", - urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/1145dd77ffb7a2845c71c8e6ca188ef55e4ff607.tar.gz"], + sha256 = "9223b4d54af4151910dede03aa58247e90df72167fcc91d5f75e73a141568036", + strip_prefix = "opencensus-cpp-e292a374fb42c6cb2743f1689234bd409f4fda20", + urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/e292a374fb42c6cb2743f1689234bd409f4fda20.tar.gz"], ), com_github_curl = dict( sha256 = "821aeb78421375f70e55381c9ad2474bf279fc454b791b7e95fc83562951c690", diff --git a/ci/WORKSPACE.filter.example b/ci/WORKSPACE.filter.example index f11c8e7911be..db20a3146ac9 100644 --- a/ci/WORKSPACE.filter.example +++ b/ci/WORKSPACE.filter.example @@ -5,21 +5,18 @@ local_repository( path = "{ENVOY_SRCDIR}", ) +load("@envoy//bazel:api_binding.bzl", "envoy_api_binding") + +envoy_api_binding() + load("@envoy//bazel:api_repositories.bzl", "envoy_api_dependencies") + envoy_api_dependencies() -load("@envoy//bazel:repositories.bzl", "envoy_dependencies", "GO_VERSION") +load("@envoy//bazel:repositories.bzl", "envoy_dependencies") envoy_dependencies() -# TODO(htuch): Roll this into envoy_dependencies() -load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") -rules_foreign_cc_dependencies() - -load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") -go_rules_dependencies() -go_register_toolchains(go_version = GO_VERSION) - -load("@envoy//bazel/toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config") +load("@envoy//bazel:dependency_imports.bzl", "envoy_dependency_imports") -rbe_toolchains_config() +envoy_dependency_imports() diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 12c0cfd19f54..154e5903fe83 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -135,9 +135,7 @@ Span::Span(const envoy::config::trace::v2::OpenCensusConfig& oc_config, ::opencensus::trace::Span&& span) : span_(std::move(span)), oc_config_(oc_config) {} -void Span::setOperation(absl::string_view operation) { - span_.AddAnnotation("setOperation", {{"operation", operation}}); -} +void Span::setOperation(absl::string_view operation) { span_.SetName(operation); } void Span::setTag(absl::string_view name, absl::string_view value) { span_.AddAttribute(name, value); diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index 83d622b70bcf..2c7cf5695fec 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -134,18 +134,17 @@ TEST(OpenCensusTracerTest, Span) { const auto& sd = (spans[0].name() == operation_name) ? spans[0] : spans[1]; ENVOY_LOG_MISC(debug, "{}", sd.DebugString()); - EXPECT_EQ("my_operation_1", sd.name()); + EXPECT_EQ("different_name", sd.name()); EXPECT_TRUE(sd.context().IsValid()); EXPECT_TRUE(sd.context().trace_options().IsSampled()); ::opencensus::trace::SpanId zeros; EXPECT_EQ(zeros, sd.parent_span_id()); parent_span_id = sd.context().span_id(); - ASSERT_EQ(4, sd.annotations().events().size()); - EXPECT_EQ("setOperation", sd.annotations().events()[0].event().description()); - EXPECT_EQ("my annotation", sd.annotations().events()[1].event().description()); - EXPECT_EQ("spawnChild", sd.annotations().events()[2].event().description()); - EXPECT_EQ("setSampled", sd.annotations().events()[3].event().description()); + ASSERT_EQ(3, sd.annotations().events().size()); + EXPECT_EQ("my annotation", sd.annotations().events()[0].event().description()); + EXPECT_EQ("spawnChild", sd.annotations().events()[1].event().description()); + EXPECT_EQ("setSampled", sd.annotations().events()[2].event().description()); EXPECT_TRUE(sd.has_ended()); } From 78fd87b788e440574005c2f6f546d2ab8bba8dc1 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 25 Jul 2019 19:33:19 -0700 Subject: [PATCH 245/542] ci: remove ipv6 from circle, increase coverage threshold (#7726) Signed-off-by: Lizan Zhou --- .circleci/config.yml | 23 ----------------------- ci/do_ci.sh | 19 ------------------- ci/do_circle_ci.sh | 5 ++--- ci/do_circle_ci_ipv6_tests.sh | 25 ------------------------- test/run_envoy_bazel_coverage.sh | 2 +- 5 files changed, 3 insertions(+), 71 deletions(-) delete mode 100755 ci/do_circle_ci_ipv6_tests.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index ea49aaaa7018..04deae705684 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,28 +94,6 @@ jobs: fingerprints: - "f6:f9:df:90:9c:4b:5f:9c:f4:69:fd:42:94:ff:88:24" - run: ci/filter_example_mirror.sh - ipv6_tests: - machine: true - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: - name: enable ipv6 - command: | - echo '{ - "ipv6": true, - "fixed-cidr-v6": "2001:db8:1::/64" - }' | sudo tee /etc/docker/daemon.json - - sudo service docker restart - - run: dig go.googlesource.com A go.googlesource.com AAAA # Debug IPv6 network issues - - run: ifconfig - - run: route -A inet -A inet6 - - run: curl -v https://go.googlesource.com - - run: curl -6 -v https://go.googlesource.com || true - - run: ./ci/do_circle_ci_ipv6_tests.sh - - store_artifacts: - path: /tmp/envoy-docker/envoy/generated/failed-testlogs coverage: executor: ubuntu-build @@ -195,7 +173,6 @@ workflows: - compile_time_options - api - filter_example_mirror - - ipv6_tests - coverage - format - clang_tidy diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 3ca2968b2c45..867e00c6c2ba 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -207,25 +207,6 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then # these tests under "-c opt" to save time in CI. bazel test ${BAZEL_BUILD_OPTIONS} ${COMPILE_TIME_OPTIONS} -c opt //test/common/common:assert_test //test/server:server_test exit 0 -elif [[ "$CI_TARGET" == "bazel.ipv6_tests" ]]; then - # This is around until Circle supports IPv6. We try to run a limited set of IPv6 tests as fast - # as possible for basic sanity testing. - - # Hack to avoid returning IPv6 DNS - sed -i 's_#precedence ::ffff:0:0/96 100_precedence ::ffff:0:0/96 100_' /etc/gai.conf - # Debug IPv6 network issues - apt-get update && apt-get install -y dnsutils net-tools curl && \ - ifconfig && \ - route -A inet -A inet6 && \ - curl -v https://go.googlesource.com && \ - curl -6 -v https://go.googlesource.com && \ - dig go.googlesource.com A go.googlesource.com AAAA - - setup_clang_toolchain - echo "Testing..." - bazel_with_collection test ${BAZEL_BUILD_OPTIONS} --test_env=ENVOY_IP_TEST_VERSIONS=v6only -c fastbuild \ - //test/integration/... //test/common/network/... - exit 0 elif [[ "$CI_TARGET" == "bazel.api" ]]; then setup_clang_toolchain echo "Building API..." diff --git a/ci/do_circle_ci.sh b/ci/do_circle_ci.sh index c51ee029c3ee..d6e9d214ea63 100755 --- a/ci/do_circle_ci.sh +++ b/ci/do_circle_ci.sh @@ -15,9 +15,8 @@ export ENVOY_SRCDIR="$(pwd)" # hard code this (basically due to how docker works). export NUM_CPUS=8 -# CircleCI doesn't support IPv6 by default, so we run all tests with IPv4, and -# a limited subset with IPv6 using "machine: true" and do_circle_ci_ipv6_tests.sh -# (see https://circleci.com/docs/2.0/executor-types/#using-machine) +# CircleCI doesn't support IPv6 by default, so we run all tests with IPv4 only. +# IPv6 tests are run with Azure Pipelines. export BAZEL_EXTRA_TEST_OPTIONS="--test_env=ENVOY_IP_TEST_VERSIONS=v4only" function finish { diff --git a/ci/do_circle_ci_ipv6_tests.sh b/ci/do_circle_ci_ipv6_tests.sh deleted file mode 100755 index e41d8c6013f6..000000000000 --- a/ci/do_circle_ci_ipv6_tests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -e - -. ./ci/envoy_build_sha.sh - -export ENVOY_SRCDIR="$(pwd)" - -export ENVOY_BUILD_DIR=/tmp/envoy-docker - -export TEST_TYPE="bazel.ipv6_tests" - -function finish { - echo "disk space at end of build:" - df -h -} -trap finish EXIT - -echo "disk space at beginning of build:" -df -h - -docker run -t -i -v "$ENVOY_BUILD_DIR":/build -v "$ENVOY_SRCDIR":/source \ - --env GCP_SERVICE_ACCOUNT_KEY --env BAZEL_REMOTE_CACHE \ - envoyproxy/envoy-build:"$ENVOY_BUILD_SHA" /bin/bash -c "cd /source && ci/do_ci.sh $TEST_TYPE" - diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 08c63609b661..da55f2095fef 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -54,7 +54,7 @@ then COVERAGE_VALUE=$(llvm-cov export "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \ -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -summary-only | \ python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['totals']['lines']['percent'])") - COVERAGE_THRESHOLD=96.9 + COVERAGE_THRESHOLD=97.0 COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc) if test ${COVERAGE_FAILED} -eq 1; then echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD} From e86231c8cc9d20ad7d816c7c9525399330f22363 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 26 Jul 2019 08:47:04 -0700 Subject: [PATCH 246/542] ci: run tsan in RBE (#7733) Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 2 ++ .circleci/config.yml | 10 ---------- test/extensions/transport_sockets/tls/BUILD | 1 + 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index 0515ec656aa5..fe3f6aa4c667 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -4,6 +4,8 @@ jobs: matrix: release: CI_TARGET: 'bazel.release' + tsan: + CI_TARGET: 'bazel.tsan' dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. timeoutInMinutes: 360 pool: diff --git a/.circleci/config.yml b/.circleci/config.yml index 04deae705684..9352925decd6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,15 +53,6 @@ jobs: - store_artifacts: path: /build/envoy/generated destination: / - tsan: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh bazel.tsan - - store_artifacts: - path: /build/envoy/generated - destination: / compile_time_options: executor: ubuntu-build @@ -169,7 +160,6 @@ workflows: tags: only: /^v.*/ - asan - - tsan - compile_time_options - api - filter_example_mirror diff --git a/test/extensions/transport_sockets/tls/BUILD b/test/extensions/transport_sockets/tls/BUILD index 15ac0e33560d..32ec88bb3d58 100644 --- a/test/extensions/transport_sockets/tls/BUILD +++ b/test/extensions/transport_sockets/tls/BUILD @@ -23,6 +23,7 @@ envoy_cc_test( "//test/extensions/transport_sockets/tls/test_data:certs", ], external_deps = ["ssl"], + shard_count = 4, deps = [ "//include/envoy/network:transport_socket_interface", "//source/common/buffer:buffer_lib", From 6b9a116b499646031d7a801a8d8ab3ad38fced23 Mon Sep 17 00:00:00 2001 From: danzh Date: Fri, 26 Jul 2019 11:48:48 -0400 Subject: [PATCH 247/542] quiche: import some quic test utilities (#7709) Signed-off-by: Dan Zhang --- bazel/external/quiche.BUILD | 183 ++++++++++++++++++ .../quiche/platform/quic_logging_impl.cc | 10 +- .../quiche/platform/quic_logging_impl.h | 2 +- test/extensions/quic_listeners/quiche/BUILD | 11 ++ .../quiche/crypto_test_utils_for_envoy.cc | 39 ++++ .../quiche/platform/quic_platform_test.cc | 12 ++ tools/spelling_dictionary.txt | 1 + 7 files changed, 255 insertions(+), 3 deletions(-) create mode 100644 test/extensions/quic_listeners/quiche/crypto_test_utils_for_envoy.cc diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 23276db96e7c..e08d8a5987eb 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -55,6 +55,7 @@ genrule( quiche_copt = [ # Remove these after upstream fix. "-Wno-unused-parameter", + "-Wno-unused-function", "-Wno-type-limits", # quic_inlined_frame.h uses offsetof() to optimize memory usage in frames. "-Wno-invalid-offsetof", @@ -2032,6 +2033,7 @@ envoy_cc_library( copts = quiche_copt, repository = "@envoy", tags = ["nofips"], + visibility = ["//visibility:public"], deps = [ ":quic_core_connection_lib", ":quic_core_crypto_crypto_handshake_lib", @@ -2574,6 +2576,7 @@ envoy_cc_library( copts = quiche_copt, repository = "@envoy", tags = ["nofips"], + visibility = ["//visibility:public"], deps = [ ":quic_core_alarm_factory_interface_lib", ":quic_core_alarm_interface_lib", @@ -2906,6 +2909,186 @@ envoy_cc_library( ], ) +envoy_cc_test_library( + name = "quic_test_tools_config_peer_lib", + srcs = ["quiche/quic/test_tools/quic_config_peer.cc"], + hdrs = ["quiche/quic/test_tools/quic_config_peer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_config_lib", + ":quic_core_packets_lib", + ":quic_platform_base", + ], +) + +envoy_cc_test_library( + name = "quic_test_tools_framer_peer_lib", + srcs = ["quiche/quic/test_tools/quic_framer_peer.cc"], + hdrs = ["quiche/quic/test_tools/quic_framer_peer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_crypto_encryption_lib", + ":quic_core_framer_lib", + ":quic_core_packets_lib", + ":quic_platform_base", + ], +) + +envoy_cc_test_library( + name = "quic_test_tools_mock_clock_lib", + srcs = ["quiche/quic/test_tools/mock_clock.cc"], + hdrs = ["quiche/quic/test_tools/mock_clock.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_time_lib", + ":quic_platform", + ], +) + +envoy_cc_test_library( + name = "quic_test_tools_mock_random_lib", + srcs = ["quiche/quic/test_tools/mock_random.cc"], + hdrs = ["quiche/quic/test_tools/mock_random.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_core_crypto_random_lib"], +) + +envoy_cc_test_library( + name = "quic_test_tools_packet_generator_peer_lib", + srcs = ["quiche/quic/test_tools/quic_packet_generator_peer.cc"], + hdrs = ["quiche/quic/test_tools/quic_packet_generator_peer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_packet_creator_lib", + ":quic_core_packet_generator_lib", + ":quic_core_packets_lib", + ], +) + +envoy_cc_test_library( + name = "quic_test_tools_sent_packet_manager_peer_lib", + srcs = ["quiche/quic/test_tools/quic_sent_packet_manager_peer.cc"], + hdrs = ["quiche/quic/test_tools/quic_sent_packet_manager_peer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_congestion_control_congestion_control_interface_lib", + ":quic_core_packets_lib", + ":quic_core_sent_packet_manager_lib", + ":quic_test_tools_unacked_packet_map_peer_lib", + ], +) + +envoy_cc_test_library( + name = "quic_test_tools_simple_quic_framer_lib", + srcs = ["quiche/quic/test_tools/simple_quic_framer.cc"], + hdrs = ["quiche/quic/test_tools/simple_quic_framer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_crypto_encryption_lib", + ":quic_core_framer_lib", + ":quic_core_packets_lib", + ":quic_platform_base", + ], +) + +envoy_cc_test_library( + name = "quic_test_tools_stream_send_buffer_peer_lib", + srcs = ["quiche/quic/test_tools/quic_stream_send_buffer_peer.cc"], + hdrs = ["quiche/quic/test_tools/quic_stream_send_buffer_peer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_core_stream_send_buffer_lib"], +) + +envoy_cc_test_library( + name = "quic_test_tools_stream_peer_lib", + srcs = ["quiche/quic/test_tools/quic_stream_peer.cc"], + hdrs = ["quiche/quic/test_tools/quic_stream_peer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [ + ":quic_core_packets_lib", + ":quic_core_session_lib", + ":quic_core_stream_send_buffer_lib", + ":quic_platform_base", + ":quic_test_tools_stream_send_buffer_peer_lib", + ], +) + +envoy_cc_test_library( + name = "quic_test_tools_test_utils_interface_lib", + srcs = [ + "quiche/quic/test_tools/crypto_test_utils.cc", + "quiche/quic/test_tools/mock_quic_session_visitor.cc", + "quiche/quic/test_tools/mock_quic_time_wait_list_manager.cc", + "quiche/quic/test_tools/quic_connection_peer.cc", + "quiche/quic/test_tools/quic_dispatcher_peer.cc", + "quiche/quic/test_tools/quic_test_utils.cc", + ], + hdrs = [ + "quiche/quic/test_tools/crypto_test_utils.h", + "quiche/quic/test_tools/mock_quic_session_visitor.h", + "quiche/quic/test_tools/mock_quic_time_wait_list_manager.h", + "quiche/quic/test_tools/quic_connection_peer.h", + "quiche/quic/test_tools/quic_dispatcher_peer.h", + "quiche/quic/test_tools/quic_test_utils.h", + ], + copts = quiche_copt, + external_deps = ["ssl"], + repository = "@envoy", + deps = [ + ":quic_core_buffer_allocator_lib", + ":quic_core_congestion_control_congestion_control_interface_lib", + ":quic_core_connection_lib", + ":quic_core_connection_stats_lib", + ":quic_core_crypto_crypto_handshake_lib", + ":quic_core_crypto_encryption_lib", + ":quic_core_crypto_proof_source_interface_lib", + ":quic_core_crypto_random_lib", + ":quic_core_data_lib", + ":quic_core_framer_lib", + ":quic_core_http_client_lib", + ":quic_core_http_spdy_session_lib", + ":quic_core_packet_creator_lib", + ":quic_core_packet_writer_interface_lib", + ":quic_core_packets_lib", + ":quic_core_received_packet_manager_lib", + ":quic_core_sent_packet_manager_lib", + ":quic_core_server_id_lib", + ":quic_core_server_lib", + ":quic_core_session_lib", + ":quic_core_time_wait_list_manager_lib", + ":quic_core_utils_lib", + ":quic_platform", + ":quic_platform_test", + ":quic_test_tools_config_peer_lib", + ":quic_test_tools_framer_peer_lib", + ":quic_test_tools_mock_clock_lib", + ":quic_test_tools_mock_random_lib", + ":quic_test_tools_packet_generator_peer_lib", + ":quic_test_tools_sent_packet_manager_peer_lib", + ":quic_test_tools_simple_quic_framer_lib", + ":quic_test_tools_stream_peer_lib", + ":spdy_core_framer_lib", + ], +) + +envoy_cc_test_library( + name = "quic_test_tools_unacked_packet_map_peer_lib", + srcs = ["quiche/quic/test_tools/quic_unacked_packet_map_peer.cc"], + hdrs = ["quiche/quic/test_tools/quic_unacked_packet_map_peer.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_core_unacked_packet_map_lib"], +) + envoy_cc_test_library( name = "epoll_server_platform", hdrs = [ diff --git a/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.cc b/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.cc index 895f1bbcb8d3..60870a742fdd 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.cc +++ b/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.cc @@ -28,14 +28,20 @@ QuicLogEmitter::~QuicLogEmitter() { // TODO(wub): Change to a thread-safe version of strerror. stream_ << ": " << strerror(saved_errno_) << " [" << saved_errno_ << "]"; } - GetLogger().log(level_, "{}", stream_.str().c_str()); + std::string content = stream_.str(); + if (!content.empty() && content.back() == '\n') { + // strip the last trailing '\n' because spd log will add a trailing '\n' to + // the output. + content.back() = '\0'; + } + GetLogger().log(level_, "{}", content.c_str()); // Normally there is no log sink and we can avoid acquiring the lock. if (g_quic_log_sink.load(std::memory_order_relaxed) != nullptr) { absl::MutexLock lock(&g_quic_log_sink_mutex); QuicLogSink* sink = g_quic_log_sink.load(std::memory_order_relaxed); if (sink != nullptr) { - sink->Log(level_, stream_.str()); + sink->Log(level_, content); } } diff --git a/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h index 13fa6dc81031..43dfe6661363 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_logging_impl.h @@ -144,7 +144,7 @@ class QuicLogEmitter { std::ostringstream stream_; }; -class NullLogStream { +class NullLogStream : public std::ostream { public: NullLogStream& stream() { return *this; } }; diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index a51b6ea79b34..6569b0d90844 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -47,3 +47,14 @@ envoy_cc_test( "@com_googlesource_quiche//:quic_core_versions_lib", ], ) + +envoy_cc_test_library( + name = "quic_test_utils_for_envoy_lib", + srcs = ["crypto_test_utils_for_envoy.cc"], + tags = ["nofips"], + deps = [ + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_source_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_verifier_lib", + "@com_googlesource_quiche//:quic_test_tools_test_utils_interface_lib", + ], +) diff --git a/test/extensions/quic_listeners/quiche/crypto_test_utils_for_envoy.cc b/test/extensions/quic_listeners/quiche/crypto_test_utils_for_envoy.cc new file mode 100644 index 000000000000..b3a94737a5e6 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/crypto_test_utils_for_envoy.cc @@ -0,0 +1,39 @@ +// NOLINT(namespace-envoy) + +// This file defines platform dependent test utility functions which is declared +// in quiche/quic/test_tools/crypto_test_utils.h. + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wtype-limits" + +#include "quiche/quic/test_tools/crypto_test_utils.h" + +#pragma GCC diagnostic pop + +#include +#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h" +#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h" + +namespace quic { +namespace test { +namespace crypto_test_utils { +std::unique_ptr ProofSourceForTesting() { + return std::make_unique(); +} + +std::unique_ptr ProofVerifierForTesting() { + return std::make_unique(); +} + +std::unique_ptr ProofVerifyContextForTesting() { + // No context needed for fake verifier. + return nullptr; +} + +} // namespace crypto_test_utils +} // namespace test +} // namespace quic diff --git a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc index 090d948bfa6f..1fce91d5148b 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_platform_test.cc @@ -361,6 +361,18 @@ TEST_F(QuicPlatformTest, QuicLog) { #define VALUE_BY_COMPILE_MODE(debug_mode_value, release_mode_value) debug_mode_value #endif +TEST_F(QuicPlatformTest, LogIoManipulators) { + GetLogger().set_level(ERROR); + QUIC_DLOG(ERROR) << "aaaa" << std::endl; + EXPECT_LOG_CONTAINS("error", "aaaa\n\n", QUIC_LOG(ERROR) << "aaaa" << std::endl << std::endl); + EXPECT_LOG_NOT_CONTAINS("error", "aaaa\n\n\n", + QUIC_LOG(ERROR) << "aaaa" << std::endl + << std::endl); + + EXPECT_LOG_CONTAINS("error", "42 in octal is 52", + QUIC_LOG(ERROR) << 42 << " in octal is " << std::oct << 42); +} + TEST_F(QuicPlatformTest, QuicDLog) { int i = 0; diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 051ce386f181..50abfff89e48 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -244,6 +244,7 @@ SIGSEGV SIGTERM SimpleAtoi SNI +SPD SPDY SPIFFE SPKI From fb7e74ee3e33a619eefeeb7bafd5abbb672c8f8b Mon Sep 17 00:00:00 2001 From: Kuat Date: Fri, 26 Jul 2019 08:54:32 -0700 Subject: [PATCH 248/542] cleanup: expose address string_view accessor (#7732) Signed-off-by: Kuat Yessenov --- include/envoy/network/address.h | 6 ++++++ source/common/network/address_impl.h | 1 + test/common/network/address_impl_test.cc | 12 ++++++++++++ test/common/network/dns_impl_test.cc | 1 + test/mocks/network/mocks.h | 1 + 5 files changed, 21 insertions(+) diff --git a/include/envoy/network/address.h b/include/envoy/network/address.h index c439cecc27af..86d32eca2452 100644 --- a/include/envoy/network/address.h +++ b/include/envoy/network/address.h @@ -13,6 +13,7 @@ #include "envoy/network/io_handle.h" #include "absl/numeric/int128.h" +#include "absl/strings/string_view.h" namespace Envoy { namespace Network { @@ -117,6 +118,11 @@ class Instance { */ virtual const std::string& asString() const PURE; + /** + * @return Similar to asString but returns a string view. + */ + virtual absl::string_view asStringView() const PURE; + /** * @return a human readable string for the address that represents the * logical/unresolved name. diff --git a/source/common/network/address_impl.h b/source/common/network/address_impl.h index 2c0b0112edfd..63e0566ffaaa 100644 --- a/source/common/network/address_impl.h +++ b/source/common/network/address_impl.h @@ -56,6 +56,7 @@ class InstanceBase : public Instance { public: // Network::Address::Instance const std::string& asString() const override { return friendly_name_; } + absl::string_view asStringView() const override { return friendly_name_; } // Default logical name is the human-readable name. const std::string& logicalName() const override { return asString(); } Type type() const override { return type_; } diff --git a/test/common/network/address_impl_test.cc b/test/common/network/address_impl_test.cc index f3a069cb1b87..fe4b75217a3d 100644 --- a/test/common/network/address_impl_test.cc +++ b/test/common/network/address_impl_test.cc @@ -144,6 +144,7 @@ TEST(Ipv4InstanceTest, SocketAddress) { Ipv4Instance address(&addr4); EXPECT_EQ("1.2.3.4:6502", address.asString()); + EXPECT_EQ("1.2.3.4:6502", address.asStringView()); EXPECT_EQ("1.2.3.4:6502", address.logicalName()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("1.2.3.4", address.ip()->addressAsString()); @@ -157,6 +158,7 @@ TEST(Ipv4InstanceTest, SocketAddress) { TEST(Ipv4InstanceTest, AddressOnly) { Ipv4Instance address("3.4.5.6"); EXPECT_EQ("3.4.5.6:0", address.asString()); + EXPECT_EQ("3.4.5.6:0", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("3.4.5.6", address.ip()->addressAsString()); EXPECT_EQ(0U, address.ip()->port()); @@ -168,6 +170,7 @@ TEST(Ipv4InstanceTest, AddressOnly) { TEST(Ipv4InstanceTest, AddressAndPort) { Ipv4Instance address("127.0.0.1", 80); EXPECT_EQ("127.0.0.1:80", address.asString()); + EXPECT_EQ("127.0.0.1:80", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("127.0.0.1", address.ip()->addressAsString()); EXPECT_FALSE(address.ip()->isAnyAddress()); @@ -180,6 +183,7 @@ TEST(Ipv4InstanceTest, AddressAndPort) { TEST(Ipv4InstanceTest, PortOnly) { Ipv4Instance address(443); EXPECT_EQ("0.0.0.0:443", address.asString()); + EXPECT_EQ("0.0.0.0:443", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("0.0.0.0", address.ip()->addressAsString()); EXPECT_TRUE(address.ip()->isAnyAddress()); @@ -192,6 +196,7 @@ TEST(Ipv4InstanceTest, PortOnly) { TEST(Ipv4InstanceTest, Multicast) { Ipv4Instance address("230.0.0.1"); EXPECT_EQ("230.0.0.1:0", address.asString()); + EXPECT_EQ("230.0.0.1:0", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("230.0.0.1", address.ip()->addressAsString()); EXPECT_FALSE(address.ip()->isAnyAddress()); @@ -204,6 +209,7 @@ TEST(Ipv4InstanceTest, Multicast) { TEST(Ipv4InstanceTest, Broadcast) { Ipv4Instance address("255.255.255.255"); EXPECT_EQ("255.255.255.255:0", address.asString()); + EXPECT_EQ("255.255.255.255:0", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("255.255.255.255", address.ip()->addressAsString()); EXPECT_EQ(0U, address.ip()->port()); @@ -225,6 +231,7 @@ TEST(Ipv6InstanceTest, SocketAddress) { Ipv6Instance address(addr6); EXPECT_EQ("[1:23::ef]:32000", address.asString()); + EXPECT_EQ("[1:23::ef]:32000", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("1:23::ef", address.ip()->addressAsString()); EXPECT_FALSE(address.ip()->isAnyAddress()); @@ -238,6 +245,7 @@ TEST(Ipv6InstanceTest, SocketAddress) { TEST(Ipv6InstanceTest, AddressOnly) { Ipv6Instance address("2001:0db8:85a3:0000:0000:8a2e:0370:7334"); EXPECT_EQ("[2001:db8:85a3::8a2e:370:7334]:0", address.asString()); + EXPECT_EQ("[2001:db8:85a3::8a2e:370:7334]:0", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("2001:db8:85a3::8a2e:370:7334", address.ip()->addressAsString()); EXPECT_EQ(0U, address.ip()->port()); @@ -250,6 +258,7 @@ TEST(Ipv6InstanceTest, AddressOnly) { TEST(Ipv6InstanceTest, AddressAndPort) { Ipv6Instance address("::0001", 80); EXPECT_EQ("[::1]:80", address.asString()); + EXPECT_EQ("[::1]:80", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("::1", address.ip()->addressAsString()); EXPECT_EQ(80U, address.ip()->port()); @@ -261,6 +270,7 @@ TEST(Ipv6InstanceTest, AddressAndPort) { TEST(Ipv6InstanceTest, PortOnly) { Ipv6Instance address(443); EXPECT_EQ("[::]:443", address.asString()); + EXPECT_EQ("[::]:443", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("::", address.ip()->addressAsString()); EXPECT_TRUE(address.ip()->isAnyAddress()); @@ -273,6 +283,7 @@ TEST(Ipv6InstanceTest, PortOnly) { TEST(Ipv6InstanceTest, Multicast) { Ipv6Instance address("FF00::"); EXPECT_EQ("[ff00::]:0", address.asString()); + EXPECT_EQ("[ff00::]:0", address.asStringView()); EXPECT_EQ(Type::Ip, address.type()); EXPECT_EQ("ff00::", address.ip()->addressAsString()); EXPECT_FALSE(address.ip()->isAnyAddress()); @@ -311,6 +322,7 @@ TEST(PipeInstanceTest, AbstractNamespace) { #if defined(__linux__) PipeInstance address("@/foo"); EXPECT_EQ("@/foo", address.asString()); + EXPECT_EQ("@/foo", address.asStringView()); EXPECT_EQ(Type::Pipe, address.type()); EXPECT_EQ(nullptr, address.ip()); #else diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index 3b83558e378e..eaa925b9e06a 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -369,6 +369,7 @@ class CustomInstance : public Address::Instance { return asString() == rhs.asString(); } const std::string& asString() const override { return antagonistic_name_; } + absl::string_view asStringView() const override { return antagonistic_name_; } const std::string& logicalName() const override { return antagonistic_name_; } Api::SysCallIntResult bind(int fd) const override { return instance_.bind(fd); } Api::SysCallIntResult connect(int fd) const override { return instance_.connect(fd); } diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 00f9b902386b..8d6401fd44ea 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -376,6 +376,7 @@ class MockResolvedAddress : public Address::Instance { MOCK_CONST_METHOD0(sockAddrLen, socklen_t()); const std::string& asString() const override { return physical_; } + absl::string_view asStringView() const override { return physical_; } const std::string& logicalName() const override { return logical_; } const std::string logical_; From cd474639b51b182733a47ce8b98fa321352c5744 Mon Sep 17 00:00:00 2001 From: Caleb Gilmour Date: Sat, 27 Jul 2019 03:55:33 +1200 Subject: [PATCH 249/542] Update Datadog tracer version to v1.0.1 (#7701) Signed-off-by: Caleb Gilmour --- bazel/repository_locations.bzl | 6 +++--- source/extensions/tracers/datadog/datadog_tracer_impl.cc | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 1ff6299a65ba..7c36dec104f3 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -120,9 +120,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/lightstep/lightstep-tracer-cpp/archive/v0.8.0.tar.gz"], ), com_github_datadog_dd_opentracing_cpp = dict( - sha256 = "a3d1c03e7af570fa64c01df259e6e9bb78637a6bd9c65c6bf7e8703e466dc22f", - strip_prefix = "dd-opentracing-cpp-0.4.2", - urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v0.4.2.tar.gz"], + sha256 = "f7fb2ad541f812c36fd78f9a38e4582d87dadb563ab80bee3f7c3a2132a425c5", + strip_prefix = "dd-opentracing-cpp-1.0.1", + urls = ["https://github.com/DataDog/dd-opentracing-cpp/archive/v1.0.1.tar.gz"], ), com_github_google_benchmark = dict( sha256 = "3c6a165b6ecc948967a1ead710d4a181d7b0fbcaa183ef7ea84604994966221a", diff --git a/source/extensions/tracers/datadog/datadog_tracer_impl.cc b/source/extensions/tracers/datadog/datadog_tracer_impl.cc index 660c3add42a8..a2117dc4749d 100644 --- a/source/extensions/tracers/datadog/datadog_tracer_impl.cc +++ b/source/extensions/tracers/datadog/datadog_tracer_impl.cc @@ -80,8 +80,8 @@ void TraceReporter::enableTimer() { void TraceReporter::flushTraces() { auto pendingTraces = encoder_->pendingTraces(); - ENVOY_LOG(debug, "flushing traces: {} traces", pendingTraces); if (pendingTraces) { + ENVOY_LOG(debug, "flushing traces: {} traces", pendingTraces); driver_.tracerStats().traces_sent_.add(pendingTraces); Http::MessagePtr message(new Http::RequestMessageImpl()); @@ -95,7 +95,7 @@ void TraceReporter::flushTraces() { Buffer::InstancePtr body(new Buffer::OwnedImpl()); body->add(encoder_->payload()); message->body() = std::move(body); - ENVOY_LOG(debug, "submitting {} trace(s) to {} with payload {}", pendingTraces, + ENVOY_LOG(debug, "submitting {} trace(s) to {} with payload size {}", pendingTraces, encoder_->path(), encoder_->payload().size()); driver_.clusterManager() From ca4185536dd9284020d918c74e5cca9244570fb8 Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Fri, 26 Jul 2019 18:58:54 +0300 Subject: [PATCH 250/542] tests: fix an assert in codec_impl_fuzz_test. (#7734) Encoding the headers may lead to stream reset, which leads to an assert when callbacks are added. Add the callbacks before encoding to prevent this. Signed-off-by: Ismo Puustinen --- ...uzz-testcase-minimized-codec_impl_fuzz_test-5107763548520448 | 1 + test/common/http/codec_impl_fuzz_test.cc | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 test/common/http/codec_impl_corpus/clusterfuzz-testcase-minimized-codec_impl_fuzz_test-5107763548520448 diff --git a/test/common/http/codec_impl_corpus/clusterfuzz-testcase-minimized-codec_impl_fuzz_test-5107763548520448 b/test/common/http/codec_impl_corpus/clusterfuzz-testcase-minimized-codec_impl_fuzz_test-5107763548520448 new file mode 100644 index 000000000000..f45f1e08127d --- /dev/null +++ b/test/common/http/codec_impl_corpus/clusterfuzz-testcase-minimized-codec_impl_fuzz_test-5107763548520448 @@ -0,0 +1 @@ +h2_settings { server { max_concurrent_streams: 3 initial_connection_window_size: 1 } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { client_drain { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { mutate { buffer: 1 offset: 1 value: 1 server: true } } actions { mutate { buffer: 1 offset: 3 value: 7 server: true } } actions { new_stream { } } actions { new_stream { } } actions { new_stream { } } actions { quiesce_drain { } } actions { new_stream { } } \ No newline at end of file diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 7d51f36665e9..4a56733f710f 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -150,10 +150,10 @@ class HttpStream : public LinkedObject { ON_CALL(response_.decoder_, decodeTrailers_(_)).WillByDefault(InvokeWithoutArgs([this] { response_.closeRemote(); })); - request_.encoder_->encodeHeaders(request_headers, end_stream); if (!end_stream) { request_.encoder_->getStream().addCallbacks(request_.stream_callbacks_); } + request_.encoder_->encodeHeaders(request_headers, end_stream); request_.stream_state_ = end_stream ? StreamState::Closed : StreamState::PendingDataOrTrailers; response_.stream_state_ = StreamState::PendingHeaders; } From a8200bce9a5ec36d089cdedd9f14c85abeef5c71 Mon Sep 17 00:00:00 2001 From: danzh Date: Fri, 26 Jul 2019 12:53:41 -0400 Subject: [PATCH 251/542] quick: switch to use Event::Dispatcher (#7711) Signed-off-by: Dan Zhang --- source/extensions/quic_listeners/quiche/BUILD | 1 + .../quic_listeners/quiche/envoy_quic_alarm.cc | 6 +++--- .../quic_listeners/quiche/envoy_quic_alarm.h | 7 ++++--- .../quiche/envoy_quic_alarm_factory.cc | 6 +++--- .../quiche/envoy_quic_alarm_factory.h | 8 ++++---- .../quic_listeners/quiche/platform/BUILD | 2 +- .../quiche/platform/envoy_quic_clock.cc | 6 +++--- .../quiche/platform/envoy_quic_clock.h | 6 +++--- test/extensions/quic_listeners/quiche/BUILD | 1 + .../quiche/envoy_quic_alarm_test.cc | 19 +++++++++---------- .../quic_listeners/quiche/platform/BUILD | 1 + .../quiche/platform/envoy_quic_clock_test.cc | 9 +++++++-- 12 files changed, 40 insertions(+), 32 deletions(-) diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 93b9f31ff664..b4b3369fe9bf 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -15,6 +15,7 @@ envoy_cc_library( external_deps = ["quiche_quic_platform"], tags = ["nofips"], deps = [ + "//include/envoy/event:dispatcher_interface", "//include/envoy/event:timer_interface", "@com_googlesource_quiche//:quic_core_alarm_interface_lib", ], diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm.cc b/source/extensions/quic_listeners/quiche/envoy_quic_alarm.cc index 695a525777ce..b490aff8b955 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_alarm.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm.cc @@ -3,10 +3,10 @@ namespace Envoy { namespace Quic { -EnvoyQuicAlarm::EnvoyQuicAlarm(Event::Scheduler& scheduler, quic::QuicClock& clock, +EnvoyQuicAlarm::EnvoyQuicAlarm(Event::Dispatcher& dispatcher, const quic::QuicClock& clock, quic::QuicArenaScopedPtr delegate) - : QuicAlarm(std::move(delegate)), scheduler_(scheduler), - timer_(scheduler_.createTimer([this]() { Fire(); })), clock_(clock) {} + : QuicAlarm(std::move(delegate)), dispatcher_(dispatcher), + timer_(dispatcher_.createTimer([this]() { Fire(); })), clock_(clock) {} void EnvoyQuicAlarm::CancelImpl() { timer_->disableTimer(); } diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm.h b/source/extensions/quic_listeners/quiche/envoy_quic_alarm.h index e50b4afc05ce..4152f4c101c3 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_alarm.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/event/dispatcher.h" #include "envoy/event/timer.h" #include "common/common/assert.h" @@ -16,7 +17,7 @@ namespace Quic { // wraps an Event::Timer object and provide interface for QUIC to interact with the timer. class EnvoyQuicAlarm : public quic::QuicAlarm { public: - EnvoyQuicAlarm(Event::Scheduler& scheduler, quic::QuicClock& clock, + EnvoyQuicAlarm(Event::Dispatcher& dispatcher, const quic::QuicClock& clock, quic::QuicArenaScopedPtr delegate); ~EnvoyQuicAlarm() override { ASSERT(!IsSet()); }; @@ -30,9 +31,9 @@ class EnvoyQuicAlarm : public quic::QuicAlarm { private: quic::QuicTime::Delta getDurationBeforeDeadline(); - Event::Scheduler& scheduler_; + Event::Dispatcher& dispatcher_; Event::TimerPtr timer_; - quic::QuicClock& clock_; + const quic::QuicClock& clock_; }; } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.cc b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.cc index 5595c9c308e9..cdea1ed16ff6 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.cc @@ -4,7 +4,7 @@ namespace Envoy { namespace Quic { quic::QuicAlarm* EnvoyQuicAlarmFactory::CreateAlarm(quic::QuicAlarm::Delegate* delegate) { - return new EnvoyQuicAlarm(scheduler_, clock_, + return new EnvoyQuicAlarm(dispatcher_, clock_, quic::QuicArenaScopedPtr(delegate)); } @@ -12,10 +12,10 @@ quic::QuicArenaScopedPtr EnvoyQuicAlarmFactory::CreateAlarm(quic::QuicArenaScopedPtr delegate, quic::QuicConnectionArena* arena) { if (arena != nullptr) { - return arena->New(scheduler_, clock_, std::move(delegate)); + return arena->New(dispatcher_, clock_, std::move(delegate)); } return quic::QuicArenaScopedPtr( - new EnvoyQuicAlarm(scheduler_, clock_, std::move(delegate))); + new EnvoyQuicAlarm(dispatcher_, clock_, std::move(delegate))); } } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h index f70b953fab71..c373ed42298f 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h @@ -19,8 +19,8 @@ namespace Quic { class EnvoyQuicAlarmFactory : public quic::QuicAlarmFactory, NonCopyable { public: - EnvoyQuicAlarmFactory(Event::Scheduler& scheduler, quic::QuicClock& clock) - : scheduler_(scheduler), clock_(clock) {} + EnvoyQuicAlarmFactory(Event::Dispatcher& dispatcher, const quic::QuicClock& clock) + : dispatcher_(dispatcher), clock_(clock) {} ~EnvoyQuicAlarmFactory() override = default; @@ -31,8 +31,8 @@ class EnvoyQuicAlarmFactory : public quic::QuicAlarmFactory, NonCopyable { quic::QuicConnectionArena* arena) override; private: - Event::Scheduler& scheduler_; - quic::QuicClock& clock_; + Event::Dispatcher& dispatcher_; + const quic::QuicClock& clock_; }; } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD index e30ad336b9e8..08c42a1364a8 100644 --- a/source/extensions/quic_listeners/quiche/platform/BUILD +++ b/source/extensions/quic_listeners/quiche/platform/BUILD @@ -243,7 +243,7 @@ envoy_cc_library( hdrs = ["envoy_quic_clock.h"], visibility = ["//visibility:public"], deps = [ - "//include/envoy/event:timer_interface", + "//include/envoy/event:dispatcher_interface", "@com_googlesource_quiche//:quic_platform", ], ) diff --git a/source/extensions/quic_listeners/quiche/platform/envoy_quic_clock.cc b/source/extensions/quic_listeners/quiche/platform/envoy_quic_clock.cc index b582956751da..b134827cc38d 100644 --- a/source/extensions/quic_listeners/quiche/platform/envoy_quic_clock.cc +++ b/source/extensions/quic_listeners/quiche/platform/envoy_quic_clock.cc @@ -9,13 +9,13 @@ quic::QuicTime EnvoyQuicClock::ApproximateNow() const { } quic::QuicTime EnvoyQuicClock::Now() const { - return quic::QuicTime::Zero() + quic::QuicTime::Delta::FromMicroseconds( - microsecondsSinceEpoch(time_system_.monotonicTime())); + return quic::QuicTime::Zero() + quic::QuicTime::Delta::FromMicroseconds(microsecondsSinceEpoch( + dispatcher_.timeSource().monotonicTime())); } quic::QuicWallTime EnvoyQuicClock::WallNow() const { return quic::QuicWallTime::FromUNIXMicroseconds( - microsecondsSinceEpoch(time_system_.systemTime())); + microsecondsSinceEpoch(dispatcher_.timeSource().systemTime())); } } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/platform/envoy_quic_clock.h b/source/extensions/quic_listeners/quiche/platform/envoy_quic_clock.h index eb90a1db1b94..5a85ebf932cc 100644 --- a/source/extensions/quic_listeners/quiche/platform/envoy_quic_clock.h +++ b/source/extensions/quic_listeners/quiche/platform/envoy_quic_clock.h @@ -2,7 +2,7 @@ #include -#include "envoy/event/timer.h" +#include "envoy/event/dispatcher.h" #include "quiche/quic/platform/api/quic_clock.h" @@ -11,7 +11,7 @@ namespace Quic { class EnvoyQuicClock : public quic::QuicClock { public: - EnvoyQuicClock(Event::TimeSystem& time_system) : time_system_(time_system) {} + EnvoyQuicClock(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} // quic::QuicClock quic::QuicTime ApproximateNow() const override; @@ -23,7 +23,7 @@ class EnvoyQuicClock : public quic::QuicClock { return std::chrono::duration_cast(time.time_since_epoch()).count(); } - Event::TimeSystem& time_system_; + Event::Dispatcher& dispatcher_; }; } // namespace Quic diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index 6569b0d90844..313ef1d0e808 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -22,6 +22,7 @@ envoy_cc_test( "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_lib", "//source/extensions/quic_listeners/quiche/platform:envoy_quic_clock_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc index 6dc18d378150..d2597b2e996b 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc @@ -1,10 +1,9 @@ -#include "common/event/libevent_scheduler.h" - #include "extensions/quic_listeners/quiche/envoy_quic_alarm.h" #include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" #include "extensions/quic_listeners/quiche/platform/envoy_quic_clock.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -32,19 +31,19 @@ class TestDelegate : public quic::QuicAlarm::Delegate { class EnvoyQuicAlarmTest : public ::testing::Test { public: EnvoyQuicAlarmTest() - : clock_(time_system_), scheduler_(time_system_.createScheduler(base_scheduler_)), - alarm_factory_(*scheduler_, clock_) {} + : api_(Api::createApiForTest(time_system_)), dispatcher_(api_->allocateDispatcher()), + clock_(*dispatcher_), alarm_factory_(*dispatcher_, clock_) {} void advanceMsAndLoop(int64_t delay_ms) { time_system_.sleep(std::chrono::milliseconds(delay_ms)); - base_scheduler_.run(Dispatcher::RunType::NonBlock); + dispatcher_->run(Dispatcher::RunType::NonBlock); } protected: Event::SimulatedTimeSystemHelper time_system_; + Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; EnvoyQuicClock clock_; - Event::LibeventScheduler base_scheduler_; - Event::SchedulerPtr scheduler_; EnvoyQuicAlarmFactory alarm_factory_; quic::QuicConnectionArena arena_; }; @@ -157,7 +156,7 @@ TEST_F(EnvoyQuicAlarmTest, SetAlarmToPastTime) { // alarm becomes active upon Set(). alarm->Set(clock_.Now() - QuicTime::Delta::FromMilliseconds(10)); EXPECT_FALSE(unowned_delegate->fired()); - base_scheduler_.run(Dispatcher::RunType::NonBlock); + dispatcher_->run(Dispatcher::RunType::NonBlock); EXPECT_TRUE(unowned_delegate->fired()); } @@ -170,7 +169,7 @@ TEST_F(EnvoyQuicAlarmTest, UpdateAlarmWithPastDeadline) { EXPECT_FALSE(unowned_delegate->fired()); // alarm becomes active upon Update(). alarm->Update(clock_.Now() - QuicTime::Delta::FromMilliseconds(1), quic::QuicTime::Delta::Zero()); - base_scheduler_.run(Dispatcher::RunType::NonBlock); + dispatcher_->run(Dispatcher::RunType::NonBlock); EXPECT_TRUE(unowned_delegate->fired()); unowned_delegate->set_fired(false); advanceMsAndLoop(1); @@ -186,7 +185,7 @@ TEST_F(EnvoyQuicAlarmTest, CancelActiveAlarm) { // alarm becomes active upon Set(). alarm->Set(clock_.Now() - QuicTime::Delta::FromMilliseconds(10)); alarm->Cancel(); - base_scheduler_.run(Dispatcher::RunType::NonBlock); + dispatcher_->run(Dispatcher::RunType::NonBlock); EXPECT_FALSE(unowned_delegate->fired()); } diff --git a/test/extensions/quic_listeners/quiche/platform/BUILD b/test/extensions/quic_listeners/quiche/platform/BUILD index 911943c915ce..1383cfbfbb17 100644 --- a/test/extensions/quic_listeners/quiche/platform/BUILD +++ b/test/extensions/quic_listeners/quiche/platform/BUILD @@ -221,5 +221,6 @@ envoy_cc_test( "//source/extensions/quic_listeners/quiche/platform:envoy_quic_clock_lib", "//test/test_common:simulated_time_system_lib", "//test/test_common:test_time_lib", + "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/quic_listeners/quiche/platform/envoy_quic_clock_test.cc b/test/extensions/quic_listeners/quiche/platform/envoy_quic_clock_test.cc index 8bdf8a987512..2a14f28b5217 100644 --- a/test/extensions/quic_listeners/quiche/platform/envoy_quic_clock_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/envoy_quic_clock_test.cc @@ -4,6 +4,7 @@ #include "test/test_common/simulated_time_system.h" #include "test/test_common/test_time.h" +#include "test/test_common/utility.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -13,7 +14,9 @@ namespace Quic { TEST(EnvoyQuicClockTest, TestNow) { Event::SimulatedTimeSystemHelper time_system; - EnvoyQuicClock clock(time_system); + Api::ApiPtr api = Api::createApiForTest(time_system); + Event::DispatcherPtr dispatcher = api->allocateDispatcher(); + EnvoyQuicClock clock(*dispatcher); uint64_t mono_time = std::chrono::duration_cast( time_system.monotonicTime().time_since_epoch()) .count(); @@ -41,7 +44,9 @@ TEST(EnvoyQuicClockTest, TestNow) { // Tests that Now() should never go back. TEST(EnvoyQuicClockTest, TestMonotonicityWithReadTimeSystem) { Event::TestRealTimeSystem time_system; - EnvoyQuicClock clock(time_system); + Api::ApiPtr api = Api::createApiForTest(time_system); + Event::DispatcherPtr dispatcher = api->allocateDispatcher(); + EnvoyQuicClock clock(*dispatcher); quic::QuicTime last_now = clock.Now(); for (int i = 0; i < 1000; ++i) { quic::QuicTime now = clock.Now(); From 27b41b888d5ba624f2a6098c8b9f395e8ef95cbf Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Fri, 26 Jul 2019 12:54:57 -0400 Subject: [PATCH 252/542] templates: add warning about reporting security issues. (#7735) While there, move SECURITY_RELEASE_PROCESS.md to SECURITY.md to make it work with GitHub's Security tab. Signed-off-by: Piotr Sikora --- GOVERNANCE.md | 8 ++++---- ISSUE_TEMPLATE.md | 4 ++++ README.md | 2 +- SECURITY_RELEASE_PROCESS.md => SECURITY.md | 0 security/email-templates.md | 4 ++-- 5 files changed, 11 insertions(+), 7 deletions(-) rename SECURITY_RELEASE_PROCESS.md => SECURITY.md (100%) diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 737240ad62b6..2408510145ce 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -52,10 +52,10 @@ can be promoted to other issue types once it's clear they are actionable (at which point the question label should be removed). * Make sure that ongoing PRs are moving forward at the right pace or closing them. -* Participate when called upon in the [security release process](SECURITY_RELEASE_PROCESS.md). Note - that although this should be a rare occurrence, if a serious vulnerability is found, the process - may take up to several full days of work to implement. This reality should be taken into account - when discussing time commitment obligations with employers. +* Participate when called upon in the [security release process](SECURITY.md). Note that although + this should be a rare occurrence, if a serious vulnerability is found, the process may take up to + several full days of work to implement. This reality should be taken into account when discussing + time commitment obligations with employers. * In general continue to be willing to spend at least 25% of ones time working on Envoy (~1.25 business days per week). * We currently maintain an "on-call" rotation within the maintainers. Each on-call is 1 week. diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 342f2df00dfe..8d87611c68e8 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,3 +1,7 @@ +**WARNING: If you want to report crashes, leaking of sensitive information, +and/or other security issues, please consider +[reporting them using appropriate channels](https://github.com/envoyproxy/envoy#reporting-security-vulnerabilities).** + **Issue Template** *Title*: *One line description* diff --git a/README.md b/README.md index 71b468dca47b..abe4e9822462 100644 --- a/README.md +++ b/README.md @@ -85,4 +85,4 @@ If you've found a vulnerability or a potential vulnerability in Envoy please let email to acknowledge your report, and we'll send an additional email when we've identified the issue positively or negatively. -For further details please see our complete [security release process](SECURITY_RELEASE_PROCESS.md). +For further details please see our complete [security release process](SECURITY.md). diff --git a/SECURITY_RELEASE_PROCESS.md b/SECURITY.md similarity index 100% rename from SECURITY_RELEASE_PROCESS.md rename to SECURITY.md diff --git a/security/email-templates.md b/security/email-templates.md index 99175d1f64e8..b4281894e26f 100644 --- a/security/email-templates.md +++ b/security/email-templates.md @@ -36,14 +36,14 @@ Hello Envoy Distributors, The Envoy security team would like to provide advanced notice to the Envoy Private Distributors List of some details on the pending Envoy $VERSION security release, following the process described at -https://github.com/envoyproxy/envoy/blob/master/SECURITY_RELEASE_PROCESS.md. +https://github.com/envoyproxy/envoy/blob/master/SECURITY.md. This release will be made available on the $ORDINALDAY of $MONTH $YEAR at $PDTHOUR PDT ($GMTHOUR GMT). This release will fix $NUMDEFECTS security defect(s). The highest rated security defect is considered $SEVERITY severity. Below we provide details of these vulnerabilities under our embargo policy -(https://github.com/envoyproxy/envoy/blob/master/SECURITY_RELEASE_PROCESS.md#embargo-policy). +(https://github.com/envoyproxy/envoy/blob/master/SECURITY.md#embargo-policy). This information should be treated as confidential until public release by the Envoy maintainers on the Envoy GitHub. From 9ea7ea21ac8212726f452810f1e8ad855a83c022 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Fri, 26 Jul 2019 17:32:21 -0400 Subject: [PATCH 253/542] http: moving strict header checking into the http/1.1 codec (#7601) Signed-off-by: Alyssa Wilk --- source/common/http/codec_client.cc | 5 ++- source/common/http/codec_client.h | 3 +- source/common/http/conn_manager_config.h | 6 +--- source/common/http/conn_manager_impl.cc | 6 ++-- source/common/http/conn_manager_impl.h | 2 -- source/common/http/conn_manager_utility.cc | 7 ++-- source/common/http/conn_manager_utility.h | 2 +- source/common/http/http1/BUILD | 1 + source/common/http/http1/codec_impl.cc | 34 ++++--------------- source/common/http/http1/codec_impl.h | 5 --- source/common/http/http1/conn_pool.cc | 7 +--- source/common/http/http2/conn_pool.cc | 2 +- source/common/upstream/health_checker_impl.cc | 6 ++-- .../network/http_connection_manager/config.cc | 15 ++++---- .../network/http_connection_manager/config.h | 3 +- source/server/http/admin.cc | 5 ++- source/server/http/admin.h | 3 +- .../http/conn_manager_impl_fuzz_test.cc | 2 +- test/common/http/conn_manager_impl_test.cc | 2 +- test/common/http/conn_manager_utility_test.cc | 10 +++--- test/common/http/http1/codec_impl_test.cc | 26 ++++---------- test/integration/BUILD | 1 + test/integration/http_integration.cc | 4 +-- test/integration/utility.cc | 2 +- 24 files changed, 51 insertions(+), 108 deletions(-) diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index fbfd421b0220..6c5488ef7883 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -136,12 +136,11 @@ void CodecClient::onData(Buffer::Instance& data) { CodecClientProd::CodecClientProd(Type type, Network::ClientConnectionPtr&& connection, Upstream::HostDescriptionConstSharedPtr host, - Event::Dispatcher& dispatcher, bool strict_header_validation) + Event::Dispatcher& dispatcher) : CodecClient(type, std::move(connection), host, dispatcher) { switch (type) { case Type::HTTP1: { - codec_ = std::make_unique(*connection_, *this, - strict_header_validation); + codec_ = std::make_unique(*connection_, *this); break; } case Type::HTTP2: { diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index cc27050e8fbe..b9cf3482dce5 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -241,8 +241,7 @@ using CodecClientPtr = std::unique_ptr; class CodecClientProd : public CodecClient { public: CodecClientProd(Type type, Network::ClientConnectionPtr&& connection, - Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher, - bool strict_header_validation); + Upstream::HostDescriptionConstSharedPtr host, Event::Dispatcher& dispatcher); }; } // namespace Http diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 78ebc7a19161..5e08e267d5b9 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -184,15 +184,11 @@ class ConnectionManagerConfig { * @param connection supplies the owning connection. * @param data supplies the currently available read data. * @param callbacks supplies the callbacks to install into the codec. - * @param strict_header_validation indicates whether or not the codec should validate the values - * of each HTTP header (NOTE: this argument only affects the H/1.1 codec; the H/2 codec always - * does this) * @return a codec or nullptr if no codec can be created. */ virtual ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - ServerConnectionCallbacks& callbacks, - const bool strict_header_validation) PURE; + ServerConnectionCallbacks& callbacks) PURE; /** * @return DateProvider& the date provider to use for diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 998642cd2cd6..1e9bbbfedc58 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -112,8 +112,7 @@ ConnectionManagerImpl::ConnectionManagerImpl(ConnectionManagerConfig& config, overload_manager ? overload_manager->getThreadLocalOverloadState().getState( Server::OverloadActionNames::get().DisableHttpKeepAlive) : Server::OverloadManager::getInactiveState()), - time_source_(time_source), strict_header_validation_(Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.strict_header_validation")) {} + time_source_(time_source) {} const HeaderMapImpl& ConnectionManagerImpl::continueHeader() { CONSTRUCT_ON_FIRST_USE(HeaderMapImpl, @@ -261,8 +260,7 @@ StreamDecoder& ConnectionManagerImpl::newStream(StreamEncoder& response_encoder, Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) { if (!codec_) { - codec_ = - config_.createCodec(read_callbacks_->connection(), data, *this, strict_header_validation_); + codec_ = config_.createCodec(read_callbacks_->connection(), data, *this); if (codec_->protocol() == Protocol::Http2) { stats_.named_.downstream_cx_http2_total_.inc(); stats_.named_.downstream_cx_http2_active_.inc(); diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 7b3a650d26dd..a72dab9069cd 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -679,8 +679,6 @@ class ConnectionManagerImpl : Logger::Loggable, const Server::OverloadActionState& overload_stop_accepting_requests_ref_; const Server::OverloadActionState& overload_disable_keepalive_ref_; TimeSource& time_source_; - - const bool strict_header_validation_; }; } // namespace Http diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index 6ec1bd5b9116..acc258499e56 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -41,14 +41,13 @@ std::string ConnectionManagerUtility::determineNextProtocol(Network::Connection& ServerConnectionPtr ConnectionManagerUtility::autoCreateCodec( Network::Connection& connection, const Buffer::Instance& data, ServerConnectionCallbacks& callbacks, Stats::Scope& scope, const Http1Settings& http1_settings, - const Http2Settings& http2_settings, const uint32_t max_request_headers_kb, - bool strict_header_validation) { + const Http2Settings& http2_settings, const uint32_t max_request_headers_kb) { if (determineNextProtocol(connection, data) == Http2::ALPN_STRING) { return std::make_unique(connection, callbacks, scope, http2_settings, max_request_headers_kb); } else { - return std::make_unique( - connection, callbacks, http1_settings, max_request_headers_kb, strict_header_validation); + return std::make_unique(connection, callbacks, http1_settings, + max_request_headers_kb); } } diff --git a/source/common/http/conn_manager_utility.h b/source/common/http/conn_manager_utility.h index c203fb154717..802d0ef07add 100644 --- a/source/common/http/conn_manager_utility.h +++ b/source/common/http/conn_manager_utility.h @@ -37,7 +37,7 @@ class ConnectionManagerUtility { autoCreateCodec(Network::Connection& connection, const Buffer::Instance& data, ServerConnectionCallbacks& callbacks, Stats::Scope& scope, const Http1Settings& http1_settings, const Http2Settings& http2_settings, - const uint32_t max_request_headers_kb, bool strict_header_validation); + const uint32_t max_request_headers_kb); /** * Mutates request headers in various ways. This functionality is broken out because of its diff --git a/source/common/http/http1/BUILD b/source/common/http/http1/BUILD index 22f47ba2f31f..98ddb6edb484 100644 --- a/source/common/http/http1/BUILD +++ b/source/common/http/http1/BUILD @@ -30,6 +30,7 @@ envoy_cc_library( "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", + "//source/common/runtime:runtime_lib", ], ) diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 863606b32c6c..255de29a6a1e 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -16,6 +16,7 @@ #include "common/http/header_utility.h" #include "common/http/headers.h" #include "common/http/utility.h" +#include "common/runtime/runtime_impl.h" namespace Envoy { namespace Http { @@ -320,20 +321,12 @@ const ToLowerTable& ConnectionImpl::toLowerTable() { return *table; } -// TODO(alyssawilk) The overloaded constructors here and on {Client,Server}ConnectionImpl -// can be cleaned up once "strict_header_validation" becomes the default behavior, rather -// than a runtime-guarded one. The overloads were a workaround for the fact that Runtime -// doesn't work from integration test call sites in scenarios where the required -// thread-local storage is not available. ConnectionImpl::ConnectionImpl(Network::Connection& connection, http_parser_type type, uint32_t max_headers_kb) - : ConnectionImpl::ConnectionImpl(connection, type, max_headers_kb, false) {} - -ConnectionImpl::ConnectionImpl(Network::Connection& connection, http_parser_type type, - uint32_t max_headers_kb, bool strict_header_validation) : connection_(connection), output_buffer_([&]() -> void { this->onBelowLowWatermark(); }, [&]() -> void { this->onAboveHighWatermark(); }), - max_headers_kb_(max_headers_kb), strict_header_validation_(strict_header_validation) { + max_headers_kb_(max_headers_kb), strict_header_validation_(Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.strict_header_validation")) { output_buffer_.setWatermarks(connection.bufferLimit()); http_parser_init(&parser_, type); parser_.data = this; @@ -516,15 +509,8 @@ void ConnectionImpl::onResetStreamBase(StreamResetReason reason) { ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, ServerConnectionCallbacks& callbacks, Http1Settings settings, uint32_t max_request_headers_kb) - : ServerConnectionImpl::ServerConnectionImpl(connection, callbacks, settings, - max_request_headers_kb, false) {} - -ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, - ServerConnectionCallbacks& callbacks, - Http1Settings settings, uint32_t max_request_headers_kb, - bool strict_header_validation) - : ConnectionImpl(connection, HTTP_REQUEST, max_request_headers_kb, strict_header_validation), - callbacks_(callbacks), codec_settings_(settings) {} + : ConnectionImpl(connection, HTTP_REQUEST, max_request_headers_kb), callbacks_(callbacks), + codec_settings_(settings) {} void ServerConnectionImpl::onEncodeComplete() { ASSERT(active_request_); @@ -695,14 +681,8 @@ void ServerConnectionImpl::onBelowLowWatermark() { } } -ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, - ConnectionCallbacks& callbacks) - : ClientConnectionImpl::ClientConnectionImpl(connection, callbacks, false) {} - -ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, ConnectionCallbacks&, - bool strict_header_validation) - : ConnectionImpl(connection, HTTP_RESPONSE, MAX_RESPONSE_HEADERS_KB, strict_header_validation) { -} +ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, ConnectionCallbacks&) + : ConnectionImpl(connection, HTTP_RESPONSE, MAX_RESPONSE_HEADERS_KB) {} bool ClientConnectionImpl::cannotHaveBody() { if ((!pending_responses_.empty() && pending_responses_.front().head_request_) || diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index a15bdbe913ff..c1a9fa5d7e93 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -174,8 +174,6 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable(Http::CodecClient::Type::HTTP2, std::move(data.connection_), - data.host_description_, dispatcher_, false); + data.host_description_, dispatcher_); } std::ostream& operator<<(std::ostream& out, HealthState state) { diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index f5596e83bfe0..04edfdba79c4 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -360,20 +360,21 @@ void HttpConnectionManagerConfig::processFilter( filter_factories.push_back(callback); } -Http::ServerConnectionPtr HttpConnectionManagerConfig::createCodec( - Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks, const bool strict_header_validation) { +Http::ServerConnectionPtr +HttpConnectionManagerConfig::createCodec(Network::Connection& connection, + const Buffer::Instance& data, + Http::ServerConnectionCallbacks& callbacks) { switch (codec_type_) { case CodecType::HTTP1: return std::make_unique( - connection, callbacks, http1_settings_, maxRequestHeadersKb(), strict_header_validation); + connection, callbacks, http1_settings_, maxRequestHeadersKb()); case CodecType::HTTP2: return std::make_unique( connection, callbacks, context_.scope(), http2_settings_, maxRequestHeadersKb()); case CodecType::AUTO: - return Http::ConnectionManagerUtility::autoCreateCodec( - connection, data, callbacks, context_.scope(), http1_settings_, http2_settings_, - maxRequestHeadersKb(), strict_header_validation); + return Http::ConnectionManagerUtility::autoCreateCodec(connection, data, callbacks, + context_.scope(), http1_settings_, + http2_settings_, maxRequestHeadersKb()); } NOT_REACHED_GCOVR_EXCL_LINE; diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index eb97a9bf21db..796b67fc9728 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -99,8 +99,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, const std::list& accessLogs() override { return access_logs_; } Http::ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks, - const bool strict_header_validation) override; + Http::ServerConnectionCallbacks& callbacks) override; Http::DateProvider& dateProvider() override { return date_provider_; } std::chrono::milliseconds drainTimeout() override { return drain_timeout_; } FilterChainFactory& filterFactory() override { return *this; } diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 17e655af186f..b7e8a22cebbb 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -1242,11 +1242,10 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server) Http::ServerConnectionPtr AdminImpl::createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks, - const bool strict_header_validation) { + Http::ServerConnectionCallbacks& callbacks) { return Http::ConnectionManagerUtility::autoCreateCodec( connection, data, callbacks, server_.stats(), Http::Http1Settings(), Http::Http2Settings(), - maxRequestHeadersKb(), strict_header_validation); + maxRequestHeadersKb()); } bool AdminImpl::createNetworkFilterChain(Network::Connection& connection, diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 9d1da1e8700d..ac0f6af807bb 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -105,8 +105,7 @@ class AdminImpl : public Admin, const std::list& accessLogs() override { return access_logs_; } Http::ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& data, - Http::ServerConnectionCallbacks& callbacks, - const bool strict_header_validation) override; + Http::ServerConnectionCallbacks& callbacks) override; Http::DateProvider& dateProvider() override { return date_provider_; } std::chrono::milliseconds drainTimeout() override { return std::chrono::milliseconds(100); } Http::FilterChainFactory& filterFactory() override { return *this; } diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index e15e88e97473..3c11424b5964 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -72,7 +72,7 @@ class FuzzConfig : public ConnectionManagerConfig { // Http::ConnectionManagerConfig const std::list& accessLogs() override { return access_logs_; } ServerConnectionPtr createCodec(Network::Connection&, const Buffer::Instance&, - ServerConnectionCallbacks&, const bool) override { + ServerConnectionCallbacks&) override { return ServerConnectionPtr{codec_}; } DateProvider& dateProvider() override { return date_provider_; } diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 7f2abde9ba58..c635e59c6928 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -241,7 +241,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan // Http::ConnectionManagerConfig const std::list& accessLogs() override { return access_logs_; } ServerConnectionPtr createCodec(Network::Connection&, const Buffer::Instance&, - ServerConnectionCallbacks&, const bool) override { + ServerConnectionCallbacks&) override { return ServerConnectionPtr{codec_}; } DateProvider& dateProvider() override { return date_provider_; } diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 48fa93d14174..46937cf17ab0 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -43,15 +43,13 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { // Http::ConnectionManagerConfig ServerConnectionPtr createCodec(Network::Connection& connection, const Buffer::Instance& instance, - ServerConnectionCallbacks& callbacks, - const bool strict_header_validation) override { - return ServerConnectionPtr{ - createCodec_(connection, instance, callbacks, strict_header_validation)}; + ServerConnectionCallbacks& callbacks) override { + return ServerConnectionPtr{createCodec_(connection, instance, callbacks)}; } MOCK_METHOD0(accessLogs, const std::list&()); - MOCK_METHOD4(createCodec_, ServerConnection*(Network::Connection&, const Buffer::Instance&, - ServerConnectionCallbacks&, const bool)); + MOCK_METHOD3(createCodec_, ServerConnection*(Network::Connection&, const Buffer::Instance&, + ServerConnectionCallbacks&)); MOCK_METHOD0(dateProvider, DateProvider&()); MOCK_METHOD0(drainTimeout, std::chrono::milliseconds()); MOCK_METHOD0(filterFactory, FilterChainFactory&()); diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index a4ceb41c5836..fec552d6f3f8 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -49,11 +49,8 @@ class Http1ServerConnectionImplTest : public testing::Test { } void initialize() { - strict_header_validation_ = - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.strict_header_validation"); - codec_ = - std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_, strict_header_validation_); + codec_ = std::make_unique(connection_, callbacks_, codec_settings_, + max_request_headers_kb_); } NiceMock connection_; @@ -67,7 +64,6 @@ class Http1ServerConnectionImplTest : public testing::Test { protected: uint32_t max_request_headers_kb_{Http::DEFAULT_MAX_REQUEST_HEADERS_KB}; - bool strict_header_validation_; Event::MockDispatcher dispatcher_; NiceMock tls_; Stats::IsolatedStoreImpl store_; @@ -88,9 +84,8 @@ void Http1ServerConnectionImplTest::expect400(Protocol p, bool allow_absolute_ur if (allow_absolute_url) { codec_settings_.allow_absolute_url_ = allow_absolute_url; - codec_ = - std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_, strict_header_validation_); + codec_ = std::make_unique(connection_, callbacks_, codec_settings_, + max_request_headers_kb_); } Http::MockStreamDecoder decoder; @@ -109,9 +104,8 @@ void Http1ServerConnectionImplTest::expectHeadersTest(Protocol p, bool allow_abs // Make a new 'codec' with the right settings if (allow_absolute_url) { codec_settings_.allow_absolute_url_ = allow_absolute_url; - codec_ = - std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_, strict_header_validation_); + codec_ = std::make_unique(connection_, callbacks_, codec_settings_, + max_request_headers_kb_); } Http::MockStreamDecoder decoder; @@ -866,12 +860,7 @@ class Http1ClientConnectionImplTest : public testing::Test { generator_, validation_visitor_, *api_)}); } - void initialize() { - strict_header_validation_ = - Runtime::runtimeFeatureEnabled("envoy.reloadable_features.strict_header_validation"); - codec_ = - std::make_unique(connection_, callbacks_, strict_header_validation_); - } + void initialize() { codec_ = std::make_unique(connection_, callbacks_); } NiceMock connection_; NiceMock callbacks_; @@ -887,7 +876,6 @@ class Http1ClientConnectionImplTest : public testing::Test { Init::MockManager init_manager_; NiceMock validation_visitor_; std::unique_ptr loader_; - bool strict_header_validation_; }; TEST_F(Http1ClientConnectionImplTest, SimpleGet) { diff --git a/test/integration/BUILD b/test/integration/BUILD index 64949483c84e..702ed04a4fa4 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -418,6 +418,7 @@ envoy_cc_test_library( "//source/common/network:filter_lib", "//source/common/network:listen_socket_lib", "//source/common/network:utility_lib", + "//source/common/runtime:runtime_lib", "//source/common/stats:isolated_store_lib", "//source/common/stats:thread_local_store_lib", "//source/common/thread_local:thread_local_lib", diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index eec74c73d452..fbd7308fd4a6 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -62,8 +62,8 @@ typeToCodecType(Http::CodecClient::Type type) { IntegrationCodecClient::IntegrationCodecClient( Event::Dispatcher& dispatcher, Network::ClientConnectionPtr&& conn, Upstream::HostDescriptionConstSharedPtr host_description, CodecClient::Type type) - : CodecClientProd(type, std::move(conn), host_description, dispatcher, false), - dispatcher_(dispatcher), callbacks_(*this), codec_callbacks_(*this) { + : CodecClientProd(type, std::move(conn), host_description, dispatcher), dispatcher_(dispatcher), + callbacks_(*this), codec_callbacks_(*this) { connection_->addConnectionCallbacks(callbacks_); setCodecConnectionCallbacks(codec_callbacks_); dispatcher.run(Event::Dispatcher::RunType::Block); diff --git a/test/integration/utility.cc b/test/integration/utility.cc index 59b336b38d6d..e0953f354001 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -76,7 +76,7 @@ IntegrationUtil::makeSingleRequest(const Network::Address::InstanceConstSharedPt type, dispatcher->createClientConnection(addr, Network::Address::InstanceConstSharedPtr(), Network::Test::createRawBufferSocket(), nullptr), - host_description, *dispatcher, false); + host_description, *dispatcher); BufferingStreamDecoderPtr response(new BufferingStreamDecoder([&]() -> void { client.close(); dispatcher->exit(); From 50df654f5f10382f5a4d64e9b77eb6d35a008593 Mon Sep 17 00:00:00 2001 From: Cynthia Coan Date: Fri, 26 Jul 2019 15:55:28 -0600 Subject: [PATCH 254/542] update filter example with what we use for tests (#7737) Description: this forces the same workspace file we use for tests: https://github.com/envoyproxy/envoy/blob/14c5371/ci/build_setup.sh#L75 to be pushed to the envoy-filter-example. this will allow not having to file a manual PR to fix the build, such as: envoyproxy/envoy-filter-example#95 Risk Level: Low Testing: locally Docs Changes: None Release Notes: None Signed-off-by: Cynthia Coan --- ci/filter_example_mirror.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/filter_example_mirror.sh b/ci/filter_example_mirror.sh index 43949f592121..1d6d5ae05b23 100755 --- a/ci/filter_example_mirror.sh +++ b/ci/filter_example_mirror.sh @@ -2,6 +2,7 @@ set -e +ENVOY_SRCDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd) CHECKOUT_DIR=../envoy-filter-example if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$CIRCLE_BRANCH" == "master" ] @@ -14,13 +15,17 @@ then git -C "$CHECKOUT_DIR" fetch git -C "$CHECKOUT_DIR" checkout -B master origin/master + echo "Updating Submodule..." # Update submodule to latest Envoy SHA ENVOY_SHA=$(git rev-parse HEAD) git -C "$CHECKOUT_DIR" submodule update --init git -C "$CHECKOUT_DIR/envoy" checkout "$ENVOY_SHA" - git -C "$CHECKOUT_DIR" commit -a -m "Update Envoy submodule to $ENVOY_SHA" - echo "Pushing..." + echo "Updating Workspace file." + sed -e "s|{ENVOY_SRCDIR}|envoy|" "${ENVOY_SRCDIR}"/ci/WORKSPACE.filter.example > "${CHECKOUT_DIR}"/WORKSPACE + + echo "Committing, and Pushing..." + git -C "$CHECKOUT_DIR" commit -a -m "Update Envoy submodule to $ENVOY_SHA" git -C "$CHECKOUT_DIR" push origin master echo "Done" fi From 2145077a88cab7c3069aec45d277723b972a773f Mon Sep 17 00:00:00 2001 From: Yangmin Zhu Date: Fri, 26 Jul 2019 23:56:28 -0700 Subject: [PATCH 255/542] jwt: refactor the JWT filter logging (#7731) Description: Refactor the JWT filter logging for better debugging experience: - Add logger ID `jwt` - Log JWT verification result when multiple tokens are used - Remove never used function `getProtoConfig()` Risk Level: Low Testing: N/A Docs Changes: N/A Release Notes: N/A Signed-off-by: Yangmin Zhu --- source/common/common/logger.h | 1 + source/extensions/filters/http/jwt_authn/BUILD | 3 +++ .../extensions/filters/http/jwt_authn/authenticator.cc | 10 +++++----- source/extensions/filters/http/jwt_authn/filter.cc | 5 ++++- source/extensions/filters/http/jwt_authn/filter.h | 2 +- .../extensions/filters/http/jwt_authn/filter_config.h | 8 +------- source/extensions/filters/http/jwt_authn/jwks_cache.cc | 2 +- source/extensions/filters/http/jwt_authn/matcher.cc | 2 +- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/source/common/common/logger.h b/source/common/common/logger.h index dcc83627ac07..42360c572b39 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -41,6 +41,7 @@ namespace Logger { FUNCTION(hystrix) \ FUNCTION(init) \ FUNCTION(io) \ + FUNCTION(jwt) \ FUNCTION(kafka) \ FUNCTION(lua) \ FUNCTION(main) \ diff --git a/source/extensions/filters/http/jwt_authn/BUILD b/source/extensions/filters/http/jwt_authn/BUILD index b566159039b3..feede1f772fe 100644 --- a/source/extensions/filters/http/jwt_authn/BUILD +++ b/source/extensions/filters/http/jwt_authn/BUILD @@ -51,6 +51,9 @@ envoy_cc_library( name = "filter_lib", srcs = ["filter.cc"], hdrs = ["filter.h"], + external_deps = [ + "jwt_verify_lib", + ], deps = [ ":filter_config_interface", ":matchers_lib", diff --git a/source/extensions/filters/http/jwt_authn/authenticator.cc b/source/extensions/filters/http/jwt_authn/authenticator.cc index c81ec50cfb56..fa361f90a237 100644 --- a/source/extensions/filters/http/jwt_authn/authenticator.cc +++ b/source/extensions/filters/http/jwt_authn/authenticator.cc @@ -25,7 +25,7 @@ namespace { /** * Object to implement Authenticator interface. */ -class AuthenticatorImpl : public Logger::Loggable, +class AuthenticatorImpl : public Logger::Loggable, public Authenticator, public Common::JwksFetcher::JwksReceiver { public: @@ -99,7 +99,7 @@ void AuthenticatorImpl::verify(Http::HeaderMap& headers, std::vectoriss_); // Check if token extracted from the location contains the issuer specified by config. if (!curr_token_->isIssuerSpecified(jwt_->iss_)) { - ENVOY_LOG(debug, "Jwt issuer {} does not match required", jwt_->iss_); doneWithStatus(Status::JwtUnknownIssuer); return; } @@ -236,11 +236,11 @@ void AuthenticatorImpl::verifyKey() { } void AuthenticatorImpl::doneWithStatus(const Status& status) { + ENVOY_LOG(debug, "JWT token verification completed with: {}", + ::google::jwt_verify::getStatusString(status)); // if on allow missing or failed this should verify all tokens, otherwise stop on ok. if ((Status::Ok == status && !is_allow_failed_) || tokens_.empty()) { tokens_.clear(); - ENVOY_LOG(debug, "Jwt authentication completed with: {}", - ::google::jwt_verify::getStatusString(status)); callback_(is_allow_failed_ ? Status::Ok : status); callback_ = nullptr; return; diff --git a/source/extensions/filters/http/jwt_authn/filter.cc b/source/extensions/filters/http/jwt_authn/filter.cc index 3dd3d3ab8c79..9c758605c3ee 100644 --- a/source/extensions/filters/http/jwt_authn/filter.cc +++ b/source/extensions/filters/http/jwt_authn/filter.cc @@ -4,6 +4,8 @@ #include "extensions/filters/http/well_known_names.h" +#include "jwt_verify_lib/status.h" + using ::google::jwt_verify::Status; namespace Envoy { @@ -55,7 +57,8 @@ void Filter::setPayload(const ProtobufWkt::Struct& payload) { } void Filter::onComplete(const Status& status) { - ENVOY_LOG(debug, "Called Filter : check complete {}", int(status)); + ENVOY_LOG(debug, "Called Filter : check complete {}", + ::google::jwt_verify::getStatusString(status)); // This stream has been reset, abort the callback. if (state_ == Responded) { return; diff --git a/source/extensions/filters/http/jwt_authn/filter.h b/source/extensions/filters/http/jwt_authn/filter.h index 6e9cef2609ac..67572114623c 100644 --- a/source/extensions/filters/http/jwt_authn/filter.h +++ b/source/extensions/filters/http/jwt_authn/filter.h @@ -17,7 +17,7 @@ namespace JwtAuthn { // The Envoy filter to process JWT auth. class Filter : public Http::StreamDecoderFilter, public Verifier::Callbacks, - public Logger::Loggable { + public Logger::Loggable { public: Filter(FilterConfigSharedPtr config); diff --git a/source/extensions/filters/http/jwt_authn/filter_config.h b/source/extensions/filters/http/jwt_authn/filter_config.h index 454fa9b1314c..adac32edb149 100644 --- a/source/extensions/filters/http/jwt_authn/filter_config.h +++ b/source/extensions/filters/http/jwt_authn/filter_config.h @@ -59,7 +59,7 @@ struct JwtAuthnFilterStats { /** * The filter config object to hold config and relevant objects. */ -class FilterConfig : public Logger::Loggable, public AuthFactory { +class FilterConfig : public Logger::Loggable, public AuthFactory { public: ~FilterConfig() override = default; @@ -93,12 +93,6 @@ class FilterConfig : public Logger::Loggable, public AuthFac JwtAuthnFilterStats& stats() { return stats_; } - // Get the Config. - const ::envoy::config::filter::http::jwt_authn::v2alpha::JwtAuthentication& - getProtoConfig() const { - return proto_config_; - } - // Get per-thread cache object. ThreadLocalCache& getCache() const { return tls_->getTyped(); } diff --git a/source/extensions/filters/http/jwt_authn/jwks_cache.cc b/source/extensions/filters/http/jwt_authn/jwks_cache.cc index cfb3faea3409..425bef1b4959 100644 --- a/source/extensions/filters/http/jwt_authn/jwks_cache.cc +++ b/source/extensions/filters/http/jwt_authn/jwks_cache.cc @@ -25,7 +25,7 @@ namespace { // Default cache expiration time in 5 minutes. constexpr int PubkeyCacheExpirationSec = 600; -class JwksDataImpl : public JwksCache::JwksData, public Logger::Loggable { +class JwksDataImpl : public JwksCache::JwksData, public Logger::Loggable { public: JwksDataImpl(const JwtProvider& jwt_provider, TimeSource& time_source, Api::Api& api) : jwt_provider_(jwt_provider), time_source_(time_source) { diff --git a/source/extensions/filters/http/jwt_authn/matcher.cc b/source/extensions/filters/http/jwt_authn/matcher.cc index 123e590a7e72..0a187c3eb451 100644 --- a/source/extensions/filters/http/jwt_authn/matcher.cc +++ b/source/extensions/filters/http/jwt_authn/matcher.cc @@ -19,7 +19,7 @@ namespace { /** * Perform a match against any HTTP header or pseudo-header. */ -class BaseMatcherImpl : public Matcher, public Logger::Loggable { +class BaseMatcherImpl : public Matcher, public Logger::Loggable { public: BaseMatcherImpl(const RequirementRule& rule) : case_sensitive_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(rule.match(), case_sensitive, true)) { From c37234b3f5d18c29ff0afe9b8607d14b804f14ff Mon Sep 17 00:00:00 2001 From: Frank Date: Sat, 27 Jul 2019 09:52:05 -0400 Subject: [PATCH 256/542] config: Specify Bootstrap as proto object to Envoy::OptionsImpl (#7722) * config: Specify Bootstrap as proto object to Envoy::OptionsImpl (#7722) Signed-off-by: Frank Fort --- include/envoy/server/BUILD | 1 + include/envoy/server/options.h | 7 +++++++ source/server/BUILD | 1 + source/server/options_impl.h | 8 ++++++++ source/server/server.cc | 11 +++++++---- test/integration/run_envoy_test.sh | 2 +- test/mocks/server/BUILD | 1 + test/mocks/server/mocks.cc | 1 + test/mocks/server/mocks.h | 3 +++ test/server/BUILD | 1 + test/server/options_impl_test.cc | 9 +++++++++ test/server/server_test.cc | 23 ++++++++++++++++++++++- 12 files changed, 62 insertions(+), 6 deletions(-) diff --git a/include/envoy/server/BUILD b/include/envoy/server/BUILD index 0180275b9b92..2dbd768aab91 100644 --- a/include/envoy/server/BUILD +++ b/include/envoy/server/BUILD @@ -119,6 +119,7 @@ envoy_cc_library( "//include/envoy/network:address_interface", "//include/envoy/stats:stats_interface", "@envoy_api//envoy/admin/v2alpha:server_info_cc", + "@envoy_api//envoy/config/bootstrap/v2:bootstrap_cc", ], ) diff --git a/include/envoy/server/options.h b/include/envoy/server/options.h index e78b499a7f3e..8904925f28e9 100644 --- a/include/envoy/server/options.h +++ b/include/envoy/server/options.h @@ -6,6 +6,7 @@ #include "envoy/admin/v2alpha/server_info.pb.h" #include "envoy/common/pure.h" +#include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "envoy/network/address.h" #include "spdlog/spdlog.h" @@ -78,6 +79,12 @@ class Options { */ virtual const std::string& configYaml() const PURE; + /** + * @return const envoy::config::bootstrap::v2::Bootstrap& a bootstrap proto object + * that merges into the config last, after configYaml and configPath. + */ + virtual const envoy::config::bootstrap::v2::Bootstrap& configProto() const PURE; + /** * @return bool allow unknown fields in the configuration? */ diff --git a/source/server/BUILD b/source/server/BUILD index f73da48b261f..2e83ab4671c6 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -213,6 +213,7 @@ envoy_cc_library( "//source/common/common:version_lib", "//source/common/protobuf:utility_lib", "//source/common/stats:stats_lib", + "@envoy_api//envoy/config/bootstrap/v2:bootstrap_cc", ], ) diff --git a/source/server/options_impl.h b/source/server/options_impl.h index 3269050f5fc4..e9663953aa99 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -5,6 +5,7 @@ #include #include "envoy/common/exception.h" +#include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "envoy/server/options.h" #include "common/common/logger.h" @@ -40,6 +41,9 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable #include "envoy/common/mutex_tracer.h" +#include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "envoy/server/admin.h" #include "envoy/server/configuration.h" #include "envoy/server/drain_manager.h" @@ -59,6 +60,7 @@ class MockOptions : public Options { MOCK_CONST_METHOD0(baseId, uint64_t()); MOCK_CONST_METHOD0(concurrency, uint32_t()); MOCK_CONST_METHOD0(configPath, const std::string&()); + MOCK_CONST_METHOD0(configProto, const envoy::config::bootstrap::v2::Bootstrap&()); MOCK_CONST_METHOD0(configYaml, const std::string&()); MOCK_CONST_METHOD0(allowUnknownFields, bool()); MOCK_CONST_METHOD0(adminAddressPath, const std::string&()); @@ -84,6 +86,7 @@ class MockOptions : public Options { MOCK_CONST_METHOD0(toCommandLineOptions, Server::CommandLineOptionsPtr()); std::string config_path_; + envoy::config::bootstrap::v2::Bootstrap config_proto_; std::string config_yaml_; std::string admin_address_path_; std::string service_cluster_name_; diff --git a/test/server/BUILD b/test/server/BUILD index c0fb46d6a3f6..e65a4896459e 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -122,6 +122,7 @@ envoy_cc_test( "//test/test_common:logging_lib", "//test/test_common:threadsafe_singleton_injector_lib", "//test/test_common:utility_lib", + "@envoy_api//envoy/config/bootstrap/v2:bootstrap_cc", ], ) diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index e3dbe1dfd7af..86ab2b378be7 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -6,6 +6,7 @@ #include #include "envoy/common/exception.h" +#include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "common/common/utility.h" @@ -109,6 +110,9 @@ TEST_F(OptionsImplTest, SetAll) { options->setBaseId(109876); options->setConcurrency(42); options->setConfigPath("foo"); + envoy::config::bootstrap::v2::Bootstrap bootstrap_foo{}; + bootstrap_foo.mutable_node()->set_id("foo"); + options->setConfigProto(bootstrap_foo); options->setConfigYaml("bogus:"); options->setAdminAddressPath("path"); options->setLocalAddressIpVersion(Network::Address::IpVersion::v6); @@ -130,6 +134,9 @@ TEST_F(OptionsImplTest, SetAll) { EXPECT_EQ(109876, options->baseId()); EXPECT_EQ(42U, options->concurrency()); EXPECT_EQ("foo", options->configPath()); + envoy::config::bootstrap::v2::Bootstrap bootstrap_bar{}; + bootstrap_bar.mutable_node()->set_id("foo"); + EXPECT_TRUE(TestUtility::protoEqual(bootstrap_bar, options->configProto())); EXPECT_EQ("bogus:", options->configYaml()); EXPECT_EQ("path", options->adminAddressPath()); EXPECT_EQ(Network::Address::IpVersion::v6, options->localAddressIpVersion()); @@ -278,6 +285,8 @@ TEST_F(OptionsImplTest, SaneTestConstructor) { EXPECT_EQ(regular_options_impl->baseId(), test_options_impl.baseId()); EXPECT_EQ(regular_options_impl->configPath(), test_options_impl.configPath()); + EXPECT_TRUE(TestUtility::protoEqual(regular_options_impl->configProto(), + test_options_impl.configProto())); EXPECT_EQ(regular_options_impl->configYaml(), test_options_impl.configYaml()); EXPECT_EQ(regular_options_impl->adminAddressPath(), test_options_impl.adminAddressPath()); EXPECT_EQ(regular_options_impl->localAddressIpVersion(), diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 6ed2242cbc2e..b5044f1b87c3 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -452,6 +452,25 @@ TEST_P(ServerInstanceImplTest, BootstrapNode) { EXPECT_EQ(VersionInfo::version(), server_->localInfo().node().build_version()); } +TEST_P(ServerInstanceImplTest, LoadsBootstrapFromConfigProtoOptions) { + options_.config_proto_.mutable_node()->set_id("foo"); + initialize("test/server/node_bootstrap.yaml"); + EXPECT_EQ("foo", server_->localInfo().node().id()); +} + +TEST_P(ServerInstanceImplTest, LoadsBootstrapFromConfigYamlAfterConfigPath) { + options_.config_yaml_ = "node:\n id: 'bar'"; + initialize("test/server/node_bootstrap.yaml"); + EXPECT_EQ("bar", server_->localInfo().node().id()); +} + +TEST_P(ServerInstanceImplTest, LoadsBootstrapFromConfigProtoOptionsLast) { + options_.config_yaml_ = "node:\n id: 'bar'"; + options_.config_proto_.mutable_node()->set_id("foo"); + initialize("test/server/node_bootstrap.yaml"); + EXPECT_EQ("foo", server_->localInfo().node().id()); +} + // Validate server localInfo() from bootstrap Node with CLI overrides. TEST_P(ServerInstanceImplTest, BootstrapNodeWithOptionsOverride) { options_.service_cluster_name_ = "some_cluster_name"; @@ -671,7 +690,9 @@ TEST_P(ServerInstanceImplTest, NoOptionsPassed) { hooks_, restart_, stats_store_, fakelock_, component_factory_, std::make_unique>(), *thread_local_, Thread::threadFactoryForTest(), Filesystem::fileSystemForTest(), nullptr)), - EnvoyException, "At least one of --config-path and --config-yaml should be non-empty"); + EnvoyException, + "At least one of --config-path or --config-yaml or Options::configProto() should be " + "non-empty"); } // Validate that when std::exception is unexpectedly thrown, we exit safely. From 0f892c2385e7051d325d22b3f603cc60facedd6e Mon Sep 17 00:00:00 2001 From: Kuat Date: Sun, 28 Jul 2019 12:42:47 -0700 Subject: [PATCH 257/542] filter: add network filters to the upstreams (#7503) Signed-off-by: Kuat Yessenov --- api/envoy/api/v2/BUILD | 2 + api/envoy/api/v2/cds.proto | 8 +- api/envoy/api/v2/cluster/BUILD | 13 ++ api/envoy/api/v2/cluster/filter.proto | 30 ++++ docs/build.sh | 1 + docs/root/api-v2/clusters/clusters.rst | 1 + .../intro/arch_overview/upstream/upstream.rst | 1 + .../upstream/upstream_filters.rst | 11 ++ docs/root/intro/version_history.rst | 1 + include/envoy/server/filter_config.h | 150 +++++++++++------- include/envoy/upstream/upstream.h | 5 + source/common/upstream/BUILD | 1 + .../upstream/health_discovery_service.cc | 6 +- source/common/upstream/upstream_impl.cc | 85 +++++++++- source/common/upstream/upstream_impl.h | 11 +- test/common/upstream/BUILD | 1 + .../upstream/cluster_manager_impl_test.cc | 56 +++++++ test/integration/BUILD | 12 ++ .../cluster_filter_integration_test.cc | 131 +++++++++++++++ test/integration/stats_integration_test.cc | 5 +- test/mocks/upstream/cluster_info.h | 1 + 21 files changed, 459 insertions(+), 73 deletions(-) create mode 100644 api/envoy/api/v2/cluster/filter.proto create mode 100644 docs/root/intro/arch_overview/upstream/upstream_filters.rst create mode 100644 test/integration/cluster_filter_integration_test.cc diff --git a/api/envoy/api/v2/BUILD b/api/envoy/api/v2/BUILD index 3cc2d6b2c2ec..48ddb2e13025 100644 --- a/api/envoy/api/v2/BUILD +++ b/api/envoy/api/v2/BUILD @@ -67,6 +67,7 @@ api_proto_library_internal( ":eds", "//envoy/api/v2/auth:cert", "//envoy/api/v2/cluster:circuit_breaker", + "//envoy/api/v2/cluster:filter", "//envoy/api/v2/cluster:outlier_detection", "//envoy/api/v2/core:address", "//envoy/api/v2/core:base", @@ -86,6 +87,7 @@ api_go_grpc_library( ":eds_go_grpc", "//envoy/api/v2/auth:cert_go_proto", "//envoy/api/v2/cluster:circuit_breaker_go_proto", + "//envoy/api/v2/cluster:filter_go_proto", "//envoy/api/v2/cluster:outlier_detection_go_proto", "//envoy/api/v2/core:address_go_proto", "//envoy/api/v2/core:base_go_proto", diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 44d2d7501479..bddcf1d56d37 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -16,6 +16,7 @@ import "envoy/api/v2/discovery.proto"; import "envoy/api/v2/core/health_check.proto"; import "envoy/api/v2/core/protocol.proto"; import "envoy/api/v2/cluster/circuit_breaker.proto"; +import "envoy/api/v2/cluster/filter.proto"; import "envoy/api/v2/cluster/outlier_detection.proto"; import "envoy/api/v2/eds.proto"; import "envoy/type/percent.proto"; @@ -51,7 +52,7 @@ service ClusterDiscoveryService { // [#protodoc-title: Clusters] // Configuration for a single upstream cluster. -// [#comment:next free field: 40] +// [#comment:next free field: 41] message Cluster { // Supplies the name of the cluster which must be unique across all clusters. // The cluster name is used when emitting @@ -632,6 +633,11 @@ message Cluster { // If this flag is not set to true, Envoy will wait until the hosts fail active health // checking before removing it from the cluster. bool drain_connections_on_host_removal = 32; + + // An optional list of network filters that make up the filter chain for + // outgoing connections made by the cluster. Order matters as the filters are + // processed sequentially as connection events happen. + repeated cluster.Filter filters = 40; } // An extensible structure containing the address Envoy should bind to when diff --git a/api/envoy/api/v2/cluster/BUILD b/api/envoy/api/v2/cluster/BUILD index ab34f59d0e4e..5589905d859b 100644 --- a/api/envoy/api/v2/cluster/BUILD +++ b/api/envoy/api/v2/cluster/BUILD @@ -33,3 +33,16 @@ api_go_proto_library( name = "outlier_detection", proto = ":outlier_detection", ) + +api_proto_library_internal( + name = "filter", + srcs = ["filter.proto"], + visibility = [ + "//envoy/api/v2:__pkg__", + ], +) + +api_go_proto_library( + name = "filter", + proto = ":filter", +) diff --git a/api/envoy/api/v2/cluster/filter.proto b/api/envoy/api/v2/cluster/filter.proto new file mode 100644 index 000000000000..03a826976f09 --- /dev/null +++ b/api/envoy/api/v2/cluster/filter.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package envoy.api.v2.cluster; + +option java_outer_classname = "FilterProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v2.cluster"; +option csharp_namespace = "Envoy.Api.V2.ClusterNS"; +option ruby_package = "Envoy.Api.V2.ClusterNS"; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Upstream filters] +// +// Upstream filters apply to the connections to the upstream cluster hosts. +message Filter { + // The name of the filter to instantiate. The name must match a + // :ref:`supported filter `. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Filter specific configuration which depends on the filter being + // instantiated. See the supported filters for further documentation. + google.protobuf.Any typed_config = 2; +} diff --git a/docs/build.sh b/docs/build.sh index 8a4c7850908f..acbc73303257 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -76,6 +76,7 @@ PROTO_RST=" /envoy/api/v2/cds/envoy/api/v2/cds.proto.rst /envoy/api/v2/cluster/outlier_detection/envoy/api/v2/cluster/outlier_detection.proto.rst /envoy/api/v2/cluster/circuit_breaker/envoy/api/v2/cluster/circuit_breaker.proto.rst + /envoy/api/v2/cluster/filter/envoy/api/v2/cluster/filter.proto.rst /envoy/api/v2/rds/envoy/api/v2/rds.proto.rst /envoy/api/v2/route/route/envoy/api/v2/route/route.proto.rst /envoy/api/v2/srds/envoy/api/v2/srds.proto.rst diff --git a/docs/root/api-v2/clusters/clusters.rst b/docs/root/api-v2/clusters/clusters.rst index 8fe24ed0ad24..d84f1020c159 100644 --- a/docs/root/api-v2/clusters/clusters.rst +++ b/docs/root/api-v2/clusters/clusters.rst @@ -8,6 +8,7 @@ Clusters ../api/v2/cds.proto ../api/v2/cluster/outlier_detection.proto ../api/v2/cluster/circuit_breaker.proto + ../api/v2/cluster/filter.proto ../api/v2/endpoint/endpoint.proto ../api/v2/eds.proto ../api/v2/core/health_check.proto diff --git a/docs/root/intro/arch_overview/upstream/upstream.rst b/docs/root/intro/arch_overview/upstream/upstream.rst index 4c8a12db3137..da1887087bff 100644 --- a/docs/root/intro/arch_overview/upstream/upstream.rst +++ b/docs/root/intro/arch_overview/upstream/upstream.rst @@ -11,3 +11,4 @@ Upstream clusters load_balancing/load_balancing outlier circuit_breaking + upstream_filters diff --git a/docs/root/intro/arch_overview/upstream/upstream_filters.rst b/docs/root/intro/arch_overview/upstream/upstream_filters.rst new file mode 100644 index 000000000000..1fe902dcf919 --- /dev/null +++ b/docs/root/intro/arch_overview/upstream/upstream_filters.rst @@ -0,0 +1,11 @@ +.. _arch_overview_upstream_filters: + +Upstream network filters +======================== + +Upstream clusters provide an ability to inject network level (L3/L4) +:ref:`filters `. The filters apply to the +connection to the upstream hosts, using the same API presented by listeners for +the downstream connections. The write callbacks are invoked for any chunk of +data sent to the upstream host, and the read callbacks are invoked for data +received from the upstream host. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index db59eeb09e7d..37e4c4ab97e9 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -14,6 +14,7 @@ Version history * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. +* upstream: added network filter chains to upstream connections, see :ref:`filters`. 1.11.0 (July 11, 2019) ====================== diff --git a/include/envoy/server/filter_config.h b/include/envoy/server/filter_config.h index 46d5fbac6165..12ea149f517b 100644 --- a/include/envoy/server/filter_config.h +++ b/include/envoy/server/filter_config.h @@ -32,18 +32,11 @@ namespace Server { namespace Configuration { /** - * Context passed to network and HTTP filters to access server resources. - * TODO(mattklein123): When we lock down visibility of the rest of the code, filters should only - * access the rest of the server via interfaces exposed here. + * Common interface for downstream and upstream network filters. */ -class FactoryContext { +class CommonFactoryContext { public: - virtual ~FactoryContext() = default; - - /** - * @return AccessLogManager for use by the entire server. - */ - virtual AccessLog::AccessLogManager& accessLogManager() PURE; + virtual ~CommonFactoryContext() = default; /** * @return Upstream::ClusterManager& singleton for use by the entire server. @@ -56,37 +49,6 @@ class FactoryContext { */ virtual Event::Dispatcher& dispatcher() PURE; - /** - * @return const Network::DrainDecision& a drain decision that filters can use to determine if - * they should be doing graceful closes on connections when possible. - */ - virtual const Network::DrainDecision& drainDecision() PURE; - - /** - * @return whether external healthchecks are currently failed or not. - */ - virtual bool healthCheckFailed() PURE; - - /** - * @return the server-wide http tracer. - */ - virtual Tracing::HttpTracer& httpTracer() PURE; - - /** - * @return the server's init manager. This can be used for extensions that need to initialize - * after cluster manager init but before the server starts listening. All extensions - * should register themselves during configuration load. initialize() will be called on - * each registered target after cluster manager init but before the server starts - * listening. Once all targets have initialized and invoked their callbacks, the server - * will start listening. - */ - virtual Init::Manager& initManager() PURE; - - /** - * @return ServerLifecycleNotifier& the lifecycle notifier for the server. - */ - virtual ServerLifecycleNotifier& lifecycleNotifier() PURE; - /** * @return information about the local environment the server is running in. */ @@ -123,6 +85,68 @@ class FactoryContext { */ virtual Server::Admin& admin() PURE; + /** + * @return TimeSource& a reference to the time source. + */ + virtual TimeSource& timeSource() PURE; + + /** + * @return ProtobufMessage::ValidationVisitor& validation visitor for filter configuration + * messages. + */ + virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; + + /** + * @return Api::Api& a reference to the api object. + */ + virtual Api::Api& api() PURE; +}; + +/** + * Context passed to network and HTTP filters to access server resources. + * TODO(mattklein123): When we lock down visibility of the rest of the code, filters should only + * access the rest of the server via interfaces exposed here. + */ +class FactoryContext : public virtual CommonFactoryContext { +public: + ~FactoryContext() override = default; + + /** + * @return AccessLogManager for use by the entire server. + */ + virtual AccessLog::AccessLogManager& accessLogManager() PURE; + + /** + * @return const Network::DrainDecision& a drain decision that filters can use to determine if + * they should be doing graceful closes on connections when possible. + */ + virtual const Network::DrainDecision& drainDecision() PURE; + + /** + * @return whether external healthchecks are currently failed or not. + */ + virtual bool healthCheckFailed() PURE; + + /** + * @return the server-wide http tracer. + */ + virtual Tracing::HttpTracer& httpTracer() PURE; + + /** + * @return the server's init manager. This can be used for extensions that need to initialize + * after cluster manager init but before the server starts listening. All extensions + * should register themselves during configuration load. initialize() will be called on + * each registered target after cluster manager init but before the server starts + * listening. Once all targets have initialized and invoked their callbacks, the server + * will start listening. + */ + virtual Init::Manager& initManager() PURE; + + /** + * @return ServerLifecycleNotifier& the lifecycle notifier for the server. + */ + virtual ServerLifecycleNotifier& lifecycleNotifier() PURE; + /** * @return Stats::Scope& the listener's stats scope. */ @@ -134,11 +158,6 @@ class FactoryContext { */ virtual const envoy::api::v2::core::Metadata& listenerMetadata() const PURE; - /** - * @return TimeSource& a reference to the time source. - */ - virtual TimeSource& timeSource() PURE; - /** * @return OverloadManager& the overload manager for the server. */ @@ -158,17 +177,6 @@ class FactoryContext { * @return ProcessContext& a reference to the process context. */ virtual ProcessContext& processContext() PURE; - - /** - * @return ProtobufMessage::ValidationVisitor& validation visitor for filter configuration - * messages. - */ - virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; - - /** - * @return Api::Api& a reference to the api object. - */ - virtual Api::Api& api() PURE; }; class ListenerFactoryContext : public virtual FactoryContext { @@ -316,6 +324,34 @@ class NamedNetworkFilterConfigFactory : public ProtocolOptionsFactory { virtual std::string name() PURE; }; +/** + * Implemented by each upstream cluster network filter and registered via + * Registry::registerFactory() or the convenience class RegisterFactory. + */ +class NamedUpstreamNetworkFilterConfigFactory : public ProtocolOptionsFactory { +public: + ~NamedUpstreamNetworkFilterConfigFactory() override = default; + + /** + * Create a particular upstream network filter factory implementation. If the implementation is + * unable to produce a factory with the provided parameters, it should throw an EnvoyException in + * the case of general error. The returned callback should always be initialized. + */ + virtual Network::FilterFactoryCb createFilterFactoryFromProto(const Protobuf::Message& config, + CommonFactoryContext& context) PURE; + + /** + * @return ProtobufTypes::MessagePtr create empty config proto message for v2. + */ + virtual ProtobufTypes::MessagePtr createEmptyConfigProto() PURE; + + /** + * @return std::string the identifying name for a particular implementation of a network filter + * produced by the factory. + */ + virtual std::string name() PURE; +}; + /** * Implemented by each HTTP filter and registered via Registry::registerFactory or the * convenience class RegisterFactory. diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 567466c18ebc..b627bc9be358 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -843,6 +843,11 @@ class ClusterInfo { */ virtual absl::optional eds_service_name() const PURE; + /** + * Create network filters on a new upstream connection. + */ + virtual void createNetworkFilterChain(Network::Connection& connection) const PURE; + protected: /** * Invoked by extensionProtocolOptionsTyped. diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 10e0e60f814f..e11007731ec6 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -425,6 +425,7 @@ envoy_cc_library( "//include/envoy/local_info:local_info_interface", "//include/envoy/network:dns_interface", "//include/envoy/runtime:runtime_interface", + "//include/envoy/server:filter_config_interface", "//include/envoy/server:transport_socket_config_interface", "//include/envoy/ssl:context_manager_interface", "//include/envoy/thread_local:thread_local_interface", diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index 07e960fbdfed..8fb6f44b1f15 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -236,9 +236,9 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params) Network::TransportSocketFactoryPtr socket_factory = Upstream::createTransportSocketFactory(params.cluster_, factory_context); - return std::make_unique(params.cluster_, params.bind_config_, params.runtime_, - std::move(socket_factory), std::move(scope), - params.added_via_api_, params.validation_visitor_); + return std::make_unique( + params.cluster_, params.bind_config_, params.runtime_, std::move(socket_factory), + std::move(scope), params.added_via_api_, params.validation_visitor_, factory_context); } void HdsCluster::startHealthchecks(AccessLog::AccessLogManager& access_log_manager, diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 69160fa95a99..12fc71e75290 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -271,6 +271,7 @@ HostImpl::createConnection(Event::Dispatcher& dispatcher, const ClusterInfo& clu cluster.transportSocketFactory().createTransportSocket(transport_socket_options), connection_options); connection->setBufferLimits(cluster.perConnectionBufferLimitBytes()); + cluster.createNetworkFilterChain(*connection); return connection; } @@ -542,12 +543,53 @@ ClusterLoadReportStats ClusterInfoImpl::generateLoadReportStats(Stats::Scope& sc return {ALL_CLUSTER_LOAD_REPORT_STATS(POOL_COUNTER(scope))}; } -ClusterInfoImpl::ClusterInfoImpl(const envoy::api::v2::Cluster& config, - const envoy::api::v2::core::BindConfig& bind_config, - Runtime::Loader& runtime, - Network::TransportSocketFactoryPtr&& socket_factory, - Stats::ScopePtr&& stats_scope, bool added_via_api, - ProtobufMessage::ValidationVisitor& validation_visitor) +// Implements the FactoryContext interface required by network filters. +class FactoryContextImpl : public Server::Configuration::CommonFactoryContext { +public: + // Create from a TransportSocketFactoryContext using parent stats_scope and runtime + // other contexts taken from TransportSocketFactoryContext. + FactoryContextImpl(Stats::Scope& stats_scope, Envoy::Runtime::Loader& runtime, + Server::Configuration::TransportSocketFactoryContext& c) + : admin_(c.admin()), stats_scope_(stats_scope), cluster_manager_(c.clusterManager()), + local_info_(c.localInfo()), dispatcher_(c.dispatcher()), random_(c.random()), + runtime_(runtime), singleton_manager_(c.singletonManager()), tls_(c.threadLocal()), + validation_visitor_(c.messageValidationVisitor()), api_(c.api()) {} + + Upstream::ClusterManager& clusterManager() override { return cluster_manager_; } + Event::Dispatcher& dispatcher() override { return dispatcher_; } + const LocalInfo::LocalInfo& localInfo() const override { return local_info_; } + Envoy::Runtime::RandomGenerator& random() override { return random_; } + Envoy::Runtime::Loader& runtime() override { return runtime_; } + Stats::Scope& scope() override { return stats_scope_; } + Singleton::Manager& singletonManager() override { return singleton_manager_; } + ThreadLocal::SlotAllocator& threadLocal() override { return tls_; } + Server::Admin& admin() override { return admin_; } + TimeSource& timeSource() override { return api().timeSource(); } + ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { + return validation_visitor_; + } + Api::Api& api() override { return api_; } + +private: + Server::Admin& admin_; + Stats::Scope& stats_scope_; + Upstream::ClusterManager& cluster_manager_; + const LocalInfo::LocalInfo& local_info_; + Event::Dispatcher& dispatcher_; + Envoy::Runtime::RandomGenerator& random_; + Envoy::Runtime::Loader& runtime_; + Singleton::Manager& singleton_manager_; + ThreadLocal::SlotAllocator& tls_; + ProtobufMessage::ValidationVisitor& validation_visitor_; + Api::Api& api_; +}; + +ClusterInfoImpl::ClusterInfoImpl( + const envoy::api::v2::Cluster& config, const envoy::api::v2::core::BindConfig& bind_config, + Runtime::Loader& runtime, Network::TransportSocketFactoryPtr&& socket_factory, + Stats::ScopePtr&& stats_scope, bool added_via_api, + ProtobufMessage::ValidationVisitor& validation_visitor, + Server::Configuration::TransportSocketFactoryContext& factory_context) : runtime_(runtime), name_(config.name()), type_(config.type()), max_requests_per_connection_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_requests_per_connection, 0)), @@ -577,7 +619,9 @@ ClusterInfoImpl::ClusterInfoImpl(const envoy::api::v2::Cluster& config, cluster_type_(config.has_cluster_type() ? absl::make_optional( config.cluster_type()) - : absl::nullopt) { + : absl::nullopt), + factory_context_( + std::make_unique(*stats_scope_, runtime, factory_context)) { switch (config.lb_policy()) { case envoy::api::v2::Cluster::ROUND_ROBIN: lb_type_ = LoadBalancerType::RoundRobin; @@ -641,6 +685,24 @@ ClusterInfoImpl::ClusterInfoImpl(const envoy::api::v2::Cluster& config, // https://github.com/lyft/protoc-gen-validate/issues/97 resolved. This just provides early // validation of sanity of fields that we should catch at config ingestion. DurationUtil::durationToMilliseconds(common_lb_config_.update_merge_window()); + + // Create upstream filter factories + auto filters = config.filters(); + for (ssize_t i = 0; i < filters.size(); i++) { + const auto& proto_config = filters[i]; + const std::string& string_name = proto_config.name(); + ENVOY_LOG(debug, " upstream filter #{}:", i); + ENVOY_LOG(debug, " name: {}", string_name); + auto& factory = Config::Utility::getAndCheckFactory< + Server::Configuration::NamedUpstreamNetworkFilterConfigFactory>(string_name); + auto message = factory.createEmptyConfigProto(); + if (!proto_config.typed_config().value().empty()) { + proto_config.typed_config().UnpackTo(message.get()); + } + Network::FilterFactoryCb callback = + factory.createFilterFactoryFromProto(*message, *factory_context_); + filter_factories_.push_back(callback); + } } ProtocolOptionsConfigConstSharedPtr @@ -677,6 +739,12 @@ Network::TransportSocketFactoryPtr createTransportSocketFactory( return config_factory.createTransportSocketFactory(*message, factory_context); } +void ClusterInfoImpl::createNetworkFilterChain(Network::Connection& connection) const { + for (const auto& factory : filter_factories_) { + factory(connection); + } +} + ClusterImplBase::ClusterImplBase( const envoy::api::v2::Cluster& cluster, Runtime::Loader& runtime, Server::Configuration::TransportSocketFactoryContext& factory_context, @@ -688,7 +756,8 @@ ClusterImplBase::ClusterImplBase( auto socket_factory = createTransportSocketFactory(cluster, factory_context); info_ = std::make_unique( cluster, factory_context.clusterManager().bindConfig(), runtime, std::move(socket_factory), - std::move(stats_scope), added_via_api, factory_context.messageValidationVisitor()); + std::move(stats_scope), added_via_api, factory_context.messageValidationVisitor(), + factory_context); // Create the default (empty) priority set before registering callbacks to // avoid getting an update the first time it is accessed. priority_set_.getOrCreateHostSet(0); diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 70a7ab3c3617..110b08bc955e 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -17,8 +17,10 @@ #include "envoy/event/timer.h" #include "envoy/local_info/local_info.h" #include "envoy/network/dns.h" +#include "envoy/network/filter.h" #include "envoy/runtime/runtime.h" #include "envoy/secret/secret_manager.h" +#include "envoy/server/filter_config.h" #include "envoy/server/transport_socket_config.h" #include "envoy/ssl/context_manager.h" #include "envoy/stats/scope.h" @@ -502,13 +504,14 @@ class PrioritySetImpl : public PrioritySet { /** * Implementation of ClusterInfo that reads from JSON. */ -class ClusterInfoImpl : public ClusterInfo { +class ClusterInfoImpl : public ClusterInfo, protected Logger::Loggable { public: ClusterInfoImpl(const envoy::api::v2::Cluster& config, const envoy::api::v2::core::BindConfig& bind_config, Runtime::Loader& runtime, Network::TransportSocketFactoryPtr&& socket_factory, Stats::ScopePtr&& stats_scope, bool added_via_api, - ProtobufMessage::ValidationVisitor& validation_visitor); + ProtobufMessage::ValidationVisitor& validation_visitor, + Server::Configuration::TransportSocketFactoryContext&); static ClusterStats generateStats(Stats::Scope& scope); static ClusterLoadReportStats generateLoadReportStats(Stats::Scope& scope); @@ -575,6 +578,8 @@ class ClusterInfoImpl : public ClusterInfo { absl::optional eds_service_name() const override { return eds_service_name_; } + void createNetworkFilterChain(Network::Connection&) const override; + private: struct ResourceManagers { ResourceManagers(const envoy::api::v2::Cluster& config, Runtime::Loader& runtime, @@ -620,6 +625,8 @@ class ClusterInfoImpl : public ClusterInfo { const bool warm_hosts_; absl::optional eds_service_name_; const absl::optional cluster_type_; + const std::unique_ptr factory_context_; + std::vector filter_factories_; }; /** diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 59956d5ca056..566e06cf6012 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -57,6 +57,7 @@ envoy_cc_test( "//test/mocks/tcp:tcp_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:registry_lib", "//test/test_common:simulated_time_system_lib", "//test/test_common:threadsafe_singleton_injector_lib", "//test/test_common:utility_lib", diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 039490906a11..82ac958f3267 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -33,6 +33,7 @@ #include "test/mocks/tcp/mocks.h" #include "test/mocks/thread_local/mocks.h" #include "test/mocks/upstream/mocks.h" +#include "test/test_common/registry.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/threadsafe_singleton_injector.h" #include "test/test_common/utility.h" @@ -2689,6 +2690,61 @@ TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsNullIsOkay) { EXPECT_NE(nullptr, cp); } +class TestUpstreamNetworkFilter : public Network::WriteFilter { +public: + Network::FilterStatus onWrite(Buffer::Instance&, bool) override { + return Network::FilterStatus::Continue; + } +}; + +class TestUpstreamNetworkFilterConfigFactory + : public Server::Configuration::NamedUpstreamNetworkFilterConfigFactory { +public: + Network::FilterFactoryCb + createFilterFactoryFromProto(const Protobuf::Message&, + Server::Configuration::CommonFactoryContext&) override { + return [](Network::FilterManager& filter_manager) -> void { + filter_manager.addWriteFilter(std::make_shared()); + }; + } + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + std::string name() override { return "envoy.test.filter"; } +}; + +// Verify that configured upstream filters are added to client connections. +TEST_F(ClusterManagerImplTest, AddUpstreamFilters) { + TestUpstreamNetworkFilterConfigFactory factory; + Registry::InjectFactory registry( + factory); + const std::string yaml = R"EOF( + static_resources: + clusters: + - name: cluster_1 + connect_timeout: 0.250s + lb_policy: ROUND_ROBIN + type: STATIC + hosts: + - socket_address: + address: "127.0.0.1" + port_value: 11001 + filters: + - name: envoy.test.filter + )EOF"; + + create(parseBootstrapFromV2Yaml(yaml)); + Network::MockClientConnection* connection = new NiceMock(); + EXPECT_CALL(*connection, addReadFilter(_)).Times(0); + EXPECT_CALL(*connection, addWriteFilter(_)).Times(1); + EXPECT_CALL(*connection, addFilter(_)).Times(0); + EXPECT_CALL(factory_.tls_.dispatcher_, createClientConnection_(_, _, _, _)) + .WillOnce(Return(connection)); + auto conn_data = cluster_manager_->tcpConnForCluster("cluster_1", nullptr, nullptr); + EXPECT_EQ(connection, conn_data.connection_.get()); + factory_.tls_.shutdownThread(); +} + class ClusterManagerInitHelperTest : public testing::Test { public: MOCK_METHOD1(onClusterInit, void(Cluster& cluster)); diff --git a/test/integration/BUILD b/test/integration/BUILD index 702ed04a4fa4..d00ef6ca405a 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -141,6 +141,18 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "cluster_filter_integration_test", + srcs = ["cluster_filter_integration_test.cc"], + deps = [ + ":integration_lib", + "//include/envoy/network:filter_interface", + "//include/envoy/registry", + "//source/extensions/filters/network/tcp_proxy:config", + "//test/config:utility_lib", + ], +) + envoy_cc_test( name = "custom_cluster_integration_test", srcs = ["custom_cluster_integration_test.cc"], diff --git a/test/integration/cluster_filter_integration_test.cc b/test/integration/cluster_filter_integration_test.cc new file mode 100644 index 000000000000..a418c3278fd8 --- /dev/null +++ b/test/integration/cluster_filter_integration_test.cc @@ -0,0 +1,131 @@ +#include "envoy/network/filter.h" +#include "envoy/registry/registry.h" + +#include "test/config/utility.h" +#include "test/integration/integration.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace { + +class PoliteFilter : public Network::Filter, Logger::Loggable { +public: + PoliteFilter(const ProtobufWkt::StringValue& value) : greeting_(value.value()) {} + + Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override { + ENVOY_CONN_LOG(debug, "polite: onData {} bytes {} end_stream", read_callbacks_->connection(), + data.length(), end_stream); + if (!read_greeted_) { + Buffer::OwnedImpl greeter(greeting_); + read_callbacks_->injectReadDataToFilterChain(greeter, false); + read_greeted_ = true; + } + return Network::FilterStatus::Continue; + } + Network::FilterStatus onWrite(Buffer::Instance& data, bool end_stream) override { + ENVOY_CONN_LOG(debug, "polite: onWrite {} bytes {} end_stream", write_callbacks_->connection(), + data.length(), end_stream); + if (!write_greeted_) { + Buffer::OwnedImpl greeter("please "); + write_callbacks_->injectWriteDataToFilterChain(greeter, false); + write_greeted_ = true; + } + return Network::FilterStatus::Continue; + } + Network::FilterStatus onNewConnection() override { + ENVOY_CONN_LOG(debug, "polite: new connection", read_callbacks_->connection()); + return Network::FilterStatus::Continue; + } + + void initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) override { + read_callbacks_ = &callbacks; + } + void initializeWriteFilterCallbacks(Network::WriteFilterCallbacks& callbacks) override { + write_callbacks_ = &callbacks; + } + +private: + const std::string greeting_; + Network::ReadFilterCallbacks* read_callbacks_{}; + Network::WriteFilterCallbacks* write_callbacks_{}; + bool read_greeted_{false}; + bool write_greeted_{false}; +}; + +class PoliteFilterConfigFactory + : public Server::Configuration::NamedUpstreamNetworkFilterConfigFactory { +public: + Network::FilterFactoryCb + createFilterFactoryFromProto(const Protobuf::Message& proto_config, + Server::Configuration::CommonFactoryContext&) override { + auto config = dynamic_cast(proto_config); + return [config](Network::FilterManager& filter_manager) -> void { + filter_manager.addFilter(std::make_shared(config)); + }; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() override { return "envoy.upstream.polite"; } +}; + +// perform static registration +REGISTER_FACTORY(PoliteFilterConfigFactory, + Server::Configuration::NamedUpstreamNetworkFilterConfigFactory); + +class ClusterFilterIntegrationTest : public testing::TestWithParam, + public BaseIntegrationTest { +public: + ClusterFilterIntegrationTest() + : BaseIntegrationTest(GetParam(), ConfigHelper::TCP_PROXY_CONFIG) {} + + void initialize() override { + enable_half_close_ = true; + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters(0); + auto* filter = cluster_0->add_filters(); + filter->set_name("envoy.upstream.polite"); + ProtobufWkt::StringValue config; + config.set_value("surely "); + filter->mutable_typed_config()->PackFrom(config); + }); + BaseIntegrationTest::initialize(); + } +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, ClusterFilterIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(ClusterFilterIntegrationTest, TestClusterFilter) { + initialize(); + + auto tcp_client = makeTcpConnection(lookupPort("listener_0")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + + std::string observed_data; + tcp_client->write("test"); + ASSERT_TRUE(fake_upstream_connection->waitForData(11, &observed_data)); + EXPECT_EQ("please test", observed_data); + + observed_data.clear(); + tcp_client->write(" everything"); + ASSERT_TRUE(fake_upstream_connection->waitForData(22, &observed_data)); + EXPECT_EQ("please test everything", observed_data); + + ASSERT_TRUE(fake_upstream_connection->write("yes")); + tcp_client->waitForData("surely yes"); + + tcp_client->write("", true); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->write("", true)); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect(true)); + tcp_client->waitForDisconnect(); +} + +} // namespace +} // namespace Envoy diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index ad2b87484f58..cc016ffbea1a 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -211,6 +211,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // 2019/06/30 7428 42742 43000 remove stats multiple inheritance, inline HeapStatData // 2019/07/06 7477 42742 43000 fork gauge representation to drop pending_increment_ // 2019/07/15 7555 42806 43000 static link libstdc++ in tests + // 2019/07/24 7503 43030 44000 add upstream filters to clusters // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -220,8 +221,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 42806); // 104 bytes higher than a debug build. - EXPECT_MEMORY_LE(m_per_cluster, 43000); + EXPECT_MEMORY_EQ(m_per_cluster, 43030); // 104 bytes higher than a debug build. + EXPECT_MEMORY_LE(m_per_cluster, 44000); } } // namespace diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index 56df9fa4dcca..6aa750b9e55b 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -105,6 +105,7 @@ class MockClusterInfo : public ClusterInfo { MOCK_CONST_METHOD0(drainConnectionsOnHostRemoval, bool()); MOCK_CONST_METHOD0(warmHosts, bool()); MOCK_CONST_METHOD0(eds_service_name, absl::optional()); + MOCK_CONST_METHOD1(createNetworkFilterChain, void(Network::Connection&)); std::string name_{"fake_cluster"}; absl::optional eds_service_name_; From 163c03d3e0b8efdbfa2b2733bd1dd3cbc34292b0 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Sun, 28 Jul 2019 12:46:23 -0700 Subject: [PATCH 258/542] router: minor cleanup (#7705) Signed-off-by: Derek Argueta --- source/common/router/config_impl.cc | 16 +++++++--------- source/common/router/config_impl.h | 21 +++++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index dc11c7bde57b..a589af954269 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -146,21 +146,19 @@ Upstream::RetryPrioritySharedPtr RetryPolicyImpl::retryPriority() const { CorsPolicyImpl::CorsPolicyImpl(const envoy::api::v2::route::CorsPolicy& config, Runtime::Loader& loader) - : config_(config), loader_(loader) { + : config_(config), loader_(loader), allow_methods_(config.allow_methods()), + allow_headers_(config.allow_headers()), expose_headers_(config.expose_headers()), + max_age_(config.max_age()), + legacy_enabled_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enabled, true)) { for (const auto& origin : config.allow_origin()) { allow_origin_.push_back(origin); } for (const auto& regex : config.allow_origin_regex()) { allow_origin_regex_.push_back(RegexUtil::parseRegex(regex)); } - allow_methods_ = config.allow_methods(); - allow_headers_ = config.allow_headers(); - expose_headers_ = config.expose_headers(); - max_age_ = config.max_age(); if (config.has_allow_credentials()) { allow_credentials_ = PROTOBUF_GET_WRAPPED_REQUIRED(config, allow_credentials); } - legacy_enabled_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enabled, true); } ShadowPolicyImpl::ShadowPolicyImpl(const envoy::api::v2::route::RouteAction& config) { @@ -181,7 +179,7 @@ ShadowPolicyImpl::ShadowPolicyImpl(const envoy::api::v2::route::RouteAction& con class HashMethodImplBase : public HashPolicyImpl::HashMethod { public: - HashMethodImplBase(bool terminal) : terminal_(terminal) {} + explicit HashMethodImplBase(bool terminal) : terminal_(terminal) {} bool terminal() const override { return terminal_; } @@ -428,8 +426,8 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, const std::string& runtime_key_prefix = route.route().weighted_clusters().runtime_key_prefix(); for (const auto& cluster : route.route().weighted_clusters().clusters()) { - std::unique_ptr cluster_entry(new WeightedClusterEntry( - this, runtime_key_prefix + "." + cluster.name(), factory_context, cluster)); + auto cluster_entry = std::make_unique( + this, runtime_key_prefix + "." + cluster.name(), factory_context, cluster); weighted_clusters_.emplace_back(std::move(cluster_entry)); total_weight += weighted_clusters_.back()->clusterWeight(); } diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 3fcbd480359b..22ed8ae088a6 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -133,12 +133,12 @@ class CorsPolicyImpl : public CorsPolicy { Runtime::Loader& loader_; std::list allow_origin_; std::list allow_origin_regex_; - std::string allow_methods_; - std::string allow_headers_; - std::string expose_headers_; - std::string max_age_{}; + const std::string allow_methods_; + const std::string allow_headers_; + const std::string expose_headers_; + const std::string max_age_; absl::optional allow_credentials_{}; - bool legacy_enabled_; + const bool legacy_enabled_; }; class ConfigImpl; @@ -265,7 +265,7 @@ class RetryPolicyImpl : public RetryPolicy { */ class ShadowPolicyImpl : public ShadowPolicy { public: - ShadowPolicyImpl(const envoy::api::v2::route::RouteAction& config); + explicit ShadowPolicyImpl(const envoy::api::v2::route::RouteAction& config); // Router::ShadowPolicy const std::string& cluster() const override { return cluster_; } @@ -284,8 +284,9 @@ class ShadowPolicyImpl : public ShadowPolicy { */ class HashPolicyImpl : public HashPolicy { public: - HashPolicyImpl(const Protobuf::RepeatedPtrField& - hash_policy); + explicit HashPolicyImpl( + const Protobuf::RepeatedPtrField& + hash_policy); // Router::HashPolicy absl::optional generateHash(const Network::Address::Instance* downstream_addr, @@ -336,7 +337,7 @@ class HedgePolicyImpl : public HedgePolicy { */ class DecoratorImpl : public Decorator { public: - DecoratorImpl(const envoy::api::v2::route::Decorator& decorator); + explicit DecoratorImpl(const envoy::api::v2::route::Decorator& decorator); // Decorator::apply void apply(Tracing::Span& span) const override; @@ -353,7 +354,7 @@ class DecoratorImpl : public Decorator { */ class RouteTracingImpl : public RouteTracing { public: - RouteTracingImpl(const envoy::api::v2::route::Tracing& tracing); + explicit RouteTracingImpl(const envoy::api::v2::route::Tracing& tracing); // Tracing::getClientSampling const envoy::type::FractionalPercent& getClientSampling() const override; From 2d0b449c8841e806088e09b1742380e92c4f1b7d Mon Sep 17 00:00:00 2001 From: Itay Donanhirsh Date: Sun, 28 Jul 2019 19:12:00 -0700 Subject: [PATCH 259/542] Update repokitteh.star (#7743) Per #7423: * When a PR contains changes to api/, @envoyproxy/api-shepherds are notified via a CC comment. * When a PR contains changes to api/udpa, @envoyproxy/udpa-wg is notified via a CC comment. * An additional GitHub check blocking merge is added that will only be marked as passing once an approval review is received from someone in @envoyproxy/api-shepherds. Risk Level: None. But if there is any issue, please revert. Testing: In test PRs under different users: #7659 Docs Changes: None. Release Notes: None Fixes #7423 Signed-off-by: Itay Donanhirsh --- repokitteh.star | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/repokitteh.star b/repokitteh.star index 9c90e4986673..b39ebd0dc130 100644 --- a/repokitteh.star +++ b/repokitteh.star @@ -2,5 +2,12 @@ use("github.com/repokitteh/modules/assign.star") use("github.com/repokitteh/modules/review.star") use("github.com/repokitteh/modules/wait.star") use("github.com/repokitteh/modules/circleci.star", secret_token=get_secret('circle_token')) +use( + "github.com/repokitteh/modules/ownerscheck.star", + paths=[ + ("envoyproxy/api-shepherds!", "api/"), + ("envoyproxy/udpa-wg", "api/udpa/"), + ], +) alias('retest', 'retry-circle') From a609257acf227a4446bcb08f3b792cd0c9e1f278 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Sun, 28 Jul 2019 20:44:41 -0700 Subject: [PATCH 260/542] accesslog: Restructure gRPC access log implementation (#7730) Signed-off-by: Ruslan Nigmatullin --- .../access_loggers/http_grpc/config.cc | 18 +- .../http_grpc/grpc_access_log_impl.cc | 123 +-- .../http_grpc/grpc_access_log_impl.h | 121 +-- .../http_grpc/grpc_access_log_impl_test.cc | 743 ++++++++++-------- 4 files changed, 556 insertions(+), 449 deletions(-) diff --git a/source/extensions/access_loggers/http_grpc/config.cc b/source/extensions/access_loggers/http_grpc/config.cc index 9e4d4f2d2dd8..a58327004da5 100644 --- a/source/extensions/access_loggers/http_grpc/config.cc +++ b/source/extensions/access_loggers/http_grpc/config.cc @@ -20,7 +20,7 @@ namespace AccessLoggers { namespace HttpGrpc { // Singleton registration via macro defined in envoy/singleton/manager.h -SINGLETON_MANAGER_REGISTRATION(grpc_access_log_streamer); +SINGLETON_MANAGER_REGISTRATION(grpc_access_logger_cache); AccessLog::InstanceSharedPtr HttpGrpcAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, @@ -30,18 +30,16 @@ HttpGrpcAccessLogFactory::createAccessLogInstance(const Protobuf::Message& confi const auto& proto_config = MessageUtil::downcastAndValidate< const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig&>(config); - std::shared_ptr grpc_access_log_streamer = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(grpc_access_log_streamer), - [&context, grpc_service = proto_config.common_config().grpc_service()] { - return std::make_shared( - context.clusterManager().grpcAsyncClientManager().factoryForGrpcService( - grpc_service, context.scope(), false), + std::shared_ptr grpc_access_logger_cache = + context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(grpc_access_logger_cache), [&context] { + return std::make_shared( + context.clusterManager().grpcAsyncClientManager(), context.scope(), context.threadLocal(), context.localInfo()); }); - return std::make_shared(std::move(filter), proto_config, - grpc_access_log_streamer); + return std::make_shared(std::move(filter), proto_config, context.threadLocal(), + grpc_access_logger_cache); } ProtobufTypes::MessagePtr HttpGrpcAccessLogFactory::createEmptyConfigProto() { diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc index 953e2e74e45e..928d3699c637 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc @@ -32,64 +32,83 @@ TLSProperties_TLSVersion tlsVersionStringToEnum(const std::string& tls_version) } }; // namespace -GrpcAccessLogStreamerImpl::GrpcAccessLogStreamerImpl(Grpc::AsyncClientFactoryPtr&& factory, - ThreadLocal::SlotAllocator& tls, - const LocalInfo::LocalInfo& local_info) - : tls_slot_(tls.allocateSlot()) { - SharedStateSharedPtr shared_state = std::make_shared(std::move(factory), local_info); - tls_slot_->set([shared_state](Event::Dispatcher&) { - return ThreadLocal::ThreadLocalObjectSharedPtr{new ThreadLocalStreamer(shared_state)}; - }); -} - -void GrpcAccessLogStreamerImpl::ThreadLocalStream::onRemoteClose(Grpc::Status::GrpcStatus, - const std::string&) { - auto it = parent_.stream_map_.find(log_name_); - ASSERT(it != parent_.stream_map_.end()); - if (it->second.stream_ != nullptr) { - // Only erase if we have a stream. Otherwise we had an inline failure and we will clear the +void GrpcAccessLoggerImpl::LocalStream::onRemoteClose(Grpc::Status::GrpcStatus, + const std::string&) { + ASSERT(parent_.stream_ != absl::nullopt); + if (parent_.stream_->stream_ != nullptr) { + // Only reset if we have a stream. Otherwise we had an inline failure and we will clear the // stream data in send(). - parent_.stream_map_.erase(it); + parent_.stream_.reset(); } } -GrpcAccessLogStreamerImpl::ThreadLocalStreamer::ThreadLocalStreamer( - const SharedStateSharedPtr& shared_state) - : client_(shared_state->factory_->create()), shared_state_(shared_state) {} +GrpcAccessLoggerImpl::GrpcAccessLoggerImpl(Grpc::RawAsyncClientPtr&& client, std::string log_name, + const LocalInfo::LocalInfo& local_info) + : client_(std::move(client)), log_name_(std::move(log_name)), local_info_(local_info) {} -void GrpcAccessLogStreamerImpl::ThreadLocalStreamer::send( - envoy::service::accesslog::v2::StreamAccessLogsMessage& message, const std::string& log_name) { - auto stream_it = stream_map_.find(log_name); - if (stream_it == stream_map_.end()) { - stream_it = stream_map_.emplace(log_name, ThreadLocalStream(*this, log_name)).first; +void GrpcAccessLoggerImpl::log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) { + // TODO(euroelessar): Add batching and flushing. + envoy::service::accesslog::v2::StreamAccessLogsMessage message; + message.mutable_http_logs()->add_log_entry()->Swap(&entry); + + if (stream_ == absl::nullopt) { + stream_.emplace(*this); } - auto& stream_entry = stream_it->second; - if (stream_entry.stream_ == nullptr) { - stream_entry.stream_ = + if (stream_->stream_ == nullptr) { + stream_->stream_ = client_->start(*Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs"), - stream_entry); + *stream_); auto* identifier = message.mutable_identifier(); - *identifier->mutable_node() = shared_state_->local_info_.node(); - identifier->set_log_name(log_name); + *identifier->mutable_node() = local_info_.node(); + identifier->set_log_name(log_name_); } - if (stream_entry.stream_ != nullptr) { - stream_entry.stream_->sendMessage(message, false); + if (stream_->stream_ != nullptr) { + stream_->stream_->sendMessage(message, false); } else { // Clear out the stream data due to stream creation failure. - stream_map_.erase(stream_it); + stream_.reset(); } } -HttpGrpcAccessLog::HttpGrpcAccessLog( - AccessLog::FilterPtr&& filter, - const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig& config, - GrpcAccessLogStreamerSharedPtr grpc_access_log_streamer) - : Common::ImplBase(std::move(filter)), config_(config), - grpc_access_log_streamer_(grpc_access_log_streamer) { +GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, + Stats::Scope& scope, + ThreadLocal::SlotAllocator& tls, + const LocalInfo::LocalInfo& local_info) + : async_client_manager_(async_client_manager), scope_(scope), tls_slot_(tls.allocateSlot()), + local_info_(local_info) { + tls_slot_->set([](Event::Dispatcher&) { return std::make_shared(); }); +} + +GrpcAccessLoggerSharedPtr GrpcAccessLoggerCacheImpl::getOrCreateLogger( + const envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) { + // TODO(euroelessar): Consider cleaning up loggers. + auto& cache = tls_slot_->getTyped(); + const std::size_t cache_key = MessageUtil::hash(config); + const auto it = cache.access_loggers_.find(cache_key); + if (it != cache.access_loggers_.end()) { + return it->second; + } + const Grpc::AsyncClientFactoryPtr factory = + async_client_manager_.factoryForGrpcService(config.grpc_service(), scope_, false); + const GrpcAccessLoggerSharedPtr logger = + std::make_shared(factory->create(), config.log_name(), local_info_); + cache.access_loggers_.emplace(cache_key, logger); + return logger; +} + +HttpGrpcAccessLog::ThreadLocalLogger::ThreadLocalLogger(GrpcAccessLoggerSharedPtr logger) + : logger_(std::move(logger)) {} + +HttpGrpcAccessLog::HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, + envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config, + ThreadLocal::SlotAllocator& tls, + GrpcAccessLoggerCacheSharedPtr access_logger_cache) + : Common::ImplBase(std::move(filter)), config_(std::move(config)), + tls_slot_(tls.allocateSlot()), access_logger_cache_(std::move(access_logger_cache)) { for (const auto& header : config_.additional_request_headers_to_log()) { request_headers_to_log_.emplace_back(header); } @@ -101,6 +120,11 @@ HttpGrpcAccessLog::HttpGrpcAccessLog( for (const auto& header : config_.additional_response_trailers_to_log()) { response_trailers_to_log_.emplace_back(header); } + + tls_slot_->set([this](Event::Dispatcher&) { + return std::make_shared( + access_logger_cache_->getOrCreateLogger(config_.common_config())); + }); } void HttpGrpcAccessLog::responseFlagsToAccessLogResponseFlags( @@ -189,12 +213,10 @@ void HttpGrpcAccessLog::emitLog(const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers, const Http::HeaderMap& response_trailers, const StreamInfo::StreamInfo& stream_info) { - envoy::service::accesslog::v2::StreamAccessLogsMessage message; - auto* log_entry = message.mutable_http_logs()->add_log_entry(); - // Common log properties. // TODO(mattklein123): Populate sample_rate field. - auto* common_properties = log_entry->mutable_common_properties(); + envoy::data::accesslog::v2::HTTPAccessLogEntry log_entry; + auto* common_properties = log_entry.mutable_common_properties(); if (stream_info.downstreamRemoteAddress() != nullptr) { Network::Utility::addressToProtobufAddress( @@ -308,20 +330,20 @@ void HttpGrpcAccessLog::emitLog(const Http::HeaderMap& request_headers, if (stream_info.protocol()) { switch (stream_info.protocol().value()) { case Http::Protocol::Http10: - log_entry->set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP10); + log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP10); break; case Http::Protocol::Http11: - log_entry->set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP11); + log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP11); break; case Http::Protocol::Http2: - log_entry->set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP2); + log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP2); break; } } // HTTP request properties. // TODO(mattklein123): Populate port field. - auto* request_properties = log_entry->mutable_request(); + auto* request_properties = log_entry.mutable_request(); if (request_headers.Scheme() != nullptr) { request_properties->set_scheme(std::string(request_headers.Scheme()->value().getStringView())); } @@ -372,7 +394,7 @@ void HttpGrpcAccessLog::emitLog(const Http::HeaderMap& request_headers, } // HTTP response properties. - auto* response_properties = log_entry->mutable_response(); + auto* response_properties = log_entry.mutable_response(); if (stream_info.responseCode()) { response_properties->mutable_response_code()->set_value(stream_info.responseCode().value()); } @@ -403,8 +425,7 @@ void HttpGrpcAccessLog::emitLog(const Http::HeaderMap& request_headers, } } - // TODO(mattklein123): Consider batching multiple logs and flushing. - grpc_access_log_streamer_->send(message, config_.common_config().log_name()); + tls_slot_->getTyped().logger_->log(std::move(log_entry)); } } // namespace HttpGrpc diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h index cf81c7990f40..c846be0ca171 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h @@ -24,63 +24,52 @@ namespace HttpGrpc { // TODO(mattklein123): Stats /** - * Interface for an access log streamer. The streamer deals with threading and sends access logs - * on the correct stream. + * Interface for an access logger. The logger provides abstraction on top of gRPC stream, deals with + * reconnects and performs batching. */ -class GrpcAccessLogStreamer { +class GrpcAccessLogger { public: - virtual ~GrpcAccessLogStreamer() = default; + virtual ~GrpcAccessLogger() = default; /** - * Send an access log. - * @param message supplies the access log to send. - * @param log_name supplies the name of the log stream to send on. + * Log http access entry. + * @param entry supplies the access log to send. */ - virtual void send(envoy::service::accesslog::v2::StreamAccessLogsMessage& message, - const std::string& log_name) PURE; + virtual void log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) PURE; }; -using GrpcAccessLogStreamerSharedPtr = std::shared_ptr; +using GrpcAccessLoggerSharedPtr = std::shared_ptr; /** - * Production implementation of GrpcAccessLogStreamer that supports per-thread and per-log - * streams. + * Interface for an access logger cache. The cache deals with threading and de-duplicates loggers + * for the same configuration. */ -class GrpcAccessLogStreamerImpl : public Singleton::Instance, public GrpcAccessLogStreamer { +class GrpcAccessLoggerCache { public: - GrpcAccessLogStreamerImpl(Grpc::AsyncClientFactoryPtr&& factory, ThreadLocal::SlotAllocator& tls, - const LocalInfo::LocalInfo& local_info); - - // GrpcAccessLogStreamer - void send(envoy::service::accesslog::v2::StreamAccessLogsMessage& message, - const std::string& log_name) override { - tls_slot_->getTyped().send(message, log_name); - } + virtual ~GrpcAccessLoggerCache() = default; -private: /** - * Shared state that is owned by the per-thread streamers. This allows the main streamer/TLS - * slot to be destroyed while the streamers hold onto the shared state. + * Get existing logger or create a new one for the given configuration. + * @param config supplies the configuration for the logger. + * @return GrpcAccessLoggerSharedPtr ready for logging requests. */ - struct SharedState { - SharedState(Grpc::AsyncClientFactoryPtr&& factory, const LocalInfo::LocalInfo& local_info) - : factory_(std::move(factory)), local_info_(local_info) {} + virtual GrpcAccessLoggerSharedPtr + getOrCreateLogger(const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) PURE; +}; - Grpc::AsyncClientFactoryPtr factory_; - const LocalInfo::LocalInfo& local_info_; - }; +using GrpcAccessLoggerCacheSharedPtr = std::shared_ptr; - using SharedStateSharedPtr = std::shared_ptr; +class GrpcAccessLoggerImpl : public GrpcAccessLogger { +public: + GrpcAccessLoggerImpl(Grpc::RawAsyncClientPtr&& client, std::string log_name, + const LocalInfo::LocalInfo& local_info); - struct ThreadLocalStreamer; + void log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) override; - /** - * Per-thread stream state. - */ - struct ThreadLocalStream +private: + struct LocalStream : public Grpc::AsyncStreamCallbacks { - ThreadLocalStream(ThreadLocalStreamer& parent, const std::string& log_name) - : parent_(parent), log_name_(log_name) {} + LocalStream(GrpcAccessLoggerImpl& parent) : parent_(parent) {} // Grpc::AsyncStreamCallbacks void onCreateInitialMetadata(Http::HeaderMap&) override {} @@ -90,27 +79,40 @@ class GrpcAccessLogStreamerImpl : public Singleton::Instance, public GrpcAccessL void onReceiveTrailingMetadata(Http::HeaderMapPtr&&) override {} void onRemoteClose(Grpc::Status::GrpcStatus status, const std::string& message) override; - ThreadLocalStreamer& parent_; - const std::string log_name_; + GrpcAccessLoggerImpl& parent_; Grpc::AsyncStream stream_{}; }; + Grpc::AsyncClient + client_; + const std::string log_name_; + absl::optional stream_; + const LocalInfo::LocalInfo& local_info_; +}; + +class GrpcAccessLoggerCacheImpl : public Singleton::Instance, public GrpcAccessLoggerCache { +public: + GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, Stats::Scope& scope, + ThreadLocal::SlotAllocator& tls, + const LocalInfo::LocalInfo& local_info); + + GrpcAccessLoggerSharedPtr getOrCreateLogger( + const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) override; + +private: /** - * Per-thread multi-stream state. + * Per-thread cache. */ - struct ThreadLocalStreamer : public ThreadLocal::ThreadLocalObject { - ThreadLocalStreamer(const SharedStateSharedPtr& shared_state); - void send(envoy::service::accesslog::v2::StreamAccessLogsMessage& message, - const std::string& log_name); - - Grpc::AsyncClient - client_; - std::unordered_map stream_map_; - SharedStateSharedPtr shared_state_; + struct ThreadLocalCache : public ThreadLocal::ThreadLocalObject { + // Access loggers indexed by the hash of logger's configuration. + absl::flat_hash_map access_loggers_; }; + Grpc::AsyncClientManager& async_client_manager_; + Stats::Scope& scope_; ThreadLocal::SlotPtr tls_slot_; + const LocalInfo::LocalInfo& local_info_; }; /** @@ -119,21 +121,32 @@ class GrpcAccessLogStreamerImpl : public Singleton::Instance, public GrpcAccessL class HttpGrpcAccessLog : public Common::ImplBase { public: HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, - const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig& config, - GrpcAccessLogStreamerSharedPtr grpc_access_log_streamer); + envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config, + ThreadLocal::SlotAllocator& tls, + GrpcAccessLoggerCacheSharedPtr access_logger_cache); static void responseFlagsToAccessLogResponseFlags( envoy::data::accesslog::v2::AccessLogCommon& common_access_log, const StreamInfo::StreamInfo& stream_info); private: + /** + * Per-thread cached logger. + */ + struct ThreadLocalLogger : public ThreadLocal::ThreadLocalObject { + ThreadLocalLogger(GrpcAccessLoggerSharedPtr logger); + + const GrpcAccessLoggerSharedPtr logger_; + }; + // Common::ImplBase void emitLog(const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers, const Http::HeaderMap& response_trailers, const StreamInfo::StreamInfo& stream_info) override; const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config_; - GrpcAccessLogStreamerSharedPtr grpc_access_log_streamer_; + const ThreadLocal::SlotPtr tls_slot_; + const GrpcAccessLoggerCacheSharedPtr access_logger_cache_; std::vector request_headers_to_log_; std::vector response_headers_to_log_; std::vector response_trailers_to_log_; diff --git a/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc index 960bddb1b96f..f78eae541e26 100644 --- a/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc @@ -1,5 +1,6 @@ #include +#include "common/buffer/zero_copy_input_stream_impl.h" #include "common/network/address_impl.h" #include "extensions/access_loggers/http_grpc/grpc_access_log_impl.h" @@ -24,18 +25,15 @@ namespace AccessLoggers { namespace HttpGrpc { namespace { -class GrpcAccessLogStreamerImplTest : public testing::Test { +class GrpcAccessLoggerImplTest : public testing::Test { public: using MockAccessLogStream = Grpc::MockAsyncStream; using AccessLogCallbacks = Grpc::AsyncStreamCallbacks; - GrpcAccessLogStreamerImplTest() { - EXPECT_CALL(*factory_, create()).WillOnce(Invoke([this] { - return Grpc::RawAsyncClientPtr{async_client_}; - })); - streamer_ = std::make_unique(Grpc::AsyncClientFactoryPtr{factory_}, - tls_, local_info_); + GrpcAccessLoggerImplTest() { + logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, + log_name_, local_info_); } void expectStreamStart(MockAccessLogStream& stream, AccessLogCallbacks** callbacks_to_set) { @@ -47,53 +45,86 @@ class GrpcAccessLogStreamerImplTest : public testing::Test { })); } - NiceMock tls_; + void expectStreamMessage(MockAccessLogStream& stream, const std::string& expected_message_yaml) { + envoy::service::accesslog::v2::StreamAccessLogsMessage expected_message; + TestUtility::loadFromYaml(expected_message_yaml, expected_message); + EXPECT_CALL(stream, sendMessageRaw_(_, false)) + .WillOnce(Invoke([expected_message](Buffer::InstancePtr& request, bool) { + envoy::service::accesslog::v2::StreamAccessLogsMessage message; + Buffer::ZeroCopyInputStreamImpl request_stream(std::move(request)); + EXPECT_TRUE(message.ParseFromZeroCopyStream(&request_stream)); + EXPECT_EQ(message.DebugString(), expected_message.DebugString()); + })); + } + + std::string log_name_ = "test_log_name"; LocalInfo::MockLocalInfo local_info_; Grpc::MockAsyncClient* async_client_{new Grpc::MockAsyncClient}; - Grpc::MockAsyncClientFactory* factory_{new Grpc::MockAsyncClientFactory}; - std::unique_ptr streamer_; + std::unique_ptr logger_; }; // Test basic stream logging flow. -TEST_F(GrpcAccessLogStreamerImplTest, BasicFlow) { +TEST_F(GrpcAccessLoggerImplTest, BasicFlow) { InSequence s; // Start a stream for the first log. - MockAccessLogStream stream1; - AccessLogCallbacks* callbacks1; - expectStreamStart(stream1, &callbacks1); - EXPECT_CALL(local_info_, node()); - EXPECT_CALL(stream1, sendMessageRaw_(_, false)); - envoy::service::accesslog::v2::StreamAccessLogsMessage message_log1; - streamer_->send(message_log1, "log1"); - - message_log1.Clear(); - EXPECT_CALL(stream1, sendMessageRaw_(_, false)); - streamer_->send(message_log1, "log1"); - - // Start a stream for the second log. - MockAccessLogStream stream2; - AccessLogCallbacks* callbacks2; - expectStreamStart(stream2, &callbacks2); + MockAccessLogStream stream; + AccessLogCallbacks* callbacks; + expectStreamStart(stream, &callbacks); EXPECT_CALL(local_info_, node()); - EXPECT_CALL(stream2, sendMessageRaw_(_, false)); - envoy::service::accesslog::v2::StreamAccessLogsMessage message_log2; - streamer_->send(message_log2, "log2"); + expectStreamMessage(stream, R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name +http_logs: + log_entry: + request: + path: /test/path1 +)EOF"); + envoy::data::accesslog::v2::HTTPAccessLogEntry entry; + entry.mutable_request()->set_path("/test/path1"); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + + expectStreamMessage(stream, R"EOF( +http_logs: + log_entry: + request: + path: /test/path2 +)EOF"); + entry.mutable_request()->set_path("/test/path2"); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); // Verify that sending an empty response message doesn't do anything bad. - callbacks1->onReceiveMessage( + callbacks->onReceiveMessage( std::make_unique()); - // Close stream 2 and make sure we make a new one. - callbacks2->onRemoteClose(Grpc::Status::Internal, "bad"); - expectStreamStart(stream2, &callbacks2); + // Close the stream and make sure we make a new one. + callbacks->onRemoteClose(Grpc::Status::Internal, "bad"); + expectStreamStart(stream, &callbacks); EXPECT_CALL(local_info_, node()); - EXPECT_CALL(stream2, sendMessageRaw_(_, false)); - streamer_->send(message_log2, "log2"); + expectStreamMessage(stream, R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name +http_logs: + log_entry: + request: + path: /test/path3 +)EOF"); + entry.mutable_request()->set_path("/test/path3"); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); } // Test that stream failure is handled correctly. -TEST_F(GrpcAccessLogStreamerImplTest, StreamFailure) { +TEST_F(GrpcAccessLoggerImplTest, StreamFailure) { InSequence s; EXPECT_CALL(*async_client_, startRaw(_, _, _)) @@ -103,15 +134,75 @@ TEST_F(GrpcAccessLogStreamerImplTest, StreamFailure) { return nullptr; })); EXPECT_CALL(local_info_, node()); - envoy::service::accesslog::v2::StreamAccessLogsMessage message_log1; - streamer_->send(message_log1, "log1"); + envoy::data::accesslog::v2::HTTPAccessLogEntry entry; + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); } -class MockGrpcAccessLogStreamer : public GrpcAccessLogStreamer { +class GrpcAccessLoggerCacheImplTest : public testing::Test { public: - // GrpcAccessLogStreamer - MOCK_METHOD2(send, void(envoy::service::accesslog::v2::StreamAccessLogsMessage& message, - const std::string& log_name)); + GrpcAccessLoggerCacheImplTest() { + logger_cache_ = std::make_unique(async_client_manager_, scope_, tls_, + local_info_); + } + + void expectClientCreation() { + factory_ = new Grpc::MockAsyncClientFactory; + async_client_ = new Grpc::MockAsyncClient; + EXPECT_CALL(async_client_manager_, factoryForGrpcService(_, _, false)) + .WillOnce(Invoke([this](const envoy::api::v2::core::GrpcService&, Stats::Scope&, bool) { + EXPECT_CALL(*factory_, create()).WillOnce(Invoke([this] { + return Grpc::RawAsyncClientPtr{async_client_}; + })); + return Grpc::AsyncClientFactoryPtr{factory_}; + })); + } + + LocalInfo::MockLocalInfo local_info_; + NiceMock tls_; + Grpc::MockAsyncClientManager async_client_manager_; + Grpc::MockAsyncClient* async_client_ = nullptr; + Grpc::MockAsyncClientFactory* factory_ = nullptr; + std::unique_ptr logger_cache_; + NiceMock scope_; +}; + +TEST_F(GrpcAccessLoggerCacheImplTest, Deduplication) { + InSequence s; + + ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig config; + config.set_log_name("log-1"); + config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("cluster-1"); + + expectClientCreation(); + GrpcAccessLoggerSharedPtr logger1 = logger_cache_->getOrCreateLogger(config); + EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config)); + + // Changing log name leads to another logger. + config.set_log_name("log-2"); + expectClientCreation(); + EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config)); + + config.set_log_name("log-1"); + EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config)); + + // Changing cluster name leads to another logger. + config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("cluster-2"); + expectClientCreation(); + EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config)); +} + +class MockGrpcAccessLogger : public GrpcAccessLogger { +public: + // GrpcAccessLogger + MOCK_METHOD1(log, void(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry)); +}; + +class MockGrpcAccessLoggerCache : public GrpcAccessLoggerCache { +public: + // GrpcAccessLoggerCache + MOCK_METHOD1(getOrCreateLogger, + GrpcAccessLoggerSharedPtr( + const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config)); }; class HttpGrpcAccessLogTest : public testing::Test { @@ -119,22 +210,26 @@ class HttpGrpcAccessLogTest : public testing::Test { void init() { ON_CALL(*filter_, evaluate(_, _, _, _)).WillByDefault(Return(true)); config_.mutable_common_config()->set_log_name("hello_log"); - access_log_ = - std::make_unique(AccessLog::FilterPtr{filter_}, config_, streamer_); + EXPECT_CALL(*logger_cache_, getOrCreateLogger(_)) + .WillOnce([this](const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) { + EXPECT_EQ(config.DebugString(), config_.common_config().DebugString()); + return logger_; + }); + access_log_ = std::make_unique(AccessLog::FilterPtr{filter_}, config_, tls_, + logger_cache_); } - void expectLog(const std::string& expected_request_msg_yaml) { + void expectLog(const std::string& expected_log_entry_yaml) { if (access_log_ == nullptr) { init(); } - envoy::service::accesslog::v2::StreamAccessLogsMessage expected_request_msg; - TestUtility::loadFromYaml(expected_request_msg_yaml, expected_request_msg); - EXPECT_CALL(*streamer_, send(_, "hello_log")) - .WillOnce(Invoke( - [expected_request_msg](envoy::service::accesslog::v2::StreamAccessLogsMessage& message, - const std::string&) { - EXPECT_EQ(message.DebugString(), expected_request_msg.DebugString()); + envoy::data::accesslog::v2::HTTPAccessLogEntry expected_log_entry; + TestUtility::loadFromYaml(expected_log_entry_yaml, expected_log_entry); + EXPECT_CALL(*logger_, log(_)) + .WillOnce( + Invoke([expected_log_entry](envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) { + EXPECT_EQ(entry.DebugString(), expected_log_entry.DebugString()); })); } @@ -147,30 +242,30 @@ class HttpGrpcAccessLogTest : public testing::Test { }; expectLog(fmt::format(R"EOF( - http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: {{}} - request: - request_method: {} - request_headers_bytes: {} - response: {{}} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: {{}} +request: + request_method: {} + request_headers_bytes: {} +response: {{}} )EOF", request_method, request_method.length() + 7)); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } AccessLog::MockFilter* filter_{new NiceMock()}; + NiceMock tls_; envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config_; - std::shared_ptr streamer_{new MockGrpcAccessLogStreamer()}; + std::shared_ptr logger_{new MockGrpcAccessLogger()}; + std::shared_ptr logger_cache_{new MockGrpcAccessLoggerCache()}; std::unique_ptr access_log_; }; @@ -188,25 +283,23 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { (*stream_info.metadata_.mutable_filter_metadata())["foo"] = ProtobufWkt::Struct(); expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - pipe: - path: "/foo" - start_time: - seconds: 3600 - time_to_last_downstream_tx_byte: - nanos: 2000000 - metadata: - filter_metadata: - foo: {} - request: {} - response: {} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + pipe: + path: "/foo" + start_time: + seconds: 3600 + time_to_last_downstream_tx_byte: + nanos: 2000000 + metadata: + filter_metadata: + foo: {} +request: {} +response: {} )EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -218,23 +311,21 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { stream_info.last_downstream_tx_byte_sent_ = std::chrono::nanoseconds(2000000); expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - time_to_last_downstream_tx_byte: - nanos: 2000000 - request: {} - response: {} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 + time_to_last_downstream_tx_byte: + nanos: 2000000 +request: {} +response: {} )EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -277,64 +368,62 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { Http::TestHeaderMapImpl response_headers{{":status", "200"}}; expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - time_to_last_rx_byte: - nanos: 2000000 - time_to_first_upstream_tx_byte: - nanos: 4000000 - time_to_last_upstream_tx_byte: - nanos: 6000000 - time_to_first_upstream_rx_byte: - nanos: 8000000 - time_to_last_upstream_rx_byte: - nanos: 10000000 - time_to_first_downstream_tx_byte: - nanos: 12000000 - time_to_last_downstream_tx_byte: - nanos: 14000000 - upstream_remote_address: - socket_address: - address: "10.0.0.1" - port_value: 443 - upstream_local_address: - socket_address: - address: "10.0.0.2" - port_value: 0 - upstream_cluster: "fake_cluster" - response_flags: - fault_injected: true - route_name: "route-name-test" - protocol_version: HTTP10 - request: - scheme: "scheme_value" - authority: "authority_value" - path: "path_value" - user_agent: "user-agent_value" - referer: "referer_value" - forwarded_for: "x-forwarded-for_value" - request_id: "x-request-id_value" - original_path: "x-envoy-original-path_value" - request_headers_bytes: 230 - request_body_bytes: 10 - request_method: "POST" - response: - response_code: - value: 200 - response_headers_bytes: 10 - response_body_bytes: 20 - response_code_details: "via_upstream" +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 + time_to_last_rx_byte: + nanos: 2000000 + time_to_first_upstream_tx_byte: + nanos: 4000000 + time_to_last_upstream_tx_byte: + nanos: 6000000 + time_to_first_upstream_rx_byte: + nanos: 8000000 + time_to_last_upstream_rx_byte: + nanos: 10000000 + time_to_first_downstream_tx_byte: + nanos: 12000000 + time_to_last_downstream_tx_byte: + nanos: 14000000 + upstream_remote_address: + socket_address: + address: "10.0.0.1" + port_value: 443 + upstream_local_address: + socket_address: + address: "10.0.0.2" + port_value: 0 + upstream_cluster: "fake_cluster" + response_flags: + fault_injected: true + route_name: "route-name-test" +protocol_version: HTTP10 +request: + scheme: "scheme_value" + authority: "authority_value" + path: "path_value" + user_agent: "user-agent_value" + referer: "referer_value" + forwarded_for: "x-forwarded-for_value" + request_id: "x-request-id_value" + original_path: "x-envoy-original-path_value" + request_headers_bytes: 230 + request_body_bytes: 10 + request_method: "POST" +response: + response_code: + value: 200 + response_headers_bytes: 10 + response_body_bytes: 20 + response_code_details: "via_upstream" )EOF"); access_log_->log(&request_headers, &response_headers, nullptr, stream_info); } @@ -350,24 +439,22 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { }; expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - upstream_transport_failure_reason: "TLS error" - request: - request_method: "METHOD_UNSPECIFIED" - request_headers_bytes: 16 - response: {} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 + upstream_transport_failure_reason: "TLS error" +request: + request_method: "METHOD_UNSPECIFIED" + request_headers_bytes: 16 +response: {} )EOF"); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } @@ -396,38 +483,36 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { }; expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - tls_properties: - tls_version: TLSv1_3 - tls_cipher_suite: 0x2cc0 - tls_sni_hostname: sni - local_certificate_properties: - subject_alt_name: - - uri: localSan1 - - uri: localSan2 - subject: localSubject - peer_certificate_properties: - subject_alt_name: - - uri: peerSan1 - - uri: peerSan2 - subject: peerSubject - tls_session_id: D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B - request: - request_method: "METHOD_UNSPECIFIED" - request_headers_bytes: 16 - response: {} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 + tls_properties: + tls_version: TLSv1_3 + tls_cipher_suite: 0x2cc0 + tls_sni_hostname: sni + local_certificate_properties: + subject_alt_name: + - uri: localSan1 + - uri: localSan2 + subject: localSubject + peer_certificate_properties: + subject_alt_name: + - uri: peerSan1 + - uri: peerSan2 + subject: peerSubject + tls_session_id: D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B +request: + request_method: "METHOD_UNSPECIFIED" + request_headers_bytes: 16 +response: {} )EOF"); access_log_->log(&request_headers, nullptr, nullptr, stream_info); } @@ -449,28 +534,26 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { }; expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - tls_properties: - tls_version: TLSv1_2 - tls_cipher_suite: 0x2f - tls_sni_hostname: sni - local_certificate_properties: {} - peer_certificate_properties: {} - request: - request_method: "METHOD_UNSPECIFIED" - response: {} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 + tls_properties: + tls_version: TLSv1_2 + tls_cipher_suite: 0x2f + tls_sni_hostname: sni + local_certificate_properties: {} + peer_certificate_properties: {} +request: + request_method: "METHOD_UNSPECIFIED" +response: {} )EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -492,28 +575,26 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { }; expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - tls_properties: - tls_version: TLSv1_1 - tls_cipher_suite: 0x2f - tls_sni_hostname: sni - local_certificate_properties: {} - peer_certificate_properties: {} - request: - request_method: "METHOD_UNSPECIFIED" - response: {} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 + tls_properties: + tls_version: TLSv1_1 + tls_cipher_suite: 0x2f + tls_sni_hostname: sni + local_certificate_properties: {} + peer_certificate_properties: {} +request: + request_method: "METHOD_UNSPECIFIED" +response: {} )EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -535,28 +616,26 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { }; expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - tls_properties: - tls_version: TLSv1 - tls_cipher_suite: 0x2f - tls_sni_hostname: sni - local_certificate_properties: {} - peer_certificate_properties: {} - request: - request_method: "METHOD_UNSPECIFIED" - response: {} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 + tls_properties: + tls_version: TLSv1 + tls_cipher_suite: 0x2f + tls_sni_hostname: sni + local_certificate_properties: {} + peer_certificate_properties: {} +request: + request_method: "METHOD_UNSPECIFIED" +response: {} )EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -578,28 +657,26 @@ TEST_F(HttpGrpcAccessLogTest, Marshalling) { }; expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - tls_properties: - tls_version: VERSION_UNSPECIFIED - tls_cipher_suite: 0x2f - tls_sni_hostname: sni - local_certificate_properties: {} - peer_certificate_properties: {} - request: - request_method: "METHOD_UNSPECIFIED" - response: {} +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 + tls_properties: + tls_version: VERSION_UNSPECIFIED + tls_cipher_suite: 0x2f + tls_sni_hostname: sni + local_certificate_properties: {} + peer_certificate_properties: {} +request: + request_method: "METHOD_UNSPECIFIED" +response: {} )EOF"); access_log_->log(nullptr, nullptr, nullptr, stream_info); } @@ -653,38 +730,36 @@ TEST_F(HttpGrpcAccessLogTest, MarshallingAdditionalHeaders) { }; expectLog(R"EOF( -http_logs: - log_entry: - common_properties: - downstream_remote_address: - socket_address: - address: "127.0.0.1" - port_value: 0 - downstream_local_address: - socket_address: - address: "127.0.0.2" - port_value: 0 - start_time: - seconds: 3600 - request: - scheme: "scheme_value" - authority: "authority_value" - path: "path_value" - request_method: "POST" - request_headers_bytes: 132 - request_headers: - "x-custom-request": "custom_value" - "x-custom-empty": "" - "x-envoy-max-retries": "3" - response: - response_headers_bytes: 92 - response_headers: - "x-custom-response": "custom_value" - "x-custom-empty": "" - "x-envoy-immediate-health-check-fail": "true" - response_trailers: - "x-logged-trailer": "value" - "x-empty-trailer": "" +common_properties: + downstream_remote_address: + socket_address: + address: "127.0.0.1" + port_value: 0 + downstream_local_address: + socket_address: + address: "127.0.0.2" + port_value: 0 + start_time: + seconds: 3600 +request: + scheme: "scheme_value" + authority: "authority_value" + path: "path_value" + request_method: "POST" + request_headers_bytes: 132 + request_headers: + "x-custom-request": "custom_value" + "x-custom-empty": "" + "x-envoy-max-retries": "3" +response: + response_headers_bytes: 92 + response_headers: + "x-custom-response": "custom_value" + "x-custom-empty": "" + "x-envoy-immediate-health-check-fail": "true" + response_trailers: + "x-logged-trailer": "value" + "x-empty-trailer": "" )EOF"); access_log_->log(&request_headers, &response_headers, &response_trailers, stream_info); } From 26ba5ae81a9b9ee446e9b6aed585b7b1dec1ea2d Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Sun, 28 Jul 2019 20:47:21 -0700 Subject: [PATCH 261/542] clang-tidy: modernize-use-default-member-init (#7745) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + .../filesystem/posix/directory_iterator_impl.h | 5 ++--- source/common/grpc/google_async_client_impl.h | 4 ++-- source/common/signal/signal_action.h | 2 +- source/common/stats/stats_matcher_impl.h | 4 ++-- source/common/stats/symbol_table_impl.h | 4 ++-- .../filters/network/kafka/serialization.h | 2 +- source/extensions/tracers/zipkin/span_context.h | 12 ++++++------ .../tracers/zipkin/zipkin_core_types.h | 4 ++-- test/common/common/lock_guard_test.cc | 6 +++--- test/common/event/dispatcher_impl_test.cc | 6 ++---- test/common/upstream/eds_test.cc | 4 ++-- test/exe/main_common_test.cc | 16 ++++++---------- .../quiche/envoy_quic_alarm_test.cc | 4 ++-- .../quiche/platform/quic_mock_log_impl.h | 4 ++-- test/integration/sds_dynamic_integration_test.cc | 4 ++-- test/tools/router_check/router.h | 4 ++-- 17 files changed, 40 insertions(+), 46 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index b315a91c40d1..c62c2829fab3 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -23,6 +23,7 @@ WarningsAsErrors: 'abseil-duration-*, modernize-make-shared, modernize-make-unique, modernize-return-braced-init-list, + modernize-use-default-member-init, modernize-use-equals-default, modernize-use-override, modernize-use-using, diff --git a/source/common/filesystem/posix/directory_iterator_impl.h b/source/common/filesystem/posix/directory_iterator_impl.h index 889ffef1b5ac..ed5460dba673 100644 --- a/source/common/filesystem/posix/directory_iterator_impl.h +++ b/source/common/filesystem/posix/directory_iterator_impl.h @@ -12,8 +12,7 @@ namespace Filesystem { class DirectoryIteratorImpl : public DirectoryIterator { public: DirectoryIteratorImpl(const std::string& directory_path); - DirectoryIteratorImpl() - : directory_path_(""), dir_(nullptr), os_sys_calls_(Api::OsSysCallsSingleton::get()) {} + DirectoryIteratorImpl() : directory_path_(""), os_sys_calls_(Api::OsSysCallsSingleton::get()) {} ~DirectoryIteratorImpl() override; @@ -32,7 +31,7 @@ class DirectoryIteratorImpl : public DirectoryIterator { FileType fileType(const std::string& name) const; std::string directory_path_; - DIR* dir_; + DIR* dir_{nullptr}; Api::OsSysCallsImpl& os_sys_calls_; }; diff --git a/source/common/grpc/google_async_client_impl.h b/source/common/grpc/google_async_client_impl.h index f2e9eb651b7e..da834f1b82a9 100644 --- a/source/common/grpc/google_async_client_impl.h +++ b/source/common/grpc/google_async_client_impl.h @@ -237,10 +237,10 @@ class GoogleAsyncStreamImpl : public RawAsyncStream, struct PendingMessage { PendingMessage(Buffer::InstancePtr request, bool end_stream); // End-of-stream with no additional message. - PendingMessage() : end_stream_(true) {} + PendingMessage() = default; const absl::optional buf_; - const bool end_stream_; + const bool end_stream_{true}; }; GoogleAsyncTag init_tag_{*this, GoogleAsyncTag::Operation::Init}; diff --git a/source/common/signal/signal_action.h b/source/common/signal/signal_action.h index b7ae17cd426b..afe0690d00fd 100644 --- a/source/common/signal/signal_action.h +++ b/source/common/signal/signal_action.h @@ -53,7 +53,7 @@ class SignalAction : NonCopyable { public: SignalAction() : guard_size_(sysconf(_SC_PAGE_SIZE)), - altstack_size_(std::max(guard_size_ * 4, static_cast(MINSIGSTKSZ))), altstack_() { + altstack_size_(std::max(guard_size_ * 4, static_cast(MINSIGSTKSZ))) { mapAndProtectStackMemory(); installSigHandlers(); } diff --git a/source/common/stats/stats_matcher_impl.h b/source/common/stats/stats_matcher_impl.h index 10658fd87e57..3712a466d3f6 100644 --- a/source/common/stats/stats_matcher_impl.h +++ b/source/common/stats/stats_matcher_impl.h @@ -21,7 +21,7 @@ class StatsMatcherImpl : public StatsMatcher { explicit StatsMatcherImpl(const envoy::config::metrics::v2::StatsConfig& config); // Default constructor simply allows everything. - StatsMatcherImpl() : is_inclusive_(true) {} + StatsMatcherImpl() = default; // StatsMatcher bool rejects(const std::string& name) const override; @@ -31,7 +31,7 @@ class StatsMatcherImpl : public StatsMatcher { private: // Bool indicating whether or not the StatsMatcher is including or excluding stats by default. See // StatsMatcherImpl::rejects() for much more detail. - bool is_inclusive_; + bool is_inclusive_{true}; std::vector matchers_; }; diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index 259e011f6b41..8860980017e2 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -310,7 +310,7 @@ class StatName { explicit StatName(const SymbolTable::Storage size_and_data) : size_and_data_(size_and_data) {} // Constructs an empty StatName object. - StatName() : size_and_data_(nullptr) {} + StatName() = default; // Constructs a StatName object with new storage, which must be of size // src.size(). This is used in the a flow where we first construct a StatName @@ -365,7 +365,7 @@ class StatName { bool empty() const { return size_and_data_ == nullptr || dataSize() == 0; } private: - const uint8_t* size_and_data_; + const uint8_t* size_and_data_{nullptr}; }; StatName StatNameStorage::statName() const { return StatName(bytes_.get()); } diff --git a/source/extensions/filters/network/kafka/serialization.h b/source/extensions/filters/network/kafka/serialization.h index ea0ce7ca9029..3e7fdc8e2ec6 100644 --- a/source/extensions/filters/network/kafka/serialization.h +++ b/source/extensions/filters/network/kafka/serialization.h @@ -60,7 +60,7 @@ template class Deserializer { */ template class IntDeserializer : public Deserializer { public: - IntDeserializer() : written_{0}, ready_(false){}; + IntDeserializer() : written_{0} {}; uint32_t feed(absl::string_view& data) override { const uint32_t available = std::min(sizeof(buf_) - written_, data.size()); diff --git a/source/extensions/tracers/zipkin/span_context.h b/source/extensions/tracers/zipkin/span_context.h index 78eaa5b5d1c3..6dd08c3b291b 100644 --- a/source/extensions/tracers/zipkin/span_context.h +++ b/source/extensions/tracers/zipkin/span_context.h @@ -20,7 +20,7 @@ class SpanContext { /** * Default constructor. Creates an empty context. */ - SpanContext() : trace_id_high_(0), trace_id_(0), id_(0), parent_id_(0), sampled_(false) {} + SpanContext() = default; /** * Constructor that creates a context object from the supplied trace, span and @@ -75,11 +75,11 @@ class SpanContext { bool sampled() const { return sampled_; } private: - const uint64_t trace_id_high_; - const uint64_t trace_id_; - const uint64_t id_; - const uint64_t parent_id_; - const bool sampled_; + const uint64_t trace_id_high_{0}; + const uint64_t trace_id_{0}; + const uint64_t id_{0}; + const uint64_t parent_id_{0}; + const bool sampled_{false}; }; } // namespace Zipkin diff --git a/source/extensions/tracers/zipkin/zipkin_core_types.h b/source/extensions/tracers/zipkin/zipkin_core_types.h index d9b5f10fc2d8..8c9ff909241f 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_types.h +++ b/source/extensions/tracers/zipkin/zipkin_core_types.h @@ -118,7 +118,7 @@ class Annotation : public ZipkinBase { /** * Default constructor. Creates an empty annotation. */ - Annotation() : timestamp_(0) {} + Annotation() = default; /** * Constructor that creates an annotation based on the given parameters. @@ -187,7 +187,7 @@ class Annotation : public ZipkinBase { const std::string toJson() override; private: - uint64_t timestamp_; + uint64_t timestamp_{0}; std::string value_; absl::optional endpoint_; }; diff --git a/test/common/common/lock_guard_test.cc b/test/common/common/lock_guard_test.cc index 55f505f9ab58..163b535378b2 100644 --- a/test/common/common/lock_guard_test.cc +++ b/test/common/common/lock_guard_test.cc @@ -8,10 +8,10 @@ namespace Thread { class ThreadTest : public testing::Test { protected: - ThreadTest() : a_(0), b_(0) {} - int a_ GUARDED_BY(a_mutex_); + ThreadTest() = default; + int a_ GUARDED_BY(a_mutex_){0}; MutexBasicLockable a_mutex_; - int b_; + int b_{0}; }; TEST_F(ThreadTest, TestLockGuard) { diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index c2b7a4407b7e..c0605a70c3ca 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -67,9 +67,7 @@ TEST(DeferredDeleteTest, DeferredDelete) { class DispatcherImplTest : public testing::Test { protected: - DispatcherImplTest() - : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher()), - work_finished_(false) { + DispatcherImplTest() : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher()) { dispatcher_thread_ = api_->threadFactory().createThread([this]() { // Must create a keepalive timer to keep the dispatcher from exiting. std::chrono::milliseconds time_interval(500); @@ -93,7 +91,7 @@ class DispatcherImplTest : public testing::Test { Thread::MutexBasicLockable mu_; Thread::CondVar cv_; - bool work_finished_; + bool work_finished_{false}; TimerPtr keepalive_timer_; }; diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index 83f8d3f27c2f..a3909a8ca79c 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -1658,7 +1658,7 @@ TEST_F(EdsTest, MalformedIP) { class EdsAssignmentTimeoutTest : public EdsTest { public: - EdsAssignmentTimeoutTest() : interval_timer_(nullptr) { + EdsAssignmentTimeoutTest() { EXPECT_CALL(dispatcher_, createTimer_(_)) .WillOnce(Invoke([this](Event::TimerCb cb) { timer_cb_ = cb; @@ -1671,7 +1671,7 @@ class EdsAssignmentTimeoutTest : public EdsTest { resetCluster(); } - Event::MockTimer* interval_timer_; + Event::MockTimer* interval_timer_{nullptr}; Event::TimerCb timer_cb_; }; diff --git a/test/exe/main_common_test.cc b/test/exe/main_common_test.cc index 1d412d13a6e1..4dadcb31875f 100644 --- a/test/exe/main_common_test.cc +++ b/test/exe/main_common_test.cc @@ -147,11 +147,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, MainCommonTest, class AdminRequestTest : public MainCommonTest { protected: - AdminRequestTest() - : envoy_return_(false), envoy_started_(false), envoy_finished_(false), - pause_before_run_(false), pause_after_run_(false) { - addArg("--disable-hot-restart"); - } + AdminRequestTest() { addArg("--disable-hot-restart"); } // Runs an admin request specified in path, blocking until completion, and // returning the response body. @@ -234,11 +230,11 @@ class AdminRequestTest : public MainCommonTest { absl::Notification finished_; absl::Notification resume_; absl::Notification pause_point_; - bool envoy_return_; - bool envoy_started_; - bool envoy_finished_; - bool pause_before_run_; - bool pause_after_run_; + bool envoy_return_{false}; + bool envoy_started_{false}; + bool envoy_finished_{false}; + bool pause_before_run_{false}; + bool pause_after_run_{false}; }; TEST_P(AdminRequestTest, AdminRequestGetStatsAndQuit) { diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc index d2597b2e996b..9ab3753ec855 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_alarm_test.cc @@ -16,7 +16,7 @@ namespace Quic { class TestDelegate : public quic::QuicAlarm::Delegate { public: - TestDelegate() : fired_(false) {} + TestDelegate() = default; // quic::QuicAlarm::Delegate void OnAlarm() override { fired_ = true; } @@ -25,7 +25,7 @@ class TestDelegate : public quic::QuicAlarm::Delegate { void set_fired(bool fired) { fired_ = fired; } private: - bool fired_; + bool fired_{false}; }; class EnvoyQuicAlarmTest : public ::testing::Test { diff --git a/test/extensions/quic_listeners/quiche/platform/quic_mock_log_impl.h b/test/extensions/quic_listeners/quiche/platform/quic_mock_log_impl.h index ea9326cc4eb2..f5cdc5e74499 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_mock_log_impl.h +++ b/test/extensions/quic_listeners/quiche/platform/quic_mock_log_impl.h @@ -19,7 +19,7 @@ namespace quic { // destruction(or StopCapturingLogs()). class QuicEnvoyMockLog : public QuicLogSink { public: - QuicEnvoyMockLog() : is_capturing_(false) {} + QuicEnvoyMockLog() = default; ~QuicEnvoyMockLog() override { if (is_capturing_) { @@ -43,7 +43,7 @@ class QuicEnvoyMockLog : public QuicLogSink { private: QuicLogSink* original_sink_; - bool is_capturing_; + bool is_capturing_{false}; }; // ScopedDisableExitOnDFatal is used to disable exiting the program when we encounter a diff --git a/test/integration/sds_dynamic_integration_test.cc b/test/integration/sds_dynamic_integration_test.cc index f91c7a5c383a..d260bdf6f0d0 100644 --- a/test/integration/sds_dynamic_integration_test.cc +++ b/test/integration/sds_dynamic_integration_test.cc @@ -239,7 +239,7 @@ TEST_P(SdsDynamicDownstreamIntegrationTest, WrongSecretFirst) { class SdsDynamicDownstreamCertValidationContextTest : public SdsDynamicDownstreamIntegrationTest { public: - SdsDynamicDownstreamCertValidationContextTest() : use_combined_validation_context_(false) {} + SdsDynamicDownstreamCertValidationContextTest() = default; void initialize() override { config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { @@ -285,7 +285,7 @@ class SdsDynamicDownstreamCertValidationContextTest : public SdsDynamicDownstrea void enableCombinedValidationContext(bool enable) { use_combined_validation_context_ = enable; } private: - bool use_combined_validation_context_; + bool use_combined_validation_context_{false}; }; INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, SdsDynamicDownstreamCertValidationContextTest, diff --git a/test/tools/router_check/router.h b/test/tools/router_check/router.h index 026bc1343a9a..6c507f5bc8bd 100644 --- a/test/tools/router_check/router.h +++ b/test/tools/router_check/router.h @@ -29,7 +29,7 @@ namespace Envoy { * input file. */ struct ToolConfig { - ToolConfig() : random_value_(0){}; + ToolConfig() = default; /** * @param check_config tool config json object pointer. @@ -49,7 +49,7 @@ struct ToolConfig { std::unique_ptr headers_; Router::RouteConstSharedPtr route_; - int random_value_; + int random_value_{0}; private: ToolConfig(std::unique_ptr headers, int random_value); From 33f3757210b8bef71b54ea21c416f43345a5402f Mon Sep 17 00:00:00 2001 From: Yuval Kohavi Date: Mon, 29 Jul 2019 16:03:02 -0400 Subject: [PATCH 262/542] http: connection pool - Allow a cancel callback of a request cancel other requests (#7345) Signed-off-by: Yuval Kohavi --- include/envoy/http/conn_pool.h | 2 ++ source/common/http/conn_pool_base.cc | 17 ++++++++++--- source/common/http/conn_pool_base.h | 3 +++ test/common/http/http1/conn_pool_test.cc | 32 ++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/include/envoy/http/conn_pool.h b/include/envoy/http/conn_pool.h index 7dc23a943a79..cb2d6a8607bd 100644 --- a/include/envoy/http/conn_pool.h +++ b/include/envoy/http/conn_pool.h @@ -114,6 +114,8 @@ class Instance : public Event::DeferredDeletable { * callbacks is called and the routine returns nullptr. NOTE: Once a callback * is called, the handle is no longer valid and any further cancellation * should be done by resetting the stream. + * @warning Do not call cancel() from the callbacks, as the request is implicitly canceled when + * the callbacks are called. */ virtual Cancellable* newStream(Http::StreamDecoder& response_decoder, Callbacks& callbacks) PURE; diff --git a/source/common/http/conn_pool_base.cc b/source/common/http/conn_pool_base.cc index fbc930ff4083..1cec3f6a74dc 100644 --- a/source/common/http/conn_pool_base.cc +++ b/source/common/http/conn_pool_base.cc @@ -28,10 +28,10 @@ void ConnPoolImplBase::purgePendingRequests( absl::string_view failure_reason) { // NOTE: We move the existing pending requests to a temporary list. This is done so that // if retry logic submits a new request to the pool, we don't fail it inline. - std::list pending_requests_to_purge(std::move(pending_requests_)); - while (!pending_requests_to_purge.empty()) { + pending_requests_to_purge_ = std::move(pending_requests_); + while (!pending_requests_to_purge_.empty()) { PendingRequestPtr request = - pending_requests_to_purge.front()->removeFromList(pending_requests_to_purge); + pending_requests_to_purge_.front()->removeFromList(pending_requests_to_purge_); host_->cluster().stats().upstream_rq_pending_failure_eject_.inc(); request->callbacks_.onPoolFailure(ConnectionPool::PoolFailureReason::ConnectionFailure, failure_reason, host_description); @@ -40,7 +40,16 @@ void ConnPoolImplBase::purgePendingRequests( void ConnPoolImplBase::onPendingRequestCancel(PendingRequest& request) { ENVOY_LOG(debug, "cancelling pending request"); - request.removeFromList(pending_requests_); + if (!pending_requests_to_purge_.empty()) { + // If pending_requests_to_purge_ is not empty, it means that we are called from + // with-in a onPoolFailure callback invoked in purgePendingRequests (i.e. purgePendingRequests + // is down in the call stack). Remove this request from the list as it is cancelled, + // and there is no need to call its onPoolFailure callback. + request.removeFromList(pending_requests_to_purge_); + } else { + request.removeFromList(pending_requests_); + } + host_->cluster().stats().upstream_rq_cancelled_.inc(); checkForDrained(); } diff --git a/source/common/http/conn_pool_base.h b/source/common/http/conn_pool_base.h index ab4ea2de8255..0910cadea1a5 100644 --- a/source/common/http/conn_pool_base.h +++ b/source/common/http/conn_pool_base.h @@ -48,6 +48,9 @@ class ConnPoolImplBase : protected Logger::Loggable { const Upstream::HostConstSharedPtr host_; const Upstream::ResourcePriority priority_; std::list pending_requests_; + // When calling purgePendingRequests, this list will be used to hold the requests we are about + // to purge. We need this if one cancelled requests cancels a different pending request + std::list pending_requests_to_purge_; }; } // namespace Http } // namespace Envoy diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc index df25c3ee3bf5..adc3e10472f9 100644 --- a/test/common/http/http1/conn_pool_test.cc +++ b/test/common/http/http1/conn_pool_test.cc @@ -271,6 +271,38 @@ TEST_F(Http1ConnPoolImplTest, VerifyBufferLimits) { dispatcher_.clearDeferredDeleteList(); } +/** + * Verify that canceling pending connections within the callback works. + */ +TEST_F(Http1ConnPoolImplTest, VerifyCancelInCallback) { + Http::ConnectionPool::Cancellable* handle1{}; + // In this scenario, all connections must succeed, so when + // one fails, the others are canceled. + // Note: We rely on the fact that the implementation cancels the second request first, + // to simplify the test. + ConnPoolCallbacks callbacks1; + EXPECT_CALL(callbacks1.pool_failure_, ready()).Times(0); + ConnPoolCallbacks callbacks2; + EXPECT_CALL(callbacks2.pool_failure_, ready()).WillOnce(Invoke([&]() -> void { + handle1->cancel(); + })); + + NiceMock outer_decoder; + // Create the first client. + conn_pool_.expectClientCreate(); + handle1 = conn_pool_.newStream(outer_decoder, callbacks1); + ASSERT_NE(nullptr, handle1); + + // Create the second client. + Http::ConnectionPool::Cancellable* handle2 = conn_pool_.newStream(outer_decoder, callbacks2); + ASSERT_NE(nullptr, handle2); + + // Simulate connection failure. + EXPECT_CALL(conn_pool_, onClientDestroy()); + conn_pool_.test_clients_[0].connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); + dispatcher_.clearDeferredDeleteList(); +} + /** * Tests a request that generates a new connection, completes, and then a second request that uses * the same connection. From 3eaf53fe0d31536f1d7e617c17b259cd3101cbf9 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 29 Jul 2019 16:27:10 -0400 Subject: [PATCH 263/542] http: fixing a bug in commonContinue (#7720) Risk Level: High (change to HCM) Testing: new integration test, with header Docs Changes: n/a Release Notes: n/a Fixes #7595 Signed-off-by: Alyssa Wilk --- source/common/http/conn_manager_impl.cc | 19 +++---- .../stop_iteration_and_continue_filter.cc | 53 ++++++++++++++++--- test/integration/http2_integration_test.cc | 22 ++++++++ 3 files changed, 75 insertions(+), 19 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 1e9bbbfedc58..398abbf306d3 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -858,7 +858,6 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(ActiveStreamDecoderFilte static_cast((*entry).get()), static_cast(status)); const bool new_metadata_added = processNewlyAddedMetadata(); - // If end_stream is set in headers, and a filter adds new metadata, we need to delay end_stream // in headers by inserting an empty data frame with end_stream set. The empty data frame is sent // after the new metadata. @@ -1768,16 +1767,8 @@ void ConnectionManagerImpl::ActiveStreamFilterBase::commonContinue() { doMetadata(); - // Make sure we handle filters returning StopIterationNoBuffer and then commonContinue by flushing - // the terminal fin. - const bool end_stream_with_data = complete() && !trailers(); - if (bufferedData() || end_stream_with_data) { - if (end_stream_with_data && !bufferedData()) { - // In the StopIterationNoBuffer case the ConnectionManagerImpl will not have created a - // buffer but encode/decodeData expects the buffer to exist, so create one. - bufferedData() = createBuffer(); - } - doData(end_stream_with_data); + if (bufferedData()) { + doData(complete() && !trailers()); } if (trailers()) { @@ -1867,6 +1858,12 @@ bool ConnectionManagerImpl::ActiveStreamFilterBase::commonHandleAfterDataCallbac status == FilterDataStatus::StopIterationAndWatermark) { buffer_was_streaming = status == FilterDataStatus::StopIterationAndWatermark; commonHandleBufferData(provided_data); + } else if (complete() && !trailers() && !bufferedData()) { + // If this filter is doing StopIterationNoBuffer and this stream is terminated with a zero + // byte data frame, we need to create an empty buffer to make sure that when commonContinue + // is called, the pipeline resumes with an empty data frame with end_stream = true + ASSERT(end_stream_); + bufferedData() = createBuffer(); } return false; diff --git a/test/integration/filters/stop_iteration_and_continue_filter.cc b/test/integration/filters/stop_iteration_and_continue_filter.cc index f90f9d483182..aad8c51cb960 100644 --- a/test/integration/filters/stop_iteration_and_continue_filter.cc +++ b/test/integration/filters/stop_iteration_and_continue_filter.cc @@ -9,22 +9,59 @@ namespace Envoy { -// A test filter that does StopIterationNoBuffer then continues after a 0ms alarm. +// A test filter that does StopIterationNoBuffer on end stream, then continues after a 0ms alarm. class StopIterationAndContinueFilter : public Http::PassThroughFilter { public: + void setEndStreamAndDecodeTimer() { + decode_end_stream_seen_ = true; + decode_delay_timer_ = decoder_callbacks_->dispatcher().createTimer( + [this]() -> void { decoder_callbacks_->continueDecoding(); }); + decode_delay_timer_->enableTimer(std::chrono::seconds(0)); + } + + void setEndStreamAndEncodeTimer() { + encode_end_stream_seen_ = true; + encode_delay_timer_ = decoder_callbacks_->dispatcher().createTimer( + [this]() -> void { encoder_callbacks_->continueEncoding(); }); + encode_delay_timer_->enableTimer(std::chrono::seconds(0)); + } + + Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap&, bool end_stream) override { + if (end_stream) { + setEndStreamAndDecodeTimer(); + return Http::FilterHeadersStatus::StopIteration; + } + return Http::FilterHeadersStatus::Continue; + } + Http::FilterDataStatus decodeData(Buffer::Instance&, bool end_stream) override { - RELEASE_ASSERT(!end_stream_seen_, "end stream seen twice"); + RELEASE_ASSERT(!decode_end_stream_seen_, "end stream seen twice"); + if (end_stream) { + setEndStreamAndDecodeTimer(); + } + return Http::FilterDataStatus::StopIterationNoBuffer; + } + + Http::FilterHeadersStatus encodeHeaders(Http::HeaderMap&, bool end_stream) override { + if (end_stream) { + setEndStreamAndEncodeTimer(); + return Http::FilterHeadersStatus::StopIteration; + } + return Http::FilterHeadersStatus::Continue; + } + + Http::FilterDataStatus encodeData(Buffer::Instance&, bool end_stream) override { + RELEASE_ASSERT(!encode_end_stream_seen_, "end stream seen twice"); if (end_stream) { - end_stream_seen_ = true; - delay_timer_ = decoder_callbacks_->dispatcher().createTimer( - [this]() -> void { decoder_callbacks_->continueDecoding(); }); - delay_timer_->enableTimer(std::chrono::seconds(0)); + setEndStreamAndEncodeTimer(); } return Http::FilterDataStatus::StopIterationNoBuffer; } - Event::TimerPtr delay_timer_; - bool end_stream_seen_{}; + Event::TimerPtr decode_delay_timer_; + bool decode_end_stream_seen_{}; + Event::TimerPtr encode_delay_timer_; + bool encode_end_stream_seen_{}; }; class StopIterationAndContinueFilterConfig diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index 22929816f97f..be8fc519633f 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1112,6 +1112,28 @@ TEST_P(Http2IntegrationTest, PauseAndResume) { // Now send the final data frame and make sure it gets proxied. codec_client_->sendData(*request_encoder_, 0, true); ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); + upstream_request_->encodeHeaders(default_response_headers_, false); + + response->waitForHeaders(); + upstream_request_->encodeData(0, true); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); +} + +TEST_P(Http2IntegrationTest, PauseAndResumeHeadersOnly) { + config_helper_.addFilter(R"EOF( + name: stop-iteration-and-continue-filter + config: {} + )EOF"); + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); + upstream_request_->encodeHeaders(default_response_headers_, true); response->waitForEndStream(); ASSERT_TRUE(response->complete()); From e0552cc11fefd559944166ab736935db3f605d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Mon, 29 Jul 2019 16:41:32 -0400 Subject: [PATCH 264/542] test: fix RuntimeFromNonWorkerThreads (#7746) While working on #7574, I am getting consistent timeouts for this test. To repro the timeout, I added a sleep before step 2. After this change, I can't repro anymore. Risk: n/a (test only) --- test/common/runtime/runtime_impl_test.cc | 33 ++++++++++++++++++------ 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 3d39c6b8e9a7..41f80ba3b360 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -623,25 +623,42 @@ TEST_F(StaticLoaderImplTest, RuntimeFromNonWorkerThreads) { // Now set up a test thread which verifies foo -> bar // // Then change foo and make sure the test thread picks up the change. + bool read_bar = false; + bool updated_eep = false; Thread::MutexBasicLockable mutex; Thread::CondVar foo_read; Thread::CondVar foo_changed; const Snapshot* original_thread_snapshot_pointer = nullptr; auto thread = Thread::threadFactoryForTest().createThread([&]() { - Thread::LockGuard lock(mutex); - EXPECT_EQ("bar", loader_->threadsafeSnapshot()->get("foo")); - original_thread_snapshot_pointer = loader_->threadsafeSnapshot().get(); - EXPECT_EQ(original_thread_snapshot_pointer, loader_->threadsafeSnapshot().get()); - foo_read.notifyOne(); + { + Thread::LockGuard lock(mutex); + EXPECT_EQ("bar", loader_->threadsafeSnapshot()->get("foo")); + read_bar = true; + original_thread_snapshot_pointer = loader_->threadsafeSnapshot().get(); + EXPECT_EQ(original_thread_snapshot_pointer, loader_->threadsafeSnapshot().get()); + foo_read.notifyOne(); + } - foo_changed.wait(mutex); - EXPECT_EQ("eep", loader_->threadsafeSnapshot()->get("foo")); + { + Thread::LockGuard lock(mutex); + if (!updated_eep) { + foo_changed.wait(mutex); + } + EXPECT_EQ("eep", loader_->threadsafeSnapshot()->get("foo")); + } }); { Thread::LockGuard lock(mutex); - foo_read.wait(mutex); + if (!read_bar) { + foo_read.wait(mutex); + } loader_->mergeValues({{"foo", "eep"}}); + updated_eep = true; + } + + { + Thread::LockGuard lock(mutex); foo_changed.notifyOne(); EXPECT_EQ("eep", loader_->threadsafeSnapshot()->get("foo")); } From 245888b2c01561670927427883dc7b9786cae13c Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Mon, 29 Jul 2019 23:03:15 +0200 Subject: [PATCH 265/542] Visibility & _envoy_cc_test_infrastructure_library (#7749) In envoyproxy/nighthawk#101 we update coverage to the new methodology. For that to work we need to be able to consume _lib_internal_only targets which aren't visible externally today. This proposes a change so we can make that happen. The naming is fairly clear that these targets shouldn't be used otherwise. Risk Level: low Testing: N/A Signed-off-by: Otto van der Schaaf --- bazel/envoy_test.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index c836096913fa..a2d169a4f900 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -143,8 +143,8 @@ def envoy_cc_test( repository = repository, tags = test_lib_tags, copts = copts, - # Restrict only to the code coverage tools. - visibility = ["@envoy//test/coverage:__pkg__"], + # Allow public visibility so these can be consumed in coverage tests in external projects. + visibility = ["//visibility:public"], ) native.cc_test( name = name, From b122e9a69e01ebe50a4cd98eb841016fb8159847 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 29 Jul 2019 19:29:38 -0400 Subject: [PATCH 266/542] fuzz: replace invalid characters in upstream metadata for header parser (#7436) Signed-off-by: Asra Ali --- ...e-header_parser_fuzz_test-5163306626580480 | 80 +++++++++++++++++++ test/fuzz/utility.h | 21 ++++- 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 test/common/router/header_parser_corpus/clusterfuzz-testcase-header_parser_fuzz_test-5163306626580480 diff --git a/test/common/router/header_parser_corpus/clusterfuzz-testcase-header_parser_fuzz_test-5163306626580480 b/test/common/router/header_parser_corpus/clusterfuzz-testcase-header_parser_fuzz_test-5163306626580480 new file mode 100644 index 000000000000..57041ba39771 --- /dev/null +++ b/test/common/router/header_parser_corpus/clusterfuzz-testcase-header_parser_fuzz_test-5163306626580480 @@ -0,0 +1,80 @@ +headers_to_add { + header { + key: "P" + value: "%PER_REQUEST_STATE(oB]$T)%" + } +} +headers_to_add { + header { + key: "A" + value: "%START_TIME(B%444ssa4%s4%>TME(B%128sY$T_)%" + } +} +headers_to_add { + header { + key: "A" + value: "%PER_REQUEST_STATE(dB]$T)%" + } +} +headers_to_add { + header { + key: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + value: "%PER_REQUEST_STATE(dB]$T)%" + } + append { + value: true + } +} +headers_to_add { + header { + key: "]" + value: "%UPSTREAM_METADATA([\"\", \"\"])%" + } +} +stream_info { + dynamic_metadata { + filter_metadata { + key: "\000\000{\000+p" + value { + } + } + filter_metadata { + key: "\000}" + value { + } + } + filter_metadata { + key: "\000}" + value { + } + } + filter_metadata { + key: "K" + value { + fields { + key: "" + value { + } + } + } + } + } + upstream_metadata { + filter_metadata { + key: "" + value { + fields { + key: "" + value { + string_value: "c\000\000\000\000\000\000\000" + } + } + } + } + filter_metadata { + key: "-" + value { + } + } + } +} diff --git a/test/fuzz/utility.h b/test/fuzz/utility.h index 0f869149e2bf..155226aa026c 100644 --- a/test/fuzz/utility.h +++ b/test/fuzz/utility.h @@ -52,6 +52,23 @@ inline Protobuf::RepeatedPtrField repla return processed; } +inline envoy::api::v2::core::Metadata +replaceInvalidStringValues(const envoy::api::v2::core::Metadata& upstream_metadata) { + envoy::api::v2::core::Metadata processed = upstream_metadata; + for (auto& metadata_struct : *processed.mutable_filter_metadata()) { + // Metadata fields consist of keyed Structs, which is a map of dynamically typed values. These + // values can be null, a number, a string, a boolean, a list of values, or a recursive struct. + // This clears any invalid characters in string values. It may not be likely a coverage-driven + // fuzzer will explore recursive structs, so this case is not handled here. + for (auto& field : *metadata_struct.second.mutable_fields()) { + if (field.second.kind_case() == ProtobufWkt::Value::kStringValue) { + field.second.set_string_value(replaceInvalidCharacters(field.second.string_value())); + } + } + } + return processed; +} + // Convert from test proto Headers to TestHeaderMapImpl. inline Http::TestHeaderMapImpl fromHeaders( const test::fuzz::Headers& headers, @@ -103,8 +120,8 @@ inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info, test_stream_info.response_code_ = stream_info.response_code().value(); } auto upstream_host = std::make_shared>(); - auto upstream_metadata = - std::make_shared(stream_info.upstream_metadata()); + auto upstream_metadata = std::make_shared( + replaceInvalidStringValues(stream_info.upstream_metadata())); ON_CALL(*upstream_host, metadata()).WillByDefault(testing::Return(upstream_metadata)); test_stream_info.upstream_host_ = upstream_host; auto address = Network::Utility::resolveUrl("tcp://10.0.0.1:443"); From ab7ab69d7976acacc7269acfc516cfb1f5836564 Mon Sep 17 00:00:00 2001 From: Vasili Syrakis Date: Tue, 30 Jul 2019 09:30:25 +1000 Subject: [PATCH 267/542] examples: add an example of importing local lua code (#7629) Signed-off-by: Vasili Syrakis --- configs/configgen.sh | 6 +++++- examples/BUILD | 1 + examples/lua/Dockerfile-proxy | 1 + examples/lua/envoy.yaml | 4 +++- examples/lua/lib/mylibrary.lua | 7 +++++++ 5 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 examples/lua/lib/mylibrary.lua diff --git a/configs/configgen.sh b/configs/configgen.sh index 2e82ebff3dd9..cbfa59a79c85 100755 --- a/configs/configgen.sh +++ b/configs/configgen.sh @@ -8,6 +8,7 @@ OUT_DIR="$1" shift mkdir -p "$OUT_DIR/certs" +mkdir -p "$OUT_DIR/lib" "$CONFIGGEN" "$OUT_DIR" for FILE in $*; do @@ -15,6 +16,9 @@ for FILE in $*; do *.pem) cp "$FILE" "$OUT_DIR/certs" ;; + *.lua) + cp "$FILE" "$OUT_DIR/lib" + ;; *) FILENAME="$(echo $FILE | sed -e 's/.*examples\///g')" @@ -25,4 +29,4 @@ for FILE in $*; do done # tar is having issues with -C for some reason so just cd into OUT_DIR. -(cd "$OUT_DIR"; tar -hcvf example_configs.tar *.yaml certs/*.pem) +(cd "$OUT_DIR"; tar -hcvf example_configs.tar *.yaml certs/*.pem lib/*.lua) diff --git a/examples/BUILD b/examples/BUILD index a36e43453ad7..cea6483751a4 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -22,6 +22,7 @@ filegroup( "jaeger-tracing/service1-envoy-jaeger.yaml", "jaeger-tracing/service2-envoy-jaeger.yaml", "lua/envoy.yaml", + "lua/lib/mylibrary.lua", "zipkin-tracing/front-envoy-zipkin.yaml", "zipkin-tracing/service1-envoy-zipkin.yaml", "zipkin-tracing/service2-envoy-zipkin.yaml", diff --git a/examples/lua/Dockerfile-proxy b/examples/lua/Dockerfile-proxy index 92b320ea1487..5ba5c9c33a0d 100644 --- a/examples/lua/Dockerfile-proxy +++ b/examples/lua/Dockerfile-proxy @@ -1,2 +1,3 @@ FROM envoyproxy/envoy-dev:latest +ADD ./lib/mylibrary.lua /lib/mylibrary.lua CMD /usr/local/bin/envoy -c /etc/envoy.yaml -l debug --service-cluster proxy diff --git a/examples/lua/envoy.yaml b/examples/lua/envoy.yaml index aed6f977299b..4f98481091fb 100644 --- a/examples/lua/envoy.yaml +++ b/examples/lua/envoy.yaml @@ -28,8 +28,10 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.config.filter.http.lua.v2.Lua inline_code: | + local mylibrary = require("lib.mylibrary") + function envoy_on_request(request_handle) - request_handle:headers():add("foo", "bar") + request_handle:headers():add("foo", mylibrary.foobar()) end function envoy_on_response(response_handle) body_size = response_handle:body():length() diff --git a/examples/lua/lib/mylibrary.lua b/examples/lua/lib/mylibrary.lua new file mode 100644 index 000000000000..6b4ffed1b476 --- /dev/null +++ b/examples/lua/lib/mylibrary.lua @@ -0,0 +1,7 @@ +M = {} + +function M.foobar() + return "bar" +end + +return M From 3a7f9d9ae60abaa3546a5e28afba0815a2642782 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 29 Jul 2019 21:34:17 -0700 Subject: [PATCH 268/542] ci: always publish artifacts in Azure (#7754) Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index fe3f6aa4c667..13818d477f02 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -45,3 +45,4 @@ jobs: inputs: pathtoPublish: "$(Build.StagingDirectory)/envoy" artifactName: $(CI_TARGET) + condition: always() From 4b77779f8261f522749220c6ac82e8798dcc9434 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 30 Jul 2019 05:50:44 -0700 Subject: [PATCH 269/542] ci: push build image to gcr.io (#7740) * ci: push build image to gcr.io Signed-off-by: Lizan Zhou --- .circleci/config.yml | 6 +-- ci/build_container/docker_push.sh | 88 +++++++++++++++++++------------ 2 files changed, 54 insertions(+), 40 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9352925decd6..9a4680614d63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,11 +123,7 @@ jobs: - run: ci/do_circle_ci.sh check_spelling_pedantic build_image: docker: - - image: circleci/python:3.7 - environment: - # See comment in do_circle_ci.sh for why we do this. - NUM_CPUS: 8 - resource_class: xlarge + - image: google/cloud-sdk steps: - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - checkout diff --git a/ci/build_container/docker_push.sh b/ci/build_container/docker_push.sh index 57d8f1290f96..6edc2c2eeb7b 100755 --- a/ci/build_container/docker_push.sh +++ b/ci/build_container/docker_push.sh @@ -8,45 +8,63 @@ set -e branch_want_push='false' for branch in "master" do - if [ "$CIRCLE_BRANCH" == "$branch" ] - then + if [[ "$CIRCLE_BRANCH" == "$branch" ]]; then branch_want_push='true' fi done -if [ -z "$CIRCLE_PULL_REQUEST" ] && [ "$branch_want_push" == "true" ] -then - diff_want_push='false' - if [[ $(git diff HEAD^ ci/build_container/) ]]; then - echo "There are changes in the ci/build_container directory" - diff_want_push='true' - elif [[ $(git diff HEAD^ bazel/) ]]; then - echo "There are changes in the bazel directory" - diff_want_push='true' - fi - if [ "$diff_want_push" == "true" ]; then - cd ci/build_container - docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD" - - for distro in ubuntu centos - do - echo "Updating envoyproxy/envoy-build-${distro} image" - LINUX_DISTRO=$distro ./docker_build.sh - docker push envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" - docker tag envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" envoyproxy/envoy-build-"${distro}":latest - docker push envoyproxy/envoy-build-"${distro}":latest - - if [ "$distro" == "ubuntu" ] - then - echo "Updating envoyproxy/envoy-build image" - docker tag envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" envoyproxy/envoy-build:"$CIRCLE_SHA1" - docker push envoyproxy/envoy-build:"$CIRCLE_SHA1" - docker tag envoyproxy/envoy-build:"$CIRCLE_SHA1" envoyproxy/envoy-build:latest - docker push envoyproxy/envoy-build:latest - fi - done - else - echo "The ci/build_container directory has not changed" + +if [[ -z "${CIRCLE_PR_NUMBER}" && "${branch_want_push}" == "true" ]]; then + diff_base="HEAD^" +else + git fetch https://github.com/envoyproxy/envoy.git master + diff_base="$(git merge-base HEAD FETCH_HEAD)" +fi + +diff_want_build='false' +if [[ ! -z $(git diff --name-only "${diff_base}..HEAD" ci/build_container/) ]]; then + echo "There are changes in the ci/build_container directory" + diff_want_build='true' +fi + +cd ci/build_container +if [ "$diff_want_build" == "true" ]; then + for distro in ubuntu centos + do + echo "Updating envoyproxy/envoy-build-${distro} image" + LINUX_DISTRO=$distro ./docker_build.sh + done +else + echo "The ci/build_container directory has not changed" + exit 0 +fi + +if [[ -z "${CIRCLE_PR_NUMBER}" && "${branch_want_push}" == "true" ]]; then + docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD" + + if [[ ! -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then + echo ${GCP_SERVICE_ACCOUNT_KEY} | base64 --decode | gcloud auth activate-service-account --key-file=- fi + + for distro in ubuntu centos + do + echo "Updating envoyproxy/envoy-build-${distro} image" + docker push envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" + docker tag envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" envoyproxy/envoy-build-"${distro}":latest + docker push envoyproxy/envoy-build-"${distro}":latest + + if [[ "$distro" == "ubuntu" ]] + then + echo "Updating envoyproxy/envoy-build image" + docker tag envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" envoyproxy/envoy-build:"$CIRCLE_SHA1" + docker push envoyproxy/envoy-build:"$CIRCLE_SHA1" + docker tag envoyproxy/envoy-build:"$CIRCLE_SHA1" envoyproxy/envoy-build:latest + docker push envoyproxy/envoy-build:latest + + echo "Updating gcr.io/envoy-ci/envoy-build image" + docker tag envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" gcr.io/envoy-ci/envoy-build:"$CIRCLE_SHA1" + docker push gcr.io/envoy-ci/envoy-build:"$CIRCLE_SHA1" + fi + done else echo 'Ignoring PR branch for docker push.' fi From 23d481ed4d55b60bb5a93ceb29039b409c7cc40a Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 30 Jul 2019 05:51:12 -0700 Subject: [PATCH 270/542] ci: use bazelisk in macCI and build_image (#7739) * ci: use bazelisk in macCI and build_image Signed-off-by: Lizan Zhou --- .bazelversion | 1 + ci/build_container/build_container_centos.sh | 8 -------- ci/build_container/build_container_common.sh | 21 ++++++++++++++------ ci/build_container/build_container_ubuntu.sh | 8 +------- ci/mac_ci_setup.sh | 3 +-- 5 files changed, 18 insertions(+), 23 deletions(-) create mode 100644 .bazelversion diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 000000000000..697f087f376a --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +0.28.0 diff --git a/ci/build_container/build_container_centos.sh b/ci/build_container/build_container_centos.sh index bf45bcc22a65..b1d15d166db0 100755 --- a/ci/build_container/build_container_centos.sh +++ b/ci/build_container/build_container_centos.sh @@ -12,14 +12,6 @@ yum install -y devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-binutils java- ln -s /usr/bin/cmake3 /usr/bin/cmake ln -s /usr/bin/ninja-build /usr/bin/ninja -BAZEL_VERSION="$(curl -s https://api.github.com/repos/bazelbuild/bazel/releases/latest | - python -c "import json, sys; print json.load(sys.stdin)['tag_name']")" -BAZEL_INSTALLER="bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" -curl -OL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/${BAZEL_INSTALLER}" -chmod u+x "./${BAZEL_INSTALLER}" -"./${BAZEL_INSTALLER}" -rm "./${BAZEL_INSTALLER}" - # SLES 11 has older glibc than CentOS 7, so pre-built binary for it works on CentOS 7 LLVM_VERSION=8.0.0 LLVM_RELEASE="clang+llvm-${LLVM_VERSION}-x86_64-linux-sles11.3" diff --git a/ci/build_container/build_container_common.sh b/ci/build_container/build_container_common.sh index 4d3218d6482c..a80685ee6ba5 100755 --- a/ci/build_container/build_container_common.sh +++ b/ci/build_container/build_container_common.sh @@ -1,8 +1,17 @@ #!/bin/bash -e -# buildifier -VERSION=0.25.0 -SHA256=6e6aea35b2ea2b4951163f686dfbfe47b49c840c56b873b3a7afe60939772fc1 -curl --location --output /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/"$VERSION"/buildifier \ - && echo "$SHA256" '/usr/local/bin/buildifier' | sha256sum --check \ - && chmod +x /usr/local/bin/buildifier +if [[ "$(uname -m)" == "x86_64" ]]; then + # buildifier + VERSION=0.28.0 + SHA256=3d474be62f8e18190546881daf3c6337d857bf371faf23f508e9b456b0244267 + curl --location --output /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/"$VERSION"/buildifier \ + && echo "$SHA256 /usr/local/bin/buildifier" | sha256sum --check \ + && chmod +x /usr/local/bin/buildifier + + # bazelisk + VERSION=0.0.8 + SHA256=5fced4fec06bf24beb631837fa9497b6698f34041463d9188610dfa7b91f4f8d + curl --location --output /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v${VERSION}/bazelisk-linux-amd64 \ + && echo "$SHA256 /usr/local/bin/bazel" | sha256sum --check \ + && chmod +x /usr/local/bin/bazel +fi diff --git a/ci/build_container/build_container_ubuntu.sh b/ci/build_container/build_container_ubuntu.sh index 08697220d797..6ca415d11a03 100755 --- a/ci/build_container/build_container_ubuntu.sh +++ b/ci/build_container/build_container_ubuntu.sh @@ -23,7 +23,7 @@ case $ARCH in ldconfig ;; 'x86_64' ) - wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" apt-get update apt-get install -y clang-8 clang-format-8 clang-tidy-8 lld-8 libc++-8-dev libc++abi-8-dev @@ -49,12 +49,6 @@ case $ARCH in -o /usr/local/bin/bazel chmod +x /usr/local/bin/bazel ;; - 'x86_64' ) - echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list - curl https://bazel.build/bazel-release.pub.gpg | apt-key add - - apt-get update - apt-get install -y bazel - ;; esac apt-get install -y aspell rm -rf /var/lib/apt/lists/* diff --git a/ci/mac_ci_setup.sh b/ci/mac_ci_setup.sh index 8e97d63b06a9..42cb7c63faa5 100755 --- a/ci/mac_ci_setup.sh +++ b/ci/mac_ci_setup.sh @@ -5,7 +5,6 @@ # Setup bazelbuild tap brew tap bazelbuild/tap -brew tap-pin bazelbuild/tap function is_installed { brew ls --versions "$1" >/dev/null @@ -25,7 +24,7 @@ if ! brew update; then exit 1 fi -DEPS="automake bazelbuild/tap/bazel cmake coreutils go libtool wget ninja" +DEPS="automake bazelbuild/tap/bazelisk cmake coreutils go libtool wget ninja" for DEP in ${DEPS} do is_installed "${DEP}" || install "${DEP}" From e2ca70a77f2631c42812dbe084690dc8362738e7 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 30 Jul 2019 13:01:16 -0700 Subject: [PATCH 271/542] ci: add missed gcloud command (#7764) Signed-off-by: Lizan Zhou --- ci/build_container/docker_push.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/build_container/docker_push.sh b/ci/build_container/docker_push.sh index 6edc2c2eeb7b..395adef8a754 100755 --- a/ci/build_container/docker_push.sh +++ b/ci/build_container/docker_push.sh @@ -14,10 +14,10 @@ do done if [[ -z "${CIRCLE_PR_NUMBER}" && "${branch_want_push}" == "true" ]]; then - diff_base="HEAD^" + diff_base="HEAD^" else - git fetch https://github.com/envoyproxy/envoy.git master - diff_base="$(git merge-base HEAD FETCH_HEAD)" + git fetch https://github.com/envoyproxy/envoy.git master + diff_base="$(git merge-base HEAD FETCH_HEAD)" fi diff_want_build='false' @@ -43,6 +43,7 @@ if [[ -z "${CIRCLE_PR_NUMBER}" && "${branch_want_push}" == "true" ]]; then if [[ ! -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then echo ${GCP_SERVICE_ACCOUNT_KEY} | base64 --decode | gcloud auth activate-service-account --key-file=- + gcloud auth configure-docker fi for distro in ubuntu centos From c4994020c23001ed7a36c2f02a6eb6f019adf79a Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 30 Jul 2019 16:01:43 -0400 Subject: [PATCH 272/542] fix clang-tidy errors in null stats (#7760) Signed-off-by: Joshua Marantz --- source/common/stats/null_counter.h | 2 +- source/common/stats/null_gauge.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/stats/null_counter.h b/source/common/stats/null_counter.h index 361ff739b954..61e86d16b644 100644 --- a/source/common/stats/null_counter.h +++ b/source/common/stats/null_counter.h @@ -20,7 +20,7 @@ class NullCounterImpl : public MetricImpl { // will not be able to access the SymbolTable& to free the symbols. An RAII // alternative would be to store the SymbolTable reference in the // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(symbolTable()); + MetricImpl::clear(symbol_table_); } void add(uint64_t) override {} diff --git a/source/common/stats/null_gauge.h b/source/common/stats/null_gauge.h index c5491caf736f..c3e7ccc46871 100644 --- a/source/common/stats/null_gauge.h +++ b/source/common/stats/null_gauge.h @@ -20,7 +20,7 @@ class NullGaugeImpl : public MetricImpl { // will not be able to access the SymbolTable& to free the symbols. An RAII // alternative would be to store the SymbolTable reference in the // MetricImpl, costing 8 bytes per stat. - MetricImpl::clear(symbolTable()); + MetricImpl::clear(symbol_table_); } void add(uint64_t) override {} From b22d2b5cf09f779962cfedaaab24969f384cbc48 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 30 Jul 2019 18:12:20 -0400 Subject: [PATCH 273/542] remove gogo nullable extension from repeated fields (#7669) Due to a seg fault issue with the gogo protobuf library [https://github.com/gogo/protobuf/issues/568], non nullable repeated fields in a proto will cause proto.Merge(dst, src) to panic. The nullable field setting was first added by @kyessenov when he was re-organizing the protos. Unfortunately, people have been copy pasting it across several areas in the Envoy proto. To keep the impact radius to a minimum, I have updated only the fields that are currently causing the segfault (in go-control-plane) for us. Its also partly against proto principles. You should be able to determine if a field is set or not. This non-nullable setting in gogo will insist on initializing the field to default values. Risk Level: to go control plane users Signed-off-by: Shriram Rajagopalan --- api/envoy/admin/v2alpha/config_dump.proto | 27 +++++++++---------- api/envoy/api/v2/cds.proto | 7 ++--- api/envoy/api/v2/core/address.proto | 3 +-- api/envoy/api/v2/discovery.proto | 4 +-- api/envoy/api/v2/eds.proto | 2 +- api/envoy/api/v2/endpoint/endpoint.proto | 2 +- api/envoy/api/v2/lds.proto | 6 ++--- api/envoy/api/v2/listener/listener.proto | 2 +- api/envoy/api/v2/rds.proto | 5 ++-- api/envoy/api/v2/route/route.proto | 4 +-- api/envoy/config/bootstrap/v2/bootstrap.proto | 6 ++--- .../config/common/tap/v2alpha/common.proto | 3 +-- .../network/dubbo_proxy/v2alpha1/route.proto | 6 ++--- .../v2/http_connection_manager.proto | 8 +++--- .../network/redis_proxy/v2/redis_proxy.proto | 4 +-- .../network/thrift_proxy/v2alpha1/route.proto | 6 ++--- 16 files changed, 43 insertions(+), 52 deletions(-) diff --git a/api/envoy/admin/v2alpha/config_dump.proto b/api/envoy/admin/v2alpha/config_dump.proto index 8ff5ba8fa102..1025b17fb117 100644 --- a/api/envoy/admin/v2alpha/config_dump.proto +++ b/api/envoy/admin/v2alpha/config_dump.proto @@ -33,7 +33,7 @@ message ConfigDump { // * *clusters*: :ref:`ClustersConfigDump ` // * *listeners*: :ref:`ListenersConfigDump ` // * *routes*: :ref:`RoutesConfigDump ` - repeated google.protobuf.Any configs = 1 [(gogoproto.nullable) = false]; + repeated google.protobuf.Any configs = 1; } // This message describes the bootstrap configuration that Envoy was started with. This includes @@ -41,7 +41,7 @@ message ConfigDump { // the static portions of an Envoy configuration by reusing the output as the bootstrap // configuration for another Envoy. message BootstrapConfigDump { - envoy.config.bootstrap.v2.Bootstrap bootstrap = 1 [(gogoproto.nullable) = false]; + envoy.config.bootstrap.v2.Bootstrap bootstrap = 1; // The timestamp when the BootstrapConfig was last updated. google.protobuf.Timestamp last_updated = 2; @@ -81,23 +81,23 @@ message ListenersConfigDump { } // The statically loaded listener configs. - repeated StaticListener static_listeners = 2 [(gogoproto.nullable) = false]; + repeated StaticListener static_listeners = 2; // The dynamically loaded active listeners. These are listeners that are available to service // data plane traffic. - repeated DynamicListener dynamic_active_listeners = 3 [(gogoproto.nullable) = false]; + repeated DynamicListener dynamic_active_listeners = 3; // The dynamically loaded warming listeners. These are listeners that are currently undergoing // warming in preparation to service data plane traffic. Note that if attempting to recreate an // Envoy configuration from a configuration dump, the warming listeners should generally be // discarded. - repeated DynamicListener dynamic_warming_listeners = 4 [(gogoproto.nullable) = false]; + repeated DynamicListener dynamic_warming_listeners = 4; // The dynamically loaded draining listeners. These are listeners that are currently undergoing // draining in preparation to stop servicing data plane traffic. Note that if attempting to // recreate an Envoy configuration from a configuration dump, the draining listeners should // generally be discarded. - repeated DynamicListener dynamic_draining_listeners = 5 [(gogoproto.nullable) = false]; + repeated DynamicListener dynamic_draining_listeners = 5; } // Envoy's cluster manager fills this message with all currently known clusters. Cluster @@ -134,17 +134,17 @@ message ClustersConfigDump { } // The statically loaded cluster configs. - repeated StaticCluster static_clusters = 2 [(gogoproto.nullable) = false]; + repeated StaticCluster static_clusters = 2; // The dynamically loaded active clusters. These are clusters that are available to service // data plane traffic. - repeated DynamicCluster dynamic_active_clusters = 3 [(gogoproto.nullable) = false]; + repeated DynamicCluster dynamic_active_clusters = 3; // The dynamically loaded warming clusters. These are clusters that are currently undergoing // warming in preparation to service data plane traffic. Note that if attempting to recreate an // Envoy configuration from a configuration dump, the warming clusters should generally be // discarded. - repeated DynamicCluster dynamic_warming_clusters = 4 [(gogoproto.nullable) = false]; + repeated DynamicCluster dynamic_warming_clusters = 4; } // Envoy's RDS implementation fills this message with all currently loaded routes, as described by @@ -175,10 +175,10 @@ message RoutesConfigDump { } // The statically loaded route configs. - repeated StaticRouteConfig static_route_configs = 2 [(gogoproto.nullable) = false]; + repeated StaticRouteConfig static_route_configs = 2; // The dynamically loaded route configs. - repeated DynamicRouteConfig dynamic_route_configs = 3 [(gogoproto.nullable) = false]; + repeated DynamicRouteConfig dynamic_route_configs = 3; } // Envoy's scoped RDS implementation fills this message with all currently loaded route @@ -214,11 +214,10 @@ message ScopedRoutesConfigDump { } // The statically loaded scoped route configs. - repeated InlineScopedRouteConfigs inline_scoped_route_configs = 1 [(gogoproto.nullable) = false]; + repeated InlineScopedRouteConfigs inline_scoped_route_configs = 1; // The dynamically loaded scoped route configs. - repeated DynamicScopedRouteConfigs dynamic_scoped_route_configs = 2 - [(gogoproto.nullable) = false]; + repeated DynamicScopedRouteConfigs dynamic_scoped_route_configs = 2; } // Envoys SDS implementation fills this message with all secrets fetched dynamically via SDS. diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index bddcf1d56d37..c8542387fc0d 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -127,11 +127,8 @@ message Cluster { EdsClusterConfig eds_cluster_config = 3; // The timeout for new network connections to hosts in the cluster. - google.protobuf.Duration connect_timeout = 4 [ - (validate.rules).duration.gt = {}, - (gogoproto.stdduration) = true, - (gogoproto.nullable) = false - ]; + google.protobuf.Duration connect_timeout = 4 + [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; // Soft limit on size of the cluster’s connections read and write buffers. If // unspecified, an implementation defined default is applied (1MiB). diff --git a/api/envoy/api/v2/core/address.proto b/api/envoy/api/v2/core/address.proto index 3d597f56bec5..88689e2648ce 100644 --- a/api/envoy/api/v2/core/address.proto +++ b/api/envoy/api/v2/core/address.proto @@ -83,8 +83,7 @@ message TcpKeepalive { message BindConfig { // The address to bind to when creating a socket. - SocketAddress source_address = 1 - [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + SocketAddress source_address = 1 [(validate.rules).message.required = true]; // Whether to set the *IP_FREEBIND* option when creating the socket. When this // flag is set to true, allows the :ref:`source_address diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index 2bbe53cfe24f..a3072a817aa4 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -65,7 +65,7 @@ message DiscoveryResponse { string version_info = 1; // The response resources. These resources are typed and depend on the API being called. - repeated google.protobuf.Any resources = 2 [(gogoproto.nullable) = false]; + repeated google.protobuf.Any resources = 2; // [#not-implemented-hide:] // Canary is used to support two Envoy command line flags: @@ -196,7 +196,7 @@ message DeltaDiscoveryResponse { // The response resources. These are typed resources, whose types must match // the type_url field. - repeated Resource resources = 2 [(gogoproto.nullable) = false]; + repeated Resource resources = 2; // field id 3 IS available! diff --git a/api/envoy/api/v2/eds.proto b/api/envoy/api/v2/eds.proto index dd66431c5d56..d680ef7ea5aa 100644 --- a/api/envoy/api/v2/eds.proto +++ b/api/envoy/api/v2/eds.proto @@ -59,7 +59,7 @@ message ClusterLoadAssignment { string cluster_name = 1 [(validate.rules).string.min_bytes = 1]; // List of endpoints to load balance to. - repeated endpoint.LocalityLbEndpoints endpoints = 2 [(gogoproto.nullable) = false]; + repeated endpoint.LocalityLbEndpoints endpoints = 2; // Map of named endpoints that can be referenced in LocalityLbEndpoints. map named_endpoints = 5; diff --git a/api/envoy/api/v2/endpoint/endpoint.proto b/api/envoy/api/v2/endpoint/endpoint.proto index 496bb8cdb278..7abb7ea5f3c9 100644 --- a/api/envoy/api/v2/endpoint/endpoint.proto +++ b/api/envoy/api/v2/endpoint/endpoint.proto @@ -94,7 +94,7 @@ message LocalityLbEndpoints { core.Locality locality = 1; // The group of endpoints belonging to the locality specified. - repeated LbEndpoint lb_endpoints = 2 [(gogoproto.nullable) = false]; + repeated LbEndpoint lb_endpoints = 2; // Optional: Per priority/region/zone/sub_zone weight; at least 1. The load // balancing weight for a locality is divided by the sum of the weights of all diff --git a/api/envoy/api/v2/lds.proto b/api/envoy/api/v2/lds.proto index e42c6e04b779..f6081fbc9af3 100644 --- a/api/envoy/api/v2/lds.proto +++ b/api/envoy/api/v2/lds.proto @@ -54,7 +54,7 @@ message Listener { // The address that the listener should listen on. In general, the address must be unique, though // that is governed by the bind rules of the OS. E.g., multiple listeners can listen on port 0 on // Linux as the actual port will be allocated by the OS. - core.Address address = 2 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + core.Address address = 2 [(validate.rules).message.required = true]; // A list of filter chains to consider for this listener. The // :ref:`FilterChain ` with the most specific @@ -63,7 +63,7 @@ message Listener { // // Example using SNI for filter chain selection can be found in the // :ref:`FAQ entry `. - repeated listener.FilterChain filter_chains = 3 [(gogoproto.nullable) = false]; + repeated listener.FilterChain filter_chains = 3; // If a connection is redirected using *iptables*, the port on which the proxy // receives it might be different from the original destination address. When this flag is set to @@ -129,7 +129,7 @@ message Listener { // :ref:`protocol ` is :ref:'UDP // `. // UDP listeners currently support a single filter. - repeated listener.ListenerFilter listener_filters = 9 [(gogoproto.nullable) = false]; + repeated listener.ListenerFilter listener_filters = 9; // The timeout to wait for all listener filters to complete operation. If the timeout is reached, // the accepted socket is closed without a connection being created. Specify 0 to disable the diff --git a/api/envoy/api/v2/listener/listener.proto b/api/envoy/api/v2/listener/listener.proto index 157def72132b..77f753501fdd 100644 --- a/api/envoy/api/v2/listener/listener.proto +++ b/api/envoy/api/v2/listener/listener.proto @@ -173,7 +173,7 @@ message FilterChain { // connections established with the listener. Order matters as the filters are // processed sequentially as connection events happen. Note: If the filter // list is empty, the connection will close by default. - repeated Filter filters = 3 [(gogoproto.nullable) = false]; + repeated Filter filters = 3; // Whether the listener should expect a PROXY protocol V1 header on new // connections. If this option is enabled, the listener will assume that that diff --git a/api/envoy/api/v2/rds.proto b/api/envoy/api/v2/rds.proto index 351c199fdeb5..2d82e4ad00cf 100644 --- a/api/envoy/api/v2/rds.proto +++ b/api/envoy/api/v2/rds.proto @@ -69,7 +69,7 @@ message RouteConfiguration { string name = 1; // An array of virtual hosts that make up the route table. - repeated route.VirtualHost virtual_hosts = 2 [(gogoproto.nullable) = false]; + repeated route.VirtualHost virtual_hosts = 2; // An array of virtual hosts will be dynamically loaded via the VHDS API. // Both *virtual_hosts* and *vhds* fields will be used when present. *virtual_hosts* can be used @@ -131,6 +131,5 @@ message RouteConfiguration { // [#not-implemented-hide:] message Vhds { // Configuration source specifier for VHDS. - envoy.api.v2.core.ConfigSource config_source = 1 - [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + envoy.api.v2.core.ConfigSource config_source = 1 [(validate.rules).message.required = true]; } diff --git a/api/envoy/api/v2/route/route.proto b/api/envoy/api/v2/route/route.proto index 8390e4348526..e14e56583853 100644 --- a/api/envoy/api/v2/route/route.proto +++ b/api/envoy/api/v2/route/route.proto @@ -58,7 +58,7 @@ message VirtualHost { // The list of routes that will be matched, in order, for incoming requests. // The first route that matches will be used. - repeated Route routes = 3 [(gogoproto.nullable) = false]; + repeated Route routes = 3; enum TlsRequirementType { // No TLS requirement for the virtual host. @@ -164,7 +164,7 @@ message Route { string name = 14; // Route matching parameters. - RouteMatch match = 1 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + RouteMatch match = 1 [(validate.rules).message.required = true]; oneof action { option (validate.required) = true; diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index 01ea30d2bcf3..7f0ea3c97ca9 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -37,7 +37,7 @@ message Bootstrap { message StaticResources { // Static :ref:`Listeners `. These listeners are // available regardless of LDS configuration. - repeated envoy.api.v2.Listener listeners = 1 [(gogoproto.nullable) = false]; + repeated envoy.api.v2.Listener listeners = 1; // If a network based configuration source is specified for :ref:`cds_config // `, it's necessary @@ -45,11 +45,11 @@ message Bootstrap { // how to speak to the management server. These cluster definitions may not // use :ref:`EDS ` (i.e. they should be static // IP or DNS-based). - repeated envoy.api.v2.Cluster clusters = 2 [(gogoproto.nullable) = false]; + repeated envoy.api.v2.Cluster clusters = 2; // These static secrets can be used by :ref:`SdsSecretConfig // ` - repeated envoy.api.v2.auth.Secret secrets = 3 [(gogoproto.nullable) = false]; + repeated envoy.api.v2.auth.Secret secrets = 3; } // Statically specified resources. StaticResources static_resources = 2; diff --git a/api/envoy/config/common/tap/v2alpha/common.proto b/api/envoy/config/common/tap/v2alpha/common.proto index b8180f3cdd98..6d3c869baffa 100644 --- a/api/envoy/config/common/tap/v2alpha/common.proto +++ b/api/envoy/config/common/tap/v2alpha/common.proto @@ -20,8 +20,7 @@ message CommonExtensionConfig { // [#not-implemented-hide:] message TapDSConfig { // Configuration for the source of TapDS updates for this Cluster. - envoy.api.v2.core.ConfigSource config_source = 1 - [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + envoy.api.v2.core.ConfigSource config_source = 1 [(validate.rules).message.required = true]; // Tap config to request from XDS server. string name = 2 [(validate.rules).string.min_bytes = 1]; diff --git a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto index 84b6d3fc5c17..39f16e1a9342 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto +++ b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto @@ -37,16 +37,16 @@ message RouteConfiguration { // The list of routes that will be matched, in order, against incoming requests. The first route // that matches will be used. - repeated Route routes = 5 [(gogoproto.nullable) = false]; + repeated Route routes = 5; } // [#comment:next free field: 3] message Route { // Route matching parameters. - RouteMatch match = 1 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + RouteMatch match = 1 [(validate.rules).message.required = true]; // Route request to some upstream cluster. - RouteAction route = 2 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + RouteAction route = 2 [(validate.rules).message.required = true]; } // [#comment:next free field: 3] diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index f0c7b0f59e1e..7d94ca5bcf81 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -428,8 +428,7 @@ message HttpConnectionManager { message Rds { // Configuration source specifier for RDS. - envoy.api.v2.core.ConfigSource config_source = 1 - [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + envoy.api.v2.core.ConfigSource config_source = 1 [(validate.rules).message.required = true]; // The name of the route configuration. This name will be passed to the RDS // API. This allows an Envoy configuration with multiple HTTP listeners (and @@ -529,8 +528,7 @@ message ScopedRoutes { // Configuration source specifier for RDS. // This config source is used to subscribe to RouteConfiguration resources specified in // ScopedRouteConfiguration messages. - envoy.api.v2.core.ConfigSource rds_config_source = 3 - [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + envoy.api.v2.core.ConfigSource rds_config_source = 3 [(validate.rules).message.required = true]; oneof config_specifier { option (validate.required) = true; @@ -554,7 +552,7 @@ message ScopedRoutes { message ScopedRds { // Configuration source specifier for scoped RDS. envoy.api.v2.core.ConfigSource scoped_rds_config_source = 1 - [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + [(validate.rules).message.required = true]; } message HttpFilter { diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index e2fdd24522e9..9f714dd66cd0 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -144,7 +144,7 @@ message RedisProxy { } // List of prefix routes. - repeated Route routes = 1 [(gogoproto.nullable) = false]; + repeated Route routes = 1; // Indicates that prefix matching should be case insensitive. bool case_insensitive = 2; @@ -190,7 +190,7 @@ message RedisProxy { // See the :ref:`configuration section // ` of the architecture overview for recommendations on // configuring the backing clusters. - PrefixRoutes prefix_routes = 5 [(gogoproto.nullable) = false]; + PrefixRoutes prefix_routes = 5; // Authenticate Redis client connections locally by forcing downstream clients to issue a 'Redis // AUTH command `_ with this password before enabling any other diff --git a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto index c516f516fab1..dcd83f2f1a2a 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto +++ b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto @@ -25,16 +25,16 @@ message RouteConfiguration { // The list of routes that will be matched, in order, against incoming requests. The first route // that matches will be used. - repeated Route routes = 2 [(gogoproto.nullable) = false]; + repeated Route routes = 2; } // [#comment:next free field: 3] message Route { // Route matching parameters. - RouteMatch match = 1 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + RouteMatch match = 1 [(validate.rules).message.required = true]; // Route request to some upstream cluster. - RouteAction route = 2 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + RouteAction route = 2 [(validate.rules).message.required = true]; } // [#comment:next free field: 5] From deccb6e215b2fc0df42481f373c6992024e184f9 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Tue, 30 Jul 2019 16:10:40 -0700 Subject: [PATCH 274/542] http: Add ability to merge slashes (#7621) Signed-off-by: Ruslan Nigmatullin --- .../v2/http_connection_manager.proto | 9 +++- docs/root/intro/version_history.rst | 1 + source/common/http/conn_manager_config.h | 6 +++ source/common/http/conn_manager_utility.cc | 9 +++- source/common/http/path_utility.cc | 17 +++++++ source/common/http/path_utility.h | 2 + .../network/http_connection_manager/config.cc | 3 +- .../network/http_connection_manager/config.h | 2 + source/server/http/admin.h | 1 + .../http/conn_manager_impl_fuzz_test.cc | 1 + test/common/http/conn_manager_impl_test.cc | 2 + test/common/http/conn_manager_utility_test.cc | 37 ++++++++++++++ test/common/http/path_utility_test.cc | 22 ++++++++ .../http_connection_manager/config_test.cc | 50 +++++++++++++++++++ 14 files changed, 158 insertions(+), 4 deletions(-) diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 7d94ca5bcf81..f579caa5ae19 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -25,7 +25,7 @@ import "gogoproto/gogo.proto"; // [#protodoc-title: HTTP connection manager] // HTTP connection manager :ref:`configuration overview `. -// [#comment:next free field: 33] +// [#comment:next free field: 34] message HttpConnectionManager { enum CodecType { option (gogoproto.goproto_enum_prefix) = false; @@ -424,6 +424,13 @@ message HttpConnectionManager { // Note that Envoy does not perform // `case normalization ` google.protobuf.BoolValue normalize_path = 30; + + // Determines if adjacent slashes in the path are merged into one before any processing of + // requests by HTTP filters or routing. This affects the upstream *:path* header as well. Without + // setting this option, incoming requests with path `//dir///file` will not match against route + // with `prefix` match set to `/dir`. Defaults to `false`. Note that slash merging is not part of + // `HTTP spec ` and is provided for convenience. + bool merge_slashes = 33; } message Rds { diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 37e4c4ab97e9..758247d95cf1 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -10,6 +10,7 @@ Version history * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. * grpc-json: added support for :ref:`ignoring unknown query parameters`. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. +* http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`HTTP inspector listener filter `. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * tls: added verification of IP address SAN fields in certificates against configured SANs in the diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 5e08e267d5b9..52ec547ea9d5 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -353,6 +353,12 @@ class ConnectionManagerConfig { * @return if the HttpConnectionManager should normalize url following RFC3986 */ virtual bool shouldNormalizePath() const PURE; + + /** + * @return if the HttpConnectionManager should merge two or more adjacent slashes in the path into + * one. + */ + virtual bool shouldMergeSlashes() const PURE; }; } // namespace Http } // namespace Envoy diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index acc258499e56..e4ff80a349bd 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -391,10 +391,15 @@ void ConnectionManagerUtility::mutateResponseHeaders(HeaderMap& response_headers bool ConnectionManagerUtility::maybeNormalizePath(HeaderMap& request_headers, const ConnectionManagerConfig& config) { ASSERT(request_headers.Path()); + bool is_valid_path = true; if (config.shouldNormalizePath()) { - return PathUtil::canonicalPath(*request_headers.Path()); + is_valid_path = PathUtil::canonicalPath(*request_headers.Path()); } - return true; + // Merge slashes after path normalization to catch potential edge cases with percent encoding. + if (is_valid_path && config.shouldMergeSlashes()) { + PathUtil::mergeSlashes(*request_headers.Path()); + } + return is_valid_path; } } // namespace Http diff --git a/source/common/http/path_utility.cc b/source/common/http/path_utility.cc index 56ce3204a468..49372fc50dbf 100644 --- a/source/common/http/path_utility.cc +++ b/source/common/http/path_utility.cc @@ -4,6 +4,8 @@ #include "common/chromium_url/url_canon_stdstring.h" #include "common/common/logger.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -52,5 +54,20 @@ bool PathUtil::canonicalPath(HeaderEntry& path_header) { return true; } +void PathUtil::mergeSlashes(HeaderEntry& path_header) { + const auto original_path = path_header.value().getStringView(); + // Only operate on path component in URL. + const absl::string_view::size_type query_start = original_path.find('?'); + const absl::string_view path = original_path.substr(0, query_start); + const absl::string_view query = absl::ClippedSubstr(original_path, query_start); + if (path.find("//") == absl::string_view::npos) { + return; + } + const absl::string_view prefix = absl::StartsWith(path, "/") ? "/" : absl::string_view(); + const absl::string_view suffix = absl::EndsWith(path, "/") ? "/" : absl::string_view(); + path_header.value(absl::StrCat( + prefix, absl::StrJoin(absl::StrSplit(path, '/', absl::SkipEmpty()), "/"), query, suffix)); +} + } // namespace Http } // namespace Envoy diff --git a/source/common/http/path_utility.h b/source/common/http/path_utility.h index ad0d32c3ff7d..a588f39de46e 100644 --- a/source/common/http/path_utility.h +++ b/source/common/http/path_utility.h @@ -13,6 +13,8 @@ class PathUtil { // Returns if the normalization succeeds. // If it is successful, the param will be updated with the normalized path. static bool canonicalPath(HeaderEntry& path_header); + // Merges two or more adjacent slashes in path part of URI into one. + static void mergeSlashes(HeaderEntry& path_header); }; } // namespace Http diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 04edfdba79c4..eac362cce4f4 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -170,7 +170,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( #else 0 #endif - ))) { + ))), + merge_slashes_(config.merge_slashes()) { // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will // be managed by the scoped routing logic instead. switch (config.route_specifier_case()) { diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index 796b67fc9728..6cd895503aa6 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -138,6 +138,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, bool proxy100Continue() const override { return proxy_100_continue_; } const Http::Http1Settings& http1Settings() const override { return http1_settings_; } bool shouldNormalizePath() const override { return normalize_path_; } + bool shouldMergeSlashes() const override { return merge_slashes_; } std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; } private: @@ -182,6 +183,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, const bool proxy_100_continue_; std::chrono::milliseconds delayed_close_timeout_; const bool normalize_path_; + const bool merge_slashes_; // Default idle timeout is 5 minutes if nothing is specified in the HCM config. static const uint64_t StreamIdleTimeoutMs = 5 * 60 * 1000; diff --git a/source/server/http/admin.h b/source/server/http/admin.h index ac0f6af807bb..a01c6b1ad21f 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -143,6 +143,7 @@ class AdminImpl : public Admin, bool proxy100Continue() const override { return false; } const Http::Http1Settings& http1Settings() const override { return http1_settings_; } bool shouldNormalizePath() const override { return true; } + bool shouldMergeSlashes() const override { return true; } Http::Code request(absl::string_view path_and_query, absl::string_view method, Http::HeaderMap& response_headers, std::string& body) override; void closeSocket(); diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 3c11424b5964..5c2bf01fe1ab 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -110,6 +110,7 @@ class FuzzConfig : public ConnectionManagerConfig { bool proxy100Continue() const override { return proxy_100_continue_; } const Http::Http1Settings& http1Settings() const override { return http1_settings_; } bool shouldNormalizePath() const override { return false; } + bool shouldMergeSlashes() const override { return false; } const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager config_; std::list access_logs_; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index c635e59c6928..9c84dfc14ef6 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -279,6 +279,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan bool proxy100Continue() const override { return proxy_100_continue_; } const Http::Http1Settings& http1Settings() const override { return http1_settings_; } bool shouldNormalizePath() const override { return normalize_path_; } + bool shouldMergeSlashes() const override { return merge_slashes_; } DangerousDeprecatedTestTime test_time_; ConnectionManagerImplHelper::RouteConfigProvider route_config_provider_; @@ -327,6 +328,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan bool preserve_external_request_id_ = false; Http::Http1Settings http1_settings_; bool normalize_path_ = false; + bool merge_slashes_ = false; NiceMock upstream_conn_; // for websocket tests NiceMock conn_pool_; // for websocket tests diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 46937cf17ab0..c228f99e3c30 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -83,6 +83,7 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_CONST_METHOD0(proxy100Continue, bool()); MOCK_CONST_METHOD0(http1Settings, const Http::Http1Settings&()); MOCK_CONST_METHOD0(shouldNormalizePath, bool()); + MOCK_CONST_METHOD0(shouldMergeSlashes, bool()); std::unique_ptr internal_address_config_ = std::make_unique(); @@ -1224,6 +1225,42 @@ TEST_F(ConnectionManagerUtilityTest, SanitizePathRelativePAth) { EXPECT_EQ(header_map.Path()->value().getStringView(), "/abc"); } +// maybeNormalizePath() does not touch adjacent slashes by default. +TEST_F(ConnectionManagerUtilityTest, MergeSlashesDefaultOff) { + ON_CALL(config_, shouldNormalizePath()).WillByDefault(Return(true)); + ON_CALL(config_, shouldMergeSlashes()).WillByDefault(Return(false)); + HeaderMapImpl original_headers; + original_headers.insertPath().value(std::string("/xyz///abc")); + + HeaderMapImpl header_map(static_cast(original_headers)); + ConnectionManagerUtility::maybeNormalizePath(header_map, config_); + EXPECT_EQ(header_map.Path()->value().getStringView(), "/xyz///abc"); +} + +// maybeNormalizePath() merges adjacent slashes. +TEST_F(ConnectionManagerUtilityTest, MergeSlashes) { + ON_CALL(config_, shouldNormalizePath()).WillByDefault(Return(true)); + ON_CALL(config_, shouldMergeSlashes()).WillByDefault(Return(true)); + HeaderMapImpl original_headers; + original_headers.insertPath().value(std::string("/xyz///abc")); + + HeaderMapImpl header_map(static_cast(original_headers)); + ConnectionManagerUtility::maybeNormalizePath(header_map, config_); + EXPECT_EQ(header_map.Path()->value().getStringView(), "/xyz/abc"); +} + +// maybeNormalizePath() merges adjacent slashes if normalization if off. +TEST_F(ConnectionManagerUtilityTest, MergeSlashesWithoutNormalization) { + ON_CALL(config_, shouldNormalizePath()).WillByDefault(Return(false)); + ON_CALL(config_, shouldMergeSlashes()).WillByDefault(Return(true)); + HeaderMapImpl original_headers; + original_headers.insertPath().value(std::string("/xyz/..//abc")); + + HeaderMapImpl header_map(static_cast(original_headers)); + ConnectionManagerUtility::maybeNormalizePath(header_map, config_); + EXPECT_EQ(header_map.Path()->value().getStringView(), "/xyz/../abc"); +} + // test preserve_external_request_id true does not reset the passed requestId if passed TEST_F(ConnectionManagerUtilityTest, PreserveExternalRequestId) { connection_.remote_address_ = std::make_shared("134.2.2.11"); diff --git a/test/common/http/path_utility_test.cc b/test/common/http/path_utility_test.cc index 2cc299465add..3ee558a2633e 100644 --- a/test/common/http/path_utility_test.cc +++ b/test/common/http/path_utility_test.cc @@ -85,5 +85,27 @@ TEST_F(PathUtilityTest, NormalizeCasePath) { // "/../c\r\n\" '\n' '\r' should be excluded by http parser // "/a/\0c", '\0' should be excluded by http parser +// Paths that are valid get normalized. +TEST_F(PathUtilityTest, MergeSlashes) { + auto mergeSlashes = [this](const std::string& path_value) { + auto& path_header = pathHeaderEntry(path_value); + PathUtil::mergeSlashes(path_header); + auto sanitized_path_value = path_header.value().getStringView(); + return std::string(sanitized_path_value); + }; + EXPECT_EQ("", mergeSlashes("")); // empty + EXPECT_EQ("a/b/c", mergeSlashes("a//b/c")); // relative + EXPECT_EQ("/a/b/c/", mergeSlashes("/a//b/c/")); // ends with slash + EXPECT_EQ("a/b/c/", mergeSlashes("a//b/c/")); // relative ends with slash + EXPECT_EQ("/a", mergeSlashes("/a")); // no-op + EXPECT_EQ("/a/b/c", mergeSlashes("//a/b/c")); // double / start + EXPECT_EQ("/a/b/c", mergeSlashes("/a//b/c")); // double / in the middle + EXPECT_EQ("/a/b/c/", mergeSlashes("/a/b/c//")); // double / end + EXPECT_EQ("/a/b/c", mergeSlashes("/a///b/c")); // triple / in the middle + EXPECT_EQ("/a/b/c", mergeSlashes("/a////b/c")); // quadruple / in the middle + EXPECT_EQ("/a/b?a=///c", mergeSlashes("/a//b?a=///c")); // slashes in the query are ignored + EXPECT_EQ("/a/b?", mergeSlashes("/a//b?")); // empty query +} + } // namespace Http } // namespace Envoy diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 9afdaa5a6c2b..d197b9b7fa94 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -412,6 +412,56 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathFalse) { EXPECT_FALSE(config.shouldNormalizePath()); } +// Validated that by default we don't merge slashes. +TEST_F(HttpConnectionManagerConfigTest, MergeSlashesDefault) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + http_filters: + - name: envoy.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + EXPECT_FALSE(config.shouldMergeSlashes()); +} + +// Validated that when configured, we merge slashes. +TEST_F(HttpConnectionManagerConfigTest, MergeSlashesTrue) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + merge_slashes: true + http_filters: + - name: envoy.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + EXPECT_TRUE(config.shouldMergeSlashes()); +} + +// Validated that when explicitly set false, we don't merge slashes. +TEST_F(HttpConnectionManagerConfigTest, MergeSlashesFalse) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + route_config: + name: local_route + merge_slashes: false + http_filters: + - name: envoy.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + EXPECT_FALSE(config.shouldMergeSlashes()); +} + TEST_F(HttpConnectionManagerConfigTest, ConfiguredRequestTimeout) { const std::string yaml_string = R"EOF( stat_prefix: ingress_http From 993095c87e360153c8fc7dc016d5a79604cb2602 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 30 Jul 2019 16:14:15 -0700 Subject: [PATCH 275/542] bazel: change cfg of rule to exec (#7758) The host environment isn't what you want when using remote execution Signed-off-by: Keith Smiley --- tools/protodoc/protodoc.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/protodoc/protodoc.bzl b/tools/protodoc/protodoc.bzl index e619f8b457c1..4d78355ba0b7 100644 --- a/tools/protodoc/protodoc.bzl +++ b/tools/protodoc/protodoc.bzl @@ -82,12 +82,12 @@ proto_doc_aspect = aspect( "_protoc": attr.label( default = Label("@com_google_protobuf//:protoc"), executable = True, - cfg = "host", + cfg = "exec", ), "_protodoc": attr.label( default = Label("//tools/protodoc"), executable = True, - cfg = "host", + cfg = "exec", ), }, implementation = _proto_doc_aspect_impl, From e1a33da2334eb29334e458181a4c66196fb66f45 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Tue, 30 Jul 2019 16:19:24 -0700 Subject: [PATCH 276/542] clang-tidy: clang-analyzer-core.DivideZero (#7765) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + source/common/common/backoff_strategy.cc | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.clang-tidy b/.clang-tidy index c62c2829fab3..09fb39413ac8 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -19,6 +19,7 @@ WarningsAsErrors: 'abseil-duration-*, bugprone-assert-side-effect, bugprone-unused-raii, bugprone-use-after-move, + clang-analyzer-core.DivideZero, modernize-deprecated-headers, modernize-make-shared, modernize-make-unique, diff --git a/source/common/common/backoff_strategy.cc b/source/common/common/backoff_strategy.cc index 71138eac1e00..a67cc3cb88d1 100644 --- a/source/common/common/backoff_strategy.cc +++ b/source/common/common/backoff_strategy.cc @@ -15,9 +15,10 @@ uint64_t JitteredBackOffStrategy::nextBackOffMs() { if (base_backoff <= max_interval_) { current_retry_++; } + ASSERT(base_backoff > 0); return std::min(random_.random() % base_backoff, max_interval_); } void JitteredBackOffStrategy::reset() { current_retry_ = 1; } -} // namespace Envoy \ No newline at end of file +} // namespace Envoy From bdd6788f1e01787d015eabd9902f4b565e5dea98 Mon Sep 17 00:00:00 2001 From: Shriram Rajagopalan Date: Tue, 30 Jul 2019 22:52:48 -0400 Subject: [PATCH 277/542] Fix missing annotation in http_uri.proto (#7769) Description: Fix missing gogo annotation. The file-level `equal_all` annotation was missing in one of the files and failed to compile in go-control-plane. https://github.com/envoyproxy/go-control-plane/pull/201 Risk Level: Low Testing: go-control-plane Docs Changes: N/A Release Notes: N/A Signed-off-by: Shriram Rajagopalan --- api/envoy/api/v2/core/http_uri.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/envoy/api/v2/core/http_uri.proto b/api/envoy/api/v2/core/http_uri.proto index 21c66c7fa63c..4c7a594f0635 100644 --- a/api/envoy/api/v2/core/http_uri.proto +++ b/api/envoy/api/v2/core/http_uri.proto @@ -11,6 +11,8 @@ import "gogoproto/gogo.proto"; import "validate/validate.proto"; +option (gogoproto.equal_all) = true; + // [#protodoc-title: HTTP Service URI ] // Envoy external URI descriptor From 284009009dc2db7bfabf16da826917b57b104243 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 31 Jul 2019 05:48:25 -0700 Subject: [PATCH 278/542] ci: reduce build image size (#7773) Brings down build image from 2.17GB to 1.15GB to speed up CI generally while keep functionalities. - all recommended packages + patch which was included in recommends but explicitly - golang (now fetched by bazel, was used by boringssl cmake build before) - gcov - wireshark (so no more GUI dependencies, tshark is the one used in tests) apt-get clean at the end. Risk Level: Low (not used anyway) Testing: local Signed-off-by: Lizan Zhou --- ci/build_container/build_container_ubuntu.sh | 34 +++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/ci/build_container/build_container_ubuntu.sh b/ci/build_container/build_container_ubuntu.sh index 6ca415d11a03..e329a67eafd7 100755 --- a/ci/build_container/build_container_ubuntu.sh +++ b/ci/build_container/build_container_ubuntu.sh @@ -7,9 +7,20 @@ ARCH="$(uname -m)" # Setup basic requirements and install them. apt-get update export DEBIAN_FRONTEND=noninteractive -apt-get install -y wget software-properties-common make cmake git python python-pip python3 python3-pip \ - unzip bc libtool ninja-build automake zip time golang gdb strace wireshark tshark tcpdump lcov \ - apt-transport-https +apt-get install -y --no-install-recommends software-properties-common apt-transport-https + +# gcc-7 +add-apt-repository -y ppa:ubuntu-toolchain-r/test +apt-get update +apt-get install -y --no-install-recommends g++-7 +update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 1000 +update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 1000 +update-alternatives --config gcc +update-alternatives --config g++ + +apt-get install -y --no-install-recommends curl wget make cmake git python python-pip python3 python3-pip \ + unzip bc libtool ninja-build automake zip time gdb strace tshark tcpdump patch \ + # clang 8. case $ARCH in 'ppc64le' ) @@ -26,21 +37,11 @@ case $ARCH in wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" apt-get update - apt-get install -y clang-8 clang-format-8 clang-tidy-8 lld-8 libc++-8-dev libc++abi-8-dev + apt-get install -y --no-install-recommends clang-8 clang-format-8 clang-tidy-8 lld-8 libc++-8-dev libc++abi-8-dev llvm-8 ;; esac -# gcc-7 -add-apt-repository -y ppa:ubuntu-toolchain-r/test -apt update -apt install -y g++-7 -update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 1000 -update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 1000 -update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-7 1000 -update-alternatives --config gcc -update-alternatives --config g++ -update-alternatives --config gcov + # Bazel and related dependencies. -apt-get install -y openjdk-8-jdk curl case $ARCH in 'ppc64le' ) BAZEL_LATEST="$(curl https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_16.04/latest/ 2>&1 \ @@ -50,6 +51,7 @@ case $ARCH in chmod +x /usr/local/bin/bazel ;; esac + apt-get install -y aspell rm -rf /var/lib/apt/lists/* @@ -63,3 +65,5 @@ setcap cap_net_raw,cap_net_admin=eip /usr/sbin/tcpdump pip3 install virtualenv ./build_container_common.sh + +apt-get clean From ca2af9723598fab4f511b59407396cc5cff9ed94 Mon Sep 17 00:00:00 2001 From: Kuat Date: Wed, 31 Jul 2019 07:42:16 -0700 Subject: [PATCH 279/542] api: promote tracing operation to listener level (#7723) Promote tracing operation field to listener level. This expands the scope of the field to support two use cases: Tracing TCP connections: istio can send connection events to create a service communication graph. Network filters can benefit from the common knowledge about the intent of the listener/filter chain (client-side vs server-side). Using ingress/egress designation for other telemetry. The direction of the traffic is a useful label on metrics, and it is not explicit at the moment, unless depending on tracing configuration in HTTP connection manager or naming convention. Both workarounds are not ideal. Risk Level: low Testing: all unit tests continue to pass Signed-off-by: Kuat Yessenov --- api/envoy/api/v2/core/base.proto | 12 ++++++++++++ api/envoy/api/v2/lds.proto | 5 ++++- .../v2/http_connection_manager.proto | 1 + include/envoy/server/filter_config.h | 6 ++++++ source/server/listener_manager_impl.h | 3 +++ test/mocks/server/mocks.h | 1 + test/server/listener_manager_impl_test.cc | 2 ++ 7 files changed, 29 insertions(+), 1 deletion(-) diff --git a/api/envoy/api/v2/core/base.proto b/api/envoy/api/v2/core/base.proto index 6375d87e6824..2553fe04de27 100644 --- a/api/envoy/api/v2/core/base.proto +++ b/api/envoy/api/v2/core/base.proto @@ -278,3 +278,15 @@ message ControlPlane { // the Envoy is connected to. string identifier = 1; } + +// Identifies the direction of the traffic relative to the local Envoy. +enum TrafficDirection { + // Default option is unspecified. + UNSPECIFIED = 0; + + // The transport is used for incoming traffic. + INBOUND = 1; + + // The transport is used for outgoing traffic. + OUTBOUND = 2; +} diff --git a/api/envoy/api/v2/lds.proto b/api/envoy/api/v2/lds.proto index f6081fbc9af3..dd29659ce89e 100644 --- a/api/envoy/api/v2/lds.proto +++ b/api/envoy/api/v2/lds.proto @@ -44,7 +44,7 @@ service ListenerDiscoveryService { } } -// [#comment:next free field: 16] +// [#comment:next free field: 17] message Listener { // The unique name by which this listener is known. If no name is provided, // Envoy will allocate an internal UUID for the listener. If the listener is to be dynamically @@ -181,4 +181,7 @@ message Listener { google.protobuf.UInt32Value tcp_fast_open_queue_length = 12; reserved 14; + + // Specifies the intended direction of the traffic relative to the local Envoy. + core.TrafficDirection traffic_direction = 16; } diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index f579caa5ae19..4d662fc2c524 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -80,6 +80,7 @@ message HttpConnectionManager { google.protobuf.BoolValue add_user_agent = 6; message Tracing { + // [#comment:TODO(kyessenov): Align this field with listener traffic direction field.] enum OperationName { option (gogoproto.goproto_enum_prefix) = false; diff --git a/include/envoy/server/filter_config.h b/include/envoy/server/filter_config.h index 12ea149f517b..38ea9070721c 100644 --- a/include/envoy/server/filter_config.h +++ b/include/envoy/server/filter_config.h @@ -116,6 +116,12 @@ class FactoryContext : public virtual CommonFactoryContext { */ virtual AccessLog::AccessLogManager& accessLogManager() PURE; + /** + * @return envoy::api::v2::core::TrafficDirection the direction of the traffic relative to the + * local proxy. + */ + virtual envoy::api::v2::core::TrafficDirection direction() const PURE; + /** * @return const Network::DrainDecision& a drain decision that filters can use to determine if * they should be doing graceful closes on connections when possible. diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 43edef1e2f3e..8435f0213065 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -298,6 +298,9 @@ class ListenerImpl : public Network::ListenerConfig, const envoy::api::v2::core::Metadata& listenerMetadata() const override { return config_.metadata(); }; + envoy::api::v2::core::TrafficDirection direction() const override { + return config_.traffic_direction(); + }; TimeSource& timeSource() override { return api().timeSource(); } void ensureSocketOptions() { if (!listen_socket_options_) { diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 7df622152422..99fb3ebc1512 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -458,6 +458,7 @@ class MockFactoryContext : public virtual FactoryContext { MOCK_METHOD0(listenerScope, Stats::Scope&()); MOCK_CONST_METHOD0(localInfo, const LocalInfo::LocalInfo&()); MOCK_CONST_METHOD0(listenerMetadata, const envoy::api::v2::core::Metadata&()); + MOCK_CONST_METHOD0(direction, envoy::api::v2::core::TrafficDirection()); MOCK_METHOD0(timeSource, TimeSource&()); Event::TestTimeSystem& timeSystem() { return time_system_; } Grpc::Context& grpcContext() override { return grpc_context_; } diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 18df21b388e1..ea19e0758c44 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -2615,6 +2615,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, Metadata) { address: socket_address: { address: 127.0.0.1, port_value: 1234 } metadata: { filter_metadata: { com.bar.foo: { baz: test_value } } } + traffic_direction: INBOUND filter_chains: - filter_chain_match: filters: @@ -2636,6 +2637,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, Metadata) { EXPECT_EQ("test_value", Config::Metadata::metadataValue(context->listenerMetadata(), "com.bar.foo", "baz") .string_value()); + EXPECT_EQ(envoy::api::v2::core::TrafficDirection::INBOUND, context->direction()); } TEST_F(ListenerManagerImplWithRealFiltersTest, OriginalDstFilter) { From 853547c41b456457d57a95c0651a7435e470607c Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Wed, 31 Jul 2019 09:33:19 -0700 Subject: [PATCH 280/542] io: reduce unknown error code log level (#7750) ``` 2019-07-29T17:11:16.00568 [2019-07-29 17:11:16.005][26039][error][misc] [external/envoy/source/common/network/io_socket_error_impl.cc:29] Unknown error code 111 details Connection refused ``` I saw the previous log line on a production box and I think this should be a debug log condition. Signed-off-by: Matt Klein --- source/common/network/io_socket_error_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/network/io_socket_error_impl.cc b/source/common/network/io_socket_error_impl.cc index 9f69361092b6..3382ac5acf2f 100644 --- a/source/common/network/io_socket_error_impl.cc +++ b/source/common/network/io_socket_error_impl.cc @@ -26,7 +26,7 @@ Api::IoError::IoErrorCode IoSocketError::getErrorCode() const { case EADDRNOTAVAIL: return IoErrorCode::AddressNotAvailable; default: - ENVOY_LOG_MISC(error, "Unknown error code {} details {}", errno_, ::strerror(errno_)); + ENVOY_LOG_MISC(debug, "Unknown error code {} details {}", errno_, ::strerror(errno_)); return IoErrorCode::UnknownError; } } From 6a414b203ee97262128e4929eae0b06c009de7e0 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Wed, 31 Jul 2019 12:40:48 -0400 Subject: [PATCH 281/542] tls: update BoringSSL to 87d1c8f2 (3809). (#7772) Signed-off-by: Piotr Sikora --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 7c36dec104f3..1335ae70623b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -13,10 +13,10 @@ REPOSITORY_LOCATIONS = dict( ), boringssl = dict( # Use commits from branch "chromium-stable-with-bazel" - sha256 = "448773376d063b1e9a19e4fd41002d1a31a968a13be20b3b66ecd4aab9cf14a8", - strip_prefix = "boringssl-e534d74f5732e1aeebd514f05271d089c530c2f9", - # chromium-75.0.3770.80 - urls = ["https://github.com/google/boringssl/archive/e534d74f5732e1aeebd514f05271d089c530c2f9.tar.gz"], + sha256 = "18edf961f8377e8d10fd8497bc8a331def9cb60a6c2a50a4c8eb322b045042d5", + strip_prefix = "boringssl-87d1c8f292e5184fd727efe84f458d89687d7742", + # chromium-76.0.3809.87 + urls = ["https://github.com/google/boringssl/archive/87d1c8f292e5184fd727efe84f458d89687d7742.tar.gz"], ), boringssl_fips = dict( sha256 = "b12ad676ee533824f698741bd127f6fbc82c46344398a6d78d25e62c6c418c73", From 59c080c8d77ced4b7cad7b95ccdce9de858cc144 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 31 Jul 2019 12:41:09 -0400 Subject: [PATCH 282/542] Explicitly mark Python binaries with python_version = PY3. (#7778) Signed-off-by: Yan Avlasov --- test/integration/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/BUILD b/test/integration/BUILD index d00ef6ca405a..8df797ff0fc8 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -70,6 +70,7 @@ py_binary( name = "capture_fuzz_gen", srcs = ["capture_fuzz_gen.py"], licenses = ["notice"], # Apache 2 + python_version = "PY3", visibility = ["//visibility:public"], deps = [ ":capture_fuzz_proto_py", From 3eedc793f1b47ec5d737d399103c287fe59ff650 Mon Sep 17 00:00:00 2001 From: htuch Date: Wed, 31 Jul 2019 16:38:25 -0400 Subject: [PATCH 283/542] security: add IBM to distributors, clarify policy. (#7781) Signed-off-by: Harvey Tuch --- SECURITY.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 1a11b6e157f4..9fff3ca28cc3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -329,7 +329,8 @@ customers, of which approximately 400 are using Seven in production. [links] We announce on our blog all upstream patches we apply to "Seven." [link to blog posts] -> 4. Not be a downstream or rebuild of another distribution. +> 4. Not be a downstream or rebuild of another distribution. If you offer Envoy as a publicly +> available infrastructure or platform service, this condition does not need to apply. This does not apply, "Seven" is a unique snowflake distribution. @@ -376,6 +377,7 @@ CrashOverride will vouch for the "Seven" distribution joining the distribution l | vulnerabilityreports@cloudfoundry.org | Cloud Foundry | | secalert@datawire.io | Datawire | | google-internal-envoy-security@google.com | Google | +| argoprod@us.ibm.com | IBM | | vulnerabilities@discuss.istio.io | Istio | | secalert@redhat.com | Red Hat | | envoy-security@solo.io | solo.io | From c586af9d3f5940acc73d6a5fd0fdc2c521b6243d Mon Sep 17 00:00:00 2001 From: Daniel Hochman Date: Wed, 31 Jul 2019 14:57:32 -0700 Subject: [PATCH 284/542] docs: fix missing ref in xDS docs and runTime case (#7784) Signed-off-by: Daniel Hochman --- api/envoy/service/discovery/v2/rtds.proto | 2 +- api/xds_protocol.rst | 4 ++-- docs/root/configuration/runtime.rst | 2 +- docs/root/intro/version_history.rst | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/envoy/service/discovery/v2/rtds.proto b/api/envoy/service/discovery/v2/rtds.proto index 582a14366ff1..c8b53d670fe1 100644 --- a/api/envoy/service/discovery/v2/rtds.proto +++ b/api/envoy/service/discovery/v2/rtds.proto @@ -14,7 +14,7 @@ import "google/protobuf/struct.proto"; import "validate/validate.proto"; -// [#protodoc-title: runTime Discovery Service (RTDS)] +// [#protodoc-title: Runtime Discovery Service (RTDS)] // RTDS :ref:`configuration overview ` // [#not-implemented-hide:] Not configuration. Workaround c++ protobuf issue with importing diff --git a/api/xds_protocol.rst b/api/xds_protocol.rst index 4cbda0a72663..f8fe9a6e0608 100644 --- a/api/xds_protocol.rst +++ b/api/xds_protocol.rst @@ -135,7 +135,7 @@ versioning across resource types. When ADS is not used, even each resource of a given resource type may have a distinct version, since the Envoy API allows distinct EDS/RDS resources to point at different :ref:`ConfigSources `. -.. xds_protocol_resource_update: +.. _xds_protocol_resource_update: Resource Update ~~~~~~~~~~~~~~~ @@ -463,7 +463,7 @@ messages. ADS is not available for REST-JSON polling. When the poll period is set to a small value, with the intention of long polling, then there is also a requirement to avoid sending a :ref:`DiscoveryResponse ` unless a change to the underlying resources has -occurred . +occurred via a :ref:`resource update `. .. |Multiple EDS requests on the same stream| image:: diagrams/eds-same-stream.svg .. |Multiple EDS requests on distinct streams| image:: diagrams/eds-distinct-stream.svg diff --git a/docs/root/configuration/runtime.rst b/docs/root/configuration/runtime.rst index b240c472d930..878b2b5a024e 100644 --- a/docs/root/configuration/runtime.rst +++ b/docs/root/configuration/runtime.rst @@ -136,7 +136,7 @@ It's beyond the scope of this document how the file system data is deployed, gar .. _config_runtime_rtds: -runTime Discovery Service (RTDS) +Runtime Discovery Service (RTDS) ++++++++++++++++++++++++++++++++ One or more runtime layers may be specified and delivered by specifying a :ref:`rtds_layer diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 758247d95cf1..369927c154ec 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -93,7 +93,7 @@ Version history `. * runtime: added support for statically :ref:`specifying the runtime in the bootstrap configuration `. -* runtime: :ref:`runTime Discovery Service (RTDS) ` support added to layered runtime configuration. +* runtime: :ref:`Runtime Discovery Service (RTDS) ` support added to layered runtime configuration. * sandbox: added :ref:`CSRF sandbox `. * server: ``--define manual_stamp=manual_stamp`` was added to allow server stamping outside of binary rules. more info in the `bazel docs `_. From a9493281bd91b797b0566fea1a8b22d61d11af90 Mon Sep 17 00:00:00 2001 From: Misha Efimov Date: Wed, 31 Jul 2019 19:32:04 -0400 Subject: [PATCH 285/542] access_log: add extension filter to allow runtime filter registration. (#7435) Signed-off-by: Misha Efimov --- .../filter/accesslog/v2/accesslog.proto | 16 ++ docs/root/extending/extending.rst | 3 +- .../observability/access_logging.rst | 10 ++ docs/root/intro/version_history.rst | 1 + source/common/access_log/access_log_impl.cc | 7 + source/common/access_log/access_log_impl.h | 34 ++++ .../common/access_log/access_log_impl_test.cc | 151 ++++++++++++++++++ 7 files changed, 221 insertions(+), 1 deletion(-) diff --git a/api/envoy/config/filter/accesslog/v2/accesslog.proto b/api/envoy/config/filter/accesslog/v2/accesslog.proto index 4eec60d7442d..66d901c2d413 100644 --- a/api/envoy/config/filter/accesslog/v2/accesslog.proto +++ b/api/envoy/config/filter/accesslog/v2/accesslog.proto @@ -76,6 +76,9 @@ message AccessLogFilter { // gRPC status filter. GrpcStatusFilter grpc_status_filter = 10; + + // Extension filter. + ExtensionFilter extension_filter = 11; } } @@ -227,3 +230,16 @@ message GrpcStatusFilter { // inferred gRPC status enumerated in statuses, and allow all other responses. bool exclude = 2; } + +// Extension filter is statically registered at runtime. +message ExtensionFilter { + // The name of the filter implementation to instantiate. The name must + // match a statically registered filter. + string name = 1; + + // Custom configuration that depends on the filter being instantiated. + oneof config_type { + google.protobuf.Struct config = 2; + google.protobuf.Any typed_config = 3; + } +} diff --git a/docs/root/extending/extending.rst b/docs/root/extending/extending.rst index a44e36785149..1551d0992428 100644 --- a/docs/root/extending/extending.rst +++ b/docs/root/extending/extending.rst @@ -3,10 +3,11 @@ Extending Envoy for custom use cases ==================================== -The Envoy architecture makes it fairly easily extensible via a variety of differerent extension +The Envoy architecture makes it fairly easily extensible via a variety of different extension types including: * :ref:`Access loggers ` +* :ref:`Access log filters ` * :ref:`Clusters ` * :ref:`Listener filters ` * :ref:`Network filters ` diff --git a/docs/root/intro/arch_overview/observability/access_logging.rst b/docs/root/intro/arch_overview/observability/access_logging.rst index 0ffd961bfbd1..feb98cc701dd 100644 --- a/docs/root/intro/arch_overview/observability/access_logging.rst +++ b/docs/root/intro/arch_overview/observability/access_logging.rst @@ -11,6 +11,16 @@ features: * Customizable access log filters that allow different types of requests and responses to be written to different access logs. +.. _arch_overview_access_log_filters: + +Access log filters +------------------ + +Envoy supports several built-in +:ref:`access log filters` and +:ref:`extension filters` +that are registered at runtime. + Access logging sinks -------------------- diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 369927c154ec..41c6509e9c7d 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -5,6 +5,7 @@ Version history ================ * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. +* config: added access log :ref:`extension filter`. * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc index 33fcd5485d04..f59c96d1b66e 100644 --- a/source/common/access_log/access_log_impl.cc +++ b/source/common/access_log/access_log_impl.cc @@ -77,6 +77,13 @@ FilterFactory::fromProto(const envoy::config::filter::accesslog::v2::AccessLogFi case envoy::config::filter::accesslog::v2::AccessLogFilter::kGrpcStatusFilter: MessageUtil::validate(config); return FilterPtr{new GrpcStatusFilter(config.grpc_status_filter())}; + case envoy::config::filter::accesslog::v2::AccessLogFilter::kExtensionFilter: + MessageUtil::validate(config); + { + auto& factory = Config::Utility::getAndCheckFactory( + config.extension_filter().name()); + return factory.createFilter(config.extension_filter(), runtime, random); + } default: NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/source/common/access_log/access_log_impl.h b/source/common/access_log/access_log_impl.h index 0941602a30b0..899e81a2902e 100644 --- a/source/common/access_log/access_log_impl.h +++ b/source/common/access_log/access_log_impl.h @@ -222,6 +222,40 @@ class GrpcStatusFilter : public Filter { protoToGrpcStatus(envoy::config::filter::accesslog::v2::GrpcStatusFilter_Status status) const; }; +/** + * Extension filter factory that reads from ExtensionFilter proto. + */ +class ExtensionFilterFactory { +public: + virtual ~ExtensionFilterFactory() = default; + + /** + * Create a particular extension filter implementation from a config proto. If the + * implementation is unable to produce a filter with the provided parameters, it should throw an + * EnvoyException. The returned pointer should never be nullptr. + * @param config supplies the custom configuration for this filter type. + * @param runtime supplies the runtime loader. + * @param random supplies the random generator. + * @return an instance of extension filter implementation from a config proto. + */ + virtual FilterPtr + createFilter(const envoy::config::filter::accesslog::v2::ExtensionFilter& config, + Runtime::Loader& runtime, Runtime::RandomGenerator& random) PURE; + + /** + * @return ProtobufTypes::MessagePtr create empty config proto message for v2. The config, which + * arrives in an opaque google.protobuf.Struct message, will be converted to JSON and then parsed + * into this empty proto. + */ + virtual ProtobufTypes::MessagePtr createEmptyConfigProto() PURE; + + /** + * @return std::string the identifying name for a particular Filter implementation + * produced by the factory. + */ + virtual std::string name() const PURE; +}; + /** * Access log factory that reads the configuration from proto. */ diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index 5e34f5c0e33d..fe29b7839499 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -3,11 +3,13 @@ #include #include +#include "envoy/config/filter/accesslog/v2/accesslog.pb.validate.h" #include "envoy/upstream/cluster_manager.h" #include "envoy/upstream/upstream.h" #include "common/access_log/access_log_impl.h" #include "common/config/filter_json.h" +#include "common/protobuf/message_validator_impl.h" #include "common/runtime/runtime_impl.h" #include "common/runtime/uuid_util.h" @@ -1132,6 +1134,155 @@ name: envoy.file_access_log log->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); } +class TestHeaderFilterFactory : public ExtensionFilterFactory { +public: + ~TestHeaderFilterFactory() override = default; + + FilterPtr createFilter(const envoy::config::filter::accesslog::v2::ExtensionFilter& config, + Runtime::Loader&, Runtime::RandomGenerator&) override { + auto factory_config = Config::Utility::translateToFactoryConfig( + config, Envoy::ProtobufMessage::getNullValidationVisitor(), *this); + const auto& header_config = + MessageUtil::downcastAndValidate( + *factory_config); + return std::make_unique(header_config); + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return "test_header_filter"; } +}; + +TEST_F(AccessLogImplTest, TestHeaderFilterPresence) { + Registry::RegisterFactory registered; + + const std::string yaml = R"EOF( +name: envoy.file_access_log +filter: + extension_filter: + name: test_header_filter + config: + header: + name: test-header +config: + path: /dev/null + )EOF"; + + InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV2Yaml(yaml), context_); + + EXPECT_CALL(*file_, write(_)).Times(0); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + + request_headers_.addCopy("test-header", "foo/bar"); + EXPECT_CALL(*file_, write(_)); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); +} + +/** + * Sample extension filter which allows every sample_rate-th request. + */ +class SampleExtensionFilter : public Filter { +public: + SampleExtensionFilter(uint32_t sample_rate) : sample_rate_(sample_rate) {} + + // AccessLog::Filter + bool evaluate(const StreamInfo::StreamInfo&, const Http::HeaderMap&, const Http::HeaderMap&, + const Http::HeaderMap&) override { + if (current_++ == 0) { + return true; + } + if (current_ >= sample_rate_) { + current_ = 0; + } + return false; + } + +private: + uint32_t current_ = 0; + uint32_t sample_rate_; +}; + +/** + * Sample extension filter factory which creates SampleExtensionFilter. + */ +class SampleExtensionFilterFactory : public ExtensionFilterFactory { +public: + ~SampleExtensionFilterFactory() override = default; + + FilterPtr createFilter(const envoy::config::filter::accesslog::v2::ExtensionFilter& config, + Runtime::Loader&, Runtime::RandomGenerator&) override { + auto factory_config = Config::Utility::translateToFactoryConfig( + config, Envoy::ProtobufMessage::getNullValidationVisitor(), *this); + const Json::ObjectSharedPtr filter_config = + MessageUtil::getJsonObjectFromMessage(*factory_config); + return std::make_unique(filter_config->getInteger("rate")); + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() const override { return "sample_extension_filter"; } +}; + +TEST_F(AccessLogImplTest, SampleExtensionFilter) { + Registry::RegisterFactory registered; + + const std::string yaml = R"EOF( +name: envoy.file_access_log +filter: + extension_filter: + name: sample_extension_filter + config: + rate: 5 +config: + path: /dev/null + )EOF"; + + InstanceSharedPtr logger = AccessLogFactory::fromProto(parseAccessLogFromV2Yaml(yaml), context_); + // For rate=5 expect 1st request to be recorded, 2nd-5th skipped, and 6th recorded. + EXPECT_CALL(*file_, write(_)); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + for (int i = 0; i <= 3; ++i) { + EXPECT_CALL(*file_, write(_)).Times(0); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); + } + EXPECT_CALL(*file_, write(_)); + logger->log(&request_headers_, &response_headers_, &response_trailers_, stream_info_); +} + +TEST_F(AccessLogImplTest, UnregisteredExtensionFilter) { + { + const std::string yaml = R"EOF( +name: envoy.file_access_log +filter: + extension_filter: + name: unregistered_extension_filter + config: + foo: bar +config: + path: /dev/null + )EOF"; + + EXPECT_THROW(AccessLogFactory::fromProto(parseAccessLogFromV2Yaml(yaml), context_), + EnvoyException); + } + + { + const std::string json = R"EOF( + { + "path": "/dev/null", + "filter": {"type": "extension_filter", "foo": "bar"} + } + )EOF"; + + EXPECT_THROW(AccessLogFactory::fromProto(parseAccessLogFromJson(json), context_), + EnvoyException); + } +} + } // namespace } // namespace AccessLog } // namespace Envoy From 2272acc8f5458c6003b79f436ccba23c8db8ede0 Mon Sep 17 00:00:00 2001 From: "leilei.gll" Date: Thu, 1 Aug 2019 12:41:54 +0800 Subject: [PATCH 286/542] dubbo_proxy: Improved code coverage (#7741) Description: Improve dubbo_proxy code coverage Risk Level: Low Testing: Unit test Docs Changes: N/A Release Notes: N/A Signed-off-by: leilei.gll --- .../network/dubbo_proxy/active_message.cc | 19 +- .../network/dubbo_proxy/active_message.h | 2 + .../network/dubbo_proxy/conn_manager.h | 3 + .../filters/network/dubbo_proxy/message.h | 2 +- .../network/dubbo_proxy/conn_manager_test.cc | 457 ++++++++++++------ .../network/dubbo_proxy/decoder_test.cc | 31 +- .../dubbo_proxy/dubbo_protocol_impl_test.cc | 2 + .../filters/network/dubbo_proxy/mocks.cc | 24 +- .../filters/network/dubbo_proxy/mocks.h | 15 + 9 files changed, 393 insertions(+), 162 deletions(-) diff --git a/source/extensions/filters/network/dubbo_proxy/active_message.cc b/source/extensions/filters/network/dubbo_proxy/active_message.cc index 4f44e3a9f638..8cb8a378b3ff 100644 --- a/source/extensions/filters/network/dubbo_proxy/active_message.cc +++ b/source/extensions/filters/network/dubbo_proxy/active_message.cc @@ -33,7 +33,9 @@ void ActiveResponseDecoder::onStreamDecoded(MessageMetadataSharedPtr metadata, metadata->message_type() == MessageType::Exception); ASSERT(metadata->hasResponseStatus()); + metadata_ = metadata; if (applyMessageEncodedFilters(metadata, ctx) != FilterStatus::Continue) { + response_status_ = DubboFilters::UpstreamResponseStatus::Complete; return; } @@ -51,7 +53,6 @@ void ActiveResponseDecoder::onStreamDecoded(MessageMetadataSharedPtr metadata, stats_.response_business_exception_.inc(); } - metadata_ = metadata; switch (metadata->response_status()) { case ResponseStatus::Ok: stats_.response_success_.inc(); @@ -195,9 +196,17 @@ ActiveMessage::~ActiveMessage() { parent_.stats().request_active_.dec(); request_timer_->complete(); for (auto& filter : decoder_filters_) { + ENVOY_LOG(debug, "destroy decoder filter"); filter->handler()->onDestroy(); } - ENVOY_LOG(debug, "ActiveMessage::~ActiveMessage()"); + + for (auto& filter : encoder_filters_) { + // Do not call on destroy twice for dual registered filters. + if (!filter->dual_filter_) { + ENVOY_LOG(debug, "destroy encoder filter"); + filter->handler()->onDestroy(); + } + } } std::list::iterator @@ -340,11 +349,7 @@ FilterStatus ActiveMessage::applyEncoderFilters(ActiveMessageEncoderFilter* filt } void ActiveMessage::sendLocalReply(const DubboFilters::DirectResponse& response, bool end_stream) { - if (!metadata_) { - // If the sendLocalReply function is called before the messageEnd callback, - // metadata_ is nullptr, metadata object needs to be created in order to generate a local reply. - metadata_ = std::make_shared(); - } + ASSERT(metadata_); metadata_->setRequestId(request_id_); parent_.sendLocalReply(*metadata_, response, end_stream); diff --git a/source/extensions/filters/network/dubbo_proxy/active_message.h b/source/extensions/filters/network/dubbo_proxy/active_message.h index 6dca717c577d..a0209fd4271a 100644 --- a/source/extensions/filters/network/dubbo_proxy/active_message.h +++ b/source/extensions/filters/network/dubbo_proxy/active_message.h @@ -124,6 +124,8 @@ class ActiveMessageEncoderFilter : public ActiveMessageFilterBase, private: DubboFilters::EncoderFilterSharedPtr handle_; + + friend class ActiveMessage; }; using ActiveMessageEncoderFilterPtr = std::unique_ptr; diff --git a/source/extensions/filters/network/dubbo_proxy/conn_manager.h b/source/extensions/filters/network/dubbo_proxy/conn_manager.h index b274c61a867a..b0599cf04ebe 100644 --- a/source/extensions/filters/network/dubbo_proxy/conn_manager.h +++ b/source/extensions/filters/network/dubbo_proxy/conn_manager.h @@ -79,6 +79,9 @@ class ConnectionManager : public Network::ReadFilter, void sendLocalReply(MessageMetadata& metadata, const DubboFilters::DirectResponse& response, bool end_stream); + // This function is for testing only. + std::list& getActiveMessagesForTest() { return active_message_list_; } + private: void dispatch(); void resetAllMessages(bool local_reset); diff --git a/source/extensions/filters/network/dubbo_proxy/message.h b/source/extensions/filters/network/dubbo_proxy/message.h index 95a435f5798d..8e74e25dcb41 100644 --- a/source/extensions/filters/network/dubbo_proxy/message.h +++ b/source/extensions/filters/network/dubbo_proxy/message.h @@ -90,7 +90,7 @@ class Context { public: using AttachmentMap = std::unordered_map; - bool hasAttachments() const { return attachments_.empty(); } + bool hasAttachments() const { return !attachments_.empty(); } const AttachmentMap& attachments() const { return attachments_; } Buffer::Instance& message_origin_data() { return message_origin_buffer_; } diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index a8dc3e35d24c..2b0429d2da77 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -35,29 +35,57 @@ namespace DubboProxy { using ConfigDubboProxy = envoy::config::filter::network::dubbo_proxy::v2alpha1::DubboProxy; +class ConnectionManagerTest; class TestConfigImpl : public ConfigImpl { public: TestConfigImpl(ConfigDubboProxy proto_config, Server::Configuration::MockFactoryContext& context, - DubboFilters::DecoderFilterSharedPtr decoder_filter, DubboFilterStats& stats) - : ConfigImpl(proto_config, context), decoder_filter_(decoder_filter), stats_(stats) {} + DubboFilterStats& stats) + : ConfigImpl(proto_config, context), stats_(stats) {} // ConfigImpl DubboFilterStats& stats() override { return stats_; } void createFilterChain(DubboFilters::FilterChainFactoryCallbacks& callbacks) override { - if (custom_filter_) { - callbacks.addDecoderFilter(custom_filter_); + if (setupChain) { + for (auto& decoder : decoder_filters_) { + callbacks.addDecoderFilter(decoder); + } + for (auto& encoder : encoder_filters_) { + callbacks.addEncoderFilter(encoder); + } + return; } - if (encoder_filter_) { - callbacks.addEncoderFilter(encoder_filter_); + if (codec_filter_) { + callbacks.addFilter(codec_filter_); } + } - if (decoder_filter_) { - callbacks.addDecoderFilter(decoder_filter_); + void setupFilterChain(int num_decoder_filters, int num_encoder_filters) { + for (int i = 0; i < num_decoder_filters; i++) { + decoder_filters_.push_back(std::make_shared>()); } + for (int i = 0; i < num_encoder_filters; i++) { + encoder_filters_.push_back(std::make_shared>()); + } + setupChain = true; + } - if (codec_filter_) { - callbacks.addFilter(codec_filter_); + void expectFilterCallbacks() { + for (auto& decoder : decoder_filters_) { + EXPECT_CALL(*decoder, setDecoderFilterCallbacks(_)); + } + for (auto& encoder : encoder_filters_) { + EXPECT_CALL(*encoder, setEncoderFilterCallbacks(_)); + } + } + + void expectOnDestroy() { + for (auto& decoder : decoder_filters_) { + EXPECT_CALL(*decoder, onDestroy()); + } + + for (auto& encoder : encoder_filters_) { + EXPECT_CALL(*encoder, onDestroy()); } } @@ -76,14 +104,16 @@ class TestConfigImpl : public ConfigImpl { return ConfigImpl::route(metadata, random_value); } - DubboFilters::DecoderFilterSharedPtr custom_filter_; - DubboFilters::DecoderFilterSharedPtr decoder_filter_; - DubboFilters::EncoderFilterSharedPtr encoder_filter_; DubboFilters::CodecFilterSharedPtr codec_filter_; DubboFilterStats& stats_; MockSerializer* serializer_{}; MockProtocol* protocol_{}; std::shared_ptr route_; + + NiceMock filter_factory_; + std::vector> decoder_filters_; + std::vector> encoder_filters_; + bool setupChain = false; }; class ConnectionManagerTest : public testing::Test { @@ -108,28 +138,23 @@ class ConnectionManagerTest : public testing::Test { } proto_config_.set_stat_prefix("test"); - decoder_filter_.reset(new NiceMock()); - config_ = - std::make_unique(proto_config_, factory_context_, decoder_filter_, stats_); + config_ = std::make_unique(proto_config_, factory_context_, stats_); if (custom_serializer_) { config_->serializer_ = custom_serializer_; } if (custom_protocol_) { config_->protocol_ = custom_protocol_; } - if (custom_filter_) { - config_->custom_filter_ = custom_filter_; - } ON_CALL(random_, random()).WillByDefault(Return(42)); - filter_ = std::make_unique( + conn_manager_ = std::make_unique( *config_, random_, filter_callbacks_.connection_.dispatcher_.timeSource()); - filter_->initializeReadFilterCallbacks(filter_callbacks_); - filter_->onNewConnection(); + conn_manager_->initializeReadFilterCallbacks(filter_callbacks_); + conn_manager_->onNewConnection(); // NOP currently. - filter_->onAboveWriteBufferHighWatermark(); - filter_->onBelowWriteBufferLowWatermark(); + conn_manager_->onAboveWriteBufferHighWatermark(); + conn_manager_->onBelowWriteBufferLowWatermark(); } void writeHessianErrorResponseMessage(Buffer::Instance& buffer, bool is_event, @@ -285,7 +310,6 @@ class ConnectionManagerTest : public testing::Test { } NiceMock factory_context_; - std::shared_ptr decoder_filter_; Stats::IsolatedStoreImpl store_; DubboFilterStats stats_; ConfigDubboProxy proto_config_; @@ -296,17 +320,16 @@ class ConnectionManagerTest : public testing::Test { Buffer::OwnedImpl write_buffer_; NiceMock filter_callbacks_; NiceMock random_; - std::unique_ptr filter_; + std::unique_ptr conn_manager_; MockSerializer* custom_serializer_{}; MockProtocol* custom_protocol_{}; - DubboFilters::DecoderFilterSharedPtr custom_filter_; }; TEST_F(ConnectionManagerTest, OnDataHandlesRequestTwoWay) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 0x0F); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.counter("test.request_twoway").value()); EXPECT_EQ(0U, store_.counter("test.request_oneway").value()); @@ -320,7 +343,7 @@ TEST_F(ConnectionManagerTest, OnDataHandlesRequestOneWay) { initializeFilter(); writeHessianRequestMessage(buffer_, true, false, 0x0F); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(0U, store_.counter("test.request_twoway").value()); EXPECT_EQ(1U, store_.counter("test.request_oneway").value()); @@ -339,18 +362,21 @@ TEST_F(ConnectionManagerTest, OnDataHandlesHeartbeatEvent) { EXPECT_CALL(filter_callbacks_.connection_, write(_, false)) .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) -> void { - ProtocolPtr protocol = filter_->config().createProtocol(); + ProtocolPtr protocol = conn_manager_->config().createProtocol(); MessageMetadataSharedPtr metadata(std::make_shared()); auto result = protocol->decodeHeader(buffer, metadata); EXPECT_TRUE(result.second); const DubboProxy::ContextImpl& ctx = *static_cast(result.first.get()); EXPECT_TRUE(ctx.is_heartbeat()); + EXPECT_TRUE(metadata->hasResponseStatus()); + EXPECT_FALSE(metadata->is_two_way()); + EXPECT_EQ(ProtocolType::Dubbo, metadata->protocol_type()); EXPECT_EQ(metadata->response_status(), ResponseStatus::Ok); EXPECT_EQ(metadata->message_type(), MessageType::HeartbeatResponse); buffer.drain(ctx.header_size()); })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0U, store_.counter("test.request").value()); @@ -364,7 +390,7 @@ TEST_F(ConnectionManagerTest, HandlesHeartbeatWithException) { EXPECT_CALL(*custom_protocol_, encode(_, _, _, _)).WillOnce(Return(false)); MessageMetadataSharedPtr meta = std::make_shared(); - EXPECT_THROW_WITH_MESSAGE(filter_->onHeartbeat(meta), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(conn_manager_->onHeartbeat(meta), EnvoyException, "failed to encode heartbeat message"); } @@ -372,12 +398,12 @@ TEST_F(ConnectionManagerTest, OnDataHandlesMessageSplitAcrossBuffers) { initializeFilter(); writePartialHessianRequestMessage(buffer_, false, false, 0x0F, true); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(0, buffer_.length()); // Complete the buffer writePartialHessianRequestMessage(buffer_, false, false, 0x0F, false); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request_twoway").value()); EXPECT_EQ(0U, store_.counter("test.request_decoding_error").value()); @@ -387,29 +413,32 @@ TEST_F(ConnectionManagerTest, OnDataHandlesProtocolError) { initializeFilter(); writeInvalidRequestMessage(buffer_); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request_decoding_error").value()); EXPECT_EQ(0, buffer_.length()); // Sniffing is now disabled. bool one_way = true; writeHessianRequestMessage(buffer_, one_way, false, 0x0F); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(0U, store_.counter("test.request").value()); } TEST_F(ConnectionManagerTest, OnDataHandlesProtocolErrorOnWrite) { initializeFilter(); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto decoder_filter = config_->decoder_filters_[0]; // Start the read buffer writePartialHessianRequestMessage(buffer_, false, false, 0x0F, true); uint64_t len = buffer_.length(); DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); len -= buffer_.length(); // Disable sniffing @@ -426,18 +455,25 @@ TEST_F(ConnectionManagerTest, OnDataHandlesProtocolErrorOnWrite) { TEST_F(ConnectionManagerTest, OnDataStopsSniffingWithTooManyPendingCalls) { initializeFilter(); - for (int i = 0; i < 64; i++) { + config_->setupFilterChain(1, 0); + // config_->expectOnDestroy(); + auto decoder_filter = config_->decoder_filters_[0]; + + int request_count = 64; + for (int i = 0; i < request_count; i++) { writeHessianRequestMessage(buffer_, false, false, i); } - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)).Times(64); + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)).Times(request_count); + EXPECT_CALL(*decoder_filter, onDestroy()).Times(request_count); + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)).Times(request_count); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(64U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); // Sniffing is now disabled. writeInvalidRequestMessage(buffer_); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -450,11 +486,15 @@ TEST_F(ConnectionManagerTest, OnWriteHandlesResponse) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, request_id); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -483,11 +523,15 @@ TEST_F(ConnectionManagerTest, HandlesResponseContainExceptionInfo) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.counter("test.request_decoding_success").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -514,11 +558,15 @@ TEST_F(ConnectionManagerTest, HandlesResponseError) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -542,11 +590,15 @@ TEST_F(ConnectionManagerTest, OnWriteHandlesResponseException) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); writeInvalidRequestMessage(write_buffer_); @@ -568,21 +620,25 @@ TEST_F(ConnectionManagerTest, OnWriteHandlesResponseException) { // Tests stop iteration/resume with multiple filters. TEST_F(ConnectionManagerTest, OnDataResumesWithNextFilter) { - auto* filter = new NiceMock(); - custom_filter_.reset(filter); - initializeFilter(); + + config_->setupFilterChain(2, 0); + config_->expectOnDestroy(); + auto first_filter = config_->decoder_filters_[0]; + auto second_filter = config_->decoder_filters_[1]; + writeHessianRequestMessage(buffer_, false, false, 0x0F); DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*first_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)); + EXPECT_CALL(*second_filter, setDecoderFilterCallbacks(_)); // First filter stops iteration. { - EXPECT_CALL(*filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::StopIteration)); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_CALL(*first_filter, onMessageDecoded(_, _)) + .WillOnce(Return(FilterStatus::StopIteration)); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(0U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -591,8 +647,8 @@ TEST_F(ConnectionManagerTest, OnDataResumesWithNextFilter) { // Resume processing. { InSequence s; - EXPECT_CALL(*filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*first_filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*second_filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); callbacks->continueDecoding(); } @@ -602,17 +658,20 @@ TEST_F(ConnectionManagerTest, OnDataResumesWithNextFilter) { // Tests multiple filters are invoked in the correct order. TEST_F(ConnectionManagerTest, OnDataHandlesDubboCallWithMultipleFilters) { - auto* filter = new NiceMock(); - custom_filter_.reset(filter); initializeFilter(); + config_->setupFilterChain(2, 0); + config_->expectOnDestroy(); + auto first_filter = config_->decoder_filters_[0]; + auto second_filter = config_->decoder_filters_[1]; + writeHessianRequestMessage(buffer_, false, false, 0x0F); InSequence s; - EXPECT_CALL(*filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*first_filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*second_filter, onMessageDecoded(_, _)).WillOnce(Return(FilterStatus::Continue)); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); } @@ -620,19 +679,23 @@ TEST_F(ConnectionManagerTest, OnDataHandlesDubboCallWithMultipleFilters) { TEST_F(ConnectionManagerTest, PipelinedRequestAndResponse) { initializeFilter(); + config_->setupFilterChain(1, 0); + auto decoder_filter = config_->decoder_filters_[0]; + writeHessianRequestMessage(buffer_, false, false, 1); writeHessianRequestMessage(buffer_, false, false, 2); std::list callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillRepeatedly(Invoke( [&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks.push_back(&cb); })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(2U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); EXPECT_EQ(2U, store_.counter("test.request").value()); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(2); + EXPECT_CALL(*decoder_filter, onDestroy()).Times(2); writeHessianResponseMessage(write_buffer_, false, 0x01); callbacks.front()->startUpstreamResponse(); @@ -659,11 +722,15 @@ TEST_F(ConnectionManagerTest, ResetDownstreamConnection) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 0x0F); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -679,8 +746,8 @@ TEST_F(ConnectionManagerTest, OnEvent) { // No active calls { initializeFilter(); - filter_->onEvent(Network::ConnectionEvent::RemoteClose); - filter_->onEvent(Network::ConnectionEvent::LocalClose); + conn_manager_->onEvent(Network::ConnectionEvent::RemoteClose); + conn_manager_->onEvent(Network::ConnectionEvent::LocalClose); EXPECT_EQ(0U, store_.counter("test.cx_destroy_local_with_active_rq").value()); EXPECT_EQ(0U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); } @@ -690,10 +757,10 @@ TEST_F(ConnectionManagerTest, OnEvent) { initializeFilter(); writePartialHessianRequestMessage(buffer_, false, false, 1, true); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - filter_->onEvent(Network::ConnectionEvent::RemoteClose); + conn_manager_->onEvent(Network::ConnectionEvent::RemoteClose); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); @@ -703,10 +770,10 @@ TEST_F(ConnectionManagerTest, OnEvent) { { initializeFilter(); writePartialHessianRequestMessage(buffer_, false, false, 1, true); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - filter_->onEvent(Network::ConnectionEvent::LocalClose); + conn_manager_->onEvent(Network::ConnectionEvent::LocalClose); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(1U, store_.counter("test.cx_destroy_local_with_active_rq").value()); @@ -718,10 +785,10 @@ TEST_F(ConnectionManagerTest, OnEvent) { { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - filter_->onEvent(Network::ConnectionEvent::RemoteClose); + conn_manager_->onEvent(Network::ConnectionEvent::RemoteClose); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); @@ -733,10 +800,10 @@ TEST_F(ConnectionManagerTest, OnEvent) { { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - filter_->onEvent(Network::ConnectionEvent::LocalClose); + conn_manager_->onEvent(Network::ConnectionEvent::LocalClose); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(1U, store_.counter("test.cx_destroy_local_with_active_rq").value()); @@ -747,12 +814,16 @@ TEST_F(ConnectionManagerTest, OnEvent) { TEST_F(ConnectionManagerTest, ResponseWithUnknownSequenceID) { initializeFilter(); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); writeHessianRequestMessage(buffer_, false, false, 1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); writeHessianResponseMessage(write_buffer_, false, 10); @@ -763,16 +834,18 @@ TEST_F(ConnectionManagerTest, ResponseWithUnknownSequenceID) { } TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalReply) { - auto* filter = new NiceMock(); - custom_filter_.reset(filter); - initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); + config_->setupFilterChain(2, 0); + config_->expectOnDestroy(); + auto& first_filter = config_->decoder_filters_[0]; + auto& second_filter = config_->decoder_filters_[1]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*first_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)); + EXPECT_CALL(*second_filter, setDecoderFilterCallbacks(_)); const std::string fake_response("mock dubbo response"); NiceMock direct_response; @@ -784,7 +857,7 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalReply) { })); // First filter sends local reply. - EXPECT_CALL(*filter, onMessageDecoded(_, _)) + EXPECT_CALL(*first_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { callbacks->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::NoRouteFound); callbacks->sendLocalReply(direct_response, false); @@ -795,7 +868,7 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalReply) { EXPECT_EQ(fake_response, buffer.toString()); })); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(SerializationType::Hessian2, callbacks->serializationType()); EXPECT_EQ(ProtocolType::Dubbo, callbacks->protocolType()); @@ -807,16 +880,18 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalReply) { } TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalErrorReply) { - auto* filter = new NiceMock(); - custom_filter_.reset(filter); - initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 1); + config_->setupFilterChain(2, 0); + config_->expectOnDestroy(); + auto& first_filter = config_->decoder_filters_[0]; + auto& second_filter = config_->decoder_filters_[1]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*filter, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*first_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)); + EXPECT_CALL(*second_filter, setDecoderFilterCallbacks(_)); const std::string fake_response("mock dubbo response"); NiceMock direct_response; @@ -828,7 +903,7 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalErrorReply) { })); // First filter sends local reply. - EXPECT_CALL(*filter, onMessageDecoded(_, _)) + EXPECT_CALL(*first_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { callbacks->sendLocalReply(direct_response, false); return FilterStatus::StopIteration; @@ -838,7 +913,7 @@ TEST_F(ConnectionManagerTest, OnDataWithFilterSendsLocalErrorReply) { EXPECT_EQ(fake_response, buffer.toString()); })); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); @@ -851,7 +926,11 @@ TEST_F(ConnectionManagerTest, TwoWayRequestWithEndStream) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 0x0F); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { return FilterStatus::StopIteration; })); @@ -859,7 +938,7 @@ TEST_F(ConnectionManagerTest, TwoWayRequestWithEndStream) { EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) .Times(1); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - EXPECT_EQ(filter_->onData(buffer_, true), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, true), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); } @@ -867,14 +946,18 @@ TEST_F(ConnectionManagerTest, OneWayRequestWithEndStream) { initializeFilter(); writeHessianRequestMessage(buffer_, true, false, 0x0F); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { return FilterStatus::StopIteration; })); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) .Times(1); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - EXPECT_EQ(filter_->onData(buffer_, true), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, true), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); } @@ -883,7 +966,7 @@ TEST_F(ConnectionManagerTest, EmptyRequestData) { buffer_.drain(buffer_.length()); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(0); - EXPECT_EQ(filter_->onData(buffer_, true), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, true), Network::FilterStatus::StopIteration); EXPECT_EQ(0U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); } @@ -891,7 +974,11 @@ TEST_F(ConnectionManagerTest, StopHandleRequest) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 0x0F); - ON_CALL(*decoder_filter_, onMessageDecoded(_, _)) + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + + ON_CALL(*decoder_filter, onMessageDecoded(_, _)) .WillByDefault(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { return FilterStatus::StopIteration; })); @@ -899,10 +986,10 @@ TEST_F(ConnectionManagerTest, StopHandleRequest) { EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) .Times(0); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(0); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(0U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); } TEST_F(ConnectionManagerTest, HandlesHeartbeatEventWithConnectionClose) { @@ -912,7 +999,7 @@ TEST_F(ConnectionManagerTest, HandlesHeartbeatEventWithConnectionClose) { EXPECT_CALL(filter_callbacks_.connection_, write(_, false)).Times(0); filter_callbacks_.connection_.close(Network::ConnectionCloseType::FlushWrite); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0U, store_.counter("test.request").value()); @@ -934,40 +1021,48 @@ TEST_F(ConnectionManagerTest, SendsLocalReplyWithCloseConnection) { .Times(1); MessageMetadata metadata; - filter_->sendLocalReply(metadata, direct_response, true); + conn_manager_->sendLocalReply(metadata, direct_response, true); EXPECT_EQ(1U, store_.counter("test.local_response_error").value()); // The connection closed. EXPECT_CALL(direct_response, encode(_, _, _)).Times(0); - filter_->sendLocalReply(metadata, direct_response, true); + conn_manager_->sendLocalReply(metadata, direct_response, true); } TEST_F(ConnectionManagerTest, ContinueDecodingWithHalfClose) { initializeFilter(); writeHessianRequestMessage(buffer_, true, false, 0x0F); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { return FilterStatus::StopIteration; })); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)) .Times(1); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); - EXPECT_EQ(filter_->onData(buffer_, true), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, true), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.cx_destroy_remote_with_active_rq").value()); - filter_->continueDecoding(); + conn_manager_->continueDecoding(); } TEST_F(ConnectionManagerTest, RoutingSuccess) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 0x0F); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); config_->route_ = std::make_shared(); EXPECT_EQ(config_->route_, callbacks->route()); @@ -980,13 +1075,17 @@ TEST_F(ConnectionManagerTest, RoutingFailure) { initializeFilter(); writePartialHessianRequestMessage(buffer_, false, false, 0x0F, true); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)).Times(0); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)).Times(0); DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); // The metadata is nullptr. config_->route_ = std::make_shared(); @@ -997,11 +1096,15 @@ TEST_F(ConnectionManagerTest, ResetStream) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, 0x0F); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); callbacks->resetStream(); @@ -1012,11 +1115,15 @@ TEST_F(ConnectionManagerTest, NeedMoreDataForHandleResponse) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, request_id); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -1032,15 +1139,19 @@ TEST_F(ConnectionManagerTest, PendingMessageEnd) { initializeFilter(); writeHessianRequestMessage(buffer_, false, false, request_id); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { return FilterStatus::StopIteration; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(0U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); } @@ -1065,10 +1176,14 @@ serialization_type: Hessian2 initializeFilter(yaml); writeHessianRequestMessage(buffer_, false, false, 100); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata, ContextSharedPtr) -> FilterStatus { auto invo = static_cast(&metadata->invocation_info()); auto data = const_cast(invo); @@ -1077,7 +1192,7 @@ serialization_type: Hessian2 return FilterStatus::StopIteration; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(0U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -1090,12 +1205,16 @@ serialization_type: Hessian2 TEST_F(ConnectionManagerTest, TransportEndWithConnectionClose) { initializeFilter(); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); writeHessianRequestMessage(buffer_, false, false, 1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); writeHessianResponseMessage(write_buffer_, false, 1); @@ -1110,8 +1229,12 @@ TEST_F(ConnectionManagerTest, TransportEndWithConnectionClose) { TEST_F(ConnectionManagerTest, MessageDecodedReturnStopIteration) { initializeFilter(); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); // The sendLocalReply is not called and the message type is not oneway, @@ -1121,13 +1244,13 @@ TEST_F(ConnectionManagerTest, MessageDecodedReturnStopIteration) { writeHessianRequestMessage(buffer_, false, false, 1); size_t buf_size = buffer_.length(); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr ctx) -> FilterStatus { EXPECT_EQ(ctx->message_size(), buf_size); return FilterStatus::StopIteration; })); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); // Buffer data should be consumed. EXPECT_EQ(0, buffer_.length()); @@ -1139,8 +1262,12 @@ TEST_F(ConnectionManagerTest, MessageDecodedReturnStopIteration) { TEST_F(ConnectionManagerTest, SendLocalReplyInMessageDecoded) { initializeFilter(); + config_->setupFilterChain(1, 0); + config_->expectOnDestroy(); + auto& decoder_filter = config_->decoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); const std::string fake_response("mock dubbo response"); @@ -1151,8 +1278,10 @@ TEST_F(ConnectionManagerTest, SendLocalReplyInMessageDecoded) { buffer.add(fake_response); return DubboFilters::DirectResponse::ResponseType::ErrorReply; })); - EXPECT_CALL(*decoder_filter_, onMessageDecoded(_, _)) + EXPECT_CALL(*decoder_filter, onMessageDecoded(_, _)) .WillOnce(Invoke([&](MessageMetadataSharedPtr, ContextSharedPtr) -> FilterStatus { + EXPECT_EQ(1, conn_manager_->getActiveMessagesForTest().size()); + EXPECT_NE(nullptr, conn_manager_->getActiveMessagesForTest().front()->metadata()); callbacks->sendLocalReply(direct_response, false); return FilterStatus::StopIteration; })); @@ -1162,7 +1291,7 @@ TEST_F(ConnectionManagerTest, SendLocalReplyInMessageDecoded) { writeHessianRequestMessage(buffer_, false, false, 1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); // Buffer data should be consumed. EXPECT_EQ(0, buffer_.length()); @@ -1174,19 +1303,22 @@ TEST_F(ConnectionManagerTest, SendLocalReplyInMessageDecoded) { TEST_F(ConnectionManagerTest, HandleResponseWithEncoderFilter) { uint64_t request_id = 100; initializeFilter(); - config_->encoder_filter_ = std::make_unique(); - auto mock_encoder_filter = - static_cast(config_->encoder_filter_.get()); writeHessianRequestMessage(buffer_, false, false, request_id); + config_->setupFilterChain(1, 1); + auto& decoder_filter = config_->decoder_filters_[0]; + auto& encoder_filter = config_->encoder_filters_[0]; + DubboFilters::DecoderFilterCallbacks* callbacks{}; - EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + EXPECT_CALL(*decoder_filter, setDecoderFilterCallbacks(_)) .WillOnce(Invoke([&](DubboFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); - EXPECT_CALL(*mock_encoder_filter, setEncoderFilterCallbacks(_)).Times(1); + EXPECT_CALL(*encoder_filter, setEncoderFilterCallbacks(_)).Times(1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_CALL(*decoder_filter, onDestroy()).Times(1); + + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -1199,7 +1331,7 @@ TEST_F(ConnectionManagerTest, HandleResponseWithEncoderFilter) { EXPECT_GE(callbacks->streamId(), 0); size_t expect_response_length = write_buffer_.length(); - EXPECT_CALL(*mock_encoder_filter, onMessageEncoded(_, _)) + EXPECT_CALL(*encoder_filter, onMessageEncoded(_, _)) .WillOnce( Invoke([&](MessageMetadataSharedPtr metadata, ContextSharedPtr ctx) -> FilterStatus { EXPECT_EQ(metadata->request_id(), request_id); @@ -1209,7 +1341,7 @@ TEST_F(ConnectionManagerTest, HandleResponseWithEncoderFilter) { EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Complete, callbacks->upstreamData(write_buffer_)); - + EXPECT_CALL(*encoder_filter, onDestroy()).Times(1); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(1U, store_.counter("test.response").value()); @@ -1222,9 +1354,6 @@ TEST_F(ConnectionManagerTest, HandleResponseWithCodecFilter) { config_->codec_filter_ = std::make_unique(); auto mock_codec_filter = static_cast(config_->codec_filter_.get()); - config_->decoder_filter_.reset(); - decoder_filter_.reset(); - EXPECT_EQ(config_->decoder_filter_.get(), nullptr); writeHessianRequestMessage(buffer_, false, false, request_id); @@ -1239,7 +1368,7 @@ TEST_F(ConnectionManagerTest, HandleResponseWithCodecFilter) { EXPECT_CALL(*mock_codec_filter, setEncoderFilterCallbacks(_)).Times(1); - EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); EXPECT_EQ(1U, store_.counter("test.request").value()); EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); @@ -1270,6 +1399,48 @@ TEST_F(ConnectionManagerTest, HandleResponseWithCodecFilter) { EXPECT_EQ(1U, store_.counter("test.response_success").value()); } +TEST_F(ConnectionManagerTest, AddDataWithStopAndContinue) { + InSequence s; + initializeFilter(); + config_->setupFilterChain(3, 3); + + uint64_t request_id = 100; + + EXPECT_CALL(*config_->decoder_filters_[0], onMessageDecoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata, ContextSharedPtr) -> FilterStatus { + EXPECT_EQ(metadata->request_id(), request_id); + return FilterStatus::Continue; + })); + EXPECT_CALL(*config_->decoder_filters_[1], onMessageDecoded(_, _)) + .WillOnce(Return(FilterStatus::StopIteration)) + .WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*config_->decoder_filters_[2], onMessageDecoded(_, _)) + .WillOnce(Return(FilterStatus::Continue)); + writeHessianRequestMessage(buffer_, false, false, request_id); + EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); + config_->decoder_filters_[1]->callbacks_->continueDecoding(); + + // For encode direction + EXPECT_CALL(*config_->encoder_filters_[0], onMessageEncoded(_, _)) + .WillOnce(Invoke([&](MessageMetadataSharedPtr metadata, ContextSharedPtr) -> FilterStatus { + EXPECT_EQ(metadata->request_id(), request_id); + return FilterStatus::Continue; + })); + EXPECT_CALL(*config_->encoder_filters_[1], onMessageEncoded(_, _)) + .WillOnce(Return(FilterStatus::StopIteration)) + .WillOnce(Return(FilterStatus::Continue)); + EXPECT_CALL(*config_->encoder_filters_[2], onMessageEncoded(_, _)) + .WillOnce(Return(FilterStatus::Continue)); + + writeHessianResponseMessage(write_buffer_, false, request_id); + config_->decoder_filters_[0]->callbacks_->startUpstreamResponse(); + EXPECT_EQ(DubboFilters::UpstreamResponseStatus::Complete, + config_->decoder_filters_[0]->callbacks_->upstreamData(write_buffer_)); + + config_->encoder_filters_[1]->callbacks_->continueEncoding(); + config_->expectOnDestroy(); +} + } // namespace DubboProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc index aade1c594766..bbada6519a39 100644 --- a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc @@ -235,14 +235,8 @@ TEST_F(DubboDecoderTest, decodeResponseMessage) { context->set_body_size(10); return std::pair(context, true); })); - EXPECT_CALL(protocol_, decodeData(_, _, _)) - .WillOnce(Invoke([&](Buffer::Instance&, ContextSharedPtr, MessageMetadataSharedPtr) -> bool { - return true; - })); - - EXPECT_CALL(response_callbacks_, newStream()).WillOnce(Invoke([this]() -> StreamHandler& { - return handler_; - })); + EXPECT_CALL(protocol_, decodeData(_, _, _)).WillOnce(Return(true)); + EXPECT_CALL(response_callbacks_, newStream()).WillOnce(ReturnRef(handler_)); EXPECT_CALL(response_callbacks_, onHeartbeat(_)).Times(0); EXPECT_CALL(handler_, onStreamDecoded(_, _)).Times(1); @@ -251,6 +245,27 @@ TEST_F(DubboDecoderTest, decodeResponseMessage) { bool buffer_underflow; EXPECT_EQ(decoder.onData(buffer, buffer_underflow), FilterStatus::Continue); EXPECT_EQ(buffer_underflow, true); + + decoder.reset(); + + EXPECT_EQ(ProtocolType::Dubbo, decoder.protocol().type()); + EXPECT_CALL(protocol_, decodeHeader(_, _)) + .WillOnce(Invoke([](Buffer::Instance&, + MessageMetadataSharedPtr metadate) -> std::pair { + metadate->setMessageType(MessageType::Response); + auto context = std::make_shared(); + context->set_header_size(16); + context->set_body_size(10); + return std::pair(context, true); + })); + EXPECT_CALL(protocol_, decodeData(_, _, _)).WillOnce(Return(true)); + EXPECT_CALL(response_callbacks_, newStream()).WillOnce(ReturnRef(handler_)); + EXPECT_CALL(response_callbacks_, onHeartbeat(_)).Times(0); + EXPECT_CALL(handler_, onStreamDecoded(_, _)).Times(1); + + buffer_underflow = false; + EXPECT_EQ(decoder.onData(buffer, buffer_underflow), FilterStatus::Continue); + EXPECT_EQ(buffer_underflow, true); } } // namespace DubboProxy diff --git a/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc b/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc index 3ee19707380a..38e3d97ad935 100644 --- a/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc @@ -148,6 +148,8 @@ TEST(DubboProtocolImplTest, encode) { body_buffer, content, RpcResponseType::ResponseWithValue); auto context = result.first; EXPECT_EQ(context->body_size(), serialized_body_size); + EXPECT_EQ(false, context->hasAttachments()); + EXPECT_EQ(0, context->attachments().size()); buffer.drain(context->header_size()); EXPECT_TRUE(dubbo_protocol.decodeData(buffer, context, output_metadata)); diff --git a/test/extensions/filters/network/dubbo_proxy/mocks.cc b/test/extensions/filters/network/dubbo_proxy/mocks.cc index 2e93c8eebbc3..13565441e643 100644 --- a/test/extensions/filters/network/dubbo_proxy/mocks.cc +++ b/test/extensions/filters/network/dubbo_proxy/mocks.cc @@ -48,10 +48,17 @@ MockSerializer::~MockSerializer() = default; namespace DubboFilters { +MockFilterChainFactory::MockFilterChainFactory() = default; +MockFilterChainFactory::~MockFilterChainFactory() = default; + MockFilterChainFactoryCallbacks::MockFilterChainFactoryCallbacks() = default; MockFilterChainFactoryCallbacks::~MockFilterChainFactoryCallbacks() = default; -MockDecoderFilter::MockDecoderFilter() = default; +MockDecoderFilter::MockDecoderFilter() { + ON_CALL(*this, setDecoderFilterCallbacks(_)) + .WillByDefault( + Invoke([this](DecoderFilterCallbacks& callbacks) -> void { callbacks_ = &callbacks; })); +} MockDecoderFilter::~MockDecoderFilter() = default; MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { @@ -65,7 +72,11 @@ MockDecoderFilterCallbacks::MockDecoderFilterCallbacks() { } MockDecoderFilterCallbacks::~MockDecoderFilterCallbacks() = default; -MockEncoderFilter::MockEncoderFilter() = default; +MockEncoderFilter::MockEncoderFilter() { + ON_CALL(*this, setEncoderFilterCallbacks(_)) + .WillByDefault( + Invoke([this](EncoderFilterCallbacks& callbacks) -> void { callbacks_ = &callbacks; })); +} MockEncoderFilter::~MockEncoderFilter() = default; MockEncoderFilterCallbacks::MockEncoderFilterCallbacks() { @@ -79,7 +90,14 @@ MockEncoderFilterCallbacks::MockEncoderFilterCallbacks() { } MockEncoderFilterCallbacks::~MockEncoderFilterCallbacks() = default; -MockCodecFilter::MockCodecFilter() = default; +MockCodecFilter::MockCodecFilter() { + ON_CALL(*this, setDecoderFilterCallbacks(_)) + .WillByDefault(Invoke( + [this](DecoderFilterCallbacks& callbacks) -> void { decoder_callbacks_ = &callbacks; })); + ON_CALL(*this, setEncoderFilterCallbacks(_)) + .WillByDefault(Invoke( + [this](EncoderFilterCallbacks& callbacks) -> void { encoder_callbacks_ = &callbacks; })); +} MockCodecFilter::~MockCodecFilter() = default; MockFilterConfigFactory::MockFilterConfigFactory() diff --git a/test/extensions/filters/network/dubbo_proxy/mocks.h b/test/extensions/filters/network/dubbo_proxy/mocks.h index 02a5176f1485..4e55ca6b950a 100644 --- a/test/extensions/filters/network/dubbo_proxy/mocks.h +++ b/test/extensions/filters/network/dubbo_proxy/mocks.h @@ -154,6 +154,14 @@ class MockRoute; namespace DubboFilters { +class MockFilterChainFactory : public FilterChainFactory { +public: + MockFilterChainFactory(); + ~MockFilterChainFactory() override; + + MOCK_METHOD1(createFilterChain, void(DubboFilters::FilterChainFactoryCallbacks& callbacks)); +}; + class MockFilterChainFactoryCallbacks : public FilterChainFactoryCallbacks { public: MockFilterChainFactoryCallbacks(); @@ -173,6 +181,8 @@ class MockDecoderFilter : public DecoderFilter { MOCK_METHOD0(onDestroy, void()); MOCK_METHOD1(setDecoderFilterCallbacks, void(DecoderFilterCallbacks& callbacks)); MOCK_METHOD2(onMessageDecoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); + + DecoderFilterCallbacks* callbacks_{}; }; class MockDecoderFilterCallbacks : public DecoderFilterCallbacks { @@ -212,6 +222,8 @@ class MockEncoderFilter : public EncoderFilter { MOCK_METHOD0(onDestroy, void()); MOCK_METHOD1(setEncoderFilterCallbacks, void(EncoderFilterCallbacks& callbacks)); MOCK_METHOD2(onMessageEncoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); + + EncoderFilterCallbacks* callbacks_{}; }; class MockEncoderFilterCallbacks : public EncoderFilterCallbacks { @@ -249,6 +261,9 @@ class MockCodecFilter : public CodecFilter { MOCK_METHOD2(onMessageEncoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); MOCK_METHOD1(setDecoderFilterCallbacks, void(DecoderFilterCallbacks& callbacks)); MOCK_METHOD2(onMessageDecoded, FilterStatus(MessageMetadataSharedPtr, ContextSharedPtr)); + + DecoderFilterCallbacks* decoder_callbacks_{}; + EncoderFilterCallbacks* encoder_callbacks_{}; }; class MockDirectResponse : public DirectResponse { From b02c73f3ebbaf5c7852fa814eb3d61e841588068 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 1 Aug 2019 01:57:40 -0700 Subject: [PATCH 287/542] ci: add missing tools back to build image (#7788) Description: Some tools missed from #7772 Risk Level: Testing: #7786 Docs Changes: Release Notes: Signed-off-by: Lizan Zhou --- ci/build_container/build_container_ubuntu.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_container/build_container_ubuntu.sh b/ci/build_container/build_container_ubuntu.sh index e329a67eafd7..5a11ae1a5af4 100755 --- a/ci/build_container/build_container_ubuntu.sh +++ b/ci/build_container/build_container_ubuntu.sh @@ -18,8 +18,8 @@ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 1000 update-alternatives --config gcc update-alternatives --config g++ -apt-get install -y --no-install-recommends curl wget make cmake git python python-pip python3 python3-pip \ - unzip bc libtool ninja-build automake zip time gdb strace tshark tcpdump patch \ +apt-get install -y --no-install-recommends curl wget make cmake git python python-pip python-setuptools python3 python3-pip \ + unzip bc libtool ninja-build automake zip time gdb strace tshark tcpdump patch xz-utils rsync \ # clang 8. case $ARCH in From 9123183afe20fdecdac6032df4b0409f6648a976 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Thu, 1 Aug 2019 19:49:49 +0530 Subject: [PATCH 288/542] fix server state stat (#7798) Signed-off-by: Rama Chavali --- source/server/server.cc | 2 +- test/server/server_test.cc | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/server/server.cc b/source/server/server.cc index b948a32c1384..70c234a1362d 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -195,7 +195,7 @@ void InstanceImpl::flushStatsInternal() { server_stats_->days_until_first_cert_expiring_.set( sslContextManager().daysUntilFirstCertExpires()); server_stats_->state_.set( - enumToInt(Utility::serverState(initManager().state(), !healthCheckFailed()))); + enumToInt(Utility::serverState(initManager().state(), healthCheckFailed()))); InstanceUtil::flushMetricsToSinks(config_.statsSinks(), stats_store_); // TODO(ramaraochavali): consider adding different flush interval for histograms. if (stat_flush_timer_ != nullptr) { diff --git a/test/server/server_test.cc b/test/server/server_test.cc index b5044f1b87c3..9809dec3093c 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -315,6 +315,7 @@ TEST_P(ServerInstanceImplTest, EmptyShutdownLifecycleNotifications) { server_thread->join(); // Validate that initialization_time histogram value has been set. EXPECT_TRUE(stats_store_.histogram("server.initialization_time").used()); + EXPECT_EQ(0L, TestUtility::findGauge(stats_store_, "server.state")->value()); } TEST_P(ServerInstanceImplTest, LifecycleNotifications) { From 43bb295ae8d81c7a0a8849f44a5a91fbc11957a0 Mon Sep 17 00:00:00 2001 From: Derek Date: Thu, 1 Aug 2019 07:23:50 -0700 Subject: [PATCH 289/542] tools: add coverage reporting & enforcement to router check (#7727) Signed-off-by: Derek Schaller --- .../root/configuration/tools/router_check.rst | 28 +++++++++++++++++-- docs/root/intro/version_history.rst | 1 + test/tools/router_check/BUILD | 2 ++ test/tools/router_check/coverage.cc | 23 +++++++++++++++ test/tools/router_check/coverage.h | 18 ++++++++++++ test/tools/router_check/router.cc | 16 ++++++++--- test/tools/router_check/router.h | 12 +++++++- test/tools/router_check/router_check.cc | 11 ++++++++ test/tools/router_check/test/route_tests.sh | 22 ++++++++++++--- 9 files changed, 121 insertions(+), 12 deletions(-) create mode 100644 test/tools/router_check/coverage.cc create mode 100644 test/tools/router_check/coverage.h diff --git a/docs/root/configuration/tools/router_check.rst b/docs/root/configuration/tools/router_check.rst index fa6dd076eafb..5a596521e95c 100644 --- a/docs/root/configuration/tools/router_check.rst +++ b/docs/root/configuration/tools/router_check.rst @@ -3,9 +3,11 @@ Route table check tool ====================== -**NOTE: The following configuration is for the route table check tool only and is not part of the Envoy binary. -The route table check tool is a standalone binary that can be used to verify Envoy's routing for a given configuration -file.** +.. note:: + + The following configuration is for the route table check tool only and is not part of the Envoy binary. + The route table check tool is a standalone binary that can be used to verify Envoy's routing for a given configuration + file. The following specifies input to the route table check tool. The route table check tool checks if the route returned by a :ref:`router ` matches what is expected. @@ -148,3 +150,23 @@ validate value *(required, string)* The value of the header field to match. + +Coverage +-------- + +The router check tool will report route coverage at the end of a successful test run. + +.. code:: bash + + > bazel-bin/test/tools/router_check/router_check_tool --config-path ... --test-path ... --useproto + Current route coverage: 0.0744863 + +This reporting can be leveraged to enforce a minimum coverage percentage by using +the `-f` or `--fail-under` flag. If coverage falls below this percentage the test +run will fail. + +.. code:: bash + + > bazel-bin/test/tools/router_check/router_check_tool --config-path ... --test-path ... --useproto --fail-under 0.08 + Current route coverage: 0.0744863 + Failed to meet coverage requirement: 0.08 diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 41c6509e9c7d..1ae489d86fef 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -14,6 +14,7 @@ Version history * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`HTTP inspector listener filter `. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. +* router check tool: add coverage reporting & enforcement. * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. * upstream: added network filter chains to upstream connections, see :ref:`filters`. diff --git a/test/tools/router_check/BUILD b/test/tools/router_check/BUILD index ceb012625dd5..c4a1a2facf4b 100644 --- a/test/tools/router_check/BUILD +++ b/test/tools/router_check/BUILD @@ -19,6 +19,8 @@ envoy_cc_test_binary( envoy_cc_test_library( name = "router_check_main_lib", srcs = [ + "coverage.cc", + "coverage.h", "router.cc", "router.h", ], diff --git a/test/tools/router_check/coverage.cc b/test/tools/router_check/coverage.cc new file mode 100644 index 000000000000..5f8c51a51351 --- /dev/null +++ b/test/tools/router_check/coverage.cc @@ -0,0 +1,23 @@ +#include "test/tools/router_check/coverage.h" + +#include + +#include "envoy/api/v2/core/base.pb.h" + +namespace Envoy { +void Coverage::markCovered(const Envoy::Router::RouteEntry& route) { + // n.b. If we reach the end of the seen routes without finding the specified + // route we add it as seen, otherwise it's a duplicate. + if (std::find(seen_routes_.begin(), seen_routes_.end(), &route) == seen_routes_.end()) { + seen_routes_.push_back(&route); + } +} + +double Coverage::report() { + uint64_t num_routes = 0; + for (const auto& host : route_config_.virtual_hosts()) { + num_routes += host.routes_size(); + } + return 100 * static_cast(seen_routes_.size()) / num_routes; +} +} // namespace Envoy diff --git a/test/tools/router_check/coverage.h b/test/tools/router_check/coverage.h new file mode 100644 index 000000000000..778906a89fca --- /dev/null +++ b/test/tools/router_check/coverage.h @@ -0,0 +1,18 @@ +#pragma once + +#include "envoy/router/router.h" + +#include "test/mocks/server/mocks.h" + +namespace Envoy { +class Coverage : Logger::Loggable { +public: + Coverage(envoy::api::v2::RouteConfiguration config) : route_config_(config){}; + void markCovered(const Envoy::Router::RouteEntry& route); + double report(); + +private: + std::vector seen_routes_; + const envoy::api::v2::RouteConfiguration route_config_; +}; +} // namespace Envoy diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index 74b1437fca59..70ed8c6fbd19 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -74,15 +74,15 @@ RouterCheckTool RouterCheckTool::create(const std::string& router_config_file) { auto config = std::make_unique(route_config, *factory_context, false); return RouterCheckTool(std::move(factory_context), std::move(config), std::move(stats), - std::move(api)); + std::move(api), Coverage(route_config)); } RouterCheckTool::RouterCheckTool( std::unique_ptr> factory_context, std::unique_ptr config, std::unique_ptr stats, - Api::ApiPtr api) + Api::ApiPtr api, Coverage coverage) : factory_context_(std::move(factory_context)), config_(std::move(config)), - stats_(std::move(stats)), api_(std::move(api)) { + stats_(std::move(stats)), api_(std::move(api)), coverage_(std::move(coverage)) { ON_CALL(factory_context_->runtime_loader_.snapshot_, featureEnabled(_, testing::An(), testing::An())) @@ -281,7 +281,11 @@ bool RouterCheckTool::compareRewritePath(ToolConfig& tool_config, const std::str actual = tool_config.headers_->get_(Http::Headers::get().Path); } - return compareResults(actual, expected, "path_rewrite"); + const bool matches = compareResults(actual, expected, "path_rewrite"); + if (matches) { + coverage_.markCovered(*tool_config.route_->routeEntry()); + } + return matches; } bool RouterCheckTool::compareRewritePath( @@ -415,6 +419,9 @@ Options::Options(int argc, char** argv) { TCLAP::CmdLine cmd("router_check_tool", ' ', "none", true); TCLAP::SwitchArg is_proto("p", "useproto", "Use Proto test file schema", cmd, false); TCLAP::SwitchArg is_detailed("d", "details", "Show detailed test execution results", cmd, false); + TCLAP::ValueArg fail_under("f", "fail-under", + "Fail if test coverage is under a specified amount", false, + 0.0, "float", cmd); TCLAP::ValueArg config_path("c", "config-path", "Path to configuration file.", false, "", "string", cmd); TCLAP::ValueArg test_path("t", "test-path", "Path to test file.", false, "", @@ -430,6 +437,7 @@ Options::Options(int argc, char** argv) { is_proto_ = is_proto.getValue(); is_detailed_ = is_detailed.getValue(); + fail_under_ = fail_under.getValue(); if (is_proto_) { config_path_ = config_path.getValue(); diff --git a/test/tools/router_check/router.h b/test/tools/router_check/router.h index 6c507f5bc8bd..f4af695dcf7a 100644 --- a/test/tools/router_check/router.h +++ b/test/tools/router_check/router.h @@ -17,6 +17,7 @@ #include "test/test_common/global.h" #include "test/test_common/printers.h" #include "test/test_common/utility.h" +#include "test/tools/router_check/coverage.h" #include "test/tools/router_check/json/tool_config_schemas.h" #include "test/tools/router_check/validation.pb.h" #include "test/tools/router_check/validation.pb.validate.h" @@ -88,11 +89,13 @@ class RouterCheckTool : Logger::Loggable { */ void setShowDetails() { details_ = true; } + float coverage() { return coverage_.report(); } + private: RouterCheckTool( std::unique_ptr> factory_context, std::unique_ptr config, std::unique_ptr stats, - Api::ApiPtr api); + Api::ApiPtr api, Coverage coverage); bool compareCluster(ToolConfig& tool_config, const std::string& expected); bool compareCluster(ToolConfig& tool_config, @@ -143,6 +146,7 @@ class RouterCheckTool : Logger::Loggable { std::unique_ptr stats_; Api::ApiPtr api_; std::string active_runtime; + Coverage coverage_; }; /** @@ -172,6 +176,11 @@ class Options { */ const std::string& unlabelledTestPath() const { return unlabelled_test_path_; } + /** + * @return the minimum required percentage of routes coverage. + */ + double failUnder() const { return fail_under_; } + /** * @return true if proto schema test is used. */ @@ -187,6 +196,7 @@ class Options { std::string config_path_; std::string unlabelled_test_path_; std::string unlabelled_config_path_; + float fail_under_; bool is_proto_; bool is_detailed_; }; diff --git a/test/tools/router_check/router_check.cc b/test/tools/router_check/router_check.cc index 40f6acec3948..d7b27d99f3a3 100644 --- a/test/tools/router_check/router_check.cc +++ b/test/tools/router_check/router_check.cc @@ -7,6 +7,7 @@ int main(int argc, char* argv[]) { Envoy::Options options(argc, argv); + const bool enforce_coverage = options.failUnder() != 0.0; try { Envoy::RouterCheckTool checktool = options.isProto() ? Envoy::RouterCheckTool::create(options.configPath()) @@ -23,6 +24,16 @@ int main(int argc, char* argv[]) { if (!is_equal) { return EXIT_FAILURE; } + + const double current_coverage = checktool.coverage(); + std::cerr << "Current route coverage: " << current_coverage << "%" << std::endl; + if (enforce_coverage) { + if (current_coverage < options.failUnder()) { + std::cerr << "Failed to meet coverage requirement: " << options.failUnder() << "%" + << std::endl; + return EXIT_FAILURE; + } + } } catch (const Envoy::EnvoyException& ex) { std::cerr << ex.what() << std::endl; return EXIT_FAILURE; diff --git a/test/tools/router_check/test/route_tests.sh b/test/tools/router_check/test/route_tests.sh index 366c23a73333..cba937d3f7e3 100755 --- a/test/tools/router_check/test/route_tests.sh +++ b/test/tools/router_check/test/route_tests.sh @@ -16,6 +16,20 @@ do TEST_OUTPUT=$("${PATH_BIN}" "${PATH_CONFIG}/${t}.yaml" "${PATH_CONFIG}/${t}.golden.json" "--details") done +# Testing coverage flag passes +COVERAGE_CMD="${PATH_BIN} ${PATH_CONFIG}/Redirect.yaml ${PATH_CONFIG}/Redirect.golden.json --details -f " +TEST_OUTPUT=$($COVERAGE_CMD "1.0") +COVERAGE_OUTPUT=$($COVERAGE_CMD "1.0" 2>&1) || echo "${COVERAGE_OUTPUT:-no-output}" +if [[ "${COVERAGE_OUTPUT}" != *"Current route coverage: "* ]] ; then + exit 1 +fi + +# Testing coverage flag fails +COVERAGE_OUTPUT=$($COVERAGE_CMD "100" 2>&1) || echo "${COVERAGE_OUTPUT:-no-output}" +if [[ "${COVERAGE_OUTPUT}" != *"Failed to meet coverage requirement: 100%"* ]] ; then + exit 1 +fi + # Testing expected matches using --useproto # --useproto needs the test schema as a validation.proto message. TESTS+=("Runtime") @@ -31,17 +45,17 @@ TEST_OUTPUT=$("${PATH_BIN}" "-c" "${PATH_CONFIG}/Weighted.yaml" "-t" "${PATH_CON TEST_OUTPUT=$("${PATH_BIN}" "-c" "${PATH_CONFIG}/Weighted.yaml" "-t" "${PATH_CONFIG}/Weighted.golden.proto.pb_text" "--details" "--useproto") # Bad config file -echo testing bad config output +echo "testing bad config output" BAD_CONFIG_OUTPUT=$(("${PATH_BIN}" "${PATH_CONFIG}/Redirect.golden.json" "${PATH_CONFIG}/TestRoutes.yaml") 2>&1) || - echo ${BAD_CONFIG_OUTPUT:-no-output} + echo "${BAD_CONFIG_OUTPUT:-no-output}" if [[ "${BAD_CONFIG_OUTPUT}" != *"Unable to parse"* ]]; then exit 1 fi # Failure test case -echo testing failure test case +echo "testing failure test case" FAILURE_OUTPUT=$("${PATH_BIN}" "${PATH_CONFIG}/TestRoutes.yaml" "${PATH_CONFIG}/Weighted.golden.json" "--details" 2>&1) || - echo ${FAILURE_OUTPUT:-no-output} + echo "${FAILURE_OUTPUT:-no-output}" if [[ "${FAILURE_OUTPUT}" != *"expected: [cluster1], actual: [instant-server], test type: cluster_name"* ]]; then exit 1 fi From 9a9c1936c1f2ddb7939181e082683e7c48a15e19 Mon Sep 17 00:00:00 2001 From: soya3129 <43042079+soya3129@users.noreply.github.com> Date: Thu, 1 Aug 2019 12:57:18 -0400 Subject: [PATCH 290/542] http: add a test case where metadata exceed size limit (#7747) Description: Add a test to cover the case where nghttp2 returns process error because metadata exceed size limit. The PR also adds a few asserts to verify when nghttp2 returns error, no more data is passed to filters (a bug fixed by#6852). This PR is a followup for #6842. Risk Level: Low. Testing: N/A Docs Changes: N/A Release Notes: N/A Signed-off-by: Yang Song --- test/integration/http2_integration_test.cc | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index be8fc519633f..f93b8ea2df84 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -472,6 +472,32 @@ TEST_P(Http2MetadataIntegrationTest, ProxyLargeMetadataInRequest) { ASSERT_TRUE(response->complete()); } +TEST_P(Http2MetadataIntegrationTest, RequestMetadataReachSizeLimit) { + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + auto encoder_decoder = codec_client_->startRequest(default_request_headers_); + request_encoder_ = &encoder_decoder.first; + auto response = std::move(encoder_decoder.second); + std::string value = std::string(10 * 1024, '1'); + Http::MetadataMap metadata_map = {{"key", value}}; + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 1, false); + codec_client_->sendMetadata(*request_encoder_, metadata_map); + codec_client_->sendData(*request_encoder_, 1, false); + for (int i = 0; i < 200; i++) { + codec_client_->sendMetadata(*request_encoder_, metadata_map); + if (codec_client_->disconnected()) { + break; + } + } + + // Verifies client connection will be closed. + codec_client_->waitForDisconnect(); + ASSERT_FALSE(response->complete()); +} + static std::string request_metadata_filter = R"EOF( name: request-metadata-filter config: {} From 07267190c6d8b88ff55c5b9fe607221f7cdcca99 Mon Sep 17 00:00:00 2001 From: moderation Date: Thu, 1 Aug 2019 12:16:53 -0700 Subject: [PATCH 291/542] Dependency: update PGV, opencensus-proto, nghttp2, msgpack-c, bazel-gazelle & cleanup (#7787) Update dependencies: * PGV (changes) * switch opencensus-proto to 0.2.1 release (changes) /cc @kyessenov * nghttp2 1.39.1 (release notes 1.39.0, 1.39.1) * rules_go 0.19.1 (0.19.0 release notes) this is breaking the API only. Reverting to master and will sort out later. * msgpack-c 3.2.0 (release notes) * bazel-gazelle 0.18.1 (0.18.0 release notes) * comment cleanup (remove uneccesary todo and add dates to commits) * removed unused old backward dependency build file Risk Level: Medium Testing: bazel test //test/... Signed-off-by: Michael Payne --- api/bazel/repository_locations.bzl | 12 ++++++------ bazel/external/backward.BUILD | 8 -------- bazel/repository_locations.bzl | 21 ++++++++++----------- 3 files changed, 16 insertions(+), 25 deletions(-) delete mode 100644 bazel/external/backward.BUILD diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index dd661b7d7886..0d0600a984bd 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -4,11 +4,11 @@ BAZEL_SKYLIB_SHA256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc70 GOGOPROTO_RELEASE = "1.2.1" GOGOPROTO_SHA256 = "99e423905ba8921e86817607a5294ffeedb66fdd4a85efce5eb2848f715fdb3a" -OPENCENSUS_PROTO_GIT_SHA = "d5d80953a8c2ff4633087af6933cd152678434bb" # Mar 18, 2019 -OPENCENSUS_PROTO_SHA256 = "a4e87a1da21d1b3a16674332c3ee6e2689d52f3532e2ff8cb4a626c8bcdabcfc" +OPENCENSUS_PROTO_RELEASE = "0.2.1" +OPENCENSUS_PROTO_SHA256 = "bfcefa6093fc2ecdf5c9effea86e6982d0d6f9a5178b17fcf73a62e0f3fb43d0" -PGV_GIT_SHA = "dd90514d96e35023ed20201f349542b0bc64461d" -PGV_SHA256 = "f7d67677fbec5b0f561f6ee6788223fb535d3864b53a28b6678e319f5d1f4f25" +PGV_GIT_SHA = "2feaabb13a5d697b80fcb938c0ce37b24c9381ee" # Jul 26, 2018 +PGV_SHA256 = "ddefe3dcbb25d68a2e5dfea67d19c060959c2aecc782802bd4c1a5811d44dd45" GOOGLEAPIS_GIT_SHA = "be480e391cc88a75cf2a81960ef79c80d5012068" # Jul 24, 2019 GOOGLEAPIS_SHA = "c1969e5b72eab6d9b6cfcff748e45ba57294aeea1d96fd04cd081995de0605c2" @@ -46,8 +46,8 @@ REPOSITORY_LOCATIONS = dict( ), opencensus_proto = dict( sha256 = OPENCENSUS_PROTO_SHA256, - strip_prefix = "opencensus-proto-" + OPENCENSUS_PROTO_GIT_SHA + "/src", - urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/" + OPENCENSUS_PROTO_GIT_SHA + ".tar.gz"], + strip_prefix = "opencensus-proto-" + OPENCENSUS_PROTO_RELEASE + "/src", + urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/v" + OPENCENSUS_PROTO_RELEASE + ".tar.gz"], ), kafka_source = dict( sha256 = KAFKA_SOURCE_SHA, diff --git a/bazel/external/backward.BUILD b/bazel/external/backward.BUILD deleted file mode 100644 index e5b70b649541..000000000000 --- a/bazel/external/backward.BUILD +++ /dev/null @@ -1,8 +0,0 @@ -licenses(["notice"]) # Apache 2 - -cc_library( - name = "backward", - hdrs = ["backward.hpp"], - includes = ["."], - visibility = ["//visibility:public"], -) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 1335ae70623b..45af5f892c10 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1,7 +1,7 @@ REPOSITORY_LOCATIONS = dict( bazel_gazelle = dict( - sha256 = "3c681998538231a2d24d0c07ed5a7658cb72bfb5fd4bf9911157c0e9ac6a2687", - urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.17.0/bazel-gazelle-0.17.0.tar.gz"], + sha256 = "be9296bfd64882e3c08e3283c58fcb461fa6dd3c171764fcc4cf322f60615a9b", + urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.18.1/bazel-gazelle-0.18.1.tar.gz"], ), bazel_toolchains = dict( sha256 = "68e7678473090542e679ce7e6aa8a3ba5669577dede2b404f9865d556bd99f10", @@ -105,9 +105,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/nanopb/nanopb/archive/0.3.9.3.tar.gz"], ), com_github_nghttp2_nghttp2 = dict( - sha256 = "fe9a75ec44e3a2e8f7f0cb83ad91e663bbc4c5085baf37b57ee2610846d7cf5d", - strip_prefix = "nghttp2-1.38.0", - urls = ["https://github.com/nghttp2/nghttp2/releases/download/v1.38.0/nghttp2-1.38.0.tar.gz"], + sha256 = "25b623cd04dc6a863ca3b34ed6247844effe1aa5458229590b3f56a6d53cd692", + strip_prefix = "nghttp2-1.39.1", + urls = ["https://github.com/nghttp2/nghttp2/releases/download/v1.39.1/nghttp2-1.39.1.tar.gz"], ), io_opentracing_cpp = dict( sha256 = "015c4187f7a6426a2b5196f0ccd982aa87f010cf61f507ae3ce5c90523f92301", @@ -142,9 +142,6 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/libevent/libevent/archive/0d7d85c2083f7a4c9efe01c061486f332b576d28.tar.gz"], ), net_zlib = dict( - # TODO(moderation): revert to com_github_madler_zlib name pending resolution of workaround - # in rules_go https://github.com/bazelbuild/rules_go/blob/master/go/private/repositories.bzl#L87-L101 - # for issue in protocolbuffers/protobuf https://github.com/protocolbuffers/protobuf/issues/5472 sha256 = "629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff", strip_prefix = "zlib-1.2.11", urls = ["https://github.com/madler/zlib/archive/v1.2.11.tar.gz"], @@ -155,9 +152,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/jbeder/yaml-cpp/archive/0f9a586ca1dc29c2ecb8dd715a315b93e3f40f79.tar.gz"], ), com_github_msgpack_msgpack_c = dict( - sha256 = "bda49f996a73d2c6080ff0523e7b535917cd28c8a79c3a5da54fc29332d61d1e", - strip_prefix = "msgpack-c-cpp-3.1.1", - urls = ["https://github.com/msgpack/msgpack-c/archive/cpp-3.1.1.tar.gz"], + sha256 = "fbaa28c363a316fd7523f31d1745cf03eab0d1e1ea5a1c60aa0dffd4ce551afe", + strip_prefix = "msgpack-3.2.0", + urls = ["https://github.com/msgpack/msgpack-c/releases/download/cpp-3.2.0/msgpack-3.2.0.tar.gz"], ), com_github_google_jwt_verify = dict( sha256 = "8ab9a0b3f8b7eab5f1cd059920e81fdc138cd4ee657c1412af891652929885c5", @@ -203,6 +200,7 @@ REPOSITORY_LOCATIONS = dict( com_google_googletest = dict( sha256 = "cbd251a40485fddd44cdf641af6df2953d45695853af6d68aeb11c7efcde6771", strip_prefix = "googletest-d7003576dd133856432e2e07340f45926242cc3a", + # 2019-07-16 # TODO(akonradi): Switch this back to a released version later than 1.8.1 once there is # one available. urls = ["https://github.com/google/googletest/archive/d7003576dd133856432e2e07340f45926242cc3a.tar.gz"], @@ -235,6 +233,7 @@ REPOSITORY_LOCATIONS = dict( io_opencensus_cpp = dict( sha256 = "9223b4d54af4151910dede03aa58247e90df72167fcc91d5f75e73a141568036", strip_prefix = "opencensus-cpp-e292a374fb42c6cb2743f1689234bd409f4fda20", + # 2019-07-15 urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/e292a374fb42c6cb2743f1689234bd409f4fda20.tar.gz"], ), com_github_curl = dict( From 68d14ec2418cf6e099899806b07a666a3bbb5de8 Mon Sep 17 00:00:00 2001 From: Xin Date: Thu, 1 Aug 2019 16:02:46 -0400 Subject: [PATCH 292/542] [xds] refactor config provider framework (#7704) Refactor config provider to: * Move creation of Config implementation from x-ConfigProviders to x-Subscriptions, so that we can share one Config implementation across workers in a DeltaConfigSubscriptionInstance and its associated providers, and share one Config impl across all works and all providers in a ConfigSubscriptionInstance and its associated providers. * Remove DeltaMutableConfigProviderBase, and MutableConfigProviderBase as now their functionality is covered by the DeltaConfigSubscriptionInstance/ConfigSubscriptionInstance impl. Adjusted tests. Risk Level: LOW (no production code) Testing: unit tests Signed-off-by: Xin Zhuang --- source/common/config/config_provider_impl.cc | 61 +--- source/common/config/config_provider_impl.h | 302 +++++++----------- source/common/router/BUILD | 1 + source/common/router/scoped_config_impl.cc | 7 +- source/common/router/scoped_config_impl.h | 4 +- source/common/router/scoped_rds.cc | 69 ++-- source/common/router/scoped_rds.h | 24 +- .../config/config_provider_impl_test.cc | 212 ++++++------ test/mocks/config/mocks.cc | 7 - test/mocks/config/mocks.h | 14 - 10 files changed, 258 insertions(+), 443 deletions(-) diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index ed209e8427ad..11cbf993e51c 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -23,24 +23,6 @@ ConfigSubscriptionCommonBase::~ConfigSubscriptionCommonBase() { init_target_.ready(); config_provider_manager_.unbindSubscription(manager_identifier_); } - -void ConfigSubscriptionCommonBase::bindConfigProvider(MutableConfigProviderCommonBase* provider) { - // All config providers bound to a ConfigSubscriptionCommonBase must be of the same concrete - // type; this is assumed by ConfigSubscriptionInstance::checkAndApplyConfigUpdate() and is - // verified by the assertion below. NOTE: an inlined statement ASSERT() triggers a potentially - // evaluated expression warning from clang due to `typeid(**mutable_config_providers_.begin())`. - // To avoid this, we use a lambda to separate the first mutable provider dereference from the - // typeid() statement. - ASSERT([&]() { - if (!mutable_config_providers_.empty()) { - const auto& first_provider = **mutable_config_providers_.begin(); - return typeid(*provider) == typeid(first_provider); - } - return true; - }()); - mutable_config_providers_.insert(provider); -} - bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info) { @@ -55,49 +37,12 @@ bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Messa config_info_ = {new_hash, version_info}; ENVOY_LOG(debug, "{}: loading new configuration: config_name={} hash={}", name_, config_name, new_hash); - - ASSERT(!mutable_config_providers_.empty()); - ConfigProvider::ConfigConstSharedPtr new_config; - for (auto* provider : mutable_config_providers_) { - // All bound mutable config providers must be of the same type (see the ASSERT... in - // bindConfigProvider()). - // This makes it safe to call any of the provider's onConfigProtoUpdate() to get a new config - // impl, which can then be passed to all providers. - auto* typed_provider = static_cast(provider); - if (new_config == nullptr) { - if ((new_config = typed_provider->onConfigProtoUpdate(config_proto)) == nullptr) { - return false; - } - } - typed_provider->onConfigUpdate(new_config); - } - + ConfigProvider::ConfigConstSharedPtr new_config_impl = onConfigProtoUpdate(config_proto); + applyConfigUpdate([new_config_impl](ConfigProvider::ConfigConstSharedPtr) + -> ConfigProvider::ConfigConstSharedPtr { return new_config_impl; }); return true; } -void DeltaConfigSubscriptionInstance::applyDeltaConfigUpdate( - const std::function& update_fn) { - // The Config implementation is assumed to be shared across the config providers bound to this - // subscription, therefore, simply propagating the update to all worker threads for a single bound - // provider will be sufficient. - if (mutable_config_providers_.size() > 1) { - ASSERT(static_cast(*mutable_config_providers_.begin()) - ->getConfig() == static_cast( - *std::next(mutable_config_providers_.begin())) - ->getConfig()); - } - - // TODO(AndresGuedez): currently, the caller has to compute the differences in resources between - // DS API config updates and passes a granular update_fn() that adds/modifies/removes resources as - // needed. Such logic could be generalized as part of this framework such that this function owns - // the diffing and issues the corresponding call to add/modify/remove a resource according to a - // vector of functions passed by the caller. - auto* typed_provider = - static_cast(getAnyBoundMutableConfigProvider()); - ConfigSharedPtr config = typed_provider->getConfig(); - typed_provider->onConfigUpdate([config, update_fn]() { update_fn(config); }); -} - ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin, const std::string& config_name) { config_tracker_entry_ = diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 2553b3976e59..a1a7b02d71b7 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -20,11 +20,11 @@ namespace Envoy { namespace Config { // This file provides a set of base classes, (ImmutableConfigProviderBase, -// MutableConfigProviderCommonBase, MutableConfigProviderBase, DeltaMutableConfigProviderBase, -// ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, ConfigSubscriptionInstance, -// DeltaConfigSubscriptionInstance), conforming to the ConfigProvider/ConfigProviderManager -// interfaces, which in tandem provide a framework for implementing statically defined (i.e., -// immutable) and dynamic (mutable via subscriptions) configuration for Envoy. +// MutableConfigProviderCommonBase, ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, +// ConfigSubscriptionInstance, DeltaConfigSubscriptionInstance), conforming to the +// ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a framework for +// implementing statically defined (i.e., immutable) and dynamic (mutable via subscriptions) +// configuration for Envoy. // // The mutability property applies to the ConfigProvider itself and _not_ the underlying config // proto, which is always immutable. MutableConfigProviderCommonBase objects receive config proto @@ -58,11 +58,12 @@ namespace Config { // interface. // // For mutable (xDS) providers: -// 1) According to the API type, create a class derived from MutableConfigProviderBase or -// DeltaMutableConfigProviderBase and implement the required interface. -// 2) According to the API type, create a class derived from ConfigSubscriptionInstance or -// DeltaConfigSubscriptionInstance; this is the entity responsible for owning and managing the -// Envoy::Config::Subscription that provides the underlying config subscription. +// 1) According to the API type, create a class derived from MutableConfigProviderCommonBase and +// implement the required interface. +// 2) According to the API type, create a class derived from +// ConfigSubscriptionInstance or DeltaConfigSubscriptionInstance; this is the entity responsible +// for owning and managing the Envoy::Config::Subscription that provides the +// underlying config subscription, and the Config implemention shared by associated providers. // a) For a ConfigProvider::ApiType::Full subscription instance (i.e., a // ConfigSubscriptionInstance child): // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the @@ -78,8 +79,8 @@ namespace Config { // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the // underlying subscription, the corresponding ConfigSubscriptionInstance functions must be called // as well. -// - On a successful config update, applyConfigUpdate() should be called to propagate the config -// updates to all bound config providers and worker threads. +// - On a successful config update, applyConfigUpdate() should be called to propagate the +// config updates to all bound config providers and worker threads. class ConfigProviderManagerImplBase; @@ -130,12 +131,13 @@ class ImmutableConfigProviderBase : public ConfigProvider { class MutableConfigProviderCommonBase; /** - * Provides common DS API subscription functionality required by the ConfigProvider::ApiType - * specific base classes (see ConfigSubscriptionInstance and DeltaConfigSubscriptionInstance). + * Provides common DS API subscription functionality required by the ConfigProvider::ApiType. * - * To do so, this class keeps track of a set of MutableConfigProviderCommonBase instances associated - * with an underlying subscription; providers are bound/unbound as needed as they are created and - * destroyed. + * This class can not be instantiated directly; instead, it provides the foundation for + * config subscription implementations which derive from it. + * + * A subscription is intended to be co-owned by config providers with the same config source, it's + * designed to be created/destructed on admin thread only. * * xDS config providers and subscriptions are split to avoid lifetime issues with arguments * required by the config providers. An example is the Server::Configuration::FactoryContext, which @@ -143,11 +145,17 @@ class MutableConfigProviderCommonBase; * in use (see #3960). This split enables single ownership of the config providers, while enabling * shared ownership of the underlying subscription. * - * This class can not be instantiated directly; instead, it provides the foundation for - * config subscription implementations which derive from it. */ -class ConfigSubscriptionCommonBase : protected Logger::Loggable { +class ConfigSubscriptionCommonBase + : protected Logger::Loggable, + public std::enable_shared_from_this { public: + // Callback for updating a Config implementation held in each worker thread, the callback is + // called in applyConfigUpdate() with the current version Config, and is expected to return the + // new version Config. + using ConfigUpdateCb = + std::function; + struct LastConfigInfo { absl::optional last_config_hash_; std::string last_config_version_; @@ -166,6 +174,10 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggable& configInfo() const { return config_info_; } + ConfigProvider::ConfigConstSharedPtr getConfig() const { + return tls_->getTyped().config_; + } + /** * Must be called by derived classes when the onConfigUpdate() callback associated with the * underlying subscription is issued. @@ -184,25 +196,49 @@ class ConfigSubscriptionCommonBase : protected Logger::LoggablerunOnAllThreads( + [this, update_fn]() { + tls_->getTyped().config_ = update_fn(this->getConfig()); + }, + // During the update propagation, a subscription may get teared down in main thread due to + // all owners/providers destructed in a xDS update (e.g. LDS demolishes a + // RouteConfigProvider and its subscription). + // If such a race condition happens, holding a reference to the "*this" subscription + // instance in this cb will ensure the shared "*this" gets posted back to main thread, after + // all the workers finish calling the update_fn, at which point it's safe to destruct + // "*this" instance. + [shared_this, complete_cb]() { complete_cb(); }); } void setLastUpdated() { last_updated_ = time_source_.systemTime(); } @@ -212,16 +248,12 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggable mutable_config_providers_; absl::optional config_info_; + // This slot holds a Config implementation in each thread, which is intended to be shared between + // config providers from the same config source. + ThreadLocal::SlotPtr tls_; private: - void bindConfigProvider(MutableConfigProviderCommonBase* provider); - - void unbindConfigProvider(MutableConfigProviderCommonBase* provider) { - mutable_config_providers_.erase(provider); - } - Init::TargetImpl init_target_; const uint64_t manager_identifier_; ConfigProviderManagerImplBase& config_provider_manager_; @@ -235,28 +267,34 @@ class ConfigSubscriptionCommonBase : protected Logger::Loggables and // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit // reference counting would be more maintainable. - friend class MutableConfigProviderCommonBase; - friend class MutableConfigProviderBase; - friend class DeltaMutableConfigProviderBase; friend class ConfigProviderManagerImplBase; - friend class MockMutableConfigProviderBase; }; using ConfigSubscriptionCommonBaseSharedPtr = std::shared_ptr; /** * Provides common subscription functionality required by ConfigProvider::ApiType::Full DS APIs. + * A single Config instance is shared across all providers and all workers associated with this + * subscription. */ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { -protected: +public: ConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, ConfigProviderManagerImplBase& config_provider_manager, - TimeSource& time_source, const SystemTime& last_updated, - const LocalInfo::LocalInfo& local_info) - : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, - last_updated, local_info) {} + Server::Configuration::FactoryContext& factory_context) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, + factory_context) {} - ~ConfigSubscriptionInstance() override = default; + /** + * Must be called by the derived class' constructor. + * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with + * the underlying subscription, shared across all providers and workers. + */ + void initialize(const ConfigProvider::ConfigConstSharedPtr& initial_config) { + tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(initial_config); + }); + } /** * Determines whether a configuration proto is a new update, and if so, propagates it to all @@ -268,170 +306,66 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { */ bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info); -}; -using ConfigSharedPtr = std::shared_ptr; +protected: + /** + * Called when a new config proto is received via an xDS subscription. + * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config + * implementation that will be propagated to all mutable config providers sharing the + * subscription. + * Note that this function is called _once_ across all shared config providers per xDS + * subscription config update. + * @param config_proto supplies the configuration proto. + * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. + */ + virtual ConfigProvider::ConfigConstSharedPtr + onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; +}; /** * Provides common subscription functionality required by ConfigProvider::ApiType::Delta DS APIs. */ class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { protected: - DeltaConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, - ConfigProviderManagerImplBase& config_provider_manager, - TimeSource& time_source, const SystemTime& last_updated, - const LocalInfo::LocalInfo& local_info) - : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, - last_updated, local_info) {} - - ~DeltaConfigSubscriptionInstance() override = default; + using ConfigSubscriptionCommonBase::ConfigSubscriptionCommonBase; /** - * Propagates a config update to the config providers and worker threads associated with the - * subscription. - * - * @param update_fn the callback to run on each worker thread. + * Must be called by the derived class' constructor. + * @param init_cb supplies an initial Envoy::Config::ConfigProvider::Config associated with the + * underlying subscription for each worker thread. */ - void applyDeltaConfigUpdate(const std::function& update_fn); + void initialize(const std::function& init_cb) { + tls_->set([init_cb](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(init_cb()); + }); + } }; /** * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config - * providers (see MutableConfigProviderBase and DeltaMutableConfigProviderBase). + * providers. * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. */ class MutableConfigProviderCommonBase : public ConfigProvider { public: - ~MutableConfigProviderCommonBase() override { subscription_->unbindConfigProvider(this); } - // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return subscription_->lastUpdated(); } ApiType apiType() const override { return api_type_; } protected: MutableConfigProviderCommonBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, ApiType api_type) - : tls_(factory_context.threadLocal().allocateSlot()), subscription_(subscription), - api_type_(api_type) {} - - ThreadLocal::SlotPtr tls_; - ConfigSubscriptionCommonBaseSharedPtr subscription_; - -private: - ApiType api_type_; -}; + : subscription_(subscription), api_type_(api_type) {} -/** - * Provides common mutable (dynamic) config provider functionality required by - * ConfigProvider::ApiType::Full DS APIs. - */ -class MutableConfigProviderBase : public MutableConfigProviderCommonBase { -public: // Envoy::Config::ConfigProvider - // NOTE: This is being promoted to public for internal uses to avoid an unnecessary dynamic_cast - // in the public API (ConfigProvider::config()). - ConfigConstSharedPtr getConfig() const override { - return tls_->getTyped().config_; - } - - /** - * Called when a new config proto is received via an xDS subscription. - * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config - * implementation that will be propagated to all mutable config providers sharing the - * subscription. - * Note that this function is called _once_ across all shared config providers per xDS - * subscription config update. - * @param config_proto supplies the configuration proto. - * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. - */ - virtual ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; - - /** - * Must be called by the derived class' constructor. - * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with - * the underlying subscription. - */ - void initialize(const ConfigConstSharedPtr& initial_config) { - subscription_->bindConfigProvider(this); - tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared(initial_config); - }); - } - - /** - * Propagates a newly instantiated Envoy::Config::ConfigProvider::Config to all workers. - * @param config supplies the newly instantiated config. - */ - void onConfigUpdate(const ConfigConstSharedPtr& config) { - if (getConfig() == config) { - return; - } - tls_->runOnAllThreads( - [this, config]() -> void { tls_->getTyped().config_ = config; }); - } - -protected: - MutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - ApiType api_type) - : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} + ConfigConstSharedPtr getConfig() const override { return subscription_->getConfig(); } - ~MutableConfigProviderBase() override = default; + ConfigSubscriptionCommonBaseSharedPtr subscription_; private: - struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { - ThreadLocalConfig(ConfigProvider::ConfigConstSharedPtr initial_config) - : config_(std::move(initial_config)) {} - - ConfigProvider::ConfigConstSharedPtr config_; - }; -}; - -/** - * Provides common mutable (dynamic) config provider functionality required by - * ConfigProvider::ApiType::Delta DS APIs. - */ -class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { -public: - // Envoy::Config::ConfigProvider - // This promotes getConfig() to public so that internal uses can avoid an unnecessary dynamic_cast - // in the public API (ConfigProvider::config()). - ConfigConstSharedPtr getConfig() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - - /** - * Non-const overload for use within the framework. - * @return ConfigSharedPtr the config implementation associated with the provider. - */ - virtual ConfigSharedPtr getConfig() PURE; - - /** - * Propagates a delta config update to all workers. - * @param updateCb the callback to run on each worker. - */ - void onConfigUpdate(Envoy::Event::PostCb update_cb) { - tls_->runOnAllThreads(std::move(update_cb)); - } - -protected: - DeltaMutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - ApiType api_type) - : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} - - ~DeltaMutableConfigProviderBase() override = default; - - /** - * Must be called by the derived class' constructor. - * @param initializeCb supplies the initialization callback to be issued for each worker - * thread. - */ - void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { - subscription_->bindConfigProvider(this); - tls_->set(std::move(initializeCb)); - } + ApiType api_type_; }; /** @@ -454,8 +388,6 @@ class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { */ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singleton::Instance { public: - ~ConfigProviderManagerImplBase() override = default; - /** * This is invoked by the /config_dump admin handler. * @return ProtobufTypes::MessagePtr the config dump proto corresponding to the associated diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 6e07d093018a..31d354477d5e 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -175,6 +175,7 @@ envoy_cc_library( hdrs = ["scoped_rds.h"], deps = [ ":scoped_config_lib", + "//include/envoy/config:config_provider_interface", "//include/envoy/config:subscription_interface", "//include/envoy/stats:stats_interface", "//source/common/common:assert_lib", diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index ee3ce1318568..f5c8007f9e74 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -104,12 +104,11 @@ ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { return std::make_unique(std::move(key)); } -void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} +void ScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} -void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} +void ScopedConfigImpl::removeRoutingScope(const std::string&) {} -Router::ConfigConstSharedPtr -ThreadLocalScopedConfigImpl::getRouteConfig(const Http::HeaderMap&) const { +Router::ConfigConstSharedPtr ScopedConfigImpl::getRouteConfig(const Http::HeaderMap&) const { return std::make_shared(); } diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 808b442838c4..184929f94664 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -141,9 +141,9 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { * ConnectionManagerImpl::refreshCachedRoute() will call getRouterConfig() to obtain the * Router::ConfigConstSharedPtr to use for route selection. */ -class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { +class ScopedConfigImpl : public ScopedConfig { public: - ThreadLocalScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) + ScopedConfigImpl(ScopedRoutes::ScopeKeyBuilder&& scope_key_builder) : scope_key_builder_(std::move(scope_key_builder)) {} void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 86e1d1dc88b8..e30ea4296b7c 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -68,7 +68,7 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ConfigProviderInstanceType::Inline, ConfigProvider::ApiType::Delta), name_(std::move(name)), - config_(std::make_shared(std::move(scope_key_builder))), + config_(std::make_shared(std::move(scope_key_builder))), config_protos_(std::make_move_iterator(config_protos.begin()), std::make_move_iterator(config_protos.end())), rds_config_source_(std::move(rds_config_source)) {} @@ -76,12 +76,13 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const uint64_t manager_identifier, const std::string& name, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager) - : DeltaConfigSubscriptionInstance( - "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()), - name_(name), + : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, + factory_context), + name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), validation_visitor_(factory_context.messageValidationVisitor()) { @@ -91,6 +92,12 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( Grpc::Common::typeUrl( envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), *scope_, *this); + + initialize([scope_key_builder]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { + return std::make_shared( + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( + scope_key_builder)); + }); } void ScopedRdsConfigSubscription::onConfigUpdate( @@ -124,10 +131,12 @@ void ScopedRdsConfigSubscription::onConfigUpdate( ScopedRouteInfoConstSharedPtr scoped_route_info = scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); - applyDeltaConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); + applyConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + return config; }); } @@ -135,35 +144,28 @@ void ScopedRdsConfigSubscription::onConfigUpdate( const std::string scoped_route_name = scoped_route.first; ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); scoped_config_manager_.removeRoutingScope(scoped_route_name); - applyDeltaConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); + applyConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) + -> ConfigProvider::ConfigConstSharedPtr { + // In place update. + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); thread_local_scoped_config->removeRoutingScope(scoped_route_name); + return config; }); } - ConfigSubscriptionCommonBase::onConfigUpdate(); + DeltaConfigSubscriptionInstance::onConfigUpdate(); setLastConfigInfo(absl::optional({absl::nullopt, version_info})); stats_.config_reload_.inc(); } ScopedRdsConfigProvider::ScopedRdsConfigProvider( ScopedRdsConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - envoy::api::v2::core::ConfigSource rds_config_source, - const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder) - : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, - ConfigProvider::ApiType::Delta), + envoy::api::v2::core::ConfigSource rds_config_source) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), subscription_(static_cast( MutableConfigProviderCommonBase::subscription_.get())), - rds_config_source_(std::move(rds_config_source)) { - initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return std::make_shared( - envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder( - scope_key_builder)); - }); -} + rds_config_source_(std::move(rds_config_source)) {} ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { auto config_dump = std::make_unique(); @@ -207,28 +209,25 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( const Protobuf::Message& config_source_proto, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, const ConfigProviderManager::OptionalArg& optarg) { + const auto& typed_optarg = static_cast(optarg); ScopedRdsConfigSubscriptionSharedPtr subscription = ConfigProviderManagerImplBase::getSubscription( config_source_proto, factory_context.initManager(), [&config_source_proto, &factory_context, &stat_prefix, - &optarg](const uint64_t manager_identifier, - ConfigProviderManagerImplBase& config_provider_manager) + &typed_optarg](const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) -> Envoy::Config::ConfigSubscriptionCommonBaseSharedPtr { const auto& scoped_rds_config_source = dynamic_cast< const envoy::config::filter::network::http_connection_manager::v2::ScopedRds&>( config_source_proto); return std::make_shared( - scoped_rds_config_source, manager_identifier, - static_cast(optarg) - .scoped_routes_name_, - factory_context, stat_prefix, + scoped_rds_config_source, manager_identifier, typed_optarg.scoped_routes_name_, + typed_optarg.scope_key_builder_, factory_context, stat_prefix, static_cast(config_provider_manager)); }); - const auto& typed_optarg = static_cast(optarg); - return std::make_unique(std::move(subscription), factory_context, - typed_optarg.rds_config_source_, - typed_optarg.scope_key_builder_); + return std::make_unique(std::move(subscription), + typed_optarg.rds_config_source_); } ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index f0cc72f71c15..725c3ec755ea 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -86,6 +86,8 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const uint64_t manager_identifier, const std::string& name, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager); @@ -98,7 +100,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio } private: - // Envoy::Config::ConfigSubscriptionCommonBase + // Envoy::Config::DeltaConfigSubscriptionInstance void start() override { subscription_->start({}); } // Envoy::Config::SubscriptionCallbacks @@ -109,7 +111,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void onConfigUpdateFailed(const EnvoyException*) override { - ConfigSubscriptionCommonBase::onConfigUpdateFailed(); + DeltaConfigSubscriptionInstance::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, @@ -119,6 +121,8 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const std::string name_; std::unique_ptr subscription_; + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder_; Stats::ScopePtr scope_; ScopedRdsStats stats_; ScopedConfigManager scoped_config_manager_; @@ -129,25 +133,13 @@ using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr(tls_->get()); - } - private: ScopedRdsConfigSubscription* subscription_; const envoy::api::v2::core::ConfigSource rds_config_source_; diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 2b3e320a7a62..020d6f5260d8 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -4,7 +4,6 @@ #include "common/protobuf/utility.h" #include "test/common/config/dummy_config.pb.h" -#include "test/mocks/config/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -20,6 +19,22 @@ using testing::InSequence; class DummyConfigProviderManager; +class DummyConfig : public Envoy::Config::ConfigProvider::Config { +public: + DummyConfig() = default; + explicit DummyConfig(const test::common::config::DummyConfig& config_proto) { + protos_.push_back(config_proto); + } + void addProto(const test::common::config::DummyConfig& config_proto) { + protos_.push_back(config_proto); + } + + uint32_t numProtos() const { return protos_.size(); } + +private: + std::vector protos_; +}; + class StaticDummyConfigProvider : public ImmutableConfigProviderBase { public: StaticDummyConfigProvider(const test::common::config::DummyConfig& config_proto, @@ -48,12 +63,18 @@ class DummyConfigSubscription : public ConfigSubscriptionInstance, DummyConfigSubscription(const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager); - ~DummyConfigSubscription() override = default; // Envoy::Config::ConfigSubscriptionCommonBase void start() override {} + // Envoy::Config::ConfigSubscriptionInstance + ConfigProvider::ConfigConstSharedPtr + onConfigProtoUpdate(const Protobuf::Message& config_proto) override { + return std::make_shared( + static_cast(config_proto)); + } + // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override { @@ -84,33 +105,17 @@ class DummyConfigSubscription : public ConfigSubscriptionInstance, }; using DummyConfigSubscriptionSharedPtr = std::shared_ptr; -class DummyConfig : public ConfigProvider::Config { +class DummyDynamicConfigProvider : public MutableConfigProviderCommonBase { public: - DummyConfig(const test::common::config::DummyConfig&) {} -}; - -class DummyDynamicConfigProvider : public MutableConfigProviderBase { -public: - DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, - const ConfigConstSharedPtr& initial_config, - Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full), + explicit DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ApiType::Full), subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())) { - initialize(initial_config); - } + MutableConfigProviderCommonBase::subscription_.get())) {} ~DummyDynamicConfigProvider() override = default; DummyConfigSubscription& subscription() { return *subscription_; } - // Envoy::Config::MutableConfigProviderBase - ConfigProvider::ConfigConstSharedPtr - onConfigProtoUpdate(const Protobuf::Message& config) override { - return std::make_shared( - static_cast(config)); - } - // Envoy::Config::ConfigProvider const Protobuf::Message* getConfigProto() const override { if (!subscription_->config_proto().has_value()) { @@ -127,7 +132,7 @@ class DummyDynamicConfigProvider : public MutableConfigProviderBase { class DummyConfigProviderManager : public ConfigProviderManagerImplBase { public: - DummyConfigProviderManager(Server::Admin& admin) + explicit DummyConfigProviderManager(Server::Admin& admin) : ConfigProviderManagerImplBase(admin, "dummy") {} ~DummyConfigProviderManager() override = default; @@ -177,14 +182,7 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { static_cast(config_provider_manager)); }); - ConfigProvider::ConfigConstSharedPtr initial_config; - const auto* provider = static_cast( - subscription->getAnyBoundMutableConfigProvider()); - if (provider) { - initial_config = provider->getConfig(); - } - return std::make_unique(std::move(subscription), initial_config, - factory_context); + return std::make_unique(std::move(subscription)); } // Envoy::Config::ConfigProviderManager @@ -204,6 +202,15 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { } }; +DummyConfigSubscription::DummyConfigSubscription( + const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, + DummyConfigProviderManager& config_provider_manager) + : ConfigSubscriptionInstance("DummyDS", manager_identifier, config_provider_manager, + factory_context) { + // A nullptr is shared as the initial value. + initialize(nullptr); +} + StaticDummyConfigProvider::StaticDummyConfigProvider( const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, @@ -212,13 +219,6 @@ StaticDummyConfigProvider::StaticDummyConfigProvider( ConfigProviderInstanceType::Static, ApiType::Full), config_(std::make_shared(config_proto)), config_proto_(config_proto) {} -DummyConfigSubscription::DummyConfigSubscription( - const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, - DummyConfigProviderManager& config_provider_manager) - : ConfigSubscriptionInstance( - "DummyDS", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()) {} - class ConfigProviderImplTest : public testing::Test { public: void initialize() { @@ -318,7 +318,7 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { .size()); } -// A ConfigProviderManager that returns a mock ConfigProvider. +// A ConfigProviderManager that returns a dummy ConfigProvider. class DummyConfigProviderManagerMockConfigProvider : public DummyConfigProviderManager { public: DummyConfigProviderManagerMockConfigProvider(Server::Admin& admin) @@ -338,15 +338,14 @@ class DummyConfigProviderManagerMockConfigProvider : public DummyConfigProviderM manager_identifier, factory_context, static_cast(config_provider_manager)); }); - return std::make_unique(std::move(subscription), nullptr, - factory_context); + return std::make_unique(std::move(subscription)); } }; // Test that duplicate config updates will not trigger creation of a new ConfigProvider::Config. TEST_F(ConfigProviderImplTest, DuplicateConfigProto) { InSequence sequence; - // This provider manager returns a MockMutableConfigProviderBase. + // This provider manager returns a DummyDynamicConfigProvider. auto provider_manager = std::make_unique(factory_context_.admin_); envoy::api::v2::core::ApiConfigSource config_source_proto; @@ -354,18 +353,21 @@ TEST_F(ConfigProviderImplTest, DuplicateConfigProto) { ConfigProviderPtr provider = provider_manager->createXdsConfigProvider( config_source_proto, factory_context_, "dummy_prefix", ConfigProviderManager::NullOptionalArg()); - auto* typed_provider = static_cast(provider.get()); - DummyConfigSubscription& subscription = - static_cast(typed_provider->subscription()); + auto* typed_provider = static_cast(provider.get()); + auto& subscription = static_cast(typed_provider->subscription()); + EXPECT_EQ(subscription.getConfig(), nullptr); // First time issuing a configUpdate(). A new ConfigProvider::Config should be created. - EXPECT_CALL(*typed_provider, onConfigProtoUpdate(_)).Times(1); Protobuf::RepeatedPtrField untyped_dummy_configs; untyped_dummy_configs.Add()->PackFrom(parseDummyConfigFromYaml("a: a dynamic dummy config")); subscription.onConfigUpdate(untyped_dummy_configs, "1"); + EXPECT_NE(subscription.getConfig(), nullptr); + auto config_ptr = subscription.getConfig(); + EXPECT_EQ(typed_provider->config().get(), config_ptr.get()); // Second time issuing the configUpdate(), this time with a duplicate proto. A new // ConfigProvider::Config _should not_ be created. - EXPECT_CALL(*typed_provider, onConfigProtoUpdate(_)).Times(0); - subscription.onConfigUpdate(untyped_dummy_configs, "1"); + subscription.onConfigUpdate(untyped_dummy_configs, "2"); + EXPECT_EQ(config_ptr, subscription.getConfig()); + EXPECT_EQ(typed_provider->config().get(), config_ptr.get()); } // An empty config provider tests on base class' constructor. @@ -518,7 +520,31 @@ class DeltaDummyConfigSubscription : public DeltaConfigSubscriptionInstance, // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) override; + const std::string& version_info) override { + if (resources.empty()) { + return; + } + + // For simplicity, there is no logic here to track updates and/or removals to the existing + // config proto set (i.e., this is append only). Real xDS APIs will need to track additions, + // updates and removals to the config set and apply the diffs to the underlying config + // implementations. + for (const auto& resource_any : resources) { + auto dummy_config = TestUtility::anyConvert(resource_any); + proto_map_[version_info] = dummy_config; + // Propagate the new config proto to all worker threads. + applyConfigUpdate([&dummy_config](ConfigProvider::ConfigConstSharedPtr prev_config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* config = const_cast(static_cast(prev_config.get())); + // Per above, append only for now. + config->addProto(dummy_config); + return prev_config; + }); + } + + ConfigSubscriptionCommonBase::onConfigUpdate(); + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + } void onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; @@ -537,32 +563,12 @@ class DeltaDummyConfigSubscription : public DeltaConfigSubscriptionInstance, }; using DeltaDummyConfigSubscriptionSharedPtr = std::shared_ptr; -class ThreadLocalDummyConfig : public ThreadLocal::ThreadLocalObject, - public Envoy::Config::ConfigProvider::Config { +class DeltaDummyDynamicConfigProvider : public Envoy::Config::MutableConfigProviderCommonBase { public: - void addProto(const test::common::config::DummyConfig& config_proto) { - protos_.push_back(config_proto); - } - - uint32_t numProtos() const { return protos_.size(); } - -private: - std::vector protos_; -}; - -class DeltaDummyDynamicConfigProvider : public Envoy::Config::DeltaMutableConfigProviderBase { -public: - DeltaDummyDynamicConfigProvider(DeltaDummyConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - std::shared_ptr dummy_config) - : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, - ConfigProvider::ApiType::Delta), + DeltaDummyDynamicConfigProvider(DeltaDummyConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())) { - initialize([&dummy_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return (dummy_config != nullptr) ? dummy_config : std::make_shared(); - }); - } + MutableConfigProviderCommonBase::subscription_.get())) {} DeltaDummyConfigSubscription& subscription() { return *subscription_; } @@ -574,23 +580,12 @@ class DeltaDummyDynamicConfigProvider : public Envoy::Config::DeltaMutableConfig } return proto_vector; } + std::string getConfigVersion() const override { return (subscription_->configInfo().has_value()) ? subscription_->configInfo().value().last_config_version_ : ""; } - ConfigConstSharedPtr getConfig() const override { - return std::dynamic_pointer_cast(tls_->get()); - } - - // Envoy::Config::DeltaMutableConfigProviderBase - ConfigSharedPtr getConfig() override { - return std::dynamic_pointer_cast(tls_->get()); - } - - std::shared_ptr getThreadLocalDummyConfig() { - return std::dynamic_pointer_cast(tls_->get()); - } private: DeltaDummyConfigSubscription* subscription_; @@ -642,44 +637,17 @@ class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { static_cast(config_provider_manager)); }); - auto* existing_provider = static_cast( - subscription->getAnyBoundMutableConfigProvider()); - return std::make_unique( - std::move(subscription), factory_context, - (existing_provider != nullptr) ? existing_provider->getThreadLocalDummyConfig() : nullptr); + return std::make_unique(std::move(subscription)); } }; DeltaDummyConfigSubscription::DeltaDummyConfigSubscription( const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DeltaDummyConfigProviderManager& config_provider_manager) - : DeltaConfigSubscriptionInstance( - "Dummy", manager_identifier, config_provider_manager, factory_context.timeSource(), - factory_context.timeSource().systemTime(), factory_context.localInfo()) {} - -void DeltaDummyConfigSubscription::onConfigUpdate( - const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) { - if (resources.empty()) { - return; - } - - // For simplicity, there is no logic here to track updates and/or removals to the existing config - // proto set (i.e., this is append only). Real xDS APIs will need to track additions, updates and - // removals to the config set and apply the diffs to the underlying config implementations. - for (const auto& resource_any : resources) { - auto dummy_config = TestUtility::anyConvert(resource_any); - proto_map_[version_info] = dummy_config; - // Propagate the new config proto to all worker threads. - applyDeltaConfigUpdate([&dummy_config](const ConfigSharedPtr& config) { - auto* thread_local_dummy_config = static_cast(config.get()); - // Per above, append only for now. - thread_local_dummy_config->addProto(dummy_config); - }); - } - - ConfigSubscriptionCommonBase::onConfigUpdate(); - setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + : DeltaConfigSubscriptionInstance("Dummy", manager_identifier, config_provider_manager, + factory_context) { + initialize( + []() -> ConfigProvider::ConfigConstSharedPtr { return std::make_shared(); }); } class DeltaConfigProviderImplTest : public testing::Test { @@ -721,7 +689,7 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { config_source_proto, factory_context_, "dummy_prefix", ConfigProviderManager::NullOptionalArg()); - // Providers, config implementations (i.e., the ThreadLocalDummyConfig) and config protos are + // Providers, config implementations (i.e., the DummyConfig) and config protos are // expected to be shared for a given subscription. EXPECT_EQ(&dynamic_cast(*provider1).subscription(), &dynamic_cast(*provider2).subscription()); @@ -729,17 +697,17 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { EXPECT_EQ( provider1->configProtoInfoVector().value().config_protos_, provider2->configProtoInfoVector().value().config_protos_); - EXPECT_EQ(provider1->config().get(), - provider2->config().get()); + EXPECT_EQ(provider1->config().get(), + provider2->config().get()); // Validate that the config protos are propagated to the thread local config implementation. - EXPECT_EQ(provider1->config()->numProtos(), 2); + EXPECT_EQ(provider1->config()->numProtos(), 2); // Issue a second config update to validate that having multiple providers bound to the // subscription causes a single update to the underlying shared config implementation. subscription.onConfigUpdate(untyped_dummy_configs, "2"); // NOTE: the config implementation is append only and _does not_ track updates/removals to the // config proto set, so the expectation is to double the size of the set. - EXPECT_EQ(provider1->config()->numProtos(), 4); + EXPECT_EQ(provider1->config()->numProtos(), 4); EXPECT_EQ(provider1->configProtoInfoVector().value().version_, "2"); } diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index c5bee0ccb920..4a3e3099e0c1 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -47,12 +47,5 @@ MockGrpcMuxCallbacks::MockGrpcMuxCallbacks() { MockGrpcMuxCallbacks::~MockGrpcMuxCallbacks() = default; -MockMutableConfigProviderBase::MockMutableConfigProviderBase( - std::shared_ptr&& subscription, - ConfigProvider::ConfigConstSharedPtr, Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full) { - subscription_->bindConfigProvider(this); -} - } // namespace Config } // namespace Envoy diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 8ced0b8a96a5..ee7c9291782b 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -109,20 +109,6 @@ class MockGrpcStreamCallbacks : public GrpcStreamCallbacks&& subscription, - ConfigProvider::ConfigConstSharedPtr initial_config, - Server::Configuration::FactoryContext& factory_context); - - MOCK_CONST_METHOD0(getConfig, ConfigConstSharedPtr()); - MOCK_METHOD1(onConfigProtoUpdate, ConfigConstSharedPtr(const Protobuf::Message& config_proto)); - MOCK_METHOD1(initialize, void(const ConfigConstSharedPtr& initial_config)); - MOCK_METHOD1(onConfigUpdate, void(const ConfigConstSharedPtr& config)); - - ConfigSubscriptionCommonBase& subscription() { return *subscription_.get(); } -}; - class MockConfigProviderManager : public ConfigProviderManager { public: MockConfigProviderManager() = default; From 80df09408b659c2d00b04b33621cd1049924d907 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 1 Aug 2019 14:45:53 -0700 Subject: [PATCH 293/542] docs: RBE, Docker sandbox and bazel fixes (#7657) * docs: RBE, Docker sandbox and bazel fixes Signed-off-by: Lizan Zhou --- .bazelrc | 8 +++++-- bazel/PPROF.md | 8 +++---- bazel/README.md | 61 ++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/.bazelrc b/.bazelrc index 3a507d5ca130..bfcdfffde0e6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -25,7 +25,6 @@ build:asan --copt -fsanitize=address,undefined build:asan --linkopt -fsanitize=address,undefined build:asan --copt -fno-sanitize=vptr build:asan --linkopt -fno-sanitize=vptr -build:asan --linkopt -fuse-ld=lld build:asan --linkopt -ldl build:asan --define tcmalloc=disabled build:asan --build_tag_filters=-no_asan @@ -39,12 +38,17 @@ build:asan --test_env=ASAN_SYMBOLIZER_PATH # Clang ASAN/UBSAN build:clang-asan --config=asan +build:clang-asan --linkopt -fuse-ld=lld # macOS ASAN/UBSAN build:macos-asan --config=asan # Workaround, see https://github.com/bazelbuild/bazel/issues/6932 build:macos-asan --copt -Wno-macro-redefined build:macos-asan --copt -D_FORTIFY_SOURCE=0 +# Workaround, see https://github.com/bazelbuild/bazel/issues/4341 +build:macos-asan --copt -DGRPC_BAZEL_BUILD +# Dynamic link cause issues like: `dyld: malformed mach-o: load commands size (59272) > 32768` +build:macos-asan --dynamic_mode=off # Clang TSAN build:clang-tsan --define ENVOY_CONFIG_TSAN=1 @@ -110,7 +114,7 @@ build:remote-clang --config=remote build:remote-clang --config=rbe-toolchain # Docker sandbox -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build:cfc514546bc0284536893cca5fa43d7128edcd35 +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/bazel/PPROF.md b/bazel/PPROF.md index 443665785ae2..888b7810d616 100644 --- a/bazel/PPROF.md +++ b/bazel/PPROF.md @@ -93,18 +93,18 @@ Build the binary using bazel, and run the binary without any environment variabl This will dump your profiler output to the working directory. ## Memory Profiling in Tests -To support memory leaks detection, tests are built with gperftools dependencies enabled by default. +To support memory leaks detection, tests are built with gperftools dependencies enabled by default. ### Enabling Memory Profiling in Tests Use `HeapProfilerStart()`, `HeapProfilerStop()`, and `HeapProfilerDump()` to start, stop, and persist memory dumps, respectively. Please see [above](#adding-tcmalloc_dep-to-envoy) for more details. ### Bazel Configuration -By default, bazel executes tests in a sandbox, which will be deleted together with memory dumps +By default, bazel executes tests in a sandbox, which will be deleted together with memory dumps after the test run. To preserve memory dumps, bazel can be forced to run tests without -sandboxing, by setting the ```TestRunner``` parameter to ```standalone```: +sandboxing, by setting the ```TestRunner``` parameter to ```local```: ``` -bazel test --strategy=TestRunner=standalone ... +bazel test --strategy=TestRunner=local ... ``` An alternative is to set ```HEAPPROFILE``` environment variable for the test runner: diff --git a/bazel/README.md b/bazel/README.md index 657177dbccfd..497412a94170 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -73,9 +73,9 @@ for how to update or override dependencies. in your shell for buildifier to work. 1. `bazel build //source/exe:envoy-static` from the Envoy source directory. -## Building Bazel with the CI Docker image +## Building Envoy with the CI Docker image -Bazel can also be built with the Docker image used for CI, by installing Docker and executing: +Envoy can also be built with the Docker image used for CI, by installing Docker and executing: ``` ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.dev' @@ -84,6 +84,40 @@ Bazel can also be built with the Docker image used for CI, by installing Docker See also the [documentation](https://github.com/envoyproxy/envoy/tree/master/ci) for developer use of the CI Docker image. +## Building Envoy with Remote Execution + +Envoy can also be built with Bazel [Remote Executioon](https://docs.bazel.build/versions/master/remote-execution.html), +part of the CI is running with the hosted [GCP RBE](https://blog.bazel.build/2018/10/05/remote-build-execution.html) service. + +To build Envoy with a remote build services, run Bazel with your remote build service flags and with `--config=remote-clang`. +For example the following command runs build with the GCP RBE service used in CI: + +``` +bazel build //source/exe:envoy-static --config=remote-clang \ + --remote_cache=grpcs://remotebuildexecution.googleapis.com \ + --remote_executor=grpcs://remotebuildexecution.googleapis.com \ + --remote_instance_name=projects/envoy-ci/instances/default_instance +``` + +Change the value of `--remote_cache`, `--remote_executor` and `--remote_instance_name` for your remote build services. Tests can +be run in remote execution too. + +Note: Currently the test run configuration in `.bazelrc` doesn't download test binaries and test logs, +to override the behavior set [`--experimental_remote_download_outputs`](https://docs.bazel.build/versions/master/command-line-reference.html#flag--experimental_remote_download_outputs) +accordingly. + +## Building Envoy with Docker sandbox + +Building Envoy with Docker sandbox uses the same Docker image used in CI with fixed C++ toolchain configuration. It produces more consistent +output which is depending on your local C++ toolchain. It can also help debugging issues with RBE. To build Envoy with Docker sandbox: + +``` +bazel build //source/exe:envoy-static --config=docker-clang +``` + +Tests can be run in docker sandbox too. Note that the network environment, such as IPv6, may be different in the docker sandbox so you may want +set different options. See below to configure test IP versions. + ## Linking against libc++ on Linux To link Envoy against libc++, use the following commands: @@ -94,10 +128,12 @@ bazel build --config=libc++ //source/exe:envoy-static ``` Note: this assumes that both: clang compiler and libc++ library are installed in the system, and that `clang` and `clang++` are available in `$PATH`. On some systems, you might need to -include them in the search path, e.g. `export PATH=/usr/lib/llvm-7/bin:$PATH`. +include them in the search path, e.g. `export PATH=/usr/lib/llvm-8/bin:$PATH`. You might also need to ensure libc++ is installed correctly on your system, e.g. on Ubuntu this -might look like `sudo apt-get install libc++abi-7-dev libc++-7-dev`. +might look like `sudo apt-get install libc++abi-8-dev libc++-8-dev`. + +Note: this configuration currently doesn't work with Remote Execution or Docker sandbox. ## Using a compiler toolchain in a non-standard location @@ -106,11 +142,14 @@ appropriate, an arbitrary compiler toolchain and standard library location can b slight caveat is that (at the time of writing), Bazel expects the binutils in `$(dirname $CC)` to be unprefixed, e.g. `as` instead of `x86_64-linux-gnu-as`. +Note: this configuration currently doesn't work with Remote Execution or Docker sandbox, you have to generate a +custom toolchains configuration for them. See [bazelbuild/bazel-toolchains](https://github.com/bazelbuild/bazel-toolchains) +for more details. + ## Supported compiler versions -Though Envoy has been run in production compiled with GCC 4.9 extensively, we now require -GCC >= 5 due to known issues with std::string thread safety and C++14 support. Clang >= 4.0 is also -known to work. +We now require Clang >= 5.0 due to known issues with std::string thread safety and C++14 support. GCC >= 7 is also +known to work. Currently the CI is running with Clang 8. ## Clang STL debug symbols @@ -181,10 +220,10 @@ bazel test //test/common/http:async_client_impl_test --cache_test_results=no Bazel will by default run all tests inside a sandbox, which disallows access to the local filesystem. If you need to break out of the sandbox (for example to run under a local script or tool with [`--run_under`](https://docs.bazel.build/versions/master/user-manual.html#flag--run_under)), -you can run the test with `--strategy=TestRunner=standalone`, e.g.: +you can run the test with `--strategy=TestRunner=local`, e.g.: ``` -bazel test //test/common/http:async_client_impl_test --strategy=TestRunner=standalone --run_under=/some/path/foobar.sh +bazel test //test/common/http:async_client_impl_test --strategy=TestRunner=local --run_under=/some/path/foobar.sh ``` # Stack trace symbol resolution @@ -205,12 +244,12 @@ The script runs in one of two modes. To process log input from stdin, pass `-s` argument, followed by the executable file path. You can postprocess a log or pipe the output of an Envoy process. If you do not specify the `-s` argument it runs the arguments as a child process. This enables you to run a test with backtrace post processing. Bazel sandboxing must -be disabled by specifying standalone execution. Example command line with +be disabled by specifying local execution. Example command line with `run_under`: ``` bazel test -c dbg //test/server:backtrace_test ---run_under=`pwd`/tools/stack_decode.py --strategy=TestRunner=standalone +--run_under=`pwd`/tools/stack_decode.py --strategy=TestRunner=local --cache_test_results=no --test_output=all ``` From 598c1694c6aec2249cddc5112a9b0d7f1d1ee67b Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 1 Aug 2019 15:45:29 -0700 Subject: [PATCH 294/542] ci: use small image and use remote_jdk (#7786) Description: Use smaller image, bazel 0.28.1, regenerates config with gcc and libcxx Risk Level: Low Testing: CI Docs Changes: Release Notes: Signed-off-by: Lizan Zhou --- .bazelrc | 6 +- .bazelversion | 2 +- .circleci/config.yml | 2 +- bazel/repository_locations.bzl | 14 +- .../configs/clang/bazel_0.28.0/java/BUILD | 25 - .../{bazel_0.28.0 => bazel_0.28.1}/cc/BUILD | 3 +- .../cc/armeabi_cc_toolchain_config.bzl | 0 .../cc/cc_toolchain_config.bzl | 0 .../cc/cc_wrapper.sh | 0 .../config/BUILD | 4 +- .../clang_libcxx/bazel_0.28.1/cc/BUILD | 149 ++ .../cc/armeabi_cc_toolchain_config.bzl | 0 .../bazel_0.28.1}/cc/cc_toolchain_config.bzl | 0 .../bazel_0.28.1/cc/cc_wrapper.sh} | 20 +- .../clang_libcxx/bazel_0.28.1/config/BUILD | 53 + .../{bazel_0.28.0 => bazel_0.28.1}/cc/BUILD | 5 +- .../cc/armeabi_cc_toolchain_config.bzl | 82 ++ .../bazel_0.28.1/cc/cc_toolchain_config.bzl | 1202 +++++++++++++++++ .../cc/cc_wrapper.sh | 0 .../config/BUILD | 4 +- bazel/toolchains/configs/versions.bzl | 15 +- bazel/toolchains/rbe_toolchains_config.bzl | 51 +- bazel/toolchains/regenerate.sh | 7 +- 23 files changed, 1551 insertions(+), 93 deletions(-) delete mode 100644 bazel/toolchains/configs/clang/bazel_0.28.0/java/BUILD rename bazel/toolchains/configs/clang/{bazel_0.28.0 => bazel_0.28.1}/cc/BUILD (99%) rename bazel/toolchains/configs/clang/{bazel_0.28.0 => bazel_0.28.1}/cc/armeabi_cc_toolchain_config.bzl (100%) rename bazel/toolchains/configs/clang/{bazel_0.28.0 => bazel_0.28.1}/cc/cc_toolchain_config.bzl (100%) rename bazel/toolchains/configs/clang/{bazel_0.28.0 => bazel_0.28.1}/cc/cc_wrapper.sh (100%) rename bazel/toolchains/configs/clang/{bazel_0.28.0 => bazel_0.28.1}/config/BUILD (89%) create mode 100755 bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD rename bazel/toolchains/configs/{gcc/bazel_0.28.0 => clang_libcxx/bazel_0.28.1}/cc/armeabi_cc_toolchain_config.bzl (100%) rename bazel/toolchains/configs/{gcc/bazel_0.28.0 => clang_libcxx/bazel_0.28.1}/cc/cc_toolchain_config.bzl (100%) rename bazel/toolchains/configs/{gcc/bazel_0.28.0/java/BUILD => clang_libcxx/bazel_0.28.1/cc/cc_wrapper.sh} (60%) mode change 100644 => 100755 create mode 100644 bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD rename bazel/toolchains/configs/gcc/{bazel_0.28.0 => bazel_0.28.1}/cc/BUILD (98%) create mode 100755 bazel/toolchains/configs/gcc/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl create mode 100755 bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_toolchain_config.bzl rename bazel/toolchains/configs/gcc/{bazel_0.28.0 => bazel_0.28.1}/cc/cc_wrapper.sh (100%) rename bazel/toolchains/configs/gcc/{bazel_0.28.0 => bazel_0.28.1}/config/BUILD (89%) diff --git a/.bazelrc b/.bazelrc index bfcdfffde0e6..b5a66c4f69b7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,6 +16,8 @@ build --experimental_local_memory_estimate build --host_force_python=PY2 build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a build --action_env=BAZEL_LINKOPTS=-lm:-static-libgcc +build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11 +build --javabase=@bazel_tools//tools/jdk:remote_jdk11 # Basic ASAN/UBSAN that works for gcc build:asan --action_env=BAZEL_LINKLIBS= @@ -87,10 +89,6 @@ build:sizeopt -c opt --copt -Os build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH # Remote execution: https://docs.bazel.build/versions/master/remote-execution.html -build:rbe-toolchain --host_javabase=@rbe_ubuntu_clang//java:jdk -build:rbe-toolchain --javabase=@rbe_ubuntu_clang//java:jdk -build:rbe-toolchain --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 -build:rbe-toolchain --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 build:rbe-toolchain --host_platform=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform build:rbe-toolchain --platforms=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 diff --git a/.bazelversion b/.bazelversion index 697f087f376a..48f7a71df4be 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -0.28.0 +0.28.1 diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a4680614d63..f72ab219b01f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ executors: description: "A regular build executor based on ubuntu image" docker: # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here. - - image: envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d + - image: envoyproxy/envoy-build:b02c73f3ebbaf5c7852fa814eb3d61e841588068 resource_class: xlarge working_directory: /source diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 45af5f892c10..4d4dc47cf9db 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -4,12 +4,14 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.18.1/bazel-gazelle-0.18.1.tar.gz"], ), bazel_toolchains = dict( - sha256 = "68e7678473090542e679ce7e6aa8a3ba5669577dede2b404f9865d556bd99f10", - strip_prefix = "bazel-toolchains-0.28.0", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/0.28.0.tar.gz", - "https://github.com/bazelbuild/bazel-toolchains/archive/0.28.0.tar.gz", - ], + sha256 = "0710ec5a88201c4c3038ea458f7e9078cc3ad7ad61736ab287c115438eb91b1d", + strip_prefix = "bazel-toolchains-5a8611ee011d0d68498b16bf42a9c69d139bc708", + # 2019-08-01 + # Need: + # - https://github.com/bazelbuild/bazel-toolchains/pull/644 to select correct toolchain from same image + # - https://github.com/bazelbuild/bazel-toolchains/pull/650 to support no java config + # TODO(lizan): Update to release when new version is released. + urls = ["https://github.com/bazelbuild/bazel-toolchains/archive/5a8611ee011d0d68498b16bf42a9c69d139bc708.tar.gz"], ), boringssl = dict( # Use commits from branch "chromium-stable-with-bazel" diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/java/BUILD b/bazel/toolchains/configs/clang/bazel_0.28.0/java/BUILD deleted file mode 100644 index 7c273a5b0e49..000000000000 --- a/bazel/toolchains/configs/clang/bazel_0.28.0/java/BUILD +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is auto-generated by an rbe_autoconfig repository rule -# and should not be modified directly. -# See @bazel_toolchains//rules:rbe_repo.bzl - -package(default_visibility = ["//visibility:public"]) - -java_runtime( - name = "jdk", - srcs = [], - java_home = "/usr/lib/jvm/java-8-openjdk-amd64", -) diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/BUILD b/bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD similarity index 99% rename from bazel/toolchains/configs/clang/bazel_0.28.0/cc/BUILD rename to bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD index 7de0f2682246..1726bdef2c05 100755 --- a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/BUILD +++ b/bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD @@ -116,7 +116,8 @@ cc_toolchain_config( "-Wl,-z,relro,-z,now", "-B/usr/lib/llvm-8/bin", "-lm", - "-static-libgcc"], + "-static-libgcc", + "-fuse-ld=lld"], link_libs = ["-l:libstdc++.a"], opt_link_flags = ["-Wl,--gc-sections"], unfiltered_compile_flags = ["-no-canonical-prefixes", diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/clang/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl similarity index 100% rename from bazel/toolchains/configs/clang/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl rename to bazel/toolchains/configs/clang/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_toolchain_config.bzl similarity index 100% rename from bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_toolchain_config.bzl rename to bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_toolchain_config.bzl diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_wrapper.sh b/bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_wrapper.sh similarity index 100% rename from bazel/toolchains/configs/clang/bazel_0.28.0/cc/cc_wrapper.sh rename to bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_wrapper.sh diff --git a/bazel/toolchains/configs/clang/bazel_0.28.0/config/BUILD b/bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD similarity index 89% rename from bazel/toolchains/configs/clang/bazel_0.28.0/config/BUILD rename to bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD index ac2d4d5f1c55..0d77a8758817 100644 --- a/bazel/toolchains/configs/clang/bazel_0.28.0/config/BUILD +++ b/bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD @@ -29,7 +29,7 @@ toolchain( "@bazel_tools//platforms:linux", "@bazel_tools//platforms:x86_64", ], - toolchain = "//bazel/toolchains/configs/clang/bazel_0.28.0/cc:cc-compiler-k8", + toolchain = "//bazel/toolchains/configs/clang/bazel_0.28.1/cc:cc-compiler-k8", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", ) @@ -43,7 +43,7 @@ platform( remote_execution_properties = """ properties: { name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" } properties { name: "OSFamily" diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD new file mode 100755 index 000000000000..dae58f58c97a --- /dev/null +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD @@ -0,0 +1,149 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes. + +package(default_visibility = ["//visibility:public"]) + +load(":cc_toolchain_config.bzl", "cc_toolchain_config") +load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") + +licenses(["notice"]) # Apache 2.0 + +cc_library( + name = "malloc", +) + +filegroup( + name = "empty", + srcs = [], +) + +filegroup( + name = "cc_wrapper", + srcs = ["cc_wrapper.sh"], +) + +filegroup( + name = "compiler_deps", + srcs = glob(["extra_tools/**"], allow_empty = True) + [":empty"], +) + +# This is the entry point for --crosstool_top. Toolchains are found +# by lopping off the name of --crosstool_top and searching for +# the "${CPU}" entry in the toolchains attribute. +cc_toolchain_suite( + name = "toolchain", + toolchains = { + "k8|clang": ":cc-compiler-k8", + "k8": ":cc-compiler-k8", + "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a", + "armeabi-v7a": ":cc-compiler-armeabi-v7a", + }, +) + +cc_toolchain( + name = "cc-compiler-k8", + toolchain_identifier = "local", + toolchain_config = ":local", + all_files = ":compiler_deps", + ar_files = ":compiler_deps", + as_files = ":compiler_deps", + compiler_files = ":compiler_deps", + dwp_files = ":empty", + linker_files = ":compiler_deps", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 1, +) + +cc_toolchain_config( + name = "local", + cpu = "k8", + compiler = "clang", + toolchain_identifier = "local", + host_system_name = "local", + target_system_name = "local", + target_libc = "local", + abi_version = "local", + abi_libc_version = "local", + cxx_builtin_include_directories = ["/usr/local/include", + "/usr/lib/llvm-8/lib/clang/8.0.1/include", + "/usr/include/x86_64-linux-gnu", + "/usr/include", + "/usr/lib/llvm-8/include/c++/v1", + "/usr/include/clang/8.0.1/include"], + tool_paths = {"ar": "/usr/bin/ar", + "ld": "/usr/bin/ld", + "cpp": "/usr/bin/cpp", + "gcc": "/usr/lib/llvm-8/bin/clang", + "dwp": "/usr/bin/dwp", + "gcov": "/usr/lib/llvm-8/bin/llvm-profdata", + "nm": "/usr/bin/nm", + "objcopy": "/usr/bin/objcopy", + "objdump": "/usr/bin/objdump", + "strip": "/usr/bin/strip"}, + compile_flags = ["-U_FORTIFY_SOURCE", + "-fstack-protector", + "-Wall", + "-Wthread-safety", + "-Wself-assign", + "-fcolor-diagnostics", + "-fno-omit-frame-pointer"], + opt_compile_flags = ["-g0", + "-O2", + "-D_FORTIFY_SOURCE=1", + "-DNDEBUG", + "-ffunction-sections", + "-fdata-sections"], + dbg_compile_flags = ["-g"], + cxx_flags = ["-stdlib=libc++"], + link_flags = ["-fuse-ld=gold", + "-Wl,-no-as-needed", + "-Wl,-z,relro,-z,now", + "-B/usr/lib/llvm-8/bin", + "-lm", + "-static-libgcc", + "-pthread", + "-fuse-ld=lld"], + link_libs = ["-l:libc++.a", + "-l:libc++abi.a"], + opt_link_flags = ["-Wl,--gc-sections"], + unfiltered_compile_flags = ["-no-canonical-prefixes", + "-Wno-builtin-macro-redefined", + "-D__DATE__=\"redacted\"", + "-D__TIMESTAMP__=\"redacted\"", + "-D__TIME__=\"redacted\""], + coverage_compile_flags = ["-fprofile-instr-generate", "-fcoverage-mapping"], + coverage_link_flags = ["-fprofile-instr-generate"], + supports_start_end_lib = True, +) + +# Android tooling requires a default toolchain for the armeabi-v7a cpu. +cc_toolchain( + name = "cc-compiler-armeabi-v7a", + toolchain_identifier = "stub_armeabi-v7a", + toolchain_config = ":stub_armeabi-v7a", + all_files = ":empty", + ar_files = ":empty", + as_files = ":empty", + compiler_files = ":empty", + dwp_files = ":empty", + linker_files = ":empty", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 1, +) + +armeabi_cc_toolchain_config(name = "stub_armeabi-v7a") diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl similarity index 100% rename from bazel/toolchains/configs/gcc/bazel_0.28.0/cc/armeabi_cc_toolchain_config.bzl rename to bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_toolchain_config.bzl similarity index 100% rename from bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_toolchain_config.bzl rename to bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_toolchain_config.bzl diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/java/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_wrapper.sh old mode 100644 new mode 100755 similarity index 60% rename from bazel/toolchains/configs/gcc/bazel_0.28.0/java/BUILD rename to bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_wrapper.sh index 7c273a5b0e49..b7ff6355883c --- a/bazel/toolchains/configs/gcc/bazel_0.28.0/java/BUILD +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_wrapper.sh @@ -1,4 +1,6 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. +#!/bin/bash +# +# Copyright 2015 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,15 +13,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# +# Ship the environment to the C++ action +# +set -eu -# This file is auto-generated by an rbe_autoconfig repository rule -# and should not be modified directly. -# See @bazel_toolchains//rules:rbe_repo.bzl +# Set-up the environment -package(default_visibility = ["//visibility:public"]) -java_runtime( - name = "jdk", - srcs = [], - java_home = "/usr/lib/jvm/java-8-openjdk-amd64", -) +# Call the C++ compiler +/usr/lib/llvm-8/bin/clang "$@" diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD new file mode 100644 index 000000000000..01337270ff96 --- /dev/null +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD @@ -0,0 +1,53 @@ +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated by an rbe_autoconfig repository rule +# and should not be modified directly. +# See @bazel_toolchains//rules:rbe_repo.bzl + +package(default_visibility = ["//visibility:public"]) + +toolchain( + name = "cc-toolchain", + exec_compatible_with = [ + "@bazel_tools//platforms:x86_64", + "@bazel_tools//platforms:linux", + "@bazel_tools//tools/cpp:clang", + ], + target_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + toolchain = "//bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc:cc-compiler-k8", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +platform( + name = "platform", + constraint_values = [ + "@bazel_tools//platforms:x86_64", + "@bazel_tools//platforms:linux", + "@bazel_tools//tools/cpp:clang", + ], + remote_execution_properties = """ + properties: { + name: "container-image" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" + } + properties { + name: "OSFamily" + value: "Linux" + } + """, +) diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/BUILD b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD similarity index 98% rename from bazel/toolchains/configs/gcc/bazel_0.28.0/cc/BUILD rename to bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD index eb9ab72263f1..0390586c9ccd 100755 --- a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/BUILD +++ b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD @@ -91,7 +91,7 @@ cc_toolchain_config( "cpp": "/usr/bin/cpp", "gcc": "/usr/bin/gcc", "dwp": "/usr/bin/dwp", - "gcov": "/usr/bin/gcov", + "gcov": "None", "nm": "/usr/bin/nm", "objcopy": "/usr/bin/objcopy", "objdump": "/usr/bin/objdump", @@ -116,7 +116,8 @@ cc_toolchain_config( "-B/usr/bin", "-pass-exit-codes", "-lm", - "-static-libgcc"], + "-static-libgcc", + "-fuse-ld=lld"], link_libs = ["-l:libstdc++.a"], opt_link_flags = ["-Wl,--gc-sections"], unfiltered_compile_flags = ["-fno-canonical-system-headers", diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl new file mode 100755 index 000000000000..94e0720bf6c9 --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl @@ -0,0 +1,82 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Starlark cc_toolchain configuration rule""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "feature", + "tool_path", +) + +def _impl(ctx): + toolchain_identifier = "stub_armeabi-v7a" + host_system_name = "armeabi-v7a" + target_system_name = "armeabi-v7a" + target_cpu = "armeabi-v7a" + target_libc = "armeabi-v7a" + compiler = "compiler" + abi_version = "armeabi-v7a" + abi_libc_version = "armeabi-v7a" + cc_target_os = None + builtin_sysroot = None + action_configs = [] + + supports_pic_feature = feature(name = "supports_pic", enabled = True) + supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) + features = [supports_dynamic_linker_feature, supports_pic_feature] + + cxx_builtin_include_directories = [] + artifact_name_patterns = [] + make_variables = [] + + tool_paths = [ + tool_path(name = "ar", path = "/bin/false"), + tool_path(name = "compat-ld", path = "/bin/false"), + tool_path(name = "cpp", path = "/bin/false"), + tool_path(name = "dwp", path = "/bin/false"), + tool_path(name = "gcc", path = "/bin/false"), + tool_path(name = "gcov", path = "/bin/false"), + tool_path(name = "ld", path = "/bin/false"), + tool_path(name = "nm", path = "/bin/false"), + tool_path(name = "objcopy", path = "/bin/false"), + tool_path(name = "objdump", path = "/bin/false"), + tool_path(name = "strip", path = "/bin/false"), + ] + + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + artifact_name_patterns = artifact_name_patterns, + cxx_builtin_include_directories = cxx_builtin_include_directories, + toolchain_identifier = toolchain_identifier, + host_system_name = host_system_name, + target_system_name = target_system_name, + target_cpu = target_cpu, + target_libc = target_libc, + compiler = compiler, + abi_version = abi_version, + abi_libc_version = abi_libc_version, + tool_paths = tool_paths, + make_variables = make_variables, + builtin_sysroot = builtin_sysroot, + cc_target_os = cc_target_os, + ) + +armeabi_cc_toolchain_config = rule( + implementation = _impl, + attrs = {}, + provides = [CcToolchainConfigInfo], +) diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_toolchain_config.bzl new file mode 100755 index 000000000000..f2b12d962963 --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_toolchain_config.bzl @@ -0,0 +1,1202 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A Starlark cc_toolchain configuration rule""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "feature", + "feature_set", + "flag_group", + "flag_set", + "tool_path", + "variable_with_value", + "with_feature_set", +) +load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") + +all_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, + ACTION_NAMES.lto_backend, +] + +all_cpp_compile_actions = [ + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.clif_match, +] + +preprocessor_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, +] + +codegen_compile_actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, +] + +all_link_actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, +] + +def _impl(ctx): + tool_paths = [ + tool_path(name = name, path = path) + for name, path in ctx.attr.tool_paths.items() + ] + action_configs = [] + + supports_pic_feature = feature( + name = "supports_pic", + enabled = True, + ) + supports_start_end_lib_feature = feature( + name = "supports_start_end_lib", + enabled = True, + ) + + default_compile_flags_feature = feature( + name = "default_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.compile_flags, + ), + ] if ctx.attr.compile_flags else []), + ), + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.dbg_compile_flags, + ), + ] if ctx.attr.dbg_compile_flags else []), + with_features = [with_feature_set(features = ["dbg"])], + ), + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.opt_compile_flags, + ), + ] if ctx.attr.opt_compile_flags else []), + with_features = [with_feature_set(features = ["opt"])], + ), + flag_set( + actions = [ + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.cxx_flags, + ), + ] if ctx.attr.cxx_flags else []), + ), + ], + ) + + default_link_flags_feature = feature( + name = "default_link_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = ([ + flag_group( + flags = ctx.attr.link_flags, + ), + ] if ctx.attr.link_flags else []), + ), + flag_set( + actions = all_link_actions, + flag_groups = ([ + flag_group( + flags = ctx.attr.opt_link_flags, + ), + ] if ctx.attr.opt_link_flags else []), + with_features = [with_feature_set(features = ["opt"])], + ), + ], + ) + + dbg_feature = feature(name = "dbg") + + opt_feature = feature(name = "opt") + + sysroot_feature = feature( + name = "sysroot", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = ["--sysroot=%{sysroot}"], + expand_if_available = "sysroot", + ), + ], + ), + ], + ) + + fdo_optimize_feature = feature( + name = "fdo_optimize", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-use=%{fdo_profile_path}", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) + + user_compile_flags_feature = feature( + name = "user_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["%{user_compile_flags}"], + iterate_over = "user_compile_flags", + expand_if_available = "user_compile_flags", + ), + ], + ), + ], + ) + + unfiltered_compile_flags_feature = feature( + name = "unfiltered_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = ([ + flag_group( + flags = ctx.attr.unfiltered_compile_flags, + ), + ] if ctx.attr.unfiltered_compile_flags else []), + ), + ], + ) + + library_search_directories_feature = feature( + name = "library_search_directories", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-L%{library_search_directories}"], + iterate_over = "library_search_directories", + expand_if_available = "library_search_directories", + ), + ], + ), + ], + ) + + static_libgcc_feature = feature( + name = "static_libgcc", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ], + flag_groups = [flag_group(flags = ["-static-libgcc"])], + with_features = [ + with_feature_set(features = ["static_link_cpp_runtimes"]), + ], + ), + ], + ) + + pic_feature = feature( + name = "pic", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = [ + flag_group(flags = ["-fPIC"], expand_if_available = "pic"), + ], + ), + ], + ) + + per_object_debug_info_feature = feature( + name = "per_object_debug_info", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ], + flag_groups = [ + flag_group( + flags = ["-gsplit-dwarf"], + expand_if_available = "per_object_debug_info_file", + ), + ], + ), + ], + ) + + preprocessor_defines_feature = feature( + name = "preprocessor_defines", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-D%{preprocessor_defines}"], + iterate_over = "preprocessor_defines", + ), + ], + ), + ], + ) + + cs_fdo_optimize_feature = feature( + name = "cs_fdo_optimize", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.lto_backend], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-use=%{fdo_profile_path}", + "-Xclang-only=-Wno-profile-instr-unprofiled", + "-Xclang-only=-Wno-profile-instr-out-of-date", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["csprofile"], + ) + + autofdo_feature = feature( + name = "autofdo", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-fauto-profile=%{fdo_profile_path}", + "-fprofile-correction", + ], + expand_if_available = "fdo_profile_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + runtime_library_search_directories_feature = feature( + name = "runtime_library_search_directories", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "runtime_library_search_directories", + flag_groups = [ + flag_group( + flags = [ + "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_true = "is_cc_test", + ), + flag_group( + flags = [ + "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", + ], + expand_if_false = "is_cc_test", + ), + ], + expand_if_available = + "runtime_library_search_directories", + ), + ], + with_features = [ + with_feature_set(features = ["static_link_cpp_runtimes"]), + ], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "runtime_library_search_directories", + flag_groups = [ + flag_group( + flags = [ + "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", + ], + ), + ], + expand_if_available = + "runtime_library_search_directories", + ), + ], + with_features = [ + with_feature_set( + not_features = ["static_link_cpp_runtimes"], + ), + ], + ), + ], + ) + + fission_support_feature = feature( + name = "fission_support", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-Wl,--gdb-index"], + expand_if_available = "is_using_fission", + ), + ], + ), + ], + ) + + shared_flag_feature = feature( + name = "shared_flag", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [flag_group(flags = ["-shared"])], + ), + ], + ) + + random_seed_feature = feature( + name = "random_seed", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = [ + flag_group( + flags = ["-frandom-seed=%{output_file}"], + expand_if_available = "output_file", + ), + ], + ), + ], + ) + + includes_feature = feature( + name = "includes", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = ["-include", "%{includes}"], + iterate_over = "includes", + expand_if_available = "includes", + ), + ], + ), + ], + ) + + fdo_instrument_feature = feature( + name = "fdo_instrument", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-generate=%{fdo_instrument_path}", + "-fno-data-sections", + ], + expand_if_available = "fdo_instrument_path", + ), + ], + ), + ], + provides = ["profile"], + ) + + cs_fdo_instrument_feature = feature( + name = "cs_fdo_instrument", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.lto_backend, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = [ + "-fcs-profile-generate=%{cs_fdo_instrument_path}", + ], + expand_if_available = "cs_fdo_instrument_path", + ), + ], + ), + ], + provides = ["csprofile"], + ) + + include_paths_feature = feature( + name = "include_paths", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.clif_match, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = ["-iquote", "%{quote_include_paths}"], + iterate_over = "quote_include_paths", + ), + flag_group( + flags = ["-I%{include_paths}"], + iterate_over = "include_paths", + ), + flag_group( + flags = ["-isystem", "%{system_include_paths}"], + iterate_over = "system_include_paths", + ), + ], + ), + ], + ) + + symbol_counts_feature = feature( + name = "symbol_counts", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = [ + "-Wl,--print-symbol-counts=%{symbol_counts_output}", + ], + expand_if_available = "symbol_counts_output", + ), + ], + ), + ], + ) + + llvm_coverage_map_format_feature = feature( + name = "llvm_coverage_map_format", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ], + flag_groups = [ + flag_group( + flags = [ + "-fprofile-instr-generate", + "-fcoverage-mapping", + ], + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + "objc-executable", + "objc++-executable", + ], + flag_groups = [ + flag_group(flags = ["-fprofile-instr-generate"]), + ], + ), + ], + requires = [feature_set(features = ["coverage"])], + provides = ["profile"], + ) + + strip_debug_symbols_feature = feature( + name = "strip_debug_symbols", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-Wl,-S"], + expand_if_available = "strip_debug_symbols", + ), + ], + ), + ], + ) + + build_interface_libraries_feature = feature( + name = "build_interface_libraries", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = [ + "%{generate_interface_library}", + "%{interface_library_builder_path}", + "%{interface_library_input_path}", + "%{interface_library_output_path}", + ], + expand_if_available = "generate_interface_library", + ), + ], + with_features = [ + with_feature_set( + features = ["supports_interface_shared_libraries"], + ), + ], + ), + ], + ) + + libraries_to_link_feature = feature( + name = "libraries_to_link", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["-Wl,--start-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["-Wl,-whole-archive"], + expand_if_true = + "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "interface_library", + ), + ), + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "static_library", + ), + ), + flag_group( + flags = ["-l%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "dynamic_library", + ), + ), + flag_group( + flags = ["-l:%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "versioned_dynamic_library", + ), + ), + flag_group( + flags = ["-Wl,-no-whole-archive"], + expand_if_true = "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["-Wl,--end-lib"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + flag_group( + flags = ["-Wl,@%{thinlto_param_file}"], + expand_if_true = "thinlto_param_file", + ), + ], + ), + ], + ) + + user_link_flags_feature = feature( + name = "user_link_flags", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{user_link_flags}"], + iterate_over = "user_link_flags", + expand_if_available = "user_link_flags", + ), + ] + ([flag_group(flags = ctx.attr.link_libs)] if ctx.attr.link_libs else []), + ), + ], + ) + + fdo_prefetch_hints_feature = feature( + name = "fdo_prefetch_hints", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.lto_backend, + ], + flag_groups = [ + flag_group( + flags = [ + "-Xclang-only=-mllvm", + "-Xclang-only=-prefetch-hints-file=%{fdo_prefetch_hints_path}", + ], + expand_if_available = "fdo_prefetch_hints_path", + ), + ], + ), + ], + ) + + linkstamps_feature = feature( + name = "linkstamps", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{linkstamp_paths}"], + iterate_over = "linkstamp_paths", + expand_if_available = "linkstamp_paths", + ), + ], + ), + ], + ) + + gcc_coverage_map_format_feature = feature( + name = "gcc_coverage_map_format", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + "objc-executable", + "objc++-executable", + ], + flag_groups = [ + flag_group( + flags = ["-fprofile-arcs", "-ftest-coverage"], + expand_if_available = "gcov_gcno_file", + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [flag_group(flags = ["--coverage"])], + ), + ], + requires = [feature_set(features = ["coverage"])], + provides = ["profile"], + ) + + archiver_flags_feature = feature( + name = "archiver_flags", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group(flags = ["rcsD"]), + flag_group( + flags = ["%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + flag_set( + actions = [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flags = ["%{libraries_to_link.object_files}"], + iterate_over = "libraries_to_link.object_files", + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + ], + ), + ], + ) + + force_pic_flags_feature = feature( + name = "force_pic_flags", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.cpp_link_executable], + flag_groups = [ + flag_group( + flags = ["-pie"], + expand_if_available = "force_pic", + ), + ], + ), + ], + ) + + dependency_file_feature = feature( + name = "dependency_file", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.objc_compile, + ACTION_NAMES.objcpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = ["-MD", "-MF", "%{dependency_file}"], + expand_if_available = "dependency_file", + ), + ], + ), + ], + ) + + dynamic_library_linker_tool_feature = feature( + name = "dynamic_library_linker_tool", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = [" + cppLinkDynamicLibraryToolPath + "], + expand_if_available = "generate_interface_library", + ), + ], + with_features = [ + with_feature_set( + features = ["supports_interface_shared_libraries"], + ), + ], + ), + ], + ) + + output_execpath_flags_feature = feature( + name = "output_execpath_flags", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [ + flag_group( + flags = ["-o", "%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + ], + ) + + # Note that we also set --coverage for c++-link-nodeps-dynamic-library. The + # generated code contains references to gcov symbols, and the dynamic linker + # can't resolve them unless the library is linked against gcov. + coverage_feature = feature( + name = "coverage", + provides = ["profile"], + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = ([ + flag_group(flags = ctx.attr.coverage_compile_flags), + ] if ctx.attr.coverage_compile_flags else []), + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = ([ + flag_group(flags = ctx.attr.coverage_link_flags), + ] if ctx.attr.coverage_link_flags else []), + ), + ], + ) + + is_linux = ctx.attr.target_libc != "macosx" + + # TODO(#8303): Mac crosstool should also declare every feature. + if is_linux: + features = [ + dependency_file_feature, + random_seed_feature, + pic_feature, + per_object_debug_info_feature, + preprocessor_defines_feature, + includes_feature, + include_paths_feature, + fdo_instrument_feature, + cs_fdo_instrument_feature, + cs_fdo_optimize_feature, + fdo_prefetch_hints_feature, + autofdo_feature, + build_interface_libraries_feature, + dynamic_library_linker_tool_feature, + symbol_counts_feature, + shared_flag_feature, + linkstamps_feature, + output_execpath_flags_feature, + runtime_library_search_directories_feature, + library_search_directories_feature, + archiver_flags_feature, + force_pic_flags_feature, + fission_support_feature, + strip_debug_symbols_feature, + coverage_feature, + supports_pic_feature, + ] + ( + [ + supports_start_end_lib_feature, + ] if ctx.attr.supports_start_end_lib else [] + ) + [ + default_compile_flags_feature, + default_link_flags_feature, + libraries_to_link_feature, + user_link_flags_feature, + static_libgcc_feature, + fdo_optimize_feature, + supports_dynamic_linker_feature, + dbg_feature, + opt_feature, + user_compile_flags_feature, + sysroot_feature, + unfiltered_compile_flags_feature, + ] + else: + features = [ + supports_pic_feature, + ] + ( + [ + supports_start_end_lib_feature, + ] if ctx.attr.supports_start_end_lib else [] + ) + [ + coverage_feature, + default_compile_flags_feature, + default_link_flags_feature, + fdo_optimize_feature, + supports_dynamic_linker_feature, + dbg_feature, + opt_feature, + user_compile_flags_feature, + sysroot_feature, + unfiltered_compile_flags_feature, + ] + + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + cxx_builtin_include_directories = ctx.attr.cxx_builtin_include_directories, + toolchain_identifier = ctx.attr.toolchain_identifier, + host_system_name = ctx.attr.host_system_name, + target_system_name = ctx.attr.target_system_name, + target_cpu = ctx.attr.cpu, + target_libc = ctx.attr.target_libc, + compiler = ctx.attr.compiler, + abi_version = ctx.attr.abi_version, + abi_libc_version = ctx.attr.abi_libc_version, + tool_paths = tool_paths, + ) + +cc_toolchain_config = rule( + implementation = _impl, + attrs = { + "cpu": attr.string(mandatory = True), + "compiler": attr.string(mandatory = True), + "toolchain_identifier": attr.string(mandatory = True), + "host_system_name": attr.string(mandatory = True), + "target_system_name": attr.string(mandatory = True), + "target_libc": attr.string(mandatory = True), + "abi_version": attr.string(mandatory = True), + "abi_libc_version": attr.string(mandatory = True), + "cxx_builtin_include_directories": attr.string_list(), + "tool_paths": attr.string_dict(), + "compile_flags": attr.string_list(), + "dbg_compile_flags": attr.string_list(), + "opt_compile_flags": attr.string_list(), + "cxx_flags": attr.string_list(), + "link_flags": attr.string_list(), + "link_libs": attr.string_list(), + "opt_link_flags": attr.string_list(), + "unfiltered_compile_flags": attr.string_list(), + "coverage_compile_flags": attr.string_list(), + "coverage_link_flags": attr.string_list(), + "supports_start_end_lib": attr.bool(), + }, + provides = [CcToolchainConfigInfo], +) diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_wrapper.sh b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_wrapper.sh similarity index 100% rename from bazel/toolchains/configs/gcc/bazel_0.28.0/cc/cc_wrapper.sh rename to bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_wrapper.sh diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.0/config/BUILD b/bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD similarity index 89% rename from bazel/toolchains/configs/gcc/bazel_0.28.0/config/BUILD rename to bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD index 2d431bd2ce03..f417961407af 100644 --- a/bazel/toolchains/configs/gcc/bazel_0.28.0/config/BUILD +++ b/bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD @@ -29,7 +29,7 @@ toolchain( "@bazel_tools//platforms:linux", "@bazel_tools//platforms:x86_64", ], - toolchain = "//bazel/toolchains/configs/gcc/bazel_0.28.0/cc:cc-compiler-k8", + toolchain = "//bazel/toolchains/configs/gcc/bazel_0.28.1/cc:cc-compiler-k8", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", ) @@ -43,7 +43,7 @@ platform( remote_execution_properties = """ properties: { name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" } properties { name: "OSFamily" diff --git a/bazel/toolchains/configs/versions.bzl b/bazel/toolchains/configs/versions.bzl index 6704b14fa542..114a706ab69a 100644 --- a/bazel/toolchains/configs/versions.bzl +++ b/bazel/toolchains/configs/versions.bzl @@ -1,16 +1,17 @@ # Generated file, do not modify by hand # Generated by 'rbe_ubuntu_gcc_gen' rbe_autoconfig rule """Definitions to be used in rbe_repo attr of an rbe_autoconf rule """ -toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", name = "clang") -toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = True, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = "/usr/lib/jvm/java-8-openjdk-amd64", name = "gcc") -_TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1] -_BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.0": ["clang", "gcc"]} -LATEST = "sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f" -_CONTAINER_TO_CONFIG_SPEC_NAMES = {"sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f": ["clang", "gcc"]} +toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "clang") +toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx") +toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc") +_TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1,toolchain_config_spec2] +_BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.1": ["clang", "clang_libcxx", "gcc"]} +LATEST = "sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" +CONTAINER_TO_CONFIG_SPEC_NAMES = {"sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28": ["clang", "clang_libcxx", "gcc"]} _DEFAULT_TOOLCHAIN_CONFIG_SPEC = toolchain_config_spec0 TOOLCHAIN_CONFIG_AUTOGEN_SPEC = struct( bazel_to_config_spec_names_map = _BAZEL_TO_CONFIG_SPEC_NAMES, - container_to_config_spec_names_map = _CONTAINER_TO_CONFIG_SPEC_NAMES, + container_to_config_spec_names_map = CONTAINER_TO_CONFIG_SPEC_NAMES, default_toolchain_config_spec = _DEFAULT_TOOLCHAIN_CONFIG_SPEC, latest_container = LATEST, toolchain_config_specs = _TOOLCHAIN_CONFIG_SPECS, diff --git a/bazel/toolchains/rbe_toolchains_config.bzl b/bazel/toolchains/rbe_toolchains_config.bzl index c1f32be5d78a..c4a34df50929 100644 --- a/bazel/toolchains/rbe_toolchains_config.bzl +++ b/bazel/toolchains/rbe_toolchains_config.bzl @@ -1,16 +1,16 @@ +load("@bazel_skylib//lib:dicts.bzl", "dicts") load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") load("@envoy//bazel/toolchains:configs/versions.bzl", _generated_toolchain_config_suite_autogen_spec = "TOOLCHAIN_CONFIG_AUTOGEN_SPEC") _ENVOY_BUILD_IMAGE_REGISTRY = "gcr.io" _ENVOY_BUILD_IMAGE_REPOSITORY = "envoy-ci/envoy-build" -_ENVOY_BUILD_IMAGE_DIGEST = "sha256:9dbe1cba2b3340d49a25a1d286c8d49083ec986a6fead27f487e80ca334f065f" -_ENVOY_BUILD_IMAGE_JAVA_HOME = "/usr/lib/jvm/java-8-openjdk-amd64" +_ENVOY_BUILD_IMAGE_DIGEST = "sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" _CONFIGS_OUTPUT_BASE = "bazel/toolchains/configs" _CLANG_ENV = { "BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", - "BAZEL_LINKOPTS": "-lm:-static-libgcc", + "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", @@ -18,10 +18,17 @@ _CLANG_ENV = { "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", } +_CLANG_LIBCXX_ENV = dicts.add(_CLANG_ENV, { + "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", + "BAZEL_LINKOPTS": "-lm:-static-libgcc:-pthread:-fuse-ld=lld", + "BAZEL_CXXOPTS": "-stdlib=libc++", + "CXXFLAGS": "-stdlib=libc++", +}) + _GCC_ENV = { "BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", - "BAZEL_LINKOPTS": "-lm:-static-libgcc", + "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", @@ -35,46 +42,32 @@ _TOOLCHAIN_CONFIG_SUITE_SPEC = { "toolchain_config_suite_autogen_spec": _generated_toolchain_config_suite_autogen_spec, } -def _rbe_toolchains_generator(): +def _envoy_rbe_toolchain(name, env, toolchain_config_spec_name): rbe_autoconfig( - name = "rbe_ubuntu_clang_gen", - digest = _ENVOY_BUILD_IMAGE_DIGEST, + name = name + "_gen", export_configs = True, - java_home = _ENVOY_BUILD_IMAGE_JAVA_HOME, - registry = _ENVOY_BUILD_IMAGE_REGISTRY, - repository = _ENVOY_BUILD_IMAGE_REPOSITORY, - env = _CLANG_ENV, - toolchain_config_spec_name = "clang", - toolchain_config_suite_spec = _TOOLCHAIN_CONFIG_SUITE_SPEC, - use_checked_in_confs = "False", - ) - - rbe_autoconfig( - name = "rbe_ubuntu_gcc_gen", + create_java_configs = False, digest = _ENVOY_BUILD_IMAGE_DIGEST, - export_configs = True, - java_home = _ENVOY_BUILD_IMAGE_JAVA_HOME, registry = _ENVOY_BUILD_IMAGE_REGISTRY, repository = _ENVOY_BUILD_IMAGE_REPOSITORY, - env = _GCC_ENV, - toolchain_config_spec_name = "gcc", + env = env, + toolchain_config_spec_name = toolchain_config_spec_name, toolchain_config_suite_spec = _TOOLCHAIN_CONFIG_SUITE_SPEC, use_checked_in_confs = "False", ) -def _generated_rbe_toolchains(): rbe_autoconfig( - name = "rbe_ubuntu_clang", + name = name, + create_java_configs = False, digest = _ENVOY_BUILD_IMAGE_DIGEST, - export_configs = True, - java_home = _ENVOY_BUILD_IMAGE_JAVA_HOME, registry = _ENVOY_BUILD_IMAGE_REGISTRY, repository = _ENVOY_BUILD_IMAGE_REPOSITORY, - toolchain_config_spec_name = "clang", + toolchain_config_spec_name = toolchain_config_spec_name, toolchain_config_suite_spec = _TOOLCHAIN_CONFIG_SUITE_SPEC, use_checked_in_confs = "Force", ) def rbe_toolchains_config(): - _rbe_toolchains_generator() - _generated_rbe_toolchains() + _envoy_rbe_toolchain("rbe_ubuntu_clang", _CLANG_ENV, "clang") + _envoy_rbe_toolchain("rbe_ubuntu_clang_libcxx", _CLANG_LIBCXX_ENV, "clang_libcxx") + _envoy_rbe_toolchain("rbe_ubuntu_gcc", _GCC_ENV, "gcc") diff --git a/bazel/toolchains/regenerate.sh b/bazel/toolchains/regenerate.sh index c71a0506d178..b391a9efad9c 100755 --- a/bazel/toolchains/regenerate.sh +++ b/bazel/toolchains/regenerate.sh @@ -4,9 +4,10 @@ set -e export RBE_AUTOCONF_ROOT=$(bazel info workspace) -rm -rf "${RBE_AUTOCONF_ROOT}/bazel/toolchains/configs/*" +rm -rf "${RBE_AUTOCONF_ROOT}"/bazel/toolchains/configs/* cp -vf "${RBE_AUTOCONF_ROOT}/bazel/toolchains/empty.bzl" "${RBE_AUTOCONF_ROOT}/bazel/toolchains/configs/versions.bzl" # Bazel query is the right command so bazel won't fail itself. -bazel query "@rbe_ubuntu_clang_gen//..." -bazel query "@rbe_ubuntu_gcc_gen//..." +bazel query ${BAZEL_QUERY_OPTIONS} "@rbe_ubuntu_clang_gen//..." +bazel query ${BAZEL_QUERY_OPTIONS} "@rbe_ubuntu_clang_libcxx_gen//..." +bazel query ${BAZEL_QUERY_OPTIONS} "@rbe_ubuntu_gcc_gen//..." From dd501103fd52b48230e52631410f6d55275d4d77 Mon Sep 17 00:00:00 2001 From: scw00 <616955249@qq.com> Date: Fri, 2 Aug 2019 08:26:03 +0800 Subject: [PATCH 295/542] Add flag when open file (#7445) Add flag when open file (#7445) Signed-off-by: cwsong --- include/envoy/filesystem/filesystem.h | 14 +++++- .../access_log/access_log_manager_impl.cc | 10 +++- .../access_log/access_log_manager_impl.h | 3 ++ source/common/filesystem/file_shared_impl.cc | 6 +-- source/common/filesystem/file_shared_impl.h | 4 +- .../filesystem/posix/filesystem_impl.cc | 31 ++++++++++-- .../common/filesystem/posix/filesystem_impl.h | 8 ++- .../filesystem/win32/filesystem_impl.cc | 31 ++++++++++-- .../common/filesystem/win32/filesystem_impl.h | 8 ++- .../common/filesystem/filesystem_impl_test.cc | 49 ++++++++++++++++--- .../quiche/platform/quic_test_output_impl.cc | 8 ++- test/mocks/filesystem/mocks.cc | 2 +- test/mocks/filesystem/mocks.h | 2 +- 13 files changed, 145 insertions(+), 31 deletions(-) diff --git a/include/envoy/filesystem/filesystem.h b/include/envoy/filesystem/filesystem.h index 64923d508544..ae6cd015584e 100644 --- a/include/envoy/filesystem/filesystem.h +++ b/include/envoy/filesystem/filesystem.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -13,6 +14,8 @@ namespace Envoy { namespace Filesystem { +using FlagSet = std::bitset<4>; + /** * Abstraction for a basic file on disk. */ @@ -20,13 +23,20 @@ class File { public: virtual ~File() = default; + enum Operation { + Read, + Write, + Create, + Append, + }; + /** - * Open the file with O_RDWR | O_APPEND | O_CREAT + * Open the file with Flag * The file will be closed when this object is destructed * * @return bool whether the open succeeded */ - virtual Api::IoCallBoolResult open() PURE; + virtual Api::IoCallBoolResult open(FlagSet flags) PURE; /** * Write the buffer to the file. The file must be explicitly opened before writing. diff --git a/source/common/access_log/access_log_manager_impl.cc b/source/common/access_log/access_log_manager_impl.cc index cbf5f8caa4a6..dd2615421db1 100644 --- a/source/common/access_log/access_log_manager_impl.cc +++ b/source/common/access_log/access_log_manager_impl.cc @@ -41,8 +41,16 @@ AccessLogFileImpl::AccessLogFileImpl(Filesystem::FilePtr&& file, Event::Dispatch open(); } +Filesystem::FlagSet AccessLogFileImpl::defaultFlags() { + static constexpr Filesystem::FlagSet default_flags{ + 1 << Filesystem::File::Operation::Read | 1 << Filesystem::File::Operation::Write | + 1 << Filesystem::File::Operation::Create | 1 << Filesystem::File::Operation::Append}; + + return default_flags; +} + void AccessLogFileImpl::open() { - const Api::IoCallBoolResult result = file_->open(); + const Api::IoCallBoolResult result = file_->open(defaultFlags()); if (!result.rc_) { throw EnvoyException( fmt::format("unable to open file '{}': {}", file_->path(), result.err_->getErrorDetails())); diff --git a/source/common/access_log/access_log_manager_impl.h b/source/common/access_log/access_log_manager_impl.h index 4a8dedcdf367..03efbf2d030b 100644 --- a/source/common/access_log/access_log_manager_impl.h +++ b/source/common/access_log/access_log_manager_impl.h @@ -83,6 +83,9 @@ class AccessLogFileImpl : public AccessLogFile { void open(); void createFlushStructures(); + // return default flags set which used by open + static Filesystem::FlagSet defaultFlags(); + // Minimum size before the flush thread will be told to flush. static const uint64_t MIN_FLUSH_SIZE = 1024 * 64; diff --git a/source/common/filesystem/file_shared_impl.cc b/source/common/filesystem/file_shared_impl.cc index 5e235c534e63..dc0e8bfcdc32 100644 --- a/source/common/filesystem/file_shared_impl.cc +++ b/source/common/filesystem/file_shared_impl.cc @@ -9,12 +9,12 @@ Api::IoError::IoErrorCode IoFileError::getErrorCode() const { return IoErrorCode std::string IoFileError::getErrorDetails() const { return ::strerror(errno_); } -Api::IoCallBoolResult FileSharedImpl::open() { +Api::IoCallBoolResult FileSharedImpl::open(FlagSet in) { if (isOpen()) { return resultSuccess(true); } - openFile(); + openFile(in); return fd_ != -1 ? resultSuccess(true) : resultFailure(false, errno); } @@ -36,4 +36,4 @@ bool FileSharedImpl::isOpen() const { return fd_ != -1; }; std::string FileSharedImpl::path() const { return path_; }; } // namespace Filesystem -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/common/filesystem/file_shared_impl.h b/source/common/filesystem/file_shared_impl.h index 5e05b607cfa8..06e166661287 100644 --- a/source/common/filesystem/file_shared_impl.h +++ b/source/common/filesystem/file_shared_impl.h @@ -42,14 +42,14 @@ class FileSharedImpl : public File { ~FileSharedImpl() override = default; // Filesystem::File - Api::IoCallBoolResult open() override; + Api::IoCallBoolResult open(FlagSet flag) override; Api::IoCallSizeResult write(absl::string_view buffer) override; Api::IoCallBoolResult close() override; bool isOpen() const override; std::string path() const override; protected: - virtual void openFile() PURE; + virtual void openFile(FlagSet in) PURE; virtual ssize_t writeFile(absl::string_view buffer) PURE; virtual bool closeFile() PURE; diff --git a/source/common/filesystem/posix/filesystem_impl.cc b/source/common/filesystem/posix/filesystem_impl.cc index 5d1b3b03eb8d..0ea5a45efc21 100644 --- a/source/common/filesystem/posix/filesystem_impl.cc +++ b/source/common/filesystem/posix/filesystem_impl.cc @@ -28,17 +28,38 @@ FileImplPosix::~FileImplPosix() { } } -void FileImplPosix::openFile() { - const int flags = O_RDWR | O_APPEND | O_CREAT; - const int mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; - - fd_ = ::open(path_.c_str(), flags, mode); +void FileImplPosix::openFile(FlagSet in) { + const auto flags_and_mode = translateFlag(in); + fd_ = ::open(path_.c_str(), flags_and_mode.flags_, flags_and_mode.mode_); } ssize_t FileImplPosix::writeFile(absl::string_view buffer) { return ::write(fd_, buffer.data(), buffer.size()); } +FileImplPosix::FlagsAndMode FileImplPosix::translateFlag(FlagSet in) { + int out = 0; + mode_t mode = 0; + if (in.test(File::Operation::Create)) { + out |= O_CREAT; + mode |= S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; + } + + if (in.test(File::Operation::Append)) { + out |= O_APPEND; + } + + if (in.test(File::Operation::Read) && in.test(File::Operation::Write)) { + out |= O_RDWR; + } else if (in.test(File::Operation::Read)) { + out |= O_RDONLY; + } else if (in.test(File::Operation::Write)) { + out |= O_WRONLY; + } + + return {out, mode}; +} + bool FileImplPosix::closeFile() { return ::close(fd_) != -1; } FilePtr InstanceImplPosix::createFile(const std::string& path) { diff --git a/source/common/filesystem/posix/filesystem_impl.h b/source/common/filesystem/posix/filesystem_impl.h index b0c6c6388321..5777c85ef4be 100644 --- a/source/common/filesystem/posix/filesystem_impl.h +++ b/source/common/filesystem/posix/filesystem_impl.h @@ -16,8 +16,14 @@ class FileImplPosix : public FileSharedImpl { ~FileImplPosix() override; protected: + struct FlagsAndMode { + int flags_ = 0; + mode_t mode_ = 0; + }; + // Filesystem::FileSharedImpl - void openFile() override; + FlagsAndMode translateFlag(FlagSet in); + void openFile(FlagSet flags) override; ssize_t writeFile(absl::string_view buffer) override; bool closeFile() override; diff --git a/source/common/filesystem/win32/filesystem_impl.cc b/source/common/filesystem/win32/filesystem_impl.cc index 41a15235ef3c..cc08abbc924c 100644 --- a/source/common/filesystem/win32/filesystem_impl.cc +++ b/source/common/filesystem/win32/filesystem_impl.cc @@ -33,17 +33,38 @@ FileImplWin32::~FileImplWin32() { } } -void FileImplWin32::openFile() { - const int flags = _O_RDWR | _O_APPEND | _O_CREAT; - const int mode = _S_IREAD | _S_IWRITE; - - fd_ = ::_open(path_.c_str(), flags, mode); +void FileImplWin32::openFile(FlagSet in) { + const auto flags_and_mode = translateFlag(in); + fd_ = ::open(path_.c_str(), flags_and_mode.flags_, flags_and_mode.pmode_); } ssize_t FileImplWin32::writeFile(absl::string_view buffer) { return ::_write(fd_, buffer.data(), buffer.size()); } +FileImplWin32::FlagsAndMode FileImplWin32::translateFlag(FlagSet in) { + int out = 0; + int pmode = 0; + if (in.test(File::Operation::Create)) { + out |= _O_CREAT; + pmode |= _S_IREAD | _S_IWRITE; + } + + if (in.test(File::Operation::Append)) { + out |= _O_APPEND; + } + + if (in.test(File::Operation::Read) && in.test(File::Operation::Write)) { + out |= _O_RDWR; + } else if (in.test(File::Operation::Read)) { + out |= _O_RDONLY; + } else if (in.test(File::Operation::Write)) { + out |= _O_WRONLY; + } + + return {out, pmode}; +} + bool FileImplWin32::closeFile() { return ::_close(fd_) != -1; } FilePtr InstanceImplWin32::createFile(const std::string& path) { diff --git a/source/common/filesystem/win32/filesystem_impl.h b/source/common/filesystem/win32/filesystem_impl.h index 7c7add205c87..8d0a2d8ba13d 100644 --- a/source/common/filesystem/win32/filesystem_impl.h +++ b/source/common/filesystem/win32/filesystem_impl.h @@ -14,8 +14,14 @@ class FileImplWin32 : public FileSharedImpl { ~FileImplWin32(); protected: + struct FlagsAndMode { + int flags_ = 0; + int pmode_ = 0; + }; + // Filesystem::FileSharedImpl - void openFile() override; + FlagsAndMode translateFlag(FlagSet in); + void openFile(FlagSet in) override; ssize_t writeFile(absl::string_view buffer) override; bool closeFile() override; diff --git a/test/common/filesystem/filesystem_impl_test.cc b/test/common/filesystem/filesystem_impl_test.cc index f5576e822b65..c8cfdc56ae3a 100644 --- a/test/common/filesystem/filesystem_impl_test.cc +++ b/test/common/filesystem/filesystem_impl_test.cc @@ -12,6 +12,10 @@ namespace Envoy { namespace Filesystem { +static constexpr FlagSet DefaultFlags{ + 1 << Filesystem::File::Operation::Read | 1 << Filesystem::File::Operation::Write | + 1 << Filesystem::File::Operation::Create | 1 << Filesystem::File::Operation::Append}; + class FileSystemImplTest : public testing::Test { protected: int getFd(File* file) { @@ -154,7 +158,7 @@ TEST_F(FileSystemImplTest, Open) { ::unlink(new_file_path.c_str()); FilePtr file = file_system_.createFile(new_file_path); - const Api::IoCallBoolResult result = file->open(); + const Api::IoCallBoolResult result = file->open(DefaultFlags); EXPECT_TRUE(result.rc_); EXPECT_TRUE(file->isOpen()); } @@ -166,13 +170,13 @@ TEST_F(FileSystemImplTest, OpenTwice) { FilePtr file = file_system_.createFile(new_file_path); EXPECT_EQ(getFd(file.get()), -1); - const Api::IoCallBoolResult result1 = file->open(); + const Api::IoCallBoolResult result1 = file->open(DefaultFlags); const int initial_fd = getFd(file.get()); EXPECT_TRUE(result1.rc_); EXPECT_TRUE(file->isOpen()); // check that we don't leak a file descriptor - const Api::IoCallBoolResult result2 = file->open(); + const Api::IoCallBoolResult result2 = file->open(DefaultFlags); EXPECT_EQ(initial_fd, getFd(file.get())); EXPECT_TRUE(result2.rc_); EXPECT_TRUE(file->isOpen()); @@ -180,7 +184,7 @@ TEST_F(FileSystemImplTest, OpenTwice) { TEST_F(FileSystemImplTest, OpenBadFilePath) { FilePtr file = file_system_.createFile(""); - const Api::IoCallBoolResult result = file->open(); + const Api::IoCallBoolResult result = file->open(DefaultFlags); EXPECT_FALSE(result.rc_); } @@ -190,7 +194,7 @@ TEST_F(FileSystemImplTest, ExistingFile) { { FilePtr file = file_system_.createFile(file_path); - const Api::IoCallBoolResult open_result = file->open(); + const Api::IoCallBoolResult open_result = file->open(DefaultFlags); EXPECT_TRUE(open_result.rc_); std::string data(" new data"); const Api::IoCallSizeResult result = file->write(data); @@ -207,7 +211,7 @@ TEST_F(FileSystemImplTest, NonExistingFile) { { FilePtr file = file_system_.createFile(new_file_path); - const Api::IoCallBoolResult open_result = file->open(); + const Api::IoCallBoolResult open_result = file->open(DefaultFlags); EXPECT_TRUE(open_result.rc_); std::string data(" new data"); const Api::IoCallSizeResult result = file->write(data); @@ -223,7 +227,7 @@ TEST_F(FileSystemImplTest, Close) { ::unlink(new_file_path.c_str()); FilePtr file = file_system_.createFile(new_file_path); - const Api::IoCallBoolResult result1 = file->open(); + const Api::IoCallBoolResult result1 = file->open(DefaultFlags); EXPECT_TRUE(result1.rc_); EXPECT_TRUE(file->isOpen()); @@ -237,7 +241,7 @@ TEST_F(FileSystemImplTest, WriteAfterClose) { ::unlink(new_file_path.c_str()); FilePtr file = file_system_.createFile(new_file_path); - const Api::IoCallBoolResult bool_result1 = file->open(); + const Api::IoCallBoolResult bool_result1 = file->open(DefaultFlags); EXPECT_TRUE(bool_result1.rc_); const Api::IoCallBoolResult bool_result2 = file->close(); EXPECT_TRUE(bool_result2.rc_); @@ -247,5 +251,34 @@ TEST_F(FileSystemImplTest, WriteAfterClose) { EXPECT_EQ("Bad file descriptor", size_result.err_->getErrorDetails()); } +TEST_F(FileSystemImplTest, NonExistingFileAndReadOnly) { + const std::string new_file_path = TestEnvironment::temporaryPath("envoy_this_not_exist"); + ::unlink(new_file_path.c_str()); + + static constexpr FlagSet flag(static_cast(Filesystem::File::Operation::Read)); + FilePtr file = file_system_.createFile(new_file_path); + const Api::IoCallBoolResult open_result = file->open(flag); + EXPECT_FALSE(open_result.rc_); +} + +TEST_F(FileSystemImplTest, ExistingReadOnlyFileAndWrite) { + const std::string file_path = + TestEnvironment::writeStringToFileForTest("test_envoy", "existing file"); + + { + static constexpr FlagSet flag(static_cast(Filesystem::File::Operation::Read)); + FilePtr file = file_system_.createFile(file_path); + const Api::IoCallBoolResult open_result = file->open(flag); + EXPECT_TRUE(open_result.rc_); + std::string data(" new data"); + const Api::IoCallSizeResult result = file->write(data); + EXPECT_TRUE(result.rc_ < 0); + EXPECT_EQ(result.err_->getErrorDetails(), "Bad file descriptor"); + } + + auto contents = TestEnvironment::readFileToStringForTest(file_path); + EXPECT_EQ("existing file", contents); +} + } // namespace Filesystem } // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/platform/quic_test_output_impl.cc b/test/extensions/quic_listeners/quiche/platform/quic_test_output_impl.cc index 0e711a484630..fb4fb3c19d8f 100644 --- a/test/extensions/quic_listeners/quiche/platform/quic_test_output_impl.cc +++ b/test/extensions/quic_listeners/quiche/platform/quic_test_output_impl.cc @@ -42,9 +42,15 @@ void QuicRecordTestOutputToFile(const std::string& filename, QuicStringPiece dat return; } + static constexpr Envoy::Filesystem::FlagSet DefaultFlags{ + 1 << Envoy::Filesystem::File::Operation::Read | + 1 << Envoy::Filesystem::File::Operation::Write | + 1 << Envoy::Filesystem::File::Operation::Create | + 1 << Envoy::Filesystem::File::Operation::Append}; + const std::string output_path = output_dir + filename; Envoy::Filesystem::FilePtr file = file_system.createFile(output_path); - if (!file->open().rc_) { + if (!file->open(DefaultFlags).rc_) { QUIC_LOG(ERROR) << "Failed to open test output file: " << output_path; return; } diff --git a/test/mocks/filesystem/mocks.cc b/test/mocks/filesystem/mocks.cc index 5aa5806a26a0..7cf733315030 100644 --- a/test/mocks/filesystem/mocks.cc +++ b/test/mocks/filesystem/mocks.cc @@ -9,7 +9,7 @@ namespace Filesystem { MockFile::MockFile() : num_opens_(0), num_writes_(0), is_open_(false) {} MockFile::~MockFile() = default; -Api::IoCallBoolResult MockFile::open() { +Api::IoCallBoolResult MockFile::open(FlagSet) { Thread::LockGuard lock(open_mutex_); Api::IoCallBoolResult result = open_(); diff --git a/test/mocks/filesystem/mocks.h b/test/mocks/filesystem/mocks.h index b62af89b68a4..459cdfd8ce65 100644 --- a/test/mocks/filesystem/mocks.h +++ b/test/mocks/filesystem/mocks.h @@ -19,7 +19,7 @@ class MockFile : public File { ~MockFile() override; // Filesystem::File - Api::IoCallBoolResult open() override; + Api::IoCallBoolResult open(FlagSet flag) override; Api::IoCallSizeResult write(absl::string_view buffer) override; Api::IoCallBoolResult close() override; bool isOpen() const override { return is_open_; }; From cd8574de783791b3353579b489222bfda74888da Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 1 Aug 2019 18:25:53 -0700 Subject: [PATCH 296/542] ci: partially revert #7786 to fix post submit (#7807) Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 2 ++ .circleci/config.yml | 2 +- ci/build_container/build_container_ubuntu.sh | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index 13818d477f02..bd1d5524a8fa 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -33,6 +33,8 @@ jobs: BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + # TODO(lizan): remove this once new image is available + IMAGE_ID: b02c73f3ebbaf5c7852fa814eb3d61e841588068 displayName: "Run CI script" - bash: | diff --git a/.circleci/config.yml b/.circleci/config.yml index f72ab219b01f..9a4680614d63 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ executors: description: "A regular build executor based on ubuntu image" docker: # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here. - - image: envoyproxy/envoy-build:b02c73f3ebbaf5c7852fa814eb3d61e841588068 + - image: envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d resource_class: xlarge working_directory: /source diff --git a/ci/build_container/build_container_ubuntu.sh b/ci/build_container/build_container_ubuntu.sh index 5a11ae1a5af4..e7ea9f625425 100755 --- a/ci/build_container/build_container_ubuntu.sh +++ b/ci/build_container/build_container_ubuntu.sh @@ -19,7 +19,7 @@ update-alternatives --config gcc update-alternatives --config g++ apt-get install -y --no-install-recommends curl wget make cmake git python python-pip python-setuptools python3 python3-pip \ - unzip bc libtool ninja-build automake zip time gdb strace tshark tcpdump patch xz-utils rsync \ + unzip bc libtool ninja-build automake zip time gdb strace tshark tcpdump patch xz-utils rsync ssh-client # clang 8. case $ARCH in From 95a2b2681a0f6635862b787de72197f87901f398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ladr=C3=B3n=20De=20Guevara?= Date: Thu, 1 Aug 2019 22:13:18 -0500 Subject: [PATCH 297/542] docs: fix custom header example (#7806) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The example shows the request_headers_to_add in the RouteAction, and this should be in the Route Signed-off-by: Christian Ladrón de Guevara Reyes --- docs/root/configuration/http_conn_man/headers.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/root/configuration/http_conn_man/headers.rst b/docs/root/configuration/http_conn_man/headers.rst index be47a4e45232..bfb3a0682ad7 100644 --- a/docs/root/configuration/http_conn_man/headers.rst +++ b/docs/root/configuration/http_conn_man/headers.rst @@ -622,8 +622,8 @@ Supported variable names are: route: cluster: www - request_headers_to_add: - - header: - key: "x-request-start" - value: "%START_TIME(%s.%3f)%" - append: true + request_headers_to_add: + - header: + key: "x-request-start" + value: "%START_TIME(%s.%3f)%" + append: true From b1fcf27fbe46804fd8a246c67a8b47e861756f98 Mon Sep 17 00:00:00 2001 From: Yechiel Kalmenson Date: Thu, 1 Aug 2019 23:14:15 -0400 Subject: [PATCH 298/542] bump abseil to latest version (#7802) This fixes https://github.com/envoyproxy/envoy/issues/7357 on windows Visual Studio 2017 Signed-off-by: William Rowe --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 4d4dc47cf9db..c21af3bb42b2 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -26,10 +26,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://commondatastorage.googleapis.com/chromium-boringssl-docs/fips/boringssl-66005f41fbc3529ffe8d007708756720529da20d.tar.xz"], ), com_google_absl = dict( - sha256 = "7ddf863ddced6fa5bf7304103f9c7aa619c20a2fcf84475512c8d3834b9d14fa", - strip_prefix = "abseil-cpp-61c9bf3e3e1c28a4aa6d7f1be4b37fd473bb5529", - # 2019-06-05 - urls = ["https://github.com/abseil/abseil-cpp/archive/61c9bf3e3e1c28a4aa6d7f1be4b37fd473bb5529.tar.gz"], + sha256 = "3df5970908ed9a09ba51388d04661803a6af18c373866f442cede7f381e0b94a", + strip_prefix = "abseil-cpp-14550beb3b7b97195e483fb74b5efb906395c31e", + # 2019-07-31 + urls = ["https://github.com/abseil/abseil-cpp/archive/14550beb3b7b97195e483fb74b5efb906395c31e.tar.gz"], ), com_github_apache_thrift = dict( sha256 = "7d59ac4fdcb2c58037ebd4a9da5f9a49e3e034bf75b3f26d9fe48ba3d8806e6b", From fc8d6f5fe4fdfa62e736ba50122c312871e92208 Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 1 Aug 2019 23:15:14 -0400 Subject: [PATCH 299/542] fuzz: remove header prefix from Bootstrap (#7803) Signed-off-by: Asra Ali --- .../clusterfuzz-testcase-server_fuzz_test-5763613693837312 | 7 +++++++ test/server/server_fuzz_test.cc | 3 +++ 2 files changed, 10 insertions(+) create mode 100644 test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5763613693837312 diff --git a/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5763613693837312 b/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5763613693837312 new file mode 100644 index 000000000000..9c3fe726b36d --- /dev/null +++ b/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5763613693837312 @@ -0,0 +1,7 @@ +stats_sinks { + typed_config { + type_url: "type.googleapis.com/envoy.api.v2.route.Route" + value: "\022*J :222222\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\t2871770\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\0378\377\377\377\377\377\377\377\377 8\377\377\377\377\377\377\377\377\37722222222222222222220\022" + } +} +header_prefix: "type.googleapis.com/envoy.api.v2.route.Route" diff --git a/test/server/server_fuzz_test.cc b/test/server/server_fuzz_test.cc index 53db522ebbb0..2ee39d2f8d88 100644 --- a/test/server/server_fuzz_test.cc +++ b/test/server/server_fuzz_test.cc @@ -37,6 +37,9 @@ makeHermeticPathsAndPorts(Fuzz::PerTestEnvironment& test_env, // we lose here. If we don't sanitize here, we get flakes due to port bind conflicts, file // conflicts, etc. output.clear_admin(); + // The header_prefix is a write-once then read-only singleton that persists across tests. We clear + // this field so that fuzz tests don't fail over multiple iterations. + output.clear_header_prefix(); if (output.has_runtime()) { output.mutable_runtime()->set_symlink_root(test_env.temporaryPath("")); } From 06119fa22202f0dd249ed2a7042a1360dc975876 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Thu, 1 Aug 2019 20:18:16 -0700 Subject: [PATCH 300/542] clang-tidy modernize-use-nullptr (#7791) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + .../proxy_protocol/proxy_protocol_header.h | 2 +- test/common/buffer/buffer_test.cc | 2 +- .../common/tap/tap_config_base_test.cc | 16 ++++++++-------- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 09fb39413ac8..69c3b70a671c 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -26,6 +26,7 @@ WarningsAsErrors: 'abseil-duration-*, modernize-return-braced-init-list, modernize-use-default-member-init, modernize-use-equals-default, + modernize-use-nullptr, modernize-use-override, modernize-use-using, performance-faster-string-find, diff --git a/source/extensions/filters/listener/proxy_protocol/proxy_protocol_header.h b/source/extensions/filters/listener/proxy_protocol/proxy_protocol_header.h index fda95268dd2b..4d3ea34c138e 100644 --- a/source/extensions/filters/listener/proxy_protocol/proxy_protocol_header.h +++ b/source/extensions/filters/listener/proxy_protocol/proxy_protocol_header.h @@ -27,7 +27,7 @@ constexpr uint32_t PROXY_PROTO_V2_AF_UNIX = 0x3; struct WireHeader { WireHeader(size_t extensions_length) : extensions_length_(extensions_length), protocol_version_(Network::Address::IpVersion::v4), - remote_address_(0), local_address_(0), local_command_(true) {} + remote_address_(nullptr), local_address_(nullptr), local_command_(true) {} WireHeader(size_t extensions_length, Network::Address::IpVersion protocol_version, Network::Address::InstanceConstSharedPtr remote_address, Network::Address::InstanceConstSharedPtr local_address) diff --git a/test/common/buffer/buffer_test.cc b/test/common/buffer/buffer_test.cc index aaa1c34ba02b..ac8cff975e5e 100644 --- a/test/common/buffer/buffer_test.cc +++ b/test/common/buffer/buffer_test.cc @@ -42,7 +42,7 @@ class OwnedSliceTest : public testing::Test { static void expectReservationFailure(const Slice::Reservation& reservation, const Slice& slice, uint64_t reservable_size) { EXPECT_EQ(nullptr, reservation.mem_); - EXPECT_EQ(0, reservation.mem_); + EXPECT_EQ(nullptr, reservation.mem_); EXPECT_EQ(reservable_size, slice.reservableSize()); } diff --git a/test/extensions/common/tap/tap_config_base_test.cc b/test/extensions/common/tap/tap_config_base_test.cc index c7d83a796f0a..e95ac433894b 100644 --- a/test/extensions/common/tap/tap_config_base_test.cc +++ b/test/extensions/common/tap/tap_config_base_test.cc @@ -96,15 +96,15 @@ TEST(TrimSlice, All) { } { - std::vector slices = {{0x0, 5}}; + std::vector slices = {{nullptr, 5}}; Utility::trimSlices(slices, 0, 100); - const std::vector expected{{0x0, 5}}; + const std::vector expected{{nullptr, 5}}; EXPECT_EQ(expected, slices); } { - std::vector slices = {{0x0, 5}}; + std::vector slices = {{nullptr, 5}}; Utility::trimSlices(slices, 3, 3); const std::vector expected{{reinterpret_cast(0x3), 2}}; @@ -112,7 +112,7 @@ TEST(TrimSlice, All) { } { - std::vector slices = {{0x0, 5}, {0x0, 4}}; + std::vector slices = {{nullptr, 5}, {nullptr, 4}}; Utility::trimSlices(slices, 3, 3); const std::vector expected{{reinterpret_cast(0x3), 2}, @@ -121,7 +121,7 @@ TEST(TrimSlice, All) { } { - std::vector slices = {{0x0, 5}, {0x0, 4}}; + std::vector slices = {{nullptr, 5}, {nullptr, 4}}; Utility::trimSlices(slices, 6, 3); const std::vector expected{{reinterpret_cast(0x5), 0}, @@ -130,7 +130,7 @@ TEST(TrimSlice, All) { } { - std::vector slices = {{0x0, 5}, {0x0, 4}}; + std::vector slices = {{nullptr, 5}, {nullptr, 4}}; Utility::trimSlices(slices, 0, 0); const std::vector expected{{reinterpret_cast(0x0), 0}, @@ -139,7 +139,7 @@ TEST(TrimSlice, All) { } { - std::vector slices = {{0x0, 5}, {0x0, 4}}; + std::vector slices = {{nullptr, 5}, {nullptr, 4}}; Utility::trimSlices(slices, 0, 3); const std::vector expected{{reinterpret_cast(0x0), 3}, @@ -148,7 +148,7 @@ TEST(TrimSlice, All) { } { - std::vector slices = {{0x0, 5}, {0x0, 4}}; + std::vector slices = {{nullptr, 5}, {nullptr, 4}}; Utility::trimSlices(slices, 1, 3); const std::vector expected{{reinterpret_cast(0x1), 3}, From 266252e8d2880113d2c01dbe4e01bda3a0c9730d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Fri, 2 Aug 2019 11:56:46 -0400 Subject: [PATCH 301/542] zookeeper: parse server responses (#7574) This adds support for parsing server responses and watch events. One or more follow-up PRs will add support for latency measurements and access log. Signed-off-by: Raul Gutierrez Segales --- .../zookeeper_proxy_filter.rst | 35 ++ .../network/zookeeper_proxy/decoder.cc | 144 ++++++- .../filters/network/zookeeper_proxy/decoder.h | 18 +- .../filters/network/zookeeper_proxy/filter.cc | 160 ++++++- .../filters/network/zookeeper_proxy/filter.h | 38 +- .../network/zookeeper_proxy/filter_test.cc | 403 +++++++++++------- 6 files changed, 592 insertions(+), 206 deletions(-) diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst index cf8e1c9716a7..c75f434d66ac 100644 --- a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst +++ b/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst @@ -72,6 +72,34 @@ following statistics: checkwatches_rq, Counter, Number of checkwatches requests removewatches_rq, Counter, Number of removewatches requests check_rq, Counter, Number of check requests + response_bytes, Counter, Number of bytes in decoded response messages + connect_resp, Counter, Number of connect responses + ping_resp, Counter, Number of ping responses + auth_resp, Counter, Number of auth responses + watch_event, Counter, Number of watch events fired by the server + getdata_resp, Counter, Number of getdata responses + create_resp, Counter, Number of create responses + create2_resp, Counter, Number of create2 responses + createcontainer_resp, Counter, Number of createcontainer responses + createttl_resp, Counter, Number of createttl responses + setdata_resp, Counter, Number of setdata responses + getchildren_resp, Counter, Number of getchildren responses + getchildren2_resp, Counter, Number of getchildren2 responses + getephemerals_resp, Counter, Number of getephemerals responses + getallchildrennumber_resp, Counter, Number of getallchildrennumber responses + remove_resp, Counter, Number of remove responses + exists_resp, Counter, Number of exists responses + getacl_resp, Counter, Number of getacl responses + setacl_resp, Counter, Number of setacl responses + sync_resp, Counter, Number of sync responses + multi_resp, Counter, Number of multi responses + reconfig_resp, Counter, Number of reconfig responses + close_resp, Counter, Number of close responses + setauth_resp, Counter, Number of setauth responses + setwatches_resp, Counter, Number of setwatches responses + checkwatches_resp, Counter, Number of checkwatches responses + removewatches_resp, Counter, Number of removewatches responses + check_resp, Counter, Number of check responses .. _config_network_filters_zookeeper_proxy_dynamic_metadata: @@ -90,3 +118,10 @@ The ZooKeeper filter emits the following dynamic metadata for each message parse , string, "The size of the request message in bytes" , string, "True if a watch is being set, false otherwise" , string, "The version parameter, if any, given with the request" + , string, "The timeout parameter in a connect response" + , string, "The protocol version in a connect response" + , string, "The readonly flag in a connect response" + , string, "The zxid field in a response header" + , string, "The error field in a response header" + , string, "The state field in a watch event" + , string, "The event type in a a watch event" diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.cc b/source/extensions/filters/network/zookeeper_proxy/decoder.cc index db2d5fb5b919..123eb21f990f 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.cc @@ -2,6 +2,8 @@ #include +#include "common/common/enum_to_int.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -16,6 +18,8 @@ constexpr uint32_t ZXID_LENGTH = 8; constexpr uint32_t TIMEOUT_LENGTH = 4; constexpr uint32_t SESSION_LENGTH = 8; constexpr uint32_t MULTI_HEADER_LENGTH = 9; +constexpr uint32_t PROTOCOL_VERSION_LENGTH = 4; +constexpr uint32_t SERVER_HEADER_LENGTH = 16; const char* createFlagsToString(CreateFlags flags) { switch (flags) { @@ -38,17 +42,9 @@ const char* createFlagsToString(CreateFlags flags) { return "unknown"; } -void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { - ENVOY_LOG(trace, "zookeeper_proxy: decoding {} bytes at offset {}", data.length(), offset); - - // Reset the helper's cursor, to ensure the current message stays within the - // allowed max length, even when it's different than the declared length - // by the message. - // - // Note: we need to keep two cursors — offset and helper_'s internal one — because - // a buffer may contain multiple messages, so offset is global and helper_'s - // internal cursor is reset for each individual message. - helper_.reset(); +void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) { + ENVOY_LOG(trace, "zookeeper_proxy: decoding request with {} bytes at offset {}", data.length(), + offset); // Check message length. const int32_t len = helper_.peekInt32(data, offset); @@ -93,8 +89,8 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { // for two cases: auth requests can happen at any time and ping requests // must happen every 1/3 of the negotiated session timeout, to keep // the session alive. - const int32_t opcode = helper_.peekInt32(data, offset); - switch (static_cast(opcode)) { + const auto opcode = static_cast(helper_.peekInt32(data, offset)); + switch (opcode) { case OpCodes::GETDATA: parseGetDataRequest(data, offset, len); break; @@ -156,8 +152,62 @@ void DecoderImpl::decode(Buffer::Instance& data, uint64_t& offset) { callbacks_.onCloseRequest(); break; default: - throw EnvoyException(fmt::format("Unknown opcode: {}", opcode)); + throw EnvoyException(fmt::format("Unknown opcode: {}", enumToSignedInt(opcode))); + } + + requests_by_xid_[xid] = opcode; +} + +void DecoderImpl::decodeOnWrite(Buffer::Instance& data, uint64_t& offset) { + ENVOY_LOG(trace, "zookeeper_proxy: decoding response with {} bytes at offset {}", data.length(), + offset); + + // Check message length. + const int32_t len = helper_.peekInt32(data, offset); + ensureMinLength(len, INT_LENGTH + XID_LENGTH); + ensureMaxLength(len); + + const auto xid = helper_.peekInt32(data, offset); + const auto xid_code = static_cast(xid); + + // Connect responses are special, they have no full reply header + // but just an XID with no zxid nor error fields like the ones + // available for all other server generated messages. + if (xid_code == XidCodes::CONNECT_XID) { + parseConnectResponse(data, offset, len); + return; + } + + // Control responses that aren't connect, with XIDs <= 0. + const auto zxid = helper_.peekInt64(data, offset); + const auto error = helper_.peekInt32(data, offset); + switch (xid_code) { + case XidCodes::PING_XID: + callbacks_.onResponse(OpCodes::PING, xid, zxid, error); + return; + case XidCodes::AUTH_XID: + callbacks_.onResponse(OpCodes::SETAUTH, xid, zxid, error); + return; + case XidCodes::SET_WATCHES_XID: + callbacks_.onResponse(OpCodes::SETWATCHES, xid, zxid, error); + return; + case XidCodes::WATCH_XID: + parseWatchEvent(data, offset, len, zxid, error); + return; + default: + break; } + + // Find the corresponding request for this XID. + const auto it = requests_by_xid_.find(xid); + + // If this fails, it's a server-side bug. + ASSERT(it != requests_by_xid_.end()); + + const auto opcode = it->second; + requests_by_xid_.erase(it); + offset += (len - (XID_LENGTH + ZXID_LENGTH + INT_LENGTH)); + callbacks_.onResponse(opcode, xid, zxid, error); } void DecoderImpl::ensureMinLength(const int32_t len, const int32_t minlen) const { @@ -181,11 +231,7 @@ void DecoderImpl::parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_ // Skip password. skipString(data, offset); - // Read readonly flag, if it's there. - bool readonly{}; - if (data.length() >= offset + 1) { - readonly = helper_.peekBool(data, offset); - } + const bool readonly = maybeReadBool(data, offset); callbacks_.onConnect(readonly); } @@ -397,13 +443,35 @@ void DecoderImpl::skipStrings(Buffer::Instance& data, uint64_t& offset) { } } -void DecoderImpl::onData(Buffer::Instance& data) { +void DecoderImpl::onData(Buffer::Instance& data) { decode(data, DecodeType::READ); } + +void DecoderImpl::onWrite(Buffer::Instance& data) { decode(data, DecodeType::WRITE); } + +void DecoderImpl::decode(Buffer::Instance& data, DecodeType dtype) { uint64_t offset = 0; + try { while (offset < data.length()) { + // Reset the helper's cursor, to ensure the current message stays within the + // allowed max length, even when it's different than the declared length + // by the message. + // + // Note: we need to keep two cursors — offset and helper_'s internal one — because + // a buffer may contain multiple messages, so offset is global and helper_'s + // internal cursor is reset for each individual message. + helper_.reset(); + const uint64_t current = offset; - decode(data, offset); - callbacks_.onRequestBytes(offset - current); + switch (dtype) { + case DecodeType::READ: + decodeOnData(data, offset); + callbacks_.onRequestBytes(offset - current); + break; + case DecodeType::WRITE: + decodeOnWrite(data, offset); + callbacks_.onResponseBytes(offset - current); + break; + } } } catch (const EnvoyException& e) { ENVOY_LOG(debug, "zookeeper_proxy: decoding exception {}", e.what()); @@ -411,6 +479,38 @@ void DecoderImpl::onData(Buffer::Instance& data) { } } +void DecoderImpl::parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len) { + ensureMinLength(len, PROTOCOL_VERSION_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); + + const auto timeout = helper_.peekInt32(data, offset); + + // Skip session id + password. + offset += SESSION_LENGTH; + skipString(data, offset); + + const bool readonly = maybeReadBool(data, offset); + + callbacks_.onConnectResponse(0, timeout, readonly); +} + +void DecoderImpl::parseWatchEvent(Buffer::Instance& data, uint64_t& offset, const uint32_t len, + const int64_t zxid, const int32_t error) { + ensureMinLength(len, SERVER_HEADER_LENGTH + (3 * INT_LENGTH)); + + const auto event_type = helper_.peekInt32(data, offset); + const auto client_state = helper_.peekInt32(data, offset); + const auto path = helper_.peekString(data, offset); + + callbacks_.onWatchEvent(event_type, client_state, path, zxid, error); +} + +bool DecoderImpl::maybeReadBool(Buffer::Instance& data, uint64_t& offset) { + if (data.length() >= offset + 1) { + return helper_.peekBool(data, offset); + } + return false; +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.h b/source/extensions/filters/network/zookeeper_proxy/decoder.h index 4e8db8c9361c..7b03f7a13b8d 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.h @@ -95,6 +95,11 @@ class DecoderCallbacks { virtual void onCheckWatchesRequest(const std::string& path, int32_t type) PURE; virtual void onRemoveWatchesRequest(const std::string& path, int32_t type) PURE; virtual void onCloseRequest() PURE; + virtual void onResponseBytes(uint64_t bytes) PURE; + virtual void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly) PURE; + virtual void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error) PURE; + virtual void onWatchEvent(int32_t event_type, int32_t client_state, const std::string& path, + int64_t zxid, int32_t error) PURE; }; /** @@ -105,6 +110,7 @@ class Decoder { virtual ~Decoder() = default; virtual void onData(Buffer::Instance& data) PURE; + virtual void onWrite(Buffer::Instance& data) PURE; }; using DecoderPtr = std::unique_ptr; @@ -116,9 +122,14 @@ class DecoderImpl : public Decoder, Logger::Loggable { // ZooKeeperProxy::Decoder void onData(Buffer::Instance& data) override; + void onWrite(Buffer::Instance& data) override; private: - void decode(Buffer::Instance& data, uint64_t& offset); + enum class DecodeType { READ, WRITE }; + + void decode(Buffer::Instance& data, DecodeType dtype); + void decodeOnData(Buffer::Instance& data, uint64_t& offset); + void decodeOnWrite(Buffer::Instance& data, uint64_t& offset); void parseConnect(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseAuthRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); void parseGetDataRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); @@ -140,10 +151,15 @@ class DecoderImpl : public Decoder, Logger::Loggable { void ensureMinLength(int32_t len, int32_t minlen) const; void ensureMaxLength(int32_t len) const; std::string pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseWatchEvent(Buffer::Instance& data, uint64_t& offset, uint32_t len, int64_t zxid, + int32_t error); + bool maybeReadBool(Buffer::Instance& data, uint64_t& offset); DecoderCallbacks& callbacks_; const uint32_t max_packet_bytes_; BufferHelper helper_; + std::unordered_map requests_by_xid_; }; } // namespace ZooKeeperProxy diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.cc b/source/extensions/filters/network/zookeeper_proxy/filter.cc index 3a11262f298f..6cb5ab105d95 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/filter.cc @@ -22,33 +22,26 @@ ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, stats_(generateStats(stat_prefix, scope)) {} ZooKeeperFilter::ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config) - : config_(std::move(config)) {} + : config_(std::move(config)), decoder_(createDecoder(*this)) {} void ZooKeeperFilter::initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) { read_callbacks_ = &callbacks; } Network::FilterStatus ZooKeeperFilter::onData(Buffer::Instance& data, bool) { - doDecode(data); + clearDynamicMetadata(); + decoder_->onData(data); return Network::FilterStatus::Continue; } -Network::FilterStatus ZooKeeperFilter::onWrite(Buffer::Instance&, bool) { +Network::FilterStatus ZooKeeperFilter::onWrite(Buffer::Instance& data, bool) { + clearDynamicMetadata(); + decoder_->onWrite(data); return Network::FilterStatus::Continue; } Network::FilterStatus ZooKeeperFilter::onNewConnection() { return Network::FilterStatus::Continue; } -void ZooKeeperFilter::doDecode(Buffer::Instance& buffer) { - clearDynamicMetadata(); - - if (!decoder_) { - decoder_ = createDecoder(*this); - } - - decoder_->onData(buffer); -} - DecoderPtr ZooKeeperFilter::createDecoder(DecoderCallbacks& callbacks) { return std::make_unique(callbacks, config_->maxPacketBytes()); } @@ -103,6 +96,11 @@ void ZooKeeperFilter::onRequestBytes(const uint64_t bytes) { setDynamicMetadata("bytes", std::to_string(bytes)); } +void ZooKeeperFilter::onResponseBytes(const uint64_t bytes) { + config_->stats_.response_bytes_.add(bytes); + setDynamicMetadata("bytes", std::to_string(bytes)); +} + void ZooKeeperFilter::onPing() { config_->stats_.ping_rq_.inc(); setDynamicMetadata("opname", "ping"); @@ -168,8 +166,8 @@ void ZooKeeperFilter::onGetChildrenRequest(const std::string& path, const bool w } void ZooKeeperFilter::onDeleteRequest(const std::string& path, const int32_t version) { - config_->stats_.remove_rq_.inc(); - setDynamicMetadata({{"opname", "remove"}, {"path", path}, {"version", std::to_string(version)}}); + config_->stats_.delete_rq_.inc(); + setDynamicMetadata({{"opname", "delete"}, {"path", path}, {"version", std::to_string(version)}}); } void ZooKeeperFilter::onExistsRequest(const std::string& path, const bool watch) { @@ -236,6 +234,138 @@ void ZooKeeperFilter::onCloseRequest() { setDynamicMetadata("opname", "close"); } +void ZooKeeperFilter::onConnectResponse(const int32_t proto_version, const int32_t timeout, + const bool readonly) { + config_->stats_.connect_resp_.inc(); + setDynamicMetadata({{"opname", "connect_response"}, + {"protocol_version", std::to_string(proto_version)}, + {"timeout", std::to_string(timeout)}, + {"readonly", std::to_string(readonly)}}); +} + +void ZooKeeperFilter::onResponse(const OpCodes opcode, const int32_t xid, const int64_t zxid, + const int32_t error) { + std::string opname = ""; + + switch (opcode) { + case OpCodes::PING: + config_->stats_.ping_resp_.inc(); + opname = "ping_response"; + break; + case OpCodes::SETAUTH: + config_->stats_.auth_resp_.inc(); + opname = "auth_response"; + break; + case OpCodes::GETDATA: + opname = "getdata_resp"; + config_->stats_.getdata_resp_.inc(); + break; + case OpCodes::CREATE: + opname = "create_resp"; + config_->stats_.create_resp_.inc(); + break; + case OpCodes::CREATE2: + opname = "create2_resp"; + config_->stats_.create2_resp_.inc(); + break; + case OpCodes::CREATECONTAINER: + opname = "createcontainer_resp"; + config_->stats_.createcontainer_resp_.inc(); + break; + case OpCodes::CREATETTL: + opname = "createttl_resp"; + config_->stats_.createttl_resp_.inc(); + break; + case OpCodes::SETDATA: + opname = "setdata_resp"; + config_->stats_.setdata_resp_.inc(); + break; + case OpCodes::GETCHILDREN: + opname = "getchildren_resp"; + config_->stats_.getchildren_resp_.inc(); + break; + case OpCodes::GETCHILDREN2: + opname = "getchildren2_resp"; + config_->stats_.getchildren2_resp_.inc(); + break; + case OpCodes::DELETE: + opname = "delete_resp"; + config_->stats_.delete_resp_.inc(); + break; + case OpCodes::EXISTS: + opname = "exists_resp"; + config_->stats_.exists_resp_.inc(); + break; + case OpCodes::GETACL: + config_->stats_.getacl_resp_.inc(); + opname = "getacl_resp"; + break; + case OpCodes::SETACL: + opname = "setacl_resp"; + config_->stats_.setacl_resp_.inc(); + break; + case OpCodes::SYNC: + opname = "sync_resp"; + config_->stats_.sync_resp_.inc(); + break; + case OpCodes::CHECK: + opname = "check_resp"; + config_->stats_.check_resp_.inc(); + break; + case OpCodes::MULTI: + opname = "multi_resp"; + config_->stats_.multi_resp_.inc(); + break; + case OpCodes::RECONFIG: + opname = "reconfig_resp"; + config_->stats_.reconfig_resp_.inc(); + break; + case OpCodes::SETWATCHES: + opname = "setwatches_resp"; + config_->stats_.setwatches_resp_.inc(); + break; + case OpCodes::CHECKWATCHES: + opname = "checkwatches_resp"; + config_->stats_.checkwatches_resp_.inc(); + break; + case OpCodes::REMOVEWATCHES: + opname = "removewatches_resp"; + config_->stats_.removewatches_resp_.inc(); + break; + case OpCodes::GETEPHEMERALS: + opname = "getephemerals_resp"; + config_->stats_.getephemerals_resp_.inc(); + break; + case OpCodes::GETALLCHILDRENNUMBER: + opname = "getallchildrennumber_resp"; + config_->stats_.getallchildrennumber_resp_.inc(); + break; + case OpCodes::CLOSE: + opname = "close_resp"; + config_->stats_.close_resp_.inc(); + break; + default: + break; + } + + setDynamicMetadata({{"opname", opname}, + {"xid", std::to_string(xid)}, + {"zxid", std::to_string(zxid)}, + {"error", std::to_string(error)}}); +} + +void ZooKeeperFilter::onWatchEvent(const int32_t event_type, const int32_t client_state, + const std::string& path, const int64_t zxid, + const int32_t error) { + config_->stats_.watch_event_.inc(); + setDynamicMetadata({{"opname", "watch_event"}, + {"event_type", std::to_string(event_type)}, + {"client_state", std::to_string(client_state)}, + {"path", path}, + {"zxid", std::to_string(zxid)}, + {"error", std::to_string(error)}}); +} + } // namespace ZooKeeperProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.h b/source/extensions/filters/network/zookeeper_proxy/filter.h index 491a12032965..d64918c2b8d4 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/filter.h @@ -39,7 +39,7 @@ namespace ZooKeeperProxy { COUNTER(getchildren2_rq) \ COUNTER(getephemerals_rq) \ COUNTER(getallchildrennumber_rq) \ - COUNTER(remove_rq) \ + COUNTER(delete_rq) \ COUNTER(exists_rq) \ COUNTER(getacl_rq) \ COUNTER(setacl_rq) \ @@ -52,7 +52,35 @@ namespace ZooKeeperProxy { COUNTER(setwatches_rq) \ COUNTER(checkwatches_rq) \ COUNTER(removewatches_rq) \ - COUNTER(check_rq) + COUNTER(check_rq) \ + COUNTER(response_bytes) \ + COUNTER(connect_resp) \ + COUNTER(ping_resp) \ + COUNTER(auth_resp) \ + COUNTER(getdata_resp) \ + COUNTER(create_resp) \ + COUNTER(create2_resp) \ + COUNTER(createcontainer_resp) \ + COUNTER(createttl_resp) \ + COUNTER(setdata_resp) \ + COUNTER(getchildren_resp) \ + COUNTER(getchildren2_resp) \ + COUNTER(getephemerals_resp) \ + COUNTER(getallchildrennumber_resp) \ + COUNTER(delete_resp) \ + COUNTER(exists_resp) \ + COUNTER(getacl_resp) \ + COUNTER(setacl_resp) \ + COUNTER(sync_resp) \ + COUNTER(multi_resp) \ + COUNTER(reconfig_resp) \ + COUNTER(close_resp) \ + COUNTER(setauth_resp) \ + COUNTER(setwatches_resp) \ + COUNTER(checkwatches_resp) \ + COUNTER(removewatches_resp) \ + COUNTER(check_resp) \ + COUNTER(watch_event) // clang-format on /** @@ -127,8 +155,12 @@ class ZooKeeperFilter : public Network::Filter, void onGetEphemeralsRequest(const std::string& path) override; void onGetAllChildrenNumberRequest(const std::string& path) override; void onCloseRequest() override; + void onResponseBytes(uint64_t bytes) override; + void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly) override; + void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error) override; + void onWatchEvent(int32_t event_type, int32_t client_state, const std::string& path, int64_t zxid, + int32_t error) override; - void doDecode(Buffer::Instance& buffer); DecoderPtr createDecoder(DecoderCallbacks& callbacks); void setDynamicMetadata(const std::string& key, const std::string& value); void setDynamicMetadata(const std::vector>& data); diff --git a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc index fc2e852ceffe..c2f37fcc197b 100644 --- a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc @@ -57,6 +57,55 @@ class ZooKeeperFilterTest : public testing::Test { return buffer; } + Buffer::OwnedImpl encodeConnectResponse(const bool readonly = false, + const uint32_t session_timeout = 10, + const uint32_t session_id = 200, + const std::string& passwd = "") const { + Buffer::OwnedImpl buffer; + const uint32_t message_size = readonly ? 20 + passwd.length() + 1 : 20 + passwd.length(); + + buffer.writeBEInt(message_size); + buffer.writeBEInt(0); // Protocol version. + buffer.writeBEInt(session_timeout); + buffer.writeBEInt(session_id); + addString(buffer, passwd); + + if (readonly) { + const char readonly_flag = 0b1; + buffer.add(std::string(1, readonly_flag)); + } + + return buffer; + } + + Buffer::OwnedImpl encodeResponseHeader(const int32_t xid, const int64_t zxid, + const int32_t error) const { + Buffer::OwnedImpl buffer; + const uint32_t message_size = 16; + + buffer.writeBEInt(message_size); + buffer.writeBEInt(xid); + buffer.writeBEInt(zxid); + buffer.writeBEInt(error); + + return buffer; + } + + Buffer::OwnedImpl encodeWatchEvent(const std::string& path, const int32_t event_type, + const int32_t client_state) const { + Buffer::OwnedImpl buffer; + + buffer.writeBEInt(28 + path.size()); + buffer.writeBEInt(enumToSignedInt(XidCodes::WATCH_XID)); + buffer.writeBEInt(1000); + buffer.writeBEInt(0); + buffer.writeBEInt(event_type); + buffer.writeBEInt(client_state); + addString(buffer, path); + + return buffer; + } + Buffer::OwnedImpl encodeBadMessage() const { Buffer::OwnedImpl buffer; @@ -365,26 +414,20 @@ class ZooKeeperFilterTest : public testing::Test { } } - void expectSetDynamicMetadata(const std::map& expected) { - EXPECT_CALL(filter_callbacks_.connection_, streamInfo()) - .WillRepeatedly(ReturnRef(stream_info_)); - EXPECT_CALL(stream_info_, - setDynamicMetadata("envoy.filters.network.zookeeper_proxy", MapEq(expected))); - } + using StrStrMap = std::map; - void expectSetDynamicMetadata(const std::map& first, - const std::map& second) { + void expectSetDynamicMetadata(const std::vector& values) { EXPECT_CALL(filter_callbacks_.connection_, streamInfo()) .WillRepeatedly(ReturnRef(stream_info_)); - EXPECT_CALL(stream_info_, setDynamicMetadata(_, _)) - .WillOnce(Invoke([first](const std::string& key, const ProtobufWkt::Struct& obj) -> void { - EXPECT_STREQ(key.c_str(), "envoy.filters.network.zookeeper_proxy"); - protoMapEq(obj, first); - })) - .WillOnce(Invoke([second](const std::string& key, const ProtobufWkt::Struct& obj) -> void { - EXPECT_STREQ(key.c_str(), "envoy.filters.network.zookeeper_proxy"); - protoMapEq(obj, second); - })); + + auto& call = EXPECT_CALL(stream_info_, setDynamicMetadata(_, _)); + + for (const auto& value : values) { + call.WillOnce(Invoke([value](const std::string& key, const ProtobufWkt::Struct& obj) -> void { + EXPECT_STREQ(key.c_str(), "envoy.filters.network.zookeeper_proxy"); + protoMapEq(obj, value); + })); + } } void testCreate(CreateFlags flags, const OpCodes opcode = OpCodes::CREATE) { @@ -405,8 +448,8 @@ class ZooKeeperFilterTest : public testing::Test { } expectSetDynamicMetadata( - {{"opname", opname}, {"path", "/foo"}, {"create_type", createFlagsToString(flags)}}, - {{"bytes", "35"}}); + {{{"opname", opname}, {"path", "/foo"}, {"create_type", createFlagsToString(flags)}}, + {{"bytes", "35"}}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); @@ -428,6 +471,26 @@ class ZooKeeperFilterTest : public testing::Test { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } + void testRequest(Buffer::OwnedImpl& data, const std::vector& metadata_values, + const Stats::Counter& stat, const uint64_t request_bytes) { + expectSetDynamicMetadata(metadata_values); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); + EXPECT_EQ(1UL, stat.value()); + EXPECT_EQ(request_bytes, config_->stats().request_bytes_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + } + + void testResponse(const std::vector& metadata_values, const Stats::Counter& stat, + uint32_t xid = 1000) { + Buffer::OwnedImpl data = encodeResponseHeader(xid, 2000, 0); + + expectSetDynamicMetadata(metadata_values); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); + EXPECT_EQ(1UL, stat.value()); + EXPECT_EQ(20UL, config_->stats().response_bytes_.value()); + EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + } + ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; Stats::IsolatedStoreImpl scope_; @@ -441,11 +504,17 @@ TEST_F(ZooKeeperFilterTest, Connect) { Buffer::OwnedImpl data = encodeConnect(); - expectSetDynamicMetadata({{"opname", "connect"}}, {{"bytes", "32"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().connect_rq_.value()); - EXPECT_EQ(32UL, config_->stats().request_bytes_.value()); + testRequest(data, {{{"opname", "connect"}}, {{"bytes", "32"}}}, config_->stats().connect_rq_, 32); + + data = encodeConnectResponse(); + expectSetDynamicMetadata({{{"opname", "connect_response"}, + {"protocol_version", "0"}, + {"timeout", "10"}, + {"readonly", "0"}}, + {{"bytes", "24"}}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); + EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); + EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -454,12 +523,18 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { Buffer::OwnedImpl data = encodeConnect(true); - expectSetDynamicMetadata({{"opname", "connect_readonly"}}, {{"bytes", "33"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(0UL, config_->stats().connect_rq_.value()); - EXPECT_EQ(1UL, config_->stats().connect_readonly_rq_.value()); - EXPECT_EQ(33UL, config_->stats().request_bytes_.value()); + testRequest(data, {{{"opname", "connect_readonly"}}, {{"bytes", "33"}}}, + config_->stats().connect_readonly_rq_, 33); + + data = encodeConnectResponse(true); + expectSetDynamicMetadata({{{"opname", "connect_response"}, + {"protocol_version", "0"}, + {"timeout", "10"}, + {"readonly", "1"}}, + {{"bytes", "25"}}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); + EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); + EXPECT_EQ(25UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } @@ -515,12 +590,9 @@ TEST_F(ZooKeeperFilterTest, PingRequest) { Buffer::OwnedImpl data = encodePing(); - expectSetDynamicMetadata({{"opname", "ping"}}, {{"bytes", "12"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().ping_rq_.value()); - EXPECT_EQ(12UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "ping"}}, {{"bytes", "12"}}}, config_->stats().ping_rq_, 12); + testResponse({{{"opname", "ping_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().ping_resp_, enumToSignedInt(XidCodes::PING_XID)); } TEST_F(ZooKeeperFilterTest, AuthRequest) { @@ -528,12 +600,10 @@ TEST_F(ZooKeeperFilterTest, AuthRequest) { Buffer::OwnedImpl data = encodeAuth("digest"); - expectSetDynamicMetadata({{"opname", "auth"}}, {{"bytes", "36"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(scope_.counter("test.zookeeper.auth.digest_rq").value(), 1); - EXPECT_EQ(36UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "auth"}}, {{"bytes", "36"}}}, + scope_.counter("test.zookeeper.auth.digest_rq"), 36); + testResponse({{{"opname", "auth_response"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().auth_resp_, enumToSignedInt(XidCodes::AUTH_XID)); } TEST_F(ZooKeeperFilterTest, GetDataRequest) { @@ -541,13 +611,11 @@ TEST_F(ZooKeeperFilterTest, GetDataRequest) { Buffer::OwnedImpl data = encodePathWatch("/foo", true); - expectSetDynamicMetadata({{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}, - {{"bytes", "21"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, + {{{"opname", "getdata"}, {"path", "/foo"}, {"watch", "true"}}, {{"bytes", "21"}}}, + config_->stats().getdata_rq_, 21); + testResponse({{{"opname", "getdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().getdata_resp_); } TEST_F(ZooKeeperFilterTest, GetDataRequestEmptyPath) { @@ -557,37 +625,47 @@ TEST_F(ZooKeeperFilterTest, GetDataRequestEmptyPath) { // by the server. Buffer::OwnedImpl data = encodePathWatch("", true); - expectSetDynamicMetadata({{"opname", "getdata"}, {"path", ""}, {"watch", "true"}}, - {{"bytes", "17"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(17UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(1UL, config_->stats().getdata_rq_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "getdata"}, {"path", ""}, {"watch", "true"}}, {{"bytes", "17"}}}, + config_->stats().getdata_rq_, 17); + testResponse({{{"opname", "getdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().getdata_resp_); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistent) { testCreate(CreateFlags::PERSISTENT); } TEST_F(ZooKeeperFilterTest, CreateRequestPersistentSequential) { testCreate(CreateFlags::PERSISTENT_SEQUENTIAL); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().create_resp_); } TEST_F(ZooKeeperFilterTest, CreateRequestEphemeral) { testCreate(CreateFlags::EPHEMERAL); } TEST_F(ZooKeeperFilterTest, CreateRequestEphemeralSequential) { testCreate(CreateFlags::EPHEMERAL_SEQUENTIAL); + testResponse({{{"opname", "create_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().create_resp_); } TEST_F(ZooKeeperFilterTest, CreateRequestContainer) { testCreate(CreateFlags::CONTAINER, OpCodes::CREATECONTAINER); + testResponse( + {{{"opname", "createcontainer_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().createcontainer_resp_); } TEST_F(ZooKeeperFilterTest, CreateRequestTTL) { testCreate(CreateFlags::PERSISTENT_WITH_TTL, OpCodes::CREATETTL); + testResponse( + {{{"opname", "createttl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().createttl_resp_); } TEST_F(ZooKeeperFilterTest, CreateRequestTTLSequential) { - testCreate(CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL); + testCreate(CreateFlags::PERSISTENT_SEQUENTIAL_WITH_TTL, OpCodes::CREATETTL); + testResponse( + {{{"opname", "createttl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().createttl_resp_); } TEST_F(ZooKeeperFilterTest, CreateRequest2) { @@ -596,13 +674,12 @@ TEST_F(ZooKeeperFilterTest, CreateRequest2) { Buffer::OwnedImpl data = encodeCreateRequest("/foo", "bar", CreateFlags::PERSISTENT, false, enumToSignedInt(OpCodes::CREATE2)); - expectSetDynamicMetadata({{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, - {{"bytes", "35"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().create2_rq_.value()); - EXPECT_EQ(35UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest( + data, + {{{"opname", "create2"}, {"path", "/foo"}, {"create_type", "persistent"}}, {{"bytes", "35"}}}, + config_->stats().create2_rq_, 35); + testResponse({{{"opname", "create2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().create2_resp_); } TEST_F(ZooKeeperFilterTest, SetRequest) { @@ -610,12 +687,10 @@ TEST_F(ZooKeeperFilterTest, SetRequest) { Buffer::OwnedImpl data = encodeSetRequest("/foo", "bar", -1); - expectSetDynamicMetadata({{"opname", "setdata"}, {"path", "/foo"}}, {{"bytes", "31"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); - EXPECT_EQ(31UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "setdata"}, {"path", "/foo"}}, {{"bytes", "31"}}}, + config_->stats().setdata_rq_, 31); + testResponse({{{"opname", "setdata_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().setdata_resp_); } TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { @@ -623,13 +698,12 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest) { Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToSignedInt(OpCodes::GETCHILDREN)); - expectSetDynamicMetadata({{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}, - {{"bytes", "21"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().getchildren_rq_.value()); - EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest( + data, {{{"opname", "getchildren"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}, + config_->stats().getchildren_rq_, 21); + testResponse( + {{{"opname", "getchildren_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().getchildren_resp_); } TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { @@ -637,13 +711,12 @@ TEST_F(ZooKeeperFilterTest, GetChildrenRequest2) { Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToSignedInt(OpCodes::GETCHILDREN2)); - expectSetDynamicMetadata({{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}, - {{"bytes", "21"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().getchildren2_rq_.value()); - EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest( + data, {{{"opname", "getchildren2"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}, + config_->stats().getchildren2_rq_, 21); + testResponse( + {{{"opname", "getchildren2_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().getchildren2_resp_); } TEST_F(ZooKeeperFilterTest, DeleteRequest) { @@ -651,13 +724,11 @@ TEST_F(ZooKeeperFilterTest, DeleteRequest) { Buffer::OwnedImpl data = encodeDeleteRequest("/foo", -1); - expectSetDynamicMetadata({{"opname", "remove"}, {"path", "/foo"}, {"version", "-1"}}, - {{"bytes", "24"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().remove_rq_.value()); - EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, + {{{"opname", "delete"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "24"}}}, + config_->stats().delete_rq_, 24); + testResponse({{{"opname", "delete_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().delete_resp_); } TEST_F(ZooKeeperFilterTest, ExistsRequest) { @@ -665,13 +736,11 @@ TEST_F(ZooKeeperFilterTest, ExistsRequest) { Buffer::OwnedImpl data = encodePathWatch("/foo", false, enumToSignedInt(OpCodes::EXISTS)); - expectSetDynamicMetadata({{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, - {{"bytes", "21"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().exists_rq_.value()); - EXPECT_EQ(21UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, + {{{"opname", "exists"}, {"path", "/foo"}, {"watch", "false"}}, {{"bytes", "21"}}}, + config_->stats().exists_rq_, 21); + testResponse({{{"opname", "exists_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().exists_resp_); } TEST_F(ZooKeeperFilterTest, GetAclRequest) { @@ -679,12 +748,10 @@ TEST_F(ZooKeeperFilterTest, GetAclRequest) { Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GETACL)); - expectSetDynamicMetadata({{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().getacl_rq_.value()); - EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "getacl"}, {"path", "/foo"}}, {{"bytes", "20"}}}, + config_->stats().getacl_rq_, 20); + testResponse({{{"opname", "getacl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().getacl_resp_); } TEST_F(ZooKeeperFilterTest, SetAclRequest) { @@ -692,13 +759,11 @@ TEST_F(ZooKeeperFilterTest, SetAclRequest) { Buffer::OwnedImpl data = encodeSetAclRequest("/foo", "digest", "passwd", -1); - expectSetDynamicMetadata({{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}, - {{"bytes", "52"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().setacl_rq_.value()); - EXPECT_EQ(52UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, + {{{"opname", "setacl"}, {"path", "/foo"}, {"version", "-1"}}, {{"bytes", "52"}}}, + config_->stats().setacl_rq_, 52); + testResponse({{{"opname", "setacl_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().setacl_resp_); } TEST_F(ZooKeeperFilterTest, SyncRequest) { @@ -706,12 +771,10 @@ TEST_F(ZooKeeperFilterTest, SyncRequest) { Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::SYNC)); - expectSetDynamicMetadata({{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().sync_rq_.value()); - EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "sync"}, {"path", "/foo"}}, {{"bytes", "20"}}}, + config_->stats().sync_rq_, 20); + testResponse({{{"opname", "sync_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().sync_resp_); } TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { @@ -719,12 +782,11 @@ TEST_F(ZooKeeperFilterTest, GetEphemeralsRequest) { Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GETEPHEMERALS)); - expectSetDynamicMetadata({{"opname", "getephemerals"}, {"path", "/foo"}}, {{"bytes", "20"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().getephemerals_rq_.value()); - EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "getephemerals"}, {"path", "/foo"}}, {{"bytes", "20"}}}, + config_->stats().getephemerals_rq_, 20); + testResponse( + {{{"opname", "getephemerals_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().getephemerals_resp_); } TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { @@ -732,13 +794,11 @@ TEST_F(ZooKeeperFilterTest, GetAllChildrenNumberRequest) { Buffer::OwnedImpl data = encodePath("/foo", enumToSignedInt(OpCodes::GETALLCHILDRENNUMBER)); - expectSetDynamicMetadata({{"opname", "getallchildrennumber"}, {"path", "/foo"}}, - {{"bytes", "20"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().getallchildrennumber_rq_.value()); - EXPECT_EQ(20UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "getallchildrennumber"}, {"path", "/foo"}}, {{"bytes", "20"}}}, + config_->stats().getallchildrennumber_rq_, 20); + testResponse({{{"opname", "getallchildrennumber_resp"}, {"zxid", "2000"}, {"error", "0"}}, + {{"bytes", "20"}}}, + config_->stats().getallchildrennumber_resp_); } TEST_F(ZooKeeperFilterTest, CheckRequest) { @@ -746,12 +806,15 @@ TEST_F(ZooKeeperFilterTest, CheckRequest) { Buffer::OwnedImpl data = encodePathVersion("/foo", 100, enumToSignedInt(OpCodes::CHECK)); - expectSetDynamicMetadata({{"bytes", "24"}}); + expectSetDynamicMetadata({{{"bytes", "24"}}}); EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); EXPECT_EQ(1UL, config_->stats().check_rq_.value()); EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + + testResponse({{{"opname", "check_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().check_resp_); } TEST_F(ZooKeeperFilterTest, MultiRequest) { @@ -777,6 +840,9 @@ TEST_F(ZooKeeperFilterTest, MultiRequest) { EXPECT_EQ(1UL, config_->stats().setdata_rq_.value()); EXPECT_EQ(1UL, config_->stats().check_rq_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + + testResponse({{{"opname", "multi_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().multi_resp_); } TEST_F(ZooKeeperFilterTest, ReconfigRequest) { @@ -784,12 +850,10 @@ TEST_F(ZooKeeperFilterTest, ReconfigRequest) { Buffer::OwnedImpl data = encodeReconfigRequest("s1", "s2", "s3", 1000); - expectSetDynamicMetadata({{"opname", "reconfig"}}, {{"bytes", "38"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().reconfig_rq_.value()); - EXPECT_EQ(38UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "reconfig"}}, {{"bytes", "38"}}}, config_->stats().reconfig_rq_, + 38); + testResponse({{{"opname", "reconfig_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().reconfig_resp_); } TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { @@ -802,12 +866,11 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequestControlXid) { Buffer::OwnedImpl data = encodeSetWatchesRequest(dataw, existw, childw, enumToSignedInt(XidCodes::SET_WATCHES_XID)); - expectSetDynamicMetadata({{"opname", "setwatches"}}, {{"bytes", "76"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().setwatches_rq_.value()); - EXPECT_EQ(76UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "setwatches"}}, {{"bytes", "76"}}}, + config_->stats().setwatches_rq_, 76); + testResponse( + {{{"opname", "setwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().setwatches_resp_, enumToSignedInt(XidCodes::SET_WATCHES_XID)); } TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { @@ -819,12 +882,11 @@ TEST_F(ZooKeeperFilterTest, SetWatchesRequest) { Buffer::OwnedImpl data = encodeSetWatchesRequest(dataw, existw, childw); - expectSetDynamicMetadata({{"opname", "setwatches"}}, {{"bytes", "76"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().setwatches_rq_.value()); - EXPECT_EQ(76UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "setwatches"}}, {{"bytes", "76"}}}, + config_->stats().setwatches_rq_, 76); + testResponse( + {{{"opname", "setwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().setwatches_resp_); } TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { @@ -833,12 +895,11 @@ TEST_F(ZooKeeperFilterTest, CheckWatchesRequest) { Buffer::OwnedImpl data = encodePathVersion("/foo", enumToSignedInt(WatcherType::CHILDREN), enumToSignedInt(OpCodes::CHECKWATCHES)); - expectSetDynamicMetadata({{"opname", "checkwatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().checkwatches_rq_.value()); - EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "checkwatches"}, {"path", "/foo"}}, {{"bytes", "24"}}}, + config_->stats().checkwatches_rq_, 24); + testResponse( + {{{"opname", "checkwatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().checkwatches_resp_); } TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { @@ -847,12 +908,11 @@ TEST_F(ZooKeeperFilterTest, RemoveWatchesRequest) { Buffer::OwnedImpl data = encodePathVersion("/foo", enumToSignedInt(WatcherType::DATA), enumToSignedInt(OpCodes::REMOVEWATCHES)); - expectSetDynamicMetadata({{"opname", "removewatches"}, {"path", "/foo"}}, {{"bytes", "24"}}); - - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().removewatches_rq_.value()); - EXPECT_EQ(24UL, config_->stats().request_bytes_.value()); - EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + testRequest(data, {{{"opname", "removewatches"}, {"path", "/foo"}}, {{"bytes", "24"}}}, + config_->stats().removewatches_rq_, 24); + testResponse( + {{{"opname", "removewatches_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().removewatches_resp_); } TEST_F(ZooKeeperFilterTest, CloseRequest) { @@ -860,11 +920,24 @@ TEST_F(ZooKeeperFilterTest, CloseRequest) { Buffer::OwnedImpl data = encodeCloseRequest(); - expectSetDynamicMetadata({{"opname", "close"}}, {{"bytes", "12"}}); + testRequest(data, {{{"opname", "close"}}, {{"bytes", "12"}}}, config_->stats().close_rq_, 12); + testResponse({{{"opname", "close_resp"}, {"zxid", "2000"}, {"error", "0"}}, {{"bytes", "20"}}}, + config_->stats().close_resp_); +} - EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onData(data, false)); - EXPECT_EQ(1UL, config_->stats().close_rq_.value()); - EXPECT_EQ(12UL, config_->stats().request_bytes_.value()); +TEST_F(ZooKeeperFilterTest, WatchEvent) { + initialize(); + + Buffer::OwnedImpl data = encodeWatchEvent("/foo", 1, 0); + expectSetDynamicMetadata({{{"opname", "watch_event"}, + {"event_type", "1"}, + {"client_state", "0"}, + {"zxid", "1000"}, + {"error", "0"}}, + {{"bytes", "36"}}}); + EXPECT_EQ(Envoy::Network::FilterStatus::Continue, filter_->onWrite(data, false)); + EXPECT_EQ(1UL, config_->stats().watch_event_.value()); + EXPECT_EQ(36UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } From a30986b441694eee64e136e49fd419c2fe630ccf Mon Sep 17 00:00:00 2001 From: htuch Date: Fri, 2 Aug 2019 14:20:01 -0400 Subject: [PATCH 302/542] security: add @yanavlasov to Envoy security team. (#7811) Signed-off-by: Harvey Tuch --- OWNERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS.md b/OWNERS.md index 816c7eb78c11..8f5898ff1070 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -38,6 +38,7 @@ routing PRs, questions, etc. to the right place. * All maintainers * Piotr Sikora ([PiotrSikora](https://github.com/PiotrSikora)) (piotrsikora@google.com) +* Yan Avlasov ([yanavlasov](https://github.com/yanavlasov)) (yavlasov@google.com) # Emeritus maintainers From fb7384e2d272b23232a4a588129c6079ffae845e Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 2 Aug 2019 12:08:48 -0700 Subject: [PATCH 303/542] ci: roll forward build image (#7809) Description: Fixes post submit issue from #7786 Risk Level: Testing: Docs Changes: Release Notes: Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 2 -- .circleci/config.yml | 2 +- bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD | 2 +- .../toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD | 2 +- bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD | 2 +- bazel/toolchains/configs/versions.bzl | 4 ++-- bazel/toolchains/rbe_toolchains_config.bzl | 2 +- 7 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index bd1d5524a8fa..13818d477f02 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -33,8 +33,6 @@ jobs: BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - # TODO(lizan): remove this once new image is available - IMAGE_ID: b02c73f3ebbaf5c7852fa814eb3d61e841588068 displayName: "Run CI script" - bash: | diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a4680614d63..b3e37128b209 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ executors: description: "A regular build executor based on ubuntu image" docker: # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here. - - image: envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d + - image: envoyproxy/envoy-build:cd8574de783791b3353579b489222bfda74888da resource_class: xlarge working_directory: /source diff --git a/bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD b/bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD index 0d77a8758817..7af5e24e5de1 100644 --- a/bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD +++ b/bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD @@ -43,7 +43,7 @@ platform( remote_execution_properties = """ properties: { name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" } properties { name: "OSFamily" diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD index 01337270ff96..385ea14041f2 100644 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD @@ -43,7 +43,7 @@ platform( remote_execution_properties = """ properties: { name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" } properties { name: "OSFamily" diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD b/bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD index f417961407af..4584bd320cc2 100644 --- a/bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD +++ b/bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD @@ -43,7 +43,7 @@ platform( remote_execution_properties = """ properties: { name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" } properties { name: "OSFamily" diff --git a/bazel/toolchains/configs/versions.bzl b/bazel/toolchains/configs/versions.bzl index 114a706ab69a..5b6fe8e45a38 100644 --- a/bazel/toolchains/configs/versions.bzl +++ b/bazel/toolchains/configs/versions.bzl @@ -6,8 +6,8 @@ toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, cre toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc") _TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1,toolchain_config_spec2] _BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.1": ["clang", "clang_libcxx", "gcc"]} -LATEST = "sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" -CONTAINER_TO_CONFIG_SPEC_NAMES = {"sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28": ["clang", "clang_libcxx", "gcc"]} +LATEST = "sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" +CONTAINER_TO_CONFIG_SPEC_NAMES = {"sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b": ["clang", "clang_libcxx", "gcc"]} _DEFAULT_TOOLCHAIN_CONFIG_SPEC = toolchain_config_spec0 TOOLCHAIN_CONFIG_AUTOGEN_SPEC = struct( bazel_to_config_spec_names_map = _BAZEL_TO_CONFIG_SPEC_NAMES, diff --git a/bazel/toolchains/rbe_toolchains_config.bzl b/bazel/toolchains/rbe_toolchains_config.bzl index c4a34df50929..4aadfd515968 100644 --- a/bazel/toolchains/rbe_toolchains_config.bzl +++ b/bazel/toolchains/rbe_toolchains_config.bzl @@ -4,7 +4,7 @@ load("@envoy//bazel/toolchains:configs/versions.bzl", _generated_toolchain_confi _ENVOY_BUILD_IMAGE_REGISTRY = "gcr.io" _ENVOY_BUILD_IMAGE_REPOSITORY = "envoy-ci/envoy-build" -_ENVOY_BUILD_IMAGE_DIGEST = "sha256:555fb7b0aa578d11852b57c6c14fd54ab4450ad001a9f03bb5c43d5454460c28" +_ENVOY_BUILD_IMAGE_DIGEST = "sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" _CONFIGS_OUTPUT_BASE = "bazel/toolchains/configs" _CLANG_ENV = { From 0957e9cb96d50cae085f4330fa8e8be7d94e587a Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Sat, 3 Aug 2019 01:40:50 +0530 Subject: [PATCH 304/542] config: do not finish initialization on stream disconnection (#7427) Signed-off-by: Rama Chavali --- include/envoy/config/BUILD | 1 + include/envoy/config/grpc_mux.h | 5 +++- include/envoy/config/subscription.h | 15 ++++++++++- .../common/config/delta_subscription_state.cc | 8 +++--- .../config/filesystem_subscription_impl.cc | 6 ++++- source/common/config/grpc_mux_impl.cc | 6 +++-- .../config/grpc_mux_subscription_impl.cc | 25 +++++++++++++------ .../config/grpc_mux_subscription_impl.h | 3 ++- .../common/config/http_subscription_impl.cc | 7 +++--- source/common/router/rds_impl.cc | 3 ++- source/common/router/rds_impl.h | 3 ++- source/common/router/scoped_rds.h | 5 ++-- source/common/router/vhds.cc | 3 ++- source/common/router/vhds.h | 3 ++- source/common/runtime/runtime_impl.cc | 3 ++- source/common/runtime/runtime_impl.h | 3 ++- source/common/secret/sds_api.cc | 2 +- source/common/secret/sds_api.h | 3 ++- source/common/upstream/cds_api_impl.cc | 3 ++- source/common/upstream/cds_api_impl.h | 3 ++- source/common/upstream/eds.cc | 9 +++++-- source/common/upstream/eds.h | 3 ++- source/server/lds_api.cc | 3 ++- source/server/lds_api.h | 3 ++- .../config/config_provider_impl_test.cc | 9 ++++--- .../config/delta_subscription_test_harness.h | 5 ++-- .../filesystem_subscription_impl_test.cc | 3 ++- .../filesystem_subscription_test_harness.h | 2 +- test/common/config/grpc_mux_impl_test.cc | 13 ++++++---- .../config/grpc_subscription_impl_test.cc | 6 +++-- .../config/grpc_subscription_test_harness.h | 5 ++-- .../config/http_subscription_impl_test.cc | 6 +++-- .../config/http_subscription_test_harness.h | 5 ++-- .../config/subscription_factory_impl_test.cc | 4 +-- test/common/router/rds_impl_test.cc | 3 ++- test/common/router/scoped_rds_test.cc | 3 ++- test/common/runtime/runtime_impl_test.cc | 3 ++- test/common/upstream/cds_api_impl_test.cc | 3 ++- test/common/upstream/eds_test.cc | 24 +++++++++++++++--- test/integration/ads_integration_test.cc | 11 ++++++++ test/mocks/config/mocks.h | 6 +++-- test/server/lds_api_test.cc | 3 ++- 42 files changed, 172 insertions(+), 70 deletions(-) diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index ab8c3a0899f6..2b2504620c99 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -33,6 +33,7 @@ envoy_cc_library( name = "grpc_mux_interface", hdrs = ["grpc_mux.h"], deps = [ + ":subscription_interface", "//include/envoy/stats:stats_macros", "//source/common/protobuf", ], diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 3d5bf2c17c70..4888e4e7e716 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -2,6 +2,7 @@ #include "envoy/common/exception.h" #include "envoy/common/pure.h" +#include "envoy/config/subscription.h" #include "envoy/stats/stats_macros.h" #include "common/protobuf/protobuf.h" @@ -42,9 +43,11 @@ class GrpcMuxCallbacks { /** * Called when either the subscription is unable to fetch a config update or when onConfigUpdate * invokes an exception. + * @param reason supplies the update failure reason. * @param e supplies any exception data on why the fetch failed. May be nullptr. */ - virtual void onConfigUpdateFailed(const EnvoyException* e) PURE; + virtual void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) PURE; /** * Obtain the "name" of a v2 API resource in a google.protobuf.Any, e.g. the route config name for diff --git a/include/envoy/config/subscription.h b/include/envoy/config/subscription.h index 822a3eec68a6..bb9861d374ff 100644 --- a/include/envoy/config/subscription.h +++ b/include/envoy/config/subscription.h @@ -13,6 +13,18 @@ namespace Envoy { namespace Config { +/** + * Reason that a config update is failed. + */ +enum class ConfigUpdateFailureReason { + // A connection failure took place and the update could not be fetched. + ConnectionFailure, + // Config fetch timed out. + FetchTimedout, + // Update rejected because there is a problem in applying the update. + UpdateRejected +}; + class SubscriptionCallbacks { public: virtual ~SubscriptionCallbacks() = default; @@ -45,9 +57,10 @@ class SubscriptionCallbacks { /** * Called when either the Subscription is unable to fetch a config update or when onConfigUpdate * invokes an exception. + * @param reason supplies the update failure reason. * @param e supplies any exception data on why the fetch failed. May be nullptr. */ - virtual void onConfigUpdateFailed(const EnvoyException* e) PURE; + virtual void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) PURE; /** * Obtain the "name" of a v2 API resource in a google.protobuf.Any, e.g. the route config name for diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 8aa9d6f0bcbc..a0dcb5f0b66f 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -25,7 +25,8 @@ void DeltaSubscriptionState::setInitFetchTimeout(Event::Dispatcher& dispatcher) if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); - callbacks_.onConfigUpdateFailed(nullptr); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, + nullptr); }); init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } @@ -145,14 +146,15 @@ void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAc disableInitFetchTimeoutTimer(); stats_.update_rejected_.inc(); ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); - callbacks_.onConfigUpdateFailed(&e); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } void DeltaSubscriptionState::handleEstablishmentFailure() { disableInitFetchTimeoutTimer(); stats_.update_failure_.inc(); stats_.update_attempt_.inc(); - callbacks_.onConfigUpdateFailed(nullptr); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + nullptr); } envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequest() { diff --git a/source/common/config/filesystem_subscription_impl.cc b/source/common/config/filesystem_subscription_impl.cc index c9bcaedd3d7d..f14e39533075 100644 --- a/source/common/config/filesystem_subscription_impl.cc +++ b/source/common/config/filesystem_subscription_impl.cc @@ -49,11 +49,15 @@ void FilesystemSubscriptionImpl::refresh() { ENVOY_LOG(warn, "Filesystem config update rejected: {}", e.what()); ENVOY_LOG(debug, "Failed configuration:\n{}", message.DebugString()); stats_.update_rejected_.inc(); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } else { ENVOY_LOG(warn, "Filesystem config update failure: {}", e.what()); stats_.update_failure_.inc(); + // ConnectionFailure is not a meaningful error code for file system but it has been chosen so + // that the behaviour is uniform across all subscription types. + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + &e); } - callbacks_.onConfigUpdateFailed(&e); } } diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index b18bcac597ff..55ffd4ea1ad6 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -192,7 +192,8 @@ void GrpcMuxImpl::onDiscoveryResponse( api_state_[type_url].request_.set_version_info(message->version_info()); } catch (const EnvoyException& e) { for (auto watch : api_state_[type_url].watches_) { - watch->callbacks_.onConfigUpdateFailed(&e); + watch->callbacks_.onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } ::google::rpc::Status* error_detail = api_state_[type_url].request_.mutable_error_detail(); error_detail->set_code(Grpc::Status::GrpcStatus::Internal); @@ -213,7 +214,8 @@ void GrpcMuxImpl::onStreamEstablished() { void GrpcMuxImpl::onEstablishmentFailure() { for (const auto& api_state : api_state_) { for (auto watch : api_state.second.watches_) { - watch->callbacks_.onConfigUpdateFailed(nullptr); + watch->callbacks_.onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); } } } diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc index 8383036d4e52..cb50a314dae8 100644 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ b/source/common/config/grpc_mux_subscription_impl.cc @@ -22,8 +22,8 @@ GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, void GrpcMuxSubscriptionImpl::start(const std::set& resources) { if (init_fetch_timeout_.count() > 0) { init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { - ENVOY_LOG(warn, "gRPC config: initial fetch timed out for {}", type_url_); - callbacks_.onConfigUpdateFailed(nullptr); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, + nullptr); }); init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } @@ -61,18 +61,27 @@ void GrpcMuxSubscriptionImpl::onConfigUpdate( resources.size(), version_info); } -void GrpcMuxSubscriptionImpl::onConfigUpdateFailed(const EnvoyException* e) { - disableInitFetchTimeoutTimer(); - // TODO(htuch): Less fragile signal that this is failure vs. reject. - if (e == nullptr) { +void GrpcMuxSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, + const EnvoyException* e) { + switch (reason) { + case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: stats_.update_failure_.inc(); ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); - } else { + break; + case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: + disableInitFetchTimeoutTimer(); + ENVOY_LOG(warn, "gRPC config: initial fetch timed out for {}", type_url_); + break; + case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: + // We expect Envoy exception to be thrown when update is rejected. + ASSERT(e != nullptr); + disableInitFetchTimeoutTimer(); stats_.update_rejected_.inc(); ENVOY_LOG(warn, "gRPC config for {} rejected: {}", type_url_, e->what()); + break; } stats_.update_attempt_.inc(); - callbacks_.onConfigUpdateFailed(e); + callbacks_.onConfigUpdateFailed(reason, e); } std::string GrpcMuxSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index 10d08ff49f3c..9fb4cd76407c 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -29,7 +29,8 @@ class GrpcMuxSubscriptionImpl : public Subscription, // Config::GrpcMuxCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override; - void onConfigUpdateFailed(const EnvoyException* e) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override; private: diff --git a/source/common/config/http_subscription_impl.cc b/source/common/config/http_subscription_impl.cc index 78a5e6caf9c0..4ee638895578 100644 --- a/source/common/config/http_subscription_impl.cc +++ b/source/common/config/http_subscription_impl.cc @@ -39,7 +39,8 @@ void HttpSubscriptionImpl::start(const std::set& resource_names) { if (init_fetch_timeout_.count() > 0) { init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { ENVOY_LOG(warn, "REST config: initial fetch timed out for", path_); - callbacks_.onConfigUpdateFailed(nullptr); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, + nullptr); }); init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } @@ -87,7 +88,7 @@ void HttpSubscriptionImpl::parseResponse(const Http::Message& response) { } catch (const EnvoyException& e) { ENVOY_LOG(warn, "REST config update rejected: {}", e.what()); stats_.update_rejected_.inc(); - callbacks_.onConfigUpdateFailed(&e); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } } @@ -101,7 +102,7 @@ void HttpSubscriptionImpl::onFetchFailure(const EnvoyException* e) { void HttpSubscriptionImpl::handleFailure(const EnvoyException* e) { stats_.update_failure_.inc(); - callbacks_.onConfigUpdateFailed(e); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, e); } void HttpSubscriptionImpl::disableInitFetchTimeoutTimer() { diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index 781ce9db1521..e316460e0f09 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -143,7 +143,8 @@ void RdsRouteConfigSubscription::onConfigUpdate( } } -void RdsRouteConfigSubscription::onConfigUpdateFailed(const EnvoyException*) { +void RdsRouteConfigSubscription::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException*) { // We need to allow server startup to continue, even if we have a bad // config. init_target_.ready(); diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 1b627cab2711..6ec2c3e16047 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -114,7 +114,8 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string&) override; - void onConfigUpdateFailed(const EnvoyException* e) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, validation_visitor_) diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 725c3ec755ea..4fd6c72c3ff8 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -110,8 +110,9 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const Protobuf::RepeatedPtrField&, const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - void onConfigUpdateFailed(const EnvoyException*) override { - DeltaConfigSubscriptionInstance::onConfigUpdateFailed(); + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException*) override { + ConfigSubscriptionCommonBase::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index 100c9ef6034e..f48449f67897 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -46,7 +46,8 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, *scope_, *this); } -void VhdsSubscription::onConfigUpdateFailed(const EnvoyException*) { +void VhdsSubscription::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException*) { // We need to allow server startup to continue, even if we have a bad // config. init_target_.ready(); diff --git a/source/common/router/vhds.h b/source/common/router/vhds.h index 7c5b370babe1..0959bc6eca2a 100644 --- a/source/common/router/vhds.h +++ b/source/common/router/vhds.h @@ -56,7 +56,8 @@ class VhdsSubscription : Envoy::Config::SubscriptionCallbacks, } void onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override; - void onConfigUpdateFailed(const EnvoyException* e) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, validation_visitor_) diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 6ed82c39a37d..8be5d9a964f8 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -526,7 +526,8 @@ void RtdsSubscription::onConfigUpdate( onConfigUpdate(unwrapped_resource, resources[0].version()); } -void RtdsSubscription::onConfigUpdateFailed(const EnvoyException*) { +void RtdsSubscription::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException*) { // We need to allow server startup to continue, even if we have a bad // config. init_target_.ready(); diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index a41260eca02f..fd1b6d271ade 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -205,7 +205,8 @@ struct RtdsSubscription : Config::SubscriptionCallbacks, Logger::Loggable& removed_resources, const std::string&) override; - void onConfigUpdateFailed(const EnvoyException* e) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, validation_visitor_) diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index 7626433b9a25..1aa00bb8c75b 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -59,7 +59,7 @@ void SdsApi::onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override; - void onConfigUpdateFailed(const EnvoyException* e) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, validation_visitor_) .name(); diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index caea8d9f571e..2d0271f56e58 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -98,7 +98,8 @@ void CdsApiImpl::onConfigUpdate( } } -void CdsApiImpl::onConfigUpdateFailed(const EnvoyException*) { +void CdsApiImpl::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException*) { // We need to allow server startup to continue, even if we have a bad // config. runInitializeCallbackIfAny(); diff --git a/source/common/upstream/cds_api_impl.h b/source/common/upstream/cds_api_impl.h index c9c2a5d1cc78..2825213f3153 100644 --- a/source/common/upstream/cds_api_impl.h +++ b/source/common/upstream/cds_api_impl.h @@ -40,7 +40,8 @@ class CdsApiImpl : public CdsApi, void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) override; - void onConfigUpdateFailed(const EnvoyException* e) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, validation_visitor_).name(); } diff --git a/source/common/upstream/eds.cc b/source/common/upstream/eds.cc index f215651682dd..7205d5aec7d2 100644 --- a/source/common/upstream/eds.cc +++ b/source/common/upstream/eds.cc @@ -251,8 +251,13 @@ bool EdsClusterImpl::updateHostsPerLocality( return false; } -void EdsClusterImpl::onConfigUpdateFailed(const EnvoyException* e) { - UNREFERENCED_PARAMETER(e); +void EdsClusterImpl::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException*) { + // We should not call onPreInitComplete if this method is called because of stream disconnection. + // This might potentially hang the initialization forever, if init_fetch_timeout is disabled. + if (reason == Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure) { + return; + } // We need to allow server startup to continue, even if we have a bad config. onPreInitComplete(); } diff --git a/source/common/upstream/eds.h b/source/common/upstream/eds.h index 9953555db0e1..c7b3ffaba024 100644 --- a/source/common/upstream/eds.h +++ b/source/common/upstream/eds.h @@ -35,7 +35,8 @@ class EdsClusterImpl : public BaseDynamicClusterImpl, Config::SubscriptionCallba const std::string& version_info) override; void onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override; - void onConfigUpdateFailed(const EnvoyException* e) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, validation_visitor_) diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index 6f9c5c8c0ef2..581abba24b71 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -108,7 +108,8 @@ void LdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) override; - void onConfigUpdateFailed(const EnvoyException* e) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource, validation_visitor_).name(); } diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 020d6f5260d8..5e7e65eeec70 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -91,7 +91,8 @@ class DummyConfigSubscription : public ConfigSubscriptionInstance, } // Envoy::Config::SubscriptionCallbacks - void onConfigUpdateFailed(const EnvoyException*) override {} + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException*) override {} // Envoy::Config::SubscriptionCallbacks std::string resourceName(const ProtobufWkt::Any&) override { return ""; } @@ -549,7 +550,8 @@ class DeltaDummyConfigSubscription : public DeltaConfigSubscriptionInstance, const Protobuf::RepeatedPtrField&, const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - void onConfigUpdateFailed(const EnvoyException*) override { + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException*) override { ConfigSubscriptionCommonBase::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any&) override { @@ -725,7 +727,8 @@ TEST_F(DeltaConfigProviderImplTest, DeltaSubscriptionFailure) { timeSystem().setSystemTime(time); const EnvoyException ex(fmt::format("config failure")); // Verify the failure updates the lastUpdated() timestamp. - subscription.onConfigUpdateFailed(&ex); + subscription.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + &ex); EXPECT_EQ(std::chrono::time_point_cast(provider->lastUpdated()) .time_since_epoch(), time); diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index a2431c154bc8..aada398fd30c 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -127,7 +127,8 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { if (accept) { expectSendMessage({}, version); } else { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config", {}); } subscription_->onDiscoveryResponse(std::move(response)); @@ -150,7 +151,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(nullptr)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { diff --git a/test/common/config/filesystem_subscription_impl_test.cc b/test/common/config/filesystem_subscription_impl_test.cc index 36c9815d1bf0..f4ac009e58f3 100644 --- a/test/common/config/filesystem_subscription_impl_test.cc +++ b/test/common/config/filesystem_subscription_impl_test.cc @@ -19,7 +19,8 @@ class FilesystemSubscriptionImplTest : public testing::Test, TEST_F(FilesystemSubscriptionImplTest, BadJsonRecovery) { startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); updateFile(";!@#badjso n"); EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index b34028e40b53..03abeb81e72b 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -81,7 +81,7 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { if (accept) { version_ = version; } else { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); } updateFile(file_json); } diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 7a515a37f7f1..05b8356ceec5 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -147,7 +147,9 @@ TEST_F(GrpcMuxImplTest, ResetStream) { expectSendMessage("baz", {"z"}, ""); grpc_mux_->start(); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)).Times(3); + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) + .Times(3); EXPECT_CALL(random_, random()); ASSERT_TRUE(timer != nullptr); // initialized from dispatcher mock. EXPECT_CALL(*timer, enableTimer(_)); @@ -207,10 +209,11 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { { invalid_response->set_type_url("foo"); invalid_response->mutable_resources()->Add()->set_type_url("bar"); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)).WillOnce(Invoke([](const EnvoyException* e) { - EXPECT_TRUE( - IsSubstring("", "", "bar does not match foo type URL in DiscoveryResponse", e->what())); - })); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)) + .WillOnce(Invoke([](Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) { + EXPECT_TRUE(IsSubstring("", "", "bar does not match foo type URL in DiscoveryResponse", + e->what())); + })); expectSendMessage("foo", {"x", "y"}, "", "", Grpc::Status::GrpcStatus::Internal, fmt::format("bar does not match foo type URL in DiscoveryResponse {}", diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 35991e88717d..563054feea27 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -15,7 +15,8 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { InSequence s; EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); EXPECT_CALL(random_, random()); EXPECT_CALL(*timer_, enableTimer(_)); subscription_->start({"cluster0", "cluster1"}); @@ -37,7 +38,8 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); EXPECT_CALL(*timer_, enableTimer(_)); EXPECT_CALL(random_, random()); subscription_->grpcMux().grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index bcbe968620a1..96a9aae23b6e 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -105,7 +105,8 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage(last_cluster_names_, version); version_ = version; } else { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); expectSendMessage(last_cluster_names_, version_, Grpc::Status::GrpcStatus::Internal, "bad config"); } @@ -132,7 +133,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(nullptr)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { diff --git a/test/common/config/http_subscription_impl_test.cc b/test/common/config/http_subscription_impl_test.cc index 6cc81becb889..114597e2fa41 100644 --- a/test/common/config/http_subscription_impl_test.cc +++ b/test/common/config/http_subscription_impl_test.cc @@ -15,7 +15,8 @@ TEST_F(HttpSubscriptionImplTest, OnRequestReset) { startSubscription({"cluster0", "cluster1"}); EXPECT_CALL(random_gen_, random()).WillOnce(Return(0)); EXPECT_CALL(*timer_, enableTimer(_)); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); http_callbacks_->onFailure(Http::AsyncClient::FailureReason::Reset); EXPECT_TRUE(statsAre(1, 0, 0, 1, 0)); timerTick(); @@ -32,7 +33,8 @@ TEST_F(HttpSubscriptionImplTest, BadJsonRecovery) { message->body() = std::make_unique(";!@#badjso n"); EXPECT_CALL(random_gen_, random()).WillOnce(Return(0)); EXPECT_CALL(*timer_, enableTimer(_)); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); http_callbacks_->onSuccess(std::move(message)); EXPECT_TRUE(statsAre(1, 0, 0, 1, 0)); request_in_progress_ = false; diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index ecf5c056950e..7c890973ab38 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -139,7 +139,8 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { .WillOnce(ThrowOnRejectedConfig(accept)); } if (!accept) { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); } EXPECT_CALL(random_gen_, random()).WillOnce(Return(0)); EXPECT_CALL(*timer_, enableTimer(_)); @@ -152,7 +153,7 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(nullptr)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index c72b02d14608..c5cd9c4c771a 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -186,7 +186,7 @@ TEST_F(SubscriptionFactoryTest, FilesystemSubscription) { auto* watcher = new Filesystem::MockWatcher(); EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillOnce(Return(watcher)); EXPECT_CALL(*watcher, addWatch(test_path, _, _)); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); subscriptionFromConfigSource(config)->start({"foo"}); } @@ -302,7 +302,7 @@ TEST_F(SubscriptionFactoryTest, GrpcSubscription) { })); EXPECT_CALL(random_, random()); EXPECT_CALL(dispatcher_, createTimer_(_)).Times(2); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); subscriptionFromConfigSource(config)->start({"static_cluster"}); } diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 303589eb9750..e84bb925b367 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -256,7 +256,8 @@ TEST_F(RdsImplTest, FailureSubscription) { setup(); EXPECT_CALL(init_watcher_, ready()); - rds_callbacks_->onConfigUpdateFailed({}); + rds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + {}); } class RouteConfigProviderManagerImplTest : public RdsTestBase { diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 1476dac9dce0..30d3a671ed0a 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -177,7 +177,8 @@ TEST_F(ScopedRdsTest, ConfigUpdateFailure) { timeSystem().setSystemTime(time); const EnvoyException ex(fmt::format("config failure")); // Verify the failure updates the lastUpdated() timestamp. - subscription_callbacks_->onConfigUpdateFailed(&ex); + subscription_callbacks_->onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &ex); EXPECT_EQ(std::chrono::time_point_cast(provider_->lastUpdated()) .time_since_epoch(), time); diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 41f80ba3b360..dc3a0c34faee 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -828,7 +828,8 @@ TEST_F(RtdsLoaderImplTest, FailureSubscription) { setup(); EXPECT_CALL(init_watcher_, ready()); - rtds_callbacks_[0]->onConfigUpdateFailed({}); + rtds_callbacks_[0]->onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, {}); EXPECT_EQ(0, store_.counter("runtime.load_error").value()); EXPECT_EQ(1, store_.counter("runtime.load_success").value()); diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index da5d5a532a0c..72c0aeb6fed2 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -355,7 +355,8 @@ TEST_F(CdsApiImplTest, FailureSubscription) { setup(); EXPECT_CALL(initialized_, ready()); - cds_callbacks_->onConfigUpdateFailed({}); + cds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + {}); EXPECT_EQ("", cds_->versionInfo()); } diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index a3909a8ca79c..988df7fc7ee4 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -210,6 +210,14 @@ TEST_F(EdsTest, ValidateFail) { EXPECT_FALSE(initialized_); } +// Validate onConfigUpdate() on stream disconnection. +TEST_F(EdsTest, StreamDisconnection) { + initialize(); + eds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + nullptr); + EXPECT_FALSE(initialized_); +} + // Validate that onConfigUpdate() with unexpected cluster names rejects config. TEST_F(EdsTest, OnConfigUpdateWrongName) { envoy::api::v2::ClusterLoadAssignment cluster_load_assignment; @@ -217,8 +225,12 @@ TEST_F(EdsTest, OnConfigUpdateWrongName) { Protobuf::RepeatedPtrField resources; resources.Add()->PackFrom(cluster_load_assignment); initialize(); - EXPECT_THROW(eds_callbacks_->onConfigUpdate(resources, ""), EnvoyException); - eds_callbacks_->onConfigUpdateFailed(nullptr); + try { + eds_callbacks_->onConfigUpdate(resources, ""); + } catch (const EnvoyException& e) { + eds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, + &e); + } EXPECT_TRUE(initialized_); } @@ -241,8 +253,12 @@ TEST_F(EdsTest, OnConfigUpdateWrongSize) { Protobuf::RepeatedPtrField resources; resources.Add()->PackFrom(cluster_load_assignment); resources.Add()->PackFrom(cluster_load_assignment); - EXPECT_THROW(eds_callbacks_->onConfigUpdate(resources, ""), EnvoyException); - eds_callbacks_->onConfigUpdateFailed(nullptr); + try { + eds_callbacks_->onConfigUpdate(resources, ""); + } catch (const EnvoyException& e) { + eds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, + &e); + } EXPECT_TRUE(initialized_); } diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index e87fdff3f6f4..3954f0aad69f 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -394,6 +394,17 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { // i,e. no named EDS response. test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + // Disconnect and reconnect the stream. + xds_stream_->finishGrpcStream(Grpc::Status::Internal); + + AssertionResult result = xds_connection_->waitForNewStream(*dispatcher_, xds_stream_); + RELEASE_ASSERT(result, result.message()); + xds_stream_->startGrpcStream(); + + // Envoy will not finish warming of the second cluster because of the missing load assignments + // i,e. no named EDS response even after disconnect and reconnect. + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + // Finish warming the second cluster. sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index ee7c9291782b..b579cb56b722 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -37,7 +37,8 @@ template class MockSubscriptionCallbacks : public Subscript void(const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info)); - MOCK_METHOD1_T(onConfigUpdateFailed, void(const EnvoyException* e)); + MOCK_METHOD2_T(onConfigUpdateFailed, + void(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e)); MOCK_METHOD1_T(resourceName, std::string(const ProtobufWkt::Any& resource)); }; @@ -93,7 +94,8 @@ class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { MOCK_METHOD2(onConfigUpdate, void(const Protobuf::RepeatedPtrField& resources, const std::string& version_info)); - MOCK_METHOD1(onConfigUpdateFailed, void(const EnvoyException* e)); + MOCK_METHOD2(onConfigUpdateFailed, + void(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e)); MOCK_METHOD1(resourceName, std::string(const ProtobufWkt::Any& resource)); }; diff --git a/test/server/lds_api_test.cc b/test/server/lds_api_test.cc index efbd1c5f8f24..3f54aed53551 100644 --- a/test/server/lds_api_test.cc +++ b/test/server/lds_api_test.cc @@ -412,7 +412,8 @@ TEST_F(LdsApiTest, FailureSubscription) { setup(); EXPECT_CALL(init_watcher_, ready()); - lds_callbacks_->onConfigUpdateFailed({}); + lds_callbacks_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + {}); EXPECT_EQ("", lds_->versionInfo()); } From 662eccc240fef26885419013cef9bbcd18e6675e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Sat, 3 Aug 2019 10:57:15 -0400 Subject: [PATCH 305/542] Fix comment (#7816) Make it easier to grok. Signed-off-by: Raul Gutierrez Segales --- source/extensions/filters/network/zookeeper_proxy/decoder.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.cc b/source/extensions/filters/network/zookeeper_proxy/decoder.cc index 123eb21f990f..2350cbb8c941 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.cc @@ -457,8 +457,8 @@ void DecoderImpl::decode(Buffer::Instance& data, DecodeType dtype) { // by the message. // // Note: we need to keep two cursors — offset and helper_'s internal one — because - // a buffer may contain multiple messages, so offset is global and helper_'s - // internal cursor is reset for each individual message. + // a buffer may contain multiple messages, so offset is global while helper_'s + // internal cursor gets reset for each individual message. helper_.reset(); const uint64_t current = offset; From 2ca5b268b34808b74688e3f247efeb2367232041 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Sat, 3 Aug 2019 07:57:55 -0700 Subject: [PATCH 306/542] clang-tidy: modernize-loop-convert (#7790) Signed-off-by: Derek Argueta --- .clang-tidy | 1 + source/common/grpc/google_grpc_utils.cc | 4 ++-- source/common/http/codec_helper.h | 6 +++--- source/common/http/codes.cc | 4 ++-- source/extensions/filters/common/rbac/engine_impl.cc | 6 +++--- .../filters/network/mysql_proxy/mysql_filter.cc | 8 ++++---- .../filters/network/redis_proxy/conn_pool_impl.cc | 4 ++-- .../previous_priorities/previous_priorities.cc | 4 ++-- source/extensions/stat_sinks/hystrix/hystrix.cc | 5 ++--- .../extensions/tracers/zipkin/zipkin_core_types.cc | 12 ++++++------ source/server/http/admin.cc | 4 ++-- source/server/options_impl.cc | 4 ++-- test/common/network/lc_trie_test.cc | 4 ++-- test/common/tcp/conn_pool_test.cc | 4 ++-- test/common/upstream/outlier_detection_impl_test.cc | 4 ++-- test/common/upstream/upstream_impl_test.cc | 8 ++++---- .../clusters/redis/redis_cluster_integration_test.cc | 6 +++--- .../filters/http/jwt_authn/group_verifier_test.cc | 8 ++++---- .../redis_proxy/redis_proxy_integration_test.cc | 6 +++--- test/integration/fake_upstream.cc | 4 ++-- test/integration/http2_integration_test.cc | 6 +++--- test/server/config_validation/server_test.cc | 4 ++-- 22 files changed, 58 insertions(+), 58 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 69c3b70a671c..248d7e642762 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -21,6 +21,7 @@ WarningsAsErrors: 'abseil-duration-*, bugprone-use-after-move, clang-analyzer-core.DivideZero, modernize-deprecated-headers, + modernize-loop-convert, modernize-make-shared, modernize-make-unique, modernize-return-braced-init-list, diff --git a/source/common/grpc/google_grpc_utils.cc b/source/common/grpc/google_grpc_utils.cc index cba62fc4722a..21fa7ddf29e9 100644 --- a/source/common/grpc/google_grpc_utils.cc +++ b/source/common/grpc/google_grpc_utils.cc @@ -87,8 +87,8 @@ Buffer::InstancePtr GoogleGrpcUtils::makeBufferInstance(const grpc::ByteBuffer& return nullptr; } - for (size_t i = 0; i < slices.size(); i++) { - buffer->addBufferFragment(*new GrpcSliceBufferFragmentImpl(std::move(slices[i]))); + for (auto& slice : slices) { + buffer->addBufferFragment(*new GrpcSliceBufferFragmentImpl(std::move(slice))); } return buffer; } diff --git a/source/common/http/codec_helper.h b/source/common/http/codec_helper.h index 4d2b456bbcc9..3cc6d5bd6580 100644 --- a/source/common/http/codec_helper.h +++ b/source/common/http/codec_helper.h @@ -73,9 +73,9 @@ class StreamCallbackHelper { // removed multiple times. // The vector may not be safely resized without making sure the run.*Callbacks() helper // functions above still handle removeCallbacks_() calls mid-loop. - for (size_t i = 0; i < callbacks_.size(); i++) { - if (callbacks_[i] == &callbacks) { - callbacks_[i] = nullptr; + for (auto& callback : callbacks_) { + if (callback == &callbacks) { + callback = nullptr; return; } } diff --git a/source/common/http/codes.cc b/source/common/http/codes.cc index ebe4819b077c..746b38a45c5e 100644 --- a/source/common/http/codes.cc +++ b/source/common/http/codes.cc @@ -34,8 +34,8 @@ CodeStatsImpl::CodeStatsImpl(Stats::SymbolTable& symbol_table) vcluster_(stat_name_pool_.add("vcluster")), vhost_(stat_name_pool_.add("vhost")), zone_(stat_name_pool_.add("zone")) { - for (uint32_t i = 0; i < NumHttpCodes; ++i) { - rc_stat_names_[i] = nullptr; + for (auto& rc_stat_name : rc_stat_names_) { + rc_stat_name = nullptr; } // Pre-allocate response codes 200, 404, and 503, as those seem quite likely. diff --git a/source/extensions/filters/common/rbac/engine_impl.cc b/source/extensions/filters/common/rbac/engine_impl.cc index c2b1e05cd2a1..a35697d74d5f 100644 --- a/source/extensions/filters/common/rbac/engine_impl.cc +++ b/source/extensions/filters/common/rbac/engine_impl.cc @@ -23,11 +23,11 @@ bool RoleBasedAccessControlEngineImpl::allowed(const Network::Connection& connec std::string* effective_policy_id) const { bool matched = false; - for (auto it = policies_.begin(); it != policies_.end(); it++) { - if (it->second.matches(connection, headers, metadata)) { + for (const auto& policy : policies_) { + if (policy.second.matches(connection, headers, metadata)) { matched = true; if (effective_policy_id != nullptr) { - *effective_policy_id = it->first; + *effective_policy_id = policy.first; } break; } diff --git a/source/extensions/filters/network/mysql_proxy/mysql_filter.cc b/source/extensions/filters/network/mysql_proxy/mysql_filter.cc index 75f14d2d1de5..aae1a0ecd292 100644 --- a/source/extensions/filters/network/mysql_proxy/mysql_filter.cc +++ b/source/extensions/filters/network/mysql_proxy/mysql_filter.cc @@ -128,10 +128,10 @@ void MySQLFilter::onCommand(Command& command) { } hsql::TableAccessMap table_access_map; result.getStatement(i)->tablesAccessed(table_access_map); - for (auto it = table_access_map.begin(); it != table_access_map.end(); ++it) { - auto& operations = *fields[it->first].mutable_list_value(); - for (auto ot = it->second.begin(); ot != it->second.end(); ++ot) { - operations.add_values()->set_string_value(*ot); + for (auto& it : table_access_map) { + auto& operations = *fields[it.first].mutable_list_value(); + for (const auto& ot : it.second) { + operations.add_values()->set_string_value(ot); } } } diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index 3f998b191efb..44f2634aa89a 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -98,8 +98,8 @@ void InstanceImpl::ThreadLocalPool::onClusterAddOrUpdateNonVirtual( }); ASSERT(host_address_map_.empty()); - for (uint32_t i = 0; i < cluster_->prioritySet().hostSetsPerPriority().size(); i++) { - for (auto& host : cluster_->prioritySet().hostSetsPerPriority()[i]->hosts()) { + for (const auto& i : cluster_->prioritySet().hostSetsPerPriority()) { + for (auto& host : i->hosts()) { host_address_map_[host->address()->asString()] = host; } } diff --git a/source/extensions/retry/priority/previous_priorities/previous_priorities.cc b/source/extensions/retry/priority/previous_priorities/previous_priorities.cc index 701567e621f6..7a1ec35d5263 100644 --- a/source/extensions/retry/priority/previous_priorities/previous_priorities.cc +++ b/source/extensions/retry/priority/previous_priorities/previous_priorities.cc @@ -47,8 +47,8 @@ bool PreviousPrioritiesRetryPriority::adjustForAttemptedPriorities( // This allows us to fall back to the unmodified priority load when we run out of priorities // instead of failing to route requests. if (total_availability == 0) { - for (size_t i = 0; i < excluded_priorities_.size(); ++i) { - excluded_priorities_[i] = false; + for (auto&& excluded_priority : excluded_priorities_) { + excluded_priority = false; } attempted_priorities_.clear(); total_availability = diff --git a/source/extensions/stat_sinks/hystrix/hystrix.cc b/source/extensions/stat_sinks/hystrix/hystrix.cc index ab54929579ab..a0e32b90a5ea 100644 --- a/source/extensions/stat_sinks/hystrix/hystrix.cc +++ b/source/extensions/stat_sinks/hystrix/hystrix.cc @@ -39,9 +39,8 @@ void ClusterStatsCache::printToStream(std::stringstream& out_str) { void ClusterStatsCache::printRollingWindow(absl::string_view name, RollingWindow rolling_window, std::stringstream& out_str) { out_str << name << " | "; - for (auto specific_stat_vec_itr = rolling_window.begin(); - specific_stat_vec_itr != rolling_window.end(); ++specific_stat_vec_itr) { - out_str << *specific_stat_vec_itr << " | "; + for (uint64_t& specific_stat_vec_itr : rolling_window) { + out_str << specific_stat_vec_itr << " | "; } out_str << std::endl; } diff --git a/source/extensions/tracers/zipkin/zipkin_core_types.cc b/source/extensions/tracers/zipkin/zipkin_core_types.cc index e99e87312077..1730e3cd75cd 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_types.cc +++ b/source/extensions/tracers/zipkin/zipkin_core_types.cc @@ -166,8 +166,8 @@ Span::Span(const Span& span) : time_source_(span.time_source_) { } void Span::setServiceName(const std::string& service_name) { - for (auto it = annotations_.begin(); it != annotations_.end(); it++) { - it->changeEndpointServiceName(service_name); + for (auto& annotation : annotations_) { + annotation.changeEndpointServiceName(service_name); } } @@ -203,15 +203,15 @@ const std::string Span::toJson() { std::vector annotation_json_vector; - for (auto it = annotations_.begin(); it != annotations_.end(); it++) { - annotation_json_vector.push_back(it->toJson()); + for (auto& annotation : annotations_) { + annotation_json_vector.push_back(annotation.toJson()); } Util::addArrayToJson(json_string, annotation_json_vector, ZipkinJsonFieldNames::get().SPAN_ANNOTATIONS); std::vector binary_annotation_json_vector; - for (auto it = binary_annotations_.begin(); it != binary_annotations_.end(); it++) { - binary_annotation_json_vector.push_back(it->toJson()); + for (auto& binary_annotation : binary_annotations_) { + binary_annotation_json_vector.push_back(binary_annotation.toJson()); } Util::addArrayToJson(json_string, binary_annotation_json_vector, ZipkinJsonFieldNames::get().SPAN_BINARY_ANNOTATIONS); diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index b7e8a22cebbb..5faacc1eb76d 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -672,8 +672,8 @@ Http::Code AdminImpl::handlerLogging(absl::string_view url, Http::HeaderMap&, response.add("usage: /logging?= (change single level)\n"); response.add("usage: /logging?level= (change all levels)\n"); response.add("levels: "); - for (size_t i = 0; i < ARRAY_SIZE(spdlog::level::level_string_views); i++) { - response.add(fmt::format("{} ", spdlog::level::level_string_views[i])); + for (auto level_string_view : spdlog::level::level_string_views) { + response.add(fmt::format("{} ", level_string_view)); } response.add("\n"); diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc index 37befc6dc084..9db443e9848f 100644 --- a/source/server/options_impl.cc +++ b/source/server/options_impl.cc @@ -23,8 +23,8 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv, spdlog::level::level_enum default_log_level) : signal_handling_enabled_(true) { std::string log_levels_string = "Log levels: "; - for (size_t i = 0; i < ARRAY_SIZE(spdlog::level::level_string_views); i++) { - log_levels_string += fmt::format("[{}]", spdlog::level::level_string_views[i]); + for (auto level_string_view : spdlog::level::level_string_views) { + log_levels_string += fmt::format("[{}]", level_string_view); } log_levels_string += fmt::format("\nDefault is [{}]", spdlog::level::level_string_views[default_log_level]); diff --git a/test/common/network/lc_trie_test.cc b/test/common/network/lc_trie_test.cc index 75b93e2dee1a..e0cf9ac43532 100644 --- a/test/common/network/lc_trie_test.cc +++ b/test/common/network/lc_trie_test.cc @@ -21,8 +21,8 @@ class LcTrieTest : public testing::Test { for (size_t i = 0; i < cidr_range_strings.size(); i++) { std::pair> ip_tags; ip_tags.first = fmt::format("tag_{0}", i); - for (size_t j = 0; j < cidr_range_strings[i].size(); j++) { - ip_tags.second.push_back(Address::CidrRange::create(cidr_range_strings[i][j])); + for (const auto& j : cidr_range_strings[i]) { + ip_tags.second.push_back(Address::CidrRange::create(j)); } output.push_back(ip_tags); } diff --git a/test/common/tcp/conn_pool_test.cc b/test/common/tcp/conn_pool_test.cc index 5cf14a40a0f5..5bcb19698b39 100644 --- a/test/common/tcp/conn_pool_test.cc +++ b/test/common/tcp/conn_pool_test.cc @@ -126,8 +126,8 @@ class ConnPoolImplForTest : public ConnPoolImpl { protected: void onConnReleased(ConnPoolImpl::ActiveConn& conn) override { - for (auto i = test_conns_.begin(); i != test_conns_.end(); i++) { - if (conn.conn_.get() == i->connection_) { + for (auto& test_conn : test_conns_) { + if (conn.conn_.get() == test_conn.connection_) { onConnReleasedForTest(); break; } diff --git a/test/common/upstream/outlier_detection_impl_test.cc b/test/common/upstream/outlier_detection_impl_test.cc index c7a3cb08abf3..c585a5509855 100644 --- a/test/common/upstream/outlier_detection_impl_test.cc +++ b/test/common/upstream/outlier_detection_impl_test.cc @@ -87,8 +87,8 @@ class OutlierDetectorImplTest : public testing::Test { } template void loadRq(HostVector& hosts, int num_rq, T code) { - for (uint64_t i = 0; i < hosts.size(); i++) { - loadRq(hosts[i], num_rq, code); + for (auto& host : hosts) { + loadRq(host, num_rq, code); } } diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index d221c6d838ff..d36f1ec56d20 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -400,10 +400,10 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalActiveHealthSkipped) { const auto& hosts = cluster.prioritySet().hostSetsPerPriority()[0]->hosts(); EXPECT_EQ(2UL, hosts.size()); - for (size_t i = 0; i < hosts.size(); ++i) { - EXPECT_TRUE(hosts[i]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); - hosts[i]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); - hosts[i]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + for (const auto& host : hosts) { + EXPECT_TRUE(host->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); + host->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); + host->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); } } diff --git a/test/extensions/clusters/redis/redis_cluster_integration_test.cc b/test/extensions/clusters/redis/redis_cluster_integration_test.cc index 546fee9a81c4..077dc3c9d92e 100644 --- a/test/extensions/clusters/redis/redis_cluster_integration_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_integration_test.cc @@ -72,9 +72,9 @@ std::string makeBulkStringArray(std::vector&& command_strings) { std::stringstream result; result << "*" << command_strings.size() << "\r\n"; - for (uint64_t i = 0; i < command_strings.size(); i++) { - result << "$" << command_strings[i].size() << "\r\n"; - result << command_strings[i] << "\r\n"; + for (auto& command_string : command_strings) { + result << "$" << command_string.size() << "\r\n"; + result << command_string << "\r\n"; } return result.str(); diff --git a/test/extensions/filters/http/jwt_authn/group_verifier_test.cc b/test/extensions/filters/http/jwt_authn/group_verifier_test.cc index 0a9ca7f0a6bb..0bc1f57ddd05 100644 --- a/test/extensions/filters/http/jwt_authn/group_verifier_test.cc +++ b/test/extensions/filters/http/jwt_authn/group_verifier_test.cc @@ -113,16 +113,16 @@ class GroupVerifierTest : public testing::Test { std::unordered_map createAsyncMockAuthsAndVerifier(const std::vector& providers) { std::unordered_map callbacks; - for (std::size_t i = 0; i < providers.size(); ++i) { + for (const auto& provider : providers) { auto mock_auth = std::make_unique(); EXPECT_CALL(*mock_auth, doVerify(_, _, _, _)) .WillOnce(Invoke( - [&callbacks, iss = providers[i]](Http::HeaderMap&, std::vector*, - SetPayloadCallback, AuthenticatorCallback callback) { + [&callbacks, iss = provider](Http::HeaderMap&, std::vector*, + SetPayloadCallback, AuthenticatorCallback callback) { callbacks[iss] = std::move(callback); })); EXPECT_CALL(*mock_auth, onDestroy()).Times(1); - mock_auths_[providers[i]] = std::move(mock_auth); + mock_auths_[provider] = std::move(mock_auth); } createVerifier(); return callbacks; diff --git a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc index 444bbc4c1e47..c9f8c2ce10f7 100644 --- a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc +++ b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc @@ -264,9 +264,9 @@ std::string makeBulkStringArray(std::vector&& command_strings) { std::stringstream result; result << "*" << command_strings.size() << "\r\n"; - for (uint64_t i = 0; i < command_strings.size(); i++) { - result << "$" << command_strings[i].size() << "\r\n"; - result << command_strings[i] << "\r\n"; + for (auto& command_string : command_strings) { + result << "$" << command_string.size() << "\r\n"; + result << command_string << "\r\n"; } return result.str(); diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index e6f9ad9d4b7b..8fb6c8f01b6c 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -486,8 +486,8 @@ FakeUpstream::waitForHttpConnection(Event::Dispatcher& client_dispatcher, Event::TestTimeSystem& time_system = upstreams[0]->timeSystem(); auto end_time = time_system.monotonicTime() + timeout; while (time_system.monotonicTime() < end_time) { - for (auto it = upstreams.begin(); it != upstreams.end(); ++it) { - FakeUpstream& upstream = **it; + for (auto& it : upstreams) { + FakeUpstream& upstream = *it; Thread::ReleasableLockGuard lock(upstream.lock_); if (upstream.new_connections_.empty()) { time_system.waitFor(upstream.lock_, upstream.new_connection_event_, 5ms); diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index f93b8ea2df84..f59c55e59cf1 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1182,10 +1182,10 @@ Http2RingHashIntegrationTest::~Http2RingHashIntegrationTest() { codec_client_->close(); codec_client_ = nullptr; } - for (auto it = fake_upstream_connections_.begin(); it != fake_upstream_connections_.end(); ++it) { - AssertionResult result = (*it)->close(); + for (auto& fake_upstream_connection : fake_upstream_connections_) { + AssertionResult result = fake_upstream_connection->close(); RELEASE_ASSERT(result, result.message()); - result = (*it)->waitForDisconnect(); + result = fake_upstream_connection->waitForDisconnect(); RELEASE_ASSERT(result, result.message()); } } diff --git a/test/server/config_validation/server_test.cc b/test/server/config_validation/server_test.cc index 89b72ac0342b..ea1d201efce2 100644 --- a/test/server/config_validation/server_test.cc +++ b/test/server/config_validation/server_test.cc @@ -42,8 +42,8 @@ class ValidationServerTest_1 : public ValidationServerTest { auto files = TestUtility::listFiles(ValidationServerTest::directory_, false); // Strip directory part. options_ adds it for each test. - for (std::vector::iterator it = files.begin(); it != files.end(); it++) { - (*it) = it->substr(directory_.length() + 1); + for (auto& file : files) { + file = file.substr(directory_.length() + 1); } return files; } From f3121b0d02f93fe6d86d7ac2e275c4071ee2cc0a Mon Sep 17 00:00:00 2001 From: Tal Nordan Date: Sat, 3 Aug 2019 08:01:53 -0700 Subject: [PATCH 307/542] docs: fix minor style issue in tracing.rst (#7821) Remove an unnecessary bullet list to make the link to the v2 API reference inlined. Signed-off-by: Tal Nordan --- docs/root/intro/arch_overview/observability/tracing.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/root/intro/arch_overview/observability/tracing.rst b/docs/root/intro/arch_overview/observability/tracing.rst index 47c635d5cb43..bd8016424d9b 100644 --- a/docs/root/intro/arch_overview/observability/tracing.rst +++ b/docs/root/intro/arch_overview/observability/tracing.rst @@ -107,7 +107,5 @@ Envoy automatically sends spans to tracing collectors. Depending on the tracing multiple spans are stitched together using common information such as the globally unique request ID :ref:`config_http_conn_man_headers_x-request-id` (LightStep) or the trace ID configuration (Zipkin and Datadog). See - -* :ref:`v2 API reference ` - +:ref:`v2 API reference ` for more information on how to setup tracing in Envoy. From 56d03e0bd106e933ece6201439bbf4d4208f39a5 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sat, 3 Aug 2019 08:04:56 -0700 Subject: [PATCH 308/542] ci: compile_time_options in Azure RBE (#7817) Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 7 +++- .bazelrc | 22 +++++++--- .circleci/config.yml | 10 ----- .../configs/gcc/bazel_0.28.1/cc/BUILD | 3 +- bazel/toolchains/configs/versions.bzl | 2 +- bazel/toolchains/rbe_toolchains_config.bzl | 2 +- ci/build_setup.sh | 42 ++++++++++++++----- ci/do_ci.sh | 8 +++- ci/run_envoy_docker.sh | 2 +- 9 files changed, 64 insertions(+), 34 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index 13818d477f02..842e1c992e74 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -6,6 +6,10 @@ jobs: CI_TARGET: 'bazel.release' tsan: CI_TARGET: 'bazel.tsan' + gcc: + CI_TARGET: 'bazel.gcc' + compile_time_options: + CI_TARGET: 'bazel.compile_time_options' dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. timeoutInMinutes: 360 pool: @@ -29,7 +33,8 @@ jobs: workingDirectory: $(Build.SourcesDirectory) env: ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-clang --config=remote-ci --jobs=100 --curses=no" + ENVOY_RBE: "true" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote --jobs=100 --curses=no" BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) diff --git a/.bazelrc b/.bazelrc index b5a66c4f69b7..f0e8b4be98b2 100644 --- a/.bazelrc +++ b/.bazelrc @@ -92,10 +92,20 @@ build --test_env=HEAPCHECK=normal --test_env=PPROF_PATH build:rbe-toolchain --host_platform=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform build:rbe-toolchain --platforms=@envoy//bazel/toolchains:rbe_ubuntu_clang_platform build:rbe-toolchain --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 -build:rbe-toolchain --crosstool_top=@rbe_ubuntu_clang//cc:toolchain -build:rbe-toolchain --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain -build:rbe-toolchain --linkopt=-fuse-ld=lld -build:rbe-toolchain --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin + +build:rbe-toolchain-clang --config=rbe-toolchain +build:rbe-toolchain-clang --crosstool_top=@rbe_ubuntu_clang//cc:toolchain +build:rbe-toolchain-clang --extra_toolchains=@rbe_ubuntu_clang//config:cc-toolchain +build:rbe-toolchain-clang --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin + +build:rbe-toolchain-clang-libc++ --config=rbe-toolchain +build:rbe-toolchain-clang-libc++ --crosstool_top=@rbe_ubuntu_clang_libcxx//cc:toolchain +build:rbe-toolchain-clang-libc++ --extra_toolchains=@rbe_ubuntu_clang_libcxx//config:cc-toolchain +build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin + +build:rbe-toolchain-gcc --config=rbe-toolchain +build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain +build:rbe-toolchain-gcc --extra_toolchains=@rbe_ubuntu_gcc//config:cc-toolchain build:remote --spawn_strategy=remote,sandboxed,local build:remote --strategy=Javac=remote,sandboxed,local @@ -109,7 +119,7 @@ build:remote --experimental_remote_download_outputs=toplevel test:remote --experimental_remote_download_outputs=minimal build:remote-clang --config=remote -build:remote-clang --config=rbe-toolchain +build:remote-clang --config=rbe-toolchain-clang # Docker sandbox build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d @@ -122,7 +132,7 @@ build:docker-sandbox --experimental_docker_verbose build:docker-sandbox --experimental_enable_docker_sandbox build:docker-clang --config=docker-sandbox -build:docker-clang --config=rbe-toolchain +build:docker-clang --config=rbe-toolchain-clang # CI configurations build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com diff --git a/.circleci/config.yml b/.circleci/config.yml index b3e37128b209..87236193005f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,15 +54,6 @@ jobs: path: /build/envoy/generated destination: / - compile_time_options: - executor: ubuntu-build - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - run: ci/do_circle_ci.sh bazel.compile_time_options - - store_artifacts: - path: /build/envoy/generated - destination: / api: executor: ubuntu-build steps: @@ -156,7 +147,6 @@ workflows: tags: only: /^v.*/ - asan - - compile_time_options - api - filter_example_mirror - coverage diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD index 0390586c9ccd..e936d4b91522 100755 --- a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD +++ b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD @@ -116,8 +116,7 @@ cc_toolchain_config( "-B/usr/bin", "-pass-exit-codes", "-lm", - "-static-libgcc", - "-fuse-ld=lld"], + "-static-libgcc"], link_libs = ["-l:libstdc++.a"], opt_link_flags = ["-Wl,--gc-sections"], unfiltered_compile_flags = ["-fno-canonical-system-headers", diff --git a/bazel/toolchains/configs/versions.bzl b/bazel/toolchains/configs/versions.bzl index 5b6fe8e45a38..b7fee4d50322 100644 --- a/bazel/toolchains/configs/versions.bzl +++ b/bazel/toolchains/configs/versions.bzl @@ -3,7 +3,7 @@ """Definitions to be used in rbe_repo attr of an rbe_autoconf rule """ toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "clang") toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx") -toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc") +toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc") _TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1,toolchain_config_spec2] _BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.1": ["clang", "clang_libcxx", "gcc"]} LATEST = "sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" diff --git a/bazel/toolchains/rbe_toolchains_config.bzl b/bazel/toolchains/rbe_toolchains_config.bzl index 4aadfd515968..fd7210db1f87 100644 --- a/bazel/toolchains/rbe_toolchains_config.bzl +++ b/bazel/toolchains/rbe_toolchains_config.bzl @@ -28,7 +28,7 @@ _CLANG_LIBCXX_ENV = dicts.add(_CLANG_ENV, { _GCC_ENV = { "BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", - "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", + "BAZEL_LINKOPTS": "-lm:-static-libgcc", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", diff --git a/ci/build_setup.sh b/ci/build_setup.sh index d397349da3eb..834ba5db9c9d 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -11,19 +11,41 @@ export PPROF_PATH=/thirdparty_build/bin/pprof echo "ENVOY_SRCDIR=${ENVOY_SRCDIR}" function setup_gcc_toolchain() { - export CC=gcc - export CXX=g++ - export BAZEL_COMPILER=gcc - echo "$CC/$CXX toolchain configured" + if [[ -z "${ENVOY_RBE}" ]]; then + export CC=gcc + export CXX=g++ + export BAZEL_COMPILER=gcc + echo "$CC/$CXX toolchain configured" + else + export BAZEL_BUILD_OPTIONS="--config=rbe-toolchain-gcc ${BAZEL_BUILD_OPTIONS}" + fi } function setup_clang_toolchain() { - export PATH=/usr/lib/llvm-8/bin:$PATH - export CC=clang - export CXX=clang++ - export BAZEL_COMPILER=clang - export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer - echo "$CC/$CXX toolchain configured" + if [[ -z "${ENVOY_RBE}" ]]; then + export PATH=/usr/lib/llvm-8/bin:$PATH + export CC=clang + export CXX=clang++ + export BAZEL_COMPILER=clang + export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer + echo "$CC/$CXX toolchain configured" + else + export BAZEL_BUILD_OPTIONS="--config=rbe-toolchain-clang ${BAZEL_BUILD_OPTIONS}" + fi +} + +function setup_clang_libcxx_toolchain() { + if [[ -z "${ENVOY_RBE}" ]]; then + export PATH=/usr/lib/llvm-8/bin:$PATH + export CC=clang + export CXX=clang++ + export BAZEL_COMPILER=clang + export ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-8/bin/llvm-symbolizer + export BAZEL_BUILD_OPTIONS="--config=libc++ ${BAZEL_BUILD_OPTIONS}" + echo "$CC/$CXX toolchain with libc++ configured" + else + export BAZEL_BUILD_OPTIONS="--config=rbe-toolchain-clang-libc++ ${BAZEL_BUILD_OPTIONS}" + fi } # Create a fake home. Python site libs tries to do getpwuid(3) if we don't and the CI diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 867e00c6c2ba..9b4dae4199d9 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -123,6 +123,11 @@ elif [[ "$CI_TARGET" == "bazel.sizeopt" ]]; then echo "Testing ${TEST_TARGETS}" bazel test ${BAZEL_BUILD_OPTIONS} --config=sizeopt ${TEST_TARGETS} exit 0 +elif [[ "$CI_TARGET" == "bazel.gcc" ]]; then + setup_gcc_toolchain + echo "bazel fastbuild build..." + bazel_binary_build fastbuild + exit 0 elif [[ "$CI_TARGET" == "bazel.debug" ]]; then setup_clang_toolchain echo "bazel debug build with tests..." @@ -185,7 +190,6 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then # changes, this build type may need to be broken up. # TODO(mpwarres): remove quiche=enabled once QUICHE is built by default. COMPILE_TIME_OPTIONS="\ - --config libc++ \ --define signal_trace=disabled \ --define hot_restart=disabled \ --define google_grpc=disabled \ @@ -194,7 +198,7 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then --define quiche=enabled \ --define path_normalization_by_default=true \ " - setup_clang_toolchain + setup_clang_libcxx_toolchain # This doesn't go into CI but is available for developer convenience. echo "bazel with different compiletime options build with tests..." # Building all the dependencies from scratch to link them against libc++. diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index af828943fb53..a93802b26173 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -25,7 +25,7 @@ mkdir -p "${ENVOY_DOCKER_BUILD_DIR}" docker run --rm ${DOCKER_TTY_OPTION} -e HTTP_PROXY=${http_proxy} -e HTTPS_PROXY=${https_proxy} \ -u "${USER}":"${USER_GROUP}" -v "${ENVOY_DOCKER_BUILD_DIR}":/build ${GIT_VOLUME_OPTION} \ -e BAZEL_BUILD_EXTRA_OPTIONS -e BAZEL_EXTRA_TEST_OPTIONS -e BAZEL_REMOTE_CACHE \ - -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS \ + -e BAZEL_REMOTE_INSTANCE -e GCP_SERVICE_ACCOUNT_KEY -e NUM_CPUS -e ENVOY_RBE \ -v "$PWD":/source --cap-add SYS_PTRACE --cap-add NET_RAW --cap-add NET_ADMIN "${IMAGE_NAME}":"${IMAGE_ID}" \ /bin/bash -lc "groupadd --gid $(id -g) -f envoygroup && useradd -o --uid $(id -u) --gid $(id -g) --no-create-home \ --home-dir /source envoybuild && usermod -a -G pcap envoybuild && su envoybuild -c \"cd source && $*\"" From 01bf1c373a2eca05c2b83b4db00200b4d2321bbc Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 5 Aug 2019 08:46:30 -0700 Subject: [PATCH 309/542] docs: link to GetEnvoy.io for pre-built binaries (#7814) Signed-off-by: Lizan Zhou --- docs/root/install/building.rst | 7 +++++-- docs/root/start/start.rst | 5 ++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/root/install/building.rst b/docs/root/install/building.rst index 143f236c6b3e..56b13ee1f20c 100644 --- a/docs/root/install/building.rst +++ b/docs/root/install/building.rst @@ -68,9 +68,12 @@ tests. organizations track and deploy master in production. We encourage you to do the same so that issues can be reported as early as possible in the development process. +Packaged Envoy pre-built binaries for a variety of platforms are available via +`GetEnvoy.io `_. + We will consider producing additional binary types depending on community interest in helping with -CI, packaging, etc. Please open an `issue `_ in GitHub -if desired. +CI, packaging, etc. Please open an `issue in GetEnvoy `_ +for pre-built binaries for different platforms. Modifying Envoy --------------- diff --git a/docs/root/start/start.rst b/docs/root/start/start.rst index 8754d1c291e2..59a02b741b1e 100644 --- a/docs/root/start/start.rst +++ b/docs/root/start/start.rst @@ -5,9 +5,8 @@ Getting Started This section gets you started with a very simple configuration and provides some example configurations. -Envoy does not currently provide separate pre-built binaries, but does provide Docker images. This is -the fastest way to get started using Envoy. Should you wish to use Envoy outside of a -Docker container, you will need to :ref:`build it `. +The fastest way to get started using Envoy is :ref:`installing pre-built binaries `. +You can also :ref:`build it ` from source. These examples use the :ref:`v2 Envoy API `, but use only the static configuration feature of the API, which is most useful for simple requirements. For more complex requirements From c906df39d13dc416c21ea1977cd163f205d61d1c Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Mon, 5 Aug 2019 09:27:08 -0700 Subject: [PATCH 310/542] docs: fix incorrect comment (#7827) Signed-off-by: Derek Argueta --- source/extensions/filters/network/well_known_names.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/well_known_names.h b/source/extensions/filters/network/well_known_names.h index a397ee552719..f3fcf33e249f 100644 --- a/source/extensions/filters/network/well_known_names.h +++ b/source/extensions/filters/network/well_known_names.h @@ -28,7 +28,7 @@ class NetworkFilterNameValues { const std::string RateLimit = "envoy.ratelimit"; // Redis proxy filter const std::string RedisProxy = "envoy.redis_proxy"; - // IP tagging filter + // TCP proxy filter const std::string TcpProxy = "envoy.tcp_proxy"; // Authorization filter const std::string ExtAuthorization = "envoy.ext_authz"; From 3e63182c3e74e0d517d2f0d1fc2ee950bbfe21e1 Mon Sep 17 00:00:00 2001 From: easy Date: Tue, 6 Aug 2019 09:04:12 +1000 Subject: [PATCH 311/542] tracing: Update OpenCensus. (#7797) Let the config override the Stackdriver address. This can be used for proxying and testing. Signed-off-by: Emil Mikulic --- api/envoy/config/trace/v2/trace.proto | 6 ++++++ bazel/repository_locations.bzl | 8 ++++---- source/extensions/tracers/opencensus/BUILD | 1 + .../tracers/opencensus/opencensus_tracer_impl.cc | 10 +++++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/api/envoy/config/trace/v2/trace.proto b/api/envoy/config/trace/v2/trace.proto index 3fe4d1fff66f..34f11ec7e3e6 100644 --- a/api/envoy/config/trace/v2/trace.proto +++ b/api/envoy/config/trace/v2/trace.proto @@ -123,6 +123,12 @@ message OpenCensusConfig { // The Cloud project_id to use for Stackdriver tracing. string stackdriver_project_id = 4; + // (optional) By default, the Stackdriver exporter will connect to production + // Stackdriver. If stackdriver_address is non-empty, it will instead connect + // to this address, which is in the gRPC format: + // https://github.com/grpc/grpc/blob/master/doc/naming.md + string stackdriver_address = 10; + // Enables the Zipkin exporter if set to true. The url and service name must // also be set. bool zipkin_exporter_enabled = 5; diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index c21af3bb42b2..133e57d5731b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -233,10 +233,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://files.pythonhosted.org/packages/b3/b2/238e2590826bfdd113244a40d9d3eb26918bd798fc187e2360a8367068db/six-1.10.0.tar.gz"], ), io_opencensus_cpp = dict( - sha256 = "9223b4d54af4151910dede03aa58247e90df72167fcc91d5f75e73a141568036", - strip_prefix = "opencensus-cpp-e292a374fb42c6cb2743f1689234bd409f4fda20", - # 2019-07-15 - urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/e292a374fb42c6cb2743f1689234bd409f4fda20.tar.gz"], + sha256 = "8d6016e47c2e19e7acbadb6f905b8c422748c64299d71101ac8f28151677e195", + strip_prefix = "opencensus-cpp-cad0d03ff3474cf14389fc249e16847ab7b6895f", + # 2019-07-31 + urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/cad0d03ff3474cf14389fc249e16847ab7b6895f.tar.gz"], ), com_github_curl = dict( sha256 = "821aeb78421375f70e55381c9ad2474bf279fc454b791b7e95fc83562951c690", diff --git a/source/extensions/tracers/opencensus/BUILD b/source/extensions/tracers/opencensus/BUILD index 90bc4f9fc7ed..87b1854a2d2f 100644 --- a/source/extensions/tracers/opencensus/BUILD +++ b/source/extensions/tracers/opencensus/BUILD @@ -25,6 +25,7 @@ envoy_cc_library( name = "opencensus_tracer_impl", srcs = ["opencensus_tracer_impl.cc"], hdrs = ["opencensus_tracer_impl.h"], + copts = ["-Wno-unused-parameter"], external_deps = [ "opencensus_trace", "opencensus_trace_cloud_trace_context", diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 154e5903fe83..eb9a52d7dbd2 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -1,10 +1,13 @@ #include "extensions/tracers/opencensus/opencensus_tracer_impl.h" +#include + #include "envoy/http/header_map.h" #include "common/common/base64.h" #include "absl/strings/str_cat.h" +#include "google/devtools/cloudtrace/v2/tracing.grpc.pb.h" #include "opencensus/exporters/trace/stackdriver/stackdriver_exporter.h" #include "opencensus/exporters/trace/stdout/stdout_exporter.h" #include "opencensus/exporters/trace/zipkin/zipkin_exporter.h" @@ -197,7 +200,12 @@ Driver::Driver(const envoy::config::trace::v2::OpenCensusConfig& oc_config, if (oc_config.stackdriver_exporter_enabled()) { ::opencensus::exporters::trace::StackdriverOptions opts; opts.project_id = oc_config.stackdriver_project_id(); - ::opencensus::exporters::trace::StackdriverExporter::Register(opts); + if (!oc_config.stackdriver_address().empty()) { + auto channel = + grpc::CreateChannel(oc_config.stackdriver_address(), grpc::InsecureChannelCredentials()); + opts.trace_service_stub = ::google::devtools::cloudtrace::v2::TraceService::NewStub(channel); + } + ::opencensus::exporters::trace::StackdriverExporter::Register(std::move(opts)); } if (oc_config.zipkin_exporter_enabled()) { ::opencensus::exporters::trace::ZipkinExporterOptions opts(oc_config.zipkin_url()); From e6145e0d5f93235b0d9b70c424766ebbb2e46294 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Tue, 6 Aug 2019 08:42:05 -0700 Subject: [PATCH 312/542] accesslog: Add buffering and flushing to gRPC access log (#7755) Signed-off-by: Ruslan Nigmatullin --- api/envoy/config/accesslog/v2/als.proto | 13 +++ docs/root/intro/version_history.rst | 1 + .../http_grpc/grpc_access_log_impl.cc | 46 ++++++-- .../http_grpc/grpc_access_log_impl.h | 12 +++ .../http_grpc/grpc_access_log_impl_test.cc | 102 +++++++++++++++++- 5 files changed, 163 insertions(+), 11 deletions(-) diff --git a/api/envoy/config/accesslog/v2/als.proto b/api/envoy/config/accesslog/v2/als.proto index c71fe70a8c85..a7291e4e9780 100644 --- a/api/envoy/config/accesslog/v2/als.proto +++ b/api/envoy/config/accesslog/v2/als.proto @@ -9,6 +9,9 @@ option go_package = "v2"; import "envoy/api/v2/core/grpc_service.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + import "validate/validate.proto"; // [#protodoc-title: gRPC Access Log Service (ALS)] @@ -49,4 +52,14 @@ message CommonGrpcAccessLogConfig { // The gRPC service for the access log service. envoy.api.v2.core.GrpcService grpc_service = 2 [(validate.rules).message.required = true]; + + // Interval for flushing access logs to the gRPC stream. Logger will flush requests every time + // this interval is elapsed, or when batch size limit is hit, whichever comes first. Defaults to + // 1 second. + google.protobuf.Duration buffer_flush_interval = 3 [(validate.rules).duration.gt = {}]; + + // Soft size limit in bytes for access log entries buffer. Logger will buffer requests until + // this limit it hit, or every time flush interval is elapsed, whichever comes first. Setting it + // to zero effectively disables the batching. Defaults to 16384. + google.protobuf.UInt32Value buffer_size_bytes = 4; } diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 1ae489d86fef..d13c8caecd40 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -3,6 +3,7 @@ Version history 1.12.0 (pending) ================ +* access log: added :ref:`buffering ` and :ref:`periodical flushing ` support to gRPC access logger. Defaults to 16KB buffer and flushing every 1 second. * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * config: added access log :ref:`extension filter`. diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc index 928d3699c637..17a443881c5c 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc @@ -43,13 +43,33 @@ void GrpcAccessLoggerImpl::LocalStream::onRemoteClose(Grpc::Status::GrpcStatus, } GrpcAccessLoggerImpl::GrpcAccessLoggerImpl(Grpc::RawAsyncClientPtr&& client, std::string log_name, + std::chrono::milliseconds buffer_flush_interval_msec, + uint64_t buffer_size_bytes, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) - : client_(std::move(client)), log_name_(std::move(log_name)), local_info_(local_info) {} + : client_(std::move(client)), log_name_(log_name), + buffer_flush_interval_msec_(buffer_flush_interval_msec), + flush_timer_(dispatcher.createTimer([this]() { + flush(); + flush_timer_->enableTimer(buffer_flush_interval_msec_); + })), + buffer_size_bytes_(buffer_size_bytes), local_info_(local_info) { + flush_timer_->enableTimer(buffer_flush_interval_msec_); +} void GrpcAccessLoggerImpl::log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) { - // TODO(euroelessar): Add batching and flushing. - envoy::service::accesslog::v2::StreamAccessLogsMessage message; - message.mutable_http_logs()->add_log_entry()->Swap(&entry); + approximate_message_size_bytes_ += entry.ByteSizeLong(); + message_.mutable_http_logs()->add_log_entry()->Swap(&entry); + if (approximate_message_size_bytes_ >= buffer_size_bytes_) { + flush(); + } +} + +void GrpcAccessLoggerImpl::flush() { + if (!message_.has_http_logs()) { + // Nothing to flush. + return; + } if (stream_ == absl::nullopt) { stream_.emplace(*this); @@ -61,17 +81,21 @@ void GrpcAccessLoggerImpl::log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs"), *stream_); - auto* identifier = message.mutable_identifier(); + auto* identifier = message_.mutable_identifier(); *identifier->mutable_node() = local_info_.node(); identifier->set_log_name(log_name_); } if (stream_->stream_ != nullptr) { - stream_->stream_->sendMessage(message, false); + stream_->stream_->sendMessage(message_, false); } else { // Clear out the stream data due to stream creation failure. stream_.reset(); } + + // Clear the message regardless of the success. + approximate_message_size_bytes_ = 0; + message_.Clear(); } GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, @@ -80,7 +104,8 @@ GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& a const LocalInfo::LocalInfo& local_info) : async_client_manager_(async_client_manager), scope_(scope), tls_slot_(tls.allocateSlot()), local_info_(local_info) { - tls_slot_->set([](Event::Dispatcher&) { return std::make_shared(); }); + tls_slot_->set( + [](Event::Dispatcher& dispatcher) { return std::make_shared(dispatcher); }); } GrpcAccessLoggerSharedPtr GrpcAccessLoggerCacheImpl::getOrCreateLogger( @@ -94,8 +119,11 @@ GrpcAccessLoggerSharedPtr GrpcAccessLoggerCacheImpl::getOrCreateLogger( } const Grpc::AsyncClientFactoryPtr factory = async_client_manager_.factoryForGrpcService(config.grpc_service(), scope_, false); - const GrpcAccessLoggerSharedPtr logger = - std::make_shared(factory->create(), config.log_name(), local_info_); + const GrpcAccessLoggerSharedPtr logger = std::make_shared( + factory->create(), config.log_name(), + std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, buffer_flush_interval, 1000)), + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, buffer_size_bytes, 16384), cache.dispatcher_, + local_info_); cache.access_loggers_.emplace(cache_key, logger); return logger; } diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h index c846be0ca171..5fb8e152108c 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h @@ -62,6 +62,8 @@ using GrpcAccessLoggerCacheSharedPtr = std::shared_ptr; class GrpcAccessLoggerImpl : public GrpcAccessLogger { public: GrpcAccessLoggerImpl(Grpc::RawAsyncClientPtr&& client, std::string log_name, + std::chrono::milliseconds buffer_flush_interval_msec, + uint64_t buffer_size_bytes, Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info); void log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) override; @@ -83,10 +85,17 @@ class GrpcAccessLoggerImpl : public GrpcAccessLogger { Grpc::AsyncStream stream_{}; }; + void flush(); + Grpc::AsyncClient client_; const std::string log_name_; + const std::chrono::milliseconds buffer_flush_interval_msec_; + const Event::TimerPtr flush_timer_; + const uint64_t buffer_size_bytes_; + uint64_t approximate_message_size_bytes_ = 0; + envoy::service::accesslog::v2::StreamAccessLogsMessage message_; absl::optional stream_; const LocalInfo::LocalInfo& local_info_; }; @@ -105,6 +114,9 @@ class GrpcAccessLoggerCacheImpl : public Singleton::Instance, public GrpcAccessL * Per-thread cache. */ struct ThreadLocalCache : public ThreadLocal::ThreadLocalObject { + ThreadLocalCache(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} + + Event::Dispatcher& dispatcher_; // Access loggers indexed by the hash of logger's configuration. absl::flat_hash_map access_loggers_; }; diff --git a/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc index f78eae541e26..ecf1e048fd5d 100644 --- a/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc @@ -25,15 +25,20 @@ namespace AccessLoggers { namespace HttpGrpc { namespace { +constexpr std::chrono::milliseconds FlushInterval(10); + class GrpcAccessLoggerImplTest : public testing::Test { public: using MockAccessLogStream = Grpc::MockAsyncStream; using AccessLogCallbacks = Grpc::AsyncStreamCallbacks; - GrpcAccessLoggerImplTest() { + void initLogger(std::chrono::milliseconds buffer_flush_interval_msec, size_t buffer_size_bytes) { + timer_ = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*timer_, enableTimer(buffer_flush_interval_msec)); logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, - log_name_, local_info_); + log_name_, buffer_flush_interval_msec, + buffer_size_bytes, dispatcher_, local_info_); } void expectStreamStart(MockAccessLogStream& stream, AccessLogCallbacks** callbacks_to_set) { @@ -59,6 +64,8 @@ class GrpcAccessLoggerImplTest : public testing::Test { std::string log_name_ = "test_log_name"; LocalInfo::MockLocalInfo local_info_; + Event::MockTimer* timer_ = nullptr; + Event::MockDispatcher dispatcher_; Grpc::MockAsyncClient* async_client_{new Grpc::MockAsyncClient}; std::unique_ptr logger_; }; @@ -66,6 +73,7 @@ class GrpcAccessLoggerImplTest : public testing::Test { // Test basic stream logging flow. TEST_F(GrpcAccessLoggerImplTest, BasicFlow) { InSequence s; + initLogger(FlushInterval, 0); // Start a stream for the first log. MockAccessLogStream stream; @@ -126,6 +134,7 @@ TEST_F(GrpcAccessLoggerImplTest, BasicFlow) { // Test that stream failure is handled correctly. TEST_F(GrpcAccessLoggerImplTest, StreamFailure) { InSequence s; + initLogger(FlushInterval, 0); EXPECT_CALL(*async_client_, startRaw(_, _, _)) .WillOnce(Invoke( @@ -138,6 +147,95 @@ TEST_F(GrpcAccessLoggerImplTest, StreamFailure) { logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); } +// Test that log entries are batched. +TEST_F(GrpcAccessLoggerImplTest, Batching) { + InSequence s; + initLogger(FlushInterval, 100); + + MockAccessLogStream stream; + AccessLogCallbacks* callbacks; + expectStreamStart(stream, &callbacks); + EXPECT_CALL(local_info_, node()); + const std::string path1(30, '1'); + const std::string path2(30, '2'); + const std::string path3(80, '3'); + expectStreamMessage(stream, fmt::format(R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name +http_logs: + log_entry: + - request: + path: "{}" + - request: + path: "{}" + - request: + path: "{}" +)EOF", + path1, path2, path3)); + envoy::data::accesslog::v2::HTTPAccessLogEntry entry; + entry.mutable_request()->set_path(path1); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + entry.mutable_request()->set_path(path2); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + entry.mutable_request()->set_path(path3); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + + const std::string path4(120, '4'); + expectStreamMessage(stream, fmt::format(R"EOF( +http_logs: + log_entry: + request: + path: "{}" +)EOF", + path4)); + entry.mutable_request()->set_path(path4); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); +} + +// Test that log entries are flushed periodically. +TEST_F(GrpcAccessLoggerImplTest, Flushing) { + InSequence s; + initLogger(FlushInterval, 100); + + // Nothing to do yet. + EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + timer_->invokeCallback(); + + envoy::data::accesslog::v2::HTTPAccessLogEntry entry; + // Not enough data yet to trigger flush on batch size. + entry.mutable_request()->set_path("/test/path1"); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + + MockAccessLogStream stream; + AccessLogCallbacks* callbacks; + expectStreamStart(stream, &callbacks); + EXPECT_CALL(local_info_, node()); + expectStreamMessage(stream, fmt::format(R"EOF( + identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name + http_logs: + log_entry: + - request: + path: /test/path1 + )EOF")); + EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + timer_->invokeCallback(); + + // Flush on empty message does nothing. + EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + timer_->invokeCallback(); +} + class GrpcAccessLoggerCacheImplTest : public testing::Test { public: GrpcAccessLoggerCacheImplTest() { From 3286ea069278075cc24ac2f671c240e5d18d04ac Mon Sep 17 00:00:00 2001 From: soya3129 <43042079+soya3129@users.noreply.github.com> Date: Tue, 6 Aug 2019 12:27:06 -0400 Subject: [PATCH 313/542] codec: add metadata_not_supported_error to HTTP/1 codec stats (#7801) Signed-off-by: Yang Song --- .../configuration/http_conn_man/stats.rst | 13 +++++++- source/common/http/codec_client.cc | 3 +- source/common/http/conn_manager_utility.cc | 4 +-- source/common/http/http1/BUILD | 2 ++ source/common/http/http1/codec_impl.cc | 24 ++++++++------ source/common/http/http1/codec_impl.h | 31 ++++++++++++++++--- .../network/http_connection_manager/config.cc | 2 +- test/common/http/codec_impl_fuzz_test.cc | 8 +++-- test/common/http/http1/codec_impl_test.cc | 23 +++++++------- test/integration/fake_upstream.cc | 3 +- 10 files changed, 79 insertions(+), 34 deletions(-) diff --git a/docs/root/configuration/http_conn_man/stats.rst b/docs/root/configuration/http_conn_man/stats.rst index 0269fefc1d4b..4188dcea4a5c 100644 --- a/docs/root/configuration/http_conn_man/stats.rst +++ b/docs/root/configuration/http_conn_man/stats.rst @@ -98,7 +98,18 @@ following statistics: Per codec statistics ----------------------- -Each codec has the option of adding per-codec statistics. Currently only http2 has codec stats. +Each codec has the option of adding per-codec statistics. Both http1 and http2 have codec stats. + +Http1 codec statistics +~~~~~~~~~~~~~~~~~~~~~~ + +All http1 statistics are rooted at *http1.* + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + metadata_not_supported_error, Counter, Total number of metadata dropped during HTTP/1 encoding Http2 codec statistics ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index 6c5488ef7883..7d9ad41f3af0 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -140,7 +140,8 @@ CodecClientProd::CodecClientProd(Type type, Network::ClientConnectionPtr&& conne : CodecClient(type, std::move(connection), host, dispatcher) { switch (type) { case Type::HTTP1: { - codec_ = std::make_unique(*connection_, *this); + codec_ = std::make_unique(*connection_, + host->cluster().statsScope(), *this); break; } case Type::HTTP2: { diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index e4ff80a349bd..5f9d541693b6 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -46,8 +46,8 @@ ServerConnectionPtr ConnectionManagerUtility::autoCreateCodec( return std::make_unique(connection, callbacks, scope, http2_settings, max_request_headers_kb); } else { - return std::make_unique(connection, callbacks, http1_settings, - max_request_headers_kb); + return std::make_unique(connection, scope, callbacks, + http1_settings, max_request_headers_kb); } } diff --git a/source/common/http/http1/BUILD b/source/common/http/http1/BUILD index 98ddb6edb484..9756586598c1 100644 --- a/source/common/http/http1/BUILD +++ b/source/common/http/http1/BUILD @@ -18,6 +18,8 @@ envoy_cc_library( "//include/envoy/http:codec_interface", "//include/envoy/http:header_map_interface", "//include/envoy/network:connection_interface", + "//include/envoy/stats:stats_interface", + "//include/envoy/stats:stats_macros", "//source/common/buffer:buffer_lib", "//source/common/buffer:watermark_buffer_lib", "//source/common/common:assert_lib", diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 255de29a6a1e..704787276616 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -164,6 +164,10 @@ void StreamEncoderImpl::encodeData(Buffer::Instance& data, bool end_stream) { void StreamEncoderImpl::encodeTrailers(const HeaderMap&) { endEncode(); } +void StreamEncoderImpl::encodeMetadata(const MetadataMapVector&) { + connection_.stats().metadata_not_supported_error_.inc(); +} + void StreamEncoderImpl::endEncode() { if (chunk_encoding_) { connection_.buffer().add(LAST_CHUNK); @@ -321,10 +325,11 @@ const ToLowerTable& ConnectionImpl::toLowerTable() { return *table; } -ConnectionImpl::ConnectionImpl(Network::Connection& connection, http_parser_type type, - uint32_t max_headers_kb) - : connection_(connection), output_buffer_([&]() -> void { this->onBelowLowWatermark(); }, - [&]() -> void { this->onAboveHighWatermark(); }), +ConnectionImpl::ConnectionImpl(Network::Connection& connection, Stats::Scope& stats, + http_parser_type type, uint32_t max_headers_kb) + : connection_(connection), stats_{ALL_HTTP1_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http1."))}, + output_buffer_([&]() -> void { this->onBelowLowWatermark(); }, + [&]() -> void { this->onAboveHighWatermark(); }), max_headers_kb_(max_headers_kb), strict_header_validation_(Runtime::runtimeFeatureEnabled( "envoy.reloadable_features.strict_header_validation")) { output_buffer_.setWatermarks(connection.bufferLimit()); @@ -506,11 +511,11 @@ void ConnectionImpl::onResetStreamBase(StreamResetReason reason) { onResetStream(reason); } -ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, +ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, Stats::Scope& stats, ServerConnectionCallbacks& callbacks, Http1Settings settings, uint32_t max_request_headers_kb) - : ConnectionImpl(connection, HTTP_REQUEST, max_request_headers_kb), callbacks_(callbacks), - codec_settings_(settings) {} + : ConnectionImpl(connection, stats, HTTP_REQUEST, max_request_headers_kb), + callbacks_(callbacks), codec_settings_(settings) {} void ServerConnectionImpl::onEncodeComplete() { ASSERT(active_request_); @@ -681,8 +686,9 @@ void ServerConnectionImpl::onBelowLowWatermark() { } } -ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, ConnectionCallbacks&) - : ConnectionImpl(connection, HTTP_RESPONSE, MAX_RESPONSE_HEADERS_KB) {} +ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection, Stats::Scope& stats, + ConnectionCallbacks&) + : ConnectionImpl(connection, stats, HTTP_RESPONSE, MAX_RESPONSE_HEADERS_KB) {} bool ClientConnectionImpl::cannotHaveBody() { if ((!pending_responses_.empty() && pending_responses_.front().head_request_) || diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index c1a9fa5d7e93..bf6708fa8f47 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -10,6 +10,7 @@ #include "envoy/http/codec.h" #include "envoy/network/connection.h" +#include "envoy/stats/scope.h" #include "common/buffer/watermark_buffer.h" #include "common/common/assert.h" @@ -22,6 +23,21 @@ namespace Envoy { namespace Http { namespace Http1 { +/** + * All stats for the HTTP/1 codec. @see stats_macros.h + */ +// clang-format off +#define ALL_HTTP1_CODEC_STATS(COUNTER) \ + COUNTER(metadata_not_supported_error) \ +// clang-format on + +/** + * Wrapper struct for the HTTP/1 codec stats. @see stats_macros.h + */ +struct CodecStats { + ALL_HTTP1_CODEC_STATS(GENERATE_COUNTER_STRUCT) +}; + class ConnectionImpl; /** @@ -37,7 +53,7 @@ class StreamEncoderImpl : public StreamEncoder, void encodeHeaders(const HeaderMap& headers, bool end_stream) override; void encodeData(Buffer::Instance& data, bool end_stream) override; void encodeTrailers(const HeaderMap& trailers) override; - void encodeMetadata(const MetadataMapVector&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + void encodeMetadata(const MetadataMapVector&) override; Stream& getStream() override { return *this; } // Http::Stream @@ -171,13 +187,16 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable( - connection, callbacks, http1_settings_, maxRequestHeadersKb()); + connection, context_.scope(), callbacks, http1_settings_, maxRequestHeadersKb()); case CodecType::HTTP2: return std::make_unique( connection, callbacks, context_.scope(), http2_settings_, maxRequestHeadersKb()); diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 4a56733f710f..e25f56194efc 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -359,7 +359,8 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi stats_store, client_http2settings, max_request_headers_kb); } else { - client = std::make_unique(client_connection, client_callbacks); + client = std::make_unique(client_connection, stats_store, + client_callbacks); } NiceMock server_connection; @@ -371,8 +372,9 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi max_request_headers_kb); } else { const Http1Settings server_http1settings{fromHttp1Settings(input.h1_settings().server())}; - server = std::make_unique( - server_connection, server_callbacks, server_http1settings, max_request_headers_kb); + server = std::make_unique(server_connection, stats_store, + server_callbacks, server_http1settings, + max_request_headers_kb); } ReorderBuffer client_write_buf{*server}; diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index fec552d6f3f8..793a4b003f9c 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -49,8 +49,8 @@ class Http1ServerConnectionImplTest : public testing::Test { } void initialize() { - codec_ = std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_); + codec_ = std::make_unique(connection_, store_, callbacks_, + codec_settings_, max_request_headers_kb_); } NiceMock connection_; @@ -84,8 +84,8 @@ void Http1ServerConnectionImplTest::expect400(Protocol p, bool allow_absolute_ur if (allow_absolute_url) { codec_settings_.allow_absolute_url_ = allow_absolute_url; - codec_ = std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_); + codec_ = std::make_unique(connection_, store_, callbacks_, + codec_settings_, max_request_headers_kb_); } Http::MockStreamDecoder decoder; @@ -104,8 +104,8 @@ void Http1ServerConnectionImplTest::expectHeadersTest(Protocol p, bool allow_abs // Make a new 'codec' with the right settings if (allow_absolute_url) { codec_settings_.allow_absolute_url_ = allow_absolute_url; - codec_ = std::make_unique(connection_, callbacks_, codec_settings_, - max_request_headers_kb_); + codec_ = std::make_unique(connection_, store_, callbacks_, + codec_settings_, max_request_headers_kb_); } Http::MockStreamDecoder decoder; @@ -578,9 +578,7 @@ TEST_F(Http1ServerConnectionImplTest, HeaderOnlyResponseWith100Then200) { EXPECT_EQ("HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n", output); } -class Http1ServerConnectionImplDeathTest : public Http1ServerConnectionImplTest {}; - -TEST_F(Http1ServerConnectionImplDeathTest, MetadataTest) { +TEST_F(Http1ServerConnectionImplTest, MetadataTest) { initialize(); NiceMock decoder; @@ -598,7 +596,8 @@ TEST_F(Http1ServerConnectionImplDeathTest, MetadataTest) { MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); MetadataMapVector metadata_map_vector; metadata_map_vector.push_back(std::move(metadata_map_ptr)); - EXPECT_DEATH_LOG_TO_STDERR(response_encoder->encodeMetadata(metadata_map_vector), ""); + response_encoder->encodeMetadata(metadata_map_vector); + EXPECT_EQ(1, store_.counter("http1.metadata_not_supported_error").value()); } TEST_F(Http1ServerConnectionImplTest, ChunkedResponse) { @@ -860,7 +859,9 @@ class Http1ClientConnectionImplTest : public testing::Test { generator_, validation_visitor_, *api_)}); } - void initialize() { codec_ = std::make_unique(connection_, callbacks_); } + void initialize() { + codec_ = std::make_unique(connection_, store_, callbacks_); + } NiceMock connection_; NiceMock callbacks_; diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index 8fb6c8f01b6c..fd970b6de8f1 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -221,7 +221,8 @@ FakeHttpConnection::FakeHttpConnection(SharedConnectionWrapper& shared_connectio : FakeConnectionBase(shared_connection, time_system) { if (type == Type::HTTP1) { codec_ = std::make_unique( - shared_connection_.connection(), *this, Http::Http1Settings(), max_request_headers_kb); + shared_connection_.connection(), store, *this, Http::Http1Settings(), + max_request_headers_kb); } else { auto settings = Http::Http2Settings(); settings.allow_connect_ = true; From 9e552c5f0a86542eaae897d8f27bd34be85ad1df Mon Sep 17 00:00:00 2001 From: Yangmin Zhu Date: Tue, 6 Aug 2019 14:19:11 -0700 Subject: [PATCH 314/542] header to metadata: add base64 encode and protobuf value type (#7796) For the first point in #7771 for converting arbitrary protobuf value from header to metadata. Risk Level: Low Testing: Unit tests Docs Changes: Updated version_history.rst Release Notes: Updated version_history.rst Signed-off-by: Yangmin Zhu --- .../v2/header_to_metadata.proto | 19 +++ docs/root/intro/version_history.rst | 1 + .../filters/http/header_to_metadata/BUILD | 1 + .../header_to_metadata_filter.cc | 34 +++-- .../header_to_metadata_filter.h | 8 +- .../filters/http/header_to_metadata/BUILD | 1 + .../header_to_metadata_filter_test.cc | 118 +++++++++++++++++- 7 files changed, 169 insertions(+), 13 deletions(-) diff --git a/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto b/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto index 2c8c606d3f86..5e70bbfce46f 100644 --- a/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto +++ b/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto @@ -20,6 +20,21 @@ message Config { enum ValueType { STRING = 0; NUMBER = 1; + + // The value is a serialized `protobuf.Value + // `_. + PROTOBUF_VALUE = 2; + } + + // ValueEncode defines the encoding algorithm. + enum ValueEncode { + // The value is not encoded. + NONE = 0; + + // The value is encoded in `Base64 `_. + // Note: this is mostly used for STRING and PROTOBUF_VALUE to escape the + // non-ASCII characters in the header. + BASE64 = 1; } message KeyValuePair { @@ -40,6 +55,10 @@ message Config { // The value's type — defaults to string. ValueType type = 4; + + // How is the value encoded, default is NONE (not encoded). + // The value will be decoded accordingly before storing to metadata. + ValueEncode encode = 5; } // A Rule defines what metadata to apply when a header is present or missing. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index d13c8caecd40..16ade7f0fbf3 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -11,6 +11,7 @@ Version history * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. * grpc-json: added support for :ref:`ignoring unknown query parameters`. +* header to metadata: added :ref:`PROTOBUF_VALUE ` and :ref:`ValueEncode ` to support protobuf Value and Base64 encoding. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`HTTP inspector listener filter `. diff --git a/source/extensions/filters/http/header_to_metadata/BUILD b/source/extensions/filters/http/header_to_metadata/BUILD index b31109c6333c..f0ffbec64f5c 100644 --- a/source/extensions/filters/http/header_to_metadata/BUILD +++ b/source/extensions/filters/http/header_to_metadata/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( hdrs = ["header_to_metadata_filter.h"], deps = [ "//include/envoy/server:filter_config_interface", + "//source/common/common:base64_lib", "//source/extensions/filters/http:well_known_names", "@envoy_api//envoy/config/filter/http/header_to_metadata/v2:header_to_metadata_cc", ], diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc index 8d0ccee24e81..6c4379e7a9d8 100644 --- a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc +++ b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.cc @@ -1,5 +1,6 @@ #include "extensions/filters/http/header_to_metadata/header_to_metadata_filter.h" +#include "common/common/base64.h" #include "common/config/well_known_names.h" #include "common/protobuf/protobuf.h" @@ -12,11 +13,6 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace HeaderToMetadataFilter { -namespace { - -const uint32_t MAX_HEADER_VALUE_LEN = 100; - -} // namespace Config::Config(const envoy::config::filter::http::header_to_metadata::v2::Config config) { request_set_ = Config::configToVector(config.request_rules(), request_rules_); @@ -83,7 +79,7 @@ void HeaderToMetadataFilter::setEncoderFilterCallbacks( bool HeaderToMetadataFilter::addMetadata(StructMap& map, const std::string& meta_namespace, const std::string& key, absl::string_view value, - ValueType type) const { + ValueType type, ValueEncode encode) const { ProtobufWkt::Value val; if (value.empty()) { @@ -98,14 +94,23 @@ bool HeaderToMetadataFilter::addMetadata(StructMap& map, const std::string& meta return false; } + std::string decodedValue = std::string(value); + if (encode == envoy::config::filter::http::header_to_metadata::v2::Config_ValueEncode_BASE64) { + decodedValue = Base64::decodeWithoutPadding(value); + if (decodedValue.empty()) { + ENVOY_LOG(debug, "Base64 decode failed"); + return false; + } + } + // Sane enough, add the key/value. switch (type) { case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_STRING: - val.set_string_value(std::string(value)); + val.set_string_value(std::move(decodedValue)); break; case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_NUMBER: { double dval; - if (absl::SimpleAtod(StringUtil::trim(value), &dval)) { + if (absl::SimpleAtod(StringUtil::trim(decodedValue), &dval)) { val.set_number_value(dval); } else { ENVOY_LOG(debug, "value to number conversion failed"); @@ -113,6 +118,13 @@ bool HeaderToMetadataFilter::addMetadata(StructMap& map, const std::string& meta } break; } + case envoy::config::filter::http::header_to_metadata::v2::Config_ValueType_PROTOBUF_VALUE: { + if (!val.ParseFromString(decodedValue)) { + ENVOY_LOG(debug, "parse from decoded string failed"); + return false; + } + break; + } default: ENVOY_LOG(debug, "unknown value type"); return false; @@ -152,7 +164,8 @@ void HeaderToMetadataFilter::writeHeaderToMetadata(Http::HeaderMap& headers, if (!value.empty()) { const auto& nspace = decideNamespace(keyval.metadata_namespace()); - addMetadata(structs_by_namespace, nspace, keyval.key(), value, keyval.type()); + addMetadata(structs_by_namespace, nspace, keyval.key(), value, keyval.type(), + keyval.encode()); } else { ENVOY_LOG(debug, "value is empty, not adding metadata"); } @@ -166,7 +179,8 @@ void HeaderToMetadataFilter::writeHeaderToMetadata(Http::HeaderMap& headers, if (!keyval.value().empty()) { const auto& nspace = decideNamespace(keyval.metadata_namespace()); - addMetadata(structs_by_namespace, nspace, keyval.key(), keyval.value(), keyval.type()); + addMetadata(structs_by_namespace, nspace, keyval.key(), keyval.value(), keyval.type(), + keyval.encode()); } else { ENVOY_LOG(debug, "value is empty, not adding metadata"); } diff --git a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h index e10da6e3eba9..297af97d2bc5 100644 --- a/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h +++ b/source/extensions/filters/http/header_to_metadata/header_to_metadata_filter.h @@ -18,8 +18,12 @@ namespace HeaderToMetadataFilter { using Rule = envoy::config::filter::http::header_to_metadata::v2::Config::Rule; using ValueType = envoy::config::filter::http::header_to_metadata::v2::Config::ValueType; +using ValueEncode = envoy::config::filter::http::header_to_metadata::v2::Config::ValueEncode; using HeaderToMetadataRules = std::vector>; +// TODO(yangminzhu): Make MAX_HEADER_VALUE_LEN configurable. +const uint32_t MAX_HEADER_VALUE_LEN = 8 * 1024; + /** * Encapsulates the filter configuration with STL containers and provides an area for any custom * configuration logic. @@ -116,8 +120,8 @@ class HeaderToMetadataFilter : public Http::StreamFilter, */ void writeHeaderToMetadata(Http::HeaderMap& headers, const HeaderToMetadataRules& rules, Http::StreamFilterCallbacks& callbacks); - bool addMetadata(StructMap&, const std::string&, const std::string&, absl::string_view, - ValueType) const; + bool addMetadata(StructMap&, const std::string&, const std::string&, absl::string_view, ValueType, + ValueEncode) const; const std::string& decideNamespace(const std::string& nspace) const; }; diff --git a/test/extensions/filters/http/header_to_metadata/BUILD b/test/extensions/filters/http/header_to_metadata/BUILD index 0c2b80bff21b..d447d310c9fe 100644 --- a/test/extensions/filters/http/header_to_metadata/BUILD +++ b/test/extensions/filters/http/header_to_metadata/BUILD @@ -16,6 +16,7 @@ envoy_extension_cc_test( srcs = ["header_to_metadata_filter_test.cc"], extension_name = "envoy.filters.http.header_to_metadata", deps = [ + "//source/common/common:base64_lib", "//source/extensions/filters/http/header_to_metadata:header_to_metadata_filter_lib", "//test/mocks/server:server_mocks", ], diff --git a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc index 1cfa6acfe73f..9674e22e8217 100644 --- a/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc +++ b/test/extensions/filters/http/header_to_metadata/header_to_metadata_filter_test.cc @@ -1,4 +1,6 @@ +#include "common/common/base64.h" #include "common/http/header_map_impl.h" +#include "common/protobuf/protobuf.h" #include "extensions/filters/http/header_to_metadata/header_to_metadata_filter.h" @@ -68,6 +70,15 @@ MATCHER_P(MapEqNum, rhs, "") { return true; } +MATCHER_P(MapEqValue, rhs, "") { + const ProtobufWkt::Struct& obj = arg; + EXPECT_TRUE(!rhs.empty()); + for (auto const& entry : rhs) { + EXPECT_TRUE(TestUtility::protoEqual(obj.fields().at(entry.first), entry.second)); + } + return true; +} + /** * Basic use-case. */ @@ -154,6 +165,110 @@ TEST_F(HeaderToMetadataTest, NumberTypeTest) { EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); } +/** + * Test the Base64 encoded value gets written as a string. + */ +TEST_F(HeaderToMetadataTest, StringTypeInBase64UrlTest) { + const std::string response_config_yaml = R"EOF( +response_rules: + - header: x-authenticated + on_header_present: + key: auth + type: STRING + encode: BASE64 +)EOF"; + initializeFilter(response_config_yaml); + std::string data = "Non-ascii-characters"; + const auto encoded = Base64::encode(data.c_str(), data.size()); + Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}}; + std::map expected = {{"auth", data}}; + Http::TestHeaderMapImpl empty_headers; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, + setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEq(expected))); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); +} + +/** + * Test the Base64 encoded protobuf value gets written as a protobuf value. + */ +TEST_F(HeaderToMetadataTest, ProtobufValueTypeInBase64UrlTest) { + const std::string response_config_yaml = R"EOF( +response_rules: + - header: x-authenticated + on_header_present: + key: auth + type: PROTOBUF_VALUE + encode: BASE64 +)EOF"; + initializeFilter(response_config_yaml); + + ProtobufWkt::Value value; + auto* s = value.mutable_struct_value(); + + ProtobufWkt::Value v; + v.set_string_value("blafoo"); + (*s->mutable_fields())["k1"] = v; + v.set_number_value(2019.07); + (*s->mutable_fields())["k2"] = v; + v.set_bool_value(true); + (*s->mutable_fields())["k3"] = v; + + std::string data; + ASSERT_TRUE(value.SerializeToString(&data)); + const auto encoded = Base64::encode(data.c_str(), data.size()); + Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}}; + std::map expected = {{"auth", value}}; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, + setDynamicMetadata("envoy.filters.http.header_to_metadata", MapEqValue(expected))); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); +} + +/** + * Test bad Base64 encoding is not written. + */ +TEST_F(HeaderToMetadataTest, ProtobufValueTypeInBadBase64UrlTest) { + const std::string response_config_yaml = R"EOF( +response_rules: + - header: x-authenticated + on_header_present: + key: auth + type: PROTOBUF_VALUE + encode: BASE64 +)EOF"; + initializeFilter(response_config_yaml); + Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", "invalid"}}; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); +} + +/** + * Test the bad protobuf value is not written. + */ +TEST_F(HeaderToMetadataTest, BadProtobufValueTypeInBase64UrlTest) { + const std::string response_config_yaml = R"EOF( +response_rules: + - header: x-authenticated + on_header_present: + key: auth + type: PROTOBUF_VALUE + encode: BASE64 +)EOF"; + initializeFilter(response_config_yaml); + std::string data = "invalid"; + const auto encoded = Base64::encode(data.c_str(), data.size()); + Http::TestHeaderMapImpl incoming_headers{{"x-authenticated", encoded}}; + + EXPECT_CALL(encoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(incoming_headers, false)); +} + /** * Headers not present. */ @@ -221,7 +336,8 @@ TEST_F(HeaderToMetadataTest, EmptyHeaderValue) { */ TEST_F(HeaderToMetadataTest, HeaderValueTooLong) { initializeFilter(request_config_yaml); - Http::TestHeaderMapImpl incoming_headers{{"X-VERSION", std::string(101, 'x')}}; + auto length = Envoy::Extensions::HttpFilters::HeaderToMetadataFilter::MAX_HEADER_VALUE_LEN + 1; + Http::TestHeaderMapImpl incoming_headers{{"X-VERSION", std::string(length, 'x')}}; EXPECT_CALL(decoder_callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); EXPECT_CALL(req_info_, setDynamicMetadata(_, _)).Times(0); From 7e5e0b510b5bf771e7e83e7688da8851cb9ef8e8 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Wed, 7 Aug 2019 04:18:15 +0530 Subject: [PATCH 315/542] config: add init fetch timeout stat (#7822) Signed-off-by: Rama Chavali --- .../configuration/cluster_manager/cds.rst | 14 +----- docs/root/configuration/configuration.rst | 1 + docs/root/configuration/http_conn_man/rds.rst | 15 +------ docs/root/configuration/listeners/lds.rst | 14 +----- .../configuration/xds_subscription_stats.rst | 23 ++++++++++ docs/root/intro/version_history.rst | 1 + include/envoy/config/subscription.h | 1 + .../common/config/delta_subscription_state.cc | 1 + .../config/grpc_mux_subscription_impl.cc | 1 + .../common/config/http_subscription_impl.cc | 1 + .../filesystem_subscription_impl_test.cc | 8 ++-- .../filesystem_subscription_test_harness.h | 5 ++- .../config/grpc_subscription_impl_test.cc | 24 +++++----- .../config/http_subscription_impl_test.cc | 20 ++++----- test/common/config/subscription_impl_test.cc | 45 ++++++++++--------- .../common/config/subscription_test_harness.h | 7 ++- 16 files changed, 91 insertions(+), 90 deletions(-) create mode 100644 docs/root/configuration/xds_subscription_stats.rst diff --git a/docs/root/configuration/cluster_manager/cds.rst b/docs/root/configuration/cluster_manager/cds.rst index 89f2dbcd4b18..dcea74d79710 100644 --- a/docs/root/configuration/cluster_manager/cds.rst +++ b/docs/root/configuration/cluster_manager/cds.rst @@ -17,16 +17,4 @@ clusters depending on what is required. Statistics ---------- -CDS has a statistics tree rooted at *cluster_manager.cds.* with the following statistics: - -.. csv-table:: - :header: Name, Type, Description - :widths: 1, 1, 2 - - config_reload, Counter, Total API fetches that resulted in a config reload due to a different config - update_attempt, Counter, Total API fetches attempted - update_success, Counter, Total API fetches completed successfully - update_failure, Counter, Total API fetches that failed because of network errors - update_rejected, Counter, Total API fetches that failed because of schema/validation errors - version, Gauge, Hash of the contents from the last successful API fetch - control_plane.connected_state, Gauge, A boolean (1 for connected and 0 for disconnected) that indicates the current connection state with management server +CDS has a :ref:`statistics ` tree rooted at *cluster_manager.cds.* diff --git a/docs/root/configuration/configuration.rst b/docs/root/configuration/configuration.rst index d004bd250ce7..ffbce401b32d 100644 --- a/docs/root/configuration/configuration.rst +++ b/docs/root/configuration/configuration.rst @@ -20,6 +20,7 @@ Configuration reference rate_limit runtime statistics + xds_subscription_stats tools/router_check overload_manager/overload_manager secret diff --git a/docs/root/configuration/http_conn_man/rds.rst b/docs/root/configuration/http_conn_man/rds.rst index 280e1eeb692c..7787c24b4eff 100644 --- a/docs/root/configuration/http_conn_man/rds.rst +++ b/docs/root/configuration/http_conn_man/rds.rst @@ -14,17 +14,6 @@ fetch its own route configuration via the API. Statistics ---------- -RDS has a statistics tree rooted at *http..rds..*. +RDS has a :ref:`statistics ` tree rooted at *http..rds..*. Any ``:`` character in the ``route_config_name`` name gets replaced with ``_`` in the -stats tree. The stats tree contains the following statistics: - -.. csv-table:: - :header: Name, Type, Description - :widths: 1, 1, 2 - - config_reload, Counter, Total API fetches that resulted in a config reload due to a different config - update_attempt, Counter, Total API fetches attempted - update_success, Counter, Total API fetches completed successfully - update_failure, Counter, Total API fetches that failed because of network errors - update_rejected, Counter, Total API fetches that failed because of schema/validation errors - version, Gauge, Hash of the contents from the last successful API fetch +stats tree. diff --git a/docs/root/configuration/listeners/lds.rst b/docs/root/configuration/listeners/lds.rst index a90fe84f6f11..94511a1cb221 100644 --- a/docs/root/configuration/listeners/lds.rst +++ b/docs/root/configuration/listeners/lds.rst @@ -36,16 +36,4 @@ Configuration Statistics ---------- -LDS has a statistics tree rooted at *listener_manager.lds.* with the following statistics: - -.. csv-table:: - :header: Name, Type, Description - :widths: 1, 1, 2 - - config_reload, Counter, Total API fetches that resulted in a config reload due to a different config - update_attempt, Counter, Total API fetches attempted - update_success, Counter, Total API fetches completed successfully - update_failure, Counter, Total API fetches that failed because of network errors - update_rejected, Counter, Total API fetches that failed because of schema/validation errors - version, Gauge, Hash of the contents from the last successful API fetch - control_plane.connected_state, Gauge, A boolean (1 for connected and 0 for disconnected) that indicates the current connection state with management server +LDS has a :ref:`statistics ` tree rooted at *listener_manager.lds.* \ No newline at end of file diff --git a/docs/root/configuration/xds_subscription_stats.rst b/docs/root/configuration/xds_subscription_stats.rst new file mode 100644 index 000000000000..c15bbcc22a1a --- /dev/null +++ b/docs/root/configuration/xds_subscription_stats.rst @@ -0,0 +1,23 @@ +.. _subscription_statistics: + +xDS subscription statistics +=========================== + +Envoy discovers its various dynamic resources via discovery +services referred to as *xDS*. Resources are requested via :ref:`subscriptions `, +by specifying a filesystem path to watch, initiating gRPC streams or polling a REST-JSON URL. + +The following statistics are generated for all subscriptions. + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + config_reload, Counter, Total API fetches that resulted in a config reload due to a different config + init_fetch_timeout, Counter, Total :ref:`initial fetch timeouts ` + update_attempt, Counter, Total API fetches attempted + update_success, Counter, Total API fetches completed successfully + update_failure, Counter, Total API fetches that failed because of network errors + update_rejected, Counter, Total API fetches that failed because of schema/validation errors + version, Gauge, Hash of the contents from the last successful API fetch + control_plane.connected_state, Gauge, A boolean (1 for connected and 0 for disconnected) that indicates the current connection state with management server diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 16ade7f0fbf3..484bc9260060 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -9,6 +9,7 @@ Version history * config: added access log :ref:`extension filter`. * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. +* config: added stat :ref:`init_fetch_timeout `. * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. * grpc-json: added support for :ref:`ignoring unknown query parameters`. * header to metadata: added :ref:`PROTOBUF_VALUE ` and :ref:`ValueEncode ` to support protobuf Value and Base64 encoding. diff --git a/include/envoy/config/subscription.h b/include/envoy/config/subscription.h index bb9861d374ff..ee21da758d10 100644 --- a/include/envoy/config/subscription.h +++ b/include/envoy/config/subscription.h @@ -98,6 +98,7 @@ using SubscriptionPtr = std::unique_ptr; * Per subscription stats. @see stats_macros.h */ #define ALL_SUBSCRIPTION_STATS(COUNTER, GAUGE) \ + COUNTER(init_fetch_timeout) \ COUNTER(update_attempt) \ COUNTER(update_failure) \ COUNTER(update_rejected) \ diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index a0dcb5f0b66f..bec633841aff 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -24,6 +24,7 @@ DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, void DeltaSubscriptionState::setInitFetchTimeout(Event::Dispatcher& dispatcher) { if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { + stats_.init_fetch_timeout_.inc(); ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, nullptr); diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc index cb50a314dae8..a181b4efa174 100644 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ b/source/common/config/grpc_mux_subscription_impl.cc @@ -69,6 +69,7 @@ void GrpcMuxSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason rea ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); break; case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: + stats_.init_fetch_timeout_.inc(); disableInitFetchTimeoutTimer(); ENVOY_LOG(warn, "gRPC config: initial fetch timed out for {}", type_url_); break; diff --git a/source/common/config/http_subscription_impl.cc b/source/common/config/http_subscription_impl.cc index 4ee638895578..908250c79495 100644 --- a/source/common/config/http_subscription_impl.cc +++ b/source/common/config/http_subscription_impl.cc @@ -39,6 +39,7 @@ void HttpSubscriptionImpl::start(const std::set& resource_names) { if (init_fetch_timeout_.count() > 0) { init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { ENVOY_LOG(warn, "REST config: initial fetch timed out for", path_); + stats_.init_fetch_timeout_.inc(); callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, nullptr); }); diff --git a/test/common/config/filesystem_subscription_impl_test.cc b/test/common/config/filesystem_subscription_impl_test.cc index f4ac009e58f3..840748a72800 100644 --- a/test/common/config/filesystem_subscription_impl_test.cc +++ b/test/common/config/filesystem_subscription_impl_test.cc @@ -18,20 +18,20 @@ class FilesystemSubscriptionImplTest : public testing::Test, // Validate that the client can recover from bad JSON responses. TEST_F(FilesystemSubscriptionImplTest, BadJsonRecovery) { startSubscription({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); updateFile(";!@#badjso n"); - EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - EXPECT_TRUE(statsAre(3, 1, 0, 1, 7148434200721666028)); + EXPECT_TRUE(statsAre(3, 1, 0, 1, 0, 7148434200721666028)); } // Validate that a file that is initially available results in a successful update. TEST_F(FilesystemSubscriptionImplTest, InitialFile) { updateFile("{\"versionInfo\": \"0\", \"resources\": []}", false); startSubscription({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(1, 1, 0, 0, 7148434200721666028)); + EXPECT_TRUE(statsAre(1, 1, 0, 0, 0, 7148434200721666028)); } // Validate that if we fail to set a watch, we get a sensible warning. diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 03abeb81e72b..88c18d85216a 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -87,10 +87,11 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { } AssertionResult statsAre(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, - uint64_t version) override { + uint32_t init_fetch_timeout, uint64_t version) override { // The first attempt always fail unless there was a file there to begin with. return SubscriptionTestHarness::statsAre(attempt, success, rejected, - failure + (file_at_start_ ? 0 : 1), version); + failure + (file_at_start_ ? 0 : 1), init_fetch_timeout, + version); } void expectConfigUpdateFailed() override { diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 563054feea27..523466291207 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -20,7 +20,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { EXPECT_CALL(random_, random()); EXPECT_CALL(*timer_, enableTimer(_)); subscription_->start({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); // Ensure this doesn't cause an issue by sending a request, since we don't // have a gRPC stream. subscription_->updateResources({"cluster2"}); @@ -30,28 +30,28 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { expectSendMessage({"cluster2"}, ""); timer_cb_(); - EXPECT_TRUE(statsAre(3, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(3, 0, 0, 1, 0, 0)); verifyControlPlaneStats(1); } // Validate that the client can recover from a remote stream closure via retry. TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { startSubscription({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); EXPECT_CALL(*timer_, enableTimer(_)); EXPECT_CALL(random_, random()); subscription_->grpcMux().grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); - EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); verifyControlPlaneStats(0); // Retry and succeed. EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage({"cluster0", "cluster1"}, ""); timer_cb_(); - EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); } // Validate that When the management server gets multiple requests for the same version, it can @@ -59,21 +59,21 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { TEST_F(GrpcSubscriptionImplTest, RepeatedNonce) { InSequence s; startSubscription({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); // First with the initial, empty version update to "0". updateResources({"cluster2"}); - EXPECT_TRUE(statsAre(2, 0, 0, 0, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster2"}, "0", false); - EXPECT_TRUE(statsAre(3, 0, 1, 0, 0)); + EXPECT_TRUE(statsAre(3, 0, 1, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster2"}, "0", true); - EXPECT_TRUE(statsAre(4, 1, 1, 0, 7148434200721666028)); + EXPECT_TRUE(statsAre(4, 1, 1, 0, 0, 7148434200721666028)); // Now with version "0" update to "1". updateResources({"cluster3"}); - EXPECT_TRUE(statsAre(5, 1, 1, 0, 7148434200721666028)); + EXPECT_TRUE(statsAre(5, 1, 1, 0, 0, 7148434200721666028)); deliverConfigUpdate({"cluster3"}, "1", false); - EXPECT_TRUE(statsAre(6, 1, 2, 0, 7148434200721666028)); + EXPECT_TRUE(statsAre(6, 1, 2, 0, 0, 7148434200721666028)); deliverConfigUpdate({"cluster3"}, "1", true); - EXPECT_TRUE(statsAre(7, 2, 2, 0, 13237225503670494420U)); + EXPECT_TRUE(statsAre(7, 2, 2, 0, 0, 13237225503670494420U)); } } // namespace diff --git a/test/common/config/http_subscription_impl_test.cc b/test/common/config/http_subscription_impl_test.cc index 114597e2fa41..258357452c12 100644 --- a/test/common/config/http_subscription_impl_test.cc +++ b/test/common/config/http_subscription_impl_test.cc @@ -18,11 +18,11 @@ TEST_F(HttpSubscriptionImplTest, OnRequestReset) { EXPECT_CALL(callbacks_, onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); http_callbacks_->onFailure(Http::AsyncClient::FailureReason::Reset); - EXPECT_TRUE(statsAre(1, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(1, 0, 0, 1, 0, 0)); timerTick(); - EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - EXPECT_TRUE(statsAre(3, 1, 0, 1, 7148434200721666028)); + EXPECT_TRUE(statsAre(3, 1, 0, 1, 0, 7148434200721666028)); } // Validate that the client can recover from bad JSON responses. @@ -36,28 +36,28 @@ TEST_F(HttpSubscriptionImplTest, BadJsonRecovery) { EXPECT_CALL(callbacks_, onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); http_callbacks_->onSuccess(std::move(message)); - EXPECT_TRUE(statsAre(1, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(1, 0, 0, 1, 0, 0)); request_in_progress_ = false; timerTick(); - EXPECT_TRUE(statsAre(2, 0, 0, 1, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - EXPECT_TRUE(statsAre(3, 1, 0, 1, 7148434200721666028)); + EXPECT_TRUE(statsAre(3, 1, 0, 1, 0, 7148434200721666028)); } TEST_F(HttpSubscriptionImplTest, ConfigNotModified) { startSubscription({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(1, 0, 0, 0, 0)); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); timerTick(); - EXPECT_TRUE(statsAre(2, 0, 0, 0, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 0, 0, 0)); // accept and modify. deliverConfigUpdate({"cluster0", "cluster1"}, "0", true, true, "200"); - EXPECT_TRUE(statsAre(3, 1, 0, 0, 7148434200721666028)); + EXPECT_TRUE(statsAre(3, 1, 0, 0, 0, 7148434200721666028)); // accept and does not modify. deliverConfigUpdate({"cluster0", "cluster1"}, "0", true, false, "304"); - EXPECT_TRUE(statsAre(4, 1, 0, 0, 7148434200721666028)); + EXPECT_TRUE(statsAre(4, 1, 0, 0, 0, 7148434200721666028)); } } // namespace diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index 094c2dbf76b2..c284bfcda9a6 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -52,8 +52,9 @@ class SubscriptionImplTest : public testing::TestWithParam { } AssertionResult statsAre(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, - uint64_t version) { - return test_harness_->statsAre(attempt, success, rejected, failure, version); + uint32_t init_fetch_timeout, uint64_t version) { + return test_harness_->statsAre(attempt, success, rejected, failure, init_fetch_timeout, + version); } void deliverConfigUpdate(const std::vector cluster_names, const std::string& version, @@ -88,57 +89,57 @@ INSTANTIATE_TEST_SUITE_P(SubscriptionImplTest, SubscriptionImplInitFetchTimeoutT // Validate basic request-response succeeds. TEST_P(SubscriptionImplTest, InitialRequestResponse) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 7148434200721666028); + statsAre(2, 1, 0, 0, 0, 7148434200721666028); } // Validate that multiple streamed updates succeed. TEST_P(SubscriptionImplTest, ResponseStream) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 7148434200721666028); + statsAre(2, 1, 0, 0, 0, 7148434200721666028); deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); - statsAre(3, 2, 0, 0, 13237225503670494420U); + statsAre(3, 2, 0, 0, 0, 13237225503670494420U); } // Validate that the client can reject a config. TEST_P(SubscriptionImplTest, RejectConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0); + statsAre(2, 0, 1, 0, 0, 0); } // Validate that the client can reject a config and accept the same config later. TEST_P(SubscriptionImplTest, RejectAcceptConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0); + statsAre(2, 0, 1, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(3, 1, 1, 0, 7148434200721666028); + statsAre(3, 1, 1, 0, 0, 7148434200721666028); } // Validate that the client can reject a config and accept another config later. TEST_P(SubscriptionImplTest, RejectAcceptNextConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0); + statsAre(2, 0, 1, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); - statsAre(3, 1, 1, 0, 13237225503670494420U); + statsAre(3, 1, 1, 0, 0, 13237225503670494420U); } // Validate that stream updates send a message with the updated resources. TEST_P(SubscriptionImplTest, UpdateResources) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 7148434200721666028); + statsAre(2, 1, 0, 0, 0, 7148434200721666028); updateResources({"cluster2"}); - statsAre(3, 1, 0, 0, 7148434200721666028); + statsAre(3, 1, 0, 0, 0, 7148434200721666028); } // Validate that initial fetch timer is created and calls callback on timeout @@ -146,10 +147,10 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); expectConfigUpdateFailed(); callInitFetchTimeoutCb(); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 1, 0); } // Validate that initial fetch timer is disabled on config update @@ -157,7 +158,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); expectDisableInitFetchTimeoutTimer(); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); } @@ -167,7 +168,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnFail) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0); + statsAre(1, 0, 0, 0, 0, 0); expectDisableInitFetchTimeoutTimer(); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); } diff --git a/test/common/config/subscription_test_harness.h b/test/common/config/subscription_test_harness.h index 510d550ff9be..09803354e151 100644 --- a/test/common/config/subscription_test_harness.h +++ b/test/common/config/subscription_test_harness.h @@ -50,7 +50,8 @@ class SubscriptionTestHarness { const std::string& version, bool accept) PURE; virtual testing::AssertionResult statsAre(uint32_t attempt, uint32_t success, uint32_t rejected, - uint32_t failure, uint64_t version) { + uint32_t failure, uint32_t init_fetch_timeout, + uint64_t version) { // TODO(fredlas) rework update_success_ to make sense across all xDS carriers. Its value in // statsAre() calls in many tests will probably have to be changed. UNREFERENCED_PARAMETER(attempt); @@ -66,6 +67,10 @@ class SubscriptionTestHarness { return testing::AssertionFailure() << "update_failure: expected " << failure << ", got " << stats_.update_failure_.value(); } + if (init_fetch_timeout != stats_.init_fetch_timeout_.value()) { + return testing::AssertionFailure() << "init_fetch_timeout: expected " << init_fetch_timeout + << ", got " << stats_.init_fetch_timeout_.value(); + } if (version != stats_.version_.value()) { return testing::AssertionFailure() << "version: expected " << version << ", got " << stats_.version_.value(); From 41c4244d115c83f9099fc5b0e95dcca672558fd6 Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+crazyxy@users.noreply.github.com> Date: Tue, 6 Aug 2019 16:30:42 -0700 Subject: [PATCH 316/542] fix respect_dns_ttl bug (#7815) Use default DNS refresh TTL when DNS returns an NXDOMAIN Risk Level: Medium Testing: unit test, manual test Docs Changes: N/A Release Notes: N/A Fixes #Issue: #7808 Signed-off-by: Yan Xue --- source/common/upstream/strict_dns_cluster.cc | 4 ++- test/common/upstream/upstream_impl_test.cc | 28 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/source/common/upstream/strict_dns_cluster.cc b/source/common/upstream/strict_dns_cluster.cc index 71aa2b3c71e0..d1f4ef3039f1 100644 --- a/source/common/upstream/strict_dns_cluster.cc +++ b/source/common/upstream/strict_dns_cluster.cc @@ -138,8 +138,10 @@ void StrictDnsClusterImpl::ResolveTarget::startResolve() { std::chrono::milliseconds final_refresh_rate = parent_.dns_refresh_rate_ms_; - if (parent_.respect_dns_ttl_ && ttl_refresh_rate != std::chrono::seconds(0)) { + if (parent_.respect_dns_ttl_ && ttl_refresh_rate != std::chrono::seconds(0) && + !response.empty()) { final_refresh_rate = ttl_refresh_rate; + ASSERT(ttl_refresh_rate != std::chrono::seconds::max() && final_refresh_rate.count() > 0); ENVOY_LOG(debug, "DNS refresh rate reset for {}, refresh rate {} ms", dns_address_, final_refresh_rate.count()); } diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index d36f1ec56d20..eb0e898f28b5 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -920,6 +920,34 @@ TEST_F(StrictDnsClusterImplTest, RecordTtlAsDnsRefreshRate) { TestUtility::makeDnsResponse({"192.168.1.1", "192.168.1.2"}, std::chrono::seconds(5))); } +TEST_F(StrictDnsClusterImplTest, DefaultTtlAsDnsRefreshRateWhenResponseEmpty) { + ResolverData resolver(*dns_resolver_, dispatcher_); + + const std::string yaml = R"EOF( + name: name + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + dns_refresh_rate: 4s + respect_dns_ttl: true + hosts: [{ socket_address: { address: localhost1, port_value: 11001 }}] + )EOF"; + + envoy::api::v2::Cluster cluster_config = parseClusterFromV2Yaml(yaml); + Envoy::Stats::ScopePtr scope = stats_.createScope(fmt::format( + "cluster.{}.", cluster_config.alt_stat_name().empty() ? cluster_config.name() + : cluster_config.alt_stat_name())); + Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( + admin_, ssl_context_manager_, *scope, cm_, local_info_, dispatcher_, random_, stats_, + singleton_manager_, tls_, validation_visitor_, *api_); + StrictDnsClusterImpl cluster(cluster_config, runtime_, dns_resolver_, factory_context, + std::move(scope), false); + cluster.initialize([] {}); + + EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(4000))); + resolver.dns_callback_(TestUtility::makeDnsResponse({}, std::chrono::seconds(5))); +} + TEST(HostImplTest, HostCluster) { MockClusterMockPrioritySet cluster; HostSharedPtr host = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", 1); From 6ddfb94655d03aabeec42dc5ffe129e42bb43edb Mon Sep 17 00:00:00 2001 From: htuch Date: Tue, 6 Aug 2019 19:37:32 -0400 Subject: [PATCH 317/542] security: update Istio contact e-mail. (#7837) Signed-off-by: Harvey Tuch --- SECURITY.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 9fff3ca28cc3..e24c641f8861 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -369,17 +369,17 @@ CrashOverride will vouch for the "Seven" distribution joining the distribution l ### Members -| E-mail | Organization | -|-------------------------------------------|:-------------:| -| envoy-security-team@aspenmesh.io | Aspen Mesh | -| aws-app-mesh-security@amazon.com | AWS | -| security@cilium.io | Cilium | -| vulnerabilityreports@cloudfoundry.org | Cloud Foundry | -| secalert@datawire.io | Datawire | -| google-internal-envoy-security@google.com | Google | -| argoprod@us.ibm.com | IBM | -| vulnerabilities@discuss.istio.io | Istio | -| secalert@redhat.com | Red Hat | -| envoy-security@solo.io | solo.io | -| envoy-security@tetrate.io | Tetrate | -| security@vmware.com | VMware | +| E-mail | Organization | +|-------------------------------------------------------|:-------------:| +| envoy-security-team@aspenmesh.io | Aspen Mesh | +| aws-app-mesh-security@amazon.com | AWS | +| security@cilium.io | Cilium | +| vulnerabilityreports@cloudfoundry.org | Cloud Foundry | +| secalert@datawire.io | Datawire | +| google-internal-envoy-security@google.com | Google | +| argoprod@us.ibm.com | IBM | +| istio-security-vulnerability-reports@googlegroups.com | Istio | +| secalert@redhat.com | Red Hat | +| envoy-security@solo.io | solo.io | +| envoy-security@tetrate.io | Tetrate | +| security@vmware.com | VMware | From 7e216238310a5fc218904e06ba15e81e42b357e9 Mon Sep 17 00:00:00 2001 From: Cynthia Coan Date: Tue, 6 Aug 2019 22:34:37 -0600 Subject: [PATCH 318/542] fix portFromUrl when double protocol is provided (#7838) revival of #7823 this fixes a particular exception case where an end-user configures a socket address to point at: `https://google.com` instead of just the hostname: `google.com`. instead of throwing an (stoi) error. because it can't turn: `//google.com` into a port. we now through a slightly more sane: `malformed url`. it isn't as good as a protoc-gen-validate message since it's not clear why: `tcp://` gets prepended, but I think it's digestable enough to know, you shouldn't be putting https there. not to mention it is a step forward from: `stoi`. Risk Level: Low Testing: Not only are there unit tests, but you can run through the following configurations and see the new error messages. Docs Changes: None Release Notes: None Signed-off-by: Cynthia Coan --- source/common/network/utility.cc | 5 +++++ test/common/network/utility_test.cc | 2 ++ 2 files changed, 7 insertions(+) diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index b6b90c5a8acc..aeef34dd5c0b 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -90,6 +90,11 @@ uint32_t portFromUrl(const std::string& url, const std::string& scheme, throw EnvoyException(fmt::format("malformed url: {}", url)); } + size_t rcolon_index = url.rfind(':'); + if (colon_index != rcolon_index) { + throw EnvoyException(fmt::format("malformed url: {}", url)); + } + try { return std::stoi(url.substr(colon_index + 1)); } catch (const std::invalid_argument& e) { diff --git a/test/common/network/utility_test.cc b/test/common/network/utility_test.cc index fe738d359e36..335271422d67 100644 --- a/test/common/network/utility_test.cc +++ b/test/common/network/utility_test.cc @@ -29,6 +29,7 @@ TEST(NetworkUtility, Url) { EXPECT_THROW(Utility::hostFromTcpUrl("tcp://foo"), EnvoyException); EXPECT_THROW(Utility::portFromTcpUrl("tcp://foo"), EnvoyException); EXPECT_THROW(Utility::portFromTcpUrl("tcp://foo:bar"), EnvoyException); + EXPECT_THROW(Utility::portFromTcpUrl("tcp://https://foo:1234"), EnvoyException); EXPECT_THROW(Utility::hostFromTcpUrl(""), EnvoyException); EXPECT_THROW(Utility::portFromTcpUrl("tcp://foo:999999999999"), EnvoyException); } @@ -40,6 +41,7 @@ TEST(NetworkUtility, udpUrl) { EXPECT_THROW(Utility::portFromUdpUrl("bogus://foo:1234"), EnvoyException); EXPECT_THROW(Utility::hostFromUdpUrl("tcp://foo"), EnvoyException); EXPECT_THROW(Utility::portFromUdpUrl("tcp://foo:1234"), EnvoyException); + EXPECT_THROW(Utility::portFromUdpUrl("udp://https://foo:1234"), EnvoyException); EXPECT_THROW(Utility::hostFromUdpUrl(""), EnvoyException); EXPECT_THROW(Utility::portFromUdpUrl("udp://foo:999999999999"), EnvoyException); } From cb7c969f7748c20ba8cd3fa6982e66c1e941ebcd Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 6 Aug 2019 21:42:36 -0700 Subject: [PATCH 319/542] jwt_authn: clean not-used using (#7845) Signed-off-by: Wayne Zhang --- source/extensions/filters/http/jwt_authn/authenticator.cc | 1 - source/extensions/filters/http/jwt_authn/extractor.cc | 1 - source/extensions/filters/http/jwt_authn/matcher.cc | 1 - test/extensions/filters/http/jwt_authn/extractor_test.cc | 4 ---- .../filters/http/jwt_authn/filter_config_test.cc | 1 - .../filters/http/jwt_authn/filter_factory_test.cc | 2 -- test/extensions/filters/http/jwt_authn/matcher_test.cc | 8 -------- 7 files changed, 18 deletions(-) diff --git a/source/extensions/filters/http/jwt_authn/authenticator.cc b/source/extensions/filters/http/jwt_authn/authenticator.cc index fa361f90a237..e158adb398d4 100644 --- a/source/extensions/filters/http/jwt_authn/authenticator.cc +++ b/source/extensions/filters/http/jwt_authn/authenticator.cc @@ -12,7 +12,6 @@ #include "jwt_verify_lib/jwt.h" #include "jwt_verify_lib/verify.h" -using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtProvider; using ::google::jwt_verify::CheckAudience; using ::google::jwt_verify::Status; diff --git a/source/extensions/filters/http/jwt_authn/extractor.cc b/source/extensions/filters/http/jwt_authn/extractor.cc index 08103d852207..c6950a2b31c3 100644 --- a/source/extensions/filters/http/jwt_authn/extractor.cc +++ b/source/extensions/filters/http/jwt_authn/extractor.cc @@ -10,7 +10,6 @@ #include "absl/strings/match.h" using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtAuthentication; -using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtHeader; using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtProvider; using Envoy::Http::LowerCaseString; diff --git a/source/extensions/filters/http/jwt_authn/matcher.cc b/source/extensions/filters/http/jwt_authn/matcher.cc index 0a187c3eb451..53c61c2c5867 100644 --- a/source/extensions/filters/http/jwt_authn/matcher.cc +++ b/source/extensions/filters/http/jwt_authn/matcher.cc @@ -6,7 +6,6 @@ #include "absl/strings/match.h" using ::envoy::api::v2::route::RouteMatch; -using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtProvider; using ::envoy::config::filter::http::jwt_authn::v2alpha::RequirementRule; using Envoy::Router::ConfigUtility; diff --git a/test/extensions/filters/http/jwt_authn/extractor_test.cc b/test/extensions/filters/http/jwt_authn/extractor_test.cc index 1286fcdb51f8..944e06bce06a 100644 --- a/test/extensions/filters/http/jwt_authn/extractor_test.cc +++ b/test/extensions/filters/http/jwt_authn/extractor_test.cc @@ -9,10 +9,6 @@ using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtAuthentication; using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtProvider; using Envoy::Http::TestHeaderMapImpl; -using ::testing::_; -using ::testing::Invoke; -using ::testing::NiceMock; - namespace Envoy { namespace Extensions { namespace HttpFilters { diff --git a/test/extensions/filters/http/jwt_authn/filter_config_test.cc b/test/extensions/filters/http/jwt_authn/filter_config_test.cc index 769cc6de7958..1b9583ac3401 100644 --- a/test/extensions/filters/http/jwt_authn/filter_config_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_config_test.cc @@ -9,7 +9,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using ::envoy::api::v2::core::Metadata; using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtAuthentication; namespace Envoy { diff --git a/test/extensions/filters/http/jwt_authn/filter_factory_test.cc b/test/extensions/filters/http/jwt_authn/filter_factory_test.cc index 212640a161d5..6b5e5900f9c0 100644 --- a/test/extensions/filters/http/jwt_authn/filter_factory_test.cc +++ b/test/extensions/filters/http/jwt_authn/filter_factory_test.cc @@ -9,9 +9,7 @@ #include "gtest/gtest.h" using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtAuthentication; -using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtProvider; using testing::_; -using testing::Invoke; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/jwt_authn/matcher_test.cc b/test/extensions/filters/http/jwt_authn/matcher_test.cc index 2e5e334a9120..0ca3e6035089 100644 --- a/test/extensions/filters/http/jwt_authn/matcher_test.cc +++ b/test/extensions/filters/http/jwt_authn/matcher_test.cc @@ -6,16 +6,8 @@ #include "test/extensions/filters/http/jwt_authn/test_common.h" #include "test/test_common/utility.h" -using ::envoy::api::v2::route::RouteMatch; -using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtProvider; -using ::envoy::config::filter::http::jwt_authn::v2alpha::JwtRequirement; using ::envoy::config::filter::http::jwt_authn::v2alpha::RequirementRule; using Envoy::Http::TestHeaderMapImpl; -using ::testing::_; -using ::testing::Invoke; -using ::testing::NiceMock; - -using ::google::jwt_verify::Status; namespace Envoy { namespace Extensions { From 67103ba519f4102ba7732ea54d618c0b5177a37a Mon Sep 17 00:00:00 2001 From: Mike Schore Date: Wed, 7 Aug 2019 09:42:17 -0700 Subject: [PATCH 320/542] http: add onComplete to AsyncClient::StreamCallbacks (#7752) Signed-off-by: Mike Schore --- include/envoy/http/async_client.h | 7 +++ source/common/grpc/async_client_impl.cc | 4 ++ source/common/grpc/async_client_impl.h | 1 + source/common/http/async_client_impl.cc | 67 +++++++++++++++++----- source/common/http/async_client_impl.h | 4 +- test/common/http/async_client_impl_test.cc | 9 +++ test/mocks/http/mocks.h | 1 + 7 files changed, 76 insertions(+), 17 deletions(-) diff --git a/include/envoy/http/async_client.h b/include/envoy/http/async_client.h index d3bf280198c7..c80eb38db6d4 100644 --- a/include/envoy/http/async_client.h +++ b/include/envoy/http/async_client.h @@ -75,6 +75,13 @@ class AsyncClient { */ virtual void onTrailers(HeaderMapPtr&& trailers) PURE; + /** + * Called when both the local and remote have gracefully closed the stream. + * Useful for asymmetric cases where end_stream may not be bidirectionally observable. + * Note this is NOT called on stream reset. + */ + virtual void onComplete() PURE; + /** * Called when the async HTTP stream is reset. */ diff --git a/source/common/grpc/async_client_impl.cc b/source/common/grpc/async_client_impl.cc index 82837ecceb67..1f1798547378 100644 --- a/source/common/grpc/async_client_impl.cc +++ b/source/common/grpc/async_client_impl.cc @@ -166,6 +166,10 @@ void AsyncStreamImpl::streamError(Status::GrpcStatus grpc_status, const std::str resetStream(); } +void AsyncStreamImpl::onComplete() { + // No-op since stream completion is handled within other callbacks. +} + void AsyncStreamImpl::onReset() { if (http_reset_) { return; diff --git a/source/common/grpc/async_client_impl.h b/source/common/grpc/async_client_impl.h index 575c5a46301f..c360adce1223 100644 --- a/source/common/grpc/async_client_impl.h +++ b/source/common/grpc/async_client_impl.h @@ -55,6 +55,7 @@ class AsyncStreamImpl : public RawAsyncStream, void onHeaders(Http::HeaderMapPtr&& headers, bool end_stream) override; void onData(Buffer::Instance& data, bool end_stream) override; void onTrailers(Http::HeaderMapPtr&& trailers) override; + void onComplete() override; void onReset() override; // Grpc::AsyncStream diff --git a/source/common/http/async_client_impl.cc b/source/common/http/async_client_impl.cc index a54b9eea5a56..054a2ef2dede 100644 --- a/source/common/http/async_client_impl.cc +++ b/source/common/http/async_client_impl.cc @@ -142,17 +142,48 @@ void AsyncStreamImpl::sendTrailers(HeaderMap& trailers) { } void AsyncStreamImpl::closeLocal(bool end_stream) { + // TODO(goaway): This assert maybe merits reconsideration. It seems to be saying that we shouldn't + // get here when trying to send the final frame of a stream that has already been closed locally, + // but it's fine for us to get here if we're trying to send a non-final frame. There's not an + // obvious reason why the first case would be not okay but the second case okay. ASSERT(!(local_closed_ && end_stream)); + // This guard ensures that we don't attempt to clean up a stream or fire a completion callback + // for a stream that has already been closed. Both send* calls and resets can result in stream + // closure, and this state may be updated synchronously during stream interaction and callbacks. + // Additionally AsyncRequestImpl maintains behavior wherein its onComplete callback will fire + // immediately upon receiving a complete response, regardless of whether it has finished sending + // a request. + // Previous logic treated post-closure entry here as more-or-less benign (providing later-stage + // guards against redundant cleanup), but to surface consistent stream state via callbacks, + // it's necessary to be more rigorous. + // TODO(goaway): Consider deeper cleanup of assumptions here. + if (local_closed_) { + return; + } - local_closed_ |= end_stream; + local_closed_ = end_stream; if (complete()) { + stream_callbacks_.onComplete(); cleanup(); } } void AsyncStreamImpl::closeRemote(bool end_stream) { - remote_closed_ |= end_stream; + // This guard ensures that we don't attempt to clean up a stream or fire a completion callback for + // a stream that has already been closed. This function is called synchronously after callbacks + // have executed, and it's possible for callbacks to, for instance, directly reset a stream or + // close the remote manually. The test case ResetInOnHeaders covers this case specifically. + // Previous logic treated post-closure entry here as more-or-less benign (providing later-stage + // guards against redundant cleanup), but to surface consistent stream state via callbacks, it's + // necessary to be more rigorous. + // TODO(goaway): Consider deeper cleanup of assumptions here. + if (remote_closed_) { + return; + } + + remote_closed_ = end_stream; if (complete()) { + stream_callbacks_.onComplete(); cleanup(); } } @@ -184,36 +215,42 @@ AsyncRequestImpl::AsyncRequestImpl(MessagePtr&& request, AsyncClientImpl& parent void AsyncRequestImpl::initialize() { sendHeaders(request_->headers(), !request_->body()); - if (!remoteClosed() && request_->body()) { - sendData(*request_->body(), true); + // AsyncRequestImpl has historically been implemented to fire onComplete immediately upon + // receiving a complete response, regardless of whether the underlying stream was fully closed (in + // other words, regardless of whether the complete request had been sent). This had the potential + // to leak half-closed streams, which is now covered by manually firing closeLocal below. (See + // test PoolFailureWithBody for an example execution path.) + // TODO(goaway): Consider deeper cleanup of assumptions here. + if (request_->body()) { + // sendHeaders can result in synchronous stream closure in certain cases (e.g. connection pool + // failure). + if (remoteClosed()) { + // In the case that we had a locally-generated response, we manually close the stream locally + // to fire the completion callback. This is a no-op if we had a locally-generated reset + // instead. + closeLocal(true); + } else { + sendData(*request_->body(), true); + } } // TODO(mattklein123): Support request trailers. } void AsyncRequestImpl::onComplete() { callbacks_.onSuccess(std::move(response_)); } -void AsyncRequestImpl::onHeaders(HeaderMapPtr&& headers, bool end_stream) { +void AsyncRequestImpl::onHeaders(HeaderMapPtr&& headers, bool) { response_ = std::make_unique(std::move(headers)); - - if (end_stream) { - onComplete(); - } } -void AsyncRequestImpl::onData(Buffer::Instance& data, bool end_stream) { +void AsyncRequestImpl::onData(Buffer::Instance& data, bool) { if (!response_->body()) { response_->body() = std::make_unique(); } response_->body()->move(data); - - if (end_stream) { - onComplete(); - } } void AsyncRequestImpl::onTrailers(HeaderMapPtr&& trailers) { response_->trailers(std::move(trailers)); - onComplete(); } void AsyncRequestImpl::onReset() { diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index b5374c455f1a..ac49b97c47cf 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -88,6 +88,7 @@ class AsyncStreamImpl : public AsyncClient::Stream, protected: bool remoteClosed() { return remote_closed_; } + void closeLocal(bool end_stream); AsyncClientImpl& parent_; @@ -281,7 +282,6 @@ class AsyncStreamImpl : public AsyncClient::Stream, }; void cleanup(); - void closeLocal(bool end_stream); void closeRemote(bool end_stream); bool complete() { return local_closed_ && remote_closed_; } @@ -383,12 +383,12 @@ class AsyncRequestImpl final : public AsyncClient::Request, private: void initialize(); - void onComplete(); // AsyncClient::StreamCallbacks void onHeaders(HeaderMapPtr&& headers, bool end_stream) override; void onData(Buffer::Instance& data, bool end_stream) override; void onTrailers(HeaderMapPtr&& trailers) override; + void onComplete() override; void onReset() override; // Http::StreamDecoderFilterCallbacks diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index 9a58c3f4f83d..b1e30d65d5fd 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -100,6 +100,7 @@ TEST_F(AsyncClientImplTest, BasicStream) { expectResponseHeaders(stream_callbacks_, 200, false); EXPECT_CALL(stream_callbacks_, onData(BufferEqual(body.get()), true)); + EXPECT_CALL(stream_callbacks_, onComplete()); AsyncClient::Stream* stream = client_.start(stream_callbacks_, AsyncClient::StreamOptions()); stream->sendHeaders(headers, false); @@ -243,6 +244,7 @@ TEST_F(AsyncClientImplTest, RetryWithStream) { // Normal response. expectResponseHeaders(stream_callbacks_, 200, true); + EXPECT_CALL(stream_callbacks_, onComplete()); HeaderMapPtr response_headers2(new TestHeaderMapImpl{{":status", "200"}}); response_decoder_->decodeHeaders(std::move(response_headers2), true); } @@ -265,6 +267,7 @@ TEST_F(AsyncClientImplTest, MultipleStreams) { expectResponseHeaders(stream_callbacks_, 200, false); EXPECT_CALL(stream_callbacks_, onData(BufferEqual(body.get()), true)); + EXPECT_CALL(stream_callbacks_, onComplete()); AsyncClient::Stream* stream = client_.start(stream_callbacks_, AsyncClient::StreamOptions()); stream->sendHeaders(headers, false); @@ -289,6 +292,7 @@ TEST_F(AsyncClientImplTest, MultipleStreams) { EXPECT_CALL(stream_encoder2, encodeData(BufferEqual(body2.get()), true)); expectResponseHeaders(stream_callbacks2, 503, true); + EXPECT_CALL(stream_callbacks2, onComplete()); AsyncClient::Stream* stream2 = client_.start(stream_callbacks2, AsyncClient::StreamOptions()); stream2->sendHeaders(headers2, false); @@ -388,6 +392,7 @@ TEST_F(AsyncClientImplTest, StreamAndRequest) { expectResponseHeaders(stream_callbacks_, 200, false); EXPECT_CALL(stream_callbacks_, onData(BufferEqual(body.get()), true)); + EXPECT_CALL(stream_callbacks_, onComplete()); AsyncClient::Stream* stream = client_.start(stream_callbacks_, AsyncClient::StreamOptions()); stream->sendHeaders(headers, false); @@ -427,6 +432,7 @@ TEST_F(AsyncClientImplTest, StreamWithTrailers) { EXPECT_CALL(stream_callbacks_, onData(BufferEqual(body.get()), false)); TestHeaderMapImpl expected_trailers{{"some", "trailer"}}; EXPECT_CALL(stream_callbacks_, onTrailers_(HeaderMapEqualRef(&expected_trailers))); + EXPECT_CALL(stream_callbacks_, onComplete()); AsyncClient::Stream* stream = client_.start(stream_callbacks_, AsyncClient::StreamOptions()); stream->sendHeaders(headers, false); @@ -720,6 +726,7 @@ TEST_F(AsyncClientImplTest, StreamTimeout) { {":status", "504"}, {"content-length", "24"}, {"content-type", "text/plain"}}; EXPECT_CALL(stream_callbacks_, onHeaders_(HeaderMapEqualRef(&expected_timeout), false)); EXPECT_CALL(stream_callbacks_, onData(_, true)); + EXPECT_CALL(stream_callbacks_, onComplete()); AsyncClient::Stream* stream = client_.start( stream_callbacks_, AsyncClient::StreamOptions().setTimeout(std::chrono::milliseconds(40))); @@ -753,6 +760,7 @@ TEST_F(AsyncClientImplTest, StreamTimeoutHeadReply) { TestHeaderMapImpl expected_timeout{ {":status", "504"}, {"content-length", "24"}, {"content-type", "text/plain"}}; EXPECT_CALL(stream_callbacks_, onHeaders_(HeaderMapEqualRef(&expected_timeout), true)); + EXPECT_CALL(stream_callbacks_, onComplete()); AsyncClient::Stream* stream = client_.start( stream_callbacks_, AsyncClient::StreamOptions().setTimeout(std::chrono::milliseconds(40))); @@ -860,6 +868,7 @@ TEST_F(AsyncClientImplTest, MultipleDataStream) { EXPECT_CALL(stream_encoder_, encodeData(BufferEqual(body2.get()), true)); EXPECT_CALL(stream_callbacks_, onData(BufferEqual(body2.get()), true)); + EXPECT_CALL(stream_callbacks_, onComplete()); stream->sendData(*body2, true); response_decoder_->decodeData(*body2, true); diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index 13f5d6717a03..c4b4dd46d669 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -347,6 +347,7 @@ class MockAsyncClientStreamCallbacks : public AsyncClient::StreamCallbacks { MOCK_METHOD2(onHeaders_, void(HeaderMap& headers, bool end_stream)); MOCK_METHOD2(onData, void(Buffer::Instance& data, bool end_stream)); MOCK_METHOD1(onTrailers_, void(HeaderMap& headers)); + MOCK_METHOD0(onComplete, void()); MOCK_METHOD0(onReset, void()); }; From 2942421db9bf711fc92841252a25250f08cee672 Mon Sep 17 00:00:00 2001 From: soya3129 <43042079+soya3129@users.noreply.github.com> Date: Wed, 7 Aug 2019 12:43:42 -0400 Subject: [PATCH 321/542] http: introduce addEncodedMetadata() (#7756) Signed-off-by: Yang Song --- include/envoy/http/filter.h | 7 ++ source/common/http/conn_manager_impl.cc | 34 +++++++++- source/common/http/conn_manager_impl.h | 11 ++-- source/docs/h2_metadata.md | 17 ++--- test/common/http/http1/BUILD | 1 + test/common/http/http1/codec_impl_test.cc | 1 + .../encode_headers_return_stop_all_filter.cc | 12 ++++ .../filters/response_metadata_filter.cc | 27 +++----- test/integration/http2_integration_test.cc | 64 ++++++++++++------- test/integration/protocol_integration_test.cc | 6 ++ test/mocks/http/mocks.h | 1 + 11 files changed, 124 insertions(+), 57 deletions(-) diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index 609205443d25..116506541d9e 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -609,6 +609,13 @@ class StreamEncoderFilterCallbacks : public virtual StreamFilterCallbacks { */ virtual HeaderMap& addEncodedTrailers() PURE; + /** + * Adds new metadata to be encoded. + * + * @param metadata_map supplies the unique_ptr of the metadata to be encoded. + */ + virtual void addEncodedMetadata(MetadataMapPtr&& metadata_map) PURE; + /** * Called when an encoder filter goes over its high watermark. */ diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 398abbf306d3..8229c9ce9f37 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1309,6 +1309,7 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ActiveStreamEncoderFilte ENVOY_STREAM_LOG(trace, "encode headers called: filter={} status={}", *this, static_cast((*entry).get()), static_cast(status)); + (*entry)->encode_headers_called_ = true; const auto continue_iteration = (*entry)->commonHandleAfterHeadersCallback(status, encoding_headers_only_); @@ -1438,10 +1439,19 @@ void ConnectionManagerImpl::ActiveStream::encodeMetadata(ActiveStreamEncoderFilt MetadataMapPtr&& metadata_map_ptr) { resetIdleTimer(); - // Metadata currently go through all filters. - ASSERT(filter == nullptr); - std::list::iterator entry = encoder_filters_.begin(); + std::list::iterator entry = + commonEncodePrefix(filter, false, FilterIterationStartState::CanStartFromCurrent); + for (; entry != encoder_filters_.end(); entry++) { + // If the filter pointed by entry has stopped for all frame type, stores metadata and returns. + // If the filter pointed by entry hasn't returned from encodeHeaders, stores newly added + // metadata in case encodeHeaders returns StopAllIteration. The latter can happen when headers + // callbacks generate new metadata. + if (!(*entry)->encode_headers_called_ || (*entry)->stoppedAll()) { + (*entry)->getSavedResponseMetadata()->emplace_back(std::move(metadata_map_ptr)); + return; + } + FilterMetadataStatus status = (*entry)->handle_->encodeMetadata(*metadata_map_ptr); ENVOY_STREAM_LOG(trace, "encode metadata called: filter={} status={}", *this, static_cast((*entry).get()), static_cast(status)); @@ -2086,6 +2096,19 @@ Buffer::WatermarkBufferPtr ConnectionManagerImpl::ActiveStreamEncoderFilter::cre return Buffer::WatermarkBufferPtr{buffer}; } +void ConnectionManagerImpl::ActiveStreamEncoderFilter::handleMetadataAfterHeadersCallback() { + // If we drain accumulated metadata, the iteration must start with the current filter. + const bool saved_state = iterate_from_current_filter_; + iterate_from_current_filter_ = true; + // If encodeHeaders() returns StopAllIteration, we should skip draining metadata, and wait + // for doMetadata() to drain the metadata after iteration continues. + if (!stoppedAll() && saved_response_metadata_ != nullptr && + !getSavedResponseMetadata()->empty()) { + drainSavedResponseMetadata(); + } + // Restores the original value of iterate_from_current_filter_. + iterate_from_current_filter_ = saved_state; +} void ConnectionManagerImpl::ActiveStreamEncoderFilter::addEncodedData(Buffer::Instance& data, bool streaming) { return parent_.addEncodedData(*this, data, streaming); @@ -2101,6 +2124,11 @@ HeaderMap& ConnectionManagerImpl::ActiveStreamEncoderFilter::addEncodedTrailers( return parent_.addEncodedTrailers(); } +void ConnectionManagerImpl::ActiveStreamEncoderFilter::addEncodedMetadata( + MetadataMapPtr&& metadata_map_ptr) { + return parent_.encodeMetadata(this, std::move(metadata_map_ptr)); +} + void ConnectionManagerImpl::ActiveStreamEncoderFilter:: onEncoderFilterAboveWriteBufferHighWatermark() { ENVOY_STREAM_LOG(debug, "Disabling upstream stream due to filter callbacks.", parent_); diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index a72dab9069cd..1b99c6ad6c98 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -101,7 +101,7 @@ class ConnectionManagerImpl : Logger::Loggable, : parent_(parent), iteration_state_(IterationState::Continue), iterate_from_current_filter_(false), headers_continued_(false), continue_headers_continued_(false), end_stream_(false), dual_filter_(dual_filter), - decode_headers_called_(false) {} + decode_headers_called_(false), encode_headers_called_(false) {} // Functions in the following block are called after the filter finishes processing // corresponding data. Those functions handle state updates and data storage (if needed) @@ -131,7 +131,7 @@ class ConnectionManagerImpl : Logger::Loggable, virtual const HeaderMapPtr& trailers() PURE; virtual void doMetadata() PURE; // TODO(soya3129): make this pure when adding impl to encodefilter. - virtual void handleMetadataAfterHeadersCallback() {} + virtual void handleMetadataAfterHeadersCallback() PURE; // Http::StreamFilterCallbacks const Network::Connection* connection() override; @@ -173,8 +173,8 @@ class ConnectionManagerImpl : Logger::Loggable, // either because [de|en]codeHeaders() of the current filter returns StopAllIteration or because // [de|en]codeHeaders() adds new metadata to [de|en]code, but we don't know // [de|en]codeHeaders()'s return value yet. The storage is created on demand. - std::unique_ptr saved_request_metadata_; - std::unique_ptr saved_response_metadata_; + std::unique_ptr saved_request_metadata_{nullptr}; + std::unique_ptr saved_response_metadata_{nullptr}; // The state of iteration. enum class IterationState { Continue, // Iteration has not stopped for any frame type. @@ -197,6 +197,7 @@ class ConnectionManagerImpl : Logger::Loggable, bool end_stream_ : 1; const bool dual_filter_ : 1; bool decode_headers_called_ : 1; + bool encode_headers_called_ : 1; }; /** @@ -346,6 +347,7 @@ class ConnectionManagerImpl : Logger::Loggable, } getSavedResponseMetadata()->clear(); } + void handleMetadataAfterHeadersCallback() override; void doMetadata() override { if (saved_response_metadata_ != nullptr) { @@ -359,6 +361,7 @@ class ConnectionManagerImpl : Logger::Loggable, void addEncodedData(Buffer::Instance& data, bool streaming) override; void injectEncodedDataToFilterChain(Buffer::Instance& data, bool end_stream) override; HeaderMap& addEncodedTrailers() override; + void addEncodedMetadata(MetadataMapPtr&& metadata_map) override; void onEncoderFilterAboveWriteBufferHighWatermark() override; void onEncoderFilterBelowWriteBufferLowWatermark() override; void setEncoderBufferLimit(uint32_t limit) override { parent_.setBufferLimit(limit); } diff --git a/source/docs/h2_metadata.md b/source/docs/h2_metadata.md index 81fce3daac7a..c066efc36b4a 100644 --- a/source/docs/h2_metadata.md +++ b/source/docs/h2_metadata.md @@ -85,19 +85,16 @@ StreamDecoderFilter::decodeMetadata(MetadataMap metadata\_map). New metadata can be added directly to metadata\_map. If users need to add new metadata for a response to downstream, a -StreamFilter should be created. Users pass the metadata to be added to -StreamDecoderFilterCallbacks::encodeMetadata(MetadataMapPtr&& +StreamEncoderFilter should be created. Users pass the metadata to be added to +StreamEncoderFilterCallbacks::addEncodedMetadata(MetadataMapPtr&& metadata\_map\_ptr). This function can be called in -StreamFilter::encode100ContinueHeaders(HeaderMap& headers), StreamFilter::encodeHeaders(HeaderMap& headers, bool end\_stream), -StreamFilter::encodeData(Buffer::Instance& data, bool end\_stream), StreamFilter::encodeTrailers(HeaderMap& trailers). -Consequently, the new metadata will be passed through all the encoding filters. Note that, the added -metadata should not share the same key as the metadata to be consumed. Otherwise, the added metadata -will be consumed. +StreamEncoderFilter::encode100ContinueHeaders(HeaderMap& headers), StreamEncoderFilter::encodeHeaders(HeaderMap& headers, bool end\_stream), +StreamEncoderFilter::encodeData(Buffer::Instance& data, bool end\_stream), StreamEncoderFilter::encodeTrailers(HeaderMap& trailers). +Consequently, the new metadata will be passed through all the encoding filters that follow the filter +where the new metadata are added. If users receive metadata from upstream, new metadata can be added directly to -the input argument metadata\_map in StreamFilter::encodeMetadata(MetadataMap& metadata\_map). Note that, -users should never call StreamDecoderFilterCallbacks::encodeMetadata(MetadataMapPtr&& -metadata\_map\_ptr) to add new metadata in StreamFilter::encodeMetadata(MetadataMap& metadata\_map). +the input argument metadata\_map in StreamFilter::encodeMetadata(MetadataMap& metadata\_map). ### Metadata implementation diff --git a/test/common/http/http1/BUILD b/test/common/http/http1/BUILD index efee73f47d1b..bbc146417792 100644 --- a/test/common/http/http1/BUILD +++ b/test/common/http/http1/BUILD @@ -27,6 +27,7 @@ envoy_cc_test( "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:logging_lib", ], ) diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 793a4b003f9c..f2b04b6e088a 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -18,6 +18,7 @@ #include "test/mocks/protobuf/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/thread_local/mocks.h" +#include "test/test_common/logging.h" #include "test/test_common/printers.h" #include "test/test_common/utility.h" diff --git a/test/integration/filters/encode_headers_return_stop_all_filter.cc b/test/integration/filters/encode_headers_return_stop_all_filter.cc index 559121efea37..778c4897a15a 100644 --- a/test/integration/filters/encode_headers_return_stop_all_filter.cc +++ b/test/integration/filters/encode_headers_return_stop_all_filter.cc @@ -35,6 +35,10 @@ class EncodeHeadersReturnStopAllFilter : public Http::PassThroughFilter { createTimerForContinue(); + Http::MetadataMap metadata_map = {{"headers", "headers"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); + Http::HeaderEntry* entry_buffer = header_map.get(Envoy::Http::LowerCaseString("buffer_limit")); if (entry_buffer == nullptr) { return Http::FilterHeadersStatus::StopAllIterationAndBuffer; @@ -56,6 +60,10 @@ class EncodeHeadersReturnStopAllFilter : public Http::PassThroughFilter { // encodeData will only be called once after iteration resumes. EXPECT_EQ(data.length(), content_size_); } + Http::MetadataMap metadata_map = {{"data", "data"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); + Buffer::OwnedImpl added_data(std::string(added_size_, 'a')); encoder_callbacks_->addEncodedData(added_data, false); return Http::FilterDataStatus::Continue; @@ -63,6 +71,10 @@ class EncodeHeadersReturnStopAllFilter : public Http::PassThroughFilter { Http::FilterTrailersStatus encodeTrailers(Http::HeaderMap&) override { ASSERT(timer_triggered_); + Http::MetadataMap metadata_map = {{"trailers", "trailers"}}; + Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); + Buffer::OwnedImpl data(std::string(added_size_, 'a')); encoder_callbacks_->addEncodedData(data, false); return Http::FilterTrailersStatus::Continue; diff --git a/test/integration/filters/response_metadata_filter.cc b/test/integration/filters/response_metadata_filter.cc index bc3779d3692e..07cc21ac98a2 100644 --- a/test/integration/filters/response_metadata_filter.cc +++ b/test/integration/filters/response_metadata_filter.cc @@ -16,42 +16,39 @@ class ResponseMetadataStreamFilter : public Http::PassThroughFilter { public: // Inserts one new metadata_map. Http::FilterHeadersStatus encodeHeaders(Http::HeaderMap&, bool) override { - Http::MetadataMap metadata_map = { - {"headers", "headers"}, {"duplicate", "duplicate"}, {"remove", "remove"}}; + Http::MetadataMap metadata_map = {{"headers", "headers"}, {"duplicate", "duplicate"}}; Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); - decoder_callbacks_->encodeMetadata(std::move(metadata_map_ptr)); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); return Http::FilterHeadersStatus::Continue; } // Inserts one new metadata_map. Http::FilterDataStatus encodeData(Buffer::Instance&, bool) override { - Http::MetadataMap metadata_map = { - {"data", "data"}, {"duplicate", "duplicate"}, {"remove", "remove"}}; + Http::MetadataMap metadata_map = {{"data", "data"}, {"duplicate", "duplicate"}}; Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); - decoder_callbacks_->encodeMetadata(std::move(metadata_map_ptr)); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); return Http::FilterDataStatus::Continue; } // Inserts two metadata_maps by calling decoder_callbacks_->encodeMetadata() twice. Http::FilterTrailersStatus encodeTrailers(Http::HeaderMap&) override { - Http::MetadataMap metadata_map = {{"trailers", "trailers"}, {"remove", "remove"}}; + Http::MetadataMap metadata_map = {{"trailers", "trailers"}}; Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); - decoder_callbacks_->encodeMetadata(std::move(metadata_map_ptr)); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); metadata_map = {{"duplicate", "duplicate"}}; metadata_map_ptr = std::make_unique(metadata_map); - decoder_callbacks_->encodeMetadata(std::move(metadata_map_ptr)); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); return Http::FilterTrailersStatus::Continue; } // Inserts two metadata_maps by calling decoder_callbacks_->encodeMetadata() twice. Http::FilterHeadersStatus encode100ContinueHeaders(Http::HeaderMap&) override { - Http::MetadataMap metadata_map = { - {"100-continue", "100-continue"}, {"duplicate", "duplicate"}, {"remove", "remove"}}; + Http::MetadataMap metadata_map = {{"100-continue", "100-continue"}, {"duplicate", "duplicate"}}; Http::MetadataMapPtr metadata_map_ptr = std::make_unique(metadata_map); - decoder_callbacks_->encodeMetadata(std::move(metadata_map_ptr)); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); metadata_map = {{"duplicate", "duplicate"}}; metadata_map_ptr = std::make_unique(metadata_map); - decoder_callbacks_->encodeMetadata(std::move(metadata_map_ptr)); + encoder_callbacks_->addEncodedMetadata(std::move(metadata_map_ptr)); return Http::FilterHeadersStatus::Continue; } @@ -66,10 +63,6 @@ class ResponseMetadataStreamFilter : public Http::PassThroughFilter { metadata_map.erase("consume"); metadata_map.emplace("replace", "replace"); } - it = metadata_map.find("remove"); - if (it != metadata_map.end()) { - metadata_map.erase("remove"); - } it = metadata_map.find("metadata"); if (it != metadata_map.end()) { metadata_map.erase("metadata"); diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index f59c55e59cf1..d03a7d1eacc2 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -280,8 +280,7 @@ TEST_P(Http2MetadataIntegrationTest, TestResponseMetadata) { response->waitForEndStream(); ASSERT_TRUE(response->complete()); - // Verify metadata added in encodeHeaders(): "headers", "duplicate" and "keep". - std::set expected_metadata_keys = {"headers", "duplicate", "keep"}; + std::set expected_metadata_keys = {"headers", "duplicate"}; verifyExpectedMetadata(response->metadata_map(), expected_metadata_keys); // Upstream responds with headers and data. @@ -292,13 +291,9 @@ TEST_P(Http2MetadataIntegrationTest, TestResponseMetadata) { response->waitForEndStream(); ASSERT_TRUE(response->complete()); - // Verify metadata added in encodeHeaders(): "headers" and "duplicate" and metadata added in - // encodeData(): "data" and "duplicate" are received by the client. Note that "remove" is - // consumed. expected_metadata_keys.insert("data"); verifyExpectedMetadata(response->metadata_map(), expected_metadata_keys); EXPECT_EQ(response->keyCount("duplicate"), 2); - EXPECT_EQ(response->keyCount("keep"), 2); // Upstream responds with headers, data and trailers. response = codec_client_->makeRequestWithBody(default_request_headers_, 10); @@ -310,13 +305,9 @@ TEST_P(Http2MetadataIntegrationTest, TestResponseMetadata) { response->waitForEndStream(); ASSERT_TRUE(response->complete()); - // Verify metadata added in encodeHeaders(): "headers" and "duplicate", and metadata added in - // encodeData(): "data" and "duplicate", and metadata added in encodeTrailer(): "trailers" and - // "duplicate" are received by the client. Note that "remove" is consumed. expected_metadata_keys.insert("trailers"); verifyExpectedMetadata(response->metadata_map(), expected_metadata_keys); EXPECT_EQ(response->keyCount("duplicate"), 3); - EXPECT_EQ(response->keyCount("keep"), 4); // Upstream responds with headers, 100-continue and data. response = codec_client_->makeRequestWithBody(Http::TestHeaderMapImpl{{":method", "GET"}, @@ -334,14 +325,10 @@ TEST_P(Http2MetadataIntegrationTest, TestResponseMetadata) { response->waitForEndStream(); ASSERT_TRUE(response->complete()); - // Verify metadata added in encodeHeaders: "headers" and "duplicate", and metadata added in - // encodeData(): "data" and "duplicate", and metadata added in encode100Continue(): "100-continue" - // and "duplicate" are received by the client. Note that "remove" is consumed. expected_metadata_keys.erase("trailers"); expected_metadata_keys.insert("100-continue"); verifyExpectedMetadata(response->metadata_map(), expected_metadata_keys); EXPECT_EQ(response->keyCount("duplicate"), 4); - EXPECT_EQ(response->keyCount("keep"), 4); // Upstream responds with headers and metadata that will not be consumed. response = codec_client_->makeRequestWithBody(default_request_headers_, 10); @@ -355,19 +342,16 @@ TEST_P(Http2MetadataIntegrationTest, TestResponseMetadata) { response->waitForEndStream(); ASSERT_TRUE(response->complete()); - // Verify metadata added in encodeHeaders(): "headers" and "duplicate", and metadata added in - // encodeMetadata(): "aaa", "keep" and "duplicate" are received by the client. Note that "remove" - // is consumed. expected_metadata_keys.erase("data"); expected_metadata_keys.erase("100-continue"); expected_metadata_keys.insert("aaa"); + expected_metadata_keys.insert("keep"); verifyExpectedMetadata(response->metadata_map(), expected_metadata_keys); - EXPECT_EQ(response->keyCount("keep"), 2); // Upstream responds with headers, data and metadata that will be consumed. response = codec_client_->makeRequestWithBody(default_request_headers_, 10); waitForNextUpstreamRequest(); - metadata_map = {{"consume", "consume"}, {"remove", "remove"}}; + metadata_map = {{"consume", "consume"}}; metadata_map_ptr = std::make_unique(metadata_map); metadata_map_vector.clear(); metadata_map_vector.push_back(std::move(metadata_map_ptr)); @@ -377,15 +361,11 @@ TEST_P(Http2MetadataIntegrationTest, TestResponseMetadata) { response->waitForEndStream(); ASSERT_TRUE(response->complete()); - // Verify metadata added in encodeHeaders(): "headers" and "duplicate", and metadata added in - // encodeData(): "data", "duplicate", and metadata added in encodeMetadata(): "keep", "duplicate", - // "replace" are received by the client. Note that key "remove" and "consume" are consumed. expected_metadata_keys.erase("aaa"); expected_metadata_keys.insert("data"); expected_metadata_keys.insert("replace"); verifyExpectedMetadata(response->metadata_map(), expected_metadata_keys); EXPECT_EQ(response->keyCount("duplicate"), 2); - EXPECT_EQ(response->keyCount("keep"), 3); } TEST_P(Http2MetadataIntegrationTest, ProxyMultipleMetadataReachSizeLimit) { @@ -729,6 +709,44 @@ TEST_P(Http2MetadataIntegrationTest, RequestMetadataWithStopAllFilterAfterMetada testRequestMetadataWithStopAllFilter(); } +TEST_P(Http2MetadataIntegrationTest, TestAddEncodedMetadata) { + config_helper_.addFilter(R"EOF( +name: encode-headers-return-stop-all-filter +)EOF"); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + // Upstream responds with headers, data and trailers. + auto response = codec_client_->makeRequestWithBody(default_request_headers_, 10); + waitForNextUpstreamRequest(); + + const int count = 70; + const int size = 1000; + const int added_decoded_data_size = 1; + + default_response_headers_.addCopy("content_size", std::to_string(count * size)); + default_response_headers_.addCopy("added_size", std::to_string(added_decoded_data_size)); + default_response_headers_.addCopy("is_first_trigger", "value"); + + upstream_request_->encodeHeaders(default_response_headers_, false); + for (int i = 0; i < count - 1; i++) { + upstream_request_->encodeData(size, false); + } + + upstream_request_->encodeData(size, false); + Http::TestHeaderMapImpl response_trailers{{"response", "trailer"}}; + upstream_request_->encodeTrailers(response_trailers); + + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); + EXPECT_EQ(response->metadata_map().find("headers")->second, "headers"); + EXPECT_EQ(response->metadata_map().find("data")->second, "data"); + EXPECT_EQ(response->metadata_map().find("trailers")->second, "trailers"); + EXPECT_EQ(response->metadata_map().size(), 3); + EXPECT_EQ(count * size + added_decoded_data_size * 2, response->body().size()); +} + TEST_P(Http2IntegrationTest, GrpcRouterNotFound) { config_helper_.setDefaultHostAndRoute("foo.com", "/found"); initialize(); diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index ae7801e4176e..d550d793b999 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -959,6 +959,9 @@ TEST_P(DownstreamProtocolIntegrationTest, testEncodeHeadersReturnsStopAll) { config_helper_.addFilter(R"EOF( name: encode-headers-return-stop-all-filter )EOF"); + config_helper_.addConfigModifier( + [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); @@ -988,6 +991,9 @@ TEST_P(DownstreamProtocolIntegrationTest, testEncodeHeadersReturnsStopAllWaterma config_helper_.addFilter(R"EOF( name: encode-headers-return-stop-all-filter )EOF"); + config_helper_.addConfigModifier( + [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); }); // Sets initial stream window to min value to make the upstream sensitive to a low watermark. config_helper_.addConfigModifier( diff --git a/test/mocks/http/mocks.h b/test/mocks/http/mocks.h index c4b4dd46d669..2398873d21a8 100644 --- a/test/mocks/http/mocks.h +++ b/test/mocks/http/mocks.h @@ -224,6 +224,7 @@ class MockStreamEncoderFilterCallbacks : public StreamEncoderFilterCallbacks, MOCK_METHOD2(addEncodedData, void(Buffer::Instance& data, bool streaming)); MOCK_METHOD2(injectEncodedDataToFilterChain, void(Buffer::Instance& data, bool end_stream)); MOCK_METHOD0(addEncodedTrailers, HeaderMap&()); + MOCK_METHOD1(addEncodedMetadata, void(Http::MetadataMapPtr&&)); MOCK_METHOD0(continueEncoding, void()); MOCK_METHOD0(encodingBuffer, const Buffer::Instance*()); MOCK_METHOD1(modifyEncodingBuffer, void(std::function)); From 3d4f6fa5b65510c06031e2624efb4471f1c985db Mon Sep 17 00:00:00 2001 From: Auni Ahsan Date: Wed, 7 Aug 2019 12:44:25 -0400 Subject: [PATCH 322/542] Clarify lifetime requirements on Envoy::Event::Timer (#7812) Signed-off-by: Auni Ahsan --- include/envoy/event/timer.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/envoy/event/timer.h b/include/envoy/event/timer.h index 05fc533ac235..3a6771904cd7 100644 --- a/include/envoy/event/timer.h +++ b/include/envoy/event/timer.h @@ -16,7 +16,8 @@ namespace Event { using TimerCb = std::function; /** - * An abstract timer event. Free the timer to unregister any pending timeouts. + * An abstract timer event. Free the timer to unregister any pending timeouts. Must be freed before + * the dispatcher is torn down. */ class Timer { public: From 7a21becb10693dfc66f75237733a23fc9010449b Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 7 Aug 2019 09:52:10 -0700 Subject: [PATCH 323/542] docs: update ci/build docs (#7849) Signed-off-by: Lizan Zhou --- bazel/README.md | 28 ++++++++++++++++++++++------ ci/README.md | 5 +++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/bazel/README.md b/bazel/README.md index 497412a94170..b1be84925a72 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -1,14 +1,31 @@ # Building Envoy with Bazel +## Installing Bazelisk as Bazel + +It is recommended to use [Bazelisk](https://github.com/bazelbuild/bazelisk) installed as `bazel`, to avoid Bazel compatibility issues. +On Linux, run the following commands: + +``` +sudo wget -O /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v0.0.8/bazelisk-linux-amd64 +sudo chmod +x /usr/local/bin/bazel +``` + +On macOS, run the follwing command: +``` +brew install bazelbuild/tap/bazelisk +``` + +If you're building from an revision of Envoy prior to August 2019, which doesn't contains a `.bazelversion` file, run `ci/run_envoy_docker.sh "bazel version"` +to find the right version of Bazel and set the version to `USE_BAZEL_VERSION` environment variable to build. + ## Production environments To build Envoy with Bazel in a production environment, where the [Envoy dependencies](https://www.envoyproxy.io/docs/envoy/latest/install/building.html#requirements) are typically independently sourced, the following steps should be followed: -1. Install the latest version of [Bazel](https://bazel.build/versions/master/docs/install.html) in your environment. -2. Configure, build and/or install the [Envoy dependencies](https://www.envoyproxy.io/docs/envoy/latest/install/building.html#requirements). -3. `bazel build //source/exe:envoy-static` from the repository root. +1. Configure, build and/or install the [Envoy dependencies](https://www.envoyproxy.io/docs/envoy/latest/install/building.html#requirements). +1. `bazel build -c opt //source/exe:envoy-static` from the repository root. ## Quick start Bazel build for developers @@ -21,7 +38,6 @@ up-to-date with the latest security patches. See [this doc](https://github.com/envoyproxy/envoy/blob/master/bazel/EXTERNAL_DEPS.md#updating-an-external-dependency-version) for how to update or override dependencies. -1. Install the latest version of [Bazel](https://bazel.build/versions/master/docs/install.html) in your environment. 1. Install external dependencies libtool, cmake, ninja, realpath and curl libraries separately. On Ubuntu, run the following command: ``` @@ -109,7 +125,7 @@ accordingly. ## Building Envoy with Docker sandbox Building Envoy with Docker sandbox uses the same Docker image used in CI with fixed C++ toolchain configuration. It produces more consistent -output which is depending on your local C++ toolchain. It can also help debugging issues with RBE. To build Envoy with Docker sandbox: +output which is not depending on your local C++ toolchain. It can also help debugging issues with RBE. To build Envoy with Docker sandbox: ``` bazel build //source/exe:envoy-static --config=docker-clang @@ -490,7 +506,7 @@ have seen some issues with seeing the artifacts tab. If you can't see it, log ou then log back in and it should start working. The latest coverage report for master is available -[here](https://s3.amazonaws.com/lyft-envoy/coverage/report-master/coverage.html). +[here](https://s3.amazonaws.com/lyft-envoy/coverage/report-master/index.html). It's also possible to specialize the coverage build to a specified test or test dir. This is useful when doing things like exploring the coverage of a fuzzer over its corpus. This can be done by diff --git a/ci/README.md b/ci/README.md index d7375cfc10c2..1f3c4301fb30 100644 --- a/ci/README.md +++ b/ci/README.md @@ -23,9 +23,10 @@ master commit at which the binary was compiled, and `latest` corresponds to a bi Currently there are three build images: * `envoyproxy/envoy-build` — alias to `envoyproxy/envoy-build-ubuntu`. -* `envoyproxy/envoy-build-ubuntu` — based on Ubuntu 16.04 (Xenial) which uses the GCC 5.4 compiler. +* `envoyproxy/envoy-build-ubuntu` — based on Ubuntu 16.04 (Xenial) with GCC 7 and Clang 8 compiler. +* `envoyproxy/envoy-build-centos` — based on CentOS 7 with GCC 7 and Clang 8 compiler, this image is experimental and not well tested. -We also install and use the clang-8 compiler for some sanitizing runs. +We use the Clang compiler for all CI runs with tests. We have an additional CI run with GCC which builds binary only. # Building and running tests as a developer From feb56a1f8ee2cf1ea2048e20b7ba05f8199355c6 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Wed, 7 Aug 2019 09:54:24 -0700 Subject: [PATCH 324/542] ip tagging: fix old TODO (#7834) Signed-off-by: Derek Argueta --- .../filters/http/ip_tagging/ip_tagging_filter.h | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 3eaa7f22c841..b79103aab9bd 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "envoy/common/exception.h" @@ -44,28 +45,24 @@ class IpTaggingFilterConfig { throw EnvoyException("HTTP IP Tagging Filter requires ip_tags to be specified."); } - // TODO(ccaraman): Reduce the amount of copies operations performed to build the - // IP tag data set and passing it to the LcTrie constructor. std::vector>> tag_data; + tag_data.reserve(config.ip_tags().size()); for (const auto& ip_tag : config.ip_tags()) { - std::pair> ip_tag_pair; - ip_tag_pair.first = ip_tag.ip_tag_name(); - std::vector cidr_set; + cidr_set.reserve(ip_tag.ip_list().size()); for (const envoy::api::v2::core::CidrRange& entry : ip_tag.ip_list()) { // Currently, CidrRange::create doesn't guarantee that the CidrRanges are valid. Network::Address::CidrRange cidr_entry = Network::Address::CidrRange::create(entry); if (cidr_entry.isValid()) { - cidr_set.emplace_back(cidr_entry); + cidr_set.emplace_back(std::move(cidr_entry)); } else { throw EnvoyException( fmt::format("invalid ip/mask combo '{}/{}' (format is /<# mask bits>)", entry.address_prefix(), entry.prefix_len().value())); } } - ip_tag_pair.second = cidr_set; - tag_data.emplace_back(ip_tag_pair); + tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); } trie_ = std::make_unique>(tag_data); } From 6db95bd9732253dd90f4516252279fb5487f28dd Mon Sep 17 00:00:00 2001 From: asraa Date: Wed, 7 Aug 2019 17:11:01 -0400 Subject: [PATCH 325/542] fuzz: link libfuzzer engine to fuzz with bazel! (#7805) This makes a new config "asan-fuzzer" that links to the libfuzzer engine. To run a fuzz target with asan and libfuzzer, do bazel build --config=asan-fuzzer test/common/router:route_fuzz_test_driver --config=asan-fuzzer bazel-bin/test/common/router/route_fuzz_test_driver test/common/router/route_corpus -runs=-1 (-runs=-1 for indefinite runs, -runs=100 for bounded). Testing: works locally Risk: Low Signed-off-by: Asra Ali --- .bazelrc | 6 ++ bazel/envoy_test.bzl | 11 ++ ci/do_ci.sh | 2 +- test/fuzz/README.md | 165 ++++++++++++++++++------------ tools/gen_compilation_database.py | 4 +- 5 files changed, 119 insertions(+), 69 deletions(-) diff --git a/.bazelrc b/.bazelrc index f0e8b4be98b2..aac3e9c240ee 100644 --- a/.bazelrc +++ b/.bazelrc @@ -137,3 +137,9 @@ build:docker-clang --config=rbe-toolchain-clang # CI configurations build:remote-ci --remote_cache=grpcs://remotebuildexecution.googleapis.com build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com + +# Fuzz builds +build:asan-fuzzer --config=asan +build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer +build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +build:asan-fuzzer --copt=-fsanitize-coverage=trace-pc-guard \ No newline at end of file diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index a2d169a4f900..91d65b8803e5 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -115,6 +115,17 @@ def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs): tags = ["manual"] + tags, ) + native.cc_binary( + name = name + "_with_libfuzzer", + copts = envoy_copts("@envoy", test = True), + linkopts = ["-fsanitize=fuzzer"] + _envoy_test_linkopts(), + linkstatic = 1, + testonly = 1, + data = [corpus_name], + deps = [":" + test_lib_name], + tags = ["manual"] + tags, + ) + # Envoy C++ test targets should be specified with this function. def envoy_cc_test( name, diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 9b4dae4199d9..d526c87d1721 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -233,7 +233,7 @@ elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then exit 0 elif [[ "$CI_TARGET" == "bazel.clang_tidy" ]]; then setup_clang_toolchain - ci/run_clang_tidy.sh + NUM_CPUS=$NUM_CPUS ci/run_clang_tidy.sh exit 0 elif [[ "$CI_TARGET" == "bazel.coverity" ]]; then # Coverity Scan version 2017.07 fails to analyze the entirely of the Envoy diff --git a/test/fuzz/README.md b/test/fuzz/README.md index 0eb7f5a07df2..10849dd15c9b 100644 --- a/test/fuzz/README.md +++ b/test/fuzz/README.md @@ -1,33 +1,29 @@ # Envoy fuzz testing -Envoy is fuzz tested via [OSS-Fuzz](https://github.com/google/oss-fuzz). We -follow the best practices described in the OSS-Fuzz [ideal integration +Envoy is fuzz tested via [OSS-Fuzz](https://github.com/google/oss-fuzz). We follow the best +practices described in the OSS-Fuzz [ideal integration page](https://github.com/google/oss-fuzz/blob/master/docs/ideal_integration.md). ## Test environment Tests should be unit test-like, fast and not require writable access to the filesystem (beyond -temporary files), network (including loopback) or multiple processes. See the -[ClusterFuzz -environment](https://github.com/google/oss-fuzz/blob/master/docs/fuzzer_environment.md) -for further details. +temporary files), network (including loopback) or multiple processes. See the [ClusterFuzz +environment](https://github.com/google/oss-fuzz/blob/master/docs/fuzzer_environment.md) for further +details. ## Corpus -Every fuzz test must comes with a *corpus*. A corpus is a set of files that -provide example valid inputs. Fuzzing libraries will use this seed corpus to -drive mutations, e.g. via evolutionary fuzzing, to explore interesting parts of -the state space. +Every fuzz test must comes with a *corpus*. A corpus is a set of files that provide example valid +inputs. Fuzzing libraries will use this seed corpus to drive mutations, e.g. via evolutionary +fuzzing, to explore interesting parts of the state space. -The corpus also acts as a quick regression test for evaluating the fuzz tests -without the help of a fuzzing library. +The corpus also acts as a quick regression test for evaluating the fuzz tests without the help of a +fuzzing library. -The corpus is located in a directory underneath the fuzz test. E.g. suppose you -have -[`test/common/common/base64_fuzz_test.cc`](../../test/common/common/base64_fuzz_test.cc), -a corpus directory -[`test/common/common/base64_corpus`](../../test/common/common/base64_corpus) should -exist, populated with files that will act as the corpus. +The corpus is located in a directory underneath the fuzz test. E.g. suppose you have +[`test/common/common/base64_fuzz_test.cc`](../../test/common/common/base64_fuzz_test.cc), a corpus +directory [`test/common/common/base64_corpus`](../../test/common/common/base64_corpus) should exist, +populated with files that will act as the corpus. ## Test driver @@ -39,42 +35,45 @@ DEFINE_FUZZER(const uint8_t* data, size_t size) { } ``` -It is up to your test `DEFINE_FUZZER` implementation to map this buffer of data to -meaningful semantics, e.g. a stream of network bytes or a protobuf binary input. +It is up to your test `DEFINE_FUZZER` implementation to map this buffer of data to meaningful +semantics, e.g. a stream of network bytes or a protobuf binary input. -The fuzz test will be executed in two environments: +The fuzz test will be executed in three environments: -1. Under Envoy's fuzz test driver when run in the Envoy repository with - `bazel test //test/path/to/some_fuzz_test`. This provides a litmus test - indicating that the test passes CI and basic sanitizers on the supplied - corpus. +1. Under Envoy's fuzz test driver when run in the Envoy repository with `bazel test + //test/path/to/some_fuzz_test`. This provides a litmus test indicating that the test passes CI + and basic sanitizers just on the supplied corpus. + +1. Using the libFuzzer fuzzing engine and ASAN when run in the Envoy repository with `bazel run + //test/path/to/some_fuzz_test_with_libfuzzer --config asan-fuzzer`. This is where real fuzzing + takes place locally. The built binary can take libFuzzer command-line flags, including the number + of runs and the maximum input length. -2. Via fuzzing library test drivers in OSS-Fuzz. This is where the real fuzzing - takes places on a VM cluster and the seed corpus is used by fuzzers to - explore the state space. +3. Via fuzzing library test drivers in OSS-Fuzz. This is where the real fuzzing takes places on a VM + cluster and the seed corpus is used by fuzzers to explore the state space. ## Defining a new fuzz test -1. Write a fuzz test module implementing the `DEFINE_FUZZER` - interface. E.g. +1. Write a fuzz test module implementing the `DEFINE_FUZZER` interface. E.g. [`test/common/common/base64_fuzz_test.cc`](../../test/common/common/base64_fuzz_test.cc). 2. Define an `envoy_cc_fuzz_test` target, see `base64_fuzz_test` in [`test/common/common/BUILD`](../../test/common/common/BUILD). -3. Create the seed corpus directory and populate it with at least one example - input. E.g. +3. Create the seed corpus directory and populate it with at least one example input. E.g. [`test/common/common/base64_corpus`](../../test/common/common/base64_corpus). -4. Run the `envoy_cc_fuzz_test` target. E.g. `bazel test +4. Run the `envoy_cc_fuzz_test` target to test against the seed corpus. E.g. `bazel test //test/common/common:base64_fuzz_test`. - + +5. Run the `*_fuzz_test_with_libfuzzer` target against libFuzzer. E.g. `bazel run + //test/common/common:base64_fuzz_test --config asan-fuzzer`. + ## Protobuf fuzz tests -We also have integration with -[libprotobuf-mutator](https://github.com/google/libprotobuf-mutator), allowing -tests built on a protobuf input to work directly with a typed protobuf object, -rather than a raw buffer. The interface to this is as described at +We also have integration with [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator), +allowing tests built on a protobuf input to work directly with a typed protobuf object, rather than +a raw buffer. The interface to this is as described at https://github.com/google/libprotobuf-mutator#integrating-with-libfuzzer: ```c++ @@ -85,26 +84,61 @@ DEFINE_PROTO_FUZZER(const MyMessageType& input) { ## Running fuzz tests locally -Within the Envoy repository, we have various `*_fuzz_test` targets. When run -under `bazel test`, these will exercise the corpus as inputs but not actually -link and run against any fuzzer (e.g. -[`libfuzzer`](https://llvm.org/docs/LibFuzzer.html)). The actual fuzzing is -performed by the [oss-fuzz](https://github.com/google/oss-fuzz) project, with -results provided on the [ClusterFuzz dashboard](https://oss-fuzz.com). +Within the Envoy repository, we have various `*_fuzz_test` targets. When run under `bazel test`, +these will exercise the corpus as inputs but not actually link and run against any fuzzer (e.g. +[`libfuzzer`](https://llvm.org/docs/LibFuzzer.html)). + +To get actual fuzzing performed, the `*_fuzz_test_with_libfuzzer` target needs to be built with +`--config asan-fuzzer`. This links the target to the libFuzzer fuzzing engine. This is recommended +when writing new fuzz tests to check if they pick up any low hanging fruit (i.e. what you can find +on your local machine vs. the fuzz cluster). The binary takes the location of the seed corpus +directory. Fuzzing continues indefinitely until a bug is found or the number of iterations it should +perform is specified with `-runs`. For example, + +`bazel run //test/common/common:base64_fuzz_test_with_libfuzzer --config asan-fuzzer -- +test/common/common/base64_corpus -runs=1000` + +The fuzzer prints information to stderr: + +``` +INFO: Seed: 774517650 +INFO: Loaded 1 modules (1090433 guards): 1090433 [0x8875600, 0x8c9e404), +INFO: -max_len is not provided; libFuzzer will not generate inputs larger than 4096 bytes +INFO: A corpus is not provided, starting from an empty corpus +#2 INITED cov: 47488 ft: 30 corp: 1/1b lim: 4 exec/s: 0 rss: 139Mb +#5 NEW cov: 47499 ft: 47 corp: 2/3b lim: 4 exec/s: 0 rss: 139Mb L: 2/2 MS: 3 ChangeByte-ShuffleBytes-InsertByte- +... +#13145 NEW cov: 47506 ft: 205 corp: 17/501b lim: 128 exec/s: 6572 rss: 150Mb L: 128/128 MS: 1 CrossOver- +#16384 pulse cov: 47506 ft: 205 corp: 17/501b lim: 156 exec/s: 8192 rss: 151Mb +#32768 pulse cov: 47506 ft: 205 corp: 17/501b lim: 317 exec/s: 6553 rss: 157Mb +#39442 NEW cov: 47506 ft: 224 corp: 18/890b lim: 389 exec/s: 6573 rss: 160Mb L: 389/389 MS: 2 InsertByte-CrossOver- +``` + +Each output line reports statistics such as: +* `cov:` -- the number of code blocks or edges in the program's control flow graph observed so + far. The number grows as the fuzzer finds interesting inputs. +* `exec/s:` -- the number of fuzzer iterations per second. +* `corp:` -- size of the corpus in memory (number of elements, size in bytes) +* `rss:` -- memory consumption. + + +## Running fuzz tests in OSS-Fuzz + +Fuzzing against other engines is also performed by the +[oss-fuzz](https://github.com/google/oss-fuzz) project, with results provided on the [ClusterFuzz +dashboard](https://oss-fuzz.com). -It is possible to run against fuzzers locally by using the `oss-fuzz` Docker -image. This is recommended when writing new fuzz tests to check if they pick up -any low hanging fruit (i.e. what you can find on your local machine vs. the fuzz -cluster). +It is possible to run against fuzzers locally by using the `oss-fuzz` Docker image. This will +provide fuzzing against other fuzzing engines. 1. `git clone https://github.com/google/oss-fuzz.git` 2. `cd oss-fuzz` 3. `python infra/helper.py build_image envoy` -4. `python infra/helper.py build_fuzzers --sanitizer=address envoy `. The path to the Envoy source tree can be omitted if you - want to consume Envoy from GitHub at HEAD/master. -5. `python infra/helper.py run_fuzzer envoy `. The fuzz test - target will be the test name, e.g. `server_fuzz_test`. +4. `python infra/helper.py build_fuzzers --sanitizer=address envoy `. The + path to the Envoy source tree can be omitted if you want to consume Envoy from GitHub at + HEAD/master. +5. `python infra/helper.py run_fuzzer envoy `. The fuzz test target will be the + test name, e.g. `server_fuzz_test`. If there is a crash, `run_fuzzer` will emit a log line along the lines of: @@ -113,26 +147,23 @@ artifact_prefix='./'; Test unit written to ./crash-db2ee19f50162f2079dc0c5ba24fd ``` The test input can be found in `build/out/envoy`, e.g. -`build/out/envoy/crash-db2ee19f50162f2079dc0c5ba24fd0e3dcb8b9bc`. For protobuf -fuzz tests, this will be in text proto format. +`build/out/envoy/crash-db2ee19f50162f2079dc0c5ba24fd0e3dcb8b9bc`. For protobuf fuzz tests, this will +be in text proto format. -To test and validate fixes to the crash, add it to the corpus in the Envoy -source directory for the test, e.g. for `server_fuzz_test` this is -`test/server/corpus`, and run `bazel test`, e.g. `bazel test -//test/server:server_fuzz_test`. These crash cases can be added to the corpus in -followup PRs to provide fuzzers some interesting starting points for invalid -inputs. +To test and validate fixes to the crash, add it to the corpus in the Envoy source directory for the +test, e.g. for `server_fuzz_test` this is `test/server/corpus`, and run `bazel test`, e.g. `bazel +test //test/server:server_fuzz_test`. These crash cases can be added to the corpus in followup PRs +to provide fuzzers some interesting starting points for invalid inputs. ## Coverage reports -Coverage reports, where individual lines are annotated with fuzzing hit counts, -are a useful way to understand the scope and efficacy of the Envoy fuzzers. You -can generate such reports from the ClusterFuzz corpus following the general -ClusterFuzz [instructions for profiling +Coverage reports, where individual lines are annotated with fuzzing hit counts, are a useful way to +understand the scope and efficacy of the Envoy fuzzers. You can generate such reports from the +ClusterFuzz corpus following the general ClusterFuzz [instructions for profiling setup](https://github.com/google/oss-fuzz/blob/master/docs/code_coverage.md). -To filter out unrelated artifacts (e.g. Bazel cache, libfuzzer src), the -following profile command can be used: +To filter out unrelated artifacts (e.g. Bazel cache, libfuzzer src), the following profile command +can be used: ```bash python infra/helper.py profile envoy -- \ diff --git a/tools/gen_compilation_database.py b/tools/gen_compilation_database.py index 94869312b9e7..dadb9a1348c3 100755 --- a/tools/gen_compilation_database.py +++ b/tools/gen_compilation_database.py @@ -8,7 +8,9 @@ def generateCompilationDatabase(args): if args.run_bazel_build: - subprocess.check_call(["bazel", "build"] + args.bazel_targets) + subprocess.check_call( + ["bazel", "build", "--build_tag_filters=-manual", "--jobs=" + os.environ.get('NUM_CPUS')] + + args.bazel_targets) gen_compilation_database_sh = os.path.join( os.path.realpath(os.path.dirname(__file__)), "../bazel/gen_compilation_database.sh") From 7c7a6c9064d16c7d09cb3c0a194dd4fb9d2a7a95 Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Wed, 7 Aug 2019 14:42:18 -0700 Subject: [PATCH 326/542] Ensure the pending requests are popped before the callback is called. (#7843) Signed-off-by: Henry Yang --- .../network/common/redis/client_impl.cc | 15 +++-- .../network/common/redis/client_impl_test.cc | 58 +++++++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 20bc5f02079a..192177dca7a1 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -160,15 +160,20 @@ void ClientImpl::onEvent(Network::ConnectionEvent event) { void ClientImpl::onRespValue(RespValuePtr&& value) { ASSERT(!pending_requests_.empty()); PendingRequest& request = pending_requests_.front(); + const bool canceled = request.canceled_; + PoolCallbacks& callbacks = request.callbacks_; - if (request.canceled_) { + // We need to ensure the request is popped before calling the callback, since the callback might + // result in closing the connection. + pending_requests_.pop_front(); + if (canceled) { host_->cluster().stats().upstream_rq_cancelled_.inc(); } else if (config_.enableRedirection() && (value->type() == Common::Redis::RespType::Error)) { std::vector err = StringUtil::splitToken(value->asString(), " ", false); bool redirected = false; if (err.size() == 3) { if (err[0] == RedirectionResponse::get().MOVED || err[0] == RedirectionResponse::get().ASK) { - redirected = request.callbacks_.onRedirection(*value); + redirected = callbacks.onRedirection(*value); if (redirected) { host_->cluster().stats().upstream_internal_redirect_succeeded_total_.inc(); } else { @@ -177,14 +182,12 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { } } if (!redirected) { - request.callbacks_.onResponse(std::move(value)); + callbacks.onResponse(std::move(value)); } } else { - request.callbacks_.onResponse(std::move(value)); + callbacks.onResponse(std::move(value)); } - pending_requests_.pop_front(); - // If there are no remaining ops in the pipeline we need to disable the timer. // Otherwise we boost the timer since we are receiving responses and there are more to flush // out. diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index fd7d9ec9b7e5..064c740dacc6 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -810,6 +810,64 @@ TEST_F(RedisClientImplTest, MovedRedirectionNotEnabled) { client_->close(); } +TEST_F(RedisClientImplTest, RemoveFailedHealthCheck) { + // This test simulates a health check response signaling traffic should be drained from the host. + // As a result, the health checker will close the client in the call back. + InSequence s; + + setup(); + + Common::Redis::RespValue request1; + MockPoolCallbacks callbacks1; + EXPECT_CALL(*encoder_, encode(Ref(request1), _)); + EXPECT_CALL(*flush_timer_, enabled()).WillOnce(Return(false)); + PoolRequest* handle1 = client_->makeRequest(request1, callbacks1); + EXPECT_NE(nullptr, handle1); + + onConnected(); + + Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); + // Each call should result in either onResponse or onFailure, never both. + EXPECT_CALL(callbacks1, onFailure()).Times(0); + EXPECT_CALL(callbacks1, onResponse_(Ref(response1))) + .WillOnce(Invoke([&](Common::Redis::RespValuePtr&) { + // The health checker might fail the active health check based on the response content, and + // result in removing the host and closing the client. + client_->close(); + })); + EXPECT_CALL(*connect_or_op_timer_, disableTimer()).Times(2); + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); + callbacks_->onRespValue(std::move(response1)); +} + +TEST_F(RedisClientImplTest, RemoveFailedHost) { + // This test simulates a health check request failed due to remote host closing the connection. + // As a result the health checker will close the client in the call back. + InSequence s; + + setup(); + + NiceMock connection_callbacks; + client_->addConnectionCallbacks(connection_callbacks); + + Common::Redis::RespValue request1; + MockPoolCallbacks callbacks1; + EXPECT_CALL(*encoder_, encode(Ref(request1), _)); + EXPECT_CALL(*flush_timer_, enabled()).WillOnce(Return(false)); + PoolRequest* handle1 = client_->makeRequest(request1, callbacks1); + EXPECT_NE(nullptr, handle1); + + onConnected(); + + EXPECT_CALL(host_->outlier_detector_, + putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_FAILED, _)); + EXPECT_CALL(callbacks1, onFailure()).WillOnce(Invoke([&]() { client_->close(); })); + EXPECT_CALL(*connect_or_op_timer_, disableTimer()); + EXPECT_CALL(connection_callbacks, onEvent(Network::ConnectionEvent::RemoteClose)); + upstream_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); +} + TEST(RedisClientFactoryImplTest, Basic) { ClientFactoryImpl factory; Upstream::MockHost::MockCreateConnectionData conn_info; From 4c08c00b9f73c0b9741be04b903a1611fdb44f6b Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Wed, 7 Aug 2019 14:46:16 -0700 Subject: [PATCH 327/542] buffer filter: Populate content-length header (#7848) Signed-off-by: Ruslan Nigmatullin --- .../http_filters/buffer_filter.rst | 4 ++ docs/root/intro/version_history.rst | 1 + source/common/runtime/runtime_features.cc | 1 + source/extensions/filters/http/buffer/BUILD | 1 + .../filters/http/buffer/buffer_filter.cc | 15 ++++- .../filters/http/buffer/buffer_filter.h | 2 + test/extensions/filters/http/buffer/BUILD | 4 ++ .../buffer/buffer_filter_integration_test.cc | 23 +++++++ .../filters/http/buffer/buffer_filter_test.cc | 65 ++++++++++++++++++- 9 files changed, 113 insertions(+), 3 deletions(-) diff --git a/docs/root/configuration/http_filters/buffer_filter.rst b/docs/root/configuration/http_filters/buffer_filter.rst index edfd112fff05..828f1337ba5a 100644 --- a/docs/root/configuration/http_filters/buffer_filter.rst +++ b/docs/root/configuration/http_filters/buffer_filter.rst @@ -7,6 +7,10 @@ The buffer filter is used to stop filter iteration and wait for a fully buffered This is useful in different situations including protecting some applications from having to deal with partial requests and high network latency. +If enabled the buffer filter populates content-length header if it is not present in the request +already. The behavior can be disabled using the runtime feature +`envoy.reloadable_features.buffer_filter_populate_content_length`. + * :ref:`v2 API reference ` * This filter should be configured with the name *envoy.buffer*. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 484bc9260060..4ee932fd613a 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -6,6 +6,7 @@ Version history * access log: added :ref:`buffering ` and :ref:`periodical flushing ` support to gRPC access logger. Defaults to 16KB buffer and flushing every 1 second. * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. +* buffer filter: the buffer filter populates content-length header if not present, behavior can be disabled using the runtime feature `envoy.reloadable_features.buffer_filter_populate_content_length`. * config: added access log :ref:`extension filter`. * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 0b7c7fea2025..e1dc2a53bc91 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -25,6 +25,7 @@ namespace Runtime { constexpr const char* runtime_features[] = { // Enabled "envoy.reloadable_features.test_feature_true", + "envoy.reloadable_features.buffer_filter_populate_content_length", }; // This is a list of configuration fields which are disallowed by default in Envoy diff --git a/source/extensions/filters/http/buffer/BUILD b/source/extensions/filters/http/buffer/BUILD index 13a893a9e97a..bebfd70adf19 100644 --- a/source/extensions/filters/http/buffer/BUILD +++ b/source/extensions/filters/http/buffer/BUILD @@ -26,6 +26,7 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", + "//source/common/runtime:runtime_lib", "//source/extensions/filters/http:well_known_names", "@envoy_api//envoy/config/filter/http/buffer/v2:buffer_cc", ], diff --git a/source/extensions/filters/http/buffer/buffer_filter.cc b/source/extensions/filters/http/buffer/buffer_filter.cc index 62d836910004..f528cdf8422e 100644 --- a/source/extensions/filters/http/buffer/buffer_filter.cc +++ b/source/extensions/filters/http/buffer/buffer_filter.cc @@ -9,6 +9,7 @@ #include "common/http/header_map_impl.h" #include "common/http/headers.h" #include "common/http/utility.h" +#include "common/runtime/runtime_impl.h" #include "extensions/filters/http/well_known_names.h" @@ -57,7 +58,7 @@ void BufferFilter::initConfig() { settings_ = route_local ? route_local : settings_; } -Http::FilterHeadersStatus BufferFilter::decodeHeaders(Http::HeaderMap&, bool end_stream) { +Http::FilterHeadersStatus BufferFilter::decodeHeaders(Http::HeaderMap& headers, bool end_stream) { if (end_stream) { // If this is a header-only request, we don't need to do any buffering. return Http::FilterHeadersStatus::Continue; @@ -70,12 +71,22 @@ Http::FilterHeadersStatus BufferFilter::decodeHeaders(Http::HeaderMap&, bool end } callbacks_->setDecoderBufferLimit(settings_->maxRequestBytes()); + request_headers_ = &headers; return Http::FilterHeadersStatus::StopIteration; } -Http::FilterDataStatus BufferFilter::decodeData(Buffer::Instance&, bool end_stream) { +Http::FilterDataStatus BufferFilter::decodeData(Buffer::Instance& data, bool end_stream) { + content_length_ += data.length(); if (end_stream || settings_->disabled()) { + // request_headers_ is initialized iff plugin is enabled. + if (request_headers_ != nullptr && request_headers_->ContentLength() == nullptr) { + ASSERT(!settings_->disabled()); + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.buffer_filter_populate_content_length")) { + request_headers_->insertContentLength().value(content_length_); + } + } return Http::FilterDataStatus::Continue; } diff --git a/source/extensions/filters/http/buffer/buffer_filter.h b/source/extensions/filters/http/buffer/buffer_filter.h index 85754ae33f1c..db2539d5758b 100644 --- a/source/extensions/filters/http/buffer/buffer_filter.h +++ b/source/extensions/filters/http/buffer/buffer_filter.h @@ -65,6 +65,8 @@ class BufferFilter : public Http::StreamDecoderFilter { BufferFilterConfigSharedPtr config_; const BufferFilterSettings* settings_; Http::StreamDecoderFilterCallbacks* callbacks_{}; + Http::HeaderMap* request_headers_{}; + uint64_t content_length_{}; bool config_initialized_{}; }; diff --git a/test/extensions/filters/http/buffer/BUILD b/test/extensions/filters/http/buffer/BUILD index fab1080df5e9..0cbbdb99fea3 100644 --- a/test/extensions/filters/http/buffer/BUILD +++ b/test/extensions/filters/http/buffer/BUILD @@ -22,6 +22,10 @@ envoy_extension_cc_test( "//source/extensions/filters/http/buffer:buffer_filter_lib", "//test/mocks/buffer:buffer_mocks", "//test/mocks/http:http_mocks", + "//test/mocks/init:init_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", ], ) diff --git a/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc b/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc index 895712fb2382..d93bad947536 100644 --- a/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc +++ b/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc @@ -36,6 +36,29 @@ TEST_P(BufferIntegrationTest, RouterRequestAndResponseWithZeroByteBodyBuffer) { testRouterRequestAndResponseWithBody(0, 0, false); } +TEST_P(BufferIntegrationTest, RouterRequestPopulateContentLength) { + config_helper_.addFilter(ConfigHelper::DEFAULT_BUFFER_FILTER); + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + auto encoder_decoder = codec_client_->startRequest(Http::TestHeaderMapImpl{ + {":method", "POST"}, {":scheme", "http"}, {":path", "/shelf"}, {":authority", "host"}}); + request_encoder_ = &encoder_decoder.first; + IntegrationStreamDecoderPtr response = std::move(encoder_decoder.second); + codec_client_->sendData(*request_encoder_, "123", false); + codec_client_->sendData(*request_encoder_, "456", false); + codec_client_->sendData(*request_encoder_, "789", true); + + ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); + ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); + auto* content_length = upstream_request_->headers().ContentLength(); + ASSERT_NE(content_length, nullptr); + EXPECT_EQ(content_length->value().getStringView(), "9"); + response->waitForEndStream(); + ASSERT_TRUE(response->complete()); +} + TEST_P(BufferIntegrationTest, RouterRequestBufferLimitExceeded) { config_helper_.addFilter(ConfigHelper::SMALL_BUFFER_FILTER); initialize(); diff --git a/test/extensions/filters/http/buffer/buffer_filter_test.cc b/test/extensions/filters/http/buffer/buffer_filter_test.cc index 4af97078d5c7..12b715f5d29f 100644 --- a/test/extensions/filters/http/buffer/buffer_filter_test.cc +++ b/test/extensions/filters/http/buffer/buffer_filter_test.cc @@ -5,12 +5,18 @@ #include "envoy/event/dispatcher.h" #include "common/http/header_map_impl.h" +#include "common/runtime/runtime_impl.h" #include "extensions/filters/http/buffer/buffer_filter.h" #include "extensions/filters/http/well_known_names.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/thread_local/mocks.h" #include "test/test_common/printers.h" #include "gmock/gmock.h" @@ -36,8 +42,16 @@ class BufferFilterTest : public testing::Test { return std::make_shared(proto_config); } - BufferFilterTest() : config_(setupConfig()), filter_(config_) { + BufferFilterTest() : config_(setupConfig()), filter_(config_), api_(Api::createApiForTest()) { filter_.setDecoderFilterCallbacks(callbacks_); + + // Create a runtime loader, so that tests can manually manipulate runtime + // guarded features. + envoy::config::bootstrap::v2::LayeredRuntime config; + config.add_layers()->mutable_admin_layer(); + loader_ = std::make_unique(Runtime::LoaderPtr{ + new Runtime::LoaderImpl(dispatcher_, tls_, config, local_info_, init_manager_, store_, + generator_, validation_visitor_, *api_)}); } void routeLocalConfig(const Router::RouteSpecificFilterConfig* route_settings, @@ -52,6 +66,15 @@ class BufferFilterTest : public testing::Test { NiceMock callbacks_; BufferFilterConfigSharedPtr config_; BufferFilter filter_; + Event::MockDispatcher dispatcher_; + NiceMock tls_; + Stats::IsolatedStoreImpl store_; + Runtime::MockRandomGenerator generator_; + Api::ApiPtr api_; + NiceMock local_info_; + Init::MockManager init_manager_; + NiceMock validation_visitor_; + std::unique_ptr loader_; }; TEST_F(BufferFilterTest, HeaderOnlyRequest) { @@ -94,6 +117,46 @@ TEST_F(BufferFilterTest, TxResetAfterEndStream) { filter_.onDestroy(); } +TEST_F(BufferFilterTest, ContentLengthPopulation) { + InSequence s; + + Http::TestHeaderMapImpl headers; + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(headers, false)); + + Buffer::OwnedImpl data1("hello"); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, filter_.decodeData(data1, false)); + + Buffer::OwnedImpl data2(" world"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data2, true)); + ASSERT_NE(headers.ContentLength(), nullptr); + EXPECT_EQ(headers.ContentLength()->value().getStringView(), "11"); +} + +TEST_F(BufferFilterTest, ContentLengthPopulationAlreadyPresent) { + InSequence s; + + Http::TestHeaderMapImpl headers{{"content-length", "3"}}; + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(headers, false)); + + Buffer::OwnedImpl data("foo"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data, true)); + ASSERT_NE(headers.ContentLength(), nullptr); + EXPECT_EQ(headers.ContentLength()->value().getStringView(), "3"); +} + +TEST_F(BufferFilterTest, ContentLengthPopulationRuntimeGuard) { + InSequence s; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.buffer_filter_populate_content_length", "false"}}); + + Http::TestHeaderMapImpl headers; + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, filter_.decodeHeaders(headers, false)); + + Buffer::OwnedImpl data("foo"); + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data, true)); + EXPECT_EQ(headers.ContentLength(), nullptr); +} + TEST_F(BufferFilterTest, RouteConfigOverride) { envoy::config::filter::http::buffer::v2::BufferPerRoute route_cfg; auto* buf = route_cfg.mutable_buffer(); From 8b549bd994a1256fb1913d5dce050a016ed63877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20B=C3=A9ky?= Date: Wed, 7 Aug 2019 17:52:11 -0400 Subject: [PATCH 328/542] Do not use SpdyString wrapper for std::string. (#7854) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Béky --- .../quiche/platform/spdy_platform_test.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/extensions/quic_listeners/quiche/platform/spdy_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/spdy_platform_test.cc index 4d5629bd1bde..c2580ae40f30 100644 --- a/test/extensions/quic_listeners/quiche/platform/spdy_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/spdy_platform_test.cc @@ -1,4 +1,5 @@ #include +#include #include "extensions/quic_listeners/quiche/platform/flags_impl.h" @@ -13,7 +14,6 @@ #include "quiche/spdy/platform/api/spdy_flags.h" #include "quiche/spdy/platform/api/spdy_logging.h" #include "quiche/spdy/platform/api/spdy_ptr_util.h" -#include "quiche/spdy/platform/api/spdy_string.h" #include "quiche/spdy/platform/api/spdy_string_piece.h" #include "quiche/spdy/platform/api/spdy_test_helpers.h" @@ -43,15 +43,14 @@ TEST(SpdyPlatformTest, SpdyBugTracker) { } TEST(SpdyPlatformTest, SpdyHashMap) { - spdy::SpdyHashMap hmap; + spdy::SpdyHashMap hmap; hmap.insert({"foo", 2}); EXPECT_EQ(2, hmap["foo"]); } TEST(SpdyPlatformTest, SpdyHashSet) { - spdy::SpdyHashSet, - std::equal_to> - hset({"foo", "bar"}); + spdy::SpdyHashSet, std::equal_to> hset( + {"foo", "bar"}); EXPECT_EQ(1, hset.count("bar")); EXPECT_EQ(0, hset.count("qux")); } @@ -62,7 +61,7 @@ TEST(SpdyPlatformTest, SpdyEndianness) { } TEST(SpdyPlatformTest, SpdyEstimateMemoryUsage) { - spdy::SpdyString s = "foo"; + std::string s = "foo"; // Stubbed out to always return 0. EXPECT_EQ(0, spdy::SpdyEstimateMemoryUsage(s)); } @@ -99,12 +98,12 @@ TEST(SpdyPlatformTest, SpdyWrapUnique) { } TEST(SpdyPlatformTest, SpdyString) { - spdy::SpdyString s = "foo"; + std::string s = "foo"; EXPECT_EQ('o', s[1]); } TEST(SpdyPlatformTest, SpdyStringPiece) { - spdy::SpdyString s = "bar"; + std::string s = "bar"; spdy::SpdyStringPiece sp(s); EXPECT_EQ('b', sp[0]); } From 8e1dd3341d27173c211affb310d2c29b364a62d3 Mon Sep 17 00:00:00 2001 From: Tony Allen Date: Thu, 8 Aug 2019 07:43:58 -0700 Subject: [PATCH 329/542] Adaptive concurrency no-op implementation (#7819) Signed-off-by: Tony Allen --- CODEOWNERS | 2 + .../http/adaptive_concurrency/v2alpha/BUILD | 11 +++ .../v2alpha/adaptive_concurrency.proto | 11 +++ source/extensions/extensions_build_config.bzl | 5 +- .../filters/http/adaptive_concurrency/BUILD | 26 +++++ .../adaptive_concurrency_filter.cc | 47 +++++++++ .../adaptive_concurrency_filter.h | 71 ++++++++++++++ .../concurrency_controller/BUILD | 23 +++++ .../concurrency_controller.h | 53 ++++++++++ .../filters/http/well_known_names.h | 2 + .../filters/http/adaptive_concurrency/BUILD | 28 ++++++ .../adaptive_concurrency_filter_test.cc | 98 +++++++++++++++++++ 12 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD create mode 100644 api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto create mode 100644 source/extensions/filters/http/adaptive_concurrency/BUILD create mode 100644 source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc create mode 100644 source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h create mode 100644 source/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD create mode 100644 source/extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h create mode 100644 test/extensions/filters/http/adaptive_concurrency/BUILD create mode 100644 test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index a252439569d6..9576cbb5a308 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -44,5 +44,7 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/http/dynamic_forward_proxy @mattklein123 @alyssawilk # omit_canary_hosts retry predicate /*/extensions/retry/host/omit_canary_hosts @sriduth @snowp +# adaptive concurrency limit extension. +/*/extensions/filters/http/adaptive_concurrency @tonya11en @mattklein123 # http inspector /*/extensions/filters/listener/http_inspector @crazyxy @PiotrSikora @lizan diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD new file mode 100644 index 000000000000..948ceec2223d --- /dev/null +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "adaptive_concurrency", + srcs = ["adaptive_concurrency.proto"], + deps = [ + "//envoy/api/v2/core:base", + ], +) diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto new file mode 100644 index 000000000000..ff19657260e8 --- /dev/null +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package envoy.config.filter.http.adaptive_concurrency.v2alpha; + +option java_package = "io.envoyproxy.envoy.config.filter.http.adaptive_concurrency.v2alpha"; +option java_outer_classname = "AdaptiveConcurrencyProto"; +option java_multiple_files = true; +option go_package = "v2alpha"; + +message AdaptiveConcurrency { +} diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index c18d6d6fa47e..b96c047dab66 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -29,6 +29,9 @@ EXTENSIONS = { # HTTP filters # + # NOTE: The adaptive concurrency filter does not have a proper filter + # implemented right now. We are just referencing the filter lib here. + "envoy.filters.http.adaptive_concurrency": "//source/extensions/filters/http/adaptive_concurrency:adaptive_concurrency_filter_lib", "envoy.filters.http.buffer": "//source/extensions/filters/http/buffer:config", "envoy.filters.http.cors": "//source/extensions/filters/http/cors:config", "envoy.filters.http.csrf": "//source/extensions/filters/http/csrf:config", @@ -37,9 +40,9 @@ EXTENSIONS = { "envoy.filters.http.ext_authz": "//source/extensions/filters/http/ext_authz:config", "envoy.filters.http.fault": "//source/extensions/filters/http/fault:config", "envoy.filters.http.grpc_http1_bridge": "//source/extensions/filters/http/grpc_http1_bridge:config", + "envoy.filters.http.grpc_http1_reverse_bridge": "//source/extensions/filters/http/grpc_http1_reverse_bridge:config", "envoy.filters.http.grpc_json_transcoder": "//source/extensions/filters/http/grpc_json_transcoder:config", "envoy.filters.http.grpc_web": "//source/extensions/filters/http/grpc_web:config", - "envoy.filters.http.grpc_http1_reverse_bridge": "//source/extensions/filters/http/grpc_http1_reverse_bridge:config", "envoy.filters.http.gzip": "//source/extensions/filters/http/gzip:config", "envoy.filters.http.header_to_metadata": "//source/extensions/filters/http/header_to_metadata:config", "envoy.filters.http.health_check": "//source/extensions/filters/http/health_check:config", diff --git a/source/extensions/filters/http/adaptive_concurrency/BUILD b/source/extensions/filters/http/adaptive_concurrency/BUILD new file mode 100644 index 000000000000..5d86773ebf8c --- /dev/null +++ b/source/extensions/filters/http/adaptive_concurrency/BUILD @@ -0,0 +1,26 @@ +licenses(["notice"]) # Apache 2 + +# HTTP L7 filter that dynamically adjusts the number of allowed concurrent +# requests based on sampled latencies. +# Public docs: TODO (tonya11en) + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "adaptive_concurrency_filter_lib", + srcs = ["adaptive_concurrency_filter.cc"], + hdrs = ["adaptive_concurrency_filter.h"], + deps = [ + "//include/envoy/http:filter_interface", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/adaptive_concurrency/concurrency_controller:concurrency_controller_lib", + "//source/extensions/filters/http/common:pass_through_filter_lib", + "@envoy_api//envoy/config/filter/http/adaptive_concurrency/v2alpha:adaptive_concurrency_cc", + ], +) diff --git a/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc new file mode 100644 index 000000000000..1ec4dd8247e2 --- /dev/null +++ b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc @@ -0,0 +1,47 @@ +#include "extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h" + +#include +#include +#include +#include + +#include "common/common/assert.h" + +#include "extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h" +#include "extensions/filters/http/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace AdaptiveConcurrency { + +AdaptiveConcurrencyFilterConfig::AdaptiveConcurrencyFilterConfig( + const envoy::config::filter::http::adaptive_concurrency::v2alpha::AdaptiveConcurrency&, + Runtime::Loader&, std::string stats_prefix, Stats::Scope&, TimeSource& time_source) + : stats_prefix_(std::move(stats_prefix)), time_source_(time_source) {} + +AdaptiveConcurrencyFilter::AdaptiveConcurrencyFilter( + AdaptiveConcurrencyFilterConfigSharedPtr config, ConcurrencyControllerSharedPtr controller) + : config_(std::move(config)), controller_(std::move(controller)) {} + +Http::FilterHeadersStatus AdaptiveConcurrencyFilter::decodeHeaders(Http::HeaderMap&, bool) { + if (controller_->forwardingDecision() == ConcurrencyController::RequestForwardingAction::Block) { + // TODO (tonya11en): Remove filler words. + decoder_callbacks_->sendLocalReply(Http::Code::ServiceUnavailable, "filler words", nullptr, + absl::nullopt, "more filler words"); + return Http::FilterHeadersStatus::StopIteration; + } + + rq_start_time_ = config_->timeSource().monotonicTime(); + return Http::FilterHeadersStatus::Continue; +} + +void AdaptiveConcurrencyFilter::encodeComplete() { + const auto rq_latency = config_->timeSource().monotonicTime() - rq_start_time_; + controller_->recordLatencySample(rq_latency); +} + +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h new file mode 100644 index 000000000000..88070180272b --- /dev/null +++ b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include + +#include "envoy/common/time.h" +#include "envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.pb.h" +#include "envoy/http/filter.h" +#include "envoy/runtime/runtime.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" + +#include "extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h" +#include "extensions/filters/http/common/pass_through_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace AdaptiveConcurrency { + +/** + * Configuration for the adaptive concurrency limit filter. + */ +class AdaptiveConcurrencyFilterConfig { +public: + AdaptiveConcurrencyFilterConfig( + const envoy::config::filter::http::adaptive_concurrency::v2alpha::AdaptiveConcurrency& + adaptive_concurrency, + Runtime::Loader& runtime, std::string stats_prefix, Stats::Scope& scope, + TimeSource& time_source); + + TimeSource& timeSource() const { return time_source_; } + +private: + const std::string stats_prefix_; + TimeSource& time_source_; +}; + +using AdaptiveConcurrencyFilterConfigSharedPtr = + std::shared_ptr; +using ConcurrencyControllerSharedPtr = + std::shared_ptr; + +/** + * A filter that samples request latencies and dynamically adjusts the request + * concurrency window. + */ +class AdaptiveConcurrencyFilter : public Http::PassThroughFilter, + Logger::Loggable { +public: + AdaptiveConcurrencyFilter(AdaptiveConcurrencyFilterConfigSharedPtr config, + ConcurrencyControllerSharedPtr controller); + + // Http::StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap&, bool) override; + + // Http::StreamEncoderFilter + void encodeComplete() override; + +private: + AdaptiveConcurrencyFilterConfigSharedPtr config_; + const ConcurrencyControllerSharedPtr controller_; + MonotonicTime rq_start_time_; + std::unique_ptr forwarding_action_; +}; + +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD new file mode 100644 index 000000000000..d213690d63c6 --- /dev/null +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD @@ -0,0 +1,23 @@ +licenses(["notice"]) # Apache 2 + +# HTTP L7 filter that dynamically adjusts the number of allowed concurrent +# requests based on sampled latencies. +# Public docs: TODO (tonya11en) + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "concurrency_controller_lib", + srcs = [], + hdrs = [ + "concurrency_controller.h", + ], + deps = [ + ], +) diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h new file mode 100644 index 000000000000..0c0dbe456c7d --- /dev/null +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h @@ -0,0 +1,53 @@ +#pragma once + +#include + +#include "envoy/common/pure.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace AdaptiveConcurrency { +namespace ConcurrencyController { + +/** + * The controller's decision on whether a request will be forwarded. + */ +enum class RequestForwardingAction { + // The concurrency limit is exceeded, so the request cannot be forwarded. + Block, + + // The controller has allowed the request through and changed its internal + // state. The request must be forwarded. + Forward +}; + +/** + * Adaptive concurrency controller interface. All implementations of this + * interface must be thread-safe. + */ +class ConcurrencyController { +public: + virtual ~ConcurrencyController() = default; + + /** + * Called during decoding when the adaptive concurrency filter is attempting + * to forward a request. Returns its decision on whether to forward a request. + */ + virtual RequestForwardingAction forwardingDecision() PURE; + + /** + * Called during encoding when the request latency is known. Records the + * request latency to update the internal state of the controller for + * concurrency limit calculations. + * + * @param rq_latency is the clocked round-trip time for the request. + */ + virtual void recordLatencySample(const std::chrono::nanoseconds& rq_latency) PURE; +}; + +} // namespace ConcurrencyController +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/well_known_names.h b/source/extensions/filters/http/well_known_names.h index 3ce6643fa8fc..82341e13739f 100644 --- a/source/extensions/filters/http/well_known_names.h +++ b/source/extensions/filters/http/well_known_names.h @@ -54,6 +54,8 @@ class HttpFilterNameValues { const std::string HeaderToMetadata = "envoy.filters.http.header_to_metadata"; // Tap filter const std::string Tap = "envoy.filters.http.tap"; + // Adaptive concurrency limit filter + const std::string AdaptiveConcurrency = "envoy.filters.http.adaptive_concurrency"; // Original Src Filter const std::string OriginalSrc = "envoy.filters.http.original_src"; // Dynamic forward proxy filter diff --git a/test/extensions/filters/http/adaptive_concurrency/BUILD b/test/extensions/filters/http/adaptive_concurrency/BUILD new file mode 100644 index 000000000000..14aa0a108be2 --- /dev/null +++ b/test/extensions/filters/http/adaptive_concurrency/BUILD @@ -0,0 +1,28 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "adaptive_concurrency_filter_test", + srcs = ["adaptive_concurrency_filter_test.cc"], + extension_name = "envoy.filters.http.adaptive_concurrency", + deps = [ + "//source/common/http:header_map_lib", + "//source/common/http:headers_lib", + "//source/extensions/filters/http/adaptive_concurrency:adaptive_concurrency_filter_lib", + "//source/extensions/filters/http/adaptive_concurrency/concurrency_controller:concurrency_controller_lib", + "//test/mocks/http:http_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_test.cc b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_test.cc new file mode 100644 index 000000000000..60ad871dc3e5 --- /dev/null +++ b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_test.cc @@ -0,0 +1,98 @@ +#include + +#include "extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h" +#include "extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h" + +#include "test/mocks/http/mocks.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace AdaptiveConcurrency { +namespace { + +using ConcurrencyController::RequestForwardingAction; + +class MockConcurrencyController : public ConcurrencyController::ConcurrencyController { +public: + MOCK_METHOD0(forwardingDecision, RequestForwardingAction()); + MOCK_METHOD1(recordLatencySample, void(const std::chrono::nanoseconds&)); +}; + +class AdaptiveConcurrencyFilterTest : public testing::Test { +public: + AdaptiveConcurrencyFilterTest() { + filter_.reset(); + + const envoy::config::filter::http::adaptive_concurrency::v2alpha::AdaptiveConcurrency config; + auto config_ptr = std::make_shared( + config, runtime_, "testprefix.", stats_, time_system_); + + filter_ = std::make_unique(config_ptr, controller_); + filter_->setDecoderFilterCallbacks(decoder_callbacks_); + } + + std::unique_ptr filter_; + Event::SimulatedTimeSystem time_system_; + Stats::IsolatedStoreImpl stats_; + NiceMock runtime_; + std::shared_ptr controller_{new MockConcurrencyController()}; + NiceMock decoder_callbacks_; +}; + +TEST_F(AdaptiveConcurrencyFilterTest, DecodeHeadersTestForwarding) { + Http::TestHeaderMapImpl request_headers; + + EXPECT_CALL(*controller_, forwardingDecision()) + .WillOnce(Return(RequestForwardingAction::Forward)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + + Buffer::OwnedImpl request_body; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(request_body, false)); + + Http::TestHeaderMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); +} + +TEST_F(AdaptiveConcurrencyFilterTest, DecodeHeadersTestBlock) { + Http::TestHeaderMapImpl request_headers; + + EXPECT_CALL(*controller_, forwardingDecision()).WillOnce(Return(RequestForwardingAction::Block)); + EXPECT_CALL(decoder_callbacks_, sendLocalReply(Http::Code::ServiceUnavailable, _, _, _, _)); + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_->decodeHeaders(request_headers, true)); +} + +TEST_F(AdaptiveConcurrencyFilterTest, EncodeHeadersValidTest) { + auto mt = time_system_.monotonicTime(); + time_system_.setMonotonicTime(mt + std::chrono::milliseconds(123)); + + // Get the filter to record the request start time via decode. + Http::TestHeaderMapImpl request_headers; + EXPECT_CALL(*controller_, forwardingDecision()) + .WillOnce(Return(RequestForwardingAction::Forward)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); + + const std::chrono::nanoseconds advance_time = std::chrono::milliseconds(42); + mt = time_system_.monotonicTime(); + time_system_.setMonotonicTime(mt + advance_time); + + Http::TestHeaderMapImpl response_headers; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, false)); + EXPECT_CALL(*controller_, recordLatencySample(advance_time)); + filter_->encodeComplete(); +} + +} // namespace +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy From 3fdd00df5df6a101d2f835f6a60749dde8fc91e4 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Thu, 8 Aug 2019 11:03:02 -0400 Subject: [PATCH 330/542] stats: More StatName conversions (#7810) * Convert a few more counter() references to use the StatName interface. Signed-off-by: Joshua Marantz --- include/envoy/stats/symbol_table.h | 3 +- source/common/http/codes.cc | 5 +- source/common/http/codes.h | 5 +- source/common/stats/symbol_table_impl.cc | 32 +++++++- source/common/stats/symbol_table_impl.h | 48 ++++++++++- .../extensions/filters/http/dynamo/config.cc | 2 +- .../filters/http/dynamo/dynamo_stats.cc | 79 ++++++------------- .../filters/http/dynamo/dynamo_stats.h | 20 ++--- .../filters/http/ext_authz/ext_authz.cc | 8 +- .../filters/http/ext_authz/ext_authz.h | 16 +++- .../filters/network/zookeeper_proxy/BUILD | 1 + .../filters/network/zookeeper_proxy/filter.cc | 20 ++++- .../filters/network/zookeeper_proxy/filter.h | 5 +- test/common/stats/symbol_table_impl_test.cc | 36 ++++++++- .../filters/http/ext_authz/ext_authz_test.cc | 2 +- .../network/zookeeper_proxy/filter_test.cc | 2 +- tools/check_format.py | 4 +- 17 files changed, 198 insertions(+), 90 deletions(-) diff --git a/include/envoy/stats/symbol_table.h b/include/envoy/stats/symbol_table.h index 4b4c8a4c4fe9..8f15df4534da 100644 --- a/include/envoy/stats/symbol_table.h +++ b/include/envoy/stats/symbol_table.h @@ -19,6 +19,7 @@ namespace Stats { * declaration for StatName is in source/common/stats/symbol_table_impl.h */ class StatName; +using StatNameVec = std::vector; class StatNameList; @@ -105,7 +106,7 @@ class SymbolTable { * @param stat_names the names to join. * @return Storage allocated for the joined name. */ - virtual StoragePtr join(const std::vector& stat_names) const PURE; + virtual StoragePtr join(const StatNameVec& stat_names) const PURE; /** * Populates a StatNameList from a list of encodings. This is not done at diff --git a/source/common/http/codes.cc b/source/common/http/codes.cc index 746b38a45c5e..1afa8c841659 100644 --- a/source/common/http/codes.cc +++ b/source/common/http/codes.cc @@ -47,8 +47,7 @@ CodeStatsImpl::CodeStatsImpl(Stats::SymbolTable& symbol_table) upstreamRqStatName(Code::ServiceUnavailable); } -void CodeStatsImpl::incCounter(Stats::Scope& scope, - const std::vector& names) const { +void CodeStatsImpl::incCounter(Stats::Scope& scope, const Stats::StatNameVec& names) const { const Stats::SymbolTable::StoragePtr stat_name_storage = symbol_table_.join(names); scope.counterFromStatName(Stats::StatName(stat_name_storage.get())).inc(); } @@ -58,7 +57,7 @@ void CodeStatsImpl::incCounter(Stats::Scope& scope, Stats::StatName a, Stats::St scope.counterFromStatName(Stats::StatName(stat_name_storage.get())).inc(); } -void CodeStatsImpl::recordHistogram(Stats::Scope& scope, const std::vector& names, +void CodeStatsImpl::recordHistogram(Stats::Scope& scope, const Stats::StatNameVec& names, uint64_t count) const { const Stats::SymbolTable::StoragePtr stat_name_storage = symbol_table_.join(names); scope.histogramFromStatName(Stats::StatName(stat_name_storage.get())).recordValue(count); diff --git a/source/common/http/codes.h b/source/common/http/codes.h index 66e4f5019a53..45d51a29ff91 100644 --- a/source/common/http/codes.h +++ b/source/common/http/codes.h @@ -54,10 +54,9 @@ class CodeStatsImpl : public CodeStats { void writeCategory(const ResponseStatInfo& info, Stats::StatName rq_group, Stats::StatName rq_code, Stats::StatName category) const; - void incCounter(Stats::Scope& scope, const std::vector& names) const; + void incCounter(Stats::Scope& scope, const Stats::StatNameVec& names) const; void incCounter(Stats::Scope& scope, Stats::StatName a, Stats::StatName b) const; - void recordHistogram(Stats::Scope& scope, const std::vector& names, - uint64_t count) const; + void recordHistogram(Stats::Scope& scope, const Stats::StatNameVec& names, uint64_t count) const; static absl::string_view stripTrailingDot(absl::string_view prefix); Stats::StatName upstreamRqGroup(Code response_code) const; diff --git a/source/common/stats/symbol_table_impl.cc b/source/common/stats/symbol_table_impl.cc index c1541f8ecda3..9a2129eb04f0 100644 --- a/source/common/stats/symbol_table_impl.cc +++ b/source/common/stats/symbol_table_impl.cc @@ -361,7 +361,7 @@ void StatNameStorageSet::free(SymbolTable& symbol_table) { } } -SymbolTable::StoragePtr SymbolTableImpl::join(const std::vector& stat_names) const { +SymbolTable::StoragePtr SymbolTableImpl::join(const StatNameVec& stat_names) const { uint64_t num_bytes = 0; for (StatName stat_name : stat_names) { num_bytes += stat_name.dataSize(); @@ -429,5 +429,35 @@ void StatNameList::clear(SymbolTable& symbol_table) { storage_.reset(); } +StatNameSet::StatNameSet(SymbolTable& symbol_table) : pool_(symbol_table) { + builtin_stat_names_[""] = StatName(); +} + +void StatNameSet::rememberBuiltin(absl::string_view str) { + StatName stat_name; + { + absl::MutexLock lock(&mutex_); + stat_name = pool_.add(str); + } + builtin_stat_names_[str] = stat_name; +} + +Stats::StatName StatNameSet::getStatName(absl::string_view token) { + // If token was recorded as a built-in during initialization, we can + // service this request lock-free. + const auto iter = builtin_stat_names_.find(token); + if (iter != builtin_stat_names_.end()) { + return iter->second; + } + + // Other tokens require holding a lock for our local cache. + absl::MutexLock lock(&mutex_); + Stats::StatName& stat_name = dynamic_stat_names_[token]; + if (stat_name.empty()) { // Note that builtin_stat_names_ already has one for "". + stat_name = pool_.add(token); + } + return stat_name; +} + } // namespace Stats } // namespace Envoy diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index 8860980017e2..b64e9ff59df1 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -132,7 +132,7 @@ class SymbolTableImpl : public SymbolTable { bool lessThan(const StatName& a, const StatName& b) const override; void free(const StatName& stat_name) override; void incRefCount(const StatName& stat_name) override; - StoragePtr join(const std::vector& stat_names) const override; + StoragePtr join(const StatNameVec& stat_names) const override; void populateList(const absl::string_view* names, uint32_t num_names, StatNameList& list) override; StoragePtr encode(absl::string_view name) override; @@ -630,5 +630,51 @@ class StatNameStorageSet { HashSet hash_set_; }; +// Captures StatNames for lookup by string, keeping two maps: a map of +// 'built-ins' that is expected to be populated during initialization, and a map +// of dynamically discovered names. The latter map is protected by a mutex, and +// can be mutated at runtime. +// +// Ideally, builtins should be added during process initialization, in the +// outermost relevant context. And as the builtins map is not mutex protected, +// builtins must *not* be added in the request-path. +class StatNameSet { +public: + explicit StatNameSet(SymbolTable& symbol_table); + + /** + * Adds a string to the builtin map, which is not mutex protected. This map is + * always consulted first as a hit there means no lock is required. + * + * Builtins can only be added immediately after construction, as the builtins + * map is not mutex-protected. + */ + void rememberBuiltin(absl::string_view str); + + /** + * Finds a StatName by name. If 'str' has been remembered as a built-in, then + * no lock is required. Otherwise we first consult dynamic_stat_names_ under a + * lock that's private to the StatNameSet. If that's empty, we need to create + * the StatName in the pool, which requires taking a global lock. + * + * TODO(jmarantz): Potential perf issue here with contention, both on this + * set's mutex and also the SymbolTable mutex which must be taken during + * StatNamePool::add(). + */ + StatName getStatName(absl::string_view str); + + /** + * Adds a StatName using the pool, but without remembering it in any maps. + */ + StatName add(absl::string_view str) { return pool_.add(str); } + +private: + Stats::StatNamePool pool_; + absl::Mutex mutex_; + using StringStatNameMap = absl::flat_hash_map; + StringStatNameMap builtin_stat_names_; + StringStatNameMap dynamic_stat_names_ GUARDED_BY(mutex_); +}; + } // namespace Stats } // namespace Envoy diff --git a/source/extensions/filters/http/dynamo/config.cc b/source/extensions/filters/http/dynamo/config.cc index 77dde48be63c..5762c8f827e8 100644 --- a/source/extensions/filters/http/dynamo/config.cc +++ b/source/extensions/filters/http/dynamo/config.cc @@ -16,7 +16,7 @@ Http::FilterFactoryCb DynamoFilterConfig::createFilter(const std::string& stat_prefix, Server::Configuration::FactoryContext& context) { auto stats = std::make_shared(context.scope(), stat_prefix); - return [&context, stat_prefix, stats](Http::FilterChainFactoryCallbacks& callbacks) -> void { + return [&context, stats](Http::FilterChainFactoryCallbacks& callbacks) -> void { callbacks.addStreamFilter(std::make_shared( context.runtime(), stats, context.dispatcher().timeSource())); }; diff --git a/source/extensions/filters/http/dynamo/dynamo_stats.cc b/source/extensions/filters/http/dynamo/dynamo_stats.cc index 51e12d96d452..543fcc5f82d5 100644 --- a/source/extensions/filters/http/dynamo/dynamo_stats.cc +++ b/source/extensions/filters/http/dynamo/dynamo_stats.cc @@ -1,6 +1,5 @@ #include "extensions/filters/http/dynamo/dynamo_stats.h" -#include #include #include @@ -16,44 +15,44 @@ namespace HttpFilters { namespace Dynamo { DynamoStats::DynamoStats(Stats::Scope& scope, const std::string& prefix) - : scope_(scope), pool_(scope.symbolTable()), prefix_(pool_.add(prefix + "dynamodb")), - batch_failure_unprocessed_keys_(pool_.add("BatchFailureUnprocessedKeys")), - capacity_(pool_.add("capacity")), empty_response_body_(pool_.add("empty_response_body")), - error_(pool_.add("error")), invalid_req_body_(pool_.add("invalid_req_body")), - invalid_resp_body_(pool_.add("invalid_resp_body")), - multiple_tables_(pool_.add("multiple_tables")), no_table_(pool_.add("no_table")), - operation_missing_(pool_.add("operation_missing")), table_(pool_.add("table")), - table_missing_(pool_.add("table_missing")), upstream_rq_time_(pool_.add("upstream_rq_time")), - upstream_rq_total_(pool_.add("upstream_rq_total")) { - upstream_rq_total_groups_[0] = pool_.add("upstream_rq_total_unknown"); - upstream_rq_time_groups_[0] = pool_.add("upstream_rq_time_unknown"); + : scope_(scope), stat_name_set_(scope.symbolTable()), + prefix_(stat_name_set_.add(prefix + "dynamodb")), + batch_failure_unprocessed_keys_(stat_name_set_.add("BatchFailureUnprocessedKeys")), + capacity_(stat_name_set_.add("capacity")), + empty_response_body_(stat_name_set_.add("empty_response_body")), + error_(stat_name_set_.add("error")), + invalid_req_body_(stat_name_set_.add("invalid_req_body")), + invalid_resp_body_(stat_name_set_.add("invalid_resp_body")), + multiple_tables_(stat_name_set_.add("multiple_tables")), + no_table_(stat_name_set_.add("no_table")), + operation_missing_(stat_name_set_.add("operation_missing")), + table_(stat_name_set_.add("table")), table_missing_(stat_name_set_.add("table_missing")), + upstream_rq_time_(stat_name_set_.add("upstream_rq_time")), + upstream_rq_total_(stat_name_set_.add("upstream_rq_total")) { + upstream_rq_total_groups_[0] = stat_name_set_.add("upstream_rq_total_unknown"); + upstream_rq_time_groups_[0] = stat_name_set_.add("upstream_rq_time_unknown"); for (size_t i = 1; i < DynamoStats::NumGroupEntries; ++i) { - upstream_rq_total_groups_[i] = pool_.add(fmt::format("upstream_rq_total_{}xx", i)); - upstream_rq_time_groups_[i] = pool_.add(fmt::format("upstream_rq_time_{}xx", i)); + upstream_rq_total_groups_[i] = stat_name_set_.add(fmt::format("upstream_rq_total_{}xx", i)); + upstream_rq_time_groups_[i] = stat_name_set_.add(fmt::format("upstream_rq_time_{}xx", i)); } - RequestParser::forEachStatString([this](const std::string& str) { - // Thread annotation does not realize this function is only called from the - // constructor, so we need to lock the mutex. It's easier to just do that - // and it's no real penalty. - absl::MutexLock lock(&mutex_); - builtin_stat_names_[str] = pool_.add(str); - }); - builtin_stat_names_[""] = Stats::StatName(); + RequestParser::forEachStatString( + [this](const std::string& str) { stat_name_set_.rememberBuiltin(str); }); } -Stats::SymbolTable::StoragePtr DynamoStats::addPrefix(const std::vector& names) { - std::vector names_with_prefix{prefix_}; - names_with_prefix.reserve(names.end() - names.begin()); +Stats::SymbolTable::StoragePtr DynamoStats::addPrefix(const Stats::StatNameVec& names) { + Stats::StatNameVec names_with_prefix; + names_with_prefix.reserve(1 + names.size()); + names_with_prefix.push_back(prefix_); names_with_prefix.insert(names_with_prefix.end(), names.begin(), names.end()); return scope_.symbolTable().join(names_with_prefix); } -Stats::Counter& DynamoStats::counter(const std::vector& names) { +Stats::Counter& DynamoStats::counter(const Stats::StatNameVec& names) { const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(names); return scope_.counterFromStatName(Stats::StatName(stat_name_storage.get())); } -Stats::Histogram& DynamoStats::histogram(const std::vector& names) { +Stats::Histogram& DynamoStats::histogram(const Stats::StatNameVec& names) { const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(names); return scope_.histogramFromStatName(Stats::StatName(stat_name_storage.get())); } @@ -77,32 +76,6 @@ size_t DynamoStats::groupIndex(uint64_t status) { return index; } -Stats::StatName DynamoStats::getStatName(const std::string& token) { - // The Dynamo system has a few well-known tokens that we have stored during - // construction, and we can access StatNames for those without a lock. Note - // that we only mutate builtin_stat_names_ during construction. - auto iter = builtin_stat_names_.find(token); - if (iter != builtin_stat_names_.end()) { - return iter->second; - } - - // However, some of the tokens come from json data received during requests, - // so we need to store these in a mutex-protected map. Once we hold the mutex, - // we can check dynamic_stat_names_. If it's missing we'll have to add it - // to the symbol table, whose mutex is more likely to be contended, and then - // store it in dynamic_stat_names. - // - // TODO(jmarantz): Potential perf issue here with contention, both on this - // mutex and also the SymbolTable mutex which must be taken during - // StatNamePool::add(). - absl::MutexLock lock(&mutex_); - Stats::StatName& stat_name = dynamic_stat_names_[token]; - if (stat_name.empty()) { // Note that builtin_stat_names_ already has one for "". - stat_name = pool_.add(token); - } - return stat_name; -} - } // namespace Dynamo } // namespace HttpFilters } // namespace Extensions diff --git a/source/extensions/filters/http/dynamo/dynamo_stats.h b/source/extensions/filters/http/dynamo/dynamo_stats.h index 57671514e581..40837a95f0e9 100644 --- a/source/extensions/filters/http/dynamo/dynamo_stats.h +++ b/source/extensions/filters/http/dynamo/dynamo_stats.h @@ -16,8 +16,8 @@ class DynamoStats { public: DynamoStats(Stats::Scope& scope, const std::string& prefix); - Stats::Counter& counter(const std::vector& names); - Stats::Histogram& histogram(const std::vector& names); + Stats::Counter& counter(const Stats::StatNameVec& names); + Stats::Histogram& histogram(const Stats::StatNameVec& names); /** * Creates the partition id stats string. The stats format is @@ -31,18 +31,20 @@ class DynamoStats { static size_t groupIndex(uint64_t status); - Stats::StatName getStatName(const std::string& str); + /** + * Finds or creates a StatName by string, taking a global lock if needed. + * + * TODO(jmarantz): Potential perf issue here with mutex contention for names + * that have not been remembered as builtins in the constructor. + */ + Stats::StatName getStatName(const std::string& str) { return stat_name_set_.getStatName(str); } private: - Stats::SymbolTable::StoragePtr addPrefix(const std::vector& names); + Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatNameVec& names); Stats::Scope& scope_; - Stats::StatNamePool pool_ GUARDED_BY(mutex_); + Stats::StatNameSet stat_name_set_; const Stats::StatName prefix_; - absl::Mutex mutex_; - using StringStatNameMap = absl::flat_hash_map; - StringStatNameMap builtin_stat_names_; - StringStatNameMap dynamic_stat_names_ GUARDED_BY(mutex_); public: const Stats::StatName batch_failure_unprocessed_keys_; diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index 3f1a1535f8aa..87b566079927 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -169,7 +169,7 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { Http::HeaderMapImpl::appendToHeader(header_to_modify->value(), header.second); } } - cluster_->statsScope().counter("ext_authz.ok").inc(); + config_->incCounter(cluster_->statsScope(), config_->ext_authz_ok_); continueDecoding(); break; } @@ -177,7 +177,7 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { case CheckStatus::Denied: { ENVOY_STREAM_LOG(trace, "ext_authz filter rejected the request. Response status code: '{}", *callbacks_, enumToInt(response->status_code)); - cluster_->statsScope().counter("ext_authz.denied").inc(); + config_->incCounter(cluster_->statsScope(), config_->ext_authz_denied_); Http::CodeStats::ResponseStatInfo info{config_->scope(), cluster_->statsScope(), empty_stat_name, @@ -207,10 +207,10 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { } case CheckStatus::Error: { - cluster_->statsScope().counter("ext_authz.error").inc(); + config_->incCounter(cluster_->statsScope(), config_->ext_authz_error_); if (config_->failureModeAllow()) { ENVOY_STREAM_LOG(trace, "ext_authz filter allowed the request with error", *callbacks_); - cluster_->statsScope().counter("ext_authz.failure_mode_allowed").inc(); + config_->incCounter(cluster_->statsScope(), config_->ext_authz_failure_mode_allowed_); continueDecoding(); } else { ENVOY_STREAM_LOG( diff --git a/source/extensions/filters/http/ext_authz/ext_authz.h b/source/extensions/filters/http/ext_authz/ext_authz.h index 75831ed9fc5e..0b0528b53403 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.h +++ b/source/extensions/filters/http/ext_authz/ext_authz.h @@ -46,7 +46,10 @@ class FilterConfig { clear_route_cache_(config.clear_route_cache()), max_request_bytes_(config.with_request_body().max_request_bytes()), status_on_error_(toErrorCode(config.status_on_error().code())), local_info_(local_info), - scope_(scope), runtime_(runtime), http_context_(http_context) {} + scope_(scope), runtime_(runtime), http_context_(http_context), pool_(scope.symbolTable()), + ext_authz_ok_(pool_.add("ext_authz.ok")), ext_authz_denied_(pool_.add("ext_authz.denied")), + ext_authz_error_(pool_.add("ext_authz.error")), + ext_authz_failure_mode_allowed_(pool_.add("ext_authz.failure_mode_allowed")) {} bool allowPartialMessage() const { return allow_partial_message_; } @@ -68,6 +71,10 @@ class FilterConfig { Http::Context& httpContext() { return http_context_; } + void incCounter(Stats::Scope& scope, Stats::StatName name) { + scope.counterFromStatName(name).inc(); + } + private: static Http::Code toErrorCode(uint64_t status) { const auto code = static_cast(status); @@ -86,6 +93,13 @@ class FilterConfig { Stats::Scope& scope_; Runtime::Loader& runtime_; Http::Context& http_context_; + Stats::StatNamePool pool_; + +public: + const Stats::StatName ext_authz_ok_; + const Stats::StatName ext_authz_denied_; + const Stats::StatName ext_authz_error_; + const Stats::StatName ext_authz_failure_mode_allowed_; }; using FilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/network/zookeeper_proxy/BUILD b/source/extensions/filters/network/zookeeper_proxy/BUILD index 26d144167c51..8956cd975fe1 100644 --- a/source/extensions/filters/network/zookeeper_proxy/BUILD +++ b/source/extensions/filters/network/zookeeper_proxy/BUILD @@ -31,6 +31,7 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/common:enum_to_int", "//source/common/network:filter_lib", + "//source/common/stats:symbol_table_lib", "//source/extensions/filters/network:well_known_names", ], ) diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.cc b/source/extensions/filters/network/zookeeper_proxy/filter.cc index 6cb5ab105d95..1bf5870d6914 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/filter.cc @@ -18,8 +18,19 @@ namespace ZooKeeperProxy { ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, const uint32_t max_packet_bytes, Stats::Scope& scope) - : scope_(scope), max_packet_bytes_(max_packet_bytes), stat_prefix_(stat_prefix), - stats_(generateStats(stat_prefix, scope)) {} + : scope_(scope), max_packet_bytes_(max_packet_bytes), stats_(generateStats(stat_prefix, scope)), + stat_name_set_(scope.symbolTable()), stat_prefix_(stat_name_set_.add(stat_prefix)), + auth_(stat_name_set_.add("auth")) { + // https://zookeeper.apache.org/doc/r3.5.4-beta/zookeeperProgrammers.html#sc_BuiltinACLSchemes + // lists commons schemes: "world", "auth", "digest", "host", "x509", and + // "ip". These are used in filter.cc by appending "_rq". + stat_name_set_.rememberBuiltin("auth_rq"); + stat_name_set_.rememberBuiltin("digest_rq"); + stat_name_set_.rememberBuiltin("host_rq"); + stat_name_set_.rememberBuiltin("ip_rq"); + stat_name_set_.rememberBuiltin("world_rq"); + stat_name_set_.rememberBuiltin("x509_rq"); +} ZooKeeperFilter::ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config) : config_(std::move(config)), decoder_(createDecoder(*this)) {} @@ -107,7 +118,10 @@ void ZooKeeperFilter::onPing() { } void ZooKeeperFilter::onAuthRequest(const std::string& scheme) { - config_->scope_.counter(fmt::format("{}.auth.{}_rq", config_->stat_prefix_, scheme)).inc(); + Stats::SymbolTable::StoragePtr storage = config_->scope_.symbolTable().join( + {config_->stat_prefix_, config_->auth_, + config_->stat_name_set_.getStatName(absl::StrCat(scheme, "_rq"))}); + config_->scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); setDynamicMetadata("opname", "auth"); } diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.h b/source/extensions/filters/network/zookeeper_proxy/filter.h index d64918c2b8d4..c68303e8d4b2 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/filter.h @@ -12,6 +12,7 @@ #include "envoy/stats/stats_macros.h" #include "common/common/logger.h" +#include "common/stats/symbol_table_impl.h" #include "extensions/filters/network/zookeeper_proxy/decoder.h" @@ -103,8 +104,10 @@ class ZooKeeperFilterConfig { Stats::Scope& scope_; const uint32_t max_packet_bytes_; - const std::string stat_prefix_; ZooKeeperProxyStats stats_; + Stats::StatNameSet stat_name_set_; + const Stats::StatName stat_prefix_; + const Stats::StatName auth_; private: ZooKeeperProxyStats generateStats(const std::string& prefix, Stats::Scope& scope) { diff --git a/test/common/stats/symbol_table_impl_test.cc b/test/common/stats/symbol_table_impl_test.cc index bfd856eabf61..d6a533b092a5 100644 --- a/test/common/stats/symbol_table_impl_test.cc +++ b/test/common/stats/symbol_table_impl_test.cc @@ -319,10 +319,10 @@ TEST_P(StatNameTest, HashTable) { } TEST_P(StatNameTest, Sort) { - std::vector names{makeStat("a.c"), makeStat("a.b"), makeStat("d.e"), - makeStat("d.a.a"), makeStat("d.a"), makeStat("a.c")}; - const std::vector sorted_names{makeStat("a.b"), makeStat("a.c"), makeStat("a.c"), - makeStat("d.a"), makeStat("d.a.a"), makeStat("d.e")}; + StatNameVec names{makeStat("a.c"), makeStat("a.b"), makeStat("d.e"), + makeStat("d.a.a"), makeStat("d.a"), makeStat("a.c")}; + const StatNameVec sorted_names{makeStat("a.b"), makeStat("a.c"), makeStat("a.c"), + makeStat("d.a"), makeStat("d.a.a"), makeStat("d.e")}; EXPECT_NE(names, sorted_names); std::sort(names.begin(), names.end(), StatNameLessThan(*table_)); EXPECT_EQ(names, sorted_names); @@ -537,6 +537,34 @@ TEST_P(StatNameTest, SharedStatNameStorageSetSwap) { set2.free(*table_); } +TEST_P(StatNameTest, StatNameSet) { + StatNameSet set(*table_); + + // Test that we get a consistent StatName object from a remembered name. + set.rememberBuiltin("remembered"); + const Stats::StatName remembered = set.getStatName("remembered"); + EXPECT_EQ("remembered", table_->toString(remembered)); + EXPECT_EQ(remembered.data(), set.getStatName("remembered").data()); + + // Same test for a dynamically allocated name. The only difference between + // the behavior with a remembered vs dynamic name is that when looking + // up a remembered name, a mutex is not taken. But we have no easy way + // to test for that. So we'll at least cover the code. + const Stats::StatName dynamic = set.getStatName("dynamic"); + EXPECT_EQ("dynamic", table_->toString(dynamic)); + EXPECT_EQ(dynamic.data(), set.getStatName("dynamic").data()); + + // There's another corner case for the same "dynamic" name from a + // different set. Here we will get a different StatName object + // out of the second set, though it will share the same underlying + // symbol-table symbol. + StatNameSet set2(*table_); + const Stats::StatName dynamic2 = set2.getStatName("dynamic"); + EXPECT_EQ("dynamic", table_->toString(dynamic2)); + EXPECT_EQ(dynamic2.data(), set2.getStatName("dynamic").data()); + EXPECT_NE(dynamic2.data(), dynamic.data()); +} + // Tests the memory savings realized from using symbol tables with 1k // clusters. This test shows the memory drops from almost 8M to less than // 2M. Note that only SymbolTableImpl is tested for memory consumption, diff --git a/test/extensions/filters/http/ext_authz/ext_authz_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_test.cc index 76c3ca3759a9..d82354c0e917 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_test.cc @@ -63,6 +63,7 @@ template class HttpFilterTestBase : public T { addr_ = std::make_shared("1.2.3.4", 1111); } + NiceMock stats_store_; FilterConfigSharedPtr config_; Filters::Common::ExtAuthz::MockClient* client_; std::unique_ptr filter_; @@ -70,7 +71,6 @@ template class HttpFilterTestBase : public T { Filters::Common::ExtAuthz::RequestCallbacks* request_callbacks_; Http::TestHeaderMapImpl request_headers_; Buffer::OwnedImpl data_; - NiceMock stats_store_; NiceMock runtime_; NiceMock cm_; NiceMock local_info_; diff --git a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc index c2f37fcc197b..5536bf8de056 100644 --- a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc @@ -491,9 +491,9 @@ class ZooKeeperFilterTest : public testing::Test { EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); } + Stats::IsolatedStoreImpl scope_; ZooKeeperFilterConfigSharedPtr config_; std::unique_ptr filter_; - Stats::IsolatedStoreImpl scope_; std::string stat_prefix_{"test.zookeeper"}; NiceMock filter_callbacks_; NiceMock stream_info_; diff --git a/tools/check_format.py b/tools/check_format.py index 8f5759c5aa96..54eda668bc4f 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -48,11 +48,9 @@ # https://github.com/envoyproxy/envoy/pull/7573 and others. # # TODO(#4196): Eliminate this list completely and then merge #4980. -STAT_FROM_STRING_WHITELIST = ("./source/extensions/filters/http/ext_authz/ext_authz.cc", - "./source/extensions/filters/http/fault/fault_filter.cc", +STAT_FROM_STRING_WHITELIST = ("./source/extensions/filters/http/fault/fault_filter.cc", "./source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc", "./source/extensions/filters/network/mongo_proxy/proxy.cc", - "./source/extensions/filters/network/zookeeper_proxy/filter.cc", "./source/extensions/stat_sinks/common/statsd/statsd.cc", "./source/extensions/transport_sockets/tls/context_impl.cc", "./source/server/guarddog_impl.cc", From 151fa5405c63b1c46e204795a794a3a49c4c3cef Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 4 Apr 2019 21:18:15 -0700 Subject: [PATCH 331/542] added logic to reset route cache Signed-off-by: Gabriel --- source/extensions/filters/http/ext_authz/ext_authz.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index 3f1a1535f8aa..a268f2655639 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -147,10 +147,9 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { switch (response->status) { case CheckStatus::OK: { - ENVOY_STREAM_LOG(trace, "ext_authz filter added header(s) to the request:", *callbacks_); - if (config_->clearRouteCache() && - (!response->headers_to_add.empty() || !response->headers_to_append.empty())) { - ENVOY_STREAM_LOG(debug, "ext_authz is clearing route cache", *callbacks_); + ENVOY_STREAM_LOG(trace, "ext_authz filter added header(s) to the request:", * + if (!response->headers_to_add.empty() || !response->headers_to_append.empty()) { + ENVOY_STREAM_LOG(debug, "ext_authz has cleared route cache", *callbacks_); callbacks_->clearRouteCache(); } for (const auto& header : response->headers_to_add) { From ccd9dbd1035967e0d2f2112e14a45faa370d162b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 4 Apr 2019 21:25:56 -0700 Subject: [PATCH 332/542] added fix for set cookie header Signed-off-by: Gabriel --- source/extensions/filters/http/ext_authz/ext_authz.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index a268f2655639..fe9073f7c580 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -196,7 +196,10 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { "ext_authz filter added header(s) to the local response:", callbacks); for (const auto& header : headers) { ENVOY_STREAM_LOG(trace, " '{}':'{}'", callbacks, header.first.get(), header.second); - response_headers.remove(header.first); + // This is just a work-around for set-cookie. + if (header.first != Http::Headers::get().SetCookie) { + response_headers.remove(header.first); + } response_headers.addCopy(header.first, header.second); } }, From b6dbd416798ac2793a56603f8e6b2a35596f59bb Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 4 Apr 2019 21:52:08 -0700 Subject: [PATCH 333/542] fixed ambassador issue #1313 Signed-off-by: Gabriel --- source/extensions/filters/http/ext_authz/ext_authz.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index fe9073f7c580..d790caf8ab29 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -166,6 +166,8 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { if (header_to_modify) { ENVOY_STREAM_LOG(trace, " '{}':'{}'", *callbacks_, header.first.get(), header.second); Http::HeaderMapImpl::appendToHeader(header_to_modify->value(), header.second); + } else { + request_headers_->addCopy(header.first, header.second); } } cluster_->statsScope().counter("ext_authz.ok").inc(); From 88240499ad0786c0b0fa6857233c082ca27006ac Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 4 Apr 2019 22:20:08 -0700 Subject: [PATCH 334/542] added tracing support Signed-off-by: Gabriel --- source/extensions/filters/common/ext_authz/BUILD | 1 + .../filters/common/ext_authz/ext_authz.h | 14 ++++++++++++++ .../common/ext_authz/ext_authz_grpc_impl.h | 8 -------- .../common/ext_authz/ext_authz_http_impl.cc | 16 ++++++++++++++-- .../common/ext_authz/ext_authz_http_impl.h | 3 +++ 5 files changed, 32 insertions(+), 10 deletions(-) diff --git a/source/extensions/filters/common/ext_authz/BUILD b/source/extensions/filters/common/ext_authz/BUILD index 8b3d30d012fd..61b2abade1cb 100644 --- a/source/extensions/filters/common/ext_authz/BUILD +++ b/source/extensions/filters/common/ext_authz/BUILD @@ -63,6 +63,7 @@ envoy_cc_library( "//source/common/common:minimal_logger_lib", "//source/common/http:async_client_lib", "//source/common/http:codes_lib", + "//source/common/tracing:http_tracer_lib", "@envoy_api//envoy/config/filter/http/ext_authz/v2:ext_authz_cc", ], ) diff --git a/source/extensions/filters/common/ext_authz/ext_authz.h b/source/extensions/filters/common/ext_authz/ext_authz.h index f79df739884c..95769a18ca8e 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz.h +++ b/source/extensions/filters/common/ext_authz/ext_authz.h @@ -5,17 +5,31 @@ #include #include +#include "common/singleton/const_singleton.h" + #include "envoy/common/pure.h" #include "envoy/http/codes.h" #include "envoy/service/auth/v2/external_auth.pb.h" #include "envoy/tracing/http_tracer.h" + namespace Envoy { namespace Extensions { namespace Filters { namespace Common { namespace ExtAuthz { +/** + * Tracing statuses. + */ +struct ConstantValues { + const std::string TraceStatus = "ext_authz_status"; + const std::string TraceUnauthz = "ext_authz_unauthorized"; + const std::string TraceOk = "ext_authz_ok"; +}; + +typedef ConstSingleton Constants; + /** * Possible async results for a check call. */ diff --git a/source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h b/source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h index 6fa44003f50e..ecfaded7676d 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h +++ b/source/extensions/filters/common/ext_authz/ext_authz_grpc_impl.h @@ -30,14 +30,6 @@ namespace ExtAuthz { using ExtAuthzAsyncCallbacks = Grpc::AsyncRequestCallbacks; -struct ConstantValues { - const std::string TraceStatus = "ext_authz_status"; - const std::string TraceUnauthz = "ext_authz_unauthorized"; - const std::string TraceOk = "ext_authz_ok"; -}; - -using Constants = ConstSingleton; - /* * This client implementation is used when the Ext_Authz filter needs to communicate with an gRPC * authorization server. Unlike the HTTP client, the gRPC allows the server to define response diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc index aff72f7a533f..3521cbebeee1 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc @@ -145,12 +145,14 @@ Http::LowerCaseStrPairVector ClientConfig::toHeadersAdd( } RawHttpClientImpl::RawHttpClientImpl(Upstream::ClusterManager& cm, ClientConfigSharedPtr config) - : cm_(cm), config_(config) {} + : cm_(cm), config_(config), real_time_() {} RawHttpClientImpl::~RawHttpClientImpl() { ASSERT(!callbacks_); } void RawHttpClientImpl::cancel() { ASSERT(callbacks_ != nullptr); + span_->setTag(Tracing::Tags::get().Status, Tracing::Tags::get().Canceled); + span_->finishSpan(); request_->cancel(); callbacks_ = nullptr; } @@ -158,10 +160,15 @@ void RawHttpClientImpl::cancel() { // Client void RawHttpClientImpl::check(RequestCallbacks& callbacks, const envoy::service::auth::v2::CheckRequest& request, - Tracing::Span&) { + Tracing::Span& parent_span) { ASSERT(callbacks_ == nullptr); + ASSERT(span_ == nullptr); + callbacks_ = &callbacks; + span_ = parent_span.spawnChild(Tracing::EgressConfig::get(), "async "+ config_->cluster() + " egress", real_time_.systemTime()); + span_->setTag(Tracing::Tags::get().UpstreamCluster, config_->cluster()); + Http::HeaderMapPtr headers; const uint64_t request_length = request.attributes().request().http().body().size(); if (request_length > 0) { @@ -203,12 +210,15 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks, void RawHttpClientImpl::onSuccess(Http::MessagePtr&& message) { callbacks_->onComplete(toResponse(std::move(message))); + span_->finishSpan(); callbacks_ = nullptr; } void RawHttpClientImpl::onFailure(Http::AsyncClient::FailureReason reason) { ASSERT(reason == Http::AsyncClient::FailureReason::Reset); callbacks_->onComplete(std::make_unique(errorResponse())); + span_->setTag(Tracing::Tags::get().Error, Tracing::Tags::get().True); + span_->finishSpan(); callbacks_ = nullptr; } @@ -233,6 +243,7 @@ ResponsePtr RawHttpClientImpl::toResponse(Http::MessagePtr message) { SuccessResponse ok{message->headers(), config_->upstreamHeaderMatchers(), Response{CheckStatus::OK, Http::HeaderVector{}, Http::HeaderVector{}, EMPTY_STRING, Http::Code::OK}}; + span_->setTag(Constants::get().TraceStatus, Constants::get().TraceOk); return std::move(ok.response_); } @@ -240,6 +251,7 @@ ResponsePtr RawHttpClientImpl::toResponse(Http::MessagePtr message) { SuccessResponse denied{message->headers(), config_->clientHeaderMatchers(), Response{CheckStatus::Denied, Http::HeaderVector{}, Http::HeaderVector{}, message->bodyAsString(), static_cast(status_code)}}; + span_->setTag(Constants::get().TraceStatus, Constants::get().TraceUnauthz); return std::move(denied.response_); } diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h index 376ccdc64c70..8b73b287c538 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/config/filter/http/ext_authz/v2/ext_authz.pb.h" +#include "envoy/tracing/http_tracer.h" #include "envoy/upstream/cluster_manager.h" #include "common/common/logger.h" @@ -146,6 +147,8 @@ class RawHttpClientImpl : public Client, ClientConfigSharedPtr config_; Http::AsyncClient::Request* request_{}; RequestCallbacks* callbacks_{}; + RealTimeSource real_time_; + Tracing::SpanPtr span_; }; } // namespace ExtAuthz From 6ba9a07026dfe88f8a8711e88fee731523d0ebe4 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 4 Apr 2019 23:58:51 -0700 Subject: [PATCH 335/542] added support for legacy RLS Signed-off-by: Gabriel --- api/envoy/service/ratelimit/v1/BUILD | 10 ++++++++++ api/envoy/service/ratelimit/v1/rls.proto | 14 ++++++++++++++ api/envoy/service/ratelimit/v2/BUILD | 1 + source/extensions/filters/common/ratelimit/BUILD | 3 +++ .../filters/common/ratelimit/ratelimit_impl.cc | 9 +++++++-- 5 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 api/envoy/service/ratelimit/v1/BUILD create mode 100644 api/envoy/service/ratelimit/v1/rls.proto diff --git a/api/envoy/service/ratelimit/v1/BUILD b/api/envoy/service/ratelimit/v1/BUILD new file mode 100644 index 000000000000..f0344b2ad04b --- /dev/null +++ b/api/envoy/service/ratelimit/v1/BUILD @@ -0,0 +1,10 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "rls", + srcs = ["rls.proto"], + has_services = 1, + deps = ["//envoy/service/ratelimit/v2:rls"], +) diff --git a/api/envoy/service/ratelimit/v1/rls.proto b/api/envoy/service/ratelimit/v1/rls.proto new file mode 100644 index 000000000000..fec245b25db3 --- /dev/null +++ b/api/envoy/service/ratelimit/v1/rls.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +option go_package = "ratelimit"; + +option cc_generic_services = true; + +package pb.lyft.ratelimit; + +import "envoy/service/ratelimit/v2/rls.proto"; + +service RateLimitService { + rpc ShouldRateLimit(envoy.service.ratelimit.v2.RateLimitRequest) returns (envoy.service.ratelimit.v2.RateLimitResponse) { + } +} diff --git a/api/envoy/service/ratelimit/v2/BUILD b/api/envoy/service/ratelimit/v2/BUILD index 24278fbebc1f..d45814cb0536 100644 --- a/api/envoy/service/ratelimit/v2/BUILD +++ b/api/envoy/service/ratelimit/v2/BUILD @@ -6,6 +6,7 @@ api_proto_library_internal( name = "rls", srcs = ["rls.proto"], has_services = 1, + visibility = ["//visibility:public"], deps = [ "//envoy/api/v2/core:base", "//envoy/api/v2/core:grpc_service", diff --git a/source/extensions/filters/common/ratelimit/BUILD b/source/extensions/filters/common/ratelimit/BUILD index 6b0c197a810d..723805921024 100644 --- a/source/extensions/filters/common/ratelimit/BUILD +++ b/source/extensions/filters/common/ratelimit/BUILD @@ -28,6 +28,9 @@ envoy_cc_library( "@envoy_api//envoy/api/v2/ratelimit:ratelimit_cc", "@envoy_api//envoy/config/ratelimit/v2:rls_cc", "@envoy_api//envoy/service/ratelimit/v2:rls_cc", + + # Support to legacy rate-limit service + "@envoy_api//envoy/service/ratelimit/v1:rls_cc", ], ) diff --git a/source/extensions/filters/common/ratelimit/ratelimit_impl.cc b/source/extensions/filters/common/ratelimit/ratelimit_impl.cc index 201f0fee4711..6becd4038204 100644 --- a/source/extensions/filters/common/ratelimit/ratelimit_impl.cc +++ b/source/extensions/filters/common/ratelimit/ratelimit_impl.cc @@ -18,10 +18,15 @@ namespace Filters { namespace Common { namespace RateLimit { + +// Values used for selecting service paths. +// TODO(gsagula): select V2 when Ambassador gets a config for selecting non-legacy. +// constexpr char V2[] = "envoy.service.ratelimit.v2.RateLimitService.ShouldRateLimit"; +constexpr char V1[] = "pb.lyft.ratelimit.ShouldRateLimit"; + GrpcClientImpl::GrpcClientImpl(Grpc::RawAsyncClientPtr&& async_client, const absl::optional& timeout) - : service_method_(*Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.ratelimit.v2.RateLimitService.ShouldRateLimit")), + : service_method_(*Protobuf::DescriptorPool::generated_pool()->FindMethodByName(V1)), async_client_(std::move(async_client)), timeout_(timeout) {} GrpcClientImpl::~GrpcClientImpl() { ASSERT(!callbacks_); } From 9fada3c43a962ded64972710d6fe6425ff21b641 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 5 Apr 2019 00:35:33 -0700 Subject: [PATCH 336/542] updated legacy RLS name Signed-off-by: Gabriel --- source/extensions/filters/common/ratelimit/ratelimit_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/common/ratelimit/ratelimit_impl.cc b/source/extensions/filters/common/ratelimit/ratelimit_impl.cc index 6becd4038204..42c3d7877078 100644 --- a/source/extensions/filters/common/ratelimit/ratelimit_impl.cc +++ b/source/extensions/filters/common/ratelimit/ratelimit_impl.cc @@ -22,7 +22,7 @@ namespace RateLimit { // Values used for selecting service paths. // TODO(gsagula): select V2 when Ambassador gets a config for selecting non-legacy. // constexpr char V2[] = "envoy.service.ratelimit.v2.RateLimitService.ShouldRateLimit"; -constexpr char V1[] = "pb.lyft.ratelimit.ShouldRateLimit"; +constexpr char V1[] = "pb.lyft.ratelimit.RateLimitService.ShouldRateLimit"; GrpcClientImpl::GrpcClientImpl(Grpc::RawAsyncClientPtr&& async_client, const absl::optional& timeout) From 2f3fc110772cd08bda60676b3e450910acec52e7 Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Thu, 6 Jun 2019 14:22:09 -0700 Subject: [PATCH 337/542] removed deprecation flags from fault, ext-authz and route protos Signed-off-by: Gabriel Linden Sagula --- api/envoy/api/v2/route/route.proto | 4 ++-- api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/envoy/api/v2/route/route.proto b/api/envoy/api/v2/route/route.proto index 8390e4348526..a155254d1223 100644 --- a/api/envoy/api/v2/route/route.proto +++ b/api/envoy/api/v2/route/route.proto @@ -433,7 +433,7 @@ message CorsPolicy { // // **This field is deprecated**. Set the // :ref:`filter_enabled` field instead. - google.protobuf.BoolValue enabled = 7 [deprecated = true]; + google.protobuf.BoolValue enabled = 7 [deprecated = false]; // Specifies if CORS is enabled. // @@ -621,7 +621,7 @@ message RouteAction { // **This field is deprecated**. Set the // :ref:`runtime_fraction // ` field instead. - string runtime_key = 2 [deprecated = true]; + string runtime_key = 2 [deprecated = false]; // If both :ref:`runtime_key // ` and this field are not diff --git a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto index 8e2ea7661e1f..9714120196cf 100644 --- a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto +++ b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto @@ -50,7 +50,7 @@ message ExtAuthz { // useful when transitioning from alpha to release versions assuming that both definitions are // semantically compatible. Deprecation note: This field is deprecated and should only be used for // version upgrade. See release notes for more details. - bool use_alpha = 4 [deprecated = true]; + bool use_alpha = 4 [deprecated = false]; // Enables filter to buffer the client request body and send it within the authorization request. // A ``x-envoy-auth-partial-body: false|true`` metadata header will be added to the authorization From e23b2a6a6b326d347070f26fa4d9e431bdd6c66b Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Thu, 6 Jun 2019 14:48:11 -0700 Subject: [PATCH 338/542] fixed bad merge Signed-off-by: Gabriel Linden Sagula --- source/extensions/filters/http/ext_authz/ext_authz.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index d790caf8ab29..f94afb134f0b 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -147,7 +147,7 @@ void Filter::onComplete(Filters::Common::ExtAuthz::ResponsePtr&& response) { switch (response->status) { case CheckStatus::OK: { - ENVOY_STREAM_LOG(trace, "ext_authz filter added header(s) to the request:", * + ENVOY_STREAM_LOG(trace, "ext_authz filter added header(s) to the request:", *callbacks_); if (!response->headers_to_add.empty() || !response->headers_to_append.empty()) { ENVOY_STREAM_LOG(debug, "ext_authz has cleared route cache", *callbacks_); callbacks_->clearRouteCache(); From f56062992b42a7f0f7b760cc21862eaea59d5460 Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Tue, 11 Jun 2019 23:24:39 -0700 Subject: [PATCH 339/542] added the tracing name string to the client config Signed-off-by: Gabriel Linden Sagula --- source/extensions/filters/common/ext_authz/ext_authz.h | 5 ++--- .../filters/common/ext_authz/ext_authz_http_impl.cc | 6 ++++-- .../filters/common/ext_authz/ext_authz_http_impl.h | 10 ++++++++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/source/extensions/filters/common/ext_authz/ext_authz.h b/source/extensions/filters/common/ext_authz/ext_authz.h index 95769a18ca8e..f636ad61ae78 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz.h +++ b/source/extensions/filters/common/ext_authz/ext_authz.h @@ -5,13 +5,12 @@ #include #include -#include "common/singleton/const_singleton.h" - #include "envoy/common/pure.h" #include "envoy/http/codes.h" #include "envoy/service/auth/v2/external_auth.pb.h" #include "envoy/tracing/http_tracer.h" +#include "common/singleton/const_singleton.h" namespace Envoy { namespace Extensions { @@ -20,7 +19,7 @@ namespace Common { namespace ExtAuthz { /** - * Tracing statuses. + * Tracing statuses. */ struct ConstantValues { const std::string TraceStatus = "ext_authz_status"; diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc index 3521cbebeee1..fb63a28fd5c2 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc @@ -78,7 +78,8 @@ ClientConfig::ClientConfig(const envoy::config::filter::http::ext_authz::v2::Ext authorization_headers_to_add_( toHeadersAdd(config.http_service().authorization_request().headers_to_add())), cluster_name_(config.http_service().server_uri().cluster()), timeout_(timeout), - path_prefix_(path_prefix) {} + path_prefix_(path_prefix), + tracing_name_("async " + config.http_service().server_uri().cluster() + " egress") {} MatcherSharedPtr ClientConfig::toRequestMatchers(const envoy::type::matcher::ListStringMatcher& list) { @@ -166,7 +167,8 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks, callbacks_ = &callbacks; - span_ = parent_span.spawnChild(Tracing::EgressConfig::get(), "async "+ config_->cluster() + " egress", real_time_.systemTime()); + span_ = parent_span.spawnChild(Tracing::EgressConfig::get(), config_->tracingName(), + real_time_.systemTime()); span_->setTag(Tracing::Tags::get().UpstreamCluster, config_->cluster()); Http::HeaderMapPtr headers; diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h index 8b73b287c538..c2917a1b15e5 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h @@ -89,16 +89,21 @@ class ClientConfig { const MatcherSharedPtr& clientHeaderMatchers() const { return client_header_matchers_; } /** - * Returns a list of matchers used for selecting the authorization response headers that + * Returns a list of matchers used for selecting the authorization response headers that * should be send to an the upstream server. */ const MatcherSharedPtr& upstreamHeaderMatchers() const { return upstream_header_matchers_; } /** - * @return List of headers that will be add to the authorization request. + * Returns a list of headers that will be add to the authorization request. */ const Http::LowerCaseStrPairVector& headersToAdd() const { return authorization_headers_to_add_; } + /** + * Returns the name used for tracing. + */ + const std::string& tracingName() { return tracing_name_; } + private: static MatcherSharedPtr toRequestMatchers(const envoy::type::matcher::ListStringMatcher& matcher); static MatcherSharedPtr toClientMatchers(const envoy::type::matcher::ListStringMatcher& matcher); @@ -114,6 +119,7 @@ class ClientConfig { const std::string cluster_name_; const std::chrono::milliseconds timeout_; const std::string path_prefix_; + const str::string tracing_name_; }; using ClientConfigSharedPtr = std::shared_ptr; From ac42dd2d687a09419029df4314b0d341ed3186db Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Wed, 12 Jun 2019 06:57:01 -0700 Subject: [PATCH 340/542] fixed typo Signed-off-by: Gabriel Linden Sagula --- .../extensions/filters/common/ext_authz/ext_authz_http_impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h index c2917a1b15e5..11f0274af16c 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h @@ -119,7 +119,7 @@ class ClientConfig { const std::string cluster_name_; const std::chrono::milliseconds timeout_; const std::string path_prefix_; - const str::string tracing_name_; + const std::string tracing_name_; }; using ClientConfigSharedPtr = std::shared_ptr; From d82576b1267efc0e1d5771364eb27b1f8933c7dd Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Wed, 12 Jun 2019 10:15:15 -0700 Subject: [PATCH 341/542] added test for http client tracing ok Signed-off-by: Gabriel Linden Sagula --- .../filters/common/ext_authz/ext_authz_http_impl_test.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc index 62e89216af2b..3466a814233b 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc @@ -114,6 +114,7 @@ class ExtAuthzHttpClientTest : public testing::Test { ClientConfigSharedPtr config_; RawHttpClientImpl client_; MockRequestCallbacks request_callbacks_; + Tracing::MockSpan span_; }; // Test HTTP client config default values. @@ -229,7 +230,8 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOk) { EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzOkResponse(authz_response)))); - client_.onSuccess(std::move(check_response)); + EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); + client_.onSuccess(std::move(check_response), span_); } // Verify client response headers when authorization_headers_to_add is configured. From 5ce687330c7c4b861b961d26082d5e8e30d88be9 Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Wed, 12 Jun 2019 10:34:47 -0700 Subject: [PATCH 342/542] more tests for http client tracing Signed-off-by: Gabriel Linden Sagula --- .../ext_authz/ext_authz_http_impl_test.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc index 3466a814233b..b44272f8f9d1 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc @@ -252,7 +252,8 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithAddedAuthzHeaders) { EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzOkResponse(authz_response)))); - client_.onSuccess(std::move(check_response)); + EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); + client_.onSuccess(std::move(check_response), span_); } // Verify client response headers when allow_upstream_headers is configured. @@ -276,8 +277,10 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithAllowHeader) { {"bar", "foo", false}, {"x-baz", "foo", false}, {"foobar", "foo", false}}); + + EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); auto message_response = TestCommon::makeMessageResponse(check_response_headers); - client_.onSuccess(std::move(message_response)); + client_.onSuccess(std::move(message_response), span_); } // Test the client when a denied response is received. @@ -292,7 +295,8 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationDenied) { EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzDeniedResponse(authz_response)))); - client_.onSuccess(TestCommon::makeMessageResponse(expected_headers)); + EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); + client_.onSuccess(TestCommon::makeMessageResponse(expected_headers), span_); } // Verify client response headers and body when the authorization server denies the request. @@ -307,8 +311,8 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationDeniedWithAllAttributes) { client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzDeniedResponse(authz_response)))); - - client_.onSuccess(TestCommon::makeMessageResponse(expected_headers, expected_body)); + EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); + client_.onSuccess(TestCommon::makeMessageResponse(expected_headers, expected_body), span_); } // Verify client response headers when the authorization server denies the request and @@ -329,7 +333,8 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationDeniedAndAllowedClientHeaders) { {"x-foo", "bar", false}, {":status", "401", false}, {"foo", "bar", false}}); - client_.onSuccess(TestCommon::makeMessageResponse(check_response_headers, expected_body)); + EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); + client_.onSuccess(TestCommon::makeMessageResponse(check_response_headers, expected_body), span_); } // Test the client when an unknown error occurs. From bce7c36d731238966cca05e35264d431b72a1826 Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Wed, 12 Jun 2019 13:57:57 -0700 Subject: [PATCH 343/542] fixed tests Signed-off-by: Gabriel Linden Sagula --- .../filters/common/ext_authz/ext_authz_http_impl_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc index b44272f8f9d1..18cdfdce76fb 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc @@ -16,6 +16,7 @@ using testing::_; using testing::AllOf; +using testing::Eq; using testing::Invoke; using testing::Ref; using testing::Return; From fa39a836e7a963daa1683452d62cb12b2ea33cf1 Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Thu, 13 Jun 2019 03:08:55 -0700 Subject: [PATCH 344/542] added tests for http client tracing Signed-off-by: Gabriel Linden Sagula --- .../ext_authz/ext_authz_http_impl_test.cc | 76 ++++++++++++------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc index 18cdfdce76fb..b72508fef47b 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc @@ -109,13 +109,21 @@ class ExtAuthzHttpClientTest : public testing::Test { return message_ptr; } + void expectTracing() { + EXPECT_CALL(active_span_, spawnChild_(_, config_->tracingName(), _)) + .WillOnce(Return(child_span_)); + EXPECT_CALL(*child_span_, + setTag(Eq(Tracing::Tags::get().UpstreamCluster), Eq(config_->cluster()))); + } + NiceMock cm_; NiceMock async_client_; NiceMock async_request_; ClientConfigSharedPtr config_; RawHttpClientImpl client_; MockRequestCallbacks request_callbacks_; - Tracing::MockSpan span_; + Tracing::MockSpan active_span_; + Tracing::MockSpan* child_span_{new Tracing::MockSpan()}; }; // Test HTTP client config default values. @@ -146,6 +154,7 @@ TEST_F(ExtAuthzHttpClientTest, ClientConfig) { // // Check other attributes. EXPECT_EQ(config_->pathPrefix(), "/bar"); EXPECT_EQ(config_->cluster(), "ext_authz"); + EXPECT_EQ(config_->tracingName(), "async ext_authz egress"); EXPECT_EQ(config_->timeout(), std::chrono::milliseconds{250}); } @@ -227,12 +236,14 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOk) { auto check_response = TestCommon::makeMessageResponse(expected_headers); envoy::service::auth::v2::CheckRequest request; - client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); + expectTracing(); + client_.check(request_callbacks_, request, active_span_); + EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzOkResponse(authz_response)))); - - EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); - client_.onSuccess(std::move(check_response), span_); + EXPECT_CALL(*child_span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); + EXPECT_CALL(*child_span_, finishSpan()); + client_.onSuccess(std::move(check_response)); } // Verify client response headers when authorization_headers_to_add is configured. @@ -246,15 +257,17 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithAddedAuthzHeaders) { (*mutable_headers)[std::string{":x-authz-header2"}] = std::string{"forged-value"}; // Expect that header1 will be added and header2 correctly overwritten. + expectTracing(); EXPECT_CALL(async_client_, send_(AllOf(ContainsPairAsHeader(config_->headersToAdd().front()), ContainsPairAsHeader(config_->headersToAdd().back())), _, _)); - client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); + client_.check(request_callbacks_, request, active_span_); EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzOkResponse(authz_response)))); - EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); - client_.onSuccess(std::move(check_response), span_); + EXPECT_CALL(*child_span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); + EXPECT_CALL(*child_span_, finishSpan()); + client_.onSuccess(std::move(check_response)); } // Verify client response headers when allow_upstream_headers is configured. @@ -266,9 +279,10 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithAllowHeader) { TestCommon::makeAuthzResponse(CheckStatus::OK, Http::Code::OK, empty_body, expected_headers); envoy::service::auth::v2::CheckRequest request; - client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzOkResponse(authz_response)))); + expectTracing(); + client_.check(request_callbacks_, request, active_span_); const auto check_response_headers = TestCommon::makeHeaderValueOption({{":status", "200", false}, @@ -279,9 +293,10 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationOkWithAllowHeader) { {"x-baz", "foo", false}, {"foobar", "foo", false}}); - EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); + EXPECT_CALL(*child_span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_ok"))); + EXPECT_CALL(*child_span_, finishSpan()); auto message_response = TestCommon::makeMessageResponse(check_response_headers); - client_.onSuccess(std::move(message_response), span_); + client_.onSuccess(std::move(message_response)); } // Test the client when a denied response is received. @@ -291,13 +306,14 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationDenied) { CheckStatus::Denied, Http::Code::Forbidden, "", expected_headers); envoy::service::auth::v2::CheckRequest request; - client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); + expectTracing(); + client_.check(request_callbacks_, request, active_span_); + EXPECT_CALL(*child_span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); + EXPECT_CALL(*child_span_, finishSpan()); EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzDeniedResponse(authz_response)))); - - EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); - client_.onSuccess(TestCommon::makeMessageResponse(expected_headers), span_); + client_.onSuccess(TestCommon::makeMessageResponse(expected_headers)); } // Verify client response headers and body when the authorization server denies the request. @@ -308,12 +324,15 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationDeniedWithAllAttributes) { const auto authz_response = TestCommon::makeAuthzResponse( CheckStatus::Denied, Http::Code::Unauthorized, expected_body, expected_headers); + expectTracing(); envoy::service::auth::v2::CheckRequest request; - client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); + client_.check(request_callbacks_, request, active_span_); + + EXPECT_CALL(*child_span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); + EXPECT_CALL(*child_span_, finishSpan()); EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzDeniedResponse(authz_response)))); - EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); - client_.onSuccess(TestCommon::makeMessageResponse(expected_headers, expected_body), span_); + client_.onSuccess(TestCommon::makeMessageResponse(expected_headers, expected_body)); } // Verify client response headers when the authorization server denies the request and @@ -325,26 +344,32 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationDeniedAndAllowedClientHeaders) { TestCommon::makeHeaderValueOption( {{"x-foo", "bar", false}, {":status", "401", false}, {"foo", "bar", false}})); + expectTracing(); envoy::service::auth::v2::CheckRequest request; - client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); + client_.check(request_callbacks_, request, active_span_); + EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzDeniedResponse(authz_response)))); - + EXPECT_CALL(*child_span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); + EXPECT_CALL(*child_span_, finishSpan()); const auto check_response_headers = TestCommon::makeHeaderValueOption({{":method", "post", false}, {"x-foo", "bar", false}, {":status", "401", false}, {"foo", "bar", false}}); - EXPECT_CALL(span_, setTag(Eq("ext_authz_status"), Eq("ext_authz_unauthorized"))); - client_.onSuccess(TestCommon::makeMessageResponse(check_response_headers, expected_body), span_); + client_.onSuccess(TestCommon::makeMessageResponse(check_response_headers, expected_body)); } // Test the client when an unknown error occurs. TEST_F(ExtAuthzHttpClientTest, AuthorizationRequestError) { envoy::service::auth::v2::CheckRequest request; - client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); + + expectTracing(); + client_.check(request_callbacks_, request, active_span_); EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzErrorResponse(CheckStatus::Error)))); + EXPECT_CALL(*child_span_, setTag(Eq(Tracing::Tags::get().Error), Eq(Tracing::Tags::get().True))); + EXPECT_CALL(*child_span_, finishSpan()); client_.onFailure(Http::AsyncClient::FailureReason::Reset); } @@ -354,10 +379,8 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationRequest5xxError) { Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "503"}}})); envoy::service::auth::v2::CheckRequest request; client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); - EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzErrorResponse(CheckStatus::Error)))); - client_.onSuccess(std::move(check_response)); } @@ -368,10 +391,8 @@ TEST_F(ExtAuthzHttpClientTest, AuthorizationRequestErrorParsingStatusCode) { Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "foo"}}})); envoy::service::auth::v2::CheckRequest request; client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); - EXPECT_CALL(request_callbacks_, onComplete_(WhenDynamicCastTo(AuthzErrorResponse(CheckStatus::Error)))); - client_.onSuccess(std::move(check_response)); } @@ -380,7 +401,6 @@ TEST_F(ExtAuthzHttpClientTest, CancelledAuthorizationRequest) { envoy::service::auth::v2::CheckRequest request; EXPECT_CALL(async_client_, send_(_, _, _)).WillOnce(Return(&async_request_)); client_.check(request_callbacks_, request, Tracing::NullSpan::instance()); - EXPECT_CALL(async_request_, cancel()); client_.cancel(); } From b1dae7dce0478c00e4f5fc3c0c5db8663f4d76ee Mon Sep 17 00:00:00 2001 From: Gabriel Linden Sagula Date: Thu, 13 Jun 2019 03:26:53 -0700 Subject: [PATCH 345/542] use fmt instead of string concatenation Signed-off-by: Gabriel Linden Sagula --- .../extensions/filters/common/ext_authz/ext_authz_http_impl.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc index fb63a28fd5c2..11996676ec3a 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc @@ -1,6 +1,7 @@ #include "extensions/filters/common/ext_authz/ext_authz_http_impl.h" #include "common/common/enum_to_int.h" +#include "common/common/fmt.h" #include "common/http/async_client_impl.h" #include "common/http/codes.h" @@ -79,7 +80,7 @@ ClientConfig::ClientConfig(const envoy::config::filter::http::ext_authz::v2::Ext toHeadersAdd(config.http_service().authorization_request().headers_to_add())), cluster_name_(config.http_service().server_uri().cluster()), timeout_(timeout), path_prefix_(path_prefix), - tracing_name_("async " + config.http_service().server_uri().cluster() + " egress") {} + tracing_name_(fmt::format("async {} egress", config.http_service().server_uri().cluster())) {} MatcherSharedPtr ClientConfig::toRequestMatchers(const envoy::type::matcher::ListStringMatcher& list) { From c48227949d60f92e49c2aa3336193c07ca9de418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20B=C3=A9ky?= Date: Thu, 8 Aug 2019 15:44:31 -0400 Subject: [PATCH 346/542] quiche: do not use Http2String wrapper for std::string (#7856) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Béky --- .../quiche/platform/http2_platform_test.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/test/extensions/quic_listeners/quiche/platform/http2_platform_test.cc b/test/extensions/quic_listeners/quiche/platform/http2_platform_test.cc index 1bad86111ed1..d68c9a748301 100644 --- a/test/extensions/quic_listeners/quiche/platform/http2_platform_test.cc +++ b/test/extensions/quic_listeners/quiche/platform/http2_platform_test.cc @@ -5,6 +5,7 @@ // porting layer for QUICHE. #include +#include #include "extensions/quic_listeners/quiche/platform/flags_impl.h" @@ -21,7 +22,6 @@ #include "quiche/http2/platform/api/http2_optional.h" #include "quiche/http2/platform/api/http2_ptr_util.h" #include "quiche/http2/platform/api/http2_reconstruct_object.h" -#include "quiche/http2/platform/api/http2_string.h" #include "quiche/http2/platform/api/http2_string_piece.h" #include "quiche/http2/test_tools/http2_random.h" @@ -54,7 +54,7 @@ TEST(Http2PlatformTest, Http2Deque) { } TEST(Http2PlatformTest, Http2EstimateMemoryUsage) { - http2::Http2String s = "foo"; + std::string s = "foo"; // Stubbed out to always return 0. EXPECT_EQ(0, http2::Http2EstimateMemoryUsage(s)); } @@ -108,13 +108,8 @@ TEST(Http2PlatformTest, Http2ReconstructObject) { EXPECT_EQ("", s); } -TEST(Http2PlatformTest, Http2String) { - http2::Http2String s = "foo"; - EXPECT_EQ('o', s[1]); -} - TEST(Http2PlatformTest, Http2StringPiece) { - http2::Http2String s = "bar"; + std::string s = "bar"; http2::Http2StringPiece sp(s); EXPECT_EQ('b', sp[0]); } From e03936eaa0c1069a29ab42cb5d8f5cec675c1ee9 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Thu, 8 Aug 2019 12:49:19 -0700 Subject: [PATCH 347/542] OriginalDstCluster: Use ThreadAwareLoadBalancer (#7820) Signed-off-by: Jarno Rajahalme --- api/envoy/api/v2/cds.proto | 7 +- docs/root/intro/deprecated.rst | 2 +- .../common/upstream/cluster_manager_impl.cc | 10 +- .../common/upstream/original_dst_cluster.cc | 152 ++++++++---------- source/common/upstream/original_dst_cluster.h | 97 ++++++----- source/common/upstream/upstream_impl.cc | 12 +- .../upstream/cluster_manager_impl_test.cc | 6 +- .../upstream/original_dst_cluster_test.cc | 47 +++--- 8 files changed, 151 insertions(+), 182 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index c8542387fc0d..647a5015063f 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -160,7 +160,12 @@ message Cluster { // Refer to the :ref:`original destination load balancing // policy` // for an explanation. - ORIGINAL_DST_LB = 4; + // + // .. attention:: + // + // **This load balancing policy is deprecated**. Use CLUSTER_PROVIDED instead. + // + ORIGINAL_DST_LB = 4 [deprecated = true]; // Refer to the :ref:`Maglev load balancing policy` // for an explanation. diff --git a/docs/root/intro/deprecated.rst b/docs/root/intro/deprecated.rst index 6fdd161ae631..c8399d9ab087 100644 --- a/docs/root/intro/deprecated.rst +++ b/docs/root/intro/deprecated.rst @@ -12,7 +12,7 @@ Deprecated items below are listed in chronological order. Version 1.12.0 (pending) ======================== - +* The ORIGINAL_DST_LB :ref:`load balancing policy ` is deprecated, use CLUSTER_PROVIDED policy instead when configuring an :ref:`original destination cluster `. Version 1.11.0 (July 11, 2019) ============================== diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 4c29bf9a70a6..e2673034a675 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -1114,18 +1114,12 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::ClusterEntry( } case LoadBalancerType::ClusterProvided: case LoadBalancerType::RingHash: - case LoadBalancerType::Maglev: { + case LoadBalancerType::Maglev: + case LoadBalancerType::OriginalDst: { ASSERT(lb_factory_ != nullptr); lb_ = lb_factory_->create(); break; } - case LoadBalancerType::OriginalDst: { - ASSERT(lb_factory_ == nullptr); - lb_ = std::make_unique( - priority_set_, parent.parent_.active_clusters_.at(cluster->name())->cluster_, - cluster->lbOriginalDstConfig()); - break; - } } } } diff --git a/source/common/upstream/original_dst_cluster.cc b/source/common/upstream/original_dst_cluster.cc index 9cb8414aa166..ec24c36e0e47 100644 --- a/source/common/upstream/original_dst_cluster.cc +++ b/source/common/upstream/original_dst_cluster.cc @@ -16,40 +16,11 @@ namespace Envoy { namespace Upstream { -// Static cast below is guaranteed to succeed, as code instantiating the cluster -// configuration, that is run prior to this code, checks that an OriginalDstCluster is -// always configured with an OriginalDstCluster::LoadBalancer, and that an -// OriginalDstCluster::LoadBalancer is never configured with any other type of cluster, -// and throws an exception otherwise. - -OriginalDstCluster::LoadBalancer::LoadBalancer( - PrioritySet& priority_set, ClusterSharedPtr& parent, - const absl::optional& config) - : priority_set_(priority_set), parent_(std::static_pointer_cast(parent)), - info_(parent->info()), use_http_header_(config ? config.value().use_http_header() : false) { - // priority_set_ is initially empty. - priority_set_.addMemberUpdateCb( - [this](const HostVector& hosts_added, const HostVector& hosts_removed) -> void { - // Update the hosts map - // TODO(ramaraochavali): use cluster stats and move the log lines to debug. - for (const HostSharedPtr& host : hosts_removed) { - ENVOY_LOG(debug, "Removing host {}.", host->address()->asString()); - host_map_.remove(host); - } - for (const HostSharedPtr& host : hosts_added) { - if (host_map_.insert(host)) { - ENVOY_LOG(debug, "Adding host {}.", host->address()->asString()); - } - } - }); -} - HostConstSharedPtr OriginalDstCluster::LoadBalancer::chooseHost(LoadBalancerContext* context) { if (context) { - // Check if override host header is present, if yes use it otherwise check local address. Network::Address::InstanceConstSharedPtr dst_host = nullptr; - if (use_http_header_) { + if (parent_->use_http_header_) { dst_host = requestOverrideHost(context); } if (dst_host == nullptr) { @@ -63,10 +34,10 @@ HostConstSharedPtr OriginalDstCluster::LoadBalancer::chooseHost(LoadBalancerCont if (dst_host) { const Network::Address::Instance& dst_addr = *dst_host.get(); - // Check if a host with the destination address is already in the host set. - HostSharedPtr host = host_map_.find(dst_addr); - if (host) { + auto it = host_map_->find(dst_addr.asString()); + if (it != host_map_->end()) { + HostSharedPtr host(it->second); // takes a reference ENVOY_LOG(debug, "Using existing host {}.", host->address()->asString()); host->used(true); // Mark as used. return host; @@ -77,29 +48,24 @@ HostConstSharedPtr OriginalDstCluster::LoadBalancer::chooseHost(LoadBalancerCont Network::Address::InstanceConstSharedPtr host_ip_port( Network::Utility::copyInternetAddressAndPort(*dst_ip)); // Create a host we can use immediately. - host.reset( - new HostImpl(info_, info_->name() + dst_addr.asString(), std::move(host_ip_port), - envoy::api::v2::core::Metadata::default_instance(), 1, - envoy::api::v2::core::Locality().default_instance(), - envoy::api::v2::endpoint::Endpoint::HealthCheckConfig().default_instance(), - 0, envoy::api::v2::core::HealthStatus::UNKNOWN)); - + auto info = parent_->info(); + HostSharedPtr host(std::make_shared( + info, info->name() + dst_addr.asString(), std::move(host_ip_port), + envoy::api::v2::core::Metadata::default_instance(), 1, + envoy::api::v2::core::Locality().default_instance(), + envoy::api::v2::endpoint::Endpoint::HealthCheckConfig().default_instance(), 0, + envoy::api::v2::core::HealthStatus::UNKNOWN)); ENVOY_LOG(debug, "Created host {}.", host->address()->asString()); - // Add the new host to the map. We just failed to find it in - // our local map above, so insert without checking (2nd arg == false). - host_map_.insert(host, false); - - if (std::shared_ptr parent = parent_.lock()) { - // lambda cannot capture a member by value. - std::weak_ptr post_parent = parent_; - parent->dispatcher_.post([post_parent, host]() mutable { - // The main cluster may have disappeared while this post was queued. - if (std::shared_ptr parent = post_parent.lock()) { - parent->addHost(host); - } - }); - } + // Tell the cluster about the new host + // lambda cannot capture a member by value. + std::weak_ptr post_parent = parent_; + parent_->dispatcher_.post([post_parent, host]() mutable { + // The main cluster may have disappeared while this post was queued. + if (std::shared_ptr parent = post_parent.lock()) { + parent->addHost(host); + } + }); return host; } else { ENVOY_LOG(debug, "Failed to create host for {}.", dst_addr.asString()); @@ -126,7 +92,7 @@ OriginalDstCluster::LoadBalancer::requestOverrideHost(LoadBalancerContext* conte ENVOY_LOG(debug, "Using request override host {}.", request_override_host); } catch (const Envoy::EnvoyException& e) { ENVOY_LOG(debug, "original_dst_load_balancer: invalid override header value. {}", e.what()); - info_->stats().original_dst_host_invalid_.inc(); + parent_->info()->stats().original_dst_host_invalid_.inc(); } } return request_host; @@ -140,7 +106,11 @@ OriginalDstCluster::OriginalDstCluster( dispatcher_(factory_context.dispatcher()), cleanup_interval_ms_( std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, cleanup_interval, 5000))), - cleanup_timer_(dispatcher_.createTimer([this]() -> void { cleanup(); })) { + cleanup_timer_(dispatcher_.createTimer([this]() -> void { cleanup(); })), + use_http_header_(info_->lbOriginalDstConfig() + ? info_->lbOriginalDstConfig().value().use_http_header() + : false), + host_map_(std::make_shared()) { // TODO(dio): Remove hosts check once the hosts field is removed. if (config.has_load_assignment() || !config.hosts().empty()) { throw EnvoyException("ORIGINAL_DST clusters must have no load assignment or hosts configured"); @@ -149,41 +119,53 @@ OriginalDstCluster::OriginalDstCluster( } void OriginalDstCluster::addHost(HostSharedPtr& host) { - // Given the current config, only EDS clusters support multiple priorities. - ASSERT(priority_set_.hostSetsPerPriority().size() == 1); - const auto& first_host_set = priority_set_.getOrCreateHostSet(0); - HostVectorSharedPtr new_hosts(new HostVector(first_host_set.hosts())); - new_hosts->emplace_back(host); - priority_set_.updateHosts(0, - HostSetImpl::partitionHosts(new_hosts, HostsPerLocalityImpl::empty()), - {}, {std::move(host)}, {}, absl::nullopt); + HostMapSharedPtr new_host_map = std::make_shared(*getCurrentHostMap()); + auto pair = new_host_map->emplace(host->address()->asString(), host); + bool added = pair.second; + if (added) { + ENVOY_LOG(debug, "addHost() adding {}", host->address()->asString()); + setHostMap(new_host_map); + // Given the current config, only EDS clusters support multiple priorities. + ASSERT(priority_set_.hostSetsPerPriority().size() == 1); + const auto& first_host_set = priority_set_.getOrCreateHostSet(0); + HostVectorSharedPtr all_hosts(new HostVector(first_host_set.hosts())); + all_hosts->emplace_back(host); + priority_set_.updateHosts(0, + HostSetImpl::partitionHosts(all_hosts, HostsPerLocalityImpl::empty()), + {}, {std::move(host)}, {}, absl::nullopt); + } } void OriginalDstCluster::cleanup() { - HostVectorSharedPtr new_hosts(new HostVector); + HostVectorSharedPtr keeping_hosts(new HostVector); HostVector to_be_removed; - // Given the current config, only EDS clusters support multiple priorities. - ASSERT(priority_set_.hostSetsPerPriority().size() == 1); - const auto& host_set = priority_set_.getOrCreateHostSet(0); ENVOY_LOG(trace, "Stale original dst hosts cleanup triggered."); - if (!host_set.hosts().empty()) { - ENVOY_LOG(debug, "Cleaning up stale original dst hosts."); - for (const HostSharedPtr& host : host_set.hosts()) { + auto host_map = getCurrentHostMap(); + if (!host_map->empty()) { + ENVOY_LOG(trace, "Cleaning up stale original dst hosts."); + for (const auto& pair : *host_map) { + const std::string& addr = pair.first; + const HostSharedPtr& host = pair.second; if (host->used()) { - ENVOY_LOG(debug, "Keeping active host {}.", host->address()->asString()); - new_hosts->emplace_back(host); + ENVOY_LOG(trace, "Keeping active host {}.", addr); + keeping_hosts->emplace_back(host); host->used(false); // Mark to be removed during the next round. } else { - ENVOY_LOG(debug, "Removing stale host {}.", host->address()->asString()); + ENVOY_LOG(trace, "Removing stale host {}.", addr); to_be_removed.emplace_back(host); } } } if (!to_be_removed.empty()) { - priority_set_.updateHosts(0, - HostSetImpl::partitionHosts(new_hosts, HostsPerLocalityImpl::empty()), - {}, {}, to_be_removed, absl::nullopt); + HostMapSharedPtr new_host_map = std::make_shared(*host_map); + for (const HostSharedPtr& host : to_be_removed) { + new_host_map->erase(host->address()->asString()); + } + setHostMap(new_host_map); + priority_set_.updateHosts( + 0, HostSetImpl::partitionHosts(keeping_hosts, HostsPerLocalityImpl::empty()), {}, {}, + to_be_removed, absl::nullopt); } cleanup_timer_->enableTimer(cleanup_interval_ms_); @@ -194,10 +176,11 @@ OriginalDstClusterFactory::createClusterImpl( const envoy::api::v2::Cluster& cluster, ClusterFactoryContext& context, Server::Configuration::TransportSocketFactoryContext& socket_factory_context, Stats::ScopePtr&& stats_scope) { - if (cluster.lb_policy() != envoy::api::v2::Cluster::ORIGINAL_DST_LB) { + if (cluster.lb_policy() != envoy::api::v2::Cluster::ORIGINAL_DST_LB && + cluster.lb_policy() != envoy::api::v2::Cluster::CLUSTER_PROVIDED) { throw EnvoyException(fmt::format( - "cluster: LB policy {} is not valid for Cluster type {}. Only 'original_dst_lb' " - "is allowed with cluster type 'original_dst'", + "cluster: LB policy {} is not valid for Cluster type {}. Only 'CLUSTER_PROVIDED' or " + "'ORIGINAL_DST_LB' is allowed with cluster type 'ORIGINAL_DST'", envoy::api::v2::Cluster_LbPolicy_Name(cluster.lb_policy()), envoy::api::v2::Cluster_DiscoveryType_Name(cluster.type()))); } @@ -209,14 +192,15 @@ OriginalDstClusterFactory::createClusterImpl( // TODO(mattklein123): The original DST load balancer type should be deprecated and instead // the cluster should directly supply the load balancer. This will remove // a special case and allow this cluster to be compiled out as an extension. - return std::make_pair( + auto new_cluster = std::make_shared(cluster, context.runtime(), socket_factory_context, - std::move(stats_scope), context.addedViaApi()), - nullptr); + std::move(stats_scope), context.addedViaApi()); + auto lb = std::make_unique(new_cluster); + return std::make_pair(new_cluster, std::move(lb)); } /** - * Static registration for the strict dns cluster factory. @see RegisterFactory. + * Static registration for the original dst cluster factory. @see RegisterFactory. */ REGISTER_FACTORY(OriginalDstClusterFactory, ClusterFactory); diff --git a/source/common/upstream/original_dst_cluster.h b/source/common/upstream/original_dst_cluster.h index 64226fc71db4..1a88dfb1c61c 100644 --- a/source/common/upstream/original_dst_cluster.h +++ b/source/common/upstream/original_dst_cluster.h @@ -20,6 +20,9 @@ namespace Envoy { namespace Upstream { +using HostMapSharedPtr = std::shared_ptr; +using HostMapConstSharedPtr = std::shared_ptr; + /** * The OriginalDstCluster is a dynamic cluster that automatically adds hosts as needed based on the * original destination address of the downstream connection. These hosts are also automatically @@ -48,66 +51,52 @@ class OriginalDstCluster : public ClusterImplBase { */ class LoadBalancer : public Upstream::LoadBalancer { public: - LoadBalancer(PrioritySet& priority_set, ClusterSharedPtr& parent, - const absl::optional& config); + LoadBalancer(const std::shared_ptr& parent) + : parent_(parent), host_map_(parent->getCurrentHostMap()) {} // Upstream::LoadBalancer HostConstSharedPtr chooseHost(LoadBalancerContext* context) override; private: - /** - * Map from an host IP address/port to a HostSharedPtr. Due to races multiple distinct host - * objects with the same address can be created, so we need to use a multimap. - */ - class HostMap { - public: - bool insert(const HostSharedPtr& host, bool check = true) { - if (check) { - auto range = map_.equal_range(host->address()->asString()); - auto it = std::find_if( - range.first, range.second, - [&host](const decltype(map_)::value_type pair) { return pair.second == host; }); - if (it != range.second) { - return false; // 'host' already in the map, no need to insert. - } - } - map_.emplace(host->address()->asString(), host); - return true; - } - - void remove(const HostSharedPtr& host) { - auto range = map_.equal_range(host->address()->asString()); - auto it = - std::find_if(range.first, range.second, [&host](const decltype(map_)::value_type pair) { - return pair.second == host; - }); - ASSERT(it != range.second); - map_.erase(it); - } - - HostSharedPtr find(const Network::Address::Instance& address) { - auto it = map_.find(address.asString()); - - if (it != map_.end()) { - return it->second; - } - return nullptr; - } - - private: - std::unordered_multimap map_; - }; - Network::Address::InstanceConstSharedPtr requestOverrideHost(LoadBalancerContext* context); - PrioritySet& priority_set_; // Thread local priority set. - std::weak_ptr parent_; // Primary cluster managed by the main thread. - ClusterInfoConstSharedPtr info_; - const bool use_http_header_; - HostMap host_map_; + const std::shared_ptr parent_; + HostMapConstSharedPtr host_map_; }; private: + struct LoadBalancerFactory : public Upstream::LoadBalancerFactory { + LoadBalancerFactory(const std::shared_ptr& cluster) : cluster_(cluster) {} + + // Upstream::LoadBalancerFactory + Upstream::LoadBalancerPtr create() override { return std::make_unique(cluster_); } + + const std::shared_ptr cluster_; + }; + + struct ThreadAwareLoadBalancer : public Upstream::ThreadAwareLoadBalancer { + ThreadAwareLoadBalancer(const std::shared_ptr& cluster) + : cluster_(cluster) {} + + // Upstream::ThreadAwareLoadBalancer + Upstream::LoadBalancerFactorySharedPtr factory() override { + return std::make_shared(cluster_); + } + void initialize() override {} + + const std::shared_ptr cluster_; + }; + + HostMapConstSharedPtr getCurrentHostMap() { + absl::ReaderMutexLock lock(&host_map_lock_); + return host_map_; + } + + void setHostMap(const HostMapConstSharedPtr& new_host_map) { + absl::WriterMutexLock lock(&host_map_lock_); + host_map_ = new_host_map; + } + void addHost(HostSharedPtr&); void cleanup(); @@ -117,8 +106,16 @@ class OriginalDstCluster : public ClusterImplBase { Event::Dispatcher& dispatcher_; const std::chrono::milliseconds cleanup_interval_ms_; Event::TimerPtr cleanup_timer_; + const bool use_http_header_; + + absl::Mutex host_map_lock_; + HostMapConstSharedPtr host_map_ ABSL_GUARDED_BY(host_map_lock_); + + friend class OriginalDstClusterFactory; }; +using OriginalDstClusterSharedPtr = std::shared_ptr; + class OriginalDstClusterFactory : public ClusterFactoryImplBase { public: OriginalDstClusterFactory() diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 12fc71e75290..f44e01f5a5de 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -637,13 +637,13 @@ ClusterInfoImpl::ClusterInfoImpl( break; case envoy::api::v2::Cluster::ORIGINAL_DST_LB: if (config.type() != envoy::api::v2::Cluster::ORIGINAL_DST) { - throw EnvoyException(fmt::format( - "cluster: LB policy {} is not valid for Cluster type {}. Only 'original_dst_lb' " - "is allowed with cluster type 'original_dst'", - envoy::api::v2::Cluster_LbPolicy_Name(config.lb_policy()), - envoy::api::v2::Cluster_DiscoveryType_Name(config.type()))); + throw EnvoyException( + fmt::format("cluster: LB policy {} is not valid for Cluster type {}. 'ORIGINAL_DST_LB' " + "is allowed only with cluster type 'ORIGINAL_DST'", + envoy::api::v2::Cluster_LbPolicy_Name(config.lb_policy()), + envoy::api::v2::Cluster_DiscoveryType_Name(config.type()))); } - lb_type_ = LoadBalancerType::OriginalDst; + lb_type_ = LoadBalancerType::ClusterProvided; break; case envoy::api::v2::Cluster::MAGLEV: lb_type_ = LoadBalancerType::Maglev; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 82ac958f3267..7e3fff828ad9 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -545,7 +545,7 @@ TEST_F(ClusterManagerImplTest, OriginalDstLbRestriction) { EXPECT_THROW_WITH_MESSAGE( create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, "cluster: LB policy ROUND_ROBIN is not valid for Cluster type ORIGINAL_DST. Only " - "'original_dst_lb' is allowed with cluster type 'original_dst'"); + "'CLUSTER_PROVIDED' or 'ORIGINAL_DST_LB' is allowed with cluster type 'ORIGINAL_DST'"); } TEST_F(ClusterManagerImplTest, OriginalDstLbRestriction2) { @@ -568,8 +568,8 @@ TEST_F(ClusterManagerImplTest, OriginalDstLbRestriction2) { EXPECT_THROW_WITH_MESSAGE( create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, - "cluster: LB policy ORIGINAL_DST_LB is not valid for Cluster type STATIC. Only " - "'original_dst_lb' is allowed with cluster type 'original_dst'"); + "cluster: LB policy ORIGINAL_DST_LB is not valid for Cluster type STATIC. " + "'ORIGINAL_DST_LB' is allowed only with cluster type 'ORIGINAL_DST'"); } TEST_F(ClusterManagerImplTest, SubsetLoadBalancerInitialization) { diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index ba5e955b6fa6..dd0a2c1babf3 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -37,7 +37,6 @@ using testing::SaveArg; namespace Envoy { namespace Upstream { -namespace OriginalDstClusterTest { namespace { class TestLoadBalancerContext : public LoadBalancerContextBase { @@ -89,7 +88,7 @@ class OriginalDstClusterTest : public testing::Test { Stats::IsolatedStoreImpl stats_store_; Ssl::MockContextManager ssl_context_manager_; - ClusterSharedPtr cluster_; + OriginalDstClusterSharedPtr cluster_; ReadyWatcher membership_updated_; ReadyWatcher initialized_; NiceMock runtime_; @@ -109,7 +108,7 @@ TEST(OriginalDstClusterConfigTest, GoodConfig) { name: name connect_timeout: 0.25s type: original_dst - lb_policy: original_dst_lb + lb_policy: cluster_provided cleanup_interval: 1s )EOF"; // Help Emacs balance quotation marks: " @@ -198,8 +197,7 @@ TEST_F(OriginalDstClusterTest, NoContext) { // No downstream connection => no host. { TestLoadBalancerContext lb_context(nullptr); - OriginalDstCluster::LoadBalancer lb(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); + OriginalDstCluster::LoadBalancer lb(cluster_); EXPECT_CALL(dispatcher_, post(_)).Times(0); HostConstSharedPtr host = lb.chooseHost(&lb_context); EXPECT_EQ(host, nullptr); @@ -214,8 +212,7 @@ TEST_F(OriginalDstClusterTest, NoContext) { // First argument is normally the reference to the ThreadLocalCluster's HostSet, but in these // tests we do not have the thread local clusters, so we pass a reference to the HostSet of the // primary cluster. The implementation handles both cases the same. - OriginalDstCluster::LoadBalancer lb(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); + OriginalDstCluster::LoadBalancer lb(cluster_); EXPECT_CALL(dispatcher_, post(_)).Times(0); HostConstSharedPtr host = lb.chooseHost(&lb_context); EXPECT_EQ(host, nullptr); @@ -228,8 +225,7 @@ TEST_F(OriginalDstClusterTest, NoContext) { connection.local_address_ = std::make_shared("unix://foo"); EXPECT_CALL(connection, localAddressRestored()).WillRepeatedly(Return(true)); - OriginalDstCluster::LoadBalancer lb(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); + OriginalDstCluster::LoadBalancer lb(cluster_); EXPECT_CALL(dispatcher_, post(_)).Times(0); HostConstSharedPtr host = lb.chooseHost(&lb_context); EXPECT_EQ(host, nullptr); @@ -264,11 +260,10 @@ TEST_F(OriginalDstClusterTest, Membership) { connection.local_address_ = std::make_shared("10.10.11.11"); EXPECT_CALL(connection, localAddressRestored()).WillRepeatedly(Return(true)); - OriginalDstCluster::LoadBalancer lb(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); Event::PostCb post_cb; EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb)); - HostConstSharedPtr host = lb.chooseHost(&lb_context); + // Mock the cluster manager by recreating the load balancer each time to get a fresh host map + HostConstSharedPtr host = OriginalDstCluster::LoadBalancer(cluster_).chooseHost(&lb_context); post_cb(); auto cluster_hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts(); @@ -287,7 +282,8 @@ TEST_F(OriginalDstClusterTest, Membership) { *cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->address()); // Same host is returned on the 2nd call - HostConstSharedPtr host2 = lb.chooseHost(&lb_context); + // Mock the cluster manager by recreating the load balancer with the new host map + HostConstSharedPtr host2 = OriginalDstCluster::LoadBalancer(cluster_).chooseHost(&lb_context); EXPECT_EQ(host2, host); // Make host time out, no membership changes happen on the first timeout. @@ -315,7 +311,8 @@ TEST_F(OriginalDstClusterTest, Membership) { // New host gets created EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb)); - HostConstSharedPtr host3 = lb.chooseHost(&lb_context); + // Mock the cluster manager by recreating the load balancer with the new host map + HostConstSharedPtr host3 = OriginalDstCluster::LoadBalancer(cluster_).chooseHost(&lb_context); post_cb(); EXPECT_NE(host3, nullptr); EXPECT_NE(host3, host); @@ -357,9 +354,7 @@ TEST_F(OriginalDstClusterTest, Membership2) { connection2.local_address_ = std::make_shared("10.10.11.12"); EXPECT_CALL(connection2, localAddressRestored()).WillRepeatedly(Return(true)); - OriginalDstCluster::LoadBalancer lb(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); - + OriginalDstCluster::LoadBalancer lb(cluster_); EXPECT_CALL(membership_updated_, ready()); Event::PostCb post_cb; EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb)); @@ -443,8 +438,7 @@ TEST_F(OriginalDstClusterTest, Connection) { connection.local_address_ = std::make_shared("FD00::1"); EXPECT_CALL(connection, localAddressRestored()).WillRepeatedly(Return(true)); - OriginalDstCluster::LoadBalancer lb(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); + OriginalDstCluster::LoadBalancer lb(cluster_); Event::PostCb post_cb; EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb)); HostConstSharedPtr host = lb.chooseHost(&lb_context); @@ -493,18 +487,16 @@ TEST_F(OriginalDstClusterTest, MultipleClusters) { connection.local_address_ = std::make_shared("FD00::1"); EXPECT_CALL(connection, localAddressRestored()).WillRepeatedly(Return(true)); - OriginalDstCluster::LoadBalancer lb1(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); - OriginalDstCluster::LoadBalancer lb2(second, cluster_, cluster_->info()->lbOriginalDstConfig()); + OriginalDstCluster::LoadBalancer lb(cluster_); Event::PostCb post_cb; EXPECT_CALL(dispatcher_, post(_)).WillOnce(SaveArg<0>(&post_cb)); - HostConstSharedPtr host = lb1.chooseHost(&lb_context); + HostConstSharedPtr host = lb.chooseHost(&lb_context); post_cb(); ASSERT_NE(host, nullptr); EXPECT_EQ(*connection.local_address_, *host->address()); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); - // Check that lb2 also gets updated + // Check that 'second' also gets updated EXPECT_EQ(1UL, second.hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(host, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]); @@ -532,8 +524,7 @@ TEST_F(OriginalDstClusterTest, UseHttpHeaderEnabled) { 0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->healthyHostsPerLocality().get().size()); - OriginalDstCluster::LoadBalancer lb(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); + OriginalDstCluster::LoadBalancer lb(cluster_); Event::PostCb post_cb; // HTTP header override. @@ -604,8 +595,7 @@ TEST_F(OriginalDstClusterTest, UseHttpHeaderDisabled) { 0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->healthyHostsPerLocality().get().size()); - OriginalDstCluster::LoadBalancer lb(cluster_->prioritySet(), cluster_, - cluster_->info()->lbOriginalDstConfig()); + OriginalDstCluster::LoadBalancer lb(cluster_); Event::PostCb post_cb; // Downstream connection with original_dst filter, HTTP header override ignored. @@ -647,6 +637,5 @@ TEST_F(OriginalDstClusterTest, UseHttpHeaderDisabled) { } } // namespace -} // namespace OriginalDstClusterTest } // namespace Upstream } // namespace Envoy From 04477cac4177b50ff656b35040b5a421c8337894 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Thu, 8 Aug 2019 17:17:18 -0700 Subject: [PATCH 348/542] Pass CC, CXX and PATH variables to Bazel. (#7874) Without this change, the default system compiler is used in some rules (e.g. genrules). Signed-off-by: Piotr Sikora --- .bazelrc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.bazelrc b/.bazelrc index aac3e9c240ee..552052098379 100644 --- a/.bazelrc +++ b/.bazelrc @@ -19,6 +19,11 @@ build --action_env=BAZEL_LINKOPTS=-lm:-static-libgcc build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11 build --javabase=@bazel_tools//tools/jdk:remote_jdk11 +# Pass PATH, CC and CXX variables from the environment. +build --action_env=CC +build --action_env=CXX +build --action_env=PATH + # Basic ASAN/UBSAN that works for gcc build:asan --action_env=BAZEL_LINKLIBS= build:asan --action_env=BAZEL_LINKOPTS=-lstdc++:-lm @@ -73,12 +78,9 @@ build:clang-msan --copt -fsanitize-memory-track-origins=2 # Clang with libc++ # TODO(cmluciano) fix and re-enable _LIBCPP_VERSION testing for TCMALLOC in Envoy::Stats::TestUtil::hasDeterministicMallocStats # and update stats_integration_test with appropriate m_per_cluster value -build:libc++ --action_env=CC -build:libc++ --action_env=CXX build:libc++ --action_env=CXXFLAGS=-stdlib=libc++ build:libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ build:libc++ --action_env=BAZEL_LINKLIBS=-l%:libc++.a:-l%:libc++abi.a:-lm -build:libc++ --action_env=PATH build:libc++ --host_linkopt=-fuse-ld=lld build:libc++ --define force_libcpp=enabled From c92b8ba4d90a61c266b4c1f0b807910eb11b05e7 Mon Sep 17 00:00:00 2001 From: Scott LaVigne <1406859+lavignes@users.noreply.github.com> Date: Thu, 8 Aug 2019 20:03:34 -0700 Subject: [PATCH 349/542] grpc: Add AWS IAM grpc credentials extension (#7532) Signed-off-by: Scott LaVigne --- CODEOWNERS | 3 + api/docs/BUILD | 1 + .../config/grpc_credential/v2alpha/BUILD | 10 ++ .../grpc_credential/v2alpha/aws_iam.proto | 29 ++++ docs/build.sh | 1 + docs/root/api-v2/config/config.rst | 1 + .../grpc_credential/grpc_credential.rst | 8 + docs/root/intro/version_history.rst | 1 + source/extensions/extensions_build_config.bzl | 1 + .../extensions/grpc_credentials/aws_iam/BUILD | 33 ++++ .../grpc_credentials/aws_iam/config.cc | 148 ++++++++++++++++++ .../grpc_credentials/aws_iam/config.h | 62 ++++++++ .../grpc_credentials/well_known_names.h | 2 + .../extensions/grpc_credentials/aws_iam/BUILD | 23 +++ .../aws_iam/aws_iam_grpc_credentials_test.cc | 116 ++++++++++++++ test/test_common/environment.cc | 1 - 16 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 api/envoy/config/grpc_credential/v2alpha/aws_iam.proto create mode 100644 docs/root/api-v2/config/grpc_credential/grpc_credential.rst create mode 100644 source/extensions/grpc_credentials/aws_iam/BUILD create mode 100644 source/extensions/grpc_credentials/aws_iam/config.cc create mode 100644 source/extensions/grpc_credentials/aws_iam/config.h create mode 100644 test/extensions/grpc_credentials/aws_iam/BUILD create mode 100644 test/extensions/grpc_credentials/aws_iam/aws_iam_grpc_credentials_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index 9576cbb5a308..a6ef0e4aa0dc 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -44,6 +44,9 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/http/dynamic_forward_proxy @mattklein123 @alyssawilk # omit_canary_hosts retry predicate /*/extensions/retry/host/omit_canary_hosts @sriduth @snowp +# aws_iam grpc credentials +/*/extensions/grpc_credentials/aws_iam @lavignes @mattklein123 +/*/extensions/filters/http/common/aws @lavignes @mattklein123 # adaptive concurrency limit extension. /*/extensions/filters/http/adaptive_concurrency @tonya11en @mattklein123 # http inspector diff --git a/api/docs/BUILD b/api/docs/BUILD index b1ec7f0a3388..da20fe5e7171 100644 --- a/api/docs/BUILD +++ b/api/docs/BUILD @@ -71,6 +71,7 @@ proto_library( "//envoy/config/filter/network/thrift_proxy/v2alpha1:thrift_proxy", "//envoy/config/filter/thrift/rate_limit/v2alpha1:rate_limit", "//envoy/config/filter/thrift/router/v2alpha1:router", + "//envoy/config/grpc_credential/v2alpha:aws_iam", "//envoy/config/grpc_credential/v2alpha:file_based_metadata", "//envoy/config/health_checker/redis/v2:redis", "//envoy/config/metrics/v2:metrics_service", diff --git a/api/envoy/config/grpc_credential/v2alpha/BUILD b/api/envoy/config/grpc_credential/v2alpha/BUILD index 4765215c4c64..f299179ecb00 100644 --- a/api/envoy/config/grpc_credential/v2alpha/BUILD +++ b/api/envoy/config/grpc_credential/v2alpha/BUILD @@ -15,3 +15,13 @@ api_go_proto_library( "//envoy/api/v2/core:base_go_proto", ], ) + +api_proto_library_internal( + name = "aws_iam", + srcs = ["aws_iam.proto"], +) + +api_go_proto_library( + name = "aws_iam", + proto = ":aws_iam", +) diff --git a/api/envoy/config/grpc_credential/v2alpha/aws_iam.proto b/api/envoy/config/grpc_credential/v2alpha/aws_iam.proto new file mode 100644 index 000000000000..3689b80611f5 --- /dev/null +++ b/api/envoy/config/grpc_credential/v2alpha/aws_iam.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +// [#protodoc-title: Grpc Credentials AWS IAM] +// Configuration for AWS IAM Grpc Credentials Plugin + +package envoy.config.grpc_credential.v2alpha; + +option java_outer_classname = "AwsIamProto"; +option java_package = "io.envoyproxy.envoy.config.grpc_credential.v2alpha"; +option java_multiple_files = true; +option go_package = "v2alpha"; + +import "validate/validate.proto"; + +message AwsIamConfig { + // The `service namespace + // `_ + // of the Grpc endpoint. + // + // Example: appmesh + string service_name = 1 [(validate.rules).string.min_bytes = 1]; + + // The `region `_ hosting the Grpc + // endpoint. If unspecified, the extension will use the value in the ``AWS_REGION`` environment + // variable. + // + // Example: us-west-2 + string region = 2; +} diff --git a/docs/build.sh b/docs/build.sh index acbc73303257..b147712dd537 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -130,6 +130,7 @@ PROTO_RST=" /envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto.rst /envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto.rst /envoy/config/filter/thrift/router/v2alpha1/router/envoy/config/filter/thrift/router/v2alpha1/router.proto.rst + /envoy/config/grpc_credential/v2alpha/aws_iam/envoy/config/grpc_credential/v2alpha/aws_iam.proto.rst /envoy/config/health_checker/redis/v2/redis/envoy/config/health_checker/redis/v2/redis.proto.rst /envoy/config/overload/v2alpha/overload/envoy/config/overload/v2alpha/overload.proto.rst /envoy/config/rbac/v2/rbac/envoy/config/rbac/v2/rbac.proto.rst diff --git a/docs/root/api-v2/config/config.rst b/docs/root/api-v2/config/config.rst index 134d5101c7c8..c57ccbcc3f80 100644 --- a/docs/root/api-v2/config/config.rst +++ b/docs/root/api-v2/config/config.rst @@ -13,3 +13,4 @@ Extensions resource_monitor/resource_monitor common/common cluster/cluster + grpc_credential/grpc_credential diff --git a/docs/root/api-v2/config/grpc_credential/grpc_credential.rst b/docs/root/api-v2/config/grpc_credential/grpc_credential.rst new file mode 100644 index 000000000000..4f154fbf04d0 --- /dev/null +++ b/docs/root/api-v2/config/grpc_credential/grpc_credential.rst @@ -0,0 +1,8 @@ +Grpc Credentials +================ + +.. toctree:: + :glob: + :maxdepth: 1 + + v2alpha/* diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 4ee932fd613a..4d5123dd04bb 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -12,6 +12,7 @@ Version history * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * config: added stat :ref:`init_fetch_timeout `. * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. +* grpc: added :ref:`AWS IAM grpc credentials extension ` for AWS-managed xDS. * grpc-json: added support for :ref:`ignoring unknown query parameters`. * header to metadata: added :ref:`PROTOBUF_VALUE ` and :ref:`ValueEncode ` to support protobuf Value and Base64 encoding. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index b96c047dab66..95627187f702 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -18,6 +18,7 @@ EXTENSIONS = { # "envoy.grpc_credentials.file_based_metadata": "//source/extensions/grpc_credentials/file_based_metadata:config", + "envoy.grpc_credentials.aws_iam": "//source/extensions/grpc_credentials/aws_iam:config", # # Health checkers diff --git a/source/extensions/grpc_credentials/aws_iam/BUILD b/source/extensions/grpc_credentials/aws_iam/BUILD new file mode 100644 index 000000000000..8c8d36fe3b85 --- /dev/null +++ b/source/extensions/grpc_credentials/aws_iam/BUILD @@ -0,0 +1,33 @@ +licenses(["notice"]) # Apache 2 + +# AWS IAM gRPC Credentials + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + external_deps = ["grpc"], + deps = [ + "//include/envoy/grpc:google_grpc_creds_interface", + "//include/envoy/registry", + "//source/common/common:assert_lib", + "//source/common/config:utility_lib", + "//source/common/grpc:google_grpc_creds_lib", + "//source/common/http:message_lib", + "//source/common/http:utility_lib", + "//source/extensions/filters/http/common/aws:credentials_provider_impl_lib", + "//source/extensions/filters/http/common/aws:region_provider_impl_lib", + "//source/extensions/filters/http/common/aws:signer_impl_lib", + "//source/extensions/filters/http/common/aws:utility_lib", + "//source/extensions/grpc_credentials:well_known_names", + "@envoy_api//envoy/config/grpc_credential/v2alpha:aws_iam_cc", + ], +) diff --git a/source/extensions/grpc_credentials/aws_iam/config.cc b/source/extensions/grpc_credentials/aws_iam/config.cc new file mode 100644 index 000000000000..3128a9608434 --- /dev/null +++ b/source/extensions/grpc_credentials/aws_iam/config.cc @@ -0,0 +1,148 @@ +#include "extensions/grpc_credentials/aws_iam/config.h" + +#include "envoy/api/v2/core/grpc_service.pb.h" +#include "envoy/common/exception.h" +#include "envoy/config/grpc_credential/v2alpha/aws_iam.pb.validate.h" +#include "envoy/grpc/google_grpc_creds.h" +#include "envoy/registry/registry.h" + +#include "common/config/utility.h" +#include "common/grpc/google_grpc_creds_impl.h" +#include "common/http/utility.h" +#include "common/protobuf/message_validator_impl.h" + +#include "extensions/filters/http/common/aws/credentials_provider_impl.h" +#include "extensions/filters/http/common/aws/region_provider_impl.h" +#include "extensions/filters/http/common/aws/signer_impl.h" +#include "extensions/filters/http/common/aws/utility.h" + +namespace Envoy { +namespace Extensions { +namespace GrpcCredentials { +namespace AwsIam { + +std::shared_ptr AwsIamGrpcCredentialsFactory::getChannelCredentials( + const envoy::api::v2::core::GrpcService& grpc_service_config, Api::Api& api) { + + const auto& google_grpc = grpc_service_config.google_grpc(); + std::shared_ptr creds = + Grpc::CredsUtility::defaultSslChannelCredentials(grpc_service_config, api); + + std::shared_ptr call_creds; + for (const auto& credential : google_grpc.call_credentials()) { + switch (credential.credential_specifier_case()) { + case envoy::api::v2::core::GrpcService::GoogleGrpc::CallCredentials::kFromPlugin: { + if (credential.from_plugin().name() == GrpcCredentialsNames::get().AwsIam) { + AwsIamGrpcCredentialsFactory credentials_factory; + const Envoy::ProtobufTypes::MessagePtr config_message = + Envoy::Config::Utility::translateToFactoryConfig( + credential.from_plugin(), ProtobufMessage::getNullValidationVisitor(), + credentials_factory); + const auto& config = Envoy::MessageUtil::downcastAndValidate< + const envoy::config::grpc_credential::v2alpha::AwsIamConfig&>(*config_message); + auto credentials_provider = + std::make_shared( + api, HttpFilters::Common::Aws::Utility::metadataFetcher); + auto signer = std::make_unique( + config.service_name(), getRegion(config), credentials_provider, api.timeSource()); + std::shared_ptr new_call_creds = grpc::MetadataCredentialsFromPlugin( + std::make_unique(std::move(signer))); + if (call_creds == nullptr) { + call_creds = new_call_creds; + } else { + call_creds = grpc::CompositeCallCredentials(call_creds, new_call_creds); + } + } + break; + } + default: + // unused credential types + continue; + } + } + + if (call_creds != nullptr) { + return grpc::CompositeChannelCredentials(creds, call_creds); + } + + return creds; +} + +std::string AwsIamGrpcCredentialsFactory::getRegion( + const envoy::config::grpc_credential::v2alpha::AwsIamConfig& config) { + std::unique_ptr region_provider; + if (!config.region().empty()) { + region_provider = + std::make_unique(config.region()); + } else { + region_provider = std::make_unique(); + } + + if (!region_provider->getRegion().has_value()) { + throw EnvoyException("Could not determine AWS region. " + "If you are not running Envoy in EC2 or ECS, " + "provide the region in the plugin configuration."); + } + + return *region_provider->getRegion(); +} + +grpc::Status +AwsIamHeaderAuthenticator::GetMetadata(grpc::string_ref service_url, grpc::string_ref method_name, + const grpc::AuthContext&, + std::multimap* metadata) { + + auto message = buildMessageToSign(absl::string_view(service_url.data(), service_url.length()), + absl::string_view(method_name.data(), method_name.length())); + + try { + signer_->sign(message, false); + } catch (const EnvoyException& e) { + return grpc::Status(grpc::StatusCode::INTERNAL, e.what()); + } + + signedHeadersToMetadata(message.headers(), *metadata); + + return grpc::Status::OK; +} + +Http::RequestMessageImpl +AwsIamHeaderAuthenticator::buildMessageToSign(absl::string_view service_url, + absl::string_view method_name) { + + const auto uri = fmt::format("{}/{}", service_url, method_name); + absl::string_view host; + absl::string_view path; + Http::Utility::extractHostPathFromUri(uri, host, path); + + Http::RequestMessageImpl message; + message.headers().insertMethod().value().setReference(Http::Headers::get().MethodValues.Post); + message.headers().insertHost().value(host); + message.headers().insertPath().value(path); + + return message; +} + +void AwsIamHeaderAuthenticator::signedHeadersToMetadata( + const Http::HeaderMap& headers, std::multimap& metadata) { + + headers.iterate( + [](const Http::HeaderEntry& entry, void* context) -> Http::HeaderMap::Iterate { + auto* md = static_cast*>(context); + const auto& key = entry.key().getStringView(); + // Skip pseudo-headers + if (key.empty() || key[0] == ':') { + return Http::HeaderMap::Iterate::Continue; + } + md->emplace(key, entry.value().getStringView()); + return Http::HeaderMap::Iterate::Continue; + }, + &metadata); +} + +REGISTER_FACTORY(AwsIamGrpcCredentialsFactory, Grpc::GoogleGrpcCredentialsFactory); + +} // namespace AwsIam +} // namespace GrpcCredentials +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/grpc_credentials/aws_iam/config.h b/source/extensions/grpc_credentials/aws_iam/config.h new file mode 100644 index 000000000000..3682a8603b8d --- /dev/null +++ b/source/extensions/grpc_credentials/aws_iam/config.h @@ -0,0 +1,62 @@ +#pragma once + +#include "envoy/config/grpc_credential/v2alpha/aws_iam.pb.validate.h" +#include "envoy/grpc/google_grpc_creds.h" +#include "envoy/http/header_map.h" + +#include "common/http/message_impl.h" + +#include "extensions/filters/http/common/aws/signer.h" +#include "extensions/grpc_credentials/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace GrpcCredentials { +namespace AwsIam { + +/** + * AWS IAM based gRPC channel credentials factory. + */ +class AwsIamGrpcCredentialsFactory : public Grpc::GoogleGrpcCredentialsFactory { +public: + std::shared_ptr + getChannelCredentials(const envoy::api::v2::core::GrpcService& grpc_service_config, + Api::Api& api) override; + + Envoy::ProtobufTypes::MessagePtr createEmptyConfigProto() { + return std::make_unique(); + } + + std::string name() const override { return GrpcCredentialsNames::get().AwsIam; } + +private: + static std::string getRegion(const envoy::config::grpc_credential::v2alpha::AwsIamConfig& config); +}; + +/** + * Produce AWS IAM signature metadata for a gRPC call. + */ +class AwsIamHeaderAuthenticator : public grpc::MetadataCredentialsPlugin { +public: + AwsIamHeaderAuthenticator(HttpFilters::Common::Aws::SignerPtr signer) + : signer_(std::move(signer)) {} + + grpc::Status GetMetadata(grpc::string_ref, grpc::string_ref, const grpc::AuthContext&, + std::multimap* metadata) override; + + bool IsBlocking() const override { return true; } + +private: + static Http::RequestMessageImpl buildMessageToSign(absl::string_view service_url, + absl::string_view method_name); + + static void signedHeadersToMetadata(const Http::HeaderMap& headers, + std::multimap& metadata); + + const HttpFilters::Common::Aws::SignerPtr signer_; +}; + +} // namespace AwsIam +} // namespace GrpcCredentials +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/grpc_credentials/well_known_names.h b/source/extensions/grpc_credentials/well_known_names.h index bfb145770045..e654414e9b47 100644 --- a/source/extensions/grpc_credentials/well_known_names.h +++ b/source/extensions/grpc_credentials/well_known_names.h @@ -18,6 +18,8 @@ class GrpcCredentialsNameValues { const std::string AccessTokenExample = "envoy.grpc_credentials.access_token_example"; // File Based Metadata credentials const std::string FileBasedMetadata = "envoy.grpc_credentials.file_based_metadata"; + // AWS IAM + const std::string AwsIam = "envoy.grpc_credentials.aws_iam"; }; using GrpcCredentialsNames = ConstSingleton; diff --git a/test/extensions/grpc_credentials/aws_iam/BUILD b/test/extensions/grpc_credentials/aws_iam/BUILD new file mode 100644 index 000000000000..54acfea8cbc6 --- /dev/null +++ b/test/extensions/grpc_credentials/aws_iam/BUILD @@ -0,0 +1,23 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", + "envoy_select_google_grpc", +) + +envoy_package() + +envoy_cc_test( + name = "aws_iam_grpc_credentials_test", + srcs = envoy_select_google_grpc(["aws_iam_grpc_credentials_test.cc"]), + data = ["//test/config/integration/certs"], + deps = [ + "//source/extensions/grpc_credentials:well_known_names", + "//source/extensions/grpc_credentials/aws_iam:config", + "//test/common/grpc:grpc_client_integration_test_harness_lib", + "//test/integration:integration_lib", + "@envoy_api//envoy/config/grpc_credential/v2alpha:aws_iam_cc", + ] + envoy_select_google_grpc(["//source/common/grpc:google_async_client_lib"]), +) diff --git a/test/extensions/grpc_credentials/aws_iam/aws_iam_grpc_credentials_test.cc b/test/extensions/grpc_credentials/aws_iam/aws_iam_grpc_credentials_test.cc new file mode 100644 index 000000000000..7579f3a24fa2 --- /dev/null +++ b/test/extensions/grpc_credentials/aws_iam/aws_iam_grpc_credentials_test.cc @@ -0,0 +1,116 @@ +#include "envoy/config/grpc_credential/v2alpha/aws_iam.pb.h" + +#include "common/common/fmt.h" +#include "common/common/utility.h" +#include "common/grpc/google_async_client_impl.h" + +#include "extensions/grpc_credentials/well_known_names.h" + +#include "test/common/grpc/grpc_client_integration_test_harness.h" +#include "test/integration/fake_upstream.h" +#include "test/test_common/environment.h" + +#include "absl/strings/match.h" + +namespace Envoy { +namespace Grpc { +namespace { + +// AWS IAM credential validation tests. +class GrpcAwsIamClientIntegrationTest : public GrpcSslClientIntegrationTest { +public: + void SetUp() override { + GrpcSslClientIntegrationTest::SetUp(); + TestEnvironment::setEnvVar("AWS_ACCESS_KEY_ID", "test_akid", 1); + TestEnvironment::setEnvVar("AWS_SECRET_ACCESS_KEY", "test_secret", 1); + } + + void TearDown() override { + GrpcSslClientIntegrationTest::TearDown(); + TestEnvironment::unsetEnvVar("AWS_REGION"); + TestEnvironment::unsetEnvVar("AWS_ACCESS_KEY_ID"); + TestEnvironment::unsetEnvVar("AWS_SECRET_ACCESS_KEY"); + } + + void expectExtraHeaders(FakeStream& fake_stream) override { + AssertionResult result = fake_stream.waitForHeadersComplete(); + RELEASE_ASSERT(result, result.message()); + Http::TestHeaderMapImpl stream_headers(fake_stream.headers()); + const auto auth_header = stream_headers.get_("Authorization"); + const auto auth_parts = StringUtil::splitToken(auth_header, ", ", false); + ASSERT_EQ(4, auth_parts.size()); + EXPECT_EQ("AWS4-HMAC-SHA256", auth_parts[0]); + EXPECT_TRUE(absl::StartsWith(auth_parts[1], "Credential=test_akid/")); + EXPECT_TRUE(absl::EndsWith(auth_parts[1], + fmt::format("{}/{}/aws4_request", region_name_, service_name_))); + EXPECT_EQ("SignedHeaders=host;x-amz-date", auth_parts[2]); + // We don't verify correctness off the signature here, as this is part of the signer unit tests. + EXPECT_TRUE(absl::StartsWith(auth_parts[3], "Signature=")); + } + + envoy::api::v2::core::GrpcService createGoogleGrpcConfig() override { + auto config = GrpcSslClientIntegrationTest::createGoogleGrpcConfig(); + auto* google_grpc = config.mutable_google_grpc(); + google_grpc->set_credentials_factory_name(credentials_factory_name_); + auto* ssl_creds = google_grpc->mutable_channel_credentials()->mutable_ssl_credentials(); + ssl_creds->mutable_root_certs()->set_filename( + TestEnvironment::runfilesPath("test/config/integration/certs/upstreamcacert.pem")); + + std::string config_yaml; + if (region_in_env_) { + TestEnvironment::setEnvVar("AWS_REGION", region_name_, 1); + config_yaml = fmt::format(R"EOF( +service_name: {} +)EOF", + service_name_); + } else { + config_yaml = fmt::format(R"EOF( +service_name: {} +region: {} +)EOF", + service_name_, region_name_); + } + + auto* plugin_config = google_grpc->add_call_credentials()->mutable_from_plugin(); + plugin_config->set_name(credentials_factory_name_); + envoy::config::grpc_credential::v2alpha::AwsIamConfig metadata_config; + Envoy::TestUtility::loadFromYaml(config_yaml, *plugin_config->mutable_config()); + + return config; + } + + bool region_in_env_{}; + std::string service_name_{}; + std::string region_name_{}; + std::string credentials_factory_name_{}; +}; + +INSTANTIATE_TEST_SUITE_P(SslIpVersionsClientType, GrpcAwsIamClientIntegrationTest, + GRPC_CLIENT_INTEGRATION_PARAMS); + +TEST_P(GrpcAwsIamClientIntegrationTest, AwsIamGrpcAuth_ConfigRegion) { + SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); + service_name_ = "test_service"; + region_name_ = "test_region_static"; + credentials_factory_name_ = Extensions::GrpcCredentials::GrpcCredentialsNames::get().AwsIam; + initialize(); + auto request = createRequest(empty_metadata_); + request->sendReply(); + dispatcher_helper_.runDispatcher(); +} + +TEST_P(GrpcAwsIamClientIntegrationTest, AwsIamGrpcAuth_EnvRegion) { + SKIP_IF_GRPC_CLIENT(ClientType::EnvoyGrpc); + service_name_ = "test_service"; + region_name_ = "test_region_env"; + region_in_env_ = true; + credentials_factory_name_ = Extensions::GrpcCredentials::GrpcCredentialsNames::get().AwsIam; + initialize(); + auto request = createRequest(empty_metadata_); + request->sendReply(); + dispatcher_helper_.runDispatcher(); +} + +} // namespace +} // namespace Grpc +} // namespace Envoy \ No newline at end of file diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index a83054bdcdb2..86b0a0ec3bdd 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -352,7 +352,6 @@ void TestEnvironment::setEnvVar(const std::string& name, const std::string& valu ASSERT_EQ(0, rc); #endif } - void TestEnvironment::unsetEnvVar(const std::string& name) { #ifdef WIN32 const int rc = ::_putenv_s(name.c_str(), ""); From 05e3f6e07b49afe80790a3a90c03b862ecda77c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bence=20B=C3=A9ky?= Date: Thu, 8 Aug 2019 23:04:00 -0400 Subject: [PATCH 350/542] Remove dummy.h, dummy.cc, dummy_test.cc. (#7871) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bence Béky --- .../extensions/quic_listeners/quiche/dummy.cc | 17 --------------- .../extensions/quic_listeners/quiche/dummy.h | 17 --------------- .../quic_listeners/quiche/dummy_test.cc | 21 ------------------- 3 files changed, 55 deletions(-) delete mode 100644 source/extensions/quic_listeners/quiche/dummy.cc delete mode 100644 source/extensions/quic_listeners/quiche/dummy.h delete mode 100644 test/extensions/quic_listeners/quiche/dummy_test.cc diff --git a/source/extensions/quic_listeners/quiche/dummy.cc b/source/extensions/quic_listeners/quiche/dummy.cc deleted file mode 100644 index f09ec04b42bc..000000000000 --- a/source/extensions/quic_listeners/quiche/dummy.cc +++ /dev/null @@ -1,17 +0,0 @@ -#include "extensions/quic_listeners/quiche/dummy.h" - -using http2::Http2String; - -namespace Envoy { -namespace Extensions { -namespace QuicListeners { -namespace Quiche { - -// Placeholder use of a QUICHE platform type. -// TODO(mpwarres): remove once real uses of QUICHE platform added. -Http2String moreCowbell(const Http2String& s) { return s + " cowbell"; } - -} // namespace Quiche -} // namespace QuicListeners -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/dummy.h b/source/extensions/quic_listeners/quiche/dummy.h deleted file mode 100644 index c57b89486de3..000000000000 --- a/source/extensions/quic_listeners/quiche/dummy.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "quiche/http2/platform/api/http2_string.h" - -namespace Envoy { -namespace Extensions { -namespace QuicListeners { -namespace Quiche { - -// Placeholder use of a QUICHE platform type. -// TODO(mpwarres): remove once real uses of QUICHE platform added. -http2::Http2String moreCowbell(const http2::Http2String& s); - -} // namespace Quiche -} // namespace QuicListeners -} // namespace Extensions -} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/dummy_test.cc b/test/extensions/quic_listeners/quiche/dummy_test.cc deleted file mode 100644 index 77d45a589997..000000000000 --- a/test/extensions/quic_listeners/quiche/dummy_test.cc +++ /dev/null @@ -1,21 +0,0 @@ -#include "extensions/quic_listeners/quiche/dummy.h" - -#include "gtest/gtest.h" -#include "quiche/http2/platform/api/http2_string.h" - -namespace Envoy { -namespace Extensions { -namespace QuicListeners { -namespace Quiche { -namespace { - -TEST(DummyTest, Dummy) { - http2::Http2String foo = "bar"; - EXPECT_EQ("bar cowbell", moreCowbell(foo)); -} - -} // namespace -} // namespace Quiche -} // namespace QuicListeners -} // namespace Extensions -} // namespace Envoy From 85a294e1cc7ccc649810ce336b7d85fa3f271711 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Tue, 30 Jul 2019 09:43:18 -0400 Subject: [PATCH 351/542] http2: Limit the number of outbound frames (#9) Limit the number of outbound (these, waiting to be written into the socket) HTTP/2 frames. When the limit is exceeded the connection is terminated. This mitigates flood exploits where a client continually sends frames that are not subject to flow control without reading server responses. Fixes CVE-2019-9512, CVE-2019-9514 and CVE-2019-9515. Signed-off-by: Yan Avlasov --- api/envoy/api/v2/core/protocol.proto | 15 ++ .../configuration/http_conn_man/stats.rst | 2 + docs/root/intro/version_history.rst | 6 + include/envoy/buffer/buffer.h | 4 +- include/envoy/http/codec.h | 7 + source/common/buffer/buffer_impl.h | 44 +++- source/common/http/conn_manager_impl.cc | 28 ++- source/common/http/conn_manager_impl.h | 1 + source/common/http/exception.h | 8 + source/common/http/http2/codec_impl.cc | 122 +++++++++- source/common/http/http2/codec_impl.h | 78 ++++++- source/common/http/utility.cc | 5 + source/common/network/connection_impl.h | 2 + test/common/buffer/buffer_test.cc | 16 ++ test/common/buffer/owned_impl_test.cc | 51 +++++ test/common/http/conn_manager_impl_test.cc | 21 ++ test/common/http/http2/BUILD | 10 + test/common/http/http2/codec_impl_test.cc | 210 ++++++++++++++++++ test/common/http/http2/codec_impl_test_util.h | 1 + test/common/http/http2/http2_frame.cc | 148 ++++++++++++ test/common/http/http2/http2_frame.h | 126 +++++++++++ test/common/http/utility_test.cc | 3 + test/config/utility.cc | 15 ++ test/config/utility.h | 3 + test/integration/BUILD | 1 + test/integration/http2_integration_test.cc | 158 +++++++++++++ test/integration/http2_integration_test.h | 20 ++ test/integration/integration.cc | 9 + test/integration/integration.h | 5 +- .../integration/tcp_proxy_integration_test.cc | 9 + test/integration/utility.cc | 5 + test/integration/utility.h | 9 +- test/test_common/utility.cc | 2 + 33 files changed, 1123 insertions(+), 21 deletions(-) create mode 100644 test/common/http/http2/http2_frame.cc create mode 100644 test/common/http/http2/http2_frame.h diff --git a/api/envoy/api/v2/core/protocol.proto b/api/envoy/api/v2/core/protocol.proto index 200b8517abd1..becd596a0e98 100644 --- a/api/envoy/api/v2/core/protocol.proto +++ b/api/envoy/api/v2/core/protocol.proto @@ -91,6 +91,21 @@ message Http2ProtocolOptions { // docs](https://github.com/envoyproxy/envoy/blob/master/source/docs/h2_metadata.md) for more // information. bool allow_metadata = 6; + + // Limit the number of pending outbound downstream frames of all types (frames that are waiting to + // be written into the socket). Exceeding this limit triggers flood mitigation and connection is + // terminated. The "http2.outbound_flood" stat tracks the number of terminated connections due to + // flood mitigation. The default limit is 10000. + // [#comment:TODO: implement same limits for upstream outbound frames as well.] + google.protobuf.UInt32Value max_outbound_frames = 7 [(validate.rules).uint32 = {gte: 1}]; + + // Limit the number of pending outbound downstream frames of types PING, SETTINGS and RST_STREAM, + // preventing high memory utilization when receiving continuous stream of these frames. Exceeding + // this limit triggers flood mitigation and connection is terminated. The + // "http2.outbound_control_flood" stat tracks the number of terminated connections due to flood + // mitigation. The default limit is 1000. + // [#comment:TODO: implement same limits for upstream outbound frames as well.] + google.protobuf.UInt32Value max_outbound_control_frames = 8 [(validate.rules).uint32 = {gte: 1}]; } // [#not-implemented-hide:] diff --git a/docs/root/configuration/http_conn_man/stats.rst b/docs/root/configuration/http_conn_man/stats.rst index 0269fefc1d4b..3c5a0cdae14c 100644 --- a/docs/root/configuration/http_conn_man/stats.rst +++ b/docs/root/configuration/http_conn_man/stats.rst @@ -111,6 +111,8 @@ All http2 statistics are rooted at *http2.* header_overflow, Counter, Total number of connections reset due to the headers being larger than the :ref:`configured value `. headers_cb_no_stream, Counter, Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug + outbound_flood, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting `. + outbound_control_flood, Counter, "Total number of connections terminated for exceeding the limit on outbound frames of types PING, SETTINGS and RST_STREAM. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `." rx_messaging_error, Counter, Total number of invalid received frames that violated `section 8 `_ of the HTTP/2 spec. This will result in a *tx_reset* rx_reset, Counter, Total number of reset stream frames received by Envoy too_many_header_frames, Counter, Total number of times an HTTP2 connection is reset due to receiving too many headers frames. Envoy currently supports proxying at most one header frame for 100-Continue one non-100 response code header frame and one frame with trailers diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index bcd2848341e1..223773dfe5e5 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -1,6 +1,12 @@ Version history --------------- +1.11.1 (Pending) +================ +* http: added mitigation of client initiated atacks that result in flooding of the outbound queue of downstream HTTP/2 connections. +* http: added :ref:`outbound_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting ` +* http: added :ref:`outbound_control_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `. + 1.11.0 (July 11, 2019) ====================== * access log: added a new field for downstream TLS session ID to file and gRPC access logger. diff --git a/include/envoy/buffer/buffer.h b/include/envoy/buffer/buffer.h index 1f19d94d6601..468cf17fa967 100644 --- a/include/envoy/buffer/buffer.h +++ b/include/envoy/buffer/buffer.h @@ -33,6 +33,7 @@ struct RawSlice { */ class BufferFragment { public: + virtual ~BufferFragment() = default; /** * @return const void* a pointer to the referenced data. */ @@ -47,9 +48,6 @@ class BufferFragment { * Called by a buffer when the referenced data is no longer needed. */ virtual void done() PURE; - -protected: - virtual ~BufferFragment() = default; }; /** diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 4ac313f7d6f3..d05943c0f955 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -235,6 +235,8 @@ struct Http2Settings { uint32_t initial_connection_window_size_{DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE}; bool allow_connect_{DEFAULT_ALLOW_CONNECT}; bool allow_metadata_{DEFAULT_ALLOW_METADATA}; + uint32_t max_outbound_frames_{DEFAULT_MAX_OUTBOUND_FRAMES}; + uint32_t max_outbound_control_frames_{DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES}; // disable HPACK compression static const uint32_t MIN_HPACK_TABLE_SIZE = 0; @@ -272,6 +274,11 @@ struct Http2Settings { static const bool DEFAULT_ALLOW_CONNECT = false; // By default Envoy does not allow METADATA support. static const bool DEFAULT_ALLOW_METADATA = false; + + // Default limit on the number of outbound frames of all types. + static const uint32_t DEFAULT_MAX_OUTBOUND_FRAMES = 10000; + // Default limit on the number of outbound frames of types PING, SETTINGS and RST_STREAM. + static const uint32_t DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES = 1000; }; /** diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 9851686b4fbb..68b361c579d5 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -193,7 +193,8 @@ class Slice { using SlicePtr = std::unique_ptr; -class OwnedSlice : public Slice, public InlineStorage { +// OwnedSlice can not be derived from as it has variable sized array as member. +class OwnedSlice final : public Slice, public InlineStorage { public: /** * Create an empty OwnedSlice. @@ -563,5 +564,46 @@ class OwnedImpl : public LibEventInstance { Event::Libevent::BufferPtr buffer_; }; +using BufferFragmentPtr = std::unique_ptr; + +/** + * An implementation of BufferFragment where a releasor callback is called when the data is + * no longer needed. Copies data into internal buffer. + */ +class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorage { +public: + using Releasor = std::function; + + /** + * Copies the data into internal buffer. The releasor is called when the data has been + * fully drained or the buffer that contains this fragment is destroyed. + * @param data external data to reference + * @param releasor a callback function to be called when data is no longer needed. + */ + + static BufferFragmentPtr create(absl::string_view data, const Releasor& releasor) { + return BufferFragmentPtr(new (sizeof(OwnedBufferFragmentImpl) + data.size()) + OwnedBufferFragmentImpl(data, releasor)); + } + + // Buffer::BufferFragment + const void* data() const override { return data_; } + size_t size() const override { return size_; } + void done() override { releasor_(this); } + +private: + OwnedBufferFragmentImpl(absl::string_view data, const Releasor& releasor) + : releasor_(releasor), size_(data.size()) { + ASSERT(releasor != nullptr); + memcpy(data_, data.data(), data.size()); + } + + const Releasor releasor_; + const size_t size_; + uint8_t data_[]; +}; + +using OwnedBufferFragmentImplPtr = std::unique_ptr; + } // namespace Buffer } // namespace Envoy diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 8e905ac5aba0..bda3cde7403e 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -257,6 +257,19 @@ StreamDecoder& ConnectionManagerImpl::newStream(StreamEncoder& response_encoder, return **streams_.begin(); } +void ConnectionManagerImpl::handleCodecException(const char* error, + Network::ConnectionCloseType close_type) { + ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), error); + + // In the protocol error case, we need to reset all streams now. If the close_type is + // FlushWriteAndDelay, the connection might stick around long enough for a pending stream to come + // back and try to encode. In other cases it avoids needless processing of upstream responses when + // downstream connection is closed. + resetAllStreams(); + + read_callbacks_->connection().close(close_type); +} + Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) { if (!codec_) { codec_ = config_.createCodec(read_callbacks_->connection(), data, *this); @@ -275,18 +288,15 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool try { codec_->dispatch(data); + } catch (const FrameFloodException& e) { + // Abortively close flooded connections + handleCodecException(e.what(), Network::ConnectionCloseType::NoFlush); + return Network::FilterStatus::StopIteration; } catch (const CodecProtocolException& e) { + stats_.named_.downstream_cx_protocol_error_.inc(); // HTTP/1.1 codec has already sent a 400 response if possible. HTTP/2 codec has already sent // GOAWAY. - ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), e.what()); - stats_.named_.downstream_cx_protocol_error_.inc(); - - // In the protocol error case, we need to reset all streams now. Since we do a flush write and - // delayed close, the connection might stick around long enough for a pending stream to come - // back and try to encode. - resetAllStreams(); - - read_callbacks_->connection().close(Network::ConnectionCloseType::FlushWriteAndDelay); + handleCodecException(e.what(), Network::ConnectionCloseType::FlushWriteAndDelay); return Network::FilterStatus::StopIteration; } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index a7ef3ef9eae8..cafef6631093 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -582,6 +582,7 @@ class ConnectionManagerImpl : Logger::Loggable, void onDrainTimeout(); void startDrainSequence(); Tracing::HttpTracer& tracer() { return http_context_.tracer(); } + void handleCodecException(const char* error, Network::ConnectionCloseType close_type); enum class DrainState { NotDraining, Draining, Closing }; diff --git a/source/common/http/exception.h b/source/common/http/exception.h index 445ef5fb1407..1a7ef668b24b 100644 --- a/source/common/http/exception.h +++ b/source/common/http/exception.h @@ -16,6 +16,14 @@ class CodecProtocolException : public EnvoyException { CodecProtocolException(const std::string& message) : EnvoyException(message) {} }; +/** + * Raised when outbound frame queue flood is detected. + */ +class FrameFloodException : public CodecProtocolException { +public: + FrameFloodException(const std::string& message) : CodecProtocolException(message) {} +}; + /** * Raised when a response is received on a connection that did not send a request. In practice * this can only happen on HTTP/1.1 connections. diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 475aa1d314e2..fe7518c19cff 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -11,6 +11,7 @@ #include "envoy/stats/scope.h" #include "common/common/assert.h" +#include "common/common/cleanup.h" #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/stack_array.h" @@ -251,7 +252,13 @@ int ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t // https://nghttp2.org/documentation/types.html#c.nghttp2_send_data_callback static const uint64_t FRAME_HEADER_SIZE = 9; - Buffer::OwnedImpl output(framehd, FRAME_HEADER_SIZE); + Buffer::OwnedImpl output; + if (!parent_.addOutboundFrameFragment(output, framehd, FRAME_HEADER_SIZE)) { + ENVOY_CONN_LOG(debug, "error sending data frame: Too many frames in the outbound queue", + parent_.connection_); + return NGHTTP2_ERR_FLOODED; + } + output.move(pending_send_data_, length); parent_.connection_.write(output, false); return 0; @@ -348,6 +355,10 @@ void ConnectionImpl::dispatch(Buffer::Instance& data) { dispatching_ = true; ssize_t rc = nghttp2_session_mem_recv(session_, static_cast(slice.mem_), slice.len_); + if (rc == NGHTTP2_ERR_FLOODED) { + throw FrameFloodException( + "Flooding was detected in this HTTP/2 session, and it must be closed"); + } if (rc != static_cast(slice.len_)) { throw CodecProtocolException(fmt::format("{}", nghttp2_strerror(rc))); } @@ -555,9 +566,77 @@ int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) { return NGHTTP2_ERR_CALLBACK_FAILURE; } +int ConnectionImpl::onBeforeFrameSend(const nghttp2_frame* frame) { + ENVOY_CONN_LOG(trace, "about to sent frame type={}, flags={}", connection_, + static_cast(frame->hd.type), static_cast(frame->hd.flags)); + ASSERT(!is_outbound_flood_monitored_control_frame_); + // Flag flood monitored outbound control frames. + is_outbound_flood_monitored_control_frame_ = + ((frame->hd.type == NGHTTP2_PING || frame->hd.type == NGHTTP2_SETTINGS) && + frame->hd.flags & NGHTTP2_FLAG_ACK) || + frame->hd.type == NGHTTP2_RST_STREAM; + return 0; +} + +void ConnectionImpl::incrementOutboundFrameCount(bool is_outbound_flood_monitored_control_frame) { + ++outbound_frames_; + if (is_outbound_flood_monitored_control_frame) { + ++outbound_control_frames_; + } + checkOutboundQueueLimits(); +} + +bool ConnectionImpl::addOutboundFrameFragment(Buffer::OwnedImpl& output, const uint8_t* data, + size_t length) { + // Reset the outbound frame type (set in the onBeforeFrameSend callback) since the + // onBeforeFrameSend callback is not called for DATA frames. + bool is_outbound_flood_monitored_control_frame = false; + std::swap(is_outbound_flood_monitored_control_frame, is_outbound_flood_monitored_control_frame_); + try { + incrementOutboundFrameCount(is_outbound_flood_monitored_control_frame); + } catch (const FrameFloodException&) { + return false; + } + + auto fragment = Buffer::OwnedBufferFragmentImpl::create( + absl::string_view(reinterpret_cast(data), length), + is_outbound_flood_monitored_control_frame ? control_frame_buffer_releasor_ + : frame_buffer_releasor_); + + // The Buffer::OwnedBufferFragmentImpl object will be deleted in the *frame_buffer_releasor_ + // callback. + output.addBufferFragment(*fragment.release()); + return true; +} + +void ConnectionImpl::releaseOutboundFrame(const Buffer::OwnedBufferFragmentImpl* fragment) { + ASSERT(outbound_frames_ >= 1); + --outbound_frames_; + delete fragment; +} + +void ConnectionImpl::releaseOutboundControlFrame(const Buffer::OwnedBufferFragmentImpl* fragment) { + ASSERT(outbound_control_frames_ >= 1); + --outbound_control_frames_; + releaseOutboundFrame(fragment); +} + ssize_t ConnectionImpl::onSend(const uint8_t* data, size_t length) { ENVOY_CONN_LOG(trace, "send data: bytes={}", connection_, length); - Buffer::OwnedImpl buffer(data, length); + Buffer::OwnedImpl buffer; + if (!addOutboundFrameFragment(buffer, data, length)) { + ENVOY_CONN_LOG(debug, "error sending frame: Too many frames in the outbound queue.", + connection_); + return NGHTTP2_ERR_FLOODED; + } + + // While the buffer is transient the fragment it contains will be moved into the + // write_buffer_ of the underlying connection_ by the write method below. + // This creates lifetime dependency between the write_buffer_ of the underlying connection + // and the codec object. Specifically the write_buffer_ MUST be either fully drained or + // deleted before the codec object is deleted. This is presently guaranteed by the + // destruction order of the Network::ConnectionImpl object where write_buffer_ is + // destroyed before the filter_manager_ which owns the codec through Http::ConnectionManagerImpl. connection_.write(buffer, false); return length; } @@ -663,6 +742,15 @@ void ConnectionImpl::sendPendingFrames() { int rc = nghttp2_session_send(session_); if (rc != 0) { ASSERT(rc == NGHTTP2_ERR_CALLBACK_FAILURE); + // For errors caused by the pending outbound frame flood the FrameFloodException has + // to be thrown. However the nghttp2 library returns only the generic error code for + // all failure types. Check queue limits and throw FrameFloodException if they were + // exceeded. + if (outbound_frames_ > max_outbound_frames_ || + outbound_control_frames_ > max_outbound_control_frames_) { + throw FrameFloodException("Too many frames in the outbound queue."); + } + throw CodecProtocolException(fmt::format("{}", nghttp2_strerror(rc))); } @@ -810,6 +898,11 @@ ConnectionImpl::Http2Callbacks::Http2Callbacks() { return static_cast(user_data)->onFrameSend(frame); }); + nghttp2_session_callbacks_set_before_frame_send_callback( + callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int { + return static_cast(user_data)->onBeforeFrameSend(frame); + }); + nghttp2_session_callbacks_set_on_frame_not_send_callback( callbacks_, [](nghttp2_session*, const nghttp2_frame*, int, void*) -> int { // We used to always return failure here but it looks now this can get called if the other @@ -979,6 +1072,31 @@ int ServerConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na return saveHeader(frame, std::move(name), std::move(value)); } +void ServerConnectionImpl::checkOutboundQueueLimits() { + if (outbound_frames_ > max_outbound_frames_ && dispatching_downstream_data_) { + stats_.outbound_flood_.inc(); + throw FrameFloodException("Too many frames in the outbound queue."); + } + if (outbound_control_frames_ > max_outbound_control_frames_ && dispatching_downstream_data_) { + stats_.outbound_control_flood_.inc(); + throw FrameFloodException("Too many control frames in the outbound queue."); + } +} + +void ServerConnectionImpl::dispatch(Buffer::Instance& data) { + ASSERT(!dispatching_downstream_data_); + dispatching_downstream_data_ = true; + + // Make sure the dispatching_downstream_data_ is set to false even + // when ConnectionImpl::dispatch throws an exception. + Cleanup cleanup([this]() { dispatching_downstream_data_ = false; }); + + // Make sure downstream outbound queue was not flooded by the upstream frames. + checkOutboundQueueLimits(); + + ConnectionImpl::dispatch(data); +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 0b8c8bd0c4be..a552ff461ee3 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -37,16 +38,16 @@ const std::string CLIENT_MAGIC_PREFIX = "PRI * HTTP/2"; /** * All stats for the HTTP/2 codec. @see stats_macros.h */ -// clang-format off #define ALL_HTTP2_CODEC_STATS(COUNTER) \ COUNTER(header_overflow) \ COUNTER(headers_cb_no_stream) \ + COUNTER(outbound_control_flood) \ + COUNTER(outbound_flood) \ COUNTER(rx_messaging_error) \ COUNTER(rx_reset) \ COUNTER(too_many_header_frames) \ COUNTER(trailers) \ COUNTER(tx_reset) -// clang-format on /** * Wrapper struct for the HTTP/2 codec stats. @see stats_macros.h @@ -77,12 +78,21 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable(*fragment); + EXPECT_EQ(11, slice->dataSize()); + EXPECT_EQ(0, slice->reservableSize()); + EXPECT_EQ(0, memcmp(slice->data(), input, slice->dataSize())); + EXPECT_FALSE(release_callback_called); + slice.reset(nullptr); + EXPECT_TRUE(release_callback_called); +} + TEST(SliceDequeTest, CreateDelete) { bool slice1_deleted = false; bool slice2_deleted = false; diff --git a/test/common/buffer/owned_impl_test.cc b/test/common/buffer/owned_impl_test.cc index d009db570f8d..282ae1961cf5 100644 --- a/test/common/buffer/owned_impl_test.cc +++ b/test/common/buffer/owned_impl_test.cc @@ -96,6 +96,57 @@ TEST_P(OwnedImplTest, AddBufferFragmentDynamicAllocation) { EXPECT_TRUE(release_callback_called_); } +TEST_P(OwnedImplTest, AddOwnedBufferFragmentWithCleanup) { + char input[] = "hello world"; + const size_t expected_length = sizeof(input) - 1; + auto frag = OwnedBufferFragmentImpl::create( + {input, expected_length}, + [this](const OwnedBufferFragmentImpl*) { release_callback_called_ = true; }); + Buffer::OwnedImpl buffer; + verifyImplementation(buffer); + buffer.addBufferFragment(*frag); + EXPECT_EQ(expected_length, buffer.length()); + + const uint64_t partial_drain_size = 5; + buffer.drain(partial_drain_size); + EXPECT_EQ(expected_length - partial_drain_size, buffer.length()); + EXPECT_FALSE(release_callback_called_); + + buffer.drain(expected_length - partial_drain_size); + EXPECT_EQ(0, buffer.length()); + EXPECT_TRUE(release_callback_called_); +} + +// Verify that OwnedBufferFragment work correctly when input buffer is allocated on the heap. +TEST_P(OwnedImplTest, AddOwnedBufferFragmentDynamicAllocation) { + char input_stack[] = "hello world"; + const size_t expected_length = sizeof(input_stack) - 1; + char* input = new char[expected_length]; + std::copy(input_stack, input_stack + expected_length, input); + + auto* frag = OwnedBufferFragmentImpl::create({input, expected_length}, + [this, input](const OwnedBufferFragmentImpl* frag) { + release_callback_called_ = true; + delete[] input; + delete frag; + }) + .release(); + + Buffer::OwnedImpl buffer; + verifyImplementation(buffer); + buffer.addBufferFragment(*frag); + EXPECT_EQ(expected_length, buffer.length()); + + const uint64_t partial_drain_size = 5; + buffer.drain(partial_drain_size); + EXPECT_EQ(expected_length - partial_drain_size, buffer.length()); + EXPECT_FALSE(release_callback_called_); + + buffer.drain(expected_length - partial_drain_size); + EXPECT_EQ(0, buffer.length()); + EXPECT_TRUE(release_callback_called_); +} + TEST_P(OwnedImplTest, Add) { const std::string string1 = "Hello, ", string2 = "World!"; Buffer::OwnedImpl buffer; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index ce4e8af46284..6908e459e29a 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2314,6 +2314,27 @@ TEST_F(HttpConnectionManagerImplTest, DownstreamProtocolError) { conn_manager_->onData(fake_input, false); } +// Verify that FrameFloodException causes connection to be closed abortively. +TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { + InSequence s; + setup(false, ""); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + conn_manager_->newStream(response_encoder_); + throw FrameFloodException("too many outbound frames."); + })); + + EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)); + EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); + + // FrameFloodException should result in reset of the streams followed by abortive close. + EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + TEST_F(HttpConnectionManagerImplTest, IdleTimeoutNoCodec) { // Not used in the test. delete codec_; diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index 0c245388afb9..354055602126 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -32,6 +32,7 @@ envoy_cc_test( envoy_cc_test_library( name = "codec_impl_test_util", hdrs = ["codec_impl_test_util.h"], + external_deps = ["abseil_optional"], deps = [ "//source/common/http/http2:codec_lib", ], @@ -56,6 +57,15 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "http2_frame", + srcs = ["http2_frame.cc"], + hdrs = ["http2_frame.h"], + deps = [ + "//source/common/common:macros", + ], +) + envoy_cc_test_library( name = "frame_replay_lib", srcs = ["frame_replay.cc"], diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 28b4732f3bd0..bf7010767048 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -97,6 +97,8 @@ class Http2CodecImplTestFixture { setting.initial_stream_window_size_ = ::testing::get<2>(tp); setting.initial_connection_window_size_ = ::testing::get<3>(tp); setting.allow_metadata_ = allow_metadata_; + setting.max_outbound_frames_ = max_outbound_frames_; + setting.max_outbound_control_frames_ = max_outbound_control_frames_; } // corruptMetadataFramePayload assumes data contains at least 10 bytes of the beginning of a @@ -141,6 +143,8 @@ class Http2CodecImplTestFixture { bool corrupt_metadata_frame_ = false; uint32_t max_request_headers_kb_ = Http::DEFAULT_MAX_REQUEST_HEADERS_KB; + uint32_t max_outbound_frames_ = Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; + uint32_t max_outbound_control_frames_ = Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES; }; class Http2CodecImplTest : public ::testing::TestWithParam, @@ -1039,6 +1043,212 @@ TEST_P(Http2CodecImplTestAll, TestCodecHeaderCompression) { } } +// Verify that codec detects PING flood +TEST_P(Http2CodecImplTest, PingFlood) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + // Send one frame above the outbound control queue size limit + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES + 1; ++i) { + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + } + + int ack_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &ack_count](Buffer::Instance& frame, bool) { + ++ack_count; + buffer.move(frame); + })); + + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); + EXPECT_EQ(ack_count, Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES); + EXPECT_EQ(1, stats_store_.counter("http2.outbound_control_flood").value()); +} + +// Verify that outbound control frame counter decreases when send buffer is drained +TEST_P(Http2CodecImplTest, PingFloodCounterReset) { + static const int kMaxOutboundControlFrames = 100; + max_outbound_control_frames_ = kMaxOutboundControlFrames; + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + for (int i = 0; i < kMaxOutboundControlFrames; ++i) { + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + } + + int ack_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &ack_count](Buffer::Instance& frame, bool) { + ++ack_count; + buffer.move(frame); + })); + + // We should be 1 frame under the control frame flood mitigation threshold. + EXPECT_NO_THROW(client_->sendPendingFrames()); + EXPECT_EQ(ack_count, kMaxOutboundControlFrames); + + // Drain kMaxOutboundFrames / 2 slices from the send buffer + buffer.drain(buffer.length() / 2); + + // Send kMaxOutboundFrames / 2 more pings. + for (int i = 0; i < kMaxOutboundControlFrames / 2; ++i) { + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + } + // The number of outbound frames should be half of max so the connection should not be terminated. + EXPECT_NO_THROW(client_->sendPendingFrames()); + + // 1 more ping frame should overflow the outbound frame limit. + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); +} + +// Verify that codec detects flood of outbound HEADER frames +TEST_P(Http2CodecImplTest, ResponseHeadersFlood) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + int frame_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &frame_count](Buffer::Instance& frame, bool) { + ++frame_count; + buffer.move(frame); + })); + + TestHeaderMapImpl response_headers{{":status", "200"}}; + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 1; ++i) { + EXPECT_NO_THROW(response_encoder_->encodeHeaders(response_headers, false)); + } + // Presently flood mitigation is done only when processing downstream data + // So we need to send stream from downstream client to trigger mitigation + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); + + EXPECT_EQ(frame_count, Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 1); + EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); +} + +// Verify that codec detects flood of outbound DATA frames +TEST_P(Http2CodecImplTest, ResponseDataFlood) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + int frame_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &frame_count](Buffer::Instance& frame, bool) { + ++frame_count; + buffer.move(frame); + })); + + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + // Account for the single HEADERS frame above + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + // Presently flood mitigation is done only when processing downstream data + // So we need to send stream from downstream client to trigger mitigation + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); + + EXPECT_EQ(frame_count, Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 1); + EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); +} + +// Verify that outbound frame counter decreases when send buffer is drained +TEST_P(Http2CodecImplTest, ResponseDataFloodCounterReset) { + static const int kMaxOutboundFrames = 100; + max_outbound_frames_ = kMaxOutboundFrames; + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + int frame_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &frame_count](Buffer::Instance& frame, bool) { + ++frame_count; + buffer.move(frame); + })); + + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + // Account for the single HEADERS frame above + for (uint32_t i = 0; i < kMaxOutboundFrames - 1; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + + EXPECT_EQ(frame_count, kMaxOutboundFrames); + // Drain kMaxOutboundFrames / 2 slices from the send buffer + buffer.drain(buffer.length() / 2); + + for (uint32_t i = 0; i < kMaxOutboundFrames / 2 + 1; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + + // Presently flood mitigation is done only when processing downstream data + // So we need to send a frame from downstream client to trigger mitigation + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); +} + +// Verify that control frames are added to the counter of outbound frames of all types. +TEST_P(Http2CodecImplTest, PingStacksWithDataFlood) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + int frame_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &frame_count](Buffer::Instance& frame, bool) { + ++frame_count; + buffer.move(frame); + })); + + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + // Account for the single HEADERS frame above + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES - 1; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + // Send one PING frame above the outbound queue size limit + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); + + EXPECT_EQ(frame_count, Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES); + EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/test/common/http/http2/codec_impl_test_util.h b/test/common/http/http2/codec_impl_test_util.h index b642701265c4..2c4652ee0f2f 100644 --- a/test/common/http/http2/codec_impl_test_util.h +++ b/test/common/http/http2/codec_impl_test_util.h @@ -26,6 +26,7 @@ class TestClientConnectionImpl : public ClientConnectionImpl { } nghttp2_session* session() { return session_; } using ClientConnectionImpl::getStream; + using ConnectionImpl::sendPendingFrames; }; } // namespace Http2 diff --git a/test/common/http/http2/http2_frame.cc b/test/common/http/http2/http2_frame.cc new file mode 100644 index 000000000000..ca4fb858a070 --- /dev/null +++ b/test/common/http/http2/http2_frame.cc @@ -0,0 +1,148 @@ +#include "test/common/http/http2/http2_frame.h" + +#include + +#include + +namespace { + +// Make request stream ID in the network byte order +uint32_t makeRequestStreamId(uint32_t stream_id) { return htonl((stream_id << 1) | 1); } + +// All this templatized stuff is for the typesafe constexpr bitwise ORing of the "enum class" values +template struct FirstArgType { using type = First; }; + +template constexpr uint8_t orFlags(Flag flag) { return static_cast(flag); } + +template constexpr uint8_t orFlags(Flag first, Flags... rest) { + static_assert(std::is_same::type>::value, + "All flag types must be the same!"); + return static_cast(first) | orFlags(rest...); +} + +} // namespace + +namespace Envoy { +namespace Http { +namespace Http2 { + +const char Http2Frame::Preamble[25] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; + +void Http2Frame::setHeader(absl::string_view header) { + ASSERT(header.size() >= HeaderSize); + data_.assign(HeaderSize, 0); + memcpy(&data_[0], header.data(), HeaderSize); + data_.resize(HeaderSize + payloadSize()); +} + +void Http2Frame::setPayload(absl::string_view payload) { + ASSERT(payload.size() >= payloadSize()); + memcpy(&data_[HeaderSize], payload.data(), payloadSize()); +} + +uint32_t Http2Frame::payloadSize() const { + return (uint32_t(data_[0]) << 16) + (uint32_t(data_[1]) << 8) + uint32_t(data_[2]); +} + +Http2Frame::ResponseStatus Http2Frame::responseStatus() const { + if (empty() || Type::HEADERS != type() || size() <= HeaderSize || + ((data_[HeaderSize] & 0x80) == 0)) { + return ResponseStatus::UNKNOWN; + } + // See https://tools.ietf.org/html/rfc7541#appendix-A for header values + switch (static_cast(data_[HeaderSize] & 0x7f)) { + case StaticHeaderIndex::STATUS_200: + return ResponseStatus::_200; + case StaticHeaderIndex::STATUS_404: + return ResponseStatus::_404; + default: + break; + } + return ResponseStatus::UNKNOWN; +} + +void Http2Frame::buildHeader(Type type, uint32_t payload_size, uint8_t flags, uint32_t stream_id) { + data_.assign(payload_size + HeaderSize, 0); + setPayloadSize(payload_size); + data_[3] = static_cast(type); + data_[4] = flags; + if (stream_id) { + memcpy(&data_[5], &stream_id, sizeof(stream_id)); + } +} + +void Http2Frame::setPayloadSize(uint32_t size) { + data_[0] = (size >> 16) & 0xff; + data_[1] = (size >> 8) & 0xff; + data_[2] = size & 0xff; +} + +void Http2Frame::appendHpackInt(uint64_t value, unsigned char prefix_mask) { + if (value < prefix_mask) { + data_.push_back(value); + } else { + data_.push_back(prefix_mask); + value -= prefix_mask; + + while (value >= 128) { + data_.push_back((value & 0x7f) | 0x80); + value >>= 7; + } + data_.push_back(value); + } +} + +// See https://tools.ietf.org/html/rfc7541#section-6.1 for header representations + +void Http2Frame::appendStaticHeader(StaticHeaderIndex index) { + data_.push_back(0x80 | static_cast(index)); +} + +void Http2Frame::appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::string_view value) { + appendHpackInt(static_cast(index), 0xf); + appendHpackInt(value.size(), 0x7f); + appendData(value); +} + +Http2Frame Http2Frame::makePingFrame(absl::string_view data) { + static constexpr size_t kPingPayloadSize = 8; + Http2Frame frame; + frame.buildHeader(Type::PING, kPingPayloadSize); + if (!data.empty()) { + memcpy(&frame.data_[HeaderSize], data.data(), std::min(kPingPayloadSize, data.size())); + } + return frame; +} + +Http2Frame Http2Frame::makeEmptySettingsFrame(SettingsFlags flags) { + Http2Frame frame; + frame.buildHeader(Type::SETTINGS, 0, static_cast(flags)); + return frame; +} + +Http2Frame Http2Frame::makeMalformedRequest(uint32_t stream_index) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_STREAM, HeadersFlags::END_HEADERS), + makeRequestStreamId(stream_index)); + frame.appendStaticHeader( + StaticHeaderIndex::STATUS_200); // send :status as request header, which is invalid + frame.adjustPayloadSize(); + return frame; +} + +Http2Frame Http2Frame::makeRequest(uint32_t stream_index, absl::string_view host, + absl::string_view path) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_STREAM, HeadersFlags::END_HEADERS), + makeRequestStreamId(stream_index)); + frame.appendStaticHeader(StaticHeaderIndex::METHOD_GET); + frame.appendStaticHeader(StaticHeaderIndex::SCHEME_HTTPS); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::PATH, path); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::HOST, host); + frame.adjustPayloadSize(); + return frame; +} + +} // namespace Http2 +} // namespace Http +} // namespace Envoy diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h new file mode 100644 index 000000000000..88a051a2f133 --- /dev/null +++ b/test/common/http/http2/http2_frame.h @@ -0,0 +1,126 @@ +#pragma once + +#include +#include +#include + +#include "common/common/assert.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Http { +namespace Http2 { + +// Rudimentary facility for building and parsing of HTTP2 frames for unit tests +class Http2Frame { + using DataContainer = std::vector; + +public: + Http2Frame() = default; + + using iterator = DataContainer::iterator; + using const_iterator = DataContainer::const_iterator; + + static constexpr size_t HeaderSize = 9; + static const char Preamble[25]; + + enum class Type : uint8_t { + DATA = 0, + HEADERS, + PRIORITY, + RST_STREAM, + SETTINGS, + PUSH_PROMISE, + PING, + GOAWAY, + WINDOW_UPDATE, + CONTINUATION + }; + + enum class SettingsFlags : uint8_t { + NONE = 0, + ACK = 1, + }; + + enum class HeadersFlags : uint8_t { + NONE = 0, + END_STREAM = 1, + END_HEADERS = 4, + }; + + // See https://tools.ietf.org/html/rfc7541#appendix-A for static header indexes + enum class StaticHeaderIndex : uint8_t { + UNKNOWN, + METHOD_GET = 2, + PATH = 4, + STATUS_200 = 8, + STATUS_404 = 13, + SCHEME_HTTPS = 7, + HOST = 38, + }; + + enum class ResponseStatus { UNKNOWN, _200, _404 }; + + // Methods for creating HTTP2 frames + static Http2Frame makePingFrame(absl::string_view data = nullptr); + static Http2Frame makeEmptySettingsFrame(SettingsFlags flags = SettingsFlags::NONE); + static Http2Frame makeMalformedRequest(uint32_t stream_index); + static Http2Frame makeRequest(uint32_t stream_index, absl::string_view host, + absl::string_view path); + + Type type() const { return static_cast(data_[3]); } + ResponseStatus responseStatus() const; + + // Copy HTTP2 header. The `header` parameter must at least be HeaderSize long. + // Allocates payload size based on the value in the header. + void setHeader(absl::string_view header); + + // Copy payloadSize() bytes from the `payload`. The `payload` must be at least payloadSize() long. + void setPayload(absl::string_view payload); + + // Convert to `std::string` for convenience. + explicit operator std::string() const { + if (data_.empty()) { + return {}; + } + return std::string(reinterpret_cast(data()), size()); + } + + uint32_t payloadSize() const; + // Total size of the frame + size_t size() const { return data_.size(); } + // Access to the raw frame bytes + const uint8_t* data() const { return data_.data(); } + iterator begin() { return data_.begin(); } + iterator end() { return data_.end(); } + const_iterator begin() const { return data_.begin(); } + const_iterator end() const { return data_.end(); } + bool empty() const { return data_.empty(); } + +private: + void buildHeader(Type type, uint32_t payload_size = 0, uint8_t flags = 0, uint32_t stream_id = 0); + void setPayloadSize(uint32_t size); + + // This method appends HPACK encoded uint64_t to the payload. adjustPayloadSize() must be called + // after calling this method (possibly multiple times) to write new payload length to the HTTP2 + // header. + void appendHpackInt(uint64_t value, unsigned char prefix_mask); + void appendData(absl::string_view data) { data_.insert(data_.end(), data.begin(), data.end()); } + + // Headers are directly encoded + void appendStaticHeader(StaticHeaderIndex index); + void appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::string_view value); + + // This method updates payload length in the HTTP2 header based on the size of the data_ + void adjustPayloadSize() { + ASSERT(size() >= HeaderSize); + setPayloadSize(size() - HeaderSize); + } + + DataContainer data_; +}; + +} // namespace Http2 +} // namespace Http +} // namespace Envoy diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index daab4c044d18..4c27ead4a842 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -265,6 +265,9 @@ TEST(HttpUtility, parseHttp2Settings) { http2_settings.initial_stream_window_size_); EXPECT_EQ(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE, http2_settings.initial_connection_window_size_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES, http2_settings.max_outbound_frames_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES, + http2_settings.max_outbound_control_frames_); } { diff --git a/test/config/utility.cc b/test/config/utility.cc index 8a90d921b1ec..94fcc4bc7202 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -650,6 +650,21 @@ void ConfigHelper::setLds(absl::string_view version_info) { TestUtility::renameFile(file, lds_filename); } +void ConfigHelper::setOutboundFramesLimits(uint32_t max_all_frames, uint32_t max_control_frames) { + auto filter = getFilterFromListener("envoy.http_connection_manager"); + if (filter) { + envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager hcm_config; + loadHttpConnectionManager(hcm_config); + if (hcm_config.codec_type() == + envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::HTTP2) { + auto* options = hcm_config.mutable_http2_protocol_options(); + options->mutable_max_outbound_frames()->set_value(max_all_frames); + options->mutable_max_outbound_control_frames()->set_value(max_control_frames); + storeHttpConnectionManager(hcm_config); + } + } +} + CdsHelper::CdsHelper() : cds_path_(TestEnvironment::writeStringToFileForTest("cds.pb_text", "")) {} void CdsHelper::setCds(const std::vector& clusters) { diff --git a/test/config/utility.h b/test/config/utility.h index 5be493a51b9f..0abf01c3bf60 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -149,6 +149,9 @@ class ConfigHelper { // and write it to the lds file. void setLds(absl::string_view version_info); + // Set limits on pending outbound frames. + void setOutboundFramesLimits(uint32_t max_all_frames, uint32_t max_control_frames); + // Return the bootstrap configuration for hand-off to Envoy. const envoy::config::bootstrap::v2::Bootstrap& bootstrap() { return bootstrap_; } diff --git a/test/integration/BUILD b/test/integration/BUILD index fff182f022cb..56e0101f5d08 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -207,6 +207,7 @@ envoy_cc_test( "//source/extensions/filters/http/buffer:config", "//source/extensions/filters/http/dynamo:config", "//source/extensions/filters/http/health_check:config", + "//test/common/http/http2:http2_frame", "//test/integration/filters:stop_iteration_and_continue", "//test/mocks/http:http_mocks", "//test/mocks/upstream:upstream_mocks", diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index cddc0a62530c..ea7e686adf18 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1,5 +1,6 @@ #include "test/integration/http2_integration_test.h" +#include #include #include "common/buffer/buffer_impl.h" @@ -1071,4 +1072,161 @@ TEST_P(Http2RingHashIntegrationTest, CookieRoutingWithCookieWithTtlSet) { EXPECT_EQ(served_by.size(), 1); } +namespace { +const int64_t TransmitThreshold = 100 * 1024 * 1024; +} // namespace + +void Http2FloodMitigationTest::beginSession() { + setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + // set lower outbound frame limits to make tests run faster + config_helper_.setOutboundFramesLimits(1000, 100); + initialize(); + tcp_client_ = makeTcpConnection(lookupPort("http")); + startHttp2Session(); +} + +Http2Frame Http2FloodMitigationTest::readFrame() { + Http2Frame frame; + tcp_client_->waitForData(frame.HeaderSize); + frame.setHeader(tcp_client_->data()); + tcp_client_->clearData(frame.HeaderSize); + auto len = frame.payloadSize(); + if (len) { + tcp_client_->waitForData(len); + frame.setPayload(tcp_client_->data()); + tcp_client_->clearData(len); + } + return frame; +} + +void Http2FloodMitigationTest::sendFame(const Http2Frame& frame) { + ASSERT_TRUE(tcp_client_->connected()); + tcp_client_->write(std::string(frame), false, false); +} + +void Http2FloodMitigationTest::startHttp2Session() { + tcp_client_->write(Http2Frame::Preamble, false, false); + + // Send empty initial SETTINGS frame. + auto settings = Http2Frame::makeEmptySettingsFrame(); + tcp_client_->write(std::string(settings), false, false); + + // Read initial SETTINGS frame from the server. + readFrame(); + + // Send an SETTINGS ACK. + settings = Http2Frame::makeEmptySettingsFrame(Http2Frame::SettingsFlags::ACK); + tcp_client_->write(std::string(settings), false, false); + + // read pending SETTINGS and WINDOW_UPDATE frames + readFrame(); + readFrame(); +} + +// Verify that the server detects the flood of the given frame. +void Http2FloodMitigationTest::floodServer(const Http2Frame& frame) { + config_helper_.setBufferLimits(1024, 1024); // Set buffer limits upstream and downstream. + beginSession(); + + // pack the as many frames as we can into 16k buffer + const int FrameCount = (16 * 1024) / frame.size(); + std::vector buf(FrameCount * frame.size()); + for (auto pos = buf.begin(); pos != buf.end();) { + pos = std::copy(frame.begin(), frame.end(), pos); + } + + tcp_client_->readDisable(true); + int64_t total_bytes_sent = 0; + // If the flood protection is not working this loop will keep going + // forever until it is killed by blaze timer or run out of memory. + // Add early stop if we have sent more than 100M of frames, as it this + // point it is obvious something is wrong. + while (total_bytes_sent < TransmitThreshold && tcp_client_->connected()) { + tcp_client_->write({buf.begin(), buf.end()}, false, false); + total_bytes_sent += buf.size(); + } + + EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; + EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +// Verify that the server detects the flood using specified request parameters. +void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_view path, + Http2Frame::ResponseStatus expected_http_status) { + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(request_idx, host, path); + sendFame(request); + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::HEADERS, frame.type()); + EXPECT_EQ(expected_http_status, frame.responseStatus()); + tcp_client_->readDisable(true); + uint64_t total_bytes_sent = 0; + while (total_bytes_sent < TransmitThreshold && tcp_client_->connected()) { + request = Http2Frame::makeRequest(++request_idx, host, path); + sendFame(request); + total_bytes_sent += request.size(); + } + EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; + EXPECT_EQ(1, test_server_->counter("http2.outbound_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +INSTANTIATE_TEST_SUITE_P(IpVersions, Http2FloodMitigationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(Http2FloodMitigationTest, Ping) { floodServer(Http2Frame::makePingFrame()); } + +TEST_P(Http2FloodMitigationTest, Settings) { floodServer(Http2Frame::makeEmptySettingsFrame()); } + +// Verify that the server can detect flood of internally generated 404 responses. +TEST_P(Http2FloodMitigationTest, 404) { + // Change the default route to be restrictive, and send a request to a non existent route. + config_helper_.setDefaultHostAndRoute("foo.com", "/found"); + beginSession(); + + // Send requests to a non existent path to generate 404s + floodServer("host", "/notfound", Http2Frame::ResponseStatus::_404); +} + +// Verify that the server can detect flood of DATA frames +TEST_P(Http2FloodMitigationTest, Data) { + // Set large buffer limits so the test is not affected by the flow control. + config_helper_.setBufferLimits(1024 * 1024 * 1024, 1024 * 1024 * 1024); + autonomous_upstream_ = true; + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + floodServer("host", "/test/long/url", Http2Frame::ResponseStatus::_200); +} + +// Verify that the server can detect flood of RST_STREAM frames. +TEST_P(Http2FloodMitigationTest, RST_STREAM) { + beginSession(); + + int i = 0; + auto request = Http::Http2::Http2Frame::makeMalformedRequest(i); + sendFame(request); + auto response = readFrame(); + // Make sure we've got RST_STREAM from the server + EXPECT_EQ(Http2Frame::Type::RST_STREAM, response.type()); + uint64_t total_bytes_sent = 0; + while (total_bytes_sent < TransmitThreshold && tcp_client_->connected()) { + request = Http::Http2::Http2Frame::makeMalformedRequest(++i); + sendFame(request); + total_bytes_sent += request.size(); + } + EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; + EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + } // namespace Envoy diff --git a/test/integration/http2_integration_test.h b/test/integration/http2_integration_test.h index c910d6e78cbd..b993b0b47e07 100644 --- a/test/integration/http2_integration_test.h +++ b/test/integration/http2_integration_test.h @@ -1,9 +1,12 @@ #pragma once +#include "test/common/http/http2/http2_frame.h" #include "test/integration/http_integration.h" #include "gtest/gtest.h" +using Envoy::Http::Http2::Http2Frame; + namespace Envoy { class Http2IntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { @@ -46,4 +49,21 @@ class Http2MetadataIntegrationTest : public Http2IntegrationTest { setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); } }; + +class Http2FloodMitigationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + Http2FloodMitigationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) {} + +protected: + void startHttp2Session(); + void floodServer(const Http2Frame& frame); + void floodServer(absl::string_view host, absl::string_view path, + Http2Frame::ResponseStatus expected_http_status); + Http2Frame readFrame(); + void sendFame(const Http2Frame& frame); + void beginSession(); + + IntegrationTcpClientPtr tcp_client_; +}; } // namespace Envoy diff --git a/test/integration/integration.cc b/test/integration/integration.cc index a34d52b6a9ca..8131d513297c 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -180,6 +180,15 @@ void IntegrationTcpClient::waitForData(const std::string& data, bool exact_match connection_->dispatcher().run(Event::Dispatcher::RunType::Block); } +void IntegrationTcpClient::waitForData(size_t length) { + if (payload_reader_->data().size() >= length) { + return; + } + + payload_reader_->setLengthToWaitFor(length); + connection_->dispatcher().run(Event::Dispatcher::RunType::Block); +} + void IntegrationTcpClient::waitForDisconnect(bool ignore_spurious_events) { if (ignore_spurious_events) { while (!disconnected_) { diff --git a/test/integration/integration.h b/test/integration/integration.h index 873dde090a2c..b263d78cdd8d 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -95,13 +95,16 @@ class IntegrationTcpClient { void close(); void waitForData(const std::string& data, bool exact_match = true); + // wait for at least `length` bytes to be received + void waitForData(size_t length); void waitForDisconnect(bool ignore_spurious_events = false); void waitForHalfClose(); void readDisable(bool disabled); void write(const std::string& data, bool end_stream = false, bool verify = true); const std::string& data() { return payload_reader_->data(); } bool connected() const { return !disconnected_; } - void clearData() { payload_reader_->clearData(); } + // clear up to the `count` number of bytes of received data + void clearData(size_t count = std::string::npos) { payload_reader_->clearData(count); } private: struct ConnectionCallbacks : public Network::ConnectionCallbacks { diff --git a/test/integration/tcp_proxy_integration_test.cc b/test/integration/tcp_proxy_integration_test.cc index c79d49e9fb0d..060c24afdb40 100644 --- a/test/integration/tcp_proxy_integration_test.cc +++ b/test/integration/tcp_proxy_integration_test.cc @@ -43,6 +43,15 @@ TEST_P(TcpProxyIntegrationTest, TcpProxyUpstreamWritesFirst) { // Make sure inexact matches work also on data already received. tcp_client->waitForData("ello", false); + // Make sure length based wait works for the data already received + tcp_client->waitForData(5); + tcp_client->waitForData(4); + + // Drain part of the received message + tcp_client->clearData(2); + tcp_client->waitForData("llo"); + tcp_client->waitForData(3); + tcp_client->write("hello"); ASSERT_TRUE(fake_upstream_connection->waitForData(5)); diff --git a/test/integration/utility.cc b/test/integration/utility.cc index ecf5fd26ae7d..b19e4df1798b 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -148,6 +148,11 @@ Network::FilterStatus WaitForPayloadReader::onData(Buffer::Instance& data, bool dispatcher_.exit(); } + if (wait_for_length_ && data_.size() >= length_to_wait_for_) { + wait_for_length_ = false; + dispatcher_.exit(); + } + return Network::FilterStatus::StopIteration; } diff --git a/test/integration/utility.h b/test/integration/utility.h index 7d3dc1b2fcdc..6554234ddc95 100644 --- a/test/integration/utility.h +++ b/test/integration/utility.h @@ -181,9 +181,14 @@ class WaitForPayloadReader : public Network::ReadFilterBaseImpl { data_to_wait_for_ = data; exact_match_ = exact_match; } + void setLengthToWaitFor(size_t length) { + ASSERT(!wait_for_length_); + length_to_wait_for_ = length; + wait_for_length_ = true; + } const std::string& data() { return data_; } bool readLastByte() { return read_end_stream_; } - void clearData() { data_.clear(); } + void clearData(size_t count = std::string::npos) { data_.erase(0, count); } private: Event::Dispatcher& dispatcher_; @@ -191,6 +196,8 @@ class WaitForPayloadReader : public Network::ReadFilterBaseImpl { std::string data_; bool exact_match_{true}; bool read_end_stream_{}; + size_t length_to_wait_for_{0}; + bool wait_for_length_{false}; }; } // namespace Envoy diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 02b950cb102d..38ec2670f90f 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -385,6 +385,8 @@ const uint32_t Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS; const uint32_t Http2Settings::DEFAULT_INITIAL_STREAM_WINDOW_SIZE; const uint32_t Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE; const uint32_t Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE; +const uint32_t Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; +const uint32_t Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES; TestHeaderMapImpl::TestHeaderMapImpl() = default; From f26194252c375ce871d6d822a8b41317ef4e46c6 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Tue, 30 Jul 2019 23:58:41 -0400 Subject: [PATCH 352/542] http2: limit the number of inbound frames. (#20) This change adds protections against flooding using PRIORITY and/or WINDOW_UPDATE frames, as well as frames with an empty payload and no end stream flag. Fixes CVE-2019-9511, CVE-2019-9513 and CVE-2019-9518. Signed-off-by: Piotr Sikora --- api/envoy/api/v2/core/protocol.proto | 42 +++++- .../configuration/http_conn_man/stats.rst | 3 + docs/root/intro/version_history.rst | 5 +- include/envoy/http/codec.h | 12 ++ source/common/http/http2/codec_impl.cc | 119 ++++++++++++++- source/common/http/http2/codec_impl.h | 53 ++++++- source/common/http/utility.cc | 9 ++ test/common/http/http2/codec_impl_test.cc | 11 ++ test/common/http/http2/http2_frame.cc | 52 +++++++ test/common/http/http2/http2_frame.h | 15 ++ test/common/http/utility_test.cc | 6 + test/integration/http2_integration_test.cc | 138 ++++++++++++++++-- test/integration/http2_integration_test.h | 4 +- test/test_common/utility.cc | 3 + 14 files changed, 452 insertions(+), 20 deletions(-) diff --git a/api/envoy/api/v2/core/protocol.proto b/api/envoy/api/v2/core/protocol.proto index becd596a0e98..a5a8ba3327d7 100644 --- a/api/envoy/api/v2/core/protocol.proto +++ b/api/envoy/api/v2/core/protocol.proto @@ -49,6 +49,7 @@ message Http1ProtocolOptions { string default_host_for_http_10 = 3; } +// [#comment:next free field: 12] message Http2ProtocolOptions { // `Maximum table size `_ // (in octets) that the encoder is permitted to use for the dynamic HPACK table. Valid values @@ -94,18 +95,53 @@ message Http2ProtocolOptions { // Limit the number of pending outbound downstream frames of all types (frames that are waiting to // be written into the socket). Exceeding this limit triggers flood mitigation and connection is - // terminated. The "http2.outbound_flood" stat tracks the number of terminated connections due to - // flood mitigation. The default limit is 10000. + // terminated. The ``http2.outbound_flood`` stat tracks the number of terminated connections due + // to flood mitigation. The default limit is 10000. // [#comment:TODO: implement same limits for upstream outbound frames as well.] google.protobuf.UInt32Value max_outbound_frames = 7 [(validate.rules).uint32 = {gte: 1}]; // Limit the number of pending outbound downstream frames of types PING, SETTINGS and RST_STREAM, // preventing high memory utilization when receiving continuous stream of these frames. Exceeding // this limit triggers flood mitigation and connection is terminated. The - // "http2.outbound_control_flood" stat tracks the number of terminated connections due to flood + // ``http2.outbound_control_flood`` stat tracks the number of terminated connections due to flood // mitigation. The default limit is 1000. // [#comment:TODO: implement same limits for upstream outbound frames as well.] google.protobuf.UInt32Value max_outbound_control_frames = 8 [(validate.rules).uint32 = {gte: 1}]; + + // Limit the number of consecutive inbound frames of types HEADERS, CONTINUATION and DATA with an + // empty payload and no end stream flag. Those frames have no legitimate use and are abusive, but + // might be a result of a broken HTTP/2 implementation. The `http2.inbound_empty_frames_flood`` + // stat tracks the number of connections terminated due to flood mitigation. + // Setting this to 0 will terminate connection upon receiving first frame with an empty payload + // and no end stream flag. The default limit is 1. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_consecutive_inbound_frames_with_empty_payload = 9; + + // Limit the number of inbound PRIORITY frames allowed per each opened stream. If the number + // of PRIORITY frames received over the lifetime of connection exceeds the value calculated + // using this formula:: + // + // max_inbound_priority_frames_per_stream * (1 + inbound_streams) + // + // the connection is terminated. The ``http2.inbound_priority_frames_flood`` stat tracks + // the number of connections terminated due to flood mitigation. The default limit is 100. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_inbound_priority_frames_per_stream = 10; + + // Limit the number of inbound WINDOW_UPDATE frames allowed per DATA frame sent. If the number + // of WINDOW_UPDATE frames received over the lifetime of connection exceeds the value calculated + // using this formula:: + // + // 1 + 2 * (inbound_streams + + // max_inbound_window_update_frames_per_data_frame_sent * outbound_data_frames) + // + // the connection is terminated. The ``http2.inbound_priority_frames_flood`` stat tracks + // the number of connections terminated due to flood mitigation. The default limit is 10. + // Setting this to 1 should be enough to support HTTP/2 implementations with basic flow control, + // but more complex implementations that try to estimate available bandwidth require at least 2. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_inbound_window_update_frames_per_data_frame_sent = 11 + [(validate.rules).uint32 = {gte: 1}]; } // [#not-implemented-hide:] diff --git a/docs/root/configuration/http_conn_man/stats.rst b/docs/root/configuration/http_conn_man/stats.rst index 3c5a0cdae14c..dc4f79879042 100644 --- a/docs/root/configuration/http_conn_man/stats.rst +++ b/docs/root/configuration/http_conn_man/stats.rst @@ -111,6 +111,9 @@ All http2 statistics are rooted at *http2.* header_overflow, Counter, Total number of connections reset due to the headers being larger than the :ref:`configured value `. headers_cb_no_stream, Counter, Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug + inbound_empty_frames_flood, Counter, Total number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. + inbound_priority_frames_flood, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type PRIORITY. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. + inbound_window_update_frames_flood, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type WINDOW_UPDATE. The limit is configured by setting the :ref:`max_inbound_window_updateframes_per_data_frame_sent config setting `. outbound_flood, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting `. outbound_control_flood, Counter, "Total number of connections terminated for exceeding the limit on outbound frames of types PING, SETTINGS and RST_STREAM. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `." rx_messaging_error, Counter, Total number of invalid received frames that violated `section 8 `_ of the HTTP/2 spec. This will result in a *tx_reset* diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 223773dfe5e5..7272960614c1 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -3,7 +3,10 @@ Version history 1.11.1 (Pending) ================ -* http: added mitigation of client initiated atacks that result in flooding of the outbound queue of downstream HTTP/2 connections. +* http: added mitigation of client initiated atacks that result in flooding of the downstream HTTP/2 connections. +* http: added :ref:`inbound_empty_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. +* http: added :ref:`inbound_priority_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound PRIORITY frames. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. +* http: added :ref:`inbound_window_update_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound WINDOW_UPDATE frames. The limit is configured by setting the :ref:`max_inbound_window_update_frames_per_data_frame_sent config setting `. * http: added :ref:`outbound_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting ` * http: added :ref:`outbound_control_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `. diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index d05943c0f955..6c2b09b2f636 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -237,6 +237,11 @@ struct Http2Settings { bool allow_metadata_{DEFAULT_ALLOW_METADATA}; uint32_t max_outbound_frames_{DEFAULT_MAX_OUTBOUND_FRAMES}; uint32_t max_outbound_control_frames_{DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES}; + uint32_t max_consecutive_inbound_frames_with_empty_payload_{ + DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD}; + uint32_t max_inbound_priority_frames_per_stream_{DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM}; + uint32_t max_inbound_window_update_frames_per_data_frame_sent_{ + DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT}; // disable HPACK compression static const uint32_t MIN_HPACK_TABLE_SIZE = 0; @@ -279,6 +284,13 @@ struct Http2Settings { static const uint32_t DEFAULT_MAX_OUTBOUND_FRAMES = 10000; // Default limit on the number of outbound frames of types PING, SETTINGS and RST_STREAM. static const uint32_t DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES = 1000; + // Default limit on the number of consecutive inbound frames with an empty payload + // and no end stream flag. + static const uint32_t DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD = 1; + // Default limit on the number of inbound frames of type PRIORITY (per stream). + static const uint32_t DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM = 100; + // Default limit on the number of inbound frames of type WINDOW_UPDATE (per DATA frame sent). + static const uint32_t DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT = 10; }; /** diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index fe7518c19cff..d59c16676a75 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -252,6 +252,8 @@ int ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t // https://nghttp2.org/documentation/types.html#c.nghttp2_send_data_callback static const uint64_t FRAME_HEADER_SIZE = 9; + parent_.outbound_data_frames_++; + Buffer::OwnedImpl output; if (!parent_.addOutboundFrameFragment(output, framehd, FRAME_HEADER_SIZE)) { ENVOY_CONN_LOG(debug, "error sending data frame: Too many frames in the outbound queue", @@ -355,7 +357,7 @@ void ConnectionImpl::dispatch(Buffer::Instance& data) { dispatching_ = true; ssize_t rc = nghttp2_session_mem_recv(session_, static_cast(slice.mem_), slice.len_); - if (rc == NGHTTP2_ERR_FLOODED) { + if (rc == NGHTTP2_ERR_FLOODED || flood_detected_) { throw FrameFloodException( "Flooding was detected in this HTTP/2 session, and it must be closed"); } @@ -408,9 +410,36 @@ void ConnectionImpl::shutdownNotice() { sendPendingFrames(); } +int ConnectionImpl::onBeforeFrameReceived(const nghttp2_frame_hd* hd) { + ENVOY_CONN_LOG(trace, "about to recv frame type={}, flags={}", connection_, + static_cast(hd->type), static_cast(hd->flags)); + + // Track all the frames without padding here, since this is the only callback we receive + // for some of them (e.g. CONTINUATION frame, frames sent on closed streams, etc.). + // HEADERS frame is tracked in onBeginHeaders(), DATA frame is tracked in onFrameReceived(). + if (hd->type != NGHTTP2_HEADERS && hd->type != NGHTTP2_DATA) { + if (!trackInboundFrames(hd, 0)) { + return NGHTTP2_ERR_FLOODED; + } + } + + return 0; +} + int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { ENVOY_CONN_LOG(trace, "recv frame type={}", connection_, static_cast(frame->hd.type)); + // onFrameReceived() is called with a complete HEADERS frame assembled from all the HEADERS + // and CONTINUATION frames, but we track them separately: HEADERS frames in onBeginHeaders() + // and CONTINUATION frames in onBeforeFrameReceived(). + ASSERT(frame->hd.type != NGHTTP2_CONTINUATION); + + if (frame->hd.type == NGHTTP2_DATA) { + if (!trackInboundFrames(&frame->hd, frame->data.padlen)) { + return NGHTTP2_ERR_FLOODED; + } + } + // Only raise GOAWAY once, since we don't currently expose stream information. Shutdown // notifications are the same as a normal GOAWAY. if (frame->hd.type == NGHTTP2_GOAWAY && !raised_goaway_) { @@ -567,7 +596,7 @@ int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) { } int ConnectionImpl::onBeforeFrameSend(const nghttp2_frame* frame) { - ENVOY_CONN_LOG(trace, "about to sent frame type={}, flags={}", connection_, + ENVOY_CONN_LOG(trace, "about to send frame type={}, flags={}", connection_, static_cast(frame->hd.type), static_cast(frame->hd.flags)); ASSERT(!is_outbound_flood_monitored_control_frame_); // Flag flood monitored outbound control frames. @@ -882,6 +911,11 @@ ConnectionImpl::Http2Callbacks::Http2Callbacks() { return static_cast(user_data)->onData(stream_id, data, len); }); + nghttp2_session_callbacks_set_on_begin_frame_callback( + callbacks_, [](nghttp2_session*, const nghttp2_frame_hd* hd, void* user_data) -> int { + return static_cast(user_data)->onBeforeFrameReceived(hd); + }); + nghttp2_session_callbacks_set_on_frame_recv_callback( callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int { return static_cast(user_data)->onFrameReceived(frame); @@ -1042,6 +1076,11 @@ ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, int ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) { // For a server connection, we should never get push promise frames. ASSERT(frame->hd.type == NGHTTP2_HEADERS); + + if (!trackInboundFrames(&frame->hd, frame->headers.padlen)) { + return NGHTTP2_ERR_FLOODED; + } + if (frame->headers.cat != NGHTTP2_HCAT_REQUEST) { stats_.trailers_.inc(); ASSERT(frame->headers.cat == NGHTTP2_HCAT_HEADERS); @@ -1072,6 +1111,82 @@ int ServerConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na return saveHeader(frame, std::move(name), std::move(value)); } +bool ServerConnectionImpl::trackInboundFrames(const nghttp2_frame_hd* hd, uint32_t padding_length) { + ENVOY_CONN_LOG(trace, "track inbound frame type={} flags={} length={} padding_length={}", + connection_, static_cast(hd->type), static_cast(hd->flags), + static_cast(hd->length), padding_length); + switch (hd->type) { + case NGHTTP2_HEADERS: + case NGHTTP2_CONTINUATION: + // Track new streams. + if (hd->flags & NGHTTP2_FLAG_END_HEADERS) { + inbound_streams_++; + } + FALLTHRU; + case NGHTTP2_DATA: + // Track frames with an empty payload and no end stream flag. + if (hd->length - padding_length == 0 && !(hd->flags & NGHTTP2_FLAG_END_STREAM)) { + ENVOY_CONN_LOG(trace, "frame with an empty payload and no end stream flag.", connection_); + consecutive_inbound_frames_with_empty_payload_++; + } else { + consecutive_inbound_frames_with_empty_payload_ = 0; + } + break; + case NGHTTP2_PRIORITY: + inbound_priority_frames_++; + break; + case NGHTTP2_WINDOW_UPDATE: + inbound_window_update_frames_++; + break; + default: + break; + } + + if (!checkInboundFrameLimits()) { + // NGHTTP2_ERR_FLOODED is overridden within nghttp2 library and it doesn't propagate + // all the way to nghttp2_session_mem_recv() where we need it. + flood_detected_ = true; + return false; + } + + return true; +} + +bool ServerConnectionImpl::checkInboundFrameLimits() { + ASSERT(dispatching_downstream_data_); + + if (consecutive_inbound_frames_with_empty_payload_ > + max_consecutive_inbound_frames_with_empty_payload_) { + ENVOY_CONN_LOG(trace, + "error reading frame: Too many consecutive frames with an empty payload " + "received in this HTTP/2 session.", + connection_); + stats_.inbound_empty_frames_flood_.inc(); + return false; + } + + if (inbound_priority_frames_ > max_inbound_priority_frames_per_stream_ * (1 + inbound_streams_)) { + ENVOY_CONN_LOG(trace, + "error reading frame: Too many PRIORITY frames received in this HTTP/2 session.", + connection_); + stats_.inbound_priority_frames_flood_.inc(); + return false; + } + + if (inbound_window_update_frames_ > + 1 + 2 * (inbound_streams_ + + max_inbound_window_update_frames_per_data_frame_sent_ * outbound_data_frames_)) { + ENVOY_CONN_LOG( + trace, + "error reading frame: Too many WINDOW_UPDATE frames received in this HTTP/2 session.", + connection_); + stats_.inbound_window_update_frames_flood_.inc(); + return false; + } + + return true; +} + void ServerConnectionImpl::checkOutboundQueueLimits() { if (outbound_frames_ > max_outbound_frames_ && dispatching_downstream_data_) { stats_.outbound_flood_.inc(); diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index a552ff461ee3..abacca058216 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -41,6 +41,9 @@ const std::string CLIENT_MAGIC_PREFIX = "PRI * HTTP/2"; #define ALL_HTTP2_CODEC_STATS(COUNTER) \ COUNTER(header_overflow) \ COUNTER(headers_cb_no_stream) \ + COUNTER(inbound_empty_frames_flood) \ + COUNTER(inbound_priority_frames_flood) \ + COUNTER(inbound_window_update_frames_flood) \ COUNTER(outbound_control_flood) \ COUNTER(outbound_flood) \ COUNTER(rx_messaging_error) \ @@ -79,7 +82,7 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable, diff --git a/test/common/http/http2/http2_frame.cc b/test/common/http/http2/http2_frame.cc index ca4fb858a070..fc96cf77b852 100644 --- a/test/common/http/http2/http2_frame.cc +++ b/test/common/http/http2/http2_frame.cc @@ -120,6 +120,45 @@ Http2Frame Http2Frame::makeEmptySettingsFrame(SettingsFlags flags) { return frame; } +Http2Frame Http2Frame::makeEmptyHeadersFrame(uint32_t stream_index, HeadersFlags flags) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, static_cast(flags), + makeRequestStreamId(stream_index)); + return frame; +} + +Http2Frame Http2Frame::makeEmptyContinuationFrame(uint32_t stream_index, HeadersFlags flags) { + Http2Frame frame; + frame.buildHeader(Type::CONTINUATION, 0, static_cast(flags), + makeRequestStreamId(stream_index)); + return frame; +} + +Http2Frame Http2Frame::makeEmptyDataFrame(uint32_t stream_index, DataFlags flags) { + Http2Frame frame; + frame.buildHeader(Type::DATA, 0, static_cast(flags), makeRequestStreamId(stream_index)); + return frame; +} + +Http2Frame Http2Frame::makePriorityFrame(uint32_t stream_index, uint32_t dependent_index) { + static constexpr size_t kPriorityPayloadSize = 5; + Http2Frame frame; + frame.buildHeader(Type::PRIORITY, kPriorityPayloadSize, 0, makeRequestStreamId(stream_index)); + uint32_t dependent_net = makeRequestStreamId(dependent_index); + memcpy(&frame.data_[HeaderSize], reinterpret_cast(&dependent_net), sizeof(uint32_t)); + return frame; +} + +Http2Frame Http2Frame::makeWindowUpdateFrame(uint32_t stream_index, uint32_t increment) { + static constexpr size_t kWindowUpdatePayloadSize = 4; + Http2Frame frame; + frame.buildHeader(Type::WINDOW_UPDATE, kWindowUpdatePayloadSize, 0, + makeRequestStreamId(stream_index)); + uint32_t increment_net = htonl(increment); + memcpy(&frame.data_[HeaderSize], reinterpret_cast(&increment_net), sizeof(uint32_t)); + return frame; +} + Http2Frame Http2Frame::makeMalformedRequest(uint32_t stream_index) { Http2Frame frame; frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_STREAM, HeadersFlags::END_HEADERS), @@ -143,6 +182,19 @@ Http2Frame Http2Frame::makeRequest(uint32_t stream_index, absl::string_view host return frame; } +Http2Frame Http2Frame::makePostRequest(uint32_t stream_index, absl::string_view host, + absl::string_view path) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_HEADERS), + makeRequestStreamId(stream_index)); + frame.appendStaticHeader(StaticHeaderIndex::METHOD_POST); + frame.appendStaticHeader(StaticHeaderIndex::SCHEME_HTTPS); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::PATH, path); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::HOST, host); + frame.adjustPayloadSize(); + return frame; +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h index 88a051a2f133..760ae69e3575 100644 --- a/test/common/http/http2/http2_frame.h +++ b/test/common/http/http2/http2_frame.h @@ -49,10 +49,16 @@ class Http2Frame { END_HEADERS = 4, }; + enum class DataFlags : uint8_t { + NONE = 0, + END_STREAM = 1, + }; + // See https://tools.ietf.org/html/rfc7541#appendix-A for static header indexes enum class StaticHeaderIndex : uint8_t { UNKNOWN, METHOD_GET = 2, + METHOD_POST = 3, PATH = 4, STATUS_200 = 8, STATUS_404 = 13, @@ -65,9 +71,18 @@ class Http2Frame { // Methods for creating HTTP2 frames static Http2Frame makePingFrame(absl::string_view data = nullptr); static Http2Frame makeEmptySettingsFrame(SettingsFlags flags = SettingsFlags::NONE); + static Http2Frame makeEmptyHeadersFrame(uint32_t stream_index, + HeadersFlags flags = HeadersFlags::NONE); + static Http2Frame makeEmptyContinuationFrame(uint32_t stream_index, + HeadersFlags flags = HeadersFlags::NONE); + static Http2Frame makeEmptyDataFrame(uint32_t stream_index, DataFlags flags = DataFlags::NONE); + static Http2Frame makePriorityFrame(uint32_t stream_index, uint32_t dependent_index); + static Http2Frame makeWindowUpdateFrame(uint32_t stream_index, uint32_t increment); static Http2Frame makeMalformedRequest(uint32_t stream_index); static Http2Frame makeRequest(uint32_t stream_index, absl::string_view host, absl::string_view path); + static Http2Frame makePostRequest(uint32_t stream_index, absl::string_view host, + absl::string_view path); Type type() const { return static_cast(data_[3]); } ResponseStatus responseStatus() const; diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index 4c27ead4a842..388f603623c8 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -268,6 +268,12 @@ TEST(HttpUtility, parseHttp2Settings) { EXPECT_EQ(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES, http2_settings.max_outbound_frames_); EXPECT_EQ(Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES, http2_settings.max_outbound_control_frames_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD, + http2_settings.max_consecutive_inbound_frames_with_empty_payload_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM, + http2_settings.max_inbound_priority_frames_per_stream_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT, + http2_settings.max_inbound_window_update_frames_per_data_frame_sent_); } { diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index ea7e686adf18..f5b15df32ee8 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1125,10 +1125,7 @@ void Http2FloodMitigationTest::startHttp2Session() { } // Verify that the server detects the flood of the given frame. -void Http2FloodMitigationTest::floodServer(const Http2Frame& frame) { - config_helper_.setBufferLimits(1024, 1024); // Set buffer limits upstream and downstream. - beginSession(); - +void Http2FloodMitigationTest::floodServer(const Http2Frame& frame, const std::string& flood_stat) { // pack the as many frames as we can into 16k buffer const int FrameCount = (16 * 1024) / frame.size(); std::vector buf(FrameCount * frame.size()); @@ -1148,7 +1145,7 @@ void Http2FloodMitigationTest::floodServer(const Http2Frame& frame) { } EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; - EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value()); + EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); // Verify that connection was closed abortively EXPECT_EQ(0, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); @@ -1156,7 +1153,8 @@ void Http2FloodMitigationTest::floodServer(const Http2Frame& frame) { // Verify that the server detects the flood using specified request parameters. void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_view path, - Http2Frame::ResponseStatus expected_http_status) { + Http2Frame::ResponseStatus expected_http_status, + const std::string& flood_stat) { uint32_t request_idx = 0; auto request = Http2Frame::makeRequest(request_idx, host, path); sendFame(request); @@ -1171,7 +1169,7 @@ void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_ total_bytes_sent += request.size(); } EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; - EXPECT_EQ(1, test_server_->counter("http2.outbound_flood")->value()); + EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); // Verify that connection was closed abortively EXPECT_EQ(0, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); @@ -1181,9 +1179,15 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, Http2FloodMitigationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); -TEST_P(Http2FloodMitigationTest, Ping) { floodServer(Http2Frame::makePingFrame()); } +TEST_P(Http2FloodMitigationTest, Ping) { + beginSession(); + floodServer(Http2Frame::makePingFrame(), "http2.outbound_control_flood"); +} -TEST_P(Http2FloodMitigationTest, Settings) { floodServer(Http2Frame::makeEmptySettingsFrame()); } +TEST_P(Http2FloodMitigationTest, Settings) { + beginSession(); + floodServer(Http2Frame::makeEmptySettingsFrame(), "http2.outbound_control_flood"); +} // Verify that the server can detect flood of internally generated 404 responses. TEST_P(Http2FloodMitigationTest, 404) { @@ -1192,7 +1196,7 @@ TEST_P(Http2FloodMitigationTest, 404) { beginSession(); // Send requests to a non existent path to generate 404s - floodServer("host", "/notfound", Http2Frame::ResponseStatus::_404); + floodServer("host", "/notfound", Http2Frame::ResponseStatus::_404, "http2.outbound_flood"); } // Verify that the server can detect flood of DATA frames @@ -1203,7 +1207,7 @@ TEST_P(Http2FloodMitigationTest, Data) { beginSession(); fake_upstreams_[0]->set_allow_unexpected_disconnects(true); - floodServer("host", "/test/long/url", Http2Frame::ResponseStatus::_200); + floodServer("host", "/test/long/url", Http2Frame::ResponseStatus::_200, "http2.outbound_flood"); } // Verify that the server can detect flood of RST_STREAM frames. @@ -1229,4 +1233,116 @@ TEST_P(Http2FloodMitigationTest, RST_STREAM) { test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } +TEST_P(Http2FloodMitigationTest, EmptyHeaders) { + config_helper_.addConfigModifier( + [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options() + ->mutable_max_consecutive_inbound_frames_with_empty_payload() + ->set_value(0); + }); + beginSession(); + + uint32_t request_idx = 0; + auto request = Http2Frame::makeEmptyHeadersFrame(request_idx); + sendFame(request); + + tcp_client_->waitForDisconnect(); + + EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +TEST_P(Http2FloodMitigationTest, EmptyHeadersContinuation) { + beginSession(); + + uint32_t request_idx = 0; + auto request = Http2Frame::makeEmptyHeadersFrame(request_idx); + sendFame(request); + + for (int i = 0; i < 2; i++) { + request = Http2Frame::makeEmptyContinuationFrame(request_idx); + sendFame(request); + } + + tcp_client_->waitForDisconnect(); + + EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +TEST_P(Http2FloodMitigationTest, EmptyData) { + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + uint32_t request_idx = 0; + auto request = Http2Frame::makePostRequest(request_idx, "host", "/"); + sendFame(request); + + for (int i = 0; i < 2; i++) { + request = Http2Frame::makeEmptyDataFrame(request_idx); + sendFame(request); + } + + tcp_client_->waitForDisconnect(); + + EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +TEST_P(Http2FloodMitigationTest, PriorityIdleStream) { + beginSession(); + + floodServer(Http2Frame::makePriorityFrame(0, 1), "http2.inbound_priority_frames_flood"); +} + +TEST_P(Http2FloodMitigationTest, PriorityOpenStream) { + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Open stream. + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(request_idx, "host", "/"); + sendFame(request); + + floodServer(Http2Frame::makePriorityFrame(request_idx, request_idx + 1), + "http2.inbound_priority_frames_flood"); +} + +TEST_P(Http2FloodMitigationTest, PriorityClosedStream) { + autonomous_upstream_ = true; + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Open stream. + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(request_idx, "host", "/"); + sendFame(request); + // Reading response marks this stream as closed in nghttp2. + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::HEADERS, frame.type()); + + floodServer(Http2Frame::makePriorityFrame(request_idx, request_idx + 1), + "http2.inbound_priority_frames_flood"); +} + +TEST_P(Http2FloodMitigationTest, WindowUpdate) { + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Open stream. + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(request_idx, "host", "/"); + sendFame(request); + + floodServer(Http2Frame::makeWindowUpdateFrame(request_idx, 1), + "http2.inbound_window_update_frames_flood"); +} + } // namespace Envoy diff --git a/test/integration/http2_integration_test.h b/test/integration/http2_integration_test.h index b993b0b47e07..6a8df8be372a 100644 --- a/test/integration/http2_integration_test.h +++ b/test/integration/http2_integration_test.h @@ -57,9 +57,9 @@ class Http2FloodMitigationTest : public testing::TestWithParam Date: Wed, 31 Jul 2019 11:44:27 -0400 Subject: [PATCH 353/542] http2: enable strict validation of HTTP/2 headers. (#19) Fixes CVE-2019-9516. Signed-off-by: Piotr Sikora --- api/envoy/api/v2/core/protocol.proto | 9 ++- docs/root/intro/version_history.rst | 1 + include/envoy/http/codec.h | 3 + source/common/http/http2/codec_impl.cc | 17 +++-- source/common/http/http2/codec_impl.h | 3 + source/common/http/utility.cc | 1 + test/common/http/http2/codec_impl_test.cc | 65 ++++++++++++++++++- test/common/http/http2/http2_frame.cc | 21 ++++++ test/common/http/http2/http2_frame.h | 4 ++ test/integration/http2_integration_test.cc | 56 ++++++++++++++++ test/integration/protocol_integration_test.cc | 58 +++++++++++++++++ 11 files changed, 229 insertions(+), 9 deletions(-) diff --git a/api/envoy/api/v2/core/protocol.proto b/api/envoy/api/v2/core/protocol.proto index a5a8ba3327d7..88a82077a428 100644 --- a/api/envoy/api/v2/core/protocol.proto +++ b/api/envoy/api/v2/core/protocol.proto @@ -49,7 +49,7 @@ message Http1ProtocolOptions { string default_host_for_http_10 = 3; } -// [#comment:next free field: 12] +// [#comment:next free field: 13] message Http2ProtocolOptions { // `Maximum table size `_ // (in octets) that the encoder is permitted to use for the dynamic HPACK table. Valid values @@ -142,6 +142,13 @@ message Http2ProtocolOptions { // [#comment:TODO: implement same limits for upstream inbound frames as well.] google.protobuf.UInt32Value max_inbound_window_update_frames_per_data_frame_sent = 11 [(validate.rules).uint32 = {gte: 1}]; + + // Allows invalid HTTP messaging and headers. When this option is disabled (default), then + // the whole HTTP/2 connection is terminated upon receiving invalid HEADERS frame. However, + // when this option is enabled, only the offending stream is terminated. + // + // See [RFC7540, sec. 8.1](https://tools.ietf.org/html/rfc7540#section-8.1) for details. + bool stream_error_on_invalid_http_messaging = 12; } // [#not-implemented-hide:] diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 7272960614c1..18fb19c5f68c 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -9,6 +9,7 @@ Version history * http: added :ref:`inbound_window_update_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound WINDOW_UPDATE frames. The limit is configured by setting the :ref:`max_inbound_window_update_frames_per_data_frame_sent config setting `. * http: added :ref:`outbound_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting ` * http: added :ref:`outbound_control_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `. +* http: enabled strict validation of HTTP/2 messaging. Previous behavior can be restored using :ref:`stream_error_on_invalid_http_messaging config setting `. 1.11.0 (July 11, 2019) ====================== diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 6c2b09b2f636..eb5951d3b9b6 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -235,6 +235,7 @@ struct Http2Settings { uint32_t initial_connection_window_size_{DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE}; bool allow_connect_{DEFAULT_ALLOW_CONNECT}; bool allow_metadata_{DEFAULT_ALLOW_METADATA}; + bool stream_error_on_invalid_http_messaging_{DEFAULT_STREAM_ERROR_ON_INVALID_HTTP_MESSAGING}; uint32_t max_outbound_frames_{DEFAULT_MAX_OUTBOUND_FRAMES}; uint32_t max_outbound_control_frames_{DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES}; uint32_t max_consecutive_inbound_frames_with_empty_payload_{ @@ -279,6 +280,8 @@ struct Http2Settings { static const bool DEFAULT_ALLOW_CONNECT = false; // By default Envoy does not allow METADATA support. static const bool DEFAULT_ALLOW_METADATA = false; + // By default Envoy does not allow invalid headers. + static const bool DEFAULT_STREAM_ERROR_ON_INVALID_HTTP_MESSAGING = false; // Default limit on the number of outbound frames of all types. static const uint32_t DEFAULT_MAX_OUTBOUND_FRAMES = 10000; diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index d59c16676a75..e45f07a101af 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -579,16 +579,19 @@ int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) { ENVOY_CONN_LOG(debug, "invalid frame: {} on stream {}", connection_, nghttp2_strerror(error_code), stream_id); - // The stream is about to be closed due to an invalid header or messaging. Don't kill the - // entire connection if one stream has bad headers or messaging. if (error_code == NGHTTP2_ERR_HTTP_HEADER || error_code == NGHTTP2_ERR_HTTP_MESSAGING) { stats_.rx_messaging_error_.inc(); - StreamImpl* stream = getStream(stream_id); - if (stream != nullptr) { - // See comment below in onStreamClose() for why we do this. - stream->reset_due_to_messaging_error_ = true; + + if (stream_error_on_invalid_http_messaging_) { + // The stream is about to be closed due to an invalid header or messaging. Don't kill the + // entire connection if one stream has bad headers or messaging. + StreamImpl* stream = getStream(stream_id); + if (stream != nullptr) { + // See comment below in onStreamClose() for why we do this. + stream->reset_due_to_messaging_error_ = true; + } + return 0; } - return 0; } // Cause dispatch to return with an error code. diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index abacca058216..70bc2da1e378 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -82,6 +82,8 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable(tp); setting.initial_connection_window_size_ = ::testing::get<3>(tp); setting.allow_metadata_ = allow_metadata_; + setting.stream_error_on_invalid_http_messaging_ = stream_error_on_invalid_http_messaging_; setting.max_outbound_frames_ = max_outbound_frames_; setting.max_outbound_control_frames_ = max_outbound_control_frames_; setting.max_consecutive_inbound_frames_with_empty_payload_ = @@ -128,6 +129,7 @@ class Http2CodecImplTestFixture { const Http2SettingsTuple client_settings_; const Http2SettingsTuple server_settings_; bool allow_metadata_ = false; + bool stream_error_on_invalid_http_messaging_ = false; Stats::IsolatedStoreImpl stats_store_; Http2Settings client_http2settings_; NiceMock client_connection_; @@ -202,6 +204,20 @@ TEST_P(Http2CodecImplTest, ContinueHeaders) { TEST_P(Http2CodecImplTest, InvalidContinueWithFin) { initialize(); + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + request_encoder_->encodeHeaders(request_headers, true); + + TestHeaderMapImpl continue_headers{{":status", "100"}}; + EXPECT_THROW(response_encoder_->encodeHeaders(continue_headers, true), CodecProtocolException); + EXPECT_EQ(1, stats_store_.counter("http2.rx_messaging_error").value()); +} + +TEST_P(Http2CodecImplTest, InvalidContinueWithFinAllowed) { + stream_error_on_invalid_http_messaging_ = true; + initialize(); + MockStreamCallbacks request_callbacks; request_encoder_->getStream().addCallbacks(request_callbacks); @@ -229,6 +245,23 @@ TEST_P(Http2CodecImplTest, InvalidContinueWithFin) { TEST_P(Http2CodecImplTest, InvalidRepeatContinue) { initialize(); + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + request_encoder_->encodeHeaders(request_headers, true); + + TestHeaderMapImpl continue_headers{{":status", "100"}}; + EXPECT_CALL(response_decoder_, decode100ContinueHeaders_(_)); + response_encoder_->encode100ContinueHeaders(continue_headers); + + EXPECT_THROW(response_encoder_->encodeHeaders(continue_headers, true), CodecProtocolException); + EXPECT_EQ(1, stats_store_.counter("http2.rx_messaging_error").value()); +}; + +TEST_P(Http2CodecImplTest, InvalidRepeatContinueAllowed) { + stream_error_on_invalid_http_messaging_ = true; + initialize(); + MockStreamCallbacks request_callbacks; request_encoder_->getStream().addCallbacks(request_callbacks); @@ -280,6 +313,28 @@ TEST_P(Http2CodecImplTest, Invalid103) { TEST_P(Http2CodecImplTest, Invalid204WithContentLength) { initialize(); + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + request_encoder_->encodeHeaders(request_headers, true); + + TestHeaderMapImpl response_headers{{":status", "204"}, {"content-length", "3"}}; + // What follows is a hack to get headers that should span into continuation frames. The default + // maximum frame size is 16K. We will add 3,000 headers that will take us above this size and + // not easily compress with HPACK. (I confirmed this generates 26,468 bytes of header data + // which should contain a continuation.) + for (uint i = 1; i < 3000; i++) { + response_headers.addCopy(std::to_string(i), std::to_string(i)); + } + + EXPECT_THROW(response_encoder_->encodeHeaders(response_headers, false), CodecProtocolException); + EXPECT_EQ(1, stats_store_.counter("http2.rx_messaging_error").value()); +}; + +TEST_P(Http2CodecImplTest, Invalid204WithContentLengthAllowed) { + stream_error_on_invalid_http_messaging_ = true; + initialize(); + MockStreamCallbacks request_callbacks; request_encoder_->getStream().addCallbacks(request_callbacks); @@ -329,7 +384,15 @@ TEST_P(Http2CodecImplTest, RefusedStreamReset) { response_encoder_->getStream().resetStream(StreamResetReason::LocalRefusedStreamReset); } -TEST_P(Http2CodecImplTest, InvalidFrame) { +TEST_P(Http2CodecImplTest, InvalidHeadersFrame) { + initialize(); + + EXPECT_THROW(request_encoder_->encodeHeaders(TestHeaderMapImpl{}, true), CodecProtocolException); + EXPECT_EQ(1, stats_store_.counter("http2.rx_messaging_error").value()); +} + +TEST_P(Http2CodecImplTest, InvalidHeadersFrameAllowed) { + stream_error_on_invalid_http_messaging_ = true; initialize(); MockStreamCallbacks request_callbacks; diff --git a/test/common/http/http2/http2_frame.cc b/test/common/http/http2/http2_frame.cc index fc96cf77b852..368630e1f6ec 100644 --- a/test/common/http/http2/http2_frame.cc +++ b/test/common/http/http2/http2_frame.cc @@ -104,6 +104,12 @@ void Http2Frame::appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::stri appendData(value); } +void Http2Frame::appendEmptyHeader() { + data_.push_back(0x40); + data_.push_back(0x00); + data_.push_back(0x00); +} + Http2Frame Http2Frame::makePingFrame(absl::string_view data) { static constexpr size_t kPingPayloadSize = 8; Http2Frame frame; @@ -169,6 +175,21 @@ Http2Frame Http2Frame::makeMalformedRequest(uint32_t stream_index) { return frame; } +Http2Frame Http2Frame::makeMalformedRequestWithZerolenHeader(uint32_t stream_index, + absl::string_view host, + absl::string_view path) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_STREAM, HeadersFlags::END_HEADERS), + makeRequestStreamId(stream_index)); + frame.appendStaticHeader(StaticHeaderIndex::METHOD_GET); + frame.appendStaticHeader(StaticHeaderIndex::SCHEME_HTTPS); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::PATH, path); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::HOST, host); + frame.appendEmptyHeader(); + frame.adjustPayloadSize(); + return frame; +} + Http2Frame Http2Frame::makeRequest(uint32_t stream_index, absl::string_view host, absl::string_view path) { Http2Frame frame; diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h index 760ae69e3575..52b838dbb987 100644 --- a/test/common/http/http2/http2_frame.h +++ b/test/common/http/http2/http2_frame.h @@ -79,6 +79,9 @@ class Http2Frame { static Http2Frame makePriorityFrame(uint32_t stream_index, uint32_t dependent_index); static Http2Frame makeWindowUpdateFrame(uint32_t stream_index, uint32_t increment); static Http2Frame makeMalformedRequest(uint32_t stream_index); + static Http2Frame makeMalformedRequestWithZerolenHeader(uint32_t stream_index, + absl::string_view host, + absl::string_view path); static Http2Frame makeRequest(uint32_t stream_index, absl::string_view host, absl::string_view path); static Http2Frame makePostRequest(uint32_t stream_index, absl::string_view host, @@ -126,6 +129,7 @@ class Http2Frame { // Headers are directly encoded void appendStaticHeader(StaticHeaderIndex index); void appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::string_view value); + void appendEmptyHeader(); // This method updates payload length in the HTTP2 header based on the size of the data_ void adjustPayloadSize() { diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index f5b15df32ee8..890319f40e0e 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1212,6 +1212,12 @@ TEST_P(Http2FloodMitigationTest, Data) { // Verify that the server can detect flood of RST_STREAM frames. TEST_P(Http2FloodMitigationTest, RST_STREAM) { + // Use invalid HTTP headers to trigger sending RST_STREAM frames. + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true); + }); beginSession(); int i = 0; @@ -1345,4 +1351,54 @@ TEST_P(Http2FloodMitigationTest, WindowUpdate) { "http2.inbound_window_update_frames_flood"); } +// Verify that the HTTP/2 connection is terminated upon receiving invalid HEADERS frame. +TEST_P(Http2FloodMitigationTest, ZerolenHeader) { + beginSession(); + + // Send invalid request. + uint32_t request_idx = 0; + auto request = Http2Frame::makeMalformedRequestWithZerolenHeader(request_idx, "host", "/"); + sendFame(request); + + tcp_client_->waitForDisconnect(); + + EXPECT_EQ(1, test_server_->counter("http2.rx_messaging_error")->value()); + EXPECT_EQ(1, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +// Verify that only the offending stream is terminated upon receiving invalid HEADERS frame. +TEST_P(Http2FloodMitigationTest, ZerolenHeaderAllowed) { + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true); + }); + autonomous_upstream_ = true; + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Send invalid request. + uint32_t request_idx = 0; + auto request = Http2Frame::makeMalformedRequestWithZerolenHeader(request_idx, "host", "/"); + sendFame(request); + // Make sure we've got RST_STREAM from the server. + auto response = readFrame(); + EXPECT_EQ(Http2Frame::Type::RST_STREAM, response.type()); + + // Send valid request using the same connection. + request_idx++; + request = Http2Frame::makeRequest(request_idx, "host", "/"); + sendFame(request); + response = readFrame(); + EXPECT_EQ(Http2Frame::Type::HEADERS, response.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::_200, response.responseStatus()); + + tcp_client_->close(); + + EXPECT_EQ(1, test_server_->counter("http2.rx_messaging_error")->value()); + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + } // namespace Envoy diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 1e7c6bcb0b72..20def1351f43 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -592,6 +592,36 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLength) { {"content-length", "-1"}}); auto response = std::move(encoder_decoder.second); + codec_client_->waitForDisconnect(); + + if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + ASSERT_TRUE(response->complete()); + EXPECT_EQ("400", response->headers().Status()->value().getStringView()); + } else { + ASSERT_TRUE(response->reset()); + EXPECT_EQ(Http::StreamResetReason::ConnectionTermination, response->reset_reason()); + } +} + +// TODO(PiotrSikora): move this HTTP/2 only variant to http2_integration_test.cc. +TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLengthAllowed) { + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true); + }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto encoder_decoder = + codec_client_->startRequest(Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":authority", "host"}, + {"content-length", "-1"}}); + auto response = std::move(encoder_decoder.second); + if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { codec_client_->waitForDisconnect(); } else { @@ -618,6 +648,34 @@ TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengths) { {"content-length", "3,2"}}); auto response = std::move(encoder_decoder.second); + codec_client_->waitForDisconnect(); + + if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + ASSERT_TRUE(response->complete()); + EXPECT_EQ("400", response->headers().Status()->value().getStringView()); + } else { + ASSERT_TRUE(response->reset()); + EXPECT_EQ(Http::StreamResetReason::ConnectionTermination, response->reset_reason()); + } +} + +// TODO(PiotrSikora): move this HTTP/2 only variant to http2_integration_test.cc. +TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengthsAllowed) { + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true); + }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto encoder_decoder = + codec_client_->startRequest(Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":authority", "host"}, + {"content-length", "3,2"}}); + auto response = std::move(encoder_decoder.second); + if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { codec_client_->waitForDisconnect(); } else { From 9cd0ce4472a7c5ee22f004ca789d767a60024353 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 31 Jul 2019 16:14:34 -0400 Subject: [PATCH 354/542] Always disable reads when connection is closed with the FlushWriteAndDelay (#16) Signed-off-by: Yan Avlasov --- source/common/http/conn_manager_impl.cc | 20 +++++------ source/common/http/conn_manager_impl.h | 2 +- source/common/network/connection_impl.cc | 2 ++ test/common/http/conn_manager_impl_test.cc | 3 +- test/common/network/connection_impl_test.cc | 5 +++ test/integration/http2_integration_test.cc | 38 ++++++++++++++------- 6 files changed, 43 insertions(+), 27 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index bda3cde7403e..d3e0f93cf76c 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -257,17 +257,16 @@ StreamDecoder& ConnectionManagerImpl::newStream(StreamEncoder& response_encoder, return **streams_.begin(); } -void ConnectionManagerImpl::handleCodecException(const char* error, - Network::ConnectionCloseType close_type) { +void ConnectionManagerImpl::handleCodecException(const char* error) { ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), error); - // In the protocol error case, we need to reset all streams now. If the close_type is - // FlushWriteAndDelay, the connection might stick around long enough for a pending stream to come - // back and try to encode. In other cases it avoids needless processing of upstream responses when - // downstream connection is closed. + // In the protocol error case, we need to reset all streams now. The connection might stick around + // long enough for a pending stream to come back and try to encode. resetAllStreams(); - read_callbacks_->connection().close(close_type); + // HTTP/1.1 codec has already sent a 400 response if possible. HTTP/2 codec has already sent + // GOAWAY. + read_callbacks_->connection().close(Network::ConnectionCloseType::FlushWriteAndDelay); } Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) { @@ -289,14 +288,11 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool try { codec_->dispatch(data); } catch (const FrameFloodException& e) { - // Abortively close flooded connections - handleCodecException(e.what(), Network::ConnectionCloseType::NoFlush); + handleCodecException(e.what()); return Network::FilterStatus::StopIteration; } catch (const CodecProtocolException& e) { stats_.named_.downstream_cx_protocol_error_.inc(); - // HTTP/1.1 codec has already sent a 400 response if possible. HTTP/2 codec has already sent - // GOAWAY. - handleCodecException(e.what(), Network::ConnectionCloseType::FlushWriteAndDelay); + handleCodecException(e.what()); return Network::FilterStatus::StopIteration; } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index cafef6631093..ba802c09d01e 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -582,7 +582,7 @@ class ConnectionManagerImpl : Logger::Loggable, void onDrainTimeout(); void startDrainSequence(); Tracing::HttpTracer& tracer() { return http_context_.tracer(); } - void handleCodecException(const char* error, Network::ConnectionCloseType close_type); + void handleCodecException(const char* error); enum class DrainState { NotDraining, Draining, Closing }; diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index 2309112e71a4..a5f1b4e7b8fe 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -118,6 +118,8 @@ void ConnectionImpl::close(ConnectionCloseType type) { if (!inDelayedClose()) { initializeDelayedCloseTimer(); delayed_close_state_ = DelayedCloseState::CloseAfterFlushAndWait; + // Monitor for the peer closing the connection. + file_event_->setEnabled(enable_half_close_ ? 0 : Event::FileReadyType::Closed); } } else { closeSocket(ConnectionEvent::LocalClose); diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 6908e459e29a..c09bfceeab98 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2328,7 +2328,8 @@ TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); // FrameFloodException should result in reset of the streams followed by abortive close. - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); + EXPECT_CALL(filter_callbacks_.connection_, + close(Network::ConnectionCloseType::FlushWriteAndDelay)); // Kick off the incoming data. Buffer::OwnedImpl fake_input("1234"); diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 8ddfb9f31535..d93604d12c9e 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -1734,6 +1734,7 @@ TEST_F(PostCloseConnectionImplTest, ReadAfterCloseFlushWriteDelayIgnored) { // Delayed connection close. EXPECT_CALL(dispatcher_, createTimer_(_)); + EXPECT_CALL(*file_event_, setEnabled(Event::FileReadyType::Closed)); connection_->close(ConnectionCloseType::FlushWriteAndDelay); // Read event, doRead() happens on connection but no filter onData(). @@ -1758,6 +1759,10 @@ TEST_F(PostCloseConnectionImplTest, ReadAfterCloseFlushWriteDelayIgnoredWithWrit // Delayed connection close. EXPECT_CALL(dispatcher_, createTimer_(_)); + // With half-close semantics enabled we will not wait for early close notification. + // See the `Envoy::Network::ConnectionImpl::readDisable()' method for more details. + EXPECT_CALL(*file_event_, setEnabled(0)); + connection_->enableHalfClose(true); connection_->close(ConnectionCloseType::FlushWriteAndDelay); // Read event, doRead() happens on connection but no filter onData(). diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index 890319f40e0e..eb2c080503df 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1146,8 +1146,7 @@ void Http2FloodMitigationTest::floodServer(const Http2Frame& frame, const std::s EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } @@ -1169,9 +1168,10 @@ void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_ total_bytes_sent += request.size(); } EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; - EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + if (!flood_stat.empty()) { + EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); + } + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } @@ -1234,11 +1234,26 @@ TEST_P(Http2FloodMitigationTest, RST_STREAM) { } EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } +// Verify that the server stop reading downstream connection on protocol error. +TEST_P(Http2FloodMitigationTest, TooManyStreams) { + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->mutable_max_concurrent_streams()->set_value(2); + }); + autonomous_upstream_ = true; + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Exceed the number of streams allowed by the server. The server should stop reading from the + // client. Verify that the client was unable to stuff a lot of data into the server. + floodServer("host", "/test/long/url", Http2Frame::ResponseStatus::_200, ""); +} + TEST_P(Http2FloodMitigationTest, EmptyHeaders) { config_helper_.addConfigModifier( [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) @@ -1256,8 +1271,7 @@ TEST_P(Http2FloodMitigationTest, EmptyHeaders) { tcp_client_->waitForDisconnect(); EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } @@ -1276,8 +1290,7 @@ TEST_P(Http2FloodMitigationTest, EmptyHeadersContinuation) { tcp_client_->waitForDisconnect(); EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } @@ -1297,8 +1310,7 @@ TEST_P(Http2FloodMitigationTest, EmptyData) { tcp_client_->waitForDisconnect(); EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } From 9c3bab4c10b8855d99d69dee20c3c6e297cbd73d Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Tue, 30 Jul 2019 14:48:16 +0000 Subject: [PATCH 355/542] release: bump to 1.11.1. Signed-off-by: Piotr Sikora --- VERSION | 2 +- docs/root/intro/version_history.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 1cac385c6cb8..720c7384c619 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.0 +1.11.1 diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 18fb19c5f68c..b14b9be35c68 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -1,8 +1,8 @@ Version history --------------- -1.11.1 (Pending) -================ +1.11.1 (August 13, 2019) +======================== * http: added mitigation of client initiated atacks that result in flooding of the downstream HTTP/2 connections. * http: added :ref:`inbound_empty_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. * http: added :ref:`inbound_priority_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound PRIORITY frames. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. From 9af488581158a4eb1a27dcfbfa9234c73540834b Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Fri, 2 Aug 2019 13:55:07 -0400 Subject: [PATCH 356/542] Fix flaky http2 integration tests (#29) Signed-off-by: Yan Avlasov --- test/integration/http2_integration_test.cc | 17 +++++++++++++++++ test/integration/http2_integration_test.h | 1 + 2 files changed, 18 insertions(+) diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index eb2c080503df..7923a5880ea8 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1076,6 +1076,21 @@ namespace { const int64_t TransmitThreshold = 100 * 1024 * 1024; } // namespace +void Http2FloodMitigationTest::setNetworkConnectionBufferSize() { + // nghttp2 library has its own internal mitigation for outbound control frames. The mitigation is + // trigerred when there are more than 10000 PING or SETTINGS frames with ACK flag in the nghttp2 + // internal outbound queue. It is possible to trigger this mitigation in nghttp2 before triggering + // Envoy's own flood mitigation. This can happen when a buffer larger enough to contain over 10K + // PING or SETTINGS frames is dispatched to the nghttp2 library. To prevent this from happening + // the network connection receive buffer needs to be smaller than 90Kb (which is 10K SETTINGS + // frames). Set it to the arbitrarily chosen value of 32K. + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) -> void { + RELEASE_ASSERT(bootstrap.mutable_static_resources()->listeners_size() >= 1, ""); + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + listener->mutable_per_connection_buffer_limit_bytes()->set_value(32 * 1024); + }); +} + void Http2FloodMitigationTest::beginSession() { setDownstreamProtocol(Http::CodecClient::Type::HTTP2); setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); @@ -1180,11 +1195,13 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, Http2FloodMitigationTest, TestUtility::ipTestParamsToString); TEST_P(Http2FloodMitigationTest, Ping) { + setNetworkConnectionBufferSize(); beginSession(); floodServer(Http2Frame::makePingFrame(), "http2.outbound_control_flood"); } TEST_P(Http2FloodMitigationTest, Settings) { + setNetworkConnectionBufferSize(); beginSession(); floodServer(Http2Frame::makeEmptySettingsFrame(), "http2.outbound_control_flood"); } diff --git a/test/integration/http2_integration_test.h b/test/integration/http2_integration_test.h index 6a8df8be372a..6282ee13c4c8 100644 --- a/test/integration/http2_integration_test.h +++ b/test/integration/http2_integration_test.h @@ -62,6 +62,7 @@ class Http2FloodMitigationTest : public testing::TestWithParam Date: Wed, 31 Jul 2019 10:29:55 -0700 Subject: [PATCH 357/542] runtime: changing snapshot access to be const (#7677) (#26) This is a precursor to #7601 just to land the API change more quickly and make sure it sticks. Risk Level: Low Testing: existing unit tests Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk Signed-off-by: Piotr Sikora --- include/envoy/runtime/runtime.h | 4 ++-- source/common/runtime/runtime_impl.cc | 2 +- source/common/runtime/runtime_impl.h | 2 +- test/common/runtime/runtime_impl_test.cc | 4 ++-- test/mocks/runtime/mocks.h | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/envoy/runtime/runtime.h b/include/envoy/runtime/runtime.h index b3e3ebeb468a..bd3e759ee175 100644 --- a/include/envoy/runtime/runtime.h +++ b/include/envoy/runtime/runtime.h @@ -220,11 +220,11 @@ class Loader { virtual void initialize(Upstream::ClusterManager& cm) PURE; /** - * @return Snapshot& the current snapshot. This reference is safe to use for the duration of + * @return const Snapshot& the current snapshot. This reference is safe to use for the duration of * the calling routine, but may be overwritten on a future event loop cycle so should be * fetched again when needed. */ - virtual Snapshot& snapshot() PURE; + virtual const Snapshot& snapshot() PURE; /** * Merge the given map of key-value pairs into the runtime's state. To remove a previous merge for diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index a1369e6ab141..ae565aecbde3 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -557,7 +557,7 @@ void LoaderImpl::loadNewSnapshot() { }); } -Snapshot& LoaderImpl::snapshot() { return tls_->getTyped(); } +const Snapshot& LoaderImpl::snapshot() { return tls_->getTyped(); } void LoaderImpl::mergeValues(const std::unordered_map& values) { if (admin_layer_ == nullptr) { diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index 855dc1ea370e..b9344d0cc972 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -243,7 +243,7 @@ class LoaderImpl : public Loader, Logger::Loggable { // Runtime::Loader void initialize(Upstream::ClusterManager& cm) override; - Snapshot& snapshot() override; + const Snapshot& snapshot() override; void mergeValues(const std::unordered_map& values) override; private: diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index cb73ae6e73bf..e5edecfc5b88 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -155,7 +155,7 @@ TEST_F(DiskLoaderImplTest, All) { EXPECT_EQ(123UL, loader_->snapshot().getInteger("file4", 1)); bool value; - SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); + const SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); // Validate that the layer name is set properly for static layers. EXPECT_EQ("base", snapshot->getLayers()[0]->name()); @@ -538,7 +538,7 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { // Boolean getting. bool value; - SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); + const SnapshotImpl* snapshot = reinterpret_cast(&loader_->snapshot()); EXPECT_EQ(true, snapshot->getBoolean("file11", value)); EXPECT_EQ(true, value); diff --git a/test/mocks/runtime/mocks.h b/test/mocks/runtime/mocks.h index e67e34361958..5cb8230a14cc 100644 --- a/test/mocks/runtime/mocks.h +++ b/test/mocks/runtime/mocks.h @@ -63,7 +63,7 @@ class MockLoader : public Loader { ~MockLoader(); MOCK_METHOD1(initialize, void(Upstream::ClusterManager& cm)); - MOCK_METHOD0(snapshot, Snapshot&()); + MOCK_METHOD0(snapshot, const Snapshot&()); MOCK_METHOD1(mergeValues, void(const std::unordered_map&)); testing::NiceMock snapshot_; From 4e9588b5cca79f87cdf3b25f09ed8e361654f180 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Wed, 31 Jul 2019 13:04:51 -0700 Subject: [PATCH 358/542] runtime: making runtime accessible from non-worker threads (#7695) (#27) Making runtime accessible for non-worker threads, and using the new accessor for runtime features. This allows the work done in #7601, moving the strict HTTP checks out of the HCM and into the codec, where the integration tests use them from client/server threads, and other downstream Envoys might use them from non-worker threads as well. Risk Level: High (affects runtime access for all runtime features) Testing: new unit tests, integration tests use in #7601 Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk Signed-off-by: Piotr Sikora --- include/envoy/runtime/runtime.h | 8 +++- include/envoy/thread_local/thread_local.h | 10 +++++ source/common/runtime/runtime_impl.cc | 30 +++++++++++--- source/common/runtime/runtime_impl.h | 4 ++ .../common/thread_local/thread_local_impl.cc | 6 ++- .../common/thread_local/thread_local_impl.h | 1 + test/common/runtime/runtime_impl_test.cc | 40 +++++++++++++++++++ test/mocks/runtime/mocks.h | 1 + test/mocks/thread_local/mocks.h | 2 + 9 files changed, 95 insertions(+), 7 deletions(-) diff --git a/include/envoy/runtime/runtime.h b/include/envoy/runtime/runtime.h index bd3e759ee175..f850ec18fde4 100644 --- a/include/envoy/runtime/runtime.h +++ b/include/envoy/runtime/runtime.h @@ -222,10 +222,16 @@ class Loader { /** * @return const Snapshot& the current snapshot. This reference is safe to use for the duration of * the calling routine, but may be overwritten on a future event loop cycle so should be - * fetched again when needed. + * fetched again when needed. This may only be called from worker threads. */ virtual const Snapshot& snapshot() PURE; + /** + * @return shared_ptr the current snapshot. This function may safely be called + * from non-worker theads. + */ + virtual std::shared_ptr threadsafeSnapshot() PURE; + /** * Merge the given map of key-value pairs into the runtime's state. To remove a previous merge for * a key, use an empty string as the value. diff --git a/include/envoy/thread_local/thread_local.h b/include/envoy/thread_local/thread_local.h index 14fac5cd5e22..6f082a4607f5 100644 --- a/include/envoy/thread_local/thread_local.h +++ b/include/envoy/thread_local/thread_local.h @@ -28,6 +28,16 @@ class Slot { public: virtual ~Slot() = default; + /** + * Returns if there is thread local data for this thread. + * + * This should return true for Envoy worker threads and false for threads which do not have thread + * local storage allocated. + * + * @return true if registerThread has been called for this thread, false otherwise. + */ + virtual bool currentThreadRegistered() PURE; + /** * @return ThreadLocalObjectSharedPtr a thread local object stored in the slot. */ diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index ae565aecbde3..6ed82c39a37d 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -27,7 +27,8 @@ namespace Runtime { bool runtimeFeatureEnabled(absl::string_view feature) { ASSERT(absl::StartsWith(feature, "envoy.reloadable_features")); if (Runtime::LoaderSingleton::getExisting()) { - return Runtime::LoaderSingleton::getExisting()->snapshot().runtimeFeatureEnabled(feature); + return Runtime::LoaderSingleton::getExisting()->threadsafeSnapshot()->runtimeFeatureEnabled( + feature); } ENVOY_LOG_TO_LOGGER(Envoy::Logger::Registry::getLog(Envoy::Logger::Id::runtime), warn, "Unable to use runtime singleton for feature {}", feature); @@ -551,13 +552,32 @@ void RtdsSubscription::validateUpdateSize(uint32_t num_resources) { } void LoaderImpl::loadNewSnapshot() { - ThreadLocal::ThreadLocalObjectSharedPtr ptr = createNewSnapshot(); - tls_->set([ptr = std::move(ptr)](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { - return ptr; + std::shared_ptr ptr = createNewSnapshot(); + tls_->set([ptr](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::static_pointer_cast(ptr); }); + + { + absl::MutexLock lock(&snapshot_mutex_); + thread_safe_snapshot_ = ptr; + } +} + +const Snapshot& LoaderImpl::snapshot() { + ASSERT(tls_->currentThreadRegistered(), "snapshot can only be called from a worker thread"); + return tls_->getTyped(); } -const Snapshot& LoaderImpl::snapshot() { return tls_->getTyped(); } +std::shared_ptr LoaderImpl::threadsafeSnapshot() { + if (tls_->currentThreadRegistered()) { + return std::dynamic_pointer_cast(tls_->get()); + } + + { + absl::ReaderMutexLock lock(&snapshot_mutex_); + return thread_safe_snapshot_; + } +} void LoaderImpl::mergeValues(const std::unordered_map& values) { if (admin_layer_ == nullptr) { diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index b9344d0cc972..a41260eca02f 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -244,6 +244,7 @@ class LoaderImpl : public Loader, Logger::Loggable { // Runtime::Loader void initialize(Upstream::ClusterManager& cm) override; const Snapshot& snapshot() override; + std::shared_ptr threadsafeSnapshot() override; void mergeValues(const std::unordered_map& values) override; private: @@ -265,6 +266,9 @@ class LoaderImpl : public Loader, Logger::Loggable { Api::Api& api_; std::vector subscriptions_; Upstream::ClusterManager* cm_{}; + + absl::Mutex snapshot_mutex_; + std::shared_ptr thread_safe_snapshot_ GUARDED_BY(snapshot_mutex_); }; } // namespace Runtime diff --git a/source/common/thread_local/thread_local_impl.cc b/source/common/thread_local/thread_local_impl.cc index 6884aae42d9a..4e8c32fed776 100644 --- a/source/common/thread_local/thread_local_impl.cc +++ b/source/common/thread_local/thread_local_impl.cc @@ -37,8 +37,12 @@ SlotPtr InstanceImpl::allocateSlot() { return slot; } +bool InstanceImpl::SlotImpl::currentThreadRegistered() { + return thread_local_data_.data_.size() > index_; +} + ThreadLocalObjectSharedPtr InstanceImpl::SlotImpl::get() { - ASSERT(thread_local_data_.data_.size() > index_); + ASSERT(currentThreadRegistered()); return thread_local_data_.data_[index_]; } diff --git a/source/common/thread_local/thread_local_impl.h b/source/common/thread_local/thread_local_impl.h index 820cd1504a95..ad149a0916cd 100644 --- a/source/common/thread_local/thread_local_impl.h +++ b/source/common/thread_local/thread_local_impl.h @@ -34,6 +34,7 @@ class InstanceImpl : Logger::Loggable, public Instance { // ThreadLocal::Slot ThreadLocalObjectSharedPtr get() override; + bool currentThreadRegistered() override; void runOnAllThreads(Event::PostCb cb) override { parent_.runOnAllThreads(cb); } void runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback) override { parent_.runOnAllThreads(cb, main_callback); diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index e5edecfc5b88..3d39c6b8e9a7 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -610,6 +610,46 @@ TEST_F(StaticLoaderImplTest, ProtoParsing) { EXPECT_EQ(2, store_.gauge("runtime.num_layers", Stats::Gauge::ImportMode::NeverImport).value()); } +TEST_F(StaticLoaderImplTest, RuntimeFromNonWorkerThreads) { + // Force the thread to be considered a non-worker thread. + tls_.registered_ = false; + setup(); + + // Set up foo -> bar + loader_->mergeValues({{"foo", "bar"}}); + EXPECT_EQ("bar", loader_->threadsafeSnapshot()->get("foo")); + const Snapshot* original_snapshot_pointer = loader_->threadsafeSnapshot().get(); + + // Now set up a test thread which verifies foo -> bar + // + // Then change foo and make sure the test thread picks up the change. + Thread::MutexBasicLockable mutex; + Thread::CondVar foo_read; + Thread::CondVar foo_changed; + const Snapshot* original_thread_snapshot_pointer = nullptr; + auto thread = Thread::threadFactoryForTest().createThread([&]() { + Thread::LockGuard lock(mutex); + EXPECT_EQ("bar", loader_->threadsafeSnapshot()->get("foo")); + original_thread_snapshot_pointer = loader_->threadsafeSnapshot().get(); + EXPECT_EQ(original_thread_snapshot_pointer, loader_->threadsafeSnapshot().get()); + foo_read.notifyOne(); + + foo_changed.wait(mutex); + EXPECT_EQ("eep", loader_->threadsafeSnapshot()->get("foo")); + }); + + { + Thread::LockGuard lock(mutex); + foo_read.wait(mutex); + loader_->mergeValues({{"foo", "eep"}}); + foo_changed.notifyOne(); + EXPECT_EQ("eep", loader_->threadsafeSnapshot()->get("foo")); + } + + thread->join(); + EXPECT_EQ(original_thread_snapshot_pointer, original_snapshot_pointer); +} + class DiskLayerTest : public testing::Test { protected: DiskLayerTest() : api_(Api::createApiForTest()) {} diff --git a/test/mocks/runtime/mocks.h b/test/mocks/runtime/mocks.h index 5cb8230a14cc..73700781c2a8 100644 --- a/test/mocks/runtime/mocks.h +++ b/test/mocks/runtime/mocks.h @@ -64,6 +64,7 @@ class MockLoader : public Loader { MOCK_METHOD1(initialize, void(Upstream::ClusterManager& cm)); MOCK_METHOD0(snapshot, const Snapshot&()); + MOCK_METHOD0(threadsafeSnapshot, std::shared_ptr()); MOCK_METHOD1(mergeValues, void(const std::unordered_map&)); testing::NiceMock snapshot_; diff --git a/test/mocks/thread_local/mocks.h b/test/mocks/thread_local/mocks.h index 24393722887a..d4c4dc51c895 100644 --- a/test/mocks/thread_local/mocks.h +++ b/test/mocks/thread_local/mocks.h @@ -58,6 +58,7 @@ class MockInstance : public Instance { // ThreadLocal::Slot ThreadLocalObjectSharedPtr get() override { return parent_.data_[index_]; } + bool currentThreadRegistered() override { return parent_.registered_; } void runOnAllThreads(Event::PostCb cb) override { parent_.runOnAllThreads(cb); } void runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback) override { parent_.runOnAllThreads(cb, main_callback); @@ -72,6 +73,7 @@ class MockInstance : public Instance { testing::NiceMock dispatcher_; std::vector data_; bool shutdown_{}; + bool registered_{true}; }; } // namespace ThreadLocal From 71ff17c8e457d6f5be8d8d3b528ec2e6a892ca95 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Fri, 2 Aug 2019 16:56:12 -0400 Subject: [PATCH 359/542] Disable outbound flood mitigation through runtime config (#22) Signed-off-by: Yan Avlasov --- docs/root/intro/version_history.rst | 6 + source/common/http/http2/BUILD | 1 + source/common/http/http2/codec_impl.cc | 53 ++++++ source/common/http/http2/codec_impl.h | 23 +-- source/common/runtime/runtime_impl.cc | 11 ++ source/common/runtime/runtime_impl.h | 1 + test/common/http/http2/BUILD | 5 + test/common/http/http2/codec_impl_test.cc | 207 +++++++++++++++++++++- test/common/runtime/runtime_impl_test.cc | 10 ++ 9 files changed, 295 insertions(+), 22 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b14b9be35c68..b543451c10fa 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -5,11 +5,17 @@ Version history ======================== * http: added mitigation of client initiated atacks that result in flooding of the downstream HTTP/2 connections. * http: added :ref:`inbound_empty_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_consecutive_inbound_frames_with_empty_payload` overrides :ref:`max_consecutive_inbound_frames_with_empty_payload setting `. Large override value (i.e. 2147483647) effectively disables mitigation of inbound frames with empty payload. * http: added :ref:`inbound_priority_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound PRIORITY frames. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_inbound_priority_frames_per_stream` overrides :ref:`max_inbound_priority_frames_per_stream setting `. Large override value effectively disables flood mitigation of inbound PRIORITY frames. * http: added :ref:`inbound_window_update_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound WINDOW_UPDATE frames. The limit is configured by setting the :ref:`max_inbound_window_update_frames_per_data_frame_sent config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_inbound_window_update_frames_per_data_frame_sent` overrides :ref:`max_inbound_window_update_frames_per_data_frame_sent setting `. Large override value effectively disables flood mitigation of inbound WINDOW_UPDATE frames. * http: added :ref:`outbound_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting ` + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_outbound_frames` overrides :ref:`max_outbound_frames config setting `. Large override value effectively disables flood mitigation of outbound frames of all types. * http: added :ref:`outbound_control_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_outbound_control_frames` overrides :ref:`max_outbound_control_frames config setting `. Large override value effectively disables flood mitigation of outbound frames of types PING, SETTINGS and RST_STREAM. * http: enabled strict validation of HTTP/2 messaging. Previous behavior can be restored using :ref:`stream_error_on_invalid_http_messaging config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.stream_error_on_invalid_http_messaging` overrides :ref:`stream_error_on_invalid_http_messaging config setting `. 1.11.0 (July 11, 2019) ====================== diff --git a/source/common/http/http2/BUILD b/source/common/http/http2/BUILD index 7d51a2208dff..2133d6238e67 100644 --- a/source/common/http/http2/BUILD +++ b/source/common/http/http2/BUILD @@ -40,6 +40,7 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", + "//source/common/runtime:runtime_lib", ], ) diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index e45f07a101af..1752eb299f10 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -346,6 +346,59 @@ void ConnectionImpl::StreamImpl::onMetadataDecoded(MetadataMapPtr&& metadata_map decoder_->decodeMetadata(std::move(metadata_map_ptr)); } +namespace { + +const char InvalidHttpMessagingOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options.stream_error_on_invalid_http_messaging"; +const char MaxOutboundFramesOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options.max_outbound_frames"; +const char MaxOutboundControlFramesOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options.max_outbound_control_frames"; +const char MaxConsecutiveInboundFramesWithEmptyPayloadOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options." + "max_consecutive_inbound_frames_with_empty_payload"; +const char MaxInboundPriorityFramesPerStreamOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options.max_inbound_priority_frames_per_stream"; +const char MaxInboundWindowUpdateFramesPerDataFrameSentOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options." + "max_inbound_window_update_frames_per_data_frame_sent"; + +bool checkRuntimeOverride(bool config_value, const char* override_key) { + return Runtime::runtimeFeatureEnabled(override_key) ? true : config_value; +} + +} // namespace + +ConnectionImpl::ConnectionImpl(Network::Connection& connection, Stats::Scope& stats, + const Http2Settings& http2_settings, + const uint32_t max_request_headers_kb) + : stats_{ALL_HTTP2_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http2."))}, connection_(connection), + max_request_headers_kb_(max_request_headers_kb), + per_stream_buffer_limit_(http2_settings.initial_stream_window_size_), + stream_error_on_invalid_http_messaging_(checkRuntimeOverride( + http2_settings.stream_error_on_invalid_http_messaging_, InvalidHttpMessagingOverrideKey)), + flood_detected_(false), + max_outbound_frames_( + Runtime::getInteger(MaxOutboundFramesOverrideKey, http2_settings.max_outbound_frames_)), + frame_buffer_releasor_([this](const Buffer::OwnedBufferFragmentImpl* fragment) { + releaseOutboundFrame(fragment); + }), + max_outbound_control_frames_(Runtime::getInteger( + MaxOutboundControlFramesOverrideKey, http2_settings.max_outbound_control_frames_)), + control_frame_buffer_releasor_([this](const Buffer::OwnedBufferFragmentImpl* fragment) { + releaseOutboundControlFrame(fragment); + }), + max_consecutive_inbound_frames_with_empty_payload_( + Runtime::getInteger(MaxConsecutiveInboundFramesWithEmptyPayloadOverrideKey, + http2_settings.max_consecutive_inbound_frames_with_empty_payload_)), + max_inbound_priority_frames_per_stream_( + Runtime::getInteger(MaxInboundPriorityFramesPerStreamOverrideKey, + http2_settings.max_inbound_priority_frames_per_stream_)), + max_inbound_window_update_frames_per_data_frame_sent_(Runtime::getInteger( + MaxInboundWindowUpdateFramesPerDataFrameSentOverrideKey, + http2_settings.max_inbound_window_update_frames_per_data_frame_sent_)), + dispatching_(false), raised_goaway_(false), pending_deferred_reset_(false) {} + ConnectionImpl::~ConnectionImpl() { nghttp2_session_del(session_); } void ConnectionImpl::dispatch(Buffer::Instance& data) { diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 70bc2da1e378..8c1760f9f8a7 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -21,6 +21,7 @@ #include "common/http/http2/metadata_decoder.h" #include "common/http/http2/metadata_encoder.h" #include "common/http/utility.h" +#include "common/runtime/runtime_impl.h" #include "absl/types/optional.h" #include "nghttp2/nghttp2.h" @@ -78,27 +79,7 @@ class Utility { class ConnectionImpl : public virtual Connection, protected Logger::Loggable { public: ConnectionImpl(Network::Connection& connection, Stats::Scope& stats, - const Http2Settings& http2_settings, const uint32_t max_request_headers_kb) - : stats_{ALL_HTTP2_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http2."))}, - connection_(connection), max_request_headers_kb_(max_request_headers_kb), - per_stream_buffer_limit_(http2_settings.initial_stream_window_size_), - stream_error_on_invalid_http_messaging_( - http2_settings.stream_error_on_invalid_http_messaging_), - flood_detected_(false), max_outbound_frames_(http2_settings.max_outbound_frames_), - frame_buffer_releasor_([this](const Buffer::OwnedBufferFragmentImpl* fragment) { - releaseOutboundFrame(fragment); - }), - max_outbound_control_frames_(http2_settings.max_outbound_control_frames_), - control_frame_buffer_releasor_([this](const Buffer::OwnedBufferFragmentImpl* fragment) { - releaseOutboundControlFrame(fragment); - }), - max_consecutive_inbound_frames_with_empty_payload_( - http2_settings.max_consecutive_inbound_frames_with_empty_payload_), - max_inbound_priority_frames_per_stream_( - http2_settings.max_inbound_priority_frames_per_stream_), - max_inbound_window_update_frames_per_data_frame_sent_( - http2_settings.max_inbound_window_update_frames_per_data_frame_sent_), - dispatching_(false), raised_goaway_(false), pending_deferred_reset_(false) {} + const Http2Settings& http2_settings, const uint32_t max_request_headers_kb); ~ConnectionImpl(); diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 6ed82c39a37d..3c8643c1e320 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -35,6 +35,17 @@ bool runtimeFeatureEnabled(absl::string_view feature) { return RuntimeFeaturesDefaults::get().enabledByDefault(feature); } +uint64_t getInteger(absl::string_view feature, uint64_t default_value) { + ASSERT(absl::StartsWith(feature, "envoy.reloadable_features")); + if (Runtime::LoaderSingleton::getExisting()) { + return Runtime::LoaderSingleton::getExisting()->threadsafeSnapshot()->getInteger( + std::string(feature), default_value); + } + ENVOY_LOG_TO_LOGGER(Envoy::Logger::Registry::getLog(Envoy::Logger::Id::runtime), warn, + "Unable to use runtime singleton for feature {}", feature); + return default_value; +} + const size_t RandomGeneratorImpl::UUID_LENGTH = 36; uint64_t RandomGeneratorImpl::random() { diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index a41260eca02f..1d0cef2ae5c2 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -31,6 +31,7 @@ namespace Envoy { namespace Runtime { bool runtimeFeatureEnabled(absl::string_view feature); +uint64_t getInteger(absl::string_view feature, uint64_t default_value); using RuntimeSingleton = ThreadSafeSingleton; diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index 354055602126..ae51864b72d9 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -22,8 +22,13 @@ envoy_cc_test( "//source/common/http/http2:codec_lib", "//source/common/stats:stats_lib", "//test/common/http:common_lib", + "//test/common/http/http2:http2_frame", "//test/mocks/http:http_mocks", + "//test/mocks/init:init_mocks", + "//test/mocks/local_info:local_info_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:utility_lib", ], diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 0dc5bb358a2f..e738c140aff0 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -9,8 +9,13 @@ #include "common/http/http2/codec_impl.h" #include "test/common/http/common.h" +#include "test/common/http/http2/http2_frame.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/thread_local/mocks.h" #include "test/test_common/printers.h" #include "test/test_common/utility.h" @@ -164,7 +169,89 @@ class Http2CodecImplTest : public ::testing::TestWithParam(GetParam()), ::testing::get<1>(GetParam())) {} + : Http2CodecImplTestFixture(::testing::get<0>(GetParam()), ::testing::get<1>(GetParam())), + api_(Api::createApiForTest()) { + envoy::config::bootstrap::v2::LayeredRuntime config; + config.add_layers()->mutable_admin_layer(); + + // Create a runtime loader, so that tests can manually manipulate runtime + // guarded features. + loader_ = std::make_unique( + std::make_unique(dispatcher_, tls_, config, local_info_, init_manager_, + store_, generator_, validation_visitor_, *api_)); + } + +protected: + void priorityFlood() { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers, "POST"); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + nghttp2_priority_spec spec = {0, 10, 0}; + // HTTP/2 codec adds 1 to the number of active streams when computing PRIORITY frames limit + constexpr uint32_t max_allowed = + 2 * Http2Settings::DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM; + for (uint32_t i = 0; i < max_allowed + 1; ++i) { + EXPECT_EQ(0, nghttp2_submit_priority(client_->session(), NGHTTP2_FLAG_NONE, 1, &spec)); + } + } + + void windowUpdateFlood() { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + request_encoder_->encodeHeaders(request_headers, true); + + // Send one DATA frame back + EXPECT_CALL(response_decoder_, decodeHeaders_(_, false)); + EXPECT_CALL(response_decoder_, decodeData(_, false)); + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + + // See the limit formula in the + // `Envoy::Http::Http2::ServerConnectionImpl::checkInboundFrameLimits()' method. + constexpr uint32_t max_allowed = + 1 + 2 * (Http2Settings::DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT + 1); + for (uint32_t i = 0; i < max_allowed + 1; ++i) { + EXPECT_EQ(0, nghttp2_submit_window_update(client_->session(), NGHTTP2_FLAG_NONE, 1, 1)); + } + } + + void emptyDataFlood(Buffer::OwnedImpl& data) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers, "POST"); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + // HTTP/2 codec does not send empty DATA frames with no END_STREAM flag. + // To make this work, send raw bytes representing empty DATA frames bypassing client codec. + Http2Frame emptyDataFrame = Http2Frame::makeEmptyDataFrame(0); + constexpr uint32_t max_allowed = + Http2Settings::DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD; + for (uint32_t i = 0; i < max_allowed + 1; ++i) { + data.add(emptyDataFrame.data(), emptyDataFrame.size()); + } + } + +private: + Event::MockDispatcher dispatcher_; + NiceMock tls_; + Stats::IsolatedStoreImpl store_; + Runtime::MockRandomGenerator generator_; + Api::ApiPtr api_; + NiceMock local_info_; + Init::MockManager init_manager_; + NiceMock validation_visitor_; + std::unique_ptr loader_; }; TEST_P(Http2CodecImplTest, ShutdownNotice) { @@ -408,6 +495,25 @@ TEST_P(Http2CodecImplTest, InvalidHeadersFrameAllowed) { server_wrapper_.dispatch(Buffer::OwnedImpl(), *server_); } +TEST_P(Http2CodecImplTest, InvalidHeadersFrameOverriden) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.stream_error_on_invalid_http_messaging", + "true"}}); + initialize(); + + MockStreamCallbacks request_callbacks; + request_encoder_->getStream().addCallbacks(request_callbacks); + + ON_CALL(client_connection_, write(_, _)) + .WillByDefault( + Invoke([&](Buffer::Instance& data, bool) -> void { server_wrapper_.buffer_.add(data); })); + + request_encoder_->encodeHeaders(TestHeaderMapImpl{}, true); + EXPECT_CALL(server_stream_callbacks_, onResetStream(StreamResetReason::LocalReset, _)); + EXPECT_CALL(request_callbacks, onResetStream(StreamResetReason::RemoteReset, _)); + server_wrapper_.dispatch(Buffer::OwnedImpl(), *server_); +} + TEST_P(Http2CodecImplTest, TrailingHeaders) { initialize(); @@ -1144,6 +1250,28 @@ TEST_P(Http2CodecImplTest, PingFlood) { EXPECT_EQ(1, stats_store_.counter("http2.outbound_control_flood").value()); } +// Verify that codec allows PING flood when mitigation is disabled +TEST_P(Http2CodecImplTest, PingFloodMitigationDisabled) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_outbound_control_frames", + "2147483647"}}); + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + // Send one frame above the outbound control queue size limit + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES + 1; ++i) { + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + } + + EXPECT_CALL(server_connection_, write(_, _)) + .Times(Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES + 1); + EXPECT_NO_THROW(client_->sendPendingFrames()); +} + // Verify that outbound control frame counter decreases when send buffer is drained TEST_P(Http2CodecImplTest, PingFloodCounterReset) { static const int kMaxOutboundControlFrames = 100; @@ -1249,6 +1377,36 @@ TEST_P(Http2CodecImplTest, ResponseDataFlood) { EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); } +// Verify that codec allows outbound DATA flood when mitigation is disabled +TEST_P(Http2CodecImplTest, ResponseDataFloodMitigationDisabled) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_outbound_frames", "2147483647"}}); + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + // +2 is to account for HEADERS and PING ACK, that is used to trigger mitigation + EXPECT_CALL(server_connection_, write(_, _)) + .Times(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 2); + EXPECT_CALL(response_decoder_, decodeHeaders_(_, false)).Times(1); + EXPECT_CALL(response_decoder_, decodeData(_, false)) + .Times(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES); + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + // Account for the single HEADERS frame above + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + // Presently flood mitigation is done only when processing downstream data + // So we need to send stream from downstream client to trigger mitigation + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_NO_THROW(client_->sendPendingFrames()); +} + // Verify that outbound frame counter decreases when send buffer is drained TEST_P(Http2CodecImplTest, ResponseDataFloodCounterReset) { static const int kMaxOutboundFrames = 100; @@ -1323,6 +1481,53 @@ TEST_P(Http2CodecImplTest, PingStacksWithDataFlood) { EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); } +TEST_P(Http2CodecImplTest, PriorityFlood) { + priorityFlood(); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); +} + +TEST_P(Http2CodecImplTest, PriorityFloodOverride) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_inbound_priority_frames_per_stream", + "2147483647"}}); + + priorityFlood(); + EXPECT_NO_THROW(client_->sendPendingFrames()); +} + +TEST_P(Http2CodecImplTest, WindowUpdateFlood) { + windowUpdateFlood(); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); +} + +TEST_P(Http2CodecImplTest, WindowUpdateFloodOverride) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_inbound_window_update_frames_per_" + "data_frame_sent", + "2147483647"}}); + windowUpdateFlood(); + EXPECT_NO_THROW(client_->sendPendingFrames()); +} + +TEST_P(Http2CodecImplTest, EmptyDataFlood) { + Buffer::OwnedImpl data; + emptyDataFlood(data); + EXPECT_CALL(request_decoder_, decodeData(_, false)); + EXPECT_THROW(server_wrapper_.dispatch(data, *server_), FrameFloodException); +} + +TEST_P(Http2CodecImplTest, EmptyDataFloodOverride) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_consecutive_inbound_frames_with_" + "empty_payload", + "2147483647"}}); + Buffer::OwnedImpl data; + emptyDataFlood(data); + EXPECT_CALL(request_decoder_, decodeData(_, false)) + .Times(Http2Settings::DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD + 1); + EXPECT_NO_THROW(server_wrapper_.dispatch(data, *server_)); +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 3d39c6b8e9a7..4b3e040c6213 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -692,6 +692,16 @@ TEST(NoRuntime, FeatureEnabled) { EXPECT_EQ(true, runtimeFeatureEnabled("envoy.reloadable_features.test_feature_true")); } +TEST(NoRuntime, DefaultIntValues) { + // Make sure the registry is not set up. + ASSERT_TRUE(Runtime::LoaderSingleton::getExisting() == nullptr); + + // Feature defaults should still work. + EXPECT_EQ(0x1230000ABCDULL, + getInteger("envoy.reloadable_features.test_int_feature_default", 0x1230000ABCDULL)); + EXPECT_EQ(0, getInteger("envoy.reloadable_features.test_int_feature_zero", 0)); +} + // Test RTDS layer(s). class RtdsLoaderImplTest : public LoaderImplTest { public: From 26a1346a15020c1af2e2a15a1b90fec1998ac98a Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin Date: Fri, 9 Aug 2019 13:33:11 -0700 Subject: [PATCH 360/542] buffer filter: Deflake RouterRequestPopulateContentLength (#7885) Signed-off-by: Ruslan Nigmatullin --- .../filters/http/buffer/buffer_filter_integration_test.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc b/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc index d93bad947536..92325a9b6636 100644 --- a/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc +++ b/test/extensions/filters/http/buffer/buffer_filter_integration_test.cc @@ -51,12 +51,16 @@ TEST_P(BufferIntegrationTest, RouterRequestPopulateContentLength) { ASSERT_TRUE(fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, fake_upstream_connection_)); ASSERT_TRUE(fake_upstream_connection_->waitForNewStream(*dispatcher_, upstream_request_)); + upstream_request_->encodeHeaders(default_response_headers_, true); ASSERT_TRUE(upstream_request_->waitForEndStream(*dispatcher_)); + auto* content_length = upstream_request_->headers().ContentLength(); ASSERT_NE(content_length, nullptr); EXPECT_EQ(content_length->value().getStringView(), "9"); + response->waitForEndStream(); ASSERT_TRUE(response->complete()); + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); } TEST_P(BufferIntegrationTest, RouterRequestBufferLimitExceeded) { From 975c62dd2258fe23c3d8098b99261fffd43da736 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 9 Aug 2019 16:10:14 -0700 Subject: [PATCH 361/542] listener: add an option to continue on listener filters timeout (#7859) Description: Add an option to continue on listener filters timeout. Risk Level: Med (mostly guarded by config) Testing: unittest Docs Changes: Added Release Notes: Added Fixes #7195 Signed-off-by: Lizan Zhou --- api/envoy/api/v2/lds.proto | 14 +++- docs/root/intro/version_history.rst | 1 + include/envoy/network/BUILD | 1 + include/envoy/network/listener.h | 9 ++- source/server/connection_handler_impl.cc | 59 ++++++++------ source/server/connection_handler_impl.h | 2 + source/server/http/admin.h | 1 + source/server/listener_manager_impl.cc | 3 +- source/server/listener_manager_impl.h | 4 + .../proxy_protocol/proxy_protocol_test.cc | 2 + test/integration/fake_upstream.h | 1 + test/mocks/network/mocks.h | 1 + test/server/connection_handler_test.cc | 79 ++++++++++++++++--- 13 files changed, 137 insertions(+), 40 deletions(-) diff --git a/api/envoy/api/v2/lds.proto b/api/envoy/api/v2/lds.proto index dd29659ce89e..195401341c96 100644 --- a/api/envoy/api/v2/lds.proto +++ b/api/envoy/api/v2/lds.proto @@ -44,7 +44,7 @@ service ListenerDiscoveryService { } } -// [#comment:next free field: 17] +// [#comment:next free field: 18] message Listener { // The unique name by which this listener is known. If no name is provided, // Envoy will allocate an internal UUID for the listener. If the listener is to be dynamically @@ -132,10 +132,20 @@ message Listener { repeated listener.ListenerFilter listener_filters = 9; // The timeout to wait for all listener filters to complete operation. If the timeout is reached, - // the accepted socket is closed without a connection being created. Specify 0 to disable the + // the accepted socket is closed without a connection being created unless + // `continue_on_listener_filters_timeout` is set to true. Specify 0 to disable the // timeout. If not specified, a default timeout of 15s is used. google.protobuf.Duration listener_filters_timeout = 15 [(gogoproto.stdduration) = true]; + // Whether a connection should be created when listener filters timeout. Default is false. + // + // .. attention:: + // + // Some listener filters, such as :ref:`Proxy Protocol filter + // `, should not be used with this option. It will cause + // unexpected behavior when a connection is created. + bool continue_on_listener_filters_timeout = 17; + // Whether the listener should be set as a transparent socket. // When this flag is set to true, connections can be redirected to the listener using an // *iptables* *TPROXY* target, in which case the original source and destination addresses and diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 4d5123dd04bb..bc597c1b8259 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -17,6 +17,7 @@ Version history * header to metadata: added :ref:`PROTOBUF_VALUE ` and :ref:`ValueEncode ` to support protobuf Value and Base64 encoding. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. * http: added the ability to :ref:`merge adjacent slashes` in the path. +* listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * router check tool: add coverage reporting & enforcement. diff --git a/include/envoy/network/BUILD b/include/envoy/network/BUILD index 2c1a20f5e90c..0a9c502f1f59 100644 --- a/include/envoy/network/BUILD +++ b/include/envoy/network/BUILD @@ -100,6 +100,7 @@ envoy_cc_library( ":connection_interface", ":listen_socket_interface", "//include/envoy/stats:stats_interface", + "@envoy_api//envoy/api/v2:lds_cc", ], ) diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 8fd34c72795f..53b06b01be65 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -65,10 +65,17 @@ class ListenerConfig { /** * @return std::chrono::milliseconds the time to wait for all listener filters to complete * operation. If the timeout is reached, the accepted socket is closed without a - * connection being created. 0 specifies a disabled timeout. + * connection being created unless continueOnListenerFiltersTimeout() returns true. + * 0 specifies a disabled timeout. */ virtual std::chrono::milliseconds listenerFiltersTimeout() const PURE; + /** + * @return bool whether the listener should try to create a connection when listener filters + * time out. + */ + virtual bool continueOnListenerFiltersTimeout() const PURE; + /** * @return Stats::Scope& the stats scope to use for all listener specific stats. */ diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index b2fe7b2ae95c..b7e08a31441b 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -90,6 +90,7 @@ ConnectionHandlerImpl::ActiveListenerBase::ActiveListenerBase(ConnectionHandlerI : parent_(parent), listener_(std::move(listener)), stats_(generateStats(config.listenerScope())), listener_filters_timeout_(config.listenerFiltersTimeout()), + continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()), listener_tag_(config.listenerTag()), config_(config) {} ConnectionHandlerImpl::ActiveTcpListener::ActiveTcpListener(ConnectionHandlerImpl& parent, @@ -160,6 +161,9 @@ ConnectionHandlerImpl::findActiveListenerByAddress(const Network::Address::Insta void ConnectionHandlerImpl::ActiveSocket::onTimeout() { listener_.stats_.downstream_pre_cx_timeout_.inc(); ASSERT(inserted()); + if (listener_.continue_on_listener_filters_timeout_) { + newConnection(); + } unlink(); } @@ -195,32 +199,7 @@ void ConnectionHandlerImpl::ActiveSocket::continueFilterChain(bool success) { } } // Successfully ran all the accept filters. - - // Check if the socket may need to be redirected to another listener. - ActiveListenerBase* new_listener = nullptr; - - if (hand_off_restored_destination_connections_ && socket_->localAddressRestored()) { - // Find a listener associated with the original destination address. - new_listener = listener_.parent_.findActiveListenerByAddress(*socket_->localAddress()); - } - if (new_listener != nullptr) { - // TODO(sumukhs): Try to avoid dynamic_cast by coming up with a better interface design - ActiveTcpListener* tcp_listener = dynamic_cast(new_listener); - ASSERT(tcp_listener != nullptr, "ActiveSocket listener is expected to be tcp"); - // Hands off connections redirected by iptables to the listener associated with the - // original destination address. Pass 'hand_off_restored_destination_connections' as false to - // prevent further redirection. - tcp_listener->onAccept(std::move(socket_), - false /* hand_off_restored_destination_connections */); - } else { - // Set default transport protocol if none of the listener filters did it. - if (socket_->detectedTransportProtocol().empty()) { - socket_->setDetectedTransportProtocol( - Extensions::TransportSockets::TransportSocketNames::get().RawBuffer); - } - // Create a new connection on this listener. - listener_.newConnection(std::move(socket_)); - } + newConnection(); } // Filter execution concluded, unlink and delete this ActiveSocket if it was linked. @@ -229,6 +208,34 @@ void ConnectionHandlerImpl::ActiveSocket::continueFilterChain(bool success) { } } +void ConnectionHandlerImpl::ActiveSocket::newConnection() { + // Check if the socket may need to be redirected to another listener. + ActiveListenerBase* new_listener = nullptr; + + if (hand_off_restored_destination_connections_ && socket_->localAddressRestored()) { + // Find a listener associated with the original destination address. + new_listener = listener_.parent_.findActiveListenerByAddress(*socket_->localAddress()); + } + if (new_listener != nullptr) { + // TODO(sumukhs): Try to avoid dynamic_cast by coming up with a better interface design + ActiveTcpListener* tcp_listener = dynamic_cast(new_listener); + ASSERT(tcp_listener != nullptr, "ActiveSocket listener is expected to be tcp"); + // Hands off connections redirected by iptables to the listener associated with the + // original destination address. Pass 'hand_off_restored_destination_connections' as false to + // prevent further redirection. + tcp_listener->onAccept(std::move(socket_), + false /* hand_off_restored_destination_connections */); + } else { + // Set default transport protocol if none of the listener filters did it. + if (socket_->detectedTransportProtocol().empty()) { + socket_->setDetectedTransportProtocol( + Extensions::TransportSockets::TransportSocketNames::get().RawBuffer); + } + // Create a new connection on this listener. + listener_.newConnection(std::move(socket_)); + } +} + void ConnectionHandlerImpl::ActiveTcpListener::onAccept( Network::ConnectionSocketPtr&& socket, bool hand_off_restored_destination_connections) { auto active_socket = std::make_unique(*this, std::move(socket), diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index f8b83e3628f6..7ca0088abe61 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -89,6 +89,7 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { Network::ListenerPtr listener_; ListenerStats stats_; const std::chrono::milliseconds listener_filters_timeout_; + const bool continue_on_listener_filters_timeout_; const uint64_t listener_tag_; Network::ListenerConfig& config_; }; @@ -200,6 +201,7 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { void onTimeout(); void startTimer(); void unlink(); + void newConnection(); // Network::ListenerFilterManager void addAcceptFilter(Network::ListenerFilterPtr&& filter) override { diff --git a/source/server/http/admin.h b/source/server/http/admin.h index a01c6b1ad21f..e3806f58e188 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -315,6 +315,7 @@ class AdminImpl : public Admin, std::chrono::milliseconds listenerFiltersTimeout() const override { return std::chrono::milliseconds(); } + bool continueOnListenerFiltersTimeout() const override { return false; } Stats::Scope& listenerScope() override { return *scope_; } uint64_t listenerTag() const override { return 0; } const std::string& name() const override { return name_; } diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 00978ccbb933..824c42ec97ea 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -204,7 +204,8 @@ ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, const std::st local_drain_manager_(parent.factory_.createDrainManager(config.drain_type())), config_(config), version_info_(version_info), listener_filters_timeout_( - PROTOBUF_GET_MS_OR_DEFAULT(config, listener_filters_timeout, 15000)) { + PROTOBUF_GET_MS_OR_DEFAULT(config, listener_filters_timeout, 15000)), + continue_on_listener_filters_timeout_(config.continue_on_listener_filters_timeout()) { if (config.has_transparent()) { addListenSocketOptions(Network::SocketOptionFactory::buildIpTransparentOptions()); } diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 8435f0213065..9177a0ec03af 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -271,6 +271,9 @@ class ListenerImpl : public Network::ListenerConfig, std::chrono::milliseconds listenerFiltersTimeout() const override { return listener_filters_timeout_; } + bool continueOnListenerFiltersTimeout() const override { + return continue_on_listener_filters_timeout_; + } Stats::Scope& listenerScope() override { return *listener_scope_; } uint64_t listenerTag() const override { return listener_tag_; } const std::string& name() const override { return name_; } @@ -372,6 +375,7 @@ class ListenerImpl : public Network::ListenerConfig, const std::string version_info_; Network::Socket::OptionsSharedPtr listen_socket_options_; const std::chrono::milliseconds listener_filters_timeout_; + const bool continue_on_listener_filters_timeout_; // to access ListenerManagerImpl::factory_. friend class ListenerFilterChainFactoryBuilder; }; diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index 6d58343ecb7e..2f37b861df43 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -73,6 +73,7 @@ class ProxyProtocolTest : public testing::TestWithParam, std::chrono::milliseconds listenerFiltersTimeout() const override { return std::chrono::milliseconds(); } + bool continueOnListenerFiltersTimeout() const override { return false; } Stats::Scope& listenerScope() override { return parent_.stats_store_; } uint64_t listenerTag() const override { return 0; } const std::string& name() const override { return name_; } diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 8d6401fd44ea..e63d3f22957b 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -307,6 +307,7 @@ class MockListenerConfig : public ListenerConfig { MOCK_CONST_METHOD0(handOffRestoredDestinationConnections, bool()); MOCK_CONST_METHOD0(perConnectionBufferLimitBytes, uint32_t()); MOCK_CONST_METHOD0(listenerFiltersTimeout, std::chrono::milliseconds()); + MOCK_CONST_METHOD0(continueOnListenerFiltersTimeout, bool()); MOCK_METHOD0(listenerScope, Stats::Scope&()); MOCK_CONST_METHOD0(listenerTag, uint64_t()); MOCK_CONST_METHOD0(name, const std::string&()); diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 1aca75fb3158..850d55a348e7 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -38,10 +38,12 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable; - TestListener* addListener( - uint64_t tag, bool bind_to_port, bool hand_off_restored_destination_connections, - const std::string& name, - Network::Address::SocketType socket_type = Network::Address::SocketType::Stream, - std::chrono::milliseconds listener_filters_timeout = std::chrono::milliseconds(15000)) { - TestListener* listener = - new TestListener(*this, tag, bind_to_port, hand_off_restored_destination_connections, name, - socket_type, listener_filters_timeout); + TestListener* + addListener(uint64_t tag, bool bind_to_port, bool hand_off_restored_destination_connections, + const std::string& name, + Network::Address::SocketType socket_type = Network::Address::SocketType::Stream, + std::chrono::milliseconds listener_filters_timeout = std::chrono::milliseconds(15000), + bool continue_on_listener_filters_timeout = false) { + TestListener* listener = new TestListener( + *this, tag, bind_to_port, hand_off_restored_destination_connections, name, socket_type, + listener_filters_timeout, continue_on_listener_filters_timeout); listener->moveIntoListBack(TestListenerPtr{listener}, listeners_); return listener; } @@ -607,6 +614,58 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeout) { EXPECT_EQ(0UL, downstream_pre_cx_active.value()); EXPECT_EQ(1UL, stats_store_.counter("downstream_pre_cx_timeout").value()); + // Make sure we didn't continue to try create connection. + EXPECT_EQ(0UL, stats_store_.counter("no_filter_chain_match").value()); + + EXPECT_CALL(*listener, onDestroy()); +} + +// Continue on timeout during listener filter stop iteration. +TEST_F(ConnectionHandlerTest, ContinueOnListenerFilterTimeout) { + InSequence s; + + TestListener* test_listener = + addListener(1, true, false, "test_listener", Network::Address::SocketType::Stream, + std::chrono::milliseconds(15000), true); + Network::MockListener* listener = new Network::MockListener(); + Network::ListenerCallbacks* listener_callbacks; + EXPECT_CALL(dispatcher_, createListener_(_, _, _, false)) + .WillOnce(Invoke( + [&](Network::Socket&, Network::ListenerCallbacks& cb, bool, bool) -> Network::Listener* { + listener_callbacks = &cb; + return listener; + })); + EXPECT_CALL(test_listener->socket_, localAddress()); + handler_->addListener(*test_listener); + + Network::MockListenerFilter* test_filter = new Network::MockListenerFilter(); + EXPECT_CALL(factory_, createListenerFilterChain(_)) + .WillRepeatedly(Invoke([&](Network::ListenerFilterManager& manager) -> bool { + manager.addAcceptFilter(Network::ListenerFilterPtr{test_filter}); + return true; + })); + EXPECT_CALL(*test_filter, onAccept(_)) + .WillOnce(Invoke([&](Network::ListenerFilterCallbacks&) -> Network::FilterStatus { + return Network::FilterStatus::StopIteration; + })); + Event::MockTimer* timeout = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000))); + Network::MockConnectionSocket* accepted_socket = new NiceMock(); + listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}, true); + Stats::Gauge& downstream_pre_cx_active = + stats_store_.gauge("downstream_pre_cx_active", Stats::Gauge::ImportMode::Accumulate); + EXPECT_EQ(1UL, downstream_pre_cx_active.value()); + + EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(nullptr)); + EXPECT_CALL(*timeout, disableTimer()); + timeout->callback_(); + dispatcher_.clearDeferredDeleteList(); + EXPECT_EQ(0UL, downstream_pre_cx_active.value()); + EXPECT_EQ(1UL, stats_store_.counter("downstream_pre_cx_timeout").value()); + + // Make sure we continued to try create connection. + EXPECT_EQ(1UL, stats_store_.counter("no_filter_chain_match").value()); + EXPECT_CALL(*listener, onDestroy()); } From 922f2f9d9912534d6e8d05d4cc6440ca95b0e362 Mon Sep 17 00:00:00 2001 From: Michael Rebello Date: Sun, 11 Aug 2019 07:10:59 -0700 Subject: [PATCH 362/542] dynamic forward proxy: add factory declarations + linking validation (#7889) Signed-off-by: Michael Rebello --- source/extensions/clusters/dynamic_forward_proxy/cluster.h | 2 ++ source/extensions/filters/http/dynamic_forward_proxy/config.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/source/extensions/clusters/dynamic_forward_proxy/cluster.h b/source/extensions/clusters/dynamic_forward_proxy/cluster.h index 7635e91cb49b..39b0d36ecfbd 100644 --- a/source/extensions/clusters/dynamic_forward_proxy/cluster.h +++ b/source/extensions/clusters/dynamic_forward_proxy/cluster.h @@ -128,6 +128,8 @@ class ClusterFactory : public Upstream::ConfigurableClusterFactoryBase< Stats::ScopePtr&& stats_scope) override; }; +DECLARE_FACTORY(ClusterFactory); + } // namespace DynamicForwardProxy } // namespace Clusters } // namespace Extensions diff --git a/source/extensions/filters/http/dynamic_forward_proxy/config.h b/source/extensions/filters/http/dynamic_forward_proxy/config.h index 9f8596c280ea..873c510c8615 100644 --- a/source/extensions/filters/http/dynamic_forward_proxy/config.h +++ b/source/extensions/filters/http/dynamic_forward_proxy/config.h @@ -26,6 +26,8 @@ class DynamicForwardProxyFilterFactory const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; }; +DECLARE_FACTORY(DynamicForwardProxyFilterFactory); + } // namespace DynamicForwardProxy } // namespace HttpFilters } // namespace Extensions From 4616a0939972438ea47f0ac6657296fb836d1187 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Sat, 10 Aug 2019 00:12:25 +0000 Subject: [PATCH 363/542] runtime: add the ability to log downstream HTTP/2 attacks. Signed-off-by: Matt Klein Signed-off-by: Piotr Sikora --- docs/root/intro/version_history.rst | 2 +- source/common/http/conn_manager_impl.cc | 13 +++++++++ test/common/http/BUILD | 1 + test/common/http/conn_manager_impl_test.cc | 34 +++++++++++++++++++++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b543451c10fa..a27b4118bba0 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -3,7 +3,7 @@ Version history 1.11.1 (August 13, 2019) ======================== -* http: added mitigation of client initiated atacks that result in flooding of the downstream HTTP/2 connections. +* http: added mitigation of client initiated attacks that result in flooding of the downstream HTTP/2 connections. Those attacks can be logged at the "warning" level when the runtime feature `http.connection_manager.log_flood_exception` is enabled. The runtime setting defaults to disabled to avoid log spam when under attack. * http: added :ref:`inbound_empty_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. Runtime feature `envoy.reloadable_features.http2_protocol_options.max_consecutive_inbound_frames_with_empty_payload` overrides :ref:`max_consecutive_inbound_frames_with_empty_payload setting `. Large override value (i.e. 2147483647) effectively disables mitigation of inbound frames with empty payload. * http: added :ref:`inbound_priority_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound PRIORITY frames. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index d3e0f93cf76c..f6a789443a9b 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -288,6 +288,19 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool try { codec_->dispatch(data); } catch (const FrameFloodException& e) { + // TODO(mattklein123): This is an emergency substitute for the lack of connection level + // logging in the HCM. In a public follow up change we will add full support for connection + // level logging in the HCM, similar to what we have in tcp_proxy. This will allow abuse + // indicators to be stored in the connection level stream info, and then matched, sampled, + // etc. when logged. + const envoy::type::FractionalPercent default_value; // 0 + if (runtime_.snapshot().featureEnabled("http.connection_manager.log_flood_exception", + default_value)) { + ENVOY_CONN_LOG(warn, "downstream HTTP flood from IP '{}': {}", + read_callbacks_->connection(), + read_callbacks_->connection().remoteAddress()->asString(), e.what()); + } + handleCodecException(e.what()); return Network::FilterStatus::StopIteration; } catch (const CodecProtocolException& e) { diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 4f5ac9cad9f9..1fc8914b9abd 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -211,6 +211,7 @@ envoy_cc_test( "//test/mocks/ssl:ssl_mocks", "//test/mocks/tracing:tracing_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:logging_lib", "//test/test_common:test_time_lib", ], ) diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index c09bfceeab98..44e86304fb7a 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -39,6 +39,7 @@ #include "test/mocks/tracing/mocks.h" #include "test/mocks/upstream/cluster_info.h" #include "test/mocks/upstream/mocks.h" +#include "test/test_common/logging.h" #include "test/test_common/printers.h" #include "test/test_common/test_time.h" @@ -55,6 +56,7 @@ using testing::HasSubstr; using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; +using testing::Matcher; using testing::NiceMock; using testing::Ref; using testing::Return; @@ -2333,7 +2335,37 @@ TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { // Kick off the incoming data. Buffer::OwnedImpl fake_input("1234"); - conn_manager_->onData(fake_input, false); + EXPECT_LOG_NOT_CONTAINS("warning", "downstream HTTP flood", + conn_manager_->onData(fake_input, false)); +} + +// Verify that FrameFloodException causes connection to be closed abortively as well as logged +// if runtime indicates to do so. +TEST_F(HttpConnectionManagerImplTest, FrameFloodErrorWithLog) { + InSequence s; + setup(false, ""); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + conn_manager_->newStream(response_encoder_); + throw FrameFloodException("too many outbound frames."); + })); + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("http.connection_manager.log_flood_exception", + Matcher(_))) + .WillOnce(Return(true)); + + EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)); + EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); + + // FrameFloodException should result in reset of the streams followed by abortive close. + EXPECT_CALL(filter_callbacks_.connection_, + close(Network::ConnectionCloseType::FlushWriteAndDelay)); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + EXPECT_LOG_CONTAINS("warning", + "downstream HTTP flood from IP '0.0.0.0:0': too many outbound frames.", + conn_manager_->onData(fake_input, false)); } TEST_F(HttpConnectionManagerImplTest, IdleTimeoutNoCodec) { From dd5eb0c8903bc9028fb38882c033b509784c4cbb Mon Sep 17 00:00:00 2001 From: Yechiel Kalmenson Date: Mon, 12 Aug 2019 14:56:13 -0400 Subject: [PATCH 364/542] Introduce --features=compiler_param_file option for windows (#7897) This resolves compile line length issues by triggering a command args file https://github.com/bazelbuild/bazel/issues/5163 Signed-off-by: William Rowe --- ci/build_setup.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_setup.ps1 b/ci/build_setup.ps1 index 49bb4a0990da..9d4d4d51d258 100755 --- a/ci/build_setup.ps1 +++ b/ci/build_setup.ps1 @@ -18,4 +18,4 @@ echo "ENVOY_BAZEL_ROOT: $env:ENVOY_BAZEL_ROOT" echo "ENVOY_SRCDIR: $env:ENVOY_SRCDIR" $env:BAZEL_BASE_OPTIONS="--noworkspace_rc --output_base=$env:ENVOY_BAZEL_ROOT --bazelrc=$env:ENVOY_SRCDIR\windows\tools\bazel.rc" -$env:BAZEL_BUILD_OPTIONS="--strategy=Genrule=standalone --spawn_strategy=standalone --verbose_failures --jobs=$env:NUM_CPUS --show_task_finish --cache_test_results=no --test_output=all $env:BAZEL_BUILD_EXTRA_OPTIONS $env:BAZEL_EXTRA_TEST_OPTIONS" +$env:BAZEL_BUILD_OPTIONS="--features=compiler_param_file --strategy=Genrule=standalone --spawn_strategy=standalone --verbose_failures --jobs=$env:NUM_CPUS --show_task_finish --cache_test_results=no --test_output=all $env:BAZEL_BUILD_EXTRA_OPTIONS $env:BAZEL_EXTRA_TEST_OPTIONS" From 93f87c3d16db450daed47e12c43c77bc67852f35 Mon Sep 17 00:00:00 2001 From: ethan Date: Tue, 13 Aug 2019 02:56:43 +0800 Subject: [PATCH 365/542] cleanup: generator.py words correction. (#7891) Signed-off-by: Guangming Wang --- source/extensions/filters/network/kafka/protocol/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/kafka/protocol/generator.py b/source/extensions/filters/network/kafka/protocol/generator.py index a0cad7e89ffd..d407eca1ce61 100755 --- a/source/extensions/filters/network/kafka/protocol/generator.py +++ b/source/extensions/filters/network/kafka/protocol/generator.py @@ -246,7 +246,7 @@ def is_nullable(self): def is_nullable_in_version(self, version): """ - Whether thie field is nullable in given version. + Whether the field is nullable in given version. Fields can be non-nullable in earlier versions. See https://github.com/apache/kafka/tree/2.2.0-rc0/clients/src/main/resources/common/message#nullable-fields """ @@ -435,7 +435,7 @@ def __init__(self, name, fields, versions): def __compute_declaration_chain(self): """ - Computes all dependendencies, what means all non-primitive types used by this type. + Computes all dependencies, what means all non-primitive types used by this type. They need to be declared before this struct is declared. """ result = [] From 1cf877d7d566d74f1c6fafe25f3db509217e5b0a Mon Sep 17 00:00:00 2001 From: Alex Konradi Date: Mon, 12 Aug 2019 14:57:36 -0400 Subject: [PATCH 366/542] build: miscellaneous hygiene fixes (#7888) Signed-off-by: Alex Konradi --- source/common/http/http1/codec_impl.cc | 9 +++++---- source/common/http/http1/codec_impl.h | 2 +- source/common/singleton/BUILD | 1 + source/common/singleton/threadsafe_singleton.h | 2 ++ source/common/stats/symbol_table_impl.h | 4 ++-- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index 704787276616..ba631706f310 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -326,12 +326,13 @@ const ToLowerTable& ConnectionImpl::toLowerTable() { } ConnectionImpl::ConnectionImpl(Network::Connection& connection, Stats::Scope& stats, - http_parser_type type, uint32_t max_headers_kb) + http_parser_type type, uint32_t max_request_headers_kb) : connection_(connection), stats_{ALL_HTTP1_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http1."))}, output_buffer_([&]() -> void { this->onBelowLowWatermark(); }, [&]() -> void { this->onAboveHighWatermark(); }), - max_headers_kb_(max_headers_kb), strict_header_validation_(Runtime::runtimeFeatureEnabled( - "envoy.reloadable_features.strict_header_validation")) { + max_request_headers_kb_(max_request_headers_kb), + strict_header_validation_( + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.strict_header_validation")) { output_buffer_.setWatermarks(connection.bufferLimit()); http_parser_init(&parser_, type); parser_.data = this; @@ -452,7 +453,7 @@ void ConnectionImpl::onHeaderValue(const char* data, size_t length) { const uint32_t total = current_header_field_.size() + current_header_value_.size() + current_header_map_->byteSize(); - if (total > (max_headers_kb_ * 1024)) { + if (total > (max_request_headers_kb_ * 1024)) { error_code_ = Http::Code::RequestHeaderFieldsTooLarge; sendProtocolError(); throw CodecProtocolException("headers size exceeds limit"); diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index bf6708fa8f47..be9def724076 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -300,7 +300,7 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable +#include "common/common/assert.h" + #include "absl/base/call_once.h" namespace Envoy { diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index b64e9ff59df1..925ff26368a4 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -652,7 +652,7 @@ class StatNameSet { void rememberBuiltin(absl::string_view str); /** - * Finds a StatName by name. If 'str' has been remembered as a built-in, then + * Finds a StatName by name. If 'token' has been remembered as a built-in, then * no lock is required. Otherwise we first consult dynamic_stat_names_ under a * lock that's private to the StatNameSet. If that's empty, we need to create * the StatName in the pool, which requires taking a global lock. @@ -661,7 +661,7 @@ class StatNameSet { * set's mutex and also the SymbolTable mutex which must be taken during * StatNamePool::add(). */ - StatName getStatName(absl::string_view str); + StatName getStatName(absl::string_view token); /** * Adds a StatName using the pool, but without remembering it in any maps. From 312564644e3b8ff32040b0f5d3d862ecce504da9 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 12 Aug 2019 14:58:14 -0400 Subject: [PATCH 367/542] docs: add names to layers in runtime (#7887) Signed-off-by: Asra Ali --- docs/root/configuration/runtime.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/root/configuration/runtime.rst b/docs/root/configuration/runtime.rst index 878b2b5a024e..ca1aa31a9481 100644 --- a/docs/root/configuration/runtime.rst +++ b/docs/root/configuration/runtime.rst @@ -27,12 +27,16 @@ be: .. code-block:: yaml layers: - - static_layer: + - name: static_layer_0 + static_layer: health_check: min_interval: 5 - - disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy } - - disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy_override, append_service_cluster: true } - - admin_layer: {} + - name: disk_layer_0 + disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy } + - name: disk_layer_1 + disk_layer: { symlink_root: /srv/runtime/current, subdirectory: envoy_override, append_service_cluster: true } + - name: admin_layer_0 + admin_layer: {} In the deprecated :ref:`runtime ` bootstrap configuration, the layering was implicit and fixed: From 3380a14ec15bae10c9e240cc507a63cc19d40453 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 12 Aug 2019 14:59:01 -0400 Subject: [PATCH 368/542] runtime: add proto constraint to require a layer specifier (#7886) Signed-off-by: Asra Ali --- api/envoy/config/bootstrap/v2/bootstrap.proto | 2 + test/server/BUILD | 1 + ...id_layered_runtime_no_layer_specifier.yaml | 3 + ...testcase-server_fuzz_test-5647989147697152 | 191 ++++++++++++++++++ test/server/server_test.cc | 6 + 5 files changed, 203 insertions(+) create mode 100644 test/server/invalid_layered_runtime_no_layer_specifier.yaml create mode 100644 test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5647989147697152 diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index 7f0ea3c97ca9..477aeac5a141 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -295,6 +295,8 @@ message RuntimeLayer { // This follows the :ref:`runtime protobuf JSON representation encoding // `. Unlike static xDS resources, this static // layer is overridable by later layers in the runtime virtual filesystem. + option (validate.required) = true; + google.protobuf.Struct static_layer = 2; DiskLayer disk_layer = 3; AdminLayer admin_layer = 4; diff --git a/test/server/BUILD b/test/server/BUILD index e65a4896459e..5a9c5d74323a 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -252,6 +252,7 @@ envoy_cc_test( ":invalid_bootstrap.yaml", ":invalid_layered_runtime_duplicate_name.yaml", ":invalid_layered_runtime_missing_name.yaml", + ":invalid_layered_runtime_no_layer_specifier.yaml", ":invalid_runtime_bootstrap.yaml", ":node_bootstrap.yaml", ":node_bootstrap_no_admin_port.yaml", diff --git a/test/server/invalid_layered_runtime_no_layer_specifier.yaml b/test/server/invalid_layered_runtime_no_layer_specifier.yaml new file mode 100644 index 000000000000..da4d5dd56bd2 --- /dev/null +++ b/test/server/invalid_layered_runtime_no_layer_specifier.yaml @@ -0,0 +1,3 @@ +layered_runtime: + layers: + - name: "foo" diff --git a/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5647989147697152 b/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5647989147697152 new file mode 100644 index 000000000000..79f4fba815b3 --- /dev/null +++ b/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5647989147697152 @@ -0,0 +1,191 @@ +static_resources { + listeners { + name: "$" + address { + pipe { + path: "." + } + } + filter_chains { + } + } + listeners { + address { + pipe { + path: "=" + } + } + filter_chains { + } + } + listeners { + name: "\000$&$`%n�NaN0" + address { + socket_address { + address: "0.0.1.0" + port_value: 32768 + resolver_name: "@q" + } + } + } + listeners { + address { + pipe { + path: "=" + } + } + filter_chains { + use_proxy_proto { + value: true + } + } + } + listeners { + address { + pipe { + path: "=" + } + } + filter_chains { + use_proxy_proto { + value: true + } + } + listener_filters_timeout { + } + } + listeners { + name: "@" + address { + socket_address { + address: "2147483648.0.1.0" + named_port: "L" + } + } + filter_chains { + } + } + listeners { + name: "$" + address { + pipe { + path: "." + } + } + filter_chains { + } + } + listeners { + name: "@" + address { + socket_address { + address: "0.0.1.0" + port_value: 32768 + } + } + filter_chains { + } + } + listeners { + name: "\000$&$`%n�NaN0" + address { + pipe { + path: "1" + } + } + filter_chains { + use_proxy_proto { + value: true + } + } + use_original_dst { + } + } + listeners { + address { + pipe { + path: "=" + } + } + filter_chains { + use_proxy_proto { + value: true + } + } + } + listeners { + name: "@" + address { + socket_address { + address: "@" + port_value: 32768 + } + } + filter_chains { + } + } + listeners { + address { + pipe { + path: "=" + } + } + filter_chains { + use_proxy_proto { + value: true + } + } + use_original_dst { + } + } + listeners { + name: "\000$&$`%n�NaN0" + address { + pipe { + path: "@" + } + } + filter_chains { + use_proxy_proto { + } + } + use_original_dst { + } + } + listeners { + name: "@" + address { + socket_address { + address: "0.0.1.0" + port_value: 32768 + } + } + filter_chains { + } + } + listeners { + address { + pipe { + path: "=" + } + } + filter_chains { + use_proxy_proto { + value: true + } + } + } +} +admin { + access_log_path: "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + address { + pipe { + path: "$" + } + } +} +layered_runtime { + layers { + name: "preserveExt" + } +} diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 9809dec3093c..2edb98e09540 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -530,6 +530,12 @@ TEST_P(ServerInstanceImplTest, InvalidLayeredBootstrapDuplicateName) { EnvoyException, "Duplicate layer name: some_static_laye"); } +// Validate invalid layered runtime with no layer specifier is rejected. +TEST_P(ServerInstanceImplTest, InvalidLayeredBootstrapNoLayerSpecifier) { + EXPECT_THROW_WITH_REGEX(initialize("test/server/invalid_layered_runtime_no_layer_specifier.yaml"), + EnvoyException, "BootstrapValidationError.LayeredRuntime"); +} + // Regression test for segfault when server initialization fails prior to // ClusterManager initialization. TEST_P(ServerInstanceImplTest, BootstrapClusterManagerInitializationFail) { From 5210894b130079432e0a9ba6e6fbe7fec5687f75 Mon Sep 17 00:00:00 2001 From: asraa Date: Mon, 12 Aug 2019 15:01:04 -0400 Subject: [PATCH 369/542] fuzz: add a simple fuzzer for xxhash utilities (#7855) Signed-off-by: Asra Ali --- test/common/common/BUILD | 7 +++++++ test/common/common/hash_corpus/example | 1 + test/common/common/hash_fuzz_test.cc | 29 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 test/common/common/hash_corpus/example create mode 100644 test/common/common/hash_fuzz_test.cc diff --git a/test/common/common/BUILD b/test/common/common/BUILD index d72fea4b6521..4714f1ff5c72 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -55,6 +55,13 @@ envoy_cc_fuzz_test( deps = ["//source/common/common:utility_lib"], ) +envoy_cc_fuzz_test( + name = "hash_fuzz_test", + srcs = ["hash_fuzz_test.cc"], + corpus = "hash_corpus", + deps = ["//source/common/common:hash_lib"], +) + envoy_cc_test( name = "cleanup_test", srcs = ["cleanup_test.cc"], diff --git a/test/common/common/hash_corpus/example b/test/common/common/hash_corpus/example new file mode 100644 index 000000000000..95d09f2b1015 --- /dev/null +++ b/test/common/common/hash_corpus/example @@ -0,0 +1 @@ +hello world \ No newline at end of file diff --git a/test/common/common/hash_fuzz_test.cc b/test/common/common/hash_fuzz_test.cc new file mode 100644 index 000000000000..a4b3f6c032a3 --- /dev/null +++ b/test/common/common/hash_fuzz_test.cc @@ -0,0 +1,29 @@ +#include "common/common/hash.h" + +#include "test/fuzz/fuzz_runner.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Fuzz { +namespace { + +DEFINE_FUZZER(const uint8_t* buf, size_t len) { + const std::string input(reinterpret_cast(buf), len); + { HashUtil::xxHash64(input); } + { HashUtil::djb2CaseInsensitiveHash(input); } + { MurmurHash::murmurHash2_64(input); } + if (len > 0) { + // Split the input string into two parts to make a key-value pair. + const size_t split_point = *reinterpret_cast(buf) % len; + const std::string key = input.substr(0, split_point); + const std::string value = input.substr(split_point); + StringMap map; + map[key] = value; + map.find(key); + } +} + +} // namespace +} // namespace Fuzz +} // namespace Envoy From c3a75316f2fa495fc7be36efd4f291445ac7b857 Mon Sep 17 00:00:00 2001 From: easy Date: Tue, 13 Aug 2019 05:02:10 +1000 Subject: [PATCH 370/542] tracing: Add B3 support in OpenCensus driver. (#7800) Signed-off-by: Emil Mikulic --- api/envoy/config/trace/v2/trace.proto | 3 + bazel/repositories.bzl | 4 + source/extensions/tracers/opencensus/BUILD | 1 + .../opencensus/opencensus_tracer_impl.cc | 55 +++++- .../tracers/opencensus/config_test.cc | 1 + .../tracers/opencensus/tracer_test.cc | 174 +++++++++++------- 6 files changed, 165 insertions(+), 73 deletions(-) diff --git a/api/envoy/config/trace/v2/trace.proto b/api/envoy/config/trace/v2/trace.proto index 34f11ec7e3e6..e4e4dec64e95 100644 --- a/api/envoy/config/trace/v2/trace.proto +++ b/api/envoy/config/trace/v2/trace.proto @@ -150,6 +150,9 @@ message OpenCensusConfig { // "X-Cloud-Trace-Context:" header. CLOUD_TRACE_CONTEXT = 3; + + // X-B3-* headers. + B3 = 4; } // List of incoming trace context headers we will accept. First one found diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 9cb20ae5f0a3..c0b2205e0ad6 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -528,6 +528,10 @@ def _io_opencensus_cpp(): name = "opencensus_trace", actual = "@io_opencensus_cpp//opencensus/trace", ) + native.bind( + name = "opencensus_trace_b3", + actual = "@io_opencensus_cpp//opencensus/trace:b3", + ) native.bind( name = "opencensus_trace_cloud_trace_context", actual = "@io_opencensus_cpp//opencensus/trace:cloud_trace_context", diff --git a/source/extensions/tracers/opencensus/BUILD b/source/extensions/tracers/opencensus/BUILD index 87b1854a2d2f..492648fdf088 100644 --- a/source/extensions/tracers/opencensus/BUILD +++ b/source/extensions/tracers/opencensus/BUILD @@ -28,6 +28,7 @@ envoy_cc_library( copts = ["-Wno-unused-parameter"], external_deps = [ "opencensus_trace", + "opencensus_trace_b3", "opencensus_trace_cloud_trace_context", "opencensus_trace_grpc_trace_bin", "opencensus_trace_trace_context", diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index eb9a52d7dbd2..9be9d3484868 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -11,6 +11,7 @@ #include "opencensus/exporters/trace/stackdriver/stackdriver_exporter.h" #include "opencensus/exporters/trace/stdout/stdout_exporter.h" #include "opencensus/exporters/trace/zipkin/zipkin_exporter.h" +#include "opencensus/trace/propagation/b3.h" #include "opencensus/trace/propagation/cloud_trace_context.h" #include "opencensus/trace/propagation/grpc_trace_bin.h" #include "opencensus/trace/propagation/trace_context.h" @@ -32,6 +33,10 @@ class ConstantValues { const Http::LowerCaseString TRACEPARENT{"traceparent"}; const Http::LowerCaseString GRPC_TRACE_BIN{"grpc-trace-bin"}; const Http::LowerCaseString X_CLOUD_TRACE_CONTEXT{"x-cloud-trace-context"}; + const Http::LowerCaseString X_B3_TRACEID{"x-b3-traceid"}; + const Http::LowerCaseString X_B3_SPANID{"x-b3-spanid"}; + const Http::LowerCaseString X_B3_SAMPLED{"x-b3-sampled"}; + const Http::LowerCaseString X_B3_FLAGS{"x-b3-flags"}; }; using Constants = ConstSingleton; @@ -101,6 +106,35 @@ startSpanHelper(const std::string& name, bool traced, const Http::HeaderMap& req } break; } + + case OpenCensusConfig::B3: { + absl::string_view b3_trace_id; + absl::string_view b3_span_id; + absl::string_view b3_sampled; + absl::string_view b3_flags; + const Http::HeaderEntry* h_b3_trace_id = request_headers.get(Constants::get().X_B3_TRACEID); + if (h_b3_trace_id != nullptr) { + b3_trace_id = h_b3_trace_id->value().getStringView(); + } + const Http::HeaderEntry* h_b3_span_id = request_headers.get(Constants::get().X_B3_SPANID); + if (h_b3_span_id != nullptr) { + b3_span_id = h_b3_span_id->value().getStringView(); + } + const Http::HeaderEntry* h_b3_sampled = request_headers.get(Constants::get().X_B3_SAMPLED); + if (h_b3_sampled != nullptr) { + b3_sampled = h_b3_sampled->value().getStringView(); + } + const Http::HeaderEntry* h_b3_flags = request_headers.get(Constants::get().X_B3_FLAGS); + if (h_b3_flags != nullptr) { + b3_flags = h_b3_flags->value().getStringView(); + } + if (h_b3_trace_id != nullptr && h_b3_span_id != nullptr) { + found = true; + parent_ctx = ::opencensus::trace::propagation::FromB3Headers(b3_trace_id, b3_span_id, + b3_sampled, b3_flags); + } + break; + } } // First header found wins. if (found) { @@ -153,16 +187,16 @@ void Span::finishSpan() { span_.End(); } void Span::injectContext(Http::HeaderMap& request_headers) { using OpenCensusConfig = envoy::config::trace::v2::OpenCensusConfig; + const auto& ctx = span_.context(); for (const auto& outgoing : oc_config_.outgoing_trace_context()) { switch (outgoing) { case OpenCensusConfig::TRACE_CONTEXT: - request_headers.setReferenceKey( - Constants::get().TRACEPARENT, - ::opencensus::trace::propagation::ToTraceParentHeader(span_.context())); + request_headers.setReferenceKey(Constants::get().TRACEPARENT, + ::opencensus::trace::propagation::ToTraceParentHeader(ctx)); break; case OpenCensusConfig::GRPC_TRACE_BIN: { - std::string val = ::opencensus::trace::propagation::ToGrpcTraceBinHeader(span_.context()); + std::string val = ::opencensus::trace::propagation::ToGrpcTraceBinHeader(ctx); val = Base64::encode(val.data(), val.size(), /*add_padding=*/false); request_headers.setReferenceKey(Constants::get().GRPC_TRACE_BIN, val); break; @@ -171,7 +205,18 @@ void Span::injectContext(Http::HeaderMap& request_headers) { case OpenCensusConfig::CLOUD_TRACE_CONTEXT: request_headers.setReferenceKey( Constants::get().X_CLOUD_TRACE_CONTEXT, - ::opencensus::trace::propagation::ToCloudTraceContextHeader(span_.context())); + ::opencensus::trace::propagation::ToCloudTraceContextHeader(ctx)); + break; + + case OpenCensusConfig::B3: + request_headers.setReferenceKey(Constants::get().X_B3_TRACEID, + ::opencensus::trace::propagation::ToB3TraceIdHeader(ctx)); + request_headers.setReferenceKey(Constants::get().X_B3_SPANID, + ::opencensus::trace::propagation::ToB3SpanIdHeader(ctx)); + request_headers.setReferenceKey(Constants::get().X_B3_SAMPLED, + ::opencensus::trace::propagation::ToB3SampledHeader(ctx)); + // OpenCensus's trace context propagation doesn't produce the + // "X-B3-Flags:" header. break; } } diff --git a/test/extensions/tracers/opencensus/config_test.cc b/test/extensions/tracers/opencensus/config_test.cc index 1f549cd4afb1..686f3e5b5813 100644 --- a/test/extensions/tracers/opencensus/config_test.cc +++ b/test/extensions/tracers/opencensus/config_test.cc @@ -50,6 +50,7 @@ TEST(OpenCensusTracerConfigTest, OpenCensusHttpTracerWithTypedConfig) { stackdriver_project_id: test_project_id zipkin_exporter_enabled: true zipkin_url: http://127.0.0.1:9411/api/v2/spans + incoming_trace_context: b3 incoming_trace_context: trace_context incoming_trace_context: grpc_trace_bin incoming_trace_context: cloud_trace_context diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index 2c7cf5695fec..241f1d0a9a89 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -19,6 +19,7 @@ #include "gtest/gtest.h" #include "opencensus/trace/exporter/span_data.h" #include "opencensus/trace/exporter/span_exporter.h" +#include "opencensus/trace/propagation/b3.h" #include "opencensus/trace/propagation/cloud_trace_context.h" #include "opencensus/trace/propagation/grpc_trace_bin.h" #include "opencensus/trace/propagation/trace_context.h" @@ -161,78 +162,115 @@ TEST(OpenCensusTracerTest, Span) { } } -// Test that trace context propagation works. -TEST(OpenCensusTracerTest, PropagateTraceContext) { - registerSpanCatcher(); - // The test calls the helper with each kind of incoming context in turn. - auto helper = [](const std::string& header, const std::string& value) { - OpenCensusConfig oc_config; - NiceMock local_info; - oc_config.add_incoming_trace_context(OpenCensusConfig::NONE); - oc_config.add_incoming_trace_context(OpenCensusConfig::TRACE_CONTEXT); - oc_config.add_incoming_trace_context(OpenCensusConfig::GRPC_TRACE_BIN); - oc_config.add_incoming_trace_context(OpenCensusConfig::CLOUD_TRACE_CONTEXT); - oc_config.add_outgoing_trace_context(OpenCensusConfig::NONE); - oc_config.add_outgoing_trace_context(OpenCensusConfig::TRACE_CONTEXT); - oc_config.add_outgoing_trace_context(OpenCensusConfig::GRPC_TRACE_BIN); - oc_config.add_outgoing_trace_context(OpenCensusConfig::CLOUD_TRACE_CONTEXT); - std::unique_ptr driver(new OpenCensus::Driver(oc_config, local_info)); - NiceMock config; - Http::TestHeaderMapImpl request_headers{ - {":path", "/"}, - {":method", "GET"}, - {"x-request-id", "foo"}, - {header, value}, - }; - const std::string operation_name{"my_operation_2"}; - SystemTime start_time; - Http::TestHeaderMapImpl injected_headers; - { - Tracing::SpanPtr span = driver->startSpan(config, request_headers, operation_name, start_time, - {Tracing::Reason::Sampling, false}); - span->injectContext(injected_headers); - span->finishSpan(); - } +namespace { - // Retrieve SpanData from the OpenCensus trace exporter. - std::vector spans = getSpanCatcher()->catchSpans(); - ASSERT_EQ(1, spans.size()); - const auto& sd = spans[0]; - ENVOY_LOG_MISC(debug, "{}", sd.DebugString()); +using testing::PrintToString; - // Check contents. - EXPECT_TRUE(sd.has_remote_parent()); - EXPECT_EQ("6162636465666768", sd.parent_span_id().ToHex()); - EXPECT_EQ("404142434445464748494a4b4c4d4e4f", sd.context().trace_id().ToHex()); - EXPECT_TRUE(sd.context().trace_options().IsSampled()) - << "parent was sampled, child should be also"; - - // Check injectContext. - using Envoy::Http::LowerCaseString; - { - auto val = injected_headers.get(LowerCaseString("traceparent")); - ASSERT_NE(nullptr, val); - EXPECT_EQ(::opencensus::trace::propagation::ToTraceParentHeader(sd.context()), - val->value().getStringView()); - } - { - auto val = injected_headers.get(LowerCaseString("grpc-trace-bin")); - ASSERT_NE(nullptr, val); - std::string expected = ::opencensus::trace::propagation::ToGrpcTraceBinHeader(sd.context()); - expected = Base64::encode(expected.data(), expected.size(), /*add_padding=*/false); - EXPECT_EQ(expected, val->value().getStringView()); - } - { - auto val = injected_headers.get(LowerCaseString("x-cloud-trace-context")); - ASSERT_NE(nullptr, val); - EXPECT_EQ(::opencensus::trace::propagation::ToCloudTraceContextHeader(sd.context()), - val->value().getStringView()); - } +MATCHER_P2(ContainHeader, header, expected_value, + "contains the header " + PrintToString(header) + " with value " + + PrintToString(expected_value)) { + const auto found_value = arg.get(Http::LowerCaseString(header)); + if (found_value == nullptr) { + return false; + } + return found_value->value().getStringView() == expected_value; +} + +// Given incoming headers, test that trace context propagation works and generates all the expected +// outgoing headers. +void testIncomingHeaders( + const std::initializer_list>& headers) { + registerSpanCatcher(); + OpenCensusConfig oc_config; + NiceMock local_info; + oc_config.add_incoming_trace_context(OpenCensusConfig::NONE); + oc_config.add_incoming_trace_context(OpenCensusConfig::B3); + oc_config.add_incoming_trace_context(OpenCensusConfig::TRACE_CONTEXT); + oc_config.add_incoming_trace_context(OpenCensusConfig::GRPC_TRACE_BIN); + oc_config.add_incoming_trace_context(OpenCensusConfig::CLOUD_TRACE_CONTEXT); + oc_config.add_outgoing_trace_context(OpenCensusConfig::NONE); + oc_config.add_outgoing_trace_context(OpenCensusConfig::B3); + oc_config.add_outgoing_trace_context(OpenCensusConfig::TRACE_CONTEXT); + oc_config.add_outgoing_trace_context(OpenCensusConfig::GRPC_TRACE_BIN); + oc_config.add_outgoing_trace_context(OpenCensusConfig::CLOUD_TRACE_CONTEXT); + std::unique_ptr driver(new OpenCensus::Driver(oc_config, local_info)); + NiceMock config; + Http::TestHeaderMapImpl request_headers{ + {":path", "/"}, + {":method", "GET"}, + {"x-request-id", "foo"}, }; + for (const auto& kv : headers) { + request_headers.addCopy(Http::LowerCaseString(kv.first), kv.second); + } + + const std::string operation_name{"my_operation_2"}; + SystemTime start_time; + Http::TestHeaderMapImpl injected_headers; + { + Tracing::SpanPtr span = driver->startSpan(config, request_headers, operation_name, start_time, + {Tracing::Reason::Sampling, false}); + span->injectContext(injected_headers); + span->finishSpan(); + } + + // Retrieve SpanData from the OpenCensus trace exporter. + std::vector spans = getSpanCatcher()->catchSpans(); + ASSERT_EQ(1, spans.size()); + const auto& sd = spans[0]; + ENVOY_LOG_MISC(debug, "{}", sd.DebugString()); + + // Check contents. + EXPECT_TRUE(sd.has_remote_parent()); + EXPECT_EQ("6162636465666768", sd.parent_span_id().ToHex()); + EXPECT_EQ("404142434445464748494a4b4c4d4e4f", sd.context().trace_id().ToHex()); + EXPECT_TRUE(sd.context().trace_options().IsSampled()) + << "parent was sampled, child should be also"; + + // Check injectContext. + // The SpanID is unpredictable so re-serialize context to check it. + const auto& ctx = sd.context(); + const auto& hdrs = injected_headers; + EXPECT_THAT(hdrs, ContainHeader("traceparent", + ::opencensus::trace::propagation::ToTraceParentHeader(ctx))); + { + std::string expected = ::opencensus::trace::propagation::ToGrpcTraceBinHeader(ctx); + expected = Base64::encode(expected.data(), expected.size(), /*add_padding=*/false); + EXPECT_THAT(hdrs, ContainHeader("grpc-trace-bin", expected)); + } + EXPECT_THAT(hdrs, + ContainHeader("x-cloud-trace-context", + ::opencensus::trace::propagation::ToCloudTraceContextHeader(ctx))); + EXPECT_THAT(hdrs, ContainHeader("x-b3-traceid", "404142434445464748494a4b4c4d4e4f")); + EXPECT_THAT( + hdrs, ContainHeader("x-b3-spanid", ::opencensus::trace::propagation::ToB3SpanIdHeader(ctx))); + EXPECT_THAT(hdrs, ContainHeader("x-b3-sampled", "1")); +} +} // namespace + +TEST(OpenCensusTracerTest, PropagateTraceParentContext) { + testIncomingHeaders({{"traceparent", "00-404142434445464748494a4b4c4d4e4f-6162636465666768-01"}}); +} + +TEST(OpenCensusTracerTest, PropagateGrpcTraceBinContext) { + testIncomingHeaders({{"grpc-trace-bin", "AABAQUJDREVGR0hJSktMTU5PAWFiY2RlZmdoAgE"}}); +} + +TEST(OpenCensusTracerTest, PropagateCloudTraceContext) { + testIncomingHeaders( + {{"x-cloud-trace-context", "404142434445464748494a4b4c4d4e4f/7017280452245743464;o=1"}}); +} + +TEST(OpenCensusTracerTest, PropagateB3Context) { + testIncomingHeaders({{"x-b3-traceid", "404142434445464748494a4b4c4d4e4f"}, + {"x-b3-spanid", "6162636465666768"}, + {"x-b3-sampled", "1"}}); +} - helper("traceparent", "00-404142434445464748494a4b4c4d4e4f-6162636465666768-01"); - helper("grpc-trace-bin", "AABAQUJDREVGR0hJSktMTU5PAWFiY2RlZmdoAgE"); - helper("x-cloud-trace-context", "404142434445464748494a4b4c4d4e4f/7017280452245743464;o=1"); +TEST(OpenCensusTracerTest, PropagateB3ContextWithDebugFlag) { + testIncomingHeaders({{"x-b3-traceid", "404142434445464748494a4b4c4d4e4f"}, + {"x-b3-spanid", "6162636465666768"}, + {"x-b3-flags", "1"}}); // Debug flag causes sampling. } namespace { From e7fe19894d729315f364b0541f66e5848707978b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Mon, 12 Aug 2019 20:53:51 -0400 Subject: [PATCH 371/542] Register per opcode latencies (#7825) This change adds histograms for each opcode, e.g.: * getdata * setdata * ... Signed-off-by: Raul Gutierrez Segales --- docs/root/intro/version_history.rst | 1 + .../filters/network/zookeeper_proxy/config.cc | 6 ++- .../network/zookeeper_proxy/decoder.cc | 46 ++++++++++++------- .../filters/network/zookeeper_proxy/decoder.h | 22 ++++++--- .../filters/network/zookeeper_proxy/filter.cc | 28 ++++++++--- .../filters/network/zookeeper_proxy/filter.h | 11 +++-- .../filters/network/zookeeper_proxy/BUILD | 1 + .../network/zookeeper_proxy/filter_test.cc | 14 +++++- 8 files changed, 93 insertions(+), 36 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index bc597c1b8259..e9972311ec75 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -24,6 +24,7 @@ Version history * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. * upstream: added network filter chains to upstream connections, see :ref:`filters`. +* zookeeper: parse responses and emit latency stats. 1.11.0 (July 11, 2019) ====================== diff --git a/source/extensions/filters/network/zookeeper_proxy/config.cc b/source/extensions/filters/network/zookeeper_proxy/config.cc index b46bbde4fbf5..bc1fabe860cb 100644 --- a/source/extensions/filters/network/zookeeper_proxy/config.cc +++ b/source/extensions/filters/network/zookeeper_proxy/config.cc @@ -30,8 +30,10 @@ Network::FilterFactoryCb ZooKeeperConfigFactory::createFilterFactoryFromProtoTyp ZooKeeperFilterConfigSharedPtr filter_config( std::make_shared(stat_prefix, max_packet_bytes, context.scope())); - return [filter_config](Network::FilterManager& filter_manager) -> void { - filter_manager.addFilter(std::make_shared(filter_config)); + auto& time_source = context.dispatcher().timeSource(); + + return [filter_config, &time_source](Network::FilterManager& filter_manager) -> void { + filter_manager.addFilter(std::make_shared(filter_config, time_source)); }; } diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.cc b/source/extensions/filters/network/zookeeper_proxy/decoder.cc index 2350cbb8c941..3db82046a2d2 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.cc +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.cc @@ -51,6 +51,8 @@ void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) { ensureMinLength(len, INT_LENGTH + XID_LENGTH); ensureMaxLength(len); + auto start_time = time_source_.monotonicTime(); + // Control requests, with XIDs <= 0. // // These are meant to control the state of a session: @@ -65,17 +67,21 @@ void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) { switch (static_cast(xid)) { case XidCodes::CONNECT_XID: parseConnect(data, offset, len); + requests_by_xid_[xid] = {OpCodes::CONNECT, std::move(start_time)}; return; case XidCodes::PING_XID: offset += OPCODE_LENGTH; callbacks_.onPing(); + requests_by_xid_[xid] = {OpCodes::PING, std::move(start_time)}; return; case XidCodes::AUTH_XID: parseAuthRequest(data, offset, len); + requests_by_xid_[xid] = {OpCodes::SETAUTH, std::move(start_time)}; return; case XidCodes::SET_WATCHES_XID: offset += OPCODE_LENGTH; parseSetWatchesRequest(data, offset, len); + requests_by_xid_[xid] = {OpCodes::SETWATCHES, std::move(start_time)}; return; default: // WATCH_XID is generated by the server, so that and everything @@ -155,7 +161,7 @@ void DecoderImpl::decodeOnData(Buffer::Instance& data, uint64_t& offset) { throw EnvoyException(fmt::format("Unknown opcode: {}", enumToSignedInt(opcode))); } - requests_by_xid_[xid] = opcode; + requests_by_xid_[xid] = {opcode, std::move(start_time)}; } void DecoderImpl::decodeOnWrite(Buffer::Instance& data, uint64_t& offset) { @@ -170,11 +176,26 @@ void DecoderImpl::decodeOnWrite(Buffer::Instance& data, uint64_t& offset) { const auto xid = helper_.peekInt32(data, offset); const auto xid_code = static_cast(xid); + // Find the corresponding request for this XID. + const auto it = requests_by_xid_.find(xid); + + std::chrono::milliseconds latency; + OpCodes opcode; + + if (xid_code != XidCodes::WATCH_XID) { + // If this fails, it's a server-side bug. + ASSERT(it != requests_by_xid_.end()); + latency = std::chrono::duration_cast(time_source_.monotonicTime() - + it->second.start_time); + opcode = it->second.opcode; + requests_by_xid_.erase(it); + } + // Connect responses are special, they have no full reply header // but just an XID with no zxid nor error fields like the ones // available for all other server generated messages. if (xid_code == XidCodes::CONNECT_XID) { - parseConnectResponse(data, offset, len); + parseConnectResponse(data, offset, len, latency); return; } @@ -183,13 +204,13 @@ void DecoderImpl::decodeOnWrite(Buffer::Instance& data, uint64_t& offset) { const auto error = helper_.peekInt32(data, offset); switch (xid_code) { case XidCodes::PING_XID: - callbacks_.onResponse(OpCodes::PING, xid, zxid, error); + callbacks_.onResponse(OpCodes::PING, xid, zxid, error, latency); return; case XidCodes::AUTH_XID: - callbacks_.onResponse(OpCodes::SETAUTH, xid, zxid, error); + callbacks_.onResponse(OpCodes::SETAUTH, xid, zxid, error, latency); return; case XidCodes::SET_WATCHES_XID: - callbacks_.onResponse(OpCodes::SETWATCHES, xid, zxid, error); + callbacks_.onResponse(OpCodes::SETWATCHES, xid, zxid, error, latency); return; case XidCodes::WATCH_XID: parseWatchEvent(data, offset, len, zxid, error); @@ -198,16 +219,8 @@ void DecoderImpl::decodeOnWrite(Buffer::Instance& data, uint64_t& offset) { break; } - // Find the corresponding request for this XID. - const auto it = requests_by_xid_.find(xid); - - // If this fails, it's a server-side bug. - ASSERT(it != requests_by_xid_.end()); - - const auto opcode = it->second; - requests_by_xid_.erase(it); + callbacks_.onResponse(opcode, xid, zxid, error, latency); offset += (len - (XID_LENGTH + ZXID_LENGTH + INT_LENGTH)); - callbacks_.onResponse(opcode, xid, zxid, error); } void DecoderImpl::ensureMinLength(const int32_t len, const int32_t minlen) const { @@ -479,7 +492,8 @@ void DecoderImpl::decode(Buffer::Instance& data, DecodeType dtype) { } } -void DecoderImpl::parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len) { +void DecoderImpl::parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len, + const std::chrono::milliseconds& latency) { ensureMinLength(len, PROTOCOL_VERSION_LENGTH + TIMEOUT_LENGTH + SESSION_LENGTH + INT_LENGTH); const auto timeout = helper_.peekInt32(data, offset); @@ -490,7 +504,7 @@ void DecoderImpl::parseConnectResponse(Buffer::Instance& data, uint64_t& offset, const bool readonly = maybeReadBool(data, offset); - callbacks_.onConnectResponse(0, timeout, readonly); + callbacks_.onConnectResponse(0, timeout, readonly, latency); } void DecoderImpl::parseWatchEvent(Buffer::Instance& data, uint64_t& offset, const uint32_t len, diff --git a/source/extensions/filters/network/zookeeper_proxy/decoder.h b/source/extensions/filters/network/zookeeper_proxy/decoder.h index 7b03f7a13b8d..82fdca329648 100644 --- a/source/extensions/filters/network/zookeeper_proxy/decoder.h +++ b/source/extensions/filters/network/zookeeper_proxy/decoder.h @@ -96,8 +96,10 @@ class DecoderCallbacks { virtual void onRemoveWatchesRequest(const std::string& path, int32_t type) PURE; virtual void onCloseRequest() PURE; virtual void onResponseBytes(uint64_t bytes) PURE; - virtual void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly) PURE; - virtual void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error) PURE; + virtual void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly, + const std::chrono::milliseconds& latency) PURE; + virtual void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error, + const std::chrono::milliseconds& latency) PURE; virtual void onWatchEvent(int32_t event_type, int32_t client_state, const std::string& path, int64_t zxid, int32_t error) PURE; }; @@ -117,8 +119,10 @@ using DecoderPtr = std::unique_ptr; class DecoderImpl : public Decoder, Logger::Loggable { public: - explicit DecoderImpl(DecoderCallbacks& callbacks, uint32_t max_packet_bytes) - : callbacks_(callbacks), max_packet_bytes_(max_packet_bytes), helper_(max_packet_bytes) {} + explicit DecoderImpl(DecoderCallbacks& callbacks, uint32_t max_packet_bytes, + TimeSource& time_source) + : callbacks_(callbacks), max_packet_bytes_(max_packet_bytes), helper_(max_packet_bytes), + time_source_(time_source) {} // ZooKeeperProxy::Decoder void onData(Buffer::Instance& data) override; @@ -126,6 +130,10 @@ class DecoderImpl : public Decoder, Logger::Loggable { private: enum class DecodeType { READ, WRITE }; + struct RequestBegin { + OpCodes opcode; + MonotonicTime start_time; + }; void decode(Buffer::Instance& data, DecodeType dtype); void decodeOnData(Buffer::Instance& data, uint64_t& offset); @@ -151,7 +159,8 @@ class DecoderImpl : public Decoder, Logger::Loggable { void ensureMinLength(int32_t len, int32_t minlen) const; void ensureMaxLength(int32_t len) const; std::string pathOnlyRequest(Buffer::Instance& data, uint64_t& offset, uint32_t len); - void parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len); + void parseConnectResponse(Buffer::Instance& data, uint64_t& offset, uint32_t len, + const std::chrono::milliseconds& latency); void parseWatchEvent(Buffer::Instance& data, uint64_t& offset, uint32_t len, int64_t zxid, int32_t error); bool maybeReadBool(Buffer::Instance& data, uint64_t& offset); @@ -159,7 +168,8 @@ class DecoderImpl : public Decoder, Logger::Loggable { DecoderCallbacks& callbacks_; const uint32_t max_packet_bytes_; BufferHelper helper_; - std::unordered_map requests_by_xid_; + TimeSource& time_source_; + std::unordered_map requests_by_xid_; }; } // namespace ZooKeeperProxy diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.cc b/source/extensions/filters/network/zookeeper_proxy/filter.cc index 1bf5870d6914..c2f4d00dc8af 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/filter.cc @@ -20,7 +20,8 @@ ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, const uint32_t max_packet_bytes, Stats::Scope& scope) : scope_(scope), max_packet_bytes_(max_packet_bytes), stats_(generateStats(stat_prefix, scope)), stat_name_set_(scope.symbolTable()), stat_prefix_(stat_name_set_.add(stat_prefix)), - auth_(stat_name_set_.add("auth")) { + auth_(stat_name_set_.add("auth")), + connect_latency_(stat_name_set_.add("connect_response_latency")) { // https://zookeeper.apache.org/doc/r3.5.4-beta/zookeeperProgrammers.html#sc_BuiltinACLSchemes // lists commons schemes: "world", "auth", "digest", "host", "x509", and // "ip". These are used in filter.cc by appending "_rq". @@ -32,8 +33,8 @@ ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, stat_name_set_.rememberBuiltin("x509_rq"); } -ZooKeeperFilter::ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config) - : config_(std::move(config)), decoder_(createDecoder(*this)) {} +ZooKeeperFilter::ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config, TimeSource& time_source) + : config_(std::move(config)), decoder_(createDecoder(*this, time_source)) {} void ZooKeeperFilter::initializeReadFilterCallbacks(Network::ReadFilterCallbacks& callbacks) { read_callbacks_ = &callbacks; @@ -53,8 +54,8 @@ Network::FilterStatus ZooKeeperFilter::onWrite(Buffer::Instance& data, bool) { Network::FilterStatus ZooKeeperFilter::onNewConnection() { return Network::FilterStatus::Continue; } -DecoderPtr ZooKeeperFilter::createDecoder(DecoderCallbacks& callbacks) { - return std::make_unique(callbacks, config_->maxPacketBytes()); +DecoderPtr ZooKeeperFilter::createDecoder(DecoderCallbacks& callbacks, TimeSource& time_source) { + return std::make_unique(callbacks, config_->maxPacketBytes(), time_source); } void ZooKeeperFilter::setDynamicMetadata(const std::string& key, const std::string& value) { @@ -249,8 +250,15 @@ void ZooKeeperFilter::onCloseRequest() { } void ZooKeeperFilter::onConnectResponse(const int32_t proto_version, const int32_t timeout, - const bool readonly) { + const bool readonly, + const std::chrono::milliseconds& latency) { config_->stats_.connect_resp_.inc(); + + Stats::SymbolTable::StoragePtr storage = + config_->scope_.symbolTable().join({config_->stat_prefix_, config_->connect_latency_}); + config_->scope_.histogramFromStatName(Stats::StatName(storage.get())) + .recordValue(latency.count()); + setDynamicMetadata({{"opname", "connect_response"}, {"protocol_version", std::to_string(proto_version)}, {"timeout", std::to_string(timeout)}, @@ -258,7 +266,7 @@ void ZooKeeperFilter::onConnectResponse(const int32_t proto_version, const int32 } void ZooKeeperFilter::onResponse(const OpCodes opcode, const int32_t xid, const int64_t zxid, - const int32_t error) { + const int32_t error, const std::chrono::milliseconds& latency) { std::string opname = ""; switch (opcode) { @@ -362,6 +370,12 @@ void ZooKeeperFilter::onResponse(const OpCodes opcode, const int32_t xid, const break; } + Stats::SymbolTable::StoragePtr storage = config_->scope_.symbolTable().join( + {config_->stat_prefix_, + config_->stat_name_set_.getStatName(absl::StrCat(opname, "_latency"))}); + config_->scope_.histogramFromStatName(Stats::StatName(storage.get())) + .recordValue(latency.count()); + setDynamicMetadata({{"opname", opname}, {"xid", std::to_string(xid)}, {"zxid", std::to_string(zxid)}, diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.h b/source/extensions/filters/network/zookeeper_proxy/filter.h index c68303e8d4b2..7836b7a5b765 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/filter.h @@ -108,6 +108,7 @@ class ZooKeeperFilterConfig { Stats::StatNameSet stat_name_set_; const Stats::StatName stat_prefix_; const Stats::StatName auth_; + const Stats::StatName connect_latency_; private: ZooKeeperProxyStats generateStats(const std::string& prefix, Stats::Scope& scope) { @@ -124,7 +125,7 @@ class ZooKeeperFilter : public Network::Filter, DecoderCallbacks, Logger::Loggable { public: - explicit ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config); + ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config, TimeSource& time_source); // Network::ReadFilter Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override; @@ -159,12 +160,14 @@ class ZooKeeperFilter : public Network::Filter, void onGetAllChildrenNumberRequest(const std::string& path) override; void onCloseRequest() override; void onResponseBytes(uint64_t bytes) override; - void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly) override; - void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error) override; + void onConnectResponse(int32_t proto_version, int32_t timeout, bool readonly, + const std::chrono::milliseconds& latency) override; + void onResponse(OpCodes opcode, int32_t xid, int64_t zxid, int32_t error, + const std::chrono::milliseconds& latency) override; void onWatchEvent(int32_t event_type, int32_t client_state, const std::string& path, int64_t zxid, int32_t error) override; - DecoderPtr createDecoder(DecoderCallbacks& callbacks); + DecoderPtr createDecoder(DecoderCallbacks& callbacks, TimeSource& time_source); void setDynamicMetadata(const std::string& key, const std::string& value); void setDynamicMetadata(const std::vector>& data); void clearDynamicMetadata(); diff --git a/test/extensions/filters/network/zookeeper_proxy/BUILD b/test/extensions/filters/network/zookeeper_proxy/BUILD index dd96daf08d29..b841301d7dee 100644 --- a/test/extensions/filters/network/zookeeper_proxy/BUILD +++ b/test/extensions/filters/network/zookeeper_proxy/BUILD @@ -20,6 +20,7 @@ envoy_extension_cc_test( deps = [ "//source/extensions/filters/network/zookeeper_proxy:config", "//test/mocks/network:network_mocks", + "//test/test_common:simulated_time_system_lib", ], ) diff --git a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc index 5536bf8de056..a7a6ec46fc84 100644 --- a/test/extensions/filters/network/zookeeper_proxy/filter_test.cc +++ b/test/extensions/filters/network/zookeeper_proxy/filter_test.cc @@ -4,6 +4,7 @@ #include "extensions/filters/network/zookeeper_proxy/filter.h" #include "test/mocks/network/mocks.h" +#include "test/test_common/simulated_time_system.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -31,7 +32,7 @@ class ZooKeeperFilterTest : public testing::Test { void initialize() { config_ = std::make_shared(stat_prefix_, 1048576, scope_); - filter_ = std::make_unique(config_); + filter_ = std::make_unique(config_, time_system_); filter_->initializeReadFilterCallbacks(filter_callbacks_); } @@ -489,6 +490,14 @@ class ZooKeeperFilterTest : public testing::Test { EXPECT_EQ(1UL, stat.value()); EXPECT_EQ(20UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + const auto histogram_name = + fmt::format("test.zookeeper.{}_latency", metadata_values[0].find("opname")->second); + EXPECT_NE(absl::nullopt, findHistogram(histogram_name)); + } + + Stats::OptionalHistogram findHistogram(const std::string& name) { + Stats::StatNameManagedStorage storage(name, scope_.symbolTable()); + return scope_.findHistogram(storage.statName()); } Stats::IsolatedStoreImpl scope_; @@ -497,6 +506,7 @@ class ZooKeeperFilterTest : public testing::Test { std::string stat_prefix_{"test.zookeeper"}; NiceMock filter_callbacks_; NiceMock stream_info_; + Event::SimulatedTimeSystem time_system_; }; TEST_F(ZooKeeperFilterTest, Connect) { @@ -516,6 +526,7 @@ TEST_F(ZooKeeperFilterTest, Connect) { EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); EXPECT_EQ(24UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); } TEST_F(ZooKeeperFilterTest, ConnectReadonly) { @@ -536,6 +547,7 @@ TEST_F(ZooKeeperFilterTest, ConnectReadonly) { EXPECT_EQ(1UL, config_->stats().connect_resp_.value()); EXPECT_EQ(25UL, config_->stats().response_bytes_.value()); EXPECT_EQ(0UL, config_->stats().decoder_error_.value()); + EXPECT_NE(absl::nullopt, findHistogram("test.zookeeper.connect_response_latency")); } TEST_F(ZooKeeperFilterTest, Fallback) { From 7b0ce0d32a9b584626e8c16b5ae07817eade322d Mon Sep 17 00:00:00 2001 From: Michael Rebello Date: Mon, 12 Aug 2019 20:33:28 -0700 Subject: [PATCH 372/542] cluster: remove unused proto import (#7898) This warms when building: > envoy/api/v2/cluster/filter.proto:12:1: warning: Import google/protobuf/struct.proto but not used. Signed-off-by: Michael Rebello --- api/envoy/api/v2/cluster/filter.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/api/envoy/api/v2/cluster/filter.proto b/api/envoy/api/v2/cluster/filter.proto index 03a826976f09..8a287b399751 100644 --- a/api/envoy/api/v2/cluster/filter.proto +++ b/api/envoy/api/v2/cluster/filter.proto @@ -9,7 +9,6 @@ option csharp_namespace = "Envoy.Api.V2.ClusterNS"; option ruby_package = "Envoy.Api.V2.ClusterNS"; import "google/protobuf/any.proto"; -import "google/protobuf/struct.proto"; import "validate/validate.proto"; import "gogoproto/gogo.proto"; From b93886ca040795407efc641f8b41eaf35e7bf1bb Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 31 Jul 2019 00:41:26 -0400 Subject: [PATCH 373/542] http2: limit the number of outbound frames (#23) Limit the number of outbound (these, waiting to be written into the socket) HTTP/2 frames. When the limit is exceeded the connection is terminated. This mitigates flood exploits where a client continually sends frames that are not subject to flow control without reading server responses. Fixes CVE-2019-9512, CVE-2019-9514 and CVE-2019-9515. Signed-off-by: Yan Avlasov --- api/envoy/api/v2/core/protocol.proto | 15 ++ .../configuration/http_conn_man/stats.rst | 2 + docs/root/intro/version_history.rst | 6 + include/envoy/buffer/buffer.h | 4 +- include/envoy/http/codec.h | 7 + source/common/buffer/buffer_impl.h | 44 +++- source/common/http/conn_manager_impl.cc | 28 ++- source/common/http/conn_manager_impl.h | 1 + source/common/http/exception.h | 8 + source/common/http/http2/codec_impl.cc | 122 +++++++++- source/common/http/http2/codec_impl.h | 78 ++++++- source/common/http/utility.cc | 5 + source/common/network/connection_impl.h | 2 + test/common/buffer/buffer_test.cc | 16 ++ test/common/buffer/owned_impl_test.cc | 51 +++++ test/common/http/conn_manager_impl_test.cc | 21 ++ test/common/http/http2/BUILD | 10 + test/common/http/http2/codec_impl_test.cc | 210 ++++++++++++++++++ test/common/http/http2/codec_impl_test_util.h | 1 + test/common/http/http2/http2_frame.cc | 148 ++++++++++++ test/common/http/http2/http2_frame.h | 126 +++++++++++ test/common/http/utility_test.cc | 3 + test/config/utility.cc | 15 ++ test/config/utility.h | 3 + test/integration/BUILD | 1 + test/integration/http2_integration_test.cc | 179 +++++++++++++++ test/integration/http2_integration_test.h | 21 ++ test/integration/integration.cc | 9 + test/integration/integration.h | 5 +- .../integration/tcp_proxy_integration_test.cc | 9 + test/integration/utility.cc | 5 + test/integration/utility.h | 9 +- test/test_common/utility.cc | 2 + 33 files changed, 1145 insertions(+), 21 deletions(-) create mode 100644 test/common/http/http2/http2_frame.cc create mode 100644 test/common/http/http2/http2_frame.h diff --git a/api/envoy/api/v2/core/protocol.proto b/api/envoy/api/v2/core/protocol.proto index 200b8517abd1..becd596a0e98 100644 --- a/api/envoy/api/v2/core/protocol.proto +++ b/api/envoy/api/v2/core/protocol.proto @@ -91,6 +91,21 @@ message Http2ProtocolOptions { // docs](https://github.com/envoyproxy/envoy/blob/master/source/docs/h2_metadata.md) for more // information. bool allow_metadata = 6; + + // Limit the number of pending outbound downstream frames of all types (frames that are waiting to + // be written into the socket). Exceeding this limit triggers flood mitigation and connection is + // terminated. The "http2.outbound_flood" stat tracks the number of terminated connections due to + // flood mitigation. The default limit is 10000. + // [#comment:TODO: implement same limits for upstream outbound frames as well.] + google.protobuf.UInt32Value max_outbound_frames = 7 [(validate.rules).uint32 = {gte: 1}]; + + // Limit the number of pending outbound downstream frames of types PING, SETTINGS and RST_STREAM, + // preventing high memory utilization when receiving continuous stream of these frames. Exceeding + // this limit triggers flood mitigation and connection is terminated. The + // "http2.outbound_control_flood" stat tracks the number of terminated connections due to flood + // mitigation. The default limit is 1000. + // [#comment:TODO: implement same limits for upstream outbound frames as well.] + google.protobuf.UInt32Value max_outbound_control_frames = 8 [(validate.rules).uint32 = {gte: 1}]; } // [#not-implemented-hide:] diff --git a/docs/root/configuration/http_conn_man/stats.rst b/docs/root/configuration/http_conn_man/stats.rst index 4188dcea4a5c..c998145f42d9 100644 --- a/docs/root/configuration/http_conn_man/stats.rst +++ b/docs/root/configuration/http_conn_man/stats.rst @@ -122,6 +122,8 @@ All http2 statistics are rooted at *http2.* header_overflow, Counter, Total number of connections reset due to the headers being larger than the :ref:`configured value `. headers_cb_no_stream, Counter, Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug + outbound_flood, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting `. + outbound_control_flood, Counter, "Total number of connections terminated for exceeding the limit on outbound frames of types PING, SETTINGS and RST_STREAM. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `." rx_messaging_error, Counter, Total number of invalid received frames that violated `section 8 `_ of the HTTP/2 spec. This will result in a *tx_reset* rx_reset, Counter, Total number of reset stream frames received by Envoy too_many_header_frames, Counter, Total number of times an HTTP2 connection is reset due to receiving too many headers frames. Envoy currently supports proxying at most one header frame for 100-Continue one non-100 response code header frame and one frame with trailers diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index e9972311ec75..599753b85be8 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -26,6 +26,12 @@ Version history * upstream: added network filter chains to upstream connections, see :ref:`filters`. * zookeeper: parse responses and emit latency stats. +1.11.1 (August 13, 2019) +======================== +* http: added mitigation of client initiated atacks that result in flooding of the outbound queue of downstream HTTP/2 connections. +* http: added :ref:`outbound_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting ` +* http: added :ref:`outbound_control_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `. + 1.11.0 (July 11, 2019) ====================== * access log: added a new field for downstream TLS session ID to file and gRPC access logger. diff --git a/include/envoy/buffer/buffer.h b/include/envoy/buffer/buffer.h index 1f19d94d6601..468cf17fa967 100644 --- a/include/envoy/buffer/buffer.h +++ b/include/envoy/buffer/buffer.h @@ -33,6 +33,7 @@ struct RawSlice { */ class BufferFragment { public: + virtual ~BufferFragment() = default; /** * @return const void* a pointer to the referenced data. */ @@ -47,9 +48,6 @@ class BufferFragment { * Called by a buffer when the referenced data is no longer needed. */ virtual void done() PURE; - -protected: - virtual ~BufferFragment() = default; }; /** diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 4ac313f7d6f3..d05943c0f955 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -235,6 +235,8 @@ struct Http2Settings { uint32_t initial_connection_window_size_{DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE}; bool allow_connect_{DEFAULT_ALLOW_CONNECT}; bool allow_metadata_{DEFAULT_ALLOW_METADATA}; + uint32_t max_outbound_frames_{DEFAULT_MAX_OUTBOUND_FRAMES}; + uint32_t max_outbound_control_frames_{DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES}; // disable HPACK compression static const uint32_t MIN_HPACK_TABLE_SIZE = 0; @@ -272,6 +274,11 @@ struct Http2Settings { static const bool DEFAULT_ALLOW_CONNECT = false; // By default Envoy does not allow METADATA support. static const bool DEFAULT_ALLOW_METADATA = false; + + // Default limit on the number of outbound frames of all types. + static const uint32_t DEFAULT_MAX_OUTBOUND_FRAMES = 10000; + // Default limit on the number of outbound frames of types PING, SETTINGS and RST_STREAM. + static const uint32_t DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES = 1000; }; /** diff --git a/source/common/buffer/buffer_impl.h b/source/common/buffer/buffer_impl.h index 9851686b4fbb..68b361c579d5 100644 --- a/source/common/buffer/buffer_impl.h +++ b/source/common/buffer/buffer_impl.h @@ -193,7 +193,8 @@ class Slice { using SlicePtr = std::unique_ptr; -class OwnedSlice : public Slice, public InlineStorage { +// OwnedSlice can not be derived from as it has variable sized array as member. +class OwnedSlice final : public Slice, public InlineStorage { public: /** * Create an empty OwnedSlice. @@ -563,5 +564,46 @@ class OwnedImpl : public LibEventInstance { Event::Libevent::BufferPtr buffer_; }; +using BufferFragmentPtr = std::unique_ptr; + +/** + * An implementation of BufferFragment where a releasor callback is called when the data is + * no longer needed. Copies data into internal buffer. + */ +class OwnedBufferFragmentImpl final : public BufferFragment, public InlineStorage { +public: + using Releasor = std::function; + + /** + * Copies the data into internal buffer. The releasor is called when the data has been + * fully drained or the buffer that contains this fragment is destroyed. + * @param data external data to reference + * @param releasor a callback function to be called when data is no longer needed. + */ + + static BufferFragmentPtr create(absl::string_view data, const Releasor& releasor) { + return BufferFragmentPtr(new (sizeof(OwnedBufferFragmentImpl) + data.size()) + OwnedBufferFragmentImpl(data, releasor)); + } + + // Buffer::BufferFragment + const void* data() const override { return data_; } + size_t size() const override { return size_; } + void done() override { releasor_(this); } + +private: + OwnedBufferFragmentImpl(absl::string_view data, const Releasor& releasor) + : releasor_(releasor), size_(data.size()) { + ASSERT(releasor != nullptr); + memcpy(data_, data.data(), data.size()); + } + + const Releasor releasor_; + const size_t size_; + uint8_t data_[]; +}; + +using OwnedBufferFragmentImplPtr = std::unique_ptr; + } // namespace Buffer } // namespace Envoy diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 8229c9ce9f37..e461a40ea361 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -258,6 +258,19 @@ StreamDecoder& ConnectionManagerImpl::newStream(StreamEncoder& response_encoder, return **streams_.begin(); } +void ConnectionManagerImpl::handleCodecException(const char* error, + Network::ConnectionCloseType close_type) { + ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), error); + + // In the protocol error case, we need to reset all streams now. If the close_type is + // FlushWriteAndDelay, the connection might stick around long enough for a pending stream to come + // back and try to encode. In other cases it avoids needless processing of upstream responses when + // downstream connection is closed. + resetAllStreams(); + + read_callbacks_->connection().close(close_type); +} + Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) { if (!codec_) { codec_ = config_.createCodec(read_callbacks_->connection(), data, *this); @@ -276,18 +289,15 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool try { codec_->dispatch(data); + } catch (const FrameFloodException& e) { + // Abortively close flooded connections + handleCodecException(e.what(), Network::ConnectionCloseType::NoFlush); + return Network::FilterStatus::StopIteration; } catch (const CodecProtocolException& e) { + stats_.named_.downstream_cx_protocol_error_.inc(); // HTTP/1.1 codec has already sent a 400 response if possible. HTTP/2 codec has already sent // GOAWAY. - ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), e.what()); - stats_.named_.downstream_cx_protocol_error_.inc(); - - // In the protocol error case, we need to reset all streams now. Since we do a flush write and - // delayed close, the connection might stick around long enough for a pending stream to come - // back and try to encode. - resetAllStreams(); - - read_callbacks_->connection().close(Network::ConnectionCloseType::FlushWriteAndDelay); + handleCodecException(e.what(), Network::ConnectionCloseType::FlushWriteAndDelay); return Network::FilterStatus::StopIteration; } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 1b99c6ad6c98..3091f2d98c9a 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -653,6 +653,7 @@ class ConnectionManagerImpl : Logger::Loggable, void onDrainTimeout(); void startDrainSequence(); Tracing::HttpTracer& tracer() { return http_context_.tracer(); } + void handleCodecException(const char* error, Network::ConnectionCloseType close_type); enum class DrainState { NotDraining, Draining, Closing }; diff --git a/source/common/http/exception.h b/source/common/http/exception.h index 445ef5fb1407..1a7ef668b24b 100644 --- a/source/common/http/exception.h +++ b/source/common/http/exception.h @@ -16,6 +16,14 @@ class CodecProtocolException : public EnvoyException { CodecProtocolException(const std::string& message) : EnvoyException(message) {} }; +/** + * Raised when outbound frame queue flood is detected. + */ +class FrameFloodException : public CodecProtocolException { +public: + FrameFloodException(const std::string& message) : CodecProtocolException(message) {} +}; + /** * Raised when a response is received on a connection that did not send a request. In practice * this can only happen on HTTP/1.1 connections. diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 475aa1d314e2..fe7518c19cff 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -11,6 +11,7 @@ #include "envoy/stats/scope.h" #include "common/common/assert.h" +#include "common/common/cleanup.h" #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/stack_array.h" @@ -251,7 +252,13 @@ int ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t // https://nghttp2.org/documentation/types.html#c.nghttp2_send_data_callback static const uint64_t FRAME_HEADER_SIZE = 9; - Buffer::OwnedImpl output(framehd, FRAME_HEADER_SIZE); + Buffer::OwnedImpl output; + if (!parent_.addOutboundFrameFragment(output, framehd, FRAME_HEADER_SIZE)) { + ENVOY_CONN_LOG(debug, "error sending data frame: Too many frames in the outbound queue", + parent_.connection_); + return NGHTTP2_ERR_FLOODED; + } + output.move(pending_send_data_, length); parent_.connection_.write(output, false); return 0; @@ -348,6 +355,10 @@ void ConnectionImpl::dispatch(Buffer::Instance& data) { dispatching_ = true; ssize_t rc = nghttp2_session_mem_recv(session_, static_cast(slice.mem_), slice.len_); + if (rc == NGHTTP2_ERR_FLOODED) { + throw FrameFloodException( + "Flooding was detected in this HTTP/2 session, and it must be closed"); + } if (rc != static_cast(slice.len_)) { throw CodecProtocolException(fmt::format("{}", nghttp2_strerror(rc))); } @@ -555,9 +566,77 @@ int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) { return NGHTTP2_ERR_CALLBACK_FAILURE; } +int ConnectionImpl::onBeforeFrameSend(const nghttp2_frame* frame) { + ENVOY_CONN_LOG(trace, "about to sent frame type={}, flags={}", connection_, + static_cast(frame->hd.type), static_cast(frame->hd.flags)); + ASSERT(!is_outbound_flood_monitored_control_frame_); + // Flag flood monitored outbound control frames. + is_outbound_flood_monitored_control_frame_ = + ((frame->hd.type == NGHTTP2_PING || frame->hd.type == NGHTTP2_SETTINGS) && + frame->hd.flags & NGHTTP2_FLAG_ACK) || + frame->hd.type == NGHTTP2_RST_STREAM; + return 0; +} + +void ConnectionImpl::incrementOutboundFrameCount(bool is_outbound_flood_monitored_control_frame) { + ++outbound_frames_; + if (is_outbound_flood_monitored_control_frame) { + ++outbound_control_frames_; + } + checkOutboundQueueLimits(); +} + +bool ConnectionImpl::addOutboundFrameFragment(Buffer::OwnedImpl& output, const uint8_t* data, + size_t length) { + // Reset the outbound frame type (set in the onBeforeFrameSend callback) since the + // onBeforeFrameSend callback is not called for DATA frames. + bool is_outbound_flood_monitored_control_frame = false; + std::swap(is_outbound_flood_monitored_control_frame, is_outbound_flood_monitored_control_frame_); + try { + incrementOutboundFrameCount(is_outbound_flood_monitored_control_frame); + } catch (const FrameFloodException&) { + return false; + } + + auto fragment = Buffer::OwnedBufferFragmentImpl::create( + absl::string_view(reinterpret_cast(data), length), + is_outbound_flood_monitored_control_frame ? control_frame_buffer_releasor_ + : frame_buffer_releasor_); + + // The Buffer::OwnedBufferFragmentImpl object will be deleted in the *frame_buffer_releasor_ + // callback. + output.addBufferFragment(*fragment.release()); + return true; +} + +void ConnectionImpl::releaseOutboundFrame(const Buffer::OwnedBufferFragmentImpl* fragment) { + ASSERT(outbound_frames_ >= 1); + --outbound_frames_; + delete fragment; +} + +void ConnectionImpl::releaseOutboundControlFrame(const Buffer::OwnedBufferFragmentImpl* fragment) { + ASSERT(outbound_control_frames_ >= 1); + --outbound_control_frames_; + releaseOutboundFrame(fragment); +} + ssize_t ConnectionImpl::onSend(const uint8_t* data, size_t length) { ENVOY_CONN_LOG(trace, "send data: bytes={}", connection_, length); - Buffer::OwnedImpl buffer(data, length); + Buffer::OwnedImpl buffer; + if (!addOutboundFrameFragment(buffer, data, length)) { + ENVOY_CONN_LOG(debug, "error sending frame: Too many frames in the outbound queue.", + connection_); + return NGHTTP2_ERR_FLOODED; + } + + // While the buffer is transient the fragment it contains will be moved into the + // write_buffer_ of the underlying connection_ by the write method below. + // This creates lifetime dependency between the write_buffer_ of the underlying connection + // and the codec object. Specifically the write_buffer_ MUST be either fully drained or + // deleted before the codec object is deleted. This is presently guaranteed by the + // destruction order of the Network::ConnectionImpl object where write_buffer_ is + // destroyed before the filter_manager_ which owns the codec through Http::ConnectionManagerImpl. connection_.write(buffer, false); return length; } @@ -663,6 +742,15 @@ void ConnectionImpl::sendPendingFrames() { int rc = nghttp2_session_send(session_); if (rc != 0) { ASSERT(rc == NGHTTP2_ERR_CALLBACK_FAILURE); + // For errors caused by the pending outbound frame flood the FrameFloodException has + // to be thrown. However the nghttp2 library returns only the generic error code for + // all failure types. Check queue limits and throw FrameFloodException if they were + // exceeded. + if (outbound_frames_ > max_outbound_frames_ || + outbound_control_frames_ > max_outbound_control_frames_) { + throw FrameFloodException("Too many frames in the outbound queue."); + } + throw CodecProtocolException(fmt::format("{}", nghttp2_strerror(rc))); } @@ -810,6 +898,11 @@ ConnectionImpl::Http2Callbacks::Http2Callbacks() { return static_cast(user_data)->onFrameSend(frame); }); + nghttp2_session_callbacks_set_before_frame_send_callback( + callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int { + return static_cast(user_data)->onBeforeFrameSend(frame); + }); + nghttp2_session_callbacks_set_on_frame_not_send_callback( callbacks_, [](nghttp2_session*, const nghttp2_frame*, int, void*) -> int { // We used to always return failure here but it looks now this can get called if the other @@ -979,6 +1072,31 @@ int ServerConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na return saveHeader(frame, std::move(name), std::move(value)); } +void ServerConnectionImpl::checkOutboundQueueLimits() { + if (outbound_frames_ > max_outbound_frames_ && dispatching_downstream_data_) { + stats_.outbound_flood_.inc(); + throw FrameFloodException("Too many frames in the outbound queue."); + } + if (outbound_control_frames_ > max_outbound_control_frames_ && dispatching_downstream_data_) { + stats_.outbound_control_flood_.inc(); + throw FrameFloodException("Too many control frames in the outbound queue."); + } +} + +void ServerConnectionImpl::dispatch(Buffer::Instance& data) { + ASSERT(!dispatching_downstream_data_); + dispatching_downstream_data_ = true; + + // Make sure the dispatching_downstream_data_ is set to false even + // when ConnectionImpl::dispatch throws an exception. + Cleanup cleanup([this]() { dispatching_downstream_data_ = false; }); + + // Make sure downstream outbound queue was not flooded by the upstream frames. + checkOutboundQueueLimits(); + + ConnectionImpl::dispatch(data); +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index c8df6ee4499d..5c1b1fd8670d 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -37,16 +38,16 @@ const std::string CLIENT_MAGIC_PREFIX = "PRI * HTTP/2"; /** * All stats for the HTTP/2 codec. @see stats_macros.h */ -// clang-format off #define ALL_HTTP2_CODEC_STATS(COUNTER) \ COUNTER(header_overflow) \ COUNTER(headers_cb_no_stream) \ + COUNTER(outbound_control_flood) \ + COUNTER(outbound_flood) \ COUNTER(rx_messaging_error) \ COUNTER(rx_reset) \ COUNTER(too_many_header_frames) \ COUNTER(trailers) \ COUNTER(tx_reset) -// clang-format on /** * Wrapper struct for the HTTP/2 codec stats. @see stats_macros.h @@ -77,12 +78,21 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable(*fragment); + EXPECT_EQ(11, slice->dataSize()); + EXPECT_EQ(0, slice->reservableSize()); + EXPECT_EQ(0, memcmp(slice->data(), input, slice->dataSize())); + EXPECT_FALSE(release_callback_called); + slice.reset(nullptr); + EXPECT_TRUE(release_callback_called); +} + TEST(SliceDequeTest, CreateDelete) { bool slice1_deleted = false; bool slice2_deleted = false; diff --git a/test/common/buffer/owned_impl_test.cc b/test/common/buffer/owned_impl_test.cc index d009db570f8d..282ae1961cf5 100644 --- a/test/common/buffer/owned_impl_test.cc +++ b/test/common/buffer/owned_impl_test.cc @@ -96,6 +96,57 @@ TEST_P(OwnedImplTest, AddBufferFragmentDynamicAllocation) { EXPECT_TRUE(release_callback_called_); } +TEST_P(OwnedImplTest, AddOwnedBufferFragmentWithCleanup) { + char input[] = "hello world"; + const size_t expected_length = sizeof(input) - 1; + auto frag = OwnedBufferFragmentImpl::create( + {input, expected_length}, + [this](const OwnedBufferFragmentImpl*) { release_callback_called_ = true; }); + Buffer::OwnedImpl buffer; + verifyImplementation(buffer); + buffer.addBufferFragment(*frag); + EXPECT_EQ(expected_length, buffer.length()); + + const uint64_t partial_drain_size = 5; + buffer.drain(partial_drain_size); + EXPECT_EQ(expected_length - partial_drain_size, buffer.length()); + EXPECT_FALSE(release_callback_called_); + + buffer.drain(expected_length - partial_drain_size); + EXPECT_EQ(0, buffer.length()); + EXPECT_TRUE(release_callback_called_); +} + +// Verify that OwnedBufferFragment work correctly when input buffer is allocated on the heap. +TEST_P(OwnedImplTest, AddOwnedBufferFragmentDynamicAllocation) { + char input_stack[] = "hello world"; + const size_t expected_length = sizeof(input_stack) - 1; + char* input = new char[expected_length]; + std::copy(input_stack, input_stack + expected_length, input); + + auto* frag = OwnedBufferFragmentImpl::create({input, expected_length}, + [this, input](const OwnedBufferFragmentImpl* frag) { + release_callback_called_ = true; + delete[] input; + delete frag; + }) + .release(); + + Buffer::OwnedImpl buffer; + verifyImplementation(buffer); + buffer.addBufferFragment(*frag); + EXPECT_EQ(expected_length, buffer.length()); + + const uint64_t partial_drain_size = 5; + buffer.drain(partial_drain_size); + EXPECT_EQ(expected_length - partial_drain_size, buffer.length()); + EXPECT_FALSE(release_callback_called_); + + buffer.drain(expected_length - partial_drain_size); + EXPECT_EQ(0, buffer.length()); + EXPECT_TRUE(release_callback_called_); +} + TEST_P(OwnedImplTest, Add) { const std::string string1 = "Hello, ", string2 = "World!"; Buffer::OwnedImpl buffer; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 9c84dfc14ef6..1ea7083c5794 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2316,6 +2316,27 @@ TEST_F(HttpConnectionManagerImplTest, DownstreamProtocolError) { conn_manager_->onData(fake_input, false); } +// Verify that FrameFloodException causes connection to be closed abortively. +TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { + InSequence s; + setup(false, ""); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + conn_manager_->newStream(response_encoder_); + throw FrameFloodException("too many outbound frames."); + })); + + EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)); + EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); + + // FrameFloodException should result in reset of the streams followed by abortive close. + EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + TEST_F(HttpConnectionManagerImplTest, IdleTimeoutNoCodec) { // Not used in the test. delete codec_; diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index 0c245388afb9..354055602126 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -32,6 +32,7 @@ envoy_cc_test( envoy_cc_test_library( name = "codec_impl_test_util", hdrs = ["codec_impl_test_util.h"], + external_deps = ["abseil_optional"], deps = [ "//source/common/http/http2:codec_lib", ], @@ -56,6 +57,15 @@ envoy_cc_test( ], ) +envoy_cc_test_library( + name = "http2_frame", + srcs = ["http2_frame.cc"], + hdrs = ["http2_frame.h"], + deps = [ + "//source/common/common:macros", + ], +) + envoy_cc_test_library( name = "frame_replay_lib", srcs = ["frame_replay.cc"], diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 28b4732f3bd0..bf7010767048 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -97,6 +97,8 @@ class Http2CodecImplTestFixture { setting.initial_stream_window_size_ = ::testing::get<2>(tp); setting.initial_connection_window_size_ = ::testing::get<3>(tp); setting.allow_metadata_ = allow_metadata_; + setting.max_outbound_frames_ = max_outbound_frames_; + setting.max_outbound_control_frames_ = max_outbound_control_frames_; } // corruptMetadataFramePayload assumes data contains at least 10 bytes of the beginning of a @@ -141,6 +143,8 @@ class Http2CodecImplTestFixture { bool corrupt_metadata_frame_ = false; uint32_t max_request_headers_kb_ = Http::DEFAULT_MAX_REQUEST_HEADERS_KB; + uint32_t max_outbound_frames_ = Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; + uint32_t max_outbound_control_frames_ = Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES; }; class Http2CodecImplTest : public ::testing::TestWithParam, @@ -1039,6 +1043,212 @@ TEST_P(Http2CodecImplTestAll, TestCodecHeaderCompression) { } } +// Verify that codec detects PING flood +TEST_P(Http2CodecImplTest, PingFlood) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + // Send one frame above the outbound control queue size limit + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES + 1; ++i) { + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + } + + int ack_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &ack_count](Buffer::Instance& frame, bool) { + ++ack_count; + buffer.move(frame); + })); + + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); + EXPECT_EQ(ack_count, Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES); + EXPECT_EQ(1, stats_store_.counter("http2.outbound_control_flood").value()); +} + +// Verify that outbound control frame counter decreases when send buffer is drained +TEST_P(Http2CodecImplTest, PingFloodCounterReset) { + static const int kMaxOutboundControlFrames = 100; + max_outbound_control_frames_ = kMaxOutboundControlFrames; + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + for (int i = 0; i < kMaxOutboundControlFrames; ++i) { + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + } + + int ack_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &ack_count](Buffer::Instance& frame, bool) { + ++ack_count; + buffer.move(frame); + })); + + // We should be 1 frame under the control frame flood mitigation threshold. + EXPECT_NO_THROW(client_->sendPendingFrames()); + EXPECT_EQ(ack_count, kMaxOutboundControlFrames); + + // Drain kMaxOutboundFrames / 2 slices from the send buffer + buffer.drain(buffer.length() / 2); + + // Send kMaxOutboundFrames / 2 more pings. + for (int i = 0; i < kMaxOutboundControlFrames / 2; ++i) { + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + } + // The number of outbound frames should be half of max so the connection should not be terminated. + EXPECT_NO_THROW(client_->sendPendingFrames()); + + // 1 more ping frame should overflow the outbound frame limit. + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); +} + +// Verify that codec detects flood of outbound HEADER frames +TEST_P(Http2CodecImplTest, ResponseHeadersFlood) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + int frame_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &frame_count](Buffer::Instance& frame, bool) { + ++frame_count; + buffer.move(frame); + })); + + TestHeaderMapImpl response_headers{{":status", "200"}}; + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 1; ++i) { + EXPECT_NO_THROW(response_encoder_->encodeHeaders(response_headers, false)); + } + // Presently flood mitigation is done only when processing downstream data + // So we need to send stream from downstream client to trigger mitigation + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); + + EXPECT_EQ(frame_count, Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 1); + EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); +} + +// Verify that codec detects flood of outbound DATA frames +TEST_P(Http2CodecImplTest, ResponseDataFlood) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + int frame_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &frame_count](Buffer::Instance& frame, bool) { + ++frame_count; + buffer.move(frame); + })); + + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + // Account for the single HEADERS frame above + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + // Presently flood mitigation is done only when processing downstream data + // So we need to send stream from downstream client to trigger mitigation + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); + + EXPECT_EQ(frame_count, Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 1); + EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); +} + +// Verify that outbound frame counter decreases when send buffer is drained +TEST_P(Http2CodecImplTest, ResponseDataFloodCounterReset) { + static const int kMaxOutboundFrames = 100; + max_outbound_frames_ = kMaxOutboundFrames; + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + int frame_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &frame_count](Buffer::Instance& frame, bool) { + ++frame_count; + buffer.move(frame); + })); + + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + // Account for the single HEADERS frame above + for (uint32_t i = 0; i < kMaxOutboundFrames - 1; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + + EXPECT_EQ(frame_count, kMaxOutboundFrames); + // Drain kMaxOutboundFrames / 2 slices from the send buffer + buffer.drain(buffer.length() / 2); + + for (uint32_t i = 0; i < kMaxOutboundFrames / 2 + 1; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + + // Presently flood mitigation is done only when processing downstream data + // So we need to send a frame from downstream client to trigger mitigation + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); +} + +// Verify that control frames are added to the counter of outbound frames of all types. +TEST_P(Http2CodecImplTest, PingStacksWithDataFlood) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + int frame_count = 0; + Buffer::OwnedImpl buffer; + ON_CALL(server_connection_, write(_, _)) + .WillByDefault(Invoke([&buffer, &frame_count](Buffer::Instance& frame, bool) { + ++frame_count; + buffer.move(frame); + })); + + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + // Account for the single HEADERS frame above + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES - 1; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + // Send one PING frame above the outbound queue size limit + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); + + EXPECT_EQ(frame_count, Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES); + EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/test/common/http/http2/codec_impl_test_util.h b/test/common/http/http2/codec_impl_test_util.h index b642701265c4..2c4652ee0f2f 100644 --- a/test/common/http/http2/codec_impl_test_util.h +++ b/test/common/http/http2/codec_impl_test_util.h @@ -26,6 +26,7 @@ class TestClientConnectionImpl : public ClientConnectionImpl { } nghttp2_session* session() { return session_; } using ClientConnectionImpl::getStream; + using ConnectionImpl::sendPendingFrames; }; } // namespace Http2 diff --git a/test/common/http/http2/http2_frame.cc b/test/common/http/http2/http2_frame.cc new file mode 100644 index 000000000000..ca4fb858a070 --- /dev/null +++ b/test/common/http/http2/http2_frame.cc @@ -0,0 +1,148 @@ +#include "test/common/http/http2/http2_frame.h" + +#include + +#include + +namespace { + +// Make request stream ID in the network byte order +uint32_t makeRequestStreamId(uint32_t stream_id) { return htonl((stream_id << 1) | 1); } + +// All this templatized stuff is for the typesafe constexpr bitwise ORing of the "enum class" values +template struct FirstArgType { using type = First; }; + +template constexpr uint8_t orFlags(Flag flag) { return static_cast(flag); } + +template constexpr uint8_t orFlags(Flag first, Flags... rest) { + static_assert(std::is_same::type>::value, + "All flag types must be the same!"); + return static_cast(first) | orFlags(rest...); +} + +} // namespace + +namespace Envoy { +namespace Http { +namespace Http2 { + +const char Http2Frame::Preamble[25] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"; + +void Http2Frame::setHeader(absl::string_view header) { + ASSERT(header.size() >= HeaderSize); + data_.assign(HeaderSize, 0); + memcpy(&data_[0], header.data(), HeaderSize); + data_.resize(HeaderSize + payloadSize()); +} + +void Http2Frame::setPayload(absl::string_view payload) { + ASSERT(payload.size() >= payloadSize()); + memcpy(&data_[HeaderSize], payload.data(), payloadSize()); +} + +uint32_t Http2Frame::payloadSize() const { + return (uint32_t(data_[0]) << 16) + (uint32_t(data_[1]) << 8) + uint32_t(data_[2]); +} + +Http2Frame::ResponseStatus Http2Frame::responseStatus() const { + if (empty() || Type::HEADERS != type() || size() <= HeaderSize || + ((data_[HeaderSize] & 0x80) == 0)) { + return ResponseStatus::UNKNOWN; + } + // See https://tools.ietf.org/html/rfc7541#appendix-A for header values + switch (static_cast(data_[HeaderSize] & 0x7f)) { + case StaticHeaderIndex::STATUS_200: + return ResponseStatus::_200; + case StaticHeaderIndex::STATUS_404: + return ResponseStatus::_404; + default: + break; + } + return ResponseStatus::UNKNOWN; +} + +void Http2Frame::buildHeader(Type type, uint32_t payload_size, uint8_t flags, uint32_t stream_id) { + data_.assign(payload_size + HeaderSize, 0); + setPayloadSize(payload_size); + data_[3] = static_cast(type); + data_[4] = flags; + if (stream_id) { + memcpy(&data_[5], &stream_id, sizeof(stream_id)); + } +} + +void Http2Frame::setPayloadSize(uint32_t size) { + data_[0] = (size >> 16) & 0xff; + data_[1] = (size >> 8) & 0xff; + data_[2] = size & 0xff; +} + +void Http2Frame::appendHpackInt(uint64_t value, unsigned char prefix_mask) { + if (value < prefix_mask) { + data_.push_back(value); + } else { + data_.push_back(prefix_mask); + value -= prefix_mask; + + while (value >= 128) { + data_.push_back((value & 0x7f) | 0x80); + value >>= 7; + } + data_.push_back(value); + } +} + +// See https://tools.ietf.org/html/rfc7541#section-6.1 for header representations + +void Http2Frame::appendStaticHeader(StaticHeaderIndex index) { + data_.push_back(0x80 | static_cast(index)); +} + +void Http2Frame::appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::string_view value) { + appendHpackInt(static_cast(index), 0xf); + appendHpackInt(value.size(), 0x7f); + appendData(value); +} + +Http2Frame Http2Frame::makePingFrame(absl::string_view data) { + static constexpr size_t kPingPayloadSize = 8; + Http2Frame frame; + frame.buildHeader(Type::PING, kPingPayloadSize); + if (!data.empty()) { + memcpy(&frame.data_[HeaderSize], data.data(), std::min(kPingPayloadSize, data.size())); + } + return frame; +} + +Http2Frame Http2Frame::makeEmptySettingsFrame(SettingsFlags flags) { + Http2Frame frame; + frame.buildHeader(Type::SETTINGS, 0, static_cast(flags)); + return frame; +} + +Http2Frame Http2Frame::makeMalformedRequest(uint32_t stream_index) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_STREAM, HeadersFlags::END_HEADERS), + makeRequestStreamId(stream_index)); + frame.appendStaticHeader( + StaticHeaderIndex::STATUS_200); // send :status as request header, which is invalid + frame.adjustPayloadSize(); + return frame; +} + +Http2Frame Http2Frame::makeRequest(uint32_t stream_index, absl::string_view host, + absl::string_view path) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_STREAM, HeadersFlags::END_HEADERS), + makeRequestStreamId(stream_index)); + frame.appendStaticHeader(StaticHeaderIndex::METHOD_GET); + frame.appendStaticHeader(StaticHeaderIndex::SCHEME_HTTPS); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::PATH, path); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::HOST, host); + frame.adjustPayloadSize(); + return frame; +} + +} // namespace Http2 +} // namespace Http +} // namespace Envoy diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h new file mode 100644 index 000000000000..88a051a2f133 --- /dev/null +++ b/test/common/http/http2/http2_frame.h @@ -0,0 +1,126 @@ +#pragma once + +#include +#include +#include + +#include "common/common/assert.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Http { +namespace Http2 { + +// Rudimentary facility for building and parsing of HTTP2 frames for unit tests +class Http2Frame { + using DataContainer = std::vector; + +public: + Http2Frame() = default; + + using iterator = DataContainer::iterator; + using const_iterator = DataContainer::const_iterator; + + static constexpr size_t HeaderSize = 9; + static const char Preamble[25]; + + enum class Type : uint8_t { + DATA = 0, + HEADERS, + PRIORITY, + RST_STREAM, + SETTINGS, + PUSH_PROMISE, + PING, + GOAWAY, + WINDOW_UPDATE, + CONTINUATION + }; + + enum class SettingsFlags : uint8_t { + NONE = 0, + ACK = 1, + }; + + enum class HeadersFlags : uint8_t { + NONE = 0, + END_STREAM = 1, + END_HEADERS = 4, + }; + + // See https://tools.ietf.org/html/rfc7541#appendix-A for static header indexes + enum class StaticHeaderIndex : uint8_t { + UNKNOWN, + METHOD_GET = 2, + PATH = 4, + STATUS_200 = 8, + STATUS_404 = 13, + SCHEME_HTTPS = 7, + HOST = 38, + }; + + enum class ResponseStatus { UNKNOWN, _200, _404 }; + + // Methods for creating HTTP2 frames + static Http2Frame makePingFrame(absl::string_view data = nullptr); + static Http2Frame makeEmptySettingsFrame(SettingsFlags flags = SettingsFlags::NONE); + static Http2Frame makeMalformedRequest(uint32_t stream_index); + static Http2Frame makeRequest(uint32_t stream_index, absl::string_view host, + absl::string_view path); + + Type type() const { return static_cast(data_[3]); } + ResponseStatus responseStatus() const; + + // Copy HTTP2 header. The `header` parameter must at least be HeaderSize long. + // Allocates payload size based on the value in the header. + void setHeader(absl::string_view header); + + // Copy payloadSize() bytes from the `payload`. The `payload` must be at least payloadSize() long. + void setPayload(absl::string_view payload); + + // Convert to `std::string` for convenience. + explicit operator std::string() const { + if (data_.empty()) { + return {}; + } + return std::string(reinterpret_cast(data()), size()); + } + + uint32_t payloadSize() const; + // Total size of the frame + size_t size() const { return data_.size(); } + // Access to the raw frame bytes + const uint8_t* data() const { return data_.data(); } + iterator begin() { return data_.begin(); } + iterator end() { return data_.end(); } + const_iterator begin() const { return data_.begin(); } + const_iterator end() const { return data_.end(); } + bool empty() const { return data_.empty(); } + +private: + void buildHeader(Type type, uint32_t payload_size = 0, uint8_t flags = 0, uint32_t stream_id = 0); + void setPayloadSize(uint32_t size); + + // This method appends HPACK encoded uint64_t to the payload. adjustPayloadSize() must be called + // after calling this method (possibly multiple times) to write new payload length to the HTTP2 + // header. + void appendHpackInt(uint64_t value, unsigned char prefix_mask); + void appendData(absl::string_view data) { data_.insert(data_.end(), data.begin(), data.end()); } + + // Headers are directly encoded + void appendStaticHeader(StaticHeaderIndex index); + void appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::string_view value); + + // This method updates payload length in the HTTP2 header based on the size of the data_ + void adjustPayloadSize() { + ASSERT(size() >= HeaderSize); + setPayloadSize(size() - HeaderSize); + } + + DataContainer data_; +}; + +} // namespace Http2 +} // namespace Http +} // namespace Envoy diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index daab4c044d18..4c27ead4a842 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -265,6 +265,9 @@ TEST(HttpUtility, parseHttp2Settings) { http2_settings.initial_stream_window_size_); EXPECT_EQ(Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE, http2_settings.initial_connection_window_size_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES, http2_settings.max_outbound_frames_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES, + http2_settings.max_outbound_control_frames_); } { diff --git a/test/config/utility.cc b/test/config/utility.cc index cecf66ca568c..b8d3276fa60c 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -663,6 +663,21 @@ void ConfigHelper::setLds(absl::string_view version_info) { TestUtility::renameFile(file, lds_filename); } +void ConfigHelper::setOutboundFramesLimits(uint32_t max_all_frames, uint32_t max_control_frames) { + auto filter = getFilterFromListener("envoy.http_connection_manager"); + if (filter) { + envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager hcm_config; + loadHttpConnectionManager(hcm_config); + if (hcm_config.codec_type() == + envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::HTTP2) { + auto* options = hcm_config.mutable_http2_protocol_options(); + options->mutable_max_outbound_frames()->set_value(max_all_frames); + options->mutable_max_outbound_control_frames()->set_value(max_control_frames); + storeHttpConnectionManager(hcm_config); + } + } +} + CdsHelper::CdsHelper() : cds_path_(TestEnvironment::writeStringToFileForTest("cds.pb_text", "")) {} void CdsHelper::setCds(const std::vector& clusters) { diff --git a/test/config/utility.h b/test/config/utility.h index 5be493a51b9f..0abf01c3bf60 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -149,6 +149,9 @@ class ConfigHelper { // and write it to the lds file. void setLds(absl::string_view version_info); + // Set limits on pending outbound frames. + void setOutboundFramesLimits(uint32_t max_all_frames, uint32_t max_control_frames); + // Return the bootstrap configuration for hand-off to Envoy. const envoy::config::bootstrap::v2::Bootstrap& bootstrap() { return bootstrap_; } diff --git a/test/integration/BUILD b/test/integration/BUILD index 8df797ff0fc8..f100db5066d8 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -240,6 +240,7 @@ envoy_cc_test( "//source/extensions/filters/http/buffer:config", "//source/extensions/filters/http/dynamo:config", "//source/extensions/filters/http/health_check:config", + "//test/common/http/http2:http2_frame", "//test/integration/filters:metadata_stop_all_filter_config_lib", "//test/integration/filters:request_metadata_filter_config_lib", "//test/integration/filters:response_metadata_filter_config_lib", diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index d03a7d1eacc2..5fc38eef69ff 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1,5 +1,6 @@ #include "test/integration/http2_integration_test.h" +#include #include #include "common/buffer/buffer_impl.h" @@ -1426,4 +1427,182 @@ TEST_P(Http2RingHashIntegrationTest, CookieRoutingWithCookieWithTtlSet) { EXPECT_EQ(served_by.size(), 1); } +namespace { +const int64_t TransmitThreshold = 100 * 1024 * 1024; +} // namespace + +void Http2FloodMitigationTest::setNetworkConnectionBufferSize() { + // nghttp2 library has its own internal mitigation for outbound control frames. The mitigation is + // trigerred when there are more than 10000 PING or SETTINGS frames with ACK flag in the nghttp2 + // internal outbound queue. It is possible to trigger this mitigation in nghttp2 before triggering + // Envoy's own flood mitigation. This can happen when a buffer larger enough to contain over 10K + // PING or SETTINGS frames is dispatched to the nghttp2 library. To prevent this from happening + // the network connection receive buffer needs to be smaller than 90Kb (which is 10K SETTINGS + // frames). Set it to the arbitrarily chosen value of 32K. + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) -> void { + RELEASE_ASSERT(bootstrap.mutable_static_resources()->listeners_size() >= 1, ""); + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + listener->mutable_per_connection_buffer_limit_bytes()->set_value(32 * 1024); + }); +} + +void Http2FloodMitigationTest::beginSession() { + setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + // set lower outbound frame limits to make tests run faster + config_helper_.setOutboundFramesLimits(1000, 100); + initialize(); + tcp_client_ = makeTcpConnection(lookupPort("http")); + startHttp2Session(); +} + +Http2Frame Http2FloodMitigationTest::readFrame() { + Http2Frame frame; + tcp_client_->waitForData(frame.HeaderSize); + frame.setHeader(tcp_client_->data()); + tcp_client_->clearData(frame.HeaderSize); + auto len = frame.payloadSize(); + if (len) { + tcp_client_->waitForData(len); + frame.setPayload(tcp_client_->data()); + tcp_client_->clearData(len); + } + return frame; +} + +void Http2FloodMitigationTest::sendFame(const Http2Frame& frame) { + ASSERT_TRUE(tcp_client_->connected()); + tcp_client_->write(std::string(frame), false, false); +} + +void Http2FloodMitigationTest::startHttp2Session() { + tcp_client_->write(Http2Frame::Preamble, false, false); + + // Send empty initial SETTINGS frame. + auto settings = Http2Frame::makeEmptySettingsFrame(); + tcp_client_->write(std::string(settings), false, false); + + // Read initial SETTINGS frame from the server. + readFrame(); + + // Send an SETTINGS ACK. + settings = Http2Frame::makeEmptySettingsFrame(Http2Frame::SettingsFlags::ACK); + tcp_client_->write(std::string(settings), false, false); + + // read pending SETTINGS and WINDOW_UPDATE frames + readFrame(); + readFrame(); +} + +// Verify that the server detects the flood of the given frame. +void Http2FloodMitigationTest::floodServer(const Http2Frame& frame) { + config_helper_.setBufferLimits(1024, 1024); // Set buffer limits upstream and downstream. + beginSession(); + + // pack the as many frames as we can into 16k buffer + const int FrameCount = (16 * 1024) / frame.size(); + std::vector buf(FrameCount * frame.size()); + for (auto pos = buf.begin(); pos != buf.end();) { + pos = std::copy(frame.begin(), frame.end(), pos); + } + + tcp_client_->readDisable(true); + int64_t total_bytes_sent = 0; + // If the flood protection is not working this loop will keep going + // forever until it is killed by blaze timer or run out of memory. + // Add early stop if we have sent more than 100M of frames, as it this + // point it is obvious something is wrong. + while (total_bytes_sent < TransmitThreshold && tcp_client_->connected()) { + tcp_client_->write({buf.begin(), buf.end()}, false, false); + total_bytes_sent += buf.size(); + } + + EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; + EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +// Verify that the server detects the flood using specified request parameters. +void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_view path, + Http2Frame::ResponseStatus expected_http_status) { + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(request_idx, host, path); + sendFame(request); + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::HEADERS, frame.type()); + EXPECT_EQ(expected_http_status, frame.responseStatus()); + tcp_client_->readDisable(true); + uint64_t total_bytes_sent = 0; + while (total_bytes_sent < TransmitThreshold && tcp_client_->connected()) { + request = Http2Frame::makeRequest(++request_idx, host, path); + sendFame(request); + total_bytes_sent += request.size(); + } + EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; + EXPECT_EQ(1, test_server_->counter("http2.outbound_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +INSTANTIATE_TEST_SUITE_P(IpVersions, Http2FloodMitigationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(Http2FloodMitigationTest, Ping) { + setNetworkConnectionBufferSize(); + floodServer(Http2Frame::makePingFrame()); +} + +TEST_P(Http2FloodMitigationTest, Settings) { + setNetworkConnectionBufferSize(); + floodServer(Http2Frame::makeEmptySettingsFrame()); +} + +// Verify that the server can detect flood of internally generated 404 responses. +TEST_P(Http2FloodMitigationTest, 404) { + // Change the default route to be restrictive, and send a request to a non existent route. + config_helper_.setDefaultHostAndRoute("foo.com", "/found"); + beginSession(); + + // Send requests to a non existent path to generate 404s + floodServer("host", "/notfound", Http2Frame::ResponseStatus::_404); +} + +// Verify that the server can detect flood of DATA frames +TEST_P(Http2FloodMitigationTest, Data) { + // Set large buffer limits so the test is not affected by the flow control. + config_helper_.setBufferLimits(1024 * 1024 * 1024, 1024 * 1024 * 1024); + autonomous_upstream_ = true; + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + floodServer("host", "/test/long/url", Http2Frame::ResponseStatus::_200); +} + +// Verify that the server can detect flood of RST_STREAM frames. +TEST_P(Http2FloodMitigationTest, RST_STREAM) { + beginSession(); + + int i = 0; + auto request = Http::Http2::Http2Frame::makeMalformedRequest(i); + sendFame(request); + auto response = readFrame(); + // Make sure we've got RST_STREAM from the server + EXPECT_EQ(Http2Frame::Type::RST_STREAM, response.type()); + uint64_t total_bytes_sent = 0; + while (total_bytes_sent < TransmitThreshold && tcp_client_->connected()) { + request = Http::Http2::Http2Frame::makeMalformedRequest(++i); + sendFame(request); + total_bytes_sent += request.size(); + } + EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; + EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + } // namespace Envoy diff --git a/test/integration/http2_integration_test.h b/test/integration/http2_integration_test.h index efa53788f2bd..9291273e1a22 100644 --- a/test/integration/http2_integration_test.h +++ b/test/integration/http2_integration_test.h @@ -1,9 +1,12 @@ #pragma once +#include "test/common/http/http2/http2_frame.h" #include "test/integration/http_integration.h" #include "gtest/gtest.h" +using Envoy::Http::Http2::Http2Frame; + namespace Envoy { class Http2IntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { @@ -60,4 +63,22 @@ class Http2MetadataIntegrationTest : public Http2IntegrationTest { void runHeaderOnlyTest(bool send_request_body, size_t body_size); }; + +class Http2FloodMitigationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + Http2FloodMitigationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) {} + +protected: + void startHttp2Session(); + void floodServer(const Http2Frame& frame); + void floodServer(absl::string_view host, absl::string_view path, + Http2Frame::ResponseStatus expected_http_status); + Http2Frame readFrame(); + void sendFame(const Http2Frame& frame); + void setNetworkConnectionBufferSize(); + void beginSession(); + + IntegrationTcpClientPtr tcp_client_; +}; } // namespace Envoy diff --git a/test/integration/integration.cc b/test/integration/integration.cc index a34d52b6a9ca..8131d513297c 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -180,6 +180,15 @@ void IntegrationTcpClient::waitForData(const std::string& data, bool exact_match connection_->dispatcher().run(Event::Dispatcher::RunType::Block); } +void IntegrationTcpClient::waitForData(size_t length) { + if (payload_reader_->data().size() >= length) { + return; + } + + payload_reader_->setLengthToWaitFor(length); + connection_->dispatcher().run(Event::Dispatcher::RunType::Block); +} + void IntegrationTcpClient::waitForDisconnect(bool ignore_spurious_events) { if (ignore_spurious_events) { while (!disconnected_) { diff --git a/test/integration/integration.h b/test/integration/integration.h index 873dde090a2c..b263d78cdd8d 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -95,13 +95,16 @@ class IntegrationTcpClient { void close(); void waitForData(const std::string& data, bool exact_match = true); + // wait for at least `length` bytes to be received + void waitForData(size_t length); void waitForDisconnect(bool ignore_spurious_events = false); void waitForHalfClose(); void readDisable(bool disabled); void write(const std::string& data, bool end_stream = false, bool verify = true); const std::string& data() { return payload_reader_->data(); } bool connected() const { return !disconnected_; } - void clearData() { payload_reader_->clearData(); } + // clear up to the `count` number of bytes of received data + void clearData(size_t count = std::string::npos) { payload_reader_->clearData(count); } private: struct ConnectionCallbacks : public Network::ConnectionCallbacks { diff --git a/test/integration/tcp_proxy_integration_test.cc b/test/integration/tcp_proxy_integration_test.cc index c79d49e9fb0d..060c24afdb40 100644 --- a/test/integration/tcp_proxy_integration_test.cc +++ b/test/integration/tcp_proxy_integration_test.cc @@ -43,6 +43,15 @@ TEST_P(TcpProxyIntegrationTest, TcpProxyUpstreamWritesFirst) { // Make sure inexact matches work also on data already received. tcp_client->waitForData("ello", false); + // Make sure length based wait works for the data already received + tcp_client->waitForData(5); + tcp_client->waitForData(4); + + // Drain part of the received message + tcp_client->clearData(2); + tcp_client->waitForData("llo"); + tcp_client->waitForData(3); + tcp_client->write("hello"); ASSERT_TRUE(fake_upstream_connection->waitForData(5)); diff --git a/test/integration/utility.cc b/test/integration/utility.cc index e0953f354001..74dbdd298cac 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -148,6 +148,11 @@ Network::FilterStatus WaitForPayloadReader::onData(Buffer::Instance& data, bool dispatcher_.exit(); } + if (wait_for_length_ && data_.size() >= length_to_wait_for_) { + wait_for_length_ = false; + dispatcher_.exit(); + } + return Network::FilterStatus::StopIteration; } diff --git a/test/integration/utility.h b/test/integration/utility.h index 7d3dc1b2fcdc..6554234ddc95 100644 --- a/test/integration/utility.h +++ b/test/integration/utility.h @@ -181,9 +181,14 @@ class WaitForPayloadReader : public Network::ReadFilterBaseImpl { data_to_wait_for_ = data; exact_match_ = exact_match; } + void setLengthToWaitFor(size_t length) { + ASSERT(!wait_for_length_); + length_to_wait_for_ = length; + wait_for_length_ = true; + } const std::string& data() { return data_; } bool readLastByte() { return read_end_stream_; } - void clearData() { data_.clear(); } + void clearData(size_t count = std::string::npos) { data_.erase(0, count); } private: Event::Dispatcher& dispatcher_; @@ -191,6 +196,8 @@ class WaitForPayloadReader : public Network::ReadFilterBaseImpl { std::string data_; bool exact_match_{true}; bool read_end_stream_{}; + size_t length_to_wait_for_{0}; + bool wait_for_length_{false}; }; } // namespace Envoy diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 02b950cb102d..38ec2670f90f 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -385,6 +385,8 @@ const uint32_t Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS; const uint32_t Http2Settings::DEFAULT_INITIAL_STREAM_WINDOW_SIZE; const uint32_t Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE; const uint32_t Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE; +const uint32_t Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; +const uint32_t Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES; TestHeaderMapImpl::TestHeaderMapImpl() = default; From 9f16bca5044260f5ceeb49c5836b9326a75a0b49 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Wed, 31 Jul 2019 11:44:58 -0400 Subject: [PATCH 374/542] http2: limit the number of inbound frames. (#24) This change adds protections against flooding using PRIORITY and/or WINDOW_UPDATE frames, as well as frames with an empty payload and no end stream flag. Fixes CVE-2019-9511, CVE-2019-9513 and CVE-2019-9518. Signed-off-by: Piotr Sikora --- api/envoy/api/v2/core/protocol.proto | 42 +++++- .../configuration/http_conn_man/stats.rst | 3 + docs/root/intro/version_history.rst | 5 +- include/envoy/http/codec.h | 12 ++ source/common/http/http2/codec_impl.cc | 119 +++++++++++++++- source/common/http/http2/codec_impl.h | 53 ++++++- source/common/http/utility.cc | 9 ++ test/common/http/http2/codec_impl_test.cc | 11 ++ test/common/http/http2/http2_frame.cc | 52 +++++++ test/common/http/http2/http2_frame.h | 15 ++ test/common/http/utility_test.cc | 6 + test/integration/http2_integration_test.cc | 134 ++++++++++++++++-- test/integration/http2_integration_test.h | 4 +- test/test_common/utility.cc | 3 + 14 files changed, 448 insertions(+), 20 deletions(-) diff --git a/api/envoy/api/v2/core/protocol.proto b/api/envoy/api/v2/core/protocol.proto index becd596a0e98..a5a8ba3327d7 100644 --- a/api/envoy/api/v2/core/protocol.proto +++ b/api/envoy/api/v2/core/protocol.proto @@ -49,6 +49,7 @@ message Http1ProtocolOptions { string default_host_for_http_10 = 3; } +// [#comment:next free field: 12] message Http2ProtocolOptions { // `Maximum table size `_ // (in octets) that the encoder is permitted to use for the dynamic HPACK table. Valid values @@ -94,18 +95,53 @@ message Http2ProtocolOptions { // Limit the number of pending outbound downstream frames of all types (frames that are waiting to // be written into the socket). Exceeding this limit triggers flood mitigation and connection is - // terminated. The "http2.outbound_flood" stat tracks the number of terminated connections due to - // flood mitigation. The default limit is 10000. + // terminated. The ``http2.outbound_flood`` stat tracks the number of terminated connections due + // to flood mitigation. The default limit is 10000. // [#comment:TODO: implement same limits for upstream outbound frames as well.] google.protobuf.UInt32Value max_outbound_frames = 7 [(validate.rules).uint32 = {gte: 1}]; // Limit the number of pending outbound downstream frames of types PING, SETTINGS and RST_STREAM, // preventing high memory utilization when receiving continuous stream of these frames. Exceeding // this limit triggers flood mitigation and connection is terminated. The - // "http2.outbound_control_flood" stat tracks the number of terminated connections due to flood + // ``http2.outbound_control_flood`` stat tracks the number of terminated connections due to flood // mitigation. The default limit is 1000. // [#comment:TODO: implement same limits for upstream outbound frames as well.] google.protobuf.UInt32Value max_outbound_control_frames = 8 [(validate.rules).uint32 = {gte: 1}]; + + // Limit the number of consecutive inbound frames of types HEADERS, CONTINUATION and DATA with an + // empty payload and no end stream flag. Those frames have no legitimate use and are abusive, but + // might be a result of a broken HTTP/2 implementation. The `http2.inbound_empty_frames_flood`` + // stat tracks the number of connections terminated due to flood mitigation. + // Setting this to 0 will terminate connection upon receiving first frame with an empty payload + // and no end stream flag. The default limit is 1. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_consecutive_inbound_frames_with_empty_payload = 9; + + // Limit the number of inbound PRIORITY frames allowed per each opened stream. If the number + // of PRIORITY frames received over the lifetime of connection exceeds the value calculated + // using this formula:: + // + // max_inbound_priority_frames_per_stream * (1 + inbound_streams) + // + // the connection is terminated. The ``http2.inbound_priority_frames_flood`` stat tracks + // the number of connections terminated due to flood mitigation. The default limit is 100. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_inbound_priority_frames_per_stream = 10; + + // Limit the number of inbound WINDOW_UPDATE frames allowed per DATA frame sent. If the number + // of WINDOW_UPDATE frames received over the lifetime of connection exceeds the value calculated + // using this formula:: + // + // 1 + 2 * (inbound_streams + + // max_inbound_window_update_frames_per_data_frame_sent * outbound_data_frames) + // + // the connection is terminated. The ``http2.inbound_priority_frames_flood`` stat tracks + // the number of connections terminated due to flood mitigation. The default limit is 10. + // Setting this to 1 should be enough to support HTTP/2 implementations with basic flow control, + // but more complex implementations that try to estimate available bandwidth require at least 2. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_inbound_window_update_frames_per_data_frame_sent = 11 + [(validate.rules).uint32 = {gte: 1}]; } // [#not-implemented-hide:] diff --git a/docs/root/configuration/http_conn_man/stats.rst b/docs/root/configuration/http_conn_man/stats.rst index c998145f42d9..099c5aa3965d 100644 --- a/docs/root/configuration/http_conn_man/stats.rst +++ b/docs/root/configuration/http_conn_man/stats.rst @@ -122,6 +122,9 @@ All http2 statistics are rooted at *http2.* header_overflow, Counter, Total number of connections reset due to the headers being larger than the :ref:`configured value `. headers_cb_no_stream, Counter, Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug + inbound_empty_frames_flood, Counter, Total number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. + inbound_priority_frames_flood, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type PRIORITY. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. + inbound_window_update_frames_flood, Counter, Total number of connections terminated for exceeding the limit on inbound frames of type WINDOW_UPDATE. The limit is configured by setting the :ref:`max_inbound_window_updateframes_per_data_frame_sent config setting `. outbound_flood, Counter, Total number of connections terminated for exceeding the limit on outbound frames of all types. The limit is configured by setting the :ref:`max_outbound_frames config setting `. outbound_control_flood, Counter, "Total number of connections terminated for exceeding the limit on outbound frames of types PING, SETTINGS and RST_STREAM. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `." rx_messaging_error, Counter, Total number of invalid received frames that violated `section 8 `_ of the HTTP/2 spec. This will result in a *tx_reset* diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 599753b85be8..7896b6a1ab65 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -28,7 +28,10 @@ Version history 1.11.1 (August 13, 2019) ======================== -* http: added mitigation of client initiated atacks that result in flooding of the outbound queue of downstream HTTP/2 connections. +* http: added mitigation of client initiated atacks that result in flooding of the downstream HTTP/2 connections. +* http: added :ref:`inbound_empty_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. +* http: added :ref:`inbound_priority_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound PRIORITY frames. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. +* http: added :ref:`inbound_window_update_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound WINDOW_UPDATE frames. The limit is configured by setting the :ref:`max_inbound_window_update_frames_per_data_frame_sent config setting `. * http: added :ref:`outbound_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting ` * http: added :ref:`outbound_control_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `. diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index d05943c0f955..6c2b09b2f636 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -237,6 +237,11 @@ struct Http2Settings { bool allow_metadata_{DEFAULT_ALLOW_METADATA}; uint32_t max_outbound_frames_{DEFAULT_MAX_OUTBOUND_FRAMES}; uint32_t max_outbound_control_frames_{DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES}; + uint32_t max_consecutive_inbound_frames_with_empty_payload_{ + DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD}; + uint32_t max_inbound_priority_frames_per_stream_{DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM}; + uint32_t max_inbound_window_update_frames_per_data_frame_sent_{ + DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT}; // disable HPACK compression static const uint32_t MIN_HPACK_TABLE_SIZE = 0; @@ -279,6 +284,13 @@ struct Http2Settings { static const uint32_t DEFAULT_MAX_OUTBOUND_FRAMES = 10000; // Default limit on the number of outbound frames of types PING, SETTINGS and RST_STREAM. static const uint32_t DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES = 1000; + // Default limit on the number of consecutive inbound frames with an empty payload + // and no end stream flag. + static const uint32_t DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD = 1; + // Default limit on the number of inbound frames of type PRIORITY (per stream). + static const uint32_t DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM = 100; + // Default limit on the number of inbound frames of type WINDOW_UPDATE (per DATA frame sent). + static const uint32_t DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT = 10; }; /** diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index fe7518c19cff..d59c16676a75 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -252,6 +252,8 @@ int ConnectionImpl::StreamImpl::onDataSourceSend(const uint8_t* framehd, size_t // https://nghttp2.org/documentation/types.html#c.nghttp2_send_data_callback static const uint64_t FRAME_HEADER_SIZE = 9; + parent_.outbound_data_frames_++; + Buffer::OwnedImpl output; if (!parent_.addOutboundFrameFragment(output, framehd, FRAME_HEADER_SIZE)) { ENVOY_CONN_LOG(debug, "error sending data frame: Too many frames in the outbound queue", @@ -355,7 +357,7 @@ void ConnectionImpl::dispatch(Buffer::Instance& data) { dispatching_ = true; ssize_t rc = nghttp2_session_mem_recv(session_, static_cast(slice.mem_), slice.len_); - if (rc == NGHTTP2_ERR_FLOODED) { + if (rc == NGHTTP2_ERR_FLOODED || flood_detected_) { throw FrameFloodException( "Flooding was detected in this HTTP/2 session, and it must be closed"); } @@ -408,9 +410,36 @@ void ConnectionImpl::shutdownNotice() { sendPendingFrames(); } +int ConnectionImpl::onBeforeFrameReceived(const nghttp2_frame_hd* hd) { + ENVOY_CONN_LOG(trace, "about to recv frame type={}, flags={}", connection_, + static_cast(hd->type), static_cast(hd->flags)); + + // Track all the frames without padding here, since this is the only callback we receive + // for some of them (e.g. CONTINUATION frame, frames sent on closed streams, etc.). + // HEADERS frame is tracked in onBeginHeaders(), DATA frame is tracked in onFrameReceived(). + if (hd->type != NGHTTP2_HEADERS && hd->type != NGHTTP2_DATA) { + if (!trackInboundFrames(hd, 0)) { + return NGHTTP2_ERR_FLOODED; + } + } + + return 0; +} + int ConnectionImpl::onFrameReceived(const nghttp2_frame* frame) { ENVOY_CONN_LOG(trace, "recv frame type={}", connection_, static_cast(frame->hd.type)); + // onFrameReceived() is called with a complete HEADERS frame assembled from all the HEADERS + // and CONTINUATION frames, but we track them separately: HEADERS frames in onBeginHeaders() + // and CONTINUATION frames in onBeforeFrameReceived(). + ASSERT(frame->hd.type != NGHTTP2_CONTINUATION); + + if (frame->hd.type == NGHTTP2_DATA) { + if (!trackInboundFrames(&frame->hd, frame->data.padlen)) { + return NGHTTP2_ERR_FLOODED; + } + } + // Only raise GOAWAY once, since we don't currently expose stream information. Shutdown // notifications are the same as a normal GOAWAY. if (frame->hd.type == NGHTTP2_GOAWAY && !raised_goaway_) { @@ -567,7 +596,7 @@ int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) { } int ConnectionImpl::onBeforeFrameSend(const nghttp2_frame* frame) { - ENVOY_CONN_LOG(trace, "about to sent frame type={}, flags={}", connection_, + ENVOY_CONN_LOG(trace, "about to send frame type={}, flags={}", connection_, static_cast(frame->hd.type), static_cast(frame->hd.flags)); ASSERT(!is_outbound_flood_monitored_control_frame_); // Flag flood monitored outbound control frames. @@ -882,6 +911,11 @@ ConnectionImpl::Http2Callbacks::Http2Callbacks() { return static_cast(user_data)->onData(stream_id, data, len); }); + nghttp2_session_callbacks_set_on_begin_frame_callback( + callbacks_, [](nghttp2_session*, const nghttp2_frame_hd* hd, void* user_data) -> int { + return static_cast(user_data)->onBeforeFrameReceived(hd); + }); + nghttp2_session_callbacks_set_on_frame_recv_callback( callbacks_, [](nghttp2_session*, const nghttp2_frame* frame, void* user_data) -> int { return static_cast(user_data)->onFrameReceived(frame); @@ -1042,6 +1076,11 @@ ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection, int ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) { // For a server connection, we should never get push promise frames. ASSERT(frame->hd.type == NGHTTP2_HEADERS); + + if (!trackInboundFrames(&frame->hd, frame->headers.padlen)) { + return NGHTTP2_ERR_FLOODED; + } + if (frame->headers.cat != NGHTTP2_HCAT_REQUEST) { stats_.trailers_.inc(); ASSERT(frame->headers.cat == NGHTTP2_HCAT_HEADERS); @@ -1072,6 +1111,82 @@ int ServerConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na return saveHeader(frame, std::move(name), std::move(value)); } +bool ServerConnectionImpl::trackInboundFrames(const nghttp2_frame_hd* hd, uint32_t padding_length) { + ENVOY_CONN_LOG(trace, "track inbound frame type={} flags={} length={} padding_length={}", + connection_, static_cast(hd->type), static_cast(hd->flags), + static_cast(hd->length), padding_length); + switch (hd->type) { + case NGHTTP2_HEADERS: + case NGHTTP2_CONTINUATION: + // Track new streams. + if (hd->flags & NGHTTP2_FLAG_END_HEADERS) { + inbound_streams_++; + } + FALLTHRU; + case NGHTTP2_DATA: + // Track frames with an empty payload and no end stream flag. + if (hd->length - padding_length == 0 && !(hd->flags & NGHTTP2_FLAG_END_STREAM)) { + ENVOY_CONN_LOG(trace, "frame with an empty payload and no end stream flag.", connection_); + consecutive_inbound_frames_with_empty_payload_++; + } else { + consecutive_inbound_frames_with_empty_payload_ = 0; + } + break; + case NGHTTP2_PRIORITY: + inbound_priority_frames_++; + break; + case NGHTTP2_WINDOW_UPDATE: + inbound_window_update_frames_++; + break; + default: + break; + } + + if (!checkInboundFrameLimits()) { + // NGHTTP2_ERR_FLOODED is overridden within nghttp2 library and it doesn't propagate + // all the way to nghttp2_session_mem_recv() where we need it. + flood_detected_ = true; + return false; + } + + return true; +} + +bool ServerConnectionImpl::checkInboundFrameLimits() { + ASSERT(dispatching_downstream_data_); + + if (consecutive_inbound_frames_with_empty_payload_ > + max_consecutive_inbound_frames_with_empty_payload_) { + ENVOY_CONN_LOG(trace, + "error reading frame: Too many consecutive frames with an empty payload " + "received in this HTTP/2 session.", + connection_); + stats_.inbound_empty_frames_flood_.inc(); + return false; + } + + if (inbound_priority_frames_ > max_inbound_priority_frames_per_stream_ * (1 + inbound_streams_)) { + ENVOY_CONN_LOG(trace, + "error reading frame: Too many PRIORITY frames received in this HTTP/2 session.", + connection_); + stats_.inbound_priority_frames_flood_.inc(); + return false; + } + + if (inbound_window_update_frames_ > + 1 + 2 * (inbound_streams_ + + max_inbound_window_update_frames_per_data_frame_sent_ * outbound_data_frames_)) { + ENVOY_CONN_LOG( + trace, + "error reading frame: Too many WINDOW_UPDATE frames received in this HTTP/2 session.", + connection_); + stats_.inbound_window_update_frames_flood_.inc(); + return false; + } + + return true; +} + void ServerConnectionImpl::checkOutboundQueueLimits() { if (outbound_frames_ > max_outbound_frames_ && dispatching_downstream_data_) { stats_.outbound_flood_.inc(); diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 5c1b1fd8670d..9ca472576cfb 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -41,6 +41,9 @@ const std::string CLIENT_MAGIC_PREFIX = "PRI * HTTP/2"; #define ALL_HTTP2_CODEC_STATS(COUNTER) \ COUNTER(header_overflow) \ COUNTER(headers_cb_no_stream) \ + COUNTER(inbound_empty_frames_flood) \ + COUNTER(inbound_priority_frames_flood) \ + COUNTER(inbound_window_update_frames_flood) \ COUNTER(outbound_control_flood) \ COUNTER(outbound_flood) \ COUNTER(rx_messaging_error) \ @@ -79,7 +82,7 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable, diff --git a/test/common/http/http2/http2_frame.cc b/test/common/http/http2/http2_frame.cc index ca4fb858a070..fc96cf77b852 100644 --- a/test/common/http/http2/http2_frame.cc +++ b/test/common/http/http2/http2_frame.cc @@ -120,6 +120,45 @@ Http2Frame Http2Frame::makeEmptySettingsFrame(SettingsFlags flags) { return frame; } +Http2Frame Http2Frame::makeEmptyHeadersFrame(uint32_t stream_index, HeadersFlags flags) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, static_cast(flags), + makeRequestStreamId(stream_index)); + return frame; +} + +Http2Frame Http2Frame::makeEmptyContinuationFrame(uint32_t stream_index, HeadersFlags flags) { + Http2Frame frame; + frame.buildHeader(Type::CONTINUATION, 0, static_cast(flags), + makeRequestStreamId(stream_index)); + return frame; +} + +Http2Frame Http2Frame::makeEmptyDataFrame(uint32_t stream_index, DataFlags flags) { + Http2Frame frame; + frame.buildHeader(Type::DATA, 0, static_cast(flags), makeRequestStreamId(stream_index)); + return frame; +} + +Http2Frame Http2Frame::makePriorityFrame(uint32_t stream_index, uint32_t dependent_index) { + static constexpr size_t kPriorityPayloadSize = 5; + Http2Frame frame; + frame.buildHeader(Type::PRIORITY, kPriorityPayloadSize, 0, makeRequestStreamId(stream_index)); + uint32_t dependent_net = makeRequestStreamId(dependent_index); + memcpy(&frame.data_[HeaderSize], reinterpret_cast(&dependent_net), sizeof(uint32_t)); + return frame; +} + +Http2Frame Http2Frame::makeWindowUpdateFrame(uint32_t stream_index, uint32_t increment) { + static constexpr size_t kWindowUpdatePayloadSize = 4; + Http2Frame frame; + frame.buildHeader(Type::WINDOW_UPDATE, kWindowUpdatePayloadSize, 0, + makeRequestStreamId(stream_index)); + uint32_t increment_net = htonl(increment); + memcpy(&frame.data_[HeaderSize], reinterpret_cast(&increment_net), sizeof(uint32_t)); + return frame; +} + Http2Frame Http2Frame::makeMalformedRequest(uint32_t stream_index) { Http2Frame frame; frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_STREAM, HeadersFlags::END_HEADERS), @@ -143,6 +182,19 @@ Http2Frame Http2Frame::makeRequest(uint32_t stream_index, absl::string_view host return frame; } +Http2Frame Http2Frame::makePostRequest(uint32_t stream_index, absl::string_view host, + absl::string_view path) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_HEADERS), + makeRequestStreamId(stream_index)); + frame.appendStaticHeader(StaticHeaderIndex::METHOD_POST); + frame.appendStaticHeader(StaticHeaderIndex::SCHEME_HTTPS); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::PATH, path); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::HOST, host); + frame.adjustPayloadSize(); + return frame; +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h index 88a051a2f133..760ae69e3575 100644 --- a/test/common/http/http2/http2_frame.h +++ b/test/common/http/http2/http2_frame.h @@ -49,10 +49,16 @@ class Http2Frame { END_HEADERS = 4, }; + enum class DataFlags : uint8_t { + NONE = 0, + END_STREAM = 1, + }; + // See https://tools.ietf.org/html/rfc7541#appendix-A for static header indexes enum class StaticHeaderIndex : uint8_t { UNKNOWN, METHOD_GET = 2, + METHOD_POST = 3, PATH = 4, STATUS_200 = 8, STATUS_404 = 13, @@ -65,9 +71,18 @@ class Http2Frame { // Methods for creating HTTP2 frames static Http2Frame makePingFrame(absl::string_view data = nullptr); static Http2Frame makeEmptySettingsFrame(SettingsFlags flags = SettingsFlags::NONE); + static Http2Frame makeEmptyHeadersFrame(uint32_t stream_index, + HeadersFlags flags = HeadersFlags::NONE); + static Http2Frame makeEmptyContinuationFrame(uint32_t stream_index, + HeadersFlags flags = HeadersFlags::NONE); + static Http2Frame makeEmptyDataFrame(uint32_t stream_index, DataFlags flags = DataFlags::NONE); + static Http2Frame makePriorityFrame(uint32_t stream_index, uint32_t dependent_index); + static Http2Frame makeWindowUpdateFrame(uint32_t stream_index, uint32_t increment); static Http2Frame makeMalformedRequest(uint32_t stream_index); static Http2Frame makeRequest(uint32_t stream_index, absl::string_view host, absl::string_view path); + static Http2Frame makePostRequest(uint32_t stream_index, absl::string_view host, + absl::string_view path); Type type() const { return static_cast(data_[3]); } ResponseStatus responseStatus() const; diff --git a/test/common/http/utility_test.cc b/test/common/http/utility_test.cc index 4c27ead4a842..388f603623c8 100644 --- a/test/common/http/utility_test.cc +++ b/test/common/http/utility_test.cc @@ -268,6 +268,12 @@ TEST(HttpUtility, parseHttp2Settings) { EXPECT_EQ(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES, http2_settings.max_outbound_frames_); EXPECT_EQ(Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES, http2_settings.max_outbound_control_frames_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD, + http2_settings.max_consecutive_inbound_frames_with_empty_payload_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM, + http2_settings.max_inbound_priority_frames_per_stream_); + EXPECT_EQ(Http2Settings::DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT, + http2_settings.max_inbound_window_update_frames_per_data_frame_sent_); } { diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index 5fc38eef69ff..5861292c9c42 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1495,10 +1495,7 @@ void Http2FloodMitigationTest::startHttp2Session() { } // Verify that the server detects the flood of the given frame. -void Http2FloodMitigationTest::floodServer(const Http2Frame& frame) { - config_helper_.setBufferLimits(1024, 1024); // Set buffer limits upstream and downstream. - beginSession(); - +void Http2FloodMitigationTest::floodServer(const Http2Frame& frame, const std::string& flood_stat) { // pack the as many frames as we can into 16k buffer const int FrameCount = (16 * 1024) / frame.size(); std::vector buf(FrameCount * frame.size()); @@ -1518,7 +1515,7 @@ void Http2FloodMitigationTest::floodServer(const Http2Frame& frame) { } EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; - EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value()); + EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); // Verify that connection was closed abortively EXPECT_EQ(0, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); @@ -1526,7 +1523,8 @@ void Http2FloodMitigationTest::floodServer(const Http2Frame& frame) { // Verify that the server detects the flood using specified request parameters. void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_view path, - Http2Frame::ResponseStatus expected_http_status) { + Http2Frame::ResponseStatus expected_http_status, + const std::string& flood_stat) { uint32_t request_idx = 0; auto request = Http2Frame::makeRequest(request_idx, host, path); sendFame(request); @@ -1541,7 +1539,7 @@ void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_ total_bytes_sent += request.size(); } EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; - EXPECT_EQ(1, test_server_->counter("http2.outbound_flood")->value()); + EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); // Verify that connection was closed abortively EXPECT_EQ(0, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); @@ -1553,12 +1551,14 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, Http2FloodMitigationTest, TEST_P(Http2FloodMitigationTest, Ping) { setNetworkConnectionBufferSize(); - floodServer(Http2Frame::makePingFrame()); + beginSession(); + floodServer(Http2Frame::makePingFrame(), "http2.outbound_control_flood"); } TEST_P(Http2FloodMitigationTest, Settings) { setNetworkConnectionBufferSize(); - floodServer(Http2Frame::makeEmptySettingsFrame()); + beginSession(); + floodServer(Http2Frame::makeEmptySettingsFrame(), "http2.outbound_control_flood"); } // Verify that the server can detect flood of internally generated 404 responses. @@ -1568,7 +1568,7 @@ TEST_P(Http2FloodMitigationTest, 404) { beginSession(); // Send requests to a non existent path to generate 404s - floodServer("host", "/notfound", Http2Frame::ResponseStatus::_404); + floodServer("host", "/notfound", Http2Frame::ResponseStatus::_404, "http2.outbound_flood"); } // Verify that the server can detect flood of DATA frames @@ -1579,7 +1579,7 @@ TEST_P(Http2FloodMitigationTest, Data) { beginSession(); fake_upstreams_[0]->set_allow_unexpected_disconnects(true); - floodServer("host", "/test/long/url", Http2Frame::ResponseStatus::_200); + floodServer("host", "/test/long/url", Http2Frame::ResponseStatus::_200, "http2.outbound_flood"); } // Verify that the server can detect flood of RST_STREAM frames. @@ -1605,4 +1605,116 @@ TEST_P(Http2FloodMitigationTest, RST_STREAM) { test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } +TEST_P(Http2FloodMitigationTest, EmptyHeaders) { + config_helper_.addConfigModifier( + [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options() + ->mutable_max_consecutive_inbound_frames_with_empty_payload() + ->set_value(0); + }); + beginSession(); + + uint32_t request_idx = 0; + auto request = Http2Frame::makeEmptyHeadersFrame(request_idx); + sendFame(request); + + tcp_client_->waitForDisconnect(); + + EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +TEST_P(Http2FloodMitigationTest, EmptyHeadersContinuation) { + beginSession(); + + uint32_t request_idx = 0; + auto request = Http2Frame::makeEmptyHeadersFrame(request_idx); + sendFame(request); + + for (int i = 0; i < 2; i++) { + request = Http2Frame::makeEmptyContinuationFrame(request_idx); + sendFame(request); + } + + tcp_client_->waitForDisconnect(); + + EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +TEST_P(Http2FloodMitigationTest, EmptyData) { + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + uint32_t request_idx = 0; + auto request = Http2Frame::makePostRequest(request_idx, "host", "/"); + sendFame(request); + + for (int i = 0; i < 2; i++) { + request = Http2Frame::makeEmptyDataFrame(request_idx); + sendFame(request); + } + + tcp_client_->waitForDisconnect(); + + EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); + // Verify that connection was closed abortively + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +TEST_P(Http2FloodMitigationTest, PriorityIdleStream) { + beginSession(); + + floodServer(Http2Frame::makePriorityFrame(0, 1), "http2.inbound_priority_frames_flood"); +} + +TEST_P(Http2FloodMitigationTest, PriorityOpenStream) { + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Open stream. + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(request_idx, "host", "/"); + sendFame(request); + + floodServer(Http2Frame::makePriorityFrame(request_idx, request_idx + 1), + "http2.inbound_priority_frames_flood"); +} + +TEST_P(Http2FloodMitigationTest, PriorityClosedStream) { + autonomous_upstream_ = true; + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Open stream. + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(request_idx, "host", "/"); + sendFame(request); + // Reading response marks this stream as closed in nghttp2. + auto frame = readFrame(); + EXPECT_EQ(Http2Frame::Type::HEADERS, frame.type()); + + floodServer(Http2Frame::makePriorityFrame(request_idx, request_idx + 1), + "http2.inbound_priority_frames_flood"); +} + +TEST_P(Http2FloodMitigationTest, WindowUpdate) { + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Open stream. + uint32_t request_idx = 0; + auto request = Http2Frame::makeRequest(request_idx, "host", "/"); + sendFame(request); + + floodServer(Http2Frame::makeWindowUpdateFrame(request_idx, 1), + "http2.inbound_window_update_frames_flood"); +} + } // namespace Envoy diff --git a/test/integration/http2_integration_test.h b/test/integration/http2_integration_test.h index 9291273e1a22..2c253e5f3b16 100644 --- a/test/integration/http2_integration_test.h +++ b/test/integration/http2_integration_test.h @@ -71,9 +71,9 @@ class Http2FloodMitigationTest : public testing::TestWithParam Date: Wed, 31 Jul 2019 12:59:14 -0400 Subject: [PATCH 375/542] http2: enable strict validation of HTTP/2 headers. (#25) Fixes CVE-2019-9516. Signed-off-by: Piotr Sikora --- api/envoy/api/v2/core/protocol.proto | 9 ++- docs/root/intro/version_history.rst | 1 + include/envoy/http/codec.h | 3 + source/common/http/http2/codec_impl.cc | 17 +++-- source/common/http/http2/codec_impl.h | 3 + source/common/http/utility.cc | 1 + test/common/http/http2/codec_impl_test.cc | 65 ++++++++++++++++++- test/common/http/http2/http2_frame.cc | 21 ++++++ test/common/http/http2/http2_frame.h | 4 ++ test/integration/http2_integration_test.cc | 56 ++++++++++++++++ test/integration/protocol_integration_test.cc | 58 +++++++++++++++++ 11 files changed, 229 insertions(+), 9 deletions(-) diff --git a/api/envoy/api/v2/core/protocol.proto b/api/envoy/api/v2/core/protocol.proto index a5a8ba3327d7..88a82077a428 100644 --- a/api/envoy/api/v2/core/protocol.proto +++ b/api/envoy/api/v2/core/protocol.proto @@ -49,7 +49,7 @@ message Http1ProtocolOptions { string default_host_for_http_10 = 3; } -// [#comment:next free field: 12] +// [#comment:next free field: 13] message Http2ProtocolOptions { // `Maximum table size `_ // (in octets) that the encoder is permitted to use for the dynamic HPACK table. Valid values @@ -142,6 +142,13 @@ message Http2ProtocolOptions { // [#comment:TODO: implement same limits for upstream inbound frames as well.] google.protobuf.UInt32Value max_inbound_window_update_frames_per_data_frame_sent = 11 [(validate.rules).uint32 = {gte: 1}]; + + // Allows invalid HTTP messaging and headers. When this option is disabled (default), then + // the whole HTTP/2 connection is terminated upon receiving invalid HEADERS frame. However, + // when this option is enabled, only the offending stream is terminated. + // + // See [RFC7540, sec. 8.1](https://tools.ietf.org/html/rfc7540#section-8.1) for details. + bool stream_error_on_invalid_http_messaging = 12; } // [#not-implemented-hide:] diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 7896b6a1ab65..18bc47adcac3 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -34,6 +34,7 @@ Version history * http: added :ref:`inbound_window_update_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound WINDOW_UPDATE frames. The limit is configured by setting the :ref:`max_inbound_window_update_frames_per_data_frame_sent config setting `. * http: added :ref:`outbound_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting ` * http: added :ref:`outbound_control_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `. +* http: enabled strict validation of HTTP/2 messaging. Previous behavior can be restored using :ref:`stream_error_on_invalid_http_messaging config setting `. 1.11.0 (July 11, 2019) ====================== diff --git a/include/envoy/http/codec.h b/include/envoy/http/codec.h index 6c2b09b2f636..eb5951d3b9b6 100644 --- a/include/envoy/http/codec.h +++ b/include/envoy/http/codec.h @@ -235,6 +235,7 @@ struct Http2Settings { uint32_t initial_connection_window_size_{DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE}; bool allow_connect_{DEFAULT_ALLOW_CONNECT}; bool allow_metadata_{DEFAULT_ALLOW_METADATA}; + bool stream_error_on_invalid_http_messaging_{DEFAULT_STREAM_ERROR_ON_INVALID_HTTP_MESSAGING}; uint32_t max_outbound_frames_{DEFAULT_MAX_OUTBOUND_FRAMES}; uint32_t max_outbound_control_frames_{DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES}; uint32_t max_consecutive_inbound_frames_with_empty_payload_{ @@ -279,6 +280,8 @@ struct Http2Settings { static const bool DEFAULT_ALLOW_CONNECT = false; // By default Envoy does not allow METADATA support. static const bool DEFAULT_ALLOW_METADATA = false; + // By default Envoy does not allow invalid headers. + static const bool DEFAULT_STREAM_ERROR_ON_INVALID_HTTP_MESSAGING = false; // Default limit on the number of outbound frames of all types. static const uint32_t DEFAULT_MAX_OUTBOUND_FRAMES = 10000; diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index d59c16676a75..e45f07a101af 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -579,16 +579,19 @@ int ConnectionImpl::onInvalidFrame(int32_t stream_id, int error_code) { ENVOY_CONN_LOG(debug, "invalid frame: {} on stream {}", connection_, nghttp2_strerror(error_code), stream_id); - // The stream is about to be closed due to an invalid header or messaging. Don't kill the - // entire connection if one stream has bad headers or messaging. if (error_code == NGHTTP2_ERR_HTTP_HEADER || error_code == NGHTTP2_ERR_HTTP_MESSAGING) { stats_.rx_messaging_error_.inc(); - StreamImpl* stream = getStream(stream_id); - if (stream != nullptr) { - // See comment below in onStreamClose() for why we do this. - stream->reset_due_to_messaging_error_ = true; + + if (stream_error_on_invalid_http_messaging_) { + // The stream is about to be closed due to an invalid header or messaging. Don't kill the + // entire connection if one stream has bad headers or messaging. + StreamImpl* stream = getStream(stream_id); + if (stream != nullptr) { + // See comment below in onStreamClose() for why we do this. + stream->reset_due_to_messaging_error_ = true; + } + return 0; } - return 0; } // Cause dispatch to return with an error code. diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 9ca472576cfb..15800f25a523 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -82,6 +82,8 @@ class ConnectionImpl : public virtual Connection, protected Logger::Loggable(tp); setting.initial_connection_window_size_ = ::testing::get<3>(tp); setting.allow_metadata_ = allow_metadata_; + setting.stream_error_on_invalid_http_messaging_ = stream_error_on_invalid_http_messaging_; setting.max_outbound_frames_ = max_outbound_frames_; setting.max_outbound_control_frames_ = max_outbound_control_frames_; setting.max_consecutive_inbound_frames_with_empty_payload_ = @@ -128,6 +129,7 @@ class Http2CodecImplTestFixture { const Http2SettingsTuple client_settings_; const Http2SettingsTuple server_settings_; bool allow_metadata_ = false; + bool stream_error_on_invalid_http_messaging_ = false; Stats::IsolatedStoreImpl stats_store_; Http2Settings client_http2settings_; NiceMock client_connection_; @@ -202,6 +204,20 @@ TEST_P(Http2CodecImplTest, ContinueHeaders) { TEST_P(Http2CodecImplTest, InvalidContinueWithFin) { initialize(); + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + request_encoder_->encodeHeaders(request_headers, true); + + TestHeaderMapImpl continue_headers{{":status", "100"}}; + EXPECT_THROW(response_encoder_->encodeHeaders(continue_headers, true), CodecProtocolException); + EXPECT_EQ(1, stats_store_.counter("http2.rx_messaging_error").value()); +} + +TEST_P(Http2CodecImplTest, InvalidContinueWithFinAllowed) { + stream_error_on_invalid_http_messaging_ = true; + initialize(); + MockStreamCallbacks request_callbacks; request_encoder_->getStream().addCallbacks(request_callbacks); @@ -229,6 +245,23 @@ TEST_P(Http2CodecImplTest, InvalidContinueWithFin) { TEST_P(Http2CodecImplTest, InvalidRepeatContinue) { initialize(); + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + request_encoder_->encodeHeaders(request_headers, true); + + TestHeaderMapImpl continue_headers{{":status", "100"}}; + EXPECT_CALL(response_decoder_, decode100ContinueHeaders_(_)); + response_encoder_->encode100ContinueHeaders(continue_headers); + + EXPECT_THROW(response_encoder_->encodeHeaders(continue_headers, true), CodecProtocolException); + EXPECT_EQ(1, stats_store_.counter("http2.rx_messaging_error").value()); +}; + +TEST_P(Http2CodecImplTest, InvalidRepeatContinueAllowed) { + stream_error_on_invalid_http_messaging_ = true; + initialize(); + MockStreamCallbacks request_callbacks; request_encoder_->getStream().addCallbacks(request_callbacks); @@ -280,6 +313,28 @@ TEST_P(Http2CodecImplTest, Invalid103) { TEST_P(Http2CodecImplTest, Invalid204WithContentLength) { initialize(); + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + request_encoder_->encodeHeaders(request_headers, true); + + TestHeaderMapImpl response_headers{{":status", "204"}, {"content-length", "3"}}; + // What follows is a hack to get headers that should span into continuation frames. The default + // maximum frame size is 16K. We will add 3,000 headers that will take us above this size and + // not easily compress with HPACK. (I confirmed this generates 26,468 bytes of header data + // which should contain a continuation.) + for (uint i = 1; i < 3000; i++) { + response_headers.addCopy(std::to_string(i), std::to_string(i)); + } + + EXPECT_THROW(response_encoder_->encodeHeaders(response_headers, false), CodecProtocolException); + EXPECT_EQ(1, stats_store_.counter("http2.rx_messaging_error").value()); +}; + +TEST_P(Http2CodecImplTest, Invalid204WithContentLengthAllowed) { + stream_error_on_invalid_http_messaging_ = true; + initialize(); + MockStreamCallbacks request_callbacks; request_encoder_->getStream().addCallbacks(request_callbacks); @@ -329,7 +384,15 @@ TEST_P(Http2CodecImplTest, RefusedStreamReset) { response_encoder_->getStream().resetStream(StreamResetReason::LocalRefusedStreamReset); } -TEST_P(Http2CodecImplTest, InvalidFrame) { +TEST_P(Http2CodecImplTest, InvalidHeadersFrame) { + initialize(); + + EXPECT_THROW(request_encoder_->encodeHeaders(TestHeaderMapImpl{}, true), CodecProtocolException); + EXPECT_EQ(1, stats_store_.counter("http2.rx_messaging_error").value()); +} + +TEST_P(Http2CodecImplTest, InvalidHeadersFrameAllowed) { + stream_error_on_invalid_http_messaging_ = true; initialize(); MockStreamCallbacks request_callbacks; diff --git a/test/common/http/http2/http2_frame.cc b/test/common/http/http2/http2_frame.cc index fc96cf77b852..368630e1f6ec 100644 --- a/test/common/http/http2/http2_frame.cc +++ b/test/common/http/http2/http2_frame.cc @@ -104,6 +104,12 @@ void Http2Frame::appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::stri appendData(value); } +void Http2Frame::appendEmptyHeader() { + data_.push_back(0x40); + data_.push_back(0x00); + data_.push_back(0x00); +} + Http2Frame Http2Frame::makePingFrame(absl::string_view data) { static constexpr size_t kPingPayloadSize = 8; Http2Frame frame; @@ -169,6 +175,21 @@ Http2Frame Http2Frame::makeMalformedRequest(uint32_t stream_index) { return frame; } +Http2Frame Http2Frame::makeMalformedRequestWithZerolenHeader(uint32_t stream_index, + absl::string_view host, + absl::string_view path) { + Http2Frame frame; + frame.buildHeader(Type::HEADERS, 0, orFlags(HeadersFlags::END_STREAM, HeadersFlags::END_HEADERS), + makeRequestStreamId(stream_index)); + frame.appendStaticHeader(StaticHeaderIndex::METHOD_GET); + frame.appendStaticHeader(StaticHeaderIndex::SCHEME_HTTPS); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::PATH, path); + frame.appendHeaderWithoutIndexing(StaticHeaderIndex::HOST, host); + frame.appendEmptyHeader(); + frame.adjustPayloadSize(); + return frame; +} + Http2Frame Http2Frame::makeRequest(uint32_t stream_index, absl::string_view host, absl::string_view path) { Http2Frame frame; diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h index 760ae69e3575..52b838dbb987 100644 --- a/test/common/http/http2/http2_frame.h +++ b/test/common/http/http2/http2_frame.h @@ -79,6 +79,9 @@ class Http2Frame { static Http2Frame makePriorityFrame(uint32_t stream_index, uint32_t dependent_index); static Http2Frame makeWindowUpdateFrame(uint32_t stream_index, uint32_t increment); static Http2Frame makeMalformedRequest(uint32_t stream_index); + static Http2Frame makeMalformedRequestWithZerolenHeader(uint32_t stream_index, + absl::string_view host, + absl::string_view path); static Http2Frame makeRequest(uint32_t stream_index, absl::string_view host, absl::string_view path); static Http2Frame makePostRequest(uint32_t stream_index, absl::string_view host, @@ -126,6 +129,7 @@ class Http2Frame { // Headers are directly encoded void appendStaticHeader(StaticHeaderIndex index); void appendHeaderWithoutIndexing(StaticHeaderIndex index, absl::string_view value); + void appendEmptyHeader(); // This method updates payload length in the HTTP2 header based on the size of the data_ void adjustPayloadSize() { diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index 5861292c9c42..b1a1bb3c6368 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1584,6 +1584,12 @@ TEST_P(Http2FloodMitigationTest, Data) { // Verify that the server can detect flood of RST_STREAM frames. TEST_P(Http2FloodMitigationTest, RST_STREAM) { + // Use invalid HTTP headers to trigger sending RST_STREAM frames. + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true); + }); beginSession(); int i = 0; @@ -1717,4 +1723,54 @@ TEST_P(Http2FloodMitigationTest, WindowUpdate) { "http2.inbound_window_update_frames_flood"); } +// Verify that the HTTP/2 connection is terminated upon receiving invalid HEADERS frame. +TEST_P(Http2FloodMitigationTest, ZerolenHeader) { + beginSession(); + + // Send invalid request. + uint32_t request_idx = 0; + auto request = Http2Frame::makeMalformedRequestWithZerolenHeader(request_idx, "host", "/"); + sendFame(request); + + tcp_client_->waitForDisconnect(); + + EXPECT_EQ(1, test_server_->counter("http2.rx_messaging_error")->value()); + EXPECT_EQ(1, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + +// Verify that only the offending stream is terminated upon receiving invalid HEADERS frame. +TEST_P(Http2FloodMitigationTest, ZerolenHeaderAllowed) { + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true); + }); + autonomous_upstream_ = true; + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Send invalid request. + uint32_t request_idx = 0; + auto request = Http2Frame::makeMalformedRequestWithZerolenHeader(request_idx, "host", "/"); + sendFame(request); + // Make sure we've got RST_STREAM from the server. + auto response = readFrame(); + EXPECT_EQ(Http2Frame::Type::RST_STREAM, response.type()); + + // Send valid request using the same connection. + request_idx++; + request = Http2Frame::makeRequest(request_idx, "host", "/"); + sendFame(request); + response = readFrame(); + EXPECT_EQ(Http2Frame::Type::HEADERS, response.type()); + EXPECT_EQ(Http2Frame::ResponseStatus::_200, response.responseStatus()); + + tcp_client_->close(); + + EXPECT_EQ(1, test_server_->counter("http2.rx_messaging_error")->value()); + EXPECT_EQ(0, + test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); +} + } // namespace Envoy diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index d550d793b999..5d6bd41de406 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -592,6 +592,36 @@ TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLength) { {"content-length", "-1"}}); auto response = std::move(encoder_decoder.second); + codec_client_->waitForDisconnect(); + + if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + ASSERT_TRUE(response->complete()); + EXPECT_EQ("400", response->headers().Status()->value().getStringView()); + } else { + ASSERT_TRUE(response->reset()); + EXPECT_EQ(Http::StreamResetReason::ConnectionTermination, response->reset_reason()); + } +} + +// TODO(PiotrSikora): move this HTTP/2 only variant to http2_integration_test.cc. +TEST_P(DownstreamProtocolIntegrationTest, InvalidContentLengthAllowed) { + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true); + }); + + initialize(); + + codec_client_ = makeHttpConnection(lookupPort("http")); + + auto encoder_decoder = + codec_client_->startRequest(Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":authority", "host"}, + {"content-length", "-1"}}); + auto response = std::move(encoder_decoder.second); + if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { codec_client_->waitForDisconnect(); } else { @@ -618,6 +648,34 @@ TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengths) { {"content-length", "3,2"}}); auto response = std::move(encoder_decoder.second); + codec_client_->waitForDisconnect(); + + if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { + ASSERT_TRUE(response->complete()); + EXPECT_EQ("400", response->headers().Status()->value().getStringView()); + } else { + ASSERT_TRUE(response->reset()); + EXPECT_EQ(Http::StreamResetReason::ConnectionTermination, response->reset_reason()); + } +} + +// TODO(PiotrSikora): move this HTTP/2 only variant to http2_integration_test.cc. +TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengthsAllowed) { + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true); + }); + + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto encoder_decoder = + codec_client_->startRequest(Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/test/long/url"}, + {":authority", "host"}, + {"content-length", "3,2"}}); + auto response = std::move(encoder_decoder.second); + if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) { codec_client_->waitForDisconnect(); } else { From 79cbdcae3dda80ed24714b1c0fb1367474386c9f Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Wed, 31 Jul 2019 14:29:14 -0700 Subject: [PATCH 376/542] connection: always disable reads when connection is closed with the FlushWriteAndDelay (#28) Signed-off-by: Yan Avlasov --- source/common/http/conn_manager_impl.cc | 20 +++++------ source/common/http/conn_manager_impl.h | 2 +- source/common/network/connection_impl.cc | 2 ++ test/common/http/conn_manager_impl_test.cc | 3 +- test/common/network/connection_impl_test.cc | 5 +++ test/integration/http2_integration_test.cc | 38 ++++++++++++++------- 6 files changed, 43 insertions(+), 27 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index e461a40ea361..2d907859b8ce 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -258,17 +258,16 @@ StreamDecoder& ConnectionManagerImpl::newStream(StreamEncoder& response_encoder, return **streams_.begin(); } -void ConnectionManagerImpl::handleCodecException(const char* error, - Network::ConnectionCloseType close_type) { +void ConnectionManagerImpl::handleCodecException(const char* error) { ENVOY_CONN_LOG(debug, "dispatch error: {}", read_callbacks_->connection(), error); - // In the protocol error case, we need to reset all streams now. If the close_type is - // FlushWriteAndDelay, the connection might stick around long enough for a pending stream to come - // back and try to encode. In other cases it avoids needless processing of upstream responses when - // downstream connection is closed. + // In the protocol error case, we need to reset all streams now. The connection might stick around + // long enough for a pending stream to come back and try to encode. resetAllStreams(); - read_callbacks_->connection().close(close_type); + // HTTP/1.1 codec has already sent a 400 response if possible. HTTP/2 codec has already sent + // GOAWAY. + read_callbacks_->connection().close(Network::ConnectionCloseType::FlushWriteAndDelay); } Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool) { @@ -290,14 +289,11 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool try { codec_->dispatch(data); } catch (const FrameFloodException& e) { - // Abortively close flooded connections - handleCodecException(e.what(), Network::ConnectionCloseType::NoFlush); + handleCodecException(e.what()); return Network::FilterStatus::StopIteration; } catch (const CodecProtocolException& e) { stats_.named_.downstream_cx_protocol_error_.inc(); - // HTTP/1.1 codec has already sent a 400 response if possible. HTTP/2 codec has already sent - // GOAWAY. - handleCodecException(e.what(), Network::ConnectionCloseType::FlushWriteAndDelay); + handleCodecException(e.what()); return Network::FilterStatus::StopIteration; } diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 3091f2d98c9a..84dde922406e 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -653,7 +653,7 @@ class ConnectionManagerImpl : Logger::Loggable, void onDrainTimeout(); void startDrainSequence(); Tracing::HttpTracer& tracer() { return http_context_.tracer(); } - void handleCodecException(const char* error, Network::ConnectionCloseType close_type); + void handleCodecException(const char* error); enum class DrainState { NotDraining, Draining, Closing }; diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index 2309112e71a4..a5f1b4e7b8fe 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -118,6 +118,8 @@ void ConnectionImpl::close(ConnectionCloseType type) { if (!inDelayedClose()) { initializeDelayedCloseTimer(); delayed_close_state_ = DelayedCloseState::CloseAfterFlushAndWait; + // Monitor for the peer closing the connection. + file_event_->setEnabled(enable_half_close_ ? 0 : Event::FileReadyType::Closed); } } else { closeSocket(ConnectionEvent::LocalClose); diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 1ea7083c5794..811c5e98d473 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2330,7 +2330,8 @@ TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); // FrameFloodException should result in reset of the streams followed by abortive close. - EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); + EXPECT_CALL(filter_callbacks_.connection_, + close(Network::ConnectionCloseType::FlushWriteAndDelay)); // Kick off the incoming data. Buffer::OwnedImpl fake_input("1234"); diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 0526b0ec97dc..e4f280e96a17 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -1736,6 +1736,7 @@ TEST_F(PostCloseConnectionImplTest, ReadAfterCloseFlushWriteDelayIgnored) { // Delayed connection close. EXPECT_CALL(dispatcher_, createTimer_(_)); + EXPECT_CALL(*file_event_, setEnabled(Event::FileReadyType::Closed)); connection_->close(ConnectionCloseType::FlushWriteAndDelay); // Read event, doRead() happens on connection but no filter onData(). @@ -1760,6 +1761,10 @@ TEST_F(PostCloseConnectionImplTest, ReadAfterCloseFlushWriteDelayIgnoredWithWrit // Delayed connection close. EXPECT_CALL(dispatcher_, createTimer_(_)); + // With half-close semantics enabled we will not wait for early close notification. + // See the `Envoy::Network::ConnectionImpl::readDisable()' method for more details. + EXPECT_CALL(*file_event_, setEnabled(0)); + connection_->enableHalfClose(true); connection_->close(ConnectionCloseType::FlushWriteAndDelay); // Read event, doRead() happens on connection but no filter onData(). diff --git a/test/integration/http2_integration_test.cc b/test/integration/http2_integration_test.cc index b1a1bb3c6368..197522b3cd59 100644 --- a/test/integration/http2_integration_test.cc +++ b/test/integration/http2_integration_test.cc @@ -1516,8 +1516,7 @@ void Http2FloodMitigationTest::floodServer(const Http2Frame& frame, const std::s EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } @@ -1539,9 +1538,10 @@ void Http2FloodMitigationTest::floodServer(absl::string_view host, absl::string_ total_bytes_sent += request.size(); } EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; - EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + if (!flood_stat.empty()) { + EXPECT_EQ(1, test_server_->counter(flood_stat)->value()); + } + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } @@ -1606,11 +1606,26 @@ TEST_P(Http2FloodMitigationTest, RST_STREAM) { } EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken."; EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } +// Verify that the server stop reading downstream connection on protocol error. +TEST_P(Http2FloodMitigationTest, TooManyStreams) { + config_helper_.addConfigModifier( + [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + hcm.mutable_http2_protocol_options()->mutable_max_concurrent_streams()->set_value(2); + }); + autonomous_upstream_ = true; + beginSession(); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + + // Exceed the number of streams allowed by the server. The server should stop reading from the + // client. Verify that the client was unable to stuff a lot of data into the server. + floodServer("host", "/test/long/url", Http2Frame::ResponseStatus::_200, ""); +} + TEST_P(Http2FloodMitigationTest, EmptyHeaders) { config_helper_.addConfigModifier( [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) @@ -1628,8 +1643,7 @@ TEST_P(Http2FloodMitigationTest, EmptyHeaders) { tcp_client_->waitForDisconnect(); EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } @@ -1648,8 +1662,7 @@ TEST_P(Http2FloodMitigationTest, EmptyHeadersContinuation) { tcp_client_->waitForDisconnect(); EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } @@ -1669,8 +1682,7 @@ TEST_P(Http2FloodMitigationTest, EmptyData) { tcp_client_->waitForDisconnect(); EXPECT_EQ(1, test_server_->counter("http2.inbound_empty_frames_flood")->value()); - // Verify that connection was closed abortively - EXPECT_EQ(0, + EXPECT_EQ(1, test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value()); } From d3d5dcdb35f972ea0337af2bf911ce5c84051fa3 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Fri, 2 Aug 2019 15:16:12 -0700 Subject: [PATCH 377/542] http2: configure HTTP/2 flood mitigation through runtime. (#32) Signed-off-by: Yan Avlasov --- docs/root/intro/version_history.rst | 6 + source/common/http/http2/BUILD | 1 + source/common/http/http2/codec_impl.cc | 53 ++++++ source/common/http/http2/codec_impl.h | 23 +-- source/common/runtime/runtime_impl.cc | 11 ++ source/common/runtime/runtime_impl.h | 1 + test/common/http/http2/BUILD | 5 + test/common/http/http2/codec_impl_test.cc | 207 +++++++++++++++++++++- test/common/runtime/runtime_impl_test.cc | 10 ++ 9 files changed, 295 insertions(+), 22 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 18bc47adcac3..7149dfcc0716 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -30,11 +30,17 @@ Version history ======================== * http: added mitigation of client initiated atacks that result in flooding of the downstream HTTP/2 connections. * http: added :ref:`inbound_empty_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_consecutive_inbound_frames_with_empty_payload` overrides :ref:`max_consecutive_inbound_frames_with_empty_payload setting `. Large override value (i.e. 2147483647) effectively disables mitigation of inbound frames with empty payload. * http: added :ref:`inbound_priority_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound PRIORITY frames. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_inbound_priority_frames_per_stream` overrides :ref:`max_inbound_priority_frames_per_stream setting `. Large override value effectively disables flood mitigation of inbound PRIORITY frames. * http: added :ref:`inbound_window_update_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound WINDOW_UPDATE frames. The limit is configured by setting the :ref:`max_inbound_window_update_frames_per_data_frame_sent config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_inbound_window_update_frames_per_data_frame_sent` overrides :ref:`max_inbound_window_update_frames_per_data_frame_sent setting `. Large override value effectively disables flood mitigation of inbound WINDOW_UPDATE frames. * http: added :ref:`outbound_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit. The limit is configured by setting the :ref:`max_outbound_frames config setting ` + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_outbound_frames` overrides :ref:`max_outbound_frames config setting `. Large override value effectively disables flood mitigation of outbound frames of all types. * http: added :ref:`outbound_control_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the outbound queue limit for PING, SETTINGS and RST_STREAM frames. The limit is configured by setting the :ref:`max_outbound_control_frames config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.max_outbound_control_frames` overrides :ref:`max_outbound_control_frames config setting `. Large override value effectively disables flood mitigation of outbound frames of types PING, SETTINGS and RST_STREAM. * http: enabled strict validation of HTTP/2 messaging. Previous behavior can be restored using :ref:`stream_error_on_invalid_http_messaging config setting `. + Runtime feature `envoy.reloadable_features.http2_protocol_options.stream_error_on_invalid_http_messaging` overrides :ref:`stream_error_on_invalid_http_messaging config setting `. 1.11.0 (July 11, 2019) ====================== diff --git a/source/common/http/http2/BUILD b/source/common/http/http2/BUILD index 7d51a2208dff..2133d6238e67 100644 --- a/source/common/http/http2/BUILD +++ b/source/common/http/http2/BUILD @@ -40,6 +40,7 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", + "//source/common/runtime:runtime_lib", ], ) diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index e45f07a101af..1752eb299f10 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -346,6 +346,59 @@ void ConnectionImpl::StreamImpl::onMetadataDecoded(MetadataMapPtr&& metadata_map decoder_->decodeMetadata(std::move(metadata_map_ptr)); } +namespace { + +const char InvalidHttpMessagingOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options.stream_error_on_invalid_http_messaging"; +const char MaxOutboundFramesOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options.max_outbound_frames"; +const char MaxOutboundControlFramesOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options.max_outbound_control_frames"; +const char MaxConsecutiveInboundFramesWithEmptyPayloadOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options." + "max_consecutive_inbound_frames_with_empty_payload"; +const char MaxInboundPriorityFramesPerStreamOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options.max_inbound_priority_frames_per_stream"; +const char MaxInboundWindowUpdateFramesPerDataFrameSentOverrideKey[] = + "envoy.reloadable_features.http2_protocol_options." + "max_inbound_window_update_frames_per_data_frame_sent"; + +bool checkRuntimeOverride(bool config_value, const char* override_key) { + return Runtime::runtimeFeatureEnabled(override_key) ? true : config_value; +} + +} // namespace + +ConnectionImpl::ConnectionImpl(Network::Connection& connection, Stats::Scope& stats, + const Http2Settings& http2_settings, + const uint32_t max_request_headers_kb) + : stats_{ALL_HTTP2_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http2."))}, connection_(connection), + max_request_headers_kb_(max_request_headers_kb), + per_stream_buffer_limit_(http2_settings.initial_stream_window_size_), + stream_error_on_invalid_http_messaging_(checkRuntimeOverride( + http2_settings.stream_error_on_invalid_http_messaging_, InvalidHttpMessagingOverrideKey)), + flood_detected_(false), + max_outbound_frames_( + Runtime::getInteger(MaxOutboundFramesOverrideKey, http2_settings.max_outbound_frames_)), + frame_buffer_releasor_([this](const Buffer::OwnedBufferFragmentImpl* fragment) { + releaseOutboundFrame(fragment); + }), + max_outbound_control_frames_(Runtime::getInteger( + MaxOutboundControlFramesOverrideKey, http2_settings.max_outbound_control_frames_)), + control_frame_buffer_releasor_([this](const Buffer::OwnedBufferFragmentImpl* fragment) { + releaseOutboundControlFrame(fragment); + }), + max_consecutive_inbound_frames_with_empty_payload_( + Runtime::getInteger(MaxConsecutiveInboundFramesWithEmptyPayloadOverrideKey, + http2_settings.max_consecutive_inbound_frames_with_empty_payload_)), + max_inbound_priority_frames_per_stream_( + Runtime::getInteger(MaxInboundPriorityFramesPerStreamOverrideKey, + http2_settings.max_inbound_priority_frames_per_stream_)), + max_inbound_window_update_frames_per_data_frame_sent_(Runtime::getInteger( + MaxInboundWindowUpdateFramesPerDataFrameSentOverrideKey, + http2_settings.max_inbound_window_update_frames_per_data_frame_sent_)), + dispatching_(false), raised_goaway_(false), pending_deferred_reset_(false) {} + ConnectionImpl::~ConnectionImpl() { nghttp2_session_del(session_); } void ConnectionImpl::dispatch(Buffer::Instance& data) { diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index 15800f25a523..b9ba8ec31809 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -21,6 +21,7 @@ #include "common/http/http2/metadata_decoder.h" #include "common/http/http2/metadata_encoder.h" #include "common/http/utility.h" +#include "common/runtime/runtime_impl.h" #include "absl/types/optional.h" #include "nghttp2/nghttp2.h" @@ -78,27 +79,7 @@ class Utility { class ConnectionImpl : public virtual Connection, protected Logger::Loggable { public: ConnectionImpl(Network::Connection& connection, Stats::Scope& stats, - const Http2Settings& http2_settings, const uint32_t max_request_headers_kb) - : stats_{ALL_HTTP2_CODEC_STATS(POOL_COUNTER_PREFIX(stats, "http2."))}, - connection_(connection), max_request_headers_kb_(max_request_headers_kb), - per_stream_buffer_limit_(http2_settings.initial_stream_window_size_), - stream_error_on_invalid_http_messaging_( - http2_settings.stream_error_on_invalid_http_messaging_), - flood_detected_(false), max_outbound_frames_(http2_settings.max_outbound_frames_), - frame_buffer_releasor_([this](const Buffer::OwnedBufferFragmentImpl* fragment) { - releaseOutboundFrame(fragment); - }), - max_outbound_control_frames_(http2_settings.max_outbound_control_frames_), - control_frame_buffer_releasor_([this](const Buffer::OwnedBufferFragmentImpl* fragment) { - releaseOutboundControlFrame(fragment); - }), - max_consecutive_inbound_frames_with_empty_payload_( - http2_settings.max_consecutive_inbound_frames_with_empty_payload_), - max_inbound_priority_frames_per_stream_( - http2_settings.max_inbound_priority_frames_per_stream_), - max_inbound_window_update_frames_per_data_frame_sent_( - http2_settings.max_inbound_window_update_frames_per_data_frame_sent_), - dispatching_(false), raised_goaway_(false), pending_deferred_reset_(false) {} + const Http2Settings& http2_settings, const uint32_t max_request_headers_kb); ~ConnectionImpl() override; diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 8be5d9a964f8..2b67fdeafb78 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -35,6 +35,17 @@ bool runtimeFeatureEnabled(absl::string_view feature) { return RuntimeFeaturesDefaults::get().enabledByDefault(feature); } +uint64_t getInteger(absl::string_view feature, uint64_t default_value) { + ASSERT(absl::StartsWith(feature, "envoy.reloadable_features")); + if (Runtime::LoaderSingleton::getExisting()) { + return Runtime::LoaderSingleton::getExisting()->threadsafeSnapshot()->getInteger( + std::string(feature), default_value); + } + ENVOY_LOG_TO_LOGGER(Envoy::Logger::Registry::getLog(Envoy::Logger::Id::runtime), warn, + "Unable to use runtime singleton for feature {}", feature); + return default_value; +} + const size_t RandomGeneratorImpl::UUID_LENGTH = 36; uint64_t RandomGeneratorImpl::random() { diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index fd1b6d271ade..06be894c5aec 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -31,6 +31,7 @@ namespace Envoy { namespace Runtime { bool runtimeFeatureEnabled(absl::string_view feature); +uint64_t getInteger(absl::string_view feature, uint64_t default_value); using RuntimeSingleton = ThreadSafeSingleton; diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index 354055602126..ae51864b72d9 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -22,8 +22,13 @@ envoy_cc_test( "//source/common/http/http2:codec_lib", "//source/common/stats:stats_lib", "//test/common/http:common_lib", + "//test/common/http/http2:http2_frame", "//test/mocks/http:http_mocks", + "//test/mocks/init:init_mocks", + "//test/mocks/local_info:local_info_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:utility_lib", ], diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index 0dc5bb358a2f..e738c140aff0 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -9,8 +9,13 @@ #include "common/http/http2/codec_impl.h" #include "test/common/http/common.h" +#include "test/common/http/http2/http2_frame.h" #include "test/mocks/http/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/thread_local/mocks.h" #include "test/test_common/printers.h" #include "test/test_common/utility.h" @@ -164,7 +169,89 @@ class Http2CodecImplTest : public ::testing::TestWithParam(GetParam()), ::testing::get<1>(GetParam())) {} + : Http2CodecImplTestFixture(::testing::get<0>(GetParam()), ::testing::get<1>(GetParam())), + api_(Api::createApiForTest()) { + envoy::config::bootstrap::v2::LayeredRuntime config; + config.add_layers()->mutable_admin_layer(); + + // Create a runtime loader, so that tests can manually manipulate runtime + // guarded features. + loader_ = std::make_unique( + std::make_unique(dispatcher_, tls_, config, local_info_, init_manager_, + store_, generator_, validation_visitor_, *api_)); + } + +protected: + void priorityFlood() { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers, "POST"); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + nghttp2_priority_spec spec = {0, 10, 0}; + // HTTP/2 codec adds 1 to the number of active streams when computing PRIORITY frames limit + constexpr uint32_t max_allowed = + 2 * Http2Settings::DEFAULT_MAX_INBOUND_PRIORITY_FRAMES_PER_STREAM; + for (uint32_t i = 0; i < max_allowed + 1; ++i) { + EXPECT_EQ(0, nghttp2_submit_priority(client_->session(), NGHTTP2_FLAG_NONE, 1, &spec)); + } + } + + void windowUpdateFlood() { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, true)); + request_encoder_->encodeHeaders(request_headers, true); + + // Send one DATA frame back + EXPECT_CALL(response_decoder_, decodeHeaders_(_, false)); + EXPECT_CALL(response_decoder_, decodeData(_, false)); + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + + // See the limit formula in the + // `Envoy::Http::Http2::ServerConnectionImpl::checkInboundFrameLimits()' method. + constexpr uint32_t max_allowed = + 1 + 2 * (Http2Settings::DEFAULT_MAX_INBOUND_WINDOW_UPDATE_FRAMES_PER_DATA_FRAME_SENT + 1); + for (uint32_t i = 0; i < max_allowed + 1; ++i) { + EXPECT_EQ(0, nghttp2_submit_window_update(client_->session(), NGHTTP2_FLAG_NONE, 1, 1)); + } + } + + void emptyDataFlood(Buffer::OwnedImpl& data) { + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers, "POST"); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + // HTTP/2 codec does not send empty DATA frames with no END_STREAM flag. + // To make this work, send raw bytes representing empty DATA frames bypassing client codec. + Http2Frame emptyDataFrame = Http2Frame::makeEmptyDataFrame(0); + constexpr uint32_t max_allowed = + Http2Settings::DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD; + for (uint32_t i = 0; i < max_allowed + 1; ++i) { + data.add(emptyDataFrame.data(), emptyDataFrame.size()); + } + } + +private: + Event::MockDispatcher dispatcher_; + NiceMock tls_; + Stats::IsolatedStoreImpl store_; + Runtime::MockRandomGenerator generator_; + Api::ApiPtr api_; + NiceMock local_info_; + Init::MockManager init_manager_; + NiceMock validation_visitor_; + std::unique_ptr loader_; }; TEST_P(Http2CodecImplTest, ShutdownNotice) { @@ -408,6 +495,25 @@ TEST_P(Http2CodecImplTest, InvalidHeadersFrameAllowed) { server_wrapper_.dispatch(Buffer::OwnedImpl(), *server_); } +TEST_P(Http2CodecImplTest, InvalidHeadersFrameOverriden) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.stream_error_on_invalid_http_messaging", + "true"}}); + initialize(); + + MockStreamCallbacks request_callbacks; + request_encoder_->getStream().addCallbacks(request_callbacks); + + ON_CALL(client_connection_, write(_, _)) + .WillByDefault( + Invoke([&](Buffer::Instance& data, bool) -> void { server_wrapper_.buffer_.add(data); })); + + request_encoder_->encodeHeaders(TestHeaderMapImpl{}, true); + EXPECT_CALL(server_stream_callbacks_, onResetStream(StreamResetReason::LocalReset, _)); + EXPECT_CALL(request_callbacks, onResetStream(StreamResetReason::RemoteReset, _)); + server_wrapper_.dispatch(Buffer::OwnedImpl(), *server_); +} + TEST_P(Http2CodecImplTest, TrailingHeaders) { initialize(); @@ -1144,6 +1250,28 @@ TEST_P(Http2CodecImplTest, PingFlood) { EXPECT_EQ(1, stats_store_.counter("http2.outbound_control_flood").value()); } +// Verify that codec allows PING flood when mitigation is disabled +TEST_P(Http2CodecImplTest, PingFloodMitigationDisabled) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_outbound_control_frames", + "2147483647"}}); + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + // Send one frame above the outbound control queue size limit + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES + 1; ++i) { + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + } + + EXPECT_CALL(server_connection_, write(_, _)) + .Times(Http2Settings::DEFAULT_MAX_OUTBOUND_CONTROL_FRAMES + 1); + EXPECT_NO_THROW(client_->sendPendingFrames()); +} + // Verify that outbound control frame counter decreases when send buffer is drained TEST_P(Http2CodecImplTest, PingFloodCounterReset) { static const int kMaxOutboundControlFrames = 100; @@ -1249,6 +1377,36 @@ TEST_P(Http2CodecImplTest, ResponseDataFlood) { EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); } +// Verify that codec allows outbound DATA flood when mitigation is disabled +TEST_P(Http2CodecImplTest, ResponseDataFloodMitigationDisabled) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_outbound_frames", "2147483647"}}); + initialize(); + + TestHeaderMapImpl request_headers; + HttpTestUtility::addDefaultHeaders(request_headers); + EXPECT_CALL(request_decoder_, decodeHeaders_(_, false)); + request_encoder_->encodeHeaders(request_headers, false); + + // +2 is to account for HEADERS and PING ACK, that is used to trigger mitigation + EXPECT_CALL(server_connection_, write(_, _)) + .Times(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 2); + EXPECT_CALL(response_decoder_, decodeHeaders_(_, false)).Times(1); + EXPECT_CALL(response_decoder_, decodeData(_, false)) + .Times(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES); + TestHeaderMapImpl response_headers{{":status", "200"}}; + response_encoder_->encodeHeaders(response_headers, false); + // Account for the single HEADERS frame above + for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; ++i) { + Buffer::OwnedImpl data("0"); + EXPECT_NO_THROW(response_encoder_->encodeData(data, false)); + } + // Presently flood mitigation is done only when processing downstream data + // So we need to send stream from downstream client to trigger mitigation + EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr)); + EXPECT_NO_THROW(client_->sendPendingFrames()); +} + // Verify that outbound frame counter decreases when send buffer is drained TEST_P(Http2CodecImplTest, ResponseDataFloodCounterReset) { static const int kMaxOutboundFrames = 100; @@ -1323,6 +1481,53 @@ TEST_P(Http2CodecImplTest, PingStacksWithDataFlood) { EXPECT_EQ(1, stats_store_.counter("http2.outbound_flood").value()); } +TEST_P(Http2CodecImplTest, PriorityFlood) { + priorityFlood(); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); +} + +TEST_P(Http2CodecImplTest, PriorityFloodOverride) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_inbound_priority_frames_per_stream", + "2147483647"}}); + + priorityFlood(); + EXPECT_NO_THROW(client_->sendPendingFrames()); +} + +TEST_P(Http2CodecImplTest, WindowUpdateFlood) { + windowUpdateFlood(); + EXPECT_THROW(client_->sendPendingFrames(), FrameFloodException); +} + +TEST_P(Http2CodecImplTest, WindowUpdateFloodOverride) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_inbound_window_update_frames_per_" + "data_frame_sent", + "2147483647"}}); + windowUpdateFlood(); + EXPECT_NO_THROW(client_->sendPendingFrames()); +} + +TEST_P(Http2CodecImplTest, EmptyDataFlood) { + Buffer::OwnedImpl data; + emptyDataFlood(data); + EXPECT_CALL(request_decoder_, decodeData(_, false)); + EXPECT_THROW(server_wrapper_.dispatch(data, *server_), FrameFloodException); +} + +TEST_P(Http2CodecImplTest, EmptyDataFloodOverride) { + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.http2_protocol_options.max_consecutive_inbound_frames_with_" + "empty_payload", + "2147483647"}}); + Buffer::OwnedImpl data; + emptyDataFlood(data); + EXPECT_CALL(request_decoder_, decodeData(_, false)) + .Times(Http2Settings::DEFAULT_MAX_CONSECUTIVE_INBOUND_FRAMES_WITH_EMPTY_PAYLOAD + 1); + EXPECT_NO_THROW(server_wrapper_.dispatch(data, *server_)); +} + } // namespace Http2 } // namespace Http } // namespace Envoy diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index dc3a0c34faee..82fedad731af 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -709,6 +709,16 @@ TEST(NoRuntime, FeatureEnabled) { EXPECT_EQ(true, runtimeFeatureEnabled("envoy.reloadable_features.test_feature_true")); } +TEST(NoRuntime, DefaultIntValues) { + // Make sure the registry is not set up. + ASSERT_TRUE(Runtime::LoaderSingleton::getExisting() == nullptr); + + // Feature defaults should still work. + EXPECT_EQ(0x1230000ABCDULL, + getInteger("envoy.reloadable_features.test_int_feature_default", 0x1230000ABCDULL)); + EXPECT_EQ(0, getInteger("envoy.reloadable_features.test_int_feature_zero", 0)); +} + // Test RTDS layer(s). class RtdsLoaderImplTest : public LoaderImplTest { public: From f2129cbfeff3b206fb5d00bbae06049a4a21383e Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 9 Aug 2019 22:04:31 -0700 Subject: [PATCH 378/542] http2: configure logging of HTTP/2 flood attacks through runtime. (#34) Signed-off-by: Matt Klein --- docs/root/intro/version_history.rst | 2 +- source/common/http/conn_manager_impl.cc | 13 +++++++++ test/common/http/BUILD | 1 + test/common/http/conn_manager_impl_test.cc | 34 +++++++++++++++++++++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 7149dfcc0716..b188266557da 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -28,7 +28,7 @@ Version history 1.11.1 (August 13, 2019) ======================== -* http: added mitigation of client initiated atacks that result in flooding of the downstream HTTP/2 connections. +* http: added mitigation of client initiated attacks that result in flooding of the downstream HTTP/2 connections. Those attacks can be logged at the "warning" level when the runtime feature `http.connection_manager.log_flood_exception` is enabled. The runtime setting defaults to disabled to avoid log spam when under attack. * http: added :ref:`inbound_empty_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on consecutive inbound frames with an empty payload and no end stream flag. The limit is configured by setting the :ref:`max_consecutive_inbound_frames_with_empty_payload config setting `. Runtime feature `envoy.reloadable_features.http2_protocol_options.max_consecutive_inbound_frames_with_empty_payload` overrides :ref:`max_consecutive_inbound_frames_with_empty_payload setting `. Large override value (i.e. 2147483647) effectively disables mitigation of inbound frames with empty payload. * http: added :ref:`inbound_priority_frames_flood ` counter stat to the HTTP/2 codec stats, for tracking number of connections terminated for exceeding the limit on inbound PRIORITY frames. The limit is configured by setting the :ref:`max_inbound_priority_frames_per_stream config setting `. diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 2d907859b8ce..8ee08cf2b988 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -289,6 +289,19 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool try { codec_->dispatch(data); } catch (const FrameFloodException& e) { + // TODO(mattklein123): This is an emergency substitute for the lack of connection level + // logging in the HCM. In a public follow up change we will add full support for connection + // level logging in the HCM, similar to what we have in tcp_proxy. This will allow abuse + // indicators to be stored in the connection level stream info, and then matched, sampled, + // etc. when logged. + const envoy::type::FractionalPercent default_value; // 0 + if (runtime_.snapshot().featureEnabled("http.connection_manager.log_flood_exception", + default_value)) { + ENVOY_CONN_LOG(warn, "downstream HTTP flood from IP '{}': {}", + read_callbacks_->connection(), + read_callbacks_->connection().remoteAddress()->asString(), e.what()); + } + handleCodecException(e.what()); return Network::FilterStatus::StopIteration; } catch (const CodecProtocolException& e) { diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 5cfeaff671fa..56eefc2b25a6 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -211,6 +211,7 @@ envoy_cc_test( "//test/mocks/ssl:ssl_mocks", "//test/mocks/tracing:tracing_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:logging_lib", "//test/test_common:test_time_lib", ], ) diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 811c5e98d473..2ae4d2483eed 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -39,6 +39,7 @@ #include "test/mocks/tracing/mocks.h" #include "test/mocks/upstream/cluster_info.h" #include "test/mocks/upstream/mocks.h" +#include "test/test_common/logging.h" #include "test/test_common/printers.h" #include "test/test_common/test_time.h" @@ -55,6 +56,7 @@ using testing::HasSubstr; using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; +using testing::Matcher; using testing::NiceMock; using testing::Ref; using testing::Return; @@ -2335,7 +2337,37 @@ TEST_F(HttpConnectionManagerImplTest, FrameFloodError) { // Kick off the incoming data. Buffer::OwnedImpl fake_input("1234"); - conn_manager_->onData(fake_input, false); + EXPECT_LOG_NOT_CONTAINS("warning", "downstream HTTP flood", + conn_manager_->onData(fake_input, false)); +} + +// Verify that FrameFloodException causes connection to be closed abortively as well as logged +// if runtime indicates to do so. +TEST_F(HttpConnectionManagerImplTest, FrameFloodErrorWithLog) { + InSequence s; + setup(false, ""); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + conn_manager_->newStream(response_encoder_); + throw FrameFloodException("too many outbound frames."); + })); + + EXPECT_CALL(runtime_.snapshot_, featureEnabled("http.connection_manager.log_flood_exception", + Matcher(_))) + .WillOnce(Return(true)); + + EXPECT_CALL(response_encoder_.stream_, removeCallbacks(_)); + EXPECT_CALL(filter_factory_, createFilterChain(_)).Times(0); + + // FrameFloodException should result in reset of the streams followed by abortive close. + EXPECT_CALL(filter_callbacks_.connection_, + close(Network::ConnectionCloseType::FlushWriteAndDelay)); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + EXPECT_LOG_CONTAINS("warning", + "downstream HTTP flood from IP '0.0.0.0:0': too many outbound frames.", + conn_manager_->onData(fake_input, false)); } TEST_F(HttpConnectionManagerImplTest, IdleTimeoutNoCodec) { From 3f5580fb1ec23cc2e99d974d54eba054e14ee837 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 13 Aug 2019 14:39:00 -0400 Subject: [PATCH 379/542] config: enforcing terminal filters are the final filter in their respective chains (#7779) An (no longer annoyingly one-off) solution to the common problem folks run into with Envoy configs where they add their filters behind the router filter and don't get why things aren't working. Ditto for HCM, tcp_proxy etc for L4 Risk Level: Low (except for folks with broken config) Testing: new UT Docs Changes: n/a Release Notes: n/a Fixes #7767 Signed-off-by: Alyssa Wilk --- docs/root/intro/version_history.rst | 1 + include/envoy/server/filter_config.h | 10 +++ source/common/config/utility.h | 20 +++++ .../extensions/filters/http/router/config.h | 2 + .../filters/network/common/factory_base.h | 6 +- .../filters/network/dubbo_proxy/config.h | 2 +- .../extensions/filters/network/echo/config.cc | 1 + .../network/http_connection_manager/config.cc | 17 +++- .../network/http_connection_manager/config.h | 4 +- .../filters/network/redis_proxy/config.h | 2 +- .../filters/network/tcp_proxy/config.h | 2 +- .../filters/network/thrift_proxy/config.h | 2 +- source/server/listener_manager_impl.cc | 7 ++ .../network/dubbo_proxy/config_test.cc | 1 + .../http_connection_manager/config_test.cc | 85 ++++++++++++++++++- test/extensions/filters/network/rbac/BUILD | 1 + .../filters/network/rbac/integration_test.cc | 2 + .../network/redis_proxy/config_test.cc | 1 + .../filters/network/tcp_proxy/config_test.cc | 1 + .../network/thrift_proxy/config_test.cc | 1 + .../tcp_conn_pool_integration_test.cc | 1 + test/server/BUILD | 1 + test/server/listener_manager_impl_test.cc | 60 +++++++++++++ 23 files changed, 217 insertions(+), 13 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b188266557da..18986583b07e 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -6,6 +6,7 @@ Version history * access log: added :ref:`buffering ` and :ref:`periodical flushing ` support to gRPC access logger. Defaults to 16KB buffer and flushing every 1 second. * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. +* config: enforcing that terminal filters (e.g. HttpConnectionManager for L4, router for L7) be the last in their respective filter chains. * buffer filter: the buffer filter populates content-length header if not present, behavior can be disabled using the runtime feature `envoy.reloadable_features.buffer_filter_populate_content_length`. * config: added access log :ref:`extension filter`. * config: async data access for local and remote data source. diff --git a/include/envoy/server/filter_config.h b/include/envoy/server/filter_config.h index 38ea9070721c..8cd1b4058353 100644 --- a/include/envoy/server/filter_config.h +++ b/include/envoy/server/filter_config.h @@ -328,6 +328,11 @@ class NamedNetworkFilterConfigFactory : public ProtocolOptionsFactory { * produced by the factory. */ virtual std::string name() PURE; + + /** + * @return bool true if this filter must be the last filter in a filter chain, false otherwise. + */ + virtual bool isTerminalFilter() { return false; } }; /** @@ -428,6 +433,11 @@ class NamedHttpFilterConfigFactory : public ProtocolOptionsFactory { * produced by the factory. */ virtual std::string name() PURE; + + /** + * @return bool true if this filter must be the last filter in a filter chain, false otherwise. + */ + virtual bool isTerminalFilter() { return false; } }; } // namespace Configuration diff --git a/source/common/config/utility.h b/source/common/config/utility.h index 140f7f41e896..935699ecbdba 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -291,6 +291,26 @@ class Utility { * Return whether v1-style JSON filter config loading is allowed via 'deprecated_v1: true'. */ static bool allowDeprecatedV1Config(Runtime::Loader& runtime, const Json::Object& config); + + /** + * Verify any any filter designed to be terminal is configured to be terminal, and vice versa. + * @param name the name of the filter. + * @param name the type of filter. + * @param is_terminal_filter true if the filter is designed to be terminal. + * @param last_filter_in_current_config true if the filter is last in the configuration. + * @throws EnvoyException if there is a mismatch between design and configuration. + */ + static void validateTerminalFilters(const std::string& name, const char* filter_type, + bool is_terminal_filter, bool last_filter_in_current_config) { + if (is_terminal_filter && !last_filter_in_current_config) { + throw EnvoyException( + fmt::format("Error: {} must be the terminal {} filter.", name, filter_type)); + } else if (!is_terminal_filter && last_filter_in_current_config) { + throw EnvoyException( + fmt::format("Error: non-terminal filter {} is the last filter in a {} filter chain.", + name, filter_type)); + } + } }; } // namespace Config diff --git a/source/extensions/filters/http/router/config.h b/source/extensions/filters/http/router/config.h index 4dfde8845f1a..bfb7df66649d 100644 --- a/source/extensions/filters/http/router/config.h +++ b/source/extensions/filters/http/router/config.h @@ -26,6 +26,8 @@ class RouterFilterConfig createFilterFactory(const Json::Object& json_config, const std::string& stat_prefix, Server::Configuration::FactoryContext& context) override; + bool isTerminalFilter() override { return true; } + private: Http::FilterFactoryCb createFilterFactoryFromProtoTyped( const envoy::config::filter::http::router::v2::Router& proto_config, diff --git a/source/extensions/filters/network/common/factory_base.h b/source/extensions/filters/network/common/factory_base.h index 417ebdd85b20..0c241878d2e8 100644 --- a/source/extensions/filters/network/common/factory_base.h +++ b/source/extensions/filters/network/common/factory_base.h @@ -45,8 +45,11 @@ class FactoryBase : public Server::Configuration::NamedNetworkFilterConfigFactor std::string name() override { return name_; } + bool isTerminalFilter() override { return is_terminal_filter_; } + protected: - FactoryBase(const std::string& name) : name_(name) {} + FactoryBase(const std::string& name, bool is_terminal = false) + : name_(name), is_terminal_filter_(is_terminal) {} private: virtual Network::FilterFactoryCb @@ -59,6 +62,7 @@ class FactoryBase : public Server::Configuration::NamedNetworkFilterConfigFactor } const std::string name_; + const bool is_terminal_filter_; }; } // namespace Common diff --git a/source/extensions/filters/network/dubbo_proxy/config.h b/source/extensions/filters/network/dubbo_proxy/config.h index c0c09967b95a..0868acb5366e 100644 --- a/source/extensions/filters/network/dubbo_proxy/config.h +++ b/source/extensions/filters/network/dubbo_proxy/config.h @@ -24,7 +24,7 @@ class DubboProxyFilterConfigFactory : public Common::FactoryBase< envoy::config::filter::network::dubbo_proxy::v2alpha1::DubboProxy> { public: - DubboProxyFilterConfigFactory() : FactoryBase(NetworkFilterNames::get().DubboProxy) {} + DubboProxyFilterConfigFactory() : FactoryBase(NetworkFilterNames::get().DubboProxy, true) {} private: Network::FilterFactoryCb createFilterFactoryFromProtoTyped( diff --git a/source/extensions/filters/network/echo/config.cc b/source/extensions/filters/network/echo/config.cc index f4b43736ea22..f497c57eaf87 100644 --- a/source/extensions/filters/network/echo/config.cc +++ b/source/extensions/filters/network/echo/config.cc @@ -35,6 +35,7 @@ class EchoConfigFactory : public Server::Configuration::NamedNetworkFilterConfig } std::string name() override { return NetworkFilterNames::get().Echo; } + bool isTerminalFilter() override { return true; } }; /** diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 02feb699471c..cc03d2392364 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -307,7 +307,10 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( const auto& filters = config.http_filters(); for (int32_t i = 0; i < filters.size(); i++) { - processFilter(filters[i], i, "http", filter_factories_); + bool is_terminal = false; + processFilter(filters[i], i, "http", filter_factories_, is_terminal); + Config::Utility::validateTerminalFilters(filters[i].name(), "http", is_terminal, + i == filters.size() - 1); } for (const auto& upgrade_config : config.upgrade_configs()) { @@ -320,8 +323,12 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( } if (!upgrade_config.filters().empty()) { std::unique_ptr factories = std::make_unique(); - for (int32_t i = 0; i < upgrade_config.filters().size(); i++) { - processFilter(upgrade_config.filters(i), i, name, *factories); + for (int32_t j = 0; j < upgrade_config.filters().size(); j++) { + bool is_terminal = false; + processFilter(upgrade_config.filters(j), j, name, *factories, is_terminal); + Config::Utility::validateTerminalFilters(upgrade_config.filters(j).name(), "http upgrade", + is_terminal, + j == upgrade_config.filters().size() - 1); } upgrade_filter_factories_.emplace( std::make_pair(name, FilterConfig{std::move(factories), enabled})); @@ -335,7 +342,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( void HttpConnectionManagerConfig::processFilter( const envoy::config::filter::network::http_connection_manager::v2::HttpFilter& proto_config, - int i, absl::string_view prefix, std::list& filter_factories) { + int i, absl::string_view prefix, std::list& filter_factories, + bool& is_terminal) { const std::string& string_name = proto_config.name(); ENVOY_LOG(debug, " {} filter #{}", prefix, i); @@ -358,6 +366,7 @@ void HttpConnectionManagerConfig::processFilter( proto_config, context_.messageValidationVisitor(), factory); callback = factory.createFilterFactoryFromProto(*message, stats_prefix_, context_); } + is_terminal = factory.isTerminalFilter(); filter_factories.push_back(callback); } diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index 6cd895503aa6..8e8132e0aa01 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -33,7 +33,7 @@ class HttpConnectionManagerFilterConfigFactory envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager> { public: HttpConnectionManagerFilterConfigFactory() - : FactoryBase(NetworkFilterNames::get().HttpConnectionManager) {} + : FactoryBase(NetworkFilterNames::get().HttpConnectionManager, true) {} // NamedNetworkFilterConfigFactory Network::FilterFactoryCb @@ -145,7 +145,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, enum class CodecType { HTTP1, HTTP2, AUTO }; void processFilter( const envoy::config::filter::network::http_connection_manager::v2::HttpFilter& proto_config, - int i, absl::string_view prefix, FilterFactoriesList& filter_factories); + int i, absl::string_view prefix, FilterFactoriesList& filter_factories, bool& is_terminal); Server::Configuration::FactoryContext& context_; FilterFactoriesList filter_factories_; diff --git a/source/extensions/filters/network/redis_proxy/config.h b/source/extensions/filters/network/redis_proxy/config.h index 6a6bf7914e8b..37d6a7c0203c 100644 --- a/source/extensions/filters/network/redis_proxy/config.h +++ b/source/extensions/filters/network/redis_proxy/config.h @@ -41,7 +41,7 @@ class RedisProxyFilterConfigFactory envoy::config::filter::network::redis_proxy::v2::RedisProxy, envoy::config::filter::network::redis_proxy::v2::RedisProtocolOptions> { public: - RedisProxyFilterConfigFactory() : FactoryBase(NetworkFilterNames::get().RedisProxy) {} + RedisProxyFilterConfigFactory() : FactoryBase(NetworkFilterNames::get().RedisProxy, true) {} // NamedNetworkFilterConfigFactory Network::FilterFactoryCb diff --git a/source/extensions/filters/network/tcp_proxy/config.h b/source/extensions/filters/network/tcp_proxy/config.h index e5664ed45a7f..81d494cb3d5f 100644 --- a/source/extensions/filters/network/tcp_proxy/config.h +++ b/source/extensions/filters/network/tcp_proxy/config.h @@ -16,7 +16,7 @@ namespace TcpProxy { class ConfigFactory : public Common::FactoryBase { public: - ConfigFactory() : FactoryBase(NetworkFilterNames::get().TcpProxy) {} + ConfigFactory() : FactoryBase(NetworkFilterNames::get().TcpProxy, true) {} // NamedNetworkFilterConfigFactory Network::FilterFactoryCb diff --git a/source/extensions/filters/network/thrift_proxy/config.h b/source/extensions/filters/network/thrift_proxy/config.h index c71e1a2ed1c7..b51a96ee664a 100644 --- a/source/extensions/filters/network/thrift_proxy/config.h +++ b/source/extensions/filters/network/thrift_proxy/config.h @@ -43,7 +43,7 @@ class ThriftProxyFilterConfigFactory envoy::config::filter::network::thrift_proxy::v2alpha1::ThriftProxy, envoy::config::filter::network::thrift_proxy::v2alpha1::ThriftProtocolOptions> { public: - ThriftProxyFilterConfigFactory() : FactoryBase(NetworkFilterNames::get().ThriftProxy) {} + ThriftProxyFilterConfigFactory() : FactoryBase(NetworkFilterNames::get().ThriftProxy, true) {} private: Network::FilterFactoryCb createFilterFactoryFromProtoTyped( diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 824c42ec97ea..da8c12b82b64 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -64,6 +64,13 @@ std::vector ProdListenerComponentFactory::createNetwor auto& factory = Config::Utility::getAndCheckFactory( string_name); + + // TODO(alyssar) replace this block and reinstate TerminalNotLast test once echo2 is updated + if (factory.isTerminalFilter() && i != filters.size() - 1) { + throw EnvoyException( + fmt::format("Error: {} must be the terminal network filter.", filters[i].name())); + } + Network::FilterFactoryCb callback; if (Config::Utility::allowDeprecatedV1Config(context.runtime(), *filter_config)) { callback = factory.createFilterFactory(*filter_config->getObject("value", true), context); diff --git a/test/extensions/filters/network/dubbo_proxy/config_test.cc b/test/extensions/filters/network/dubbo_proxy/config_test.cc index a3c4abb57fdc..ecac96786272 100644 --- a/test/extensions/filters/network/dubbo_proxy/config_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/config_test.cc @@ -62,6 +62,7 @@ TEST_F(DubboFilterConfigTest, ValidProtoConfiguration) { NiceMock context; DubboProxyFilterConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, context); + EXPECT_TRUE(factory.isTerminalFilter()); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); cb(connection); diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index d197b9b7fa94..c300885c09fb 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -146,6 +146,65 @@ stat_prefix: router EnvoyException, "Didn't find a registered implementation for name: 'foo'"); } +TEST_F(HttpConnectionManagerConfigTest, RouterInverted) { + const std::string yaml_string = R"EOF( +codec_type: http1 +server_name: foo +stat_prefix: router +route_config: + virtual_hosts: + - name: service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: cluster +http_filters: +- name: envoy.router + config: {} +- name: envoy.health_check + config: + pass_through_mode: false + )EOF"; + + EXPECT_THROW_WITH_MESSAGE( + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), + EnvoyException, "Error: envoy.router must be the terminal http filter."); +} + +TEST_F(HttpConnectionManagerConfigTest, NonTerminalFilter) { + const std::string yaml_string = R"EOF( +codec_type: http1 +server_name: foo +stat_prefix: router +route_config: + virtual_hosts: + - name: service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: cluster +http_filters: +- name: envoy.health_check + config: + pass_through_mode: false + )EOF"; + + EXPECT_THROW_WITH_MESSAGE( + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), + EnvoyException, + "Error: non-terminal filter envoy.health_check is the last filter in a http filter chain."); +} + TEST_F(HttpConnectionManagerConfigTest, MiscConfig) { const std::string yaml_string = R"EOF( codec_type: http1 @@ -526,7 +585,7 @@ stat_prefix: router http_filters: - name: envoy.http_dynamo_filter config: {} - +- name: envoy.router )EOF"; auto proto_config = parseHttpConnectionManagerFromV2Yaml(yaml_string); @@ -535,6 +594,7 @@ stat_prefix: router EXPECT_CALL(context_.thread_local_, allocateSlot()); Network::FilterFactoryCb cb1 = factory.createFilterFactoryFromProto(proto_config, context_); Network::FilterFactoryCb cb2 = factory.createFilterFactoryFromProto(proto_config, context_); + EXPECT_TRUE(factory.isTerminalFilter()); } TEST_F(HttpConnectionManagerConfigTest, BadHttpConnectionMangerConfig) { @@ -785,13 +845,13 @@ TEST_F(FilterChainTest, createCustomUpgradeFilterChain) { auto foo_config = hcm_config.add_upgrade_configs(); foo_config->set_upgrade_type("foo"); - foo_config->add_filters()->ParseFromString("\n\fenvoy.router"); foo_config->add_filters()->ParseFromString("\n" "\x18" "envoy.http_dynamo_filter"); foo_config->add_filters()->ParseFromString("\n" "\x18" "envoy.http_dynamo_filter"); + foo_config->add_filters()->ParseFromString("\n\fenvoy.router"); HttpConnectionManagerConfig config(hcm_config, context_, date_provider_, route_config_provider_manager_, @@ -818,6 +878,27 @@ TEST_F(FilterChainTest, createCustomUpgradeFilterChain) { } } +TEST_F(FilterChainTest, createCustomUpgradeFilterChainWithRouterNotLast) { + auto hcm_config = parseHttpConnectionManagerFromV2Yaml(basic_config_); + auto websocket_config = hcm_config.add_upgrade_configs(); + websocket_config->set_upgrade_type("websocket"); + + ASSERT_TRUE(websocket_config->add_filters()->ParseFromString("\n\fenvoy.router")); + + auto foo_config = hcm_config.add_upgrade_configs(); + foo_config->set_upgrade_type("foo"); + foo_config->add_filters()->ParseFromString("\n\fenvoy.router"); + foo_config->add_filters()->ParseFromString("\n" + "\x18" + "envoy.http_dynamo_filter"); + + EXPECT_THROW_WITH_MESSAGE(HttpConnectionManagerConfig(hcm_config, context_, date_provider_, + route_config_provider_manager_, + scoped_routes_config_provider_manager_), + EnvoyException, + "Error: envoy.router must be the terminal http upgrade filter."); +} + TEST_F(FilterChainTest, invalidConfig) { auto hcm_config = parseHttpConnectionManagerFromV2Yaml(basic_config_); hcm_config.add_upgrade_configs()->set_upgrade_type("WEBSOCKET"); diff --git a/test/extensions/filters/network/rbac/BUILD b/test/extensions/filters/network/rbac/BUILD index 1990f3f3a2ce..eb3f0b1584b7 100644 --- a/test/extensions/filters/network/rbac/BUILD +++ b/test/extensions/filters/network/rbac/BUILD @@ -38,6 +38,7 @@ envoy_extension_cc_test( srcs = ["integration_test.cc"], extension_name = "envoy.filters.network.rbac", deps = [ + "//source/extensions/filters/network/echo:config", "//source/extensions/filters/network/rbac:config", "//test/integration:integration_lib", "//test/test_common:environment_lib", diff --git a/test/extensions/filters/network/rbac/integration_test.cc b/test/extensions/filters/network/rbac/integration_test.cc index b36c9bed1038..076099e67f42 100644 --- a/test/extensions/filters/network/rbac/integration_test.cc +++ b/test/extensions/filters/network/rbac/integration_test.cc @@ -37,6 +37,8 @@ class RoleBasedAccessControlNetworkFilterIntegrationTest principals: - not_id: any: true + - name: envoy.echo + config: )EOF"; } diff --git a/test/extensions/filters/network/redis_proxy/config_test.cc b/test/extensions/filters/network/redis_proxy/config_test.cc index 3f2b45bc7ac4..ff348c9c1a06 100644 --- a/test/extensions/filters/network/redis_proxy/config_test.cc +++ b/test/extensions/filters/network/redis_proxy/config_test.cc @@ -74,6 +74,7 @@ stat_prefix: foo NiceMock context; RedisProxyFilterConfigFactory factory; Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); + EXPECT_TRUE(factory.isTerminalFilter()); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); cb(connection); diff --git a/test/extensions/filters/network/tcp_proxy/config_test.cc b/test/extensions/filters/network/tcp_proxy/config_test.cc index 0de55a8acdb0..1adc0feac2b2 100644 --- a/test/extensions/filters/network/tcp_proxy/config_test.cc +++ b/test/extensions/filters/network/tcp_proxy/config_test.cc @@ -81,6 +81,7 @@ TEST(ConfigTest, ConfigTest) { config.set_stat_prefix("prefix"); config.set_cluster("cluster"); + EXPECT_TRUE(factory.isTerminalFilter()); Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(config, context); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); diff --git a/test/extensions/filters/network/thrift_proxy/config_test.cc b/test/extensions/filters/network/thrift_proxy/config_test.cc index 973461877a2b..d555bd6a1fac 100644 --- a/test/extensions/filters/network/thrift_proxy/config_test.cc +++ b/test/extensions/filters/network/thrift_proxy/config_test.cc @@ -56,6 +56,7 @@ class ThriftFilterConfigTestBase { void testConfig(envoy::config::filter::network::thrift_proxy::v2alpha1::ThriftProxy& config) { Network::FilterFactoryCb cb; EXPECT_NO_THROW({ cb = factory_.createFilterFactoryFromProto(config, context_); }); + EXPECT_TRUE(factory_.isTerminalFilter()); Network::MockConnection connection; EXPECT_CALL(connection, addReadFilter(_)); diff --git a/test/integration/tcp_conn_pool_integration_test.cc b/test/integration/tcp_conn_pool_integration_test.cc index d662c9b369dc..84ee8287c602 100644 --- a/test/integration/tcp_conn_pool_integration_test.cc +++ b/test/integration/tcp_conn_pool_integration_test.cc @@ -107,6 +107,7 @@ class TestFilterConfigFactory : public Server::Configuration::NamedNetworkFilter } std::string name() override { CONSTRUCT_ON_FIRST_USE(std::string, "envoy.test.router"); } + bool isTerminalFilter() override { return true; } }; } // namespace diff --git a/test/server/BUILD b/test/server/BUILD index 5a9c5d74323a..fcd3adfbd670 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -176,6 +176,7 @@ envoy_cc_test( "//source/extensions/filters/listener/original_dst:config", "//source/extensions/filters/listener/tls_inspector:config", "//source/extensions/filters/network/http_connection_manager:config", + "//source/extensions/filters/network/tcp_proxy:config", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:config", "//source/extensions/transport_sockets/tls:ssl_socket_lib", diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index ea19e0758c44..33f8fabede56 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -448,6 +448,65 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, BadFilterConfig) { EXPECT_THROW_WITH_REGEX(manager_->addOrUpdateListener(parseListenerFromV2Yaml(yaml), "", true), EnvoyException, "foo: Cannot find field"); } +class NonTerminalFilterFactory : public Configuration::NamedNetworkFilterConfigFactory { +public: + // Configuration::NamedNetworkFilterConfigFactory + Network::FilterFactoryCb createFilterFactory(const Json::Object&, + Configuration::FactoryContext&) override { + return [](Network::FilterManager&) -> void {}; + } + + Network::FilterFactoryCb createFilterFactoryFromProto(const Protobuf::Message&, + Configuration::FactoryContext&) override { + return [](Network::FilterManager&) -> void {}; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return std::make_unique(); + } + + std::string name() override { return "non_terminal"; } +}; + +TEST_F(ListenerManagerImplWithRealFiltersTest, DISABLED_TerminalNotLast) { + Registry::RegisterFactory + registered; + const std::string yaml = R"EOF( +address: + socket_address: + address: 127.0.0.1 + port_value: 1234 +filter_chains: +- filters: + - name: non_terminal + config: {} + )EOF"; + + EXPECT_THROW_WITH_REGEX(manager_->addOrUpdateListener(parseListenerFromV2Yaml(yaml), "", true), + EnvoyException, + "Error: non-terminal filter non_terminal is the last " + "filter in a network filter chain."); +} + +TEST_F(ListenerManagerImplWithRealFiltersTest, NotTerminalLast) { + const std::string yaml = R"EOF( +address: + socket_address: + address: 127.0.0.1 + port_value: 1234 +filter_chains: +- filters: + - name: envoy.tcp_proxy + config: {} + - name: unknown_but_will_not_be_processed + config: {} + )EOF"; + + EXPECT_THROW_WITH_REGEX(manager_->addOrUpdateListener(parseListenerFromV2Yaml(yaml), "", true), + EnvoyException, + "Error: envoy.tcp_proxy must be the terminal network filter."); +} TEST_F(ListenerManagerImplWithRealFiltersTest, BadFilterName) { const std::string yaml = R"EOF( @@ -485,6 +544,7 @@ class TestStatsConfigFactory : public Configuration::NamedNetworkFilterConfigFac } std::string name() override { return "stats_test"; } + bool isTerminalFilter() override { return true; } private: Network::FilterFactoryCb commonFilterFactory(Configuration::FactoryContext& context) { From 38a4d86407668612ccbc4cba65d11be6cbd81b32 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Tue, 13 Aug 2019 13:39:59 -0700 Subject: [PATCH 380/542] http1: remove unimplemented constructor (#7905) Removes a stale, unimplemented constructor definition. Risk Level: low Testing: n/a Doc Changes: n/a Release Notes: n/a Signed-off-by: Stephan Zuercher --- source/common/http/http1/codec_impl.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index be9def724076..e6dd04016888 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -314,10 +314,6 @@ class ServerConnectionImpl : public ServerConnection, public ConnectionImpl { ServerConnectionCallbacks& callbacks, Http1Settings settings, uint32_t max_request_headers_kb); - ServerConnectionImpl(Network::Connection& connection, ServerConnectionCallbacks& callbacks, - Http1Settings settings, uint32_t max_request_headers_kb, - bool strict_header_validation); - bool supports_http_10() override { return codec_settings_.accept_http_10_; } private: From 1dc418f6d7ff6402de3bbac2145f3e8e01562457 Mon Sep 17 00:00:00 2001 From: lberki Date: Wed, 14 Aug 2019 00:47:15 +0200 Subject: [PATCH 381/542] Patch in ecf04cc and 49f0fb9 from gRPC. (#7850) This is so that Envoy keeps building with the --incompatible_disable_legacy_proto_provider command line option of Bazel which is to be flipped in 1.0 . Description: Risk Level: Testing: Docs Changes: Release Notes: Signed-off-by: Lukacs T. Berki --- bazel/grpc-protoinfo-1.patch | 56 ++++++++++++++++++++++++++++++++++++ bazel/grpc-protoinfo-2.patch | 32 +++++++++++++++++++++ bazel/grpc-protoinfo-3.patch | 31 ++++++++++++++++++++ bazel/repositories.bzl | 12 +++++++- 4 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 bazel/grpc-protoinfo-1.patch create mode 100644 bazel/grpc-protoinfo-2.patch create mode 100644 bazel/grpc-protoinfo-3.patch diff --git a/bazel/grpc-protoinfo-1.patch b/bazel/grpc-protoinfo-1.patch new file mode 100644 index 000000000000..f91e7a035781 --- /dev/null +++ b/bazel/grpc-protoinfo-1.patch @@ -0,0 +1,56 @@ +commit 49f0fb9035120d0f5b5fa49846324c0b2d59c257 +Author: Marcel Hlopko +Date: Thu Jun 20 18:55:56 2019 +0200 + + Migrate from dep.proto to dep[ProtoInfo] + +diff --git a/WORKSPACE b/WORKSPACE +index 2db3c5db2f..60582d1a0f 100644 +--- a/WORKSPACE ++++ b/WORKSPACE +@@ -20,7 +20,7 @@ register_toolchains( + + git_repository( + name = "io_bazel_rules_python", +- commit = "8b5d0683a7d878b28fffe464779c8a53659fc645", ++ commit = "fdbb17a4118a1728d19e638a5291b4c4266ea5b8", + remote = "https://github.com/bazelbuild/rules_python.git", + ) + +diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl +index b7edcda702..581165a190 100644 +--- a/bazel/generate_cc.bzl ++++ b/bazel/generate_cc.bzl +@@ -41,11 +41,11 @@ def _join_directories(directories): + + def generate_cc_impl(ctx): + """Implementation of the generate_cc rule.""" +- protos = [f for src in ctx.attr.srcs for f in src.proto.check_deps_sources.to_list()] ++ protos = [f for src in ctx.attr.srcs for f in src[ProtoInfo].check_deps_sources.to_list()] + includes = [ + f + for src in ctx.attr.srcs +- for f in src.proto.transitive_imports.to_list() ++ for f in src[ProtoInfo].transitive_imports.to_list() + ] + outs = [] + proto_root = get_proto_root( +diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl +index 17004f3474..3df30f8262 100644 +--- a/bazel/python_rules.bzl ++++ b/bazel/python_rules.bzl +@@ -28,12 +28,12 @@ def _get_staged_proto_file(context, source_file): + def _generate_py_impl(context): + protos = [] + for src in context.attr.deps: +- for file in src.proto.direct_sources: ++ for file in src[ProtoInfo].direct_sources: + protos.append(_get_staged_proto_file(context, file)) + includes = [ + file + for src in context.attr.deps +- for file in src.proto.transitive_imports.to_list() ++ for file in src[ProtoInfo].transitive_imports.to_list() + ] + proto_root = get_proto_root(context.label.workspace_root) + format_str = (_GENERATED_GRPC_PROTO_FORMAT if context.executable.plugin else _GENERATED_PROTO_FORMAT) diff --git a/bazel/grpc-protoinfo-2.patch b/bazel/grpc-protoinfo-2.patch new file mode 100644 index 000000000000..f1d62d8aaaa7 --- /dev/null +++ b/bazel/grpc-protoinfo-2.patch @@ -0,0 +1,32 @@ +commit ecf04ccf4d8be9378166ec9e0ccf44081e211d11 +Author: Marcel Hlopko +Date: Thu Jun 20 18:57:33 2019 +0200 + + Require ProtoInfo in attributes, not "proto" + +diff --git a/bazel/generate_cc.bzl b/bazel/generate_cc.bzl +index 581165a190..87e8b9d329 100644 +--- a/bazel/generate_cc.bzl ++++ b/bazel/generate_cc.bzl +@@ -146,7 +146,7 @@ _generate_cc = rule( + "srcs": attr.label_list( + mandatory = True, + allow_empty = False, +- providers = ["proto"], ++ providers = [ProtoInfo], + ), + "plugin": attr.label( + executable = True, +diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl +index 3df30f8262..d4ff77094c 100644 +--- a/bazel/python_rules.bzl ++++ b/bazel/python_rules.bzl +@@ -99,7 +99,7 @@ __generate_py = rule( + "deps": attr.label_list( + mandatory = True, + allow_empty = False, +- providers = ["proto"], ++ providers = [ProtoInfo], + ), + "plugin": attr.label( + executable = True, diff --git a/bazel/grpc-protoinfo-3.patch b/bazel/grpc-protoinfo-3.patch new file mode 100644 index 000000000000..97eab476f17b --- /dev/null +++ b/bazel/grpc-protoinfo-3.patch @@ -0,0 +1,31 @@ +commit e2ba3aa07009292617c3cabe734e8e44099b22ac +Author: Lukacs T. Berki +Date: Tue Aug 6 14:00:11 2019 +0200 + + Update C++ code generation to work with Bazel 0.29 . + + The above Bazel version changes proto compilation slightly: some proto + files are put into a `_virtual_imports` directory and thus + `_get_include_directory` needs to be updated accordingly. + + Ideally, it would use instead the `ProtoInfo` provider to tease out the + proto import directories, but that's a bit more intrusive change. + +diff --git a/bazel/protobuf.bzl b/bazel/protobuf.bzl +index f2df7bd87b..3066e1d550 100644 +--- a/bazel/protobuf.bzl ++++ b/bazel/protobuf.bzl +@@ -59,6 +59,13 @@ def proto_path_to_generated_filename(proto_path, fmt_str): + def _get_include_directory(include): + directory = include.path + prefix_len = 0 ++ ++ virtual_imports = "/_virtual_imports/" ++ if not include.is_source and virtual_imports in include.path: ++ root, relative = include.path.split(virtual_imports, 2) ++ result = root + virtual_imports + relative.split("/", 1)[0] ++ return result ++ + if not include.is_source and directory.startswith(include.root.path): + prefix_len = len(include.root.path) + 1 + diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index c0b2205e0ad6..d0df39d541bd 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -600,7 +600,17 @@ def _com_googlesource_quiche(): ) def _com_github_grpc_grpc(): - _repository_impl("com_github_grpc_grpc") + _repository_impl( + "com_github_grpc_grpc", + patches = [ + # Workaround for https://github.com/envoyproxy/envoy/issues/7863 + "@envoy//bazel:grpc-protoinfo-1.patch", + "@envoy//bazel:grpc-protoinfo-2.patch", + # Pre-integration of https://github.com/grpc/grpc/pull/19860 + "@envoy//bazel:grpc-protoinfo-3.patch", + ], + patch_args = ["-p1"], + ) # Rebind some stuff to match what the gRPC Bazel is expecting. native.bind( From 8e07e8297d7d5e68cf6d6d7e49ba98d2500cdef7 Mon Sep 17 00:00:00 2001 From: Kuat Date: Tue, 13 Aug 2019 15:48:12 -0700 Subject: [PATCH 382/542] xds: apply node identifier optimization (#7876) Omit the node identifier from subsequent discovery requests on the same stream. Restricted to non-incremental xDS for tractability. Risk Level: low, affects xDS protocol but guarded by an option Testing: Unit/integration tests are updated Docs Changes: xDS spec clarification Release Notes: omit the node identifier from subsequent discovery requests Fixes: #7860 Signed-off-by: Kuat Yessenov --- api/envoy/api/v2/core/config_source.proto | 3 ++ api/xds_protocol.rst | 7 +++ docs/root/intro/version_history.rst | 1 + .../common/config/delta_subscription_impl.h | 1 + source/common/config/grpc_mux_impl.cc | 11 ++-- source/common/config/grpc_mux_impl.h | 4 +- source/common/config/grpc_subscription_impl.h | 7 +-- .../config/subscription_factory_impl.cc | 3 +- .../common/upstream/cluster_manager_impl.cc | 3 +- .../config/delta_subscription_test_harness.h | 5 +- .../filesystem_subscription_test_harness.h | 5 +- test/common/config/grpc_mux_impl_test.cc | 51 ++++++++++--------- .../config/grpc_subscription_impl_test.cc | 4 +- .../config/grpc_subscription_test_harness.h | 22 ++++---- .../config/http_subscription_test_harness.h | 5 +- test/common/config/subscription_impl_test.cc | 5 +- .../common/config/subscription_test_harness.h | 3 +- test/config/utility.cc | 1 + test/integration/ads_integration.cc | 2 +- test/integration/ads_integration.h | 1 + test/integration/ads_integration_test.cc | 26 +++++----- test/integration/cds_integration_test.cc | 2 +- test/integration/integration.cc | 16 +++--- test/integration/integration.h | 3 +- test/integration/rtds_integration_test.cc | 3 +- test/integration/vhds_integration_test.cc | 4 +- 26 files changed, 119 insertions(+), 79 deletions(-) diff --git a/api/envoy/api/v2/core/config_source.proto b/api/envoy/api/v2/core/config_source.proto index 913f3876115c..9b77f12f5bea 100644 --- a/api/envoy/api/v2/core/config_source.proto +++ b/api/envoy/api/v2/core/config_source.proto @@ -65,6 +65,9 @@ message ApiConfigSource { // For GRPC APIs, the rate limit settings. If present, discovery requests made by Envoy will be // rate limited. RateLimitSettings rate_limit_settings = 6; + + // Skip the node identifier in subsequent discovery requests for streaming gRPC config types. + bool set_node_on_first_message_only = 7; } // Aggregated Discovery Service (ADS) options. This is currently empty, but when diff --git a/api/xds_protocol.rst b/api/xds_protocol.rst index f8fe9a6e0608..4d43306cdd1d 100644 --- a/api/xds_protocol.rst +++ b/api/xds_protocol.rst @@ -135,6 +135,13 @@ versioning across resource types. When ADS is not used, even each resource of a given resource type may have a distinct version, since the Envoy API allows distinct EDS/RDS resources to point at different :ref:`ConfigSources `. +Only the first request on a stream is guaranteed to carry the node identifier. +The subsequent discovery requests on the same stream may carry an empty node +identifier. This holds true regardless of the acceptance of the discovery +responses on the same stream. The node identifier should always be identical if +present more than once on the stream. It is sufficient to only check the first +message for the node identifier as a result. + .. _xds_protocol_resource_update: Resource Update diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 18986583b07e..8be96d2a9f19 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -6,6 +6,7 @@ Version history * access log: added :ref:`buffering ` and :ref:`periodical flushing ` support to gRPC access logger. Defaults to 16KB buffer and flushing every 1 second. * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. +* api: added ::ref:`set_node_on_first_message_only ` option to omit the node identifier from the subsequent discovery requests on the same stream. * config: enforcing that terminal filters (e.g. HttpConnectionManager for L4, router for L7) be the last in their respective filter chains. * buffer filter: the buffer filter populates content-length header if not present, behavior can be disabled using the runtime feature `envoy.reloadable_features.buffer_filter_populate_content_length`. * config: added access log :ref:`extension filter`. diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index 8dbf3eccafd5..dbee51c688c3 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -20,6 +20,7 @@ namespace Config { * Manages the logic of a (non-aggregated) delta xDS subscription. * TODO(fredlas) add aggregation support. The plan is for that to happen in XdsGrpcContext, * which this class will then "have a" rather than "be a". + * TODO(kyessenov) implement skip_subsequent_node for delta xDS subscription. */ class DeltaSubscriptionImpl : public Subscription, public GrpcStreamCallbacks, diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 55ffd4ea1ad6..c1eda78b6dd6 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -12,11 +12,11 @@ GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings) + const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node) : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings), - - local_info_(local_info) { + local_info_(local_info), skip_subsequent_node_(skip_subsequent_node), + first_stream_request_(true) { Config::Utility::checkLocalInfo("ads", local_info); } @@ -57,8 +57,12 @@ void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { } } + if (skip_subsequent_node_ && !first_stream_request_) { + request.clear_node(); + } ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url, request.DebugString()); grpc_stream_.sendMessage(request); + first_stream_request_ = false; // clear error_detail after the request is sent if it exists. if (api_state_[type_url].request_.has_error_detail()) { @@ -206,6 +210,7 @@ void GrpcMuxImpl::onDiscoveryResponse( void GrpcMuxImpl::onWriteable() { drainRequests(); } void GrpcMuxImpl::onStreamEstablished() { + first_stream_request_ = true; for (const auto& type_url : subscriptions_) { queueDiscoveryRequest(type_url); } diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 33ef94150822..d1aa49459053 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -28,7 +28,7 @@ class GrpcMuxImpl : public GrpcMux, GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings); + const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node); ~GrpcMuxImpl() override; void start() override; @@ -104,6 +104,8 @@ class GrpcMuxImpl : public GrpcMux, GrpcStream grpc_stream_; const LocalInfo::LocalInfo& local_info_; + const bool skip_subsequent_node_; + bool first_stream_request_; std::unordered_map api_state_; // Envoy's dependency ordering. std::list subscriptions_; diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 3b9f32dd7bf3..94e920843102 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -19,9 +19,10 @@ class GrpcSubscriptionImpl : public Config::Subscription { const Protobuf::MethodDescriptor& service_method, absl::string_view type_url, SubscriptionCallbacks& callbacks, SubscriptionStats stats, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - std::chrono::milliseconds init_fetch_timeout) - : callbacks_(callbacks), grpc_mux_(local_info, std::move(async_client), dispatcher, - service_method, random, scope, rate_limit_settings), + std::chrono::milliseconds init_fetch_timeout, bool skip_subsequent_node) + : callbacks_(callbacks), + grpc_mux_(local_info, std::move(async_client), dispatcher, service_method, random, scope, + rate_limit_settings, skip_subsequent_node), grpc_mux_subscription_(grpc_mux_, callbacks_, stats, type_url, dispatcher, init_fetch_timeout) {} diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 16647dbc3154..b85c8bbb1d1c 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -56,7 +56,8 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( ->create(), dispatcher_, random_, sotwGrpcMethod(type_url), type_url, callbacks, stats, scope, Utility::parseRateLimitSettings(api_config_source), - Utility::configSourceInitialFetchTimeout(config)); + Utility::configSourceInitialFetchTimeout(config), + api_config_source.set_node_on_first_message_only()); break; case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.clusters(), api_config_source); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index e2673034a675..69dae7cef8bb 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -228,7 +228,8 @@ ClusterManagerImpl::ClusterManagerImpl( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), random_, stats_, - Envoy::Config::Utility::parseRateLimitSettings(bootstrap.dynamic_resources().ads_config())); + Envoy::Config::Utility::parseRateLimitSettings(bootstrap.dynamic_resources().ads_config()), + bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); } else { ads_mux_ = std::make_unique(); } diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index aada398fd30c..da7b82d8007a 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -55,9 +55,10 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { subscription_->start(cluster_names); } - void expectSendMessage(const std::set& cluster_names, - const std::string& version) override { + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node = false) override { UNREFERENCED_PARAMETER(version); + UNREFERENCED_PARAMETER(expect_node); expectSendMessage(cluster_names, {}, Grpc::Status::GrpcStatus::Ok, "", {}); } diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 88c18d85216a..5bed6e9b1942 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -58,10 +58,11 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { } } - void expectSendMessage(const std::set& cluster_names, - const std::string& version) override { + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node) override { UNREFERENCED_PARAMETER(cluster_names); UNREFERENCED_PARAMETER(version); + UNREFERENCED_PARAMETER(expect_node); } void deliverConfigUpdate(const std::vector& cluster_names, diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 05b8356ceec5..72b556277a62 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -52,7 +52,7 @@ class GrpcMuxImplTestBase : public testing::Test { local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_); + random_, stats_, rate_limit_settings_, true); } void setup(const RateLimitSettings& custom_rate_limit_settings) { @@ -60,16 +60,18 @@ class GrpcMuxImplTestBase : public testing::Test { local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, custom_rate_limit_settings); + random_, stats_, custom_rate_limit_settings, true); } void expectSendMessage(const std::string& type_url, const std::vector& resource_names, const std::string& version, - const std::string& nonce = "", + bool first = false, const std::string& nonce = "", const Protobuf::int32 error_code = Grpc::Status::GrpcStatus::Ok, const std::string& error_message = "") { envoy::api::v2::DiscoveryRequest expected_request; - expected_request.mutable_node()->CopyFrom(local_info_.node()); + if (first) { + expected_request.mutable_node()->CopyFrom(local_info_.node()); + } for (const auto& resource : resource_names) { expected_request.add_resource_names(resource); } @@ -111,7 +113,7 @@ TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); auto bar_sub = grpc_mux_->subscribe("bar", {}, callbacks_); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("foo", {"x", "y"}, ""); + expectSendMessage("foo", {"x", "y"}, "", true); expectSendMessage("bar", {}, ""); grpc_mux_->start(); EXPECT_EQ(1, control_plane_connected_state_.value()); @@ -142,7 +144,7 @@ TEST_F(GrpcMuxImplTest, ResetStream) { auto bar_sub = grpc_mux_->subscribe("bar", {}, callbacks_); auto baz_sub = grpc_mux_->subscribe("baz", {"z"}, callbacks_); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("foo", {"x", "y"}, ""); + expectSendMessage("foo", {"x", "y"}, "", true); expectSendMessage("bar", {}, ""); expectSendMessage("baz", {"z"}, ""); grpc_mux_->start(); @@ -156,7 +158,7 @@ TEST_F(GrpcMuxImplTest, ResetStream) { grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); EXPECT_EQ(0, control_plane_connected_state_.value()); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("foo", {"x", "y"}, ""); + expectSendMessage("foo", {"x", "y"}, "", true); expectSendMessage("bar", {}, ""); expectSendMessage("baz", {"z"}, ""); timer_cb(); @@ -173,7 +175,7 @@ TEST_F(GrpcMuxImplTest, PauseResume) { grpc_mux_->pause("foo"); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); grpc_mux_->start(); - expectSendMessage("foo", {"x", "y"}, ""); + expectSendMessage("foo", {"x", "y"}, "", true); grpc_mux_->resume("foo"); grpc_mux_->pause("bar"); expectSendMessage("foo", {"z", "x", "y"}, ""); @@ -196,7 +198,7 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("foo", {"x", "y"}, ""); + expectSendMessage("foo", {"x", "y"}, "", true); grpc_mux_->start(); { @@ -215,7 +217,7 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { e->what())); })); - expectSendMessage("foo", {"x", "y"}, "", "", Grpc::Status::GrpcStatus::Internal, + expectSendMessage("foo", {"x", "y"}, "", false, "", Grpc::Status::GrpcStatus::Internal, fmt::format("bar does not match foo type URL in DiscoveryResponse {}", invalid_response->DebugString())); grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(invalid_response)); @@ -231,7 +233,7 @@ TEST_F(GrpcMuxImplTest, WildcardWatch) { const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; auto foo_sub = grpc_mux_->subscribe(type_url, {}, callbacks_); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage(type_url, {}, ""); + expectSendMessage(type_url, {}, "", true); grpc_mux_->start(); { @@ -267,7 +269,7 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { auto bar_sub = grpc_mux_->subscribe(type_url, {"y", "z"}, bar_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); // Should dedupe the "x" resource. - expectSendMessage(type_url, {"y", "z", "x"}, ""); + expectSendMessage(type_url, {"y", "z", "x"}, "", true); grpc_mux_->start(); { @@ -345,7 +347,7 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { auto foo_sub = grpc_mux_->subscribe(type_url, {"x", "y"}, foo_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage(type_url, {"x", "y"}, ""); + expectSendMessage(type_url, {"x", "y"}, "", true); grpc_mux_->start(); std::unique_ptr response( @@ -368,7 +370,7 @@ TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { auto foo_sub = grpc_mux_->subscribe(type_url, {}, foo_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage(type_url, {}, ""); + expectSendMessage(type_url, {}, "", true); grpc_mux_->start(); std::unique_ptr response( @@ -422,7 +424,7 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { }; auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); - expectSendMessage("foo", {"x"}, ""); + expectSendMessage("foo", {"x"}, "", true); grpc_mux_->start(); // Exhausts the limit. @@ -475,7 +477,7 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithEmptyRateLimitSetti }; auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); - expectSendMessage("foo", {"x"}, ""); + expectSendMessage("foo", {"x"}, "", true); grpc_mux_->start(); // Validate that drain_request_timer is enabled when there are no tokens. @@ -531,7 +533,7 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { }; auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); - expectSendMessage("foo", {"x"}, ""); + expectSendMessage("foo", {"x"}, "", true); grpc_mux_->start(); // Validate that rate limit is not enforced for 100 requests. @@ -565,7 +567,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { grpc_mux_->start(); { // subscribe and unsubscribe to simulate a cluster added and removed - expectSendMessage(type_url, {"y"}, ""); + expectSendMessage(type_url, {"y"}, "", true); auto temp_sub = grpc_mux_->subscribe(type_url, {"y"}, callbacks_); expectSendMessage(type_url, {}, ""); } @@ -581,11 +583,11 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); // when we add the new subscription version should be 1 and nonce should be bar - expectSendMessage(type_url, {"x"}, "1", "bar"); + expectSendMessage(type_url, {"x"}, "1", false, "bar"); // simulate a new cluster x is added. add CLA subscription for it. auto sub = grpc_mux_->subscribe(type_url, {"x"}, callbacks_); - expectSendMessage(type_url, {}, "1", "bar"); + expectSendMessage(type_url, {}, "1", false, "bar"); } // Verifies that a messsage with some resources is rejected when there are no watches. @@ -598,7 +600,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { grpc_mux_->start(); // subscribe and unsubscribe (by not keeping the return watch) so that the type is known to envoy - expectSendMessage(type_url, {"y"}, ""); + expectSendMessage(type_url, {"y"}, "", true); expectSendMessage(type_url, {}, ""); grpc_mux_->subscribe(type_url, {"y"}, callbacks_); @@ -615,7 +617,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { response->add_resources()->PackFrom(load_assignment); // The message should be rejected. - expectSendMessage(type_url, {}, "", "bar"); + expectSendMessage(type_url, {}, "", false, "bar"); EXPECT_LOG_CONTAINS("warning", "Ignoring unwatched type URL " + type_url, grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response))); } @@ -627,7 +629,7 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_), + random_, stats_, rate_limit_settings_, true), EnvoyException, "ads: node 'id' and 'cluster' are required. Set it either in 'node' config or via " "--service-node and --service-cluster options."); @@ -640,12 +642,11 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { local_info_, std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_), + random_, stats_, rate_limit_settings_, true), EnvoyException, "ads: node 'id' and 'cluster' are required. Set it either in 'node' config or via " "--service-node and --service-cluster options."); } - } // namespace } // namespace Config } // namespace Envoy diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 523466291207..ea773b24f1f8 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -28,7 +28,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { // Retry and succeed. EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage({"cluster2"}, ""); + expectSendMessage({"cluster2"}, "", true); timer_cb_(); EXPECT_TRUE(statsAre(3, 0, 0, 1, 0, 0)); verifyControlPlaneStats(1); @@ -49,7 +49,7 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { // Retry and succeed. EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage({"cluster0", "cluster1"}, ""); + expectSendMessage({"cluster0", "cluster1"}, "", true); timer_cb_(); EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); } diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 96a9aae23b6e..7c58a62bcb00 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -45,20 +45,24 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { subscription_ = std::make_unique( local_info_, std::unique_ptr(async_client_), dispatcher_, random_, *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, - stats_store_, rate_limit_settings_, init_fetch_timeout); + stats_store_, rate_limit_settings_, init_fetch_timeout, true); } ~GrpcSubscriptionTestHarness() override { EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)); } - void expectSendMessage(const std::set& cluster_names, - const std::string& version) override { - expectSendMessage(cluster_names, version, Grpc::Status::GrpcStatus::Ok, ""); + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node = false) override { + expectSendMessage(cluster_names, version, expect_node, Grpc::Status::GrpcStatus::Ok, ""); } void expectSendMessage(const std::set& cluster_names, const std::string& version, - const Protobuf::int32 error_code, const std::string& error_message) { + bool expect_node, const Protobuf::int32 error_code, + const std::string& error_message) { + UNREFERENCED_PARAMETER(expect_node); envoy::api::v2::DiscoveryRequest expected_request; - expected_request.mutable_node()->CopyFrom(node_); + if (expect_node) { + expected_request.mutable_node()->CopyFrom(node_); + } for (const auto& cluster : cluster_names) { expected_request.add_resource_names(cluster); } @@ -78,7 +82,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { void startSubscription(const std::set& cluster_names) override { EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); last_cluster_names_ = cluster_names; - expectSendMessage(last_cluster_names_, ""); + expectSendMessage(last_cluster_names_, "", true); subscription_->start(cluster_names); } @@ -102,12 +106,12 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(callbacks_, onConfigUpdate(RepeatedProtoEq(response->resources()), version)) .WillOnce(ThrowOnRejectedConfig(accept)); if (accept) { - expectSendMessage(last_cluster_names_, version); + expectSendMessage(last_cluster_names_, version, false); version_ = version; } else { EXPECT_CALL(callbacks_, onConfigUpdateFailed( Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); - expectSendMessage(last_cluster_names_, version_, Grpc::Status::GrpcStatus::Internal, + expectSendMessage(last_cluster_names_, version_, false, Grpc::Status::GrpcStatus::Internal, "bad config"); } subscription_->grpcMux().onDiscoveryResponse(std::move(response)); diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index 7c890973ab38..4178c82301c6 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -58,8 +58,9 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { } } - void expectSendMessage(const std::set& cluster_names, - const std::string& version) override { + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node = false) override { + UNREFERENCED_PARAMETER(expect_node); EXPECT_CALL(cm_, httpAsyncClientForCluster("eds_cluster")); EXPECT_CALL(cm_.async_client_, send_(_, _, _)) .WillOnce(Invoke([this, cluster_names, version](Http::MessagePtr& request, diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index c284bfcda9a6..622a268c90cb 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -47,8 +47,9 @@ class SubscriptionImplTest : public testing::TestWithParam { test_harness_->updateResources(cluster_names); } - void expectSendMessage(const std::set& cluster_names, const std::string& version) { - test_harness_->expectSendMessage(cluster_names, version); + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node) { + test_harness_->expectSendMessage(cluster_names, version, expect_node); } AssertionResult statsAre(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, diff --git a/test/common/config/subscription_test_harness.h b/test/common/config/subscription_test_harness.h index 09803354e151..6a094116324d 100644 --- a/test/common/config/subscription_test_harness.h +++ b/test/common/config/subscription_test_harness.h @@ -36,9 +36,10 @@ class SubscriptionTestHarness { * Expect that an update request is sent by the Subscription implementation. * @param cluster_names cluster names to expect in the request. * @param version version_info to expect in the request. + * @param expect_node whether the node information should be expected */ virtual void expectSendMessage(const std::set& cluster_names, - const std::string& version) PURE; + const std::string& version, bool expect_node) PURE; /** * Deliver a response to the Subscription implementation and validate. diff --git a/test/config/utility.cc b/test/config/utility.cc index b8d3276fa60c..11efd7a79da7 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -172,6 +172,7 @@ std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_typ grpc_services: envoy_grpc: cluster_name: my_cds_cluster + set_node_on_first_message_only: true static_resources: clusters: - name: my_cds_cluster diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index a950bc8e5899..cf62f423aaad 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -145,7 +145,7 @@ void AdsIntegrationTest::initializeAds(const bool rate_limiting) { void AdsIntegrationTest::testBasicFlow() { // Send initial configuration, validate we can process a request. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index c03e42e645bf..024bcf04d607 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -20,6 +20,7 @@ static const std::string& AdsIntegrationConfig() { cds_config: {ads: {}} ads_config: api_type: GRPC + set_node_on_first_message_only: true static_resources: clusters: name: dummy_cluster diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 3954f0aad69f..9479194d73be 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -49,7 +49,7 @@ TEST_P(AdsIntegrationTest, Failure) { // Send initial configuration, failing each xDS once (via a type mismatch), validate we can // process a request. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse( Config::TypeUrl::get().Cluster, {buildClusterLoadAssignment("cluster_0")}, {buildClusterLoadAssignment("cluster_0")}, {}, "1"); @@ -57,7 +57,7 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Cluster, "", {}, {}, {}, Grpc::Status::GrpcStatus::Internal, + Config::TypeUrl::get().Cluster, "", {}, {}, {}, false, Grpc::Status::GrpcStatus::Internal, fmt::format("{} does not match {}", Config::TypeUrl::get().ClusterLoadAssignment, Config::TypeUrl::get().Cluster))); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, @@ -73,7 +73,7 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); EXPECT_TRUE( compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", {"cluster_0"}, {}, - {}, Grpc::Status::GrpcStatus::Internal, + {}, false, Grpc::Status::GrpcStatus::Internal, fmt::format("{} does not match {}", Config::TypeUrl::get().Cluster, Config::TypeUrl::get().ClusterLoadAssignment))); sendDiscoveryResponse( @@ -87,7 +87,7 @@ TEST_P(AdsIntegrationTest, Failure) { {buildRouteConfig("listener_0", "route_config_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Listener, "", {}, {}, {}, Grpc::Status::GrpcStatus::Internal, + Config::TypeUrl::get().Listener, "", {}, {}, {}, false, Grpc::Status::GrpcStatus::Internal, fmt::format("{} does not match {}", Config::TypeUrl::get().RouteConfiguration, Config::TypeUrl::get().Listener))); sendDiscoveryResponse( @@ -103,7 +103,7 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); EXPECT_TRUE( compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"route_config_0"}, {}, - {}, Grpc::Status::GrpcStatus::Internal, + {}, false, Grpc::Status::GrpcStatus::Internal, fmt::format("{} does not match {}", Config::TypeUrl::get().Listener, Config::TypeUrl::get().RouteConfiguration))); sendDiscoveryResponse( @@ -122,7 +122,7 @@ TEST_P(AdsIntegrationTest, DuplicateWarmingListeners) { initialize(); // Send initial configuration, validate we can process a request. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -191,7 +191,7 @@ TEST_P(AdsIntegrationTest, DuplicateInitialClusters) { // Send initial configuration, failing each xDS once (via a type mismatch), validate we can // process a request. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse( Config::TypeUrl::get().Cluster, {buildCluster("duplicate_cluster"), buildCluster("duplicate_cluster")}, @@ -206,7 +206,7 @@ TEST_P(AdsIntegrationTest, DuplicateWarmingClusters) { initialize(); // Send initial configuration, validate we can process a request. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -251,7 +251,7 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { initialize(); // Send initial configuration, validate we can process a request. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -333,7 +333,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { initialize(); // Send initial configuration, validate we can process a request. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -468,7 +468,7 @@ TEST_P(AdsIntegrationTest, RdsAfterLdsInvalidated) { // --------------------- // Initial request for any cluster, respond with cluster_0 version 1 - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -654,7 +654,7 @@ TEST_P(AdsIntegrationTest, XdsBatching) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", {"eds_cluster2", "eds_cluster"}, - {"eds_cluster2", "eds_cluster"}, {})); + {"eds_cluster2", "eds_cluster"}, {}, true)); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("eds_cluster"), buildClusterLoadAssignment("eds_cluster2")}, @@ -681,7 +681,7 @@ TEST_P(AdsIntegrationTest, ListenerDrainBeforeServerStart) { initialize(); // Initial request for cluster, response for cluster_0. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index d1b5058fab1e..a8efa18cfae1 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -94,7 +94,7 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht acceptXdsConnection(); // Do the initial compareDiscoveryRequest / sendDiscoveryResponse for cluster_1. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster1_}, {cluster1_}, {}, "55"); diff --git a/test/integration/integration.cc b/test/integration/integration.cc index 8131d513297c..b7fd1d46696c 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -541,11 +541,11 @@ AssertionResult BaseIntegrationTest::compareDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, const std::vector& expected_resource_names_added, - const std::vector& expected_resource_names_removed, + const std::vector& expected_resource_names_removed, bool expect_node, const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw) { return compareSotwDiscoveryRequest(expected_type_url, expected_version, expected_resource_names, - expected_error_code, expected_error_message); + expect_node, expected_error_code, expected_error_message); } else { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_names_added, expected_resource_names_removed, expected_error_code, @@ -555,14 +555,18 @@ AssertionResult BaseIntegrationTest::compareDiscoveryRequest( AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, - const std::vector& expected_resource_names, + const std::vector& expected_resource_names, bool expect_node, const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { envoy::api::v2::DiscoveryRequest discovery_request; VERIFY_ASSERTION(xds_stream_->waitForGrpcMessage(*dispatcher_, discovery_request)); - EXPECT_TRUE(discovery_request.has_node()); - EXPECT_FALSE(discovery_request.node().id().empty()); - EXPECT_FALSE(discovery_request.node().cluster().empty()); + if (expect_node) { + EXPECT_TRUE(discovery_request.has_node()); + EXPECT_FALSE(discovery_request.node().id().empty()); + EXPECT_FALSE(discovery_request.node().cluster().empty()); + } else { + EXPECT_FALSE(discovery_request.has_node()); + } if (expected_type_url != discovery_request.type_url()) { return AssertionFailure() << fmt::format("type_url {} does not match expected {}", diff --git a/test/integration/integration.h b/test/integration/integration.h index b263d78cdd8d..4c09b7ecc6fa 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -218,6 +218,7 @@ class BaseIntegrationTest : Logger::Loggable { const std::vector& expected_resource_names, const std::vector& expected_resource_names_added, const std::vector& expected_resource_names_removed, + bool expect_node = false, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); template @@ -250,7 +251,7 @@ class BaseIntegrationTest : Logger::Loggable { const std::string& expected_error_message = ""); AssertionResult compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, - const std::vector& expected_resource_names, + const std::vector& expected_resource_names, bool expect_node = false, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 63122162e184..02283b3927c2 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -36,6 +36,7 @@ std::string tdsBootstrapConfig(absl::string_view api_type) { grpc_services: envoy_grpc: cluster_name: rtds_cluster + set_node_on_first_message_only: true - name: some_admin_layer admin_layer: {{}} admin: @@ -119,7 +120,7 @@ TEST_P(RtdsIntegrationTest, RtdsReload) { EXPECT_EQ("", getRuntimeKey("baz")); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Runtime, "", {"some_rtds_layer"}, - {"some_rtds_layer"}, {})); + {"some_rtds_layer"}, {}, true)); auto some_rtds_layer = TestUtility::parseYaml(R"EOF( name: some_rtds_layer layer: diff --git a/test/integration/vhds_integration_test.cc b/test/integration/vhds_integration_test.cc index f31fbb9ef9b4..83c018b2badf 100644 --- a/test/integration/vhds_integration_test.cc +++ b/test/integration/vhds_integration_test.cc @@ -176,8 +176,8 @@ class VhdsIntegrationTest : public HttpIntegrationTest, xds_stream_->startGrpcStream(); fake_upstreams_[0]->set_allow_unexpected_disconnects(true); - EXPECT_TRUE( - compareSotwDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"my_route"})); + EXPECT_TRUE(compareSotwDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", + {"my_route"}, true)); sendSotwDiscoveryResponse( Config::TypeUrl::get().RouteConfiguration, {rdsConfig()}, "1"); From 128acb58c3292299edb4672c6c6baad9d10e62e9 Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Tue, 13 Aug 2019 16:12:36 -0700 Subject: [PATCH 383/542] config: WatchMap: cleaner management of watches (#7108) To be used with delta ADS. Could probably be used with the current GrpcMuxImpl. Has the SubscriptionCallbacks interface, so a GrpcMux can just directly pass onConfigUpdate() calls through to the WatchMap, which will then appropriately distribute the various resources to the various watches' SubscriptionCallbacks. #4991 Risk Level: none, not yet built into Envoy Testing: unit tests for the new class Signed-off-by: Fred Douglas --- include/envoy/config/BUILD | 8 + include/envoy/config/grpc_mux.h | 1 + source/common/config/BUILD | 102 ++++--- source/common/config/watch_map.cc | 170 ++++++++++++ source/common/config/watch_map.h | 115 ++++++++ source/docs/xDS_code_diagram.png | Bin 0 -> 97291 bytes test/common/config/BUILD | 11 + test/common/config/watch_map_test.cc | 397 +++++++++++++++++++++++++++ tools/spelling_dictionary.txt | 3 + 9 files changed, 762 insertions(+), 45 deletions(-) create mode 100644 source/common/config/watch_map.cc create mode 100644 source/common/config/watch_map.h create mode 100644 source/docs/xDS_code_diagram.png create mode 100644 test/common/config/watch_map_test.cc diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 2b2504620c99..001d3cbe46af 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -66,6 +66,14 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "watch_map_interface", + hdrs = ["watch_map.h"], + deps = [ + ":subscription_interface", + ], +) + envoy_cc_library( name = "xds_grpc_context_interface", hdrs = ["xds_grpc_context.h"], diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 4888e4e7e716..1d9920e2d5e3 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -25,6 +25,7 @@ struct ControlPlaneStats { ALL_CONTROL_PLANE_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) }; +// TODO(fredlas) redundant to SubscriptionCallbacks; remove this one. class GrpcMuxCallbacks { public: virtual ~GrpcMuxCallbacks() = default; diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 378512d4df5f..f3bcaacfdbb7 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -32,18 +32,20 @@ envoy_cc_library( ) envoy_cc_library( - name = "filesystem_subscription_lib", - srcs = ["filesystem_subscription_impl.cc"], - hdrs = ["filesystem_subscription_impl.h"], + name = "config_provider_lib", + srcs = ["config_provider_impl.cc"], + hdrs = ["config_provider_impl.h"], deps = [ - "//include/envoy/config:subscription_interface", - "//include/envoy/event:dispatcher_interface", - "//include/envoy/filesystem:filesystem_interface", - "//source/common/common:minimal_logger_lib", - "//source/common/config:utility_lib", + ":utility_lib", + "//include/envoy/config:config_provider_interface", + "//include/envoy/config:config_provider_manager_interface", + "//include/envoy/init:manager_interface", + "//include/envoy/server:admin_interface", + "//include/envoy/server:config_tracker_interface", + "//include/envoy/singleton:instance_interface", + "//include/envoy/thread_local:thread_local_interface", + "//source/common/init:target_lib", "//source/common/protobuf", - "//source/common/protobuf:message_validator_lib", - "//source/common/protobuf:utility_lib", ], ) @@ -100,6 +102,22 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "filesystem_subscription_lib", + srcs = ["filesystem_subscription_impl.cc"], + hdrs = ["filesystem_subscription_impl.h"], + deps = [ + "//include/envoy/config:subscription_interface", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/filesystem:filesystem_interface", + "//source/common/common:minimal_logger_lib", + "//source/common/config:utility_lib", + "//source/common/protobuf", + "//source/common/protobuf:message_validator_lib", + "//source/common/protobuf:utility_lib", + ], +) + envoy_cc_library( name = "grpc_stream_lib", hdrs = ["grpc_stream.h"], @@ -265,12 +283,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "resources_lib", - hdrs = ["resources.h"], - deps = ["//source/common/singleton:const_singleton"], -) - envoy_cc_library( name = "rds_json_lib", srcs = ["rds_json.cc"], @@ -288,6 +300,25 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "resources_lib", + hdrs = ["resources.h"], + deps = ["//source/common/singleton:const_singleton"], +) + +envoy_cc_library( + name = "remote_data_fetcher_lib", + srcs = ["remote_data_fetcher.cc"], + hdrs = ["remote_data_fetcher.h"], + deps = [ + "//include/envoy/upstream:cluster_manager_interface", + "//source/common/common:hex_lib", + "//source/common/crypto:utility_lib", + "//source/common/http:utility_lib", + "@envoy_api//envoy/api/v2/core:http_uri_cc", + ], +) + envoy_cc_library( name = "runtime_utility_lib", srcs = ["runtime_utility.cc"], @@ -361,42 +392,23 @@ envoy_cc_library( ) envoy_cc_library( - name = "well_known_names", - srcs = ["well_known_names.cc"], - hdrs = ["well_known_names.h"], + name = "watch_map_lib", + srcs = ["watch_map.cc"], + hdrs = ["watch_map.h"], deps = [ + "//include/envoy/config:subscription_interface", "//source/common/common:assert_lib", - "//source/common/singleton:const_singleton", - ], -) - -envoy_cc_library( - name = "config_provider_lib", - srcs = ["config_provider_impl.cc"], - hdrs = ["config_provider_impl.h"], - deps = [ - ":utility_lib", - "//include/envoy/config:config_provider_interface", - "//include/envoy/config:config_provider_manager_interface", - "//include/envoy/init:manager_interface", - "//include/envoy/server:admin_interface", - "//include/envoy/server:config_tracker_interface", - "//include/envoy/singleton:instance_interface", - "//include/envoy/thread_local:thread_local_interface", - "//source/common/init:target_lib", + "//source/common/common:minimal_logger_lib", "//source/common/protobuf", ], ) envoy_cc_library( - name = "remote_data_fetcher_lib", - srcs = ["remote_data_fetcher.cc"], - hdrs = ["remote_data_fetcher.h"], + name = "well_known_names", + srcs = ["well_known_names.cc"], + hdrs = ["well_known_names.h"], deps = [ - "//include/envoy/upstream:cluster_manager_interface", - "//source/common/common:hex_lib", - "//source/common/crypto:utility_lib", - "//source/common/http:utility_lib", - "@envoy_api//envoy/api/v2/core:http_uri_cc", + "//source/common/common:assert_lib", + "//source/common/singleton:const_singleton", ], ) diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc new file mode 100644 index 000000000000..718231009101 --- /dev/null +++ b/source/common/config/watch_map.cc @@ -0,0 +1,170 @@ +#include "common/config/watch_map.h" + +namespace Envoy { +namespace Config { + +Watch* WatchMap::addWatch(SubscriptionCallbacks& callbacks) { + auto watch = std::make_unique(callbacks); + Watch* watch_ptr = watch.get(); + wildcard_watches_.insert(watch_ptr); + watches_.insert(std::move(watch)); + return watch_ptr; +} + +void WatchMap::removeWatch(Watch* watch) { + wildcard_watches_.erase(watch); // may or may not be in there, but we want it gone. + watches_.erase(watch); +} + +AddedRemoved WatchMap::updateWatchInterest(Watch* watch, + const std::set& update_to_these_names) { + if (update_to_these_names.empty()) { + wildcard_watches_.insert(watch); + } else { + wildcard_watches_.erase(watch); + } + + std::vector newly_added_to_watch; + std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), + watch->resource_names_.begin(), watch->resource_names_.end(), + std::inserter(newly_added_to_watch, newly_added_to_watch.begin())); + + std::vector newly_removed_from_watch; + std::set_difference(watch->resource_names_.begin(), watch->resource_names_.end(), + update_to_these_names.begin(), update_to_these_names.end(), + std::inserter(newly_removed_from_watch, newly_removed_from_watch.begin())); + + watch->resource_names_ = update_to_these_names; + + return AddedRemoved(findAdditions(newly_added_to_watch, watch), + findRemovals(newly_removed_from_watch, watch)); +} + +absl::flat_hash_set WatchMap::watchesInterestedIn(const std::string& resource_name) { + absl::flat_hash_set ret = wildcard_watches_; + auto watches_interested = watch_interest_.find(resource_name); + if (watches_interested != watch_interest_.end()) { + for (const auto& watch : watches_interested->second) { + ret.insert(watch); + } + } + return ret; +} + +void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + if (watches_.empty()) { + ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); + return; + } + SubscriptionCallbacks& name_getter = (*watches_.begin())->callbacks_; + + // Build a map from watches, to the set of updated resources that each watch cares about. Each + // entry in the map is then a nice little bundle that can be fed directly into the individual + // onConfigUpdate()s. + absl::flat_hash_map> per_watch_updates; + for (const auto& r : resources) { + const absl::flat_hash_set& interested_in_r = + watchesInterestedIn(name_getter.resourceName(r)); + for (const auto& interested_watch : interested_in_r) { + per_watch_updates[interested_watch].Add()->CopyFrom(r); + } + } + + // We just bundled up the updates into nice per-watch packages. Now, deliver them. + for (auto& watch : watches_) { + auto this_watch_updates = per_watch_updates.find(watch); + if (this_watch_updates == per_watch_updates.end()) { + // This update included no resources this watch cares about - so we do an empty + // onConfigUpdate(), to notify the watch that its resources - if they existed before this - + // were dropped. + watch->callbacks_.onConfigUpdate({}, version_info); + } else { + watch->callbacks_.onConfigUpdate(this_watch_updates->second, version_info); + } + } +} + +void WatchMap::onConfigUpdate( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + // Build a pair of maps: from watches, to the set of resources {added,removed} that each watch + // cares about. Each entry in the map-pair is then a nice little bundle that can be fed directly + // into the individual onConfigUpdate()s. + absl::flat_hash_map> per_watch_added; + for (const auto& r : added_resources) { + const absl::flat_hash_set& interested_in_r = watchesInterestedIn(r.name()); + for (const auto& interested_watch : interested_in_r) { + per_watch_added[interested_watch].Add()->CopyFrom(r); + } + } + absl::flat_hash_map> per_watch_removed; + for (const auto& r : removed_resources) { + const absl::flat_hash_set& interested_in_r = watchesInterestedIn(r); + for (const auto& interested_watch : interested_in_r) { + *per_watch_removed[interested_watch].Add() = r; + } + } + + // We just bundled up the updates into nice per-watch packages. Now, deliver them. + for (const auto& added : per_watch_added) { + const Watch* cur_watch = added.first; + auto removed = per_watch_removed.find(cur_watch); + if (removed == per_watch_removed.end()) { + // additions only, no removals + cur_watch->callbacks_.onConfigUpdate(added.second, {}, system_version_info); + } else { + // both additions and removals + cur_watch->callbacks_.onConfigUpdate(added.second, removed->second, system_version_info); + // Drop the removals now, so the final removals-only pass won't use them. + per_watch_removed.erase(removed); + } + } + // Any removals-only updates will not have been picked up in the per_watch_added loop. + for (auto& removed : per_watch_removed) { + removed.first->callbacks_.onConfigUpdate({}, removed.second, system_version_info); + } +} + +void WatchMap::onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) { + for (auto& watch : watches_) { + watch->callbacks_.onConfigUpdateFailed(reason, e); + } +} + +std::set WatchMap::findAdditions(const std::vector& newly_added_to_watch, + Watch* watch) { + std::set newly_added_to_subscription; + for (const auto& name : newly_added_to_watch) { + auto entry = watch_interest_.find(name); + if (entry == watch_interest_.end()) { + newly_added_to_subscription.insert(name); + watch_interest_[name] = {watch}; + } else { + entry->second.insert(watch); + } + } + return newly_added_to_subscription; +} + +std::set +WatchMap::findRemovals(const std::vector& newly_removed_from_watch, Watch* watch) { + std::set newly_removed_from_subscription; + for (const auto& name : newly_removed_from_watch) { + auto entry = watch_interest_.find(name); + RELEASE_ASSERT( + entry != watch_interest_.end(), + fmt::format("WatchMap: tried to remove a watch from untracked resource {}", name)); + + entry->second.erase(watch); + if (entry->second.empty()) { + watch_interest_.erase(entry); + newly_removed_from_subscription.insert(name); + } + } + return newly_removed_from_subscription; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h new file mode 100644 index 000000000000..5e75e5e88dd7 --- /dev/null +++ b/source/common/config/watch_map.h @@ -0,0 +1,115 @@ +#pragma once + +#include +#include +#include + +#include "envoy/config/subscription.h" + +#include "common/common/assert.h" +#include "common/common/logger.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + +namespace Envoy { +namespace Config { + +struct AddedRemoved { + AddedRemoved(std::set&& added, std::set&& removed) + : added_(std::move(added)), removed_(std::move(removed)) {} + std::set added_; + std::set removed_; +}; + +struct Watch { + Watch(SubscriptionCallbacks& callbacks) : callbacks_(callbacks) {} + SubscriptionCallbacks& callbacks_; + std::set resource_names_; // must be sorted set, for set_difference. +}; + +// NOTE: Users are responsible for eventually calling removeWatch() on the Watch* returned +// by addWatch(). We don't expect there to be new users of this class beyond +// NewGrpcMuxImpl and DeltaSubscriptionImpl (TODO(fredlas) to be renamed). +// +// Manages "watches" of xDS resources. Several xDS callers might ask for a subscription to the same +// resource name "X". The xDS machinery must return to each their very own subscription to X. +// The xDS machinery's "watch" concept accomplishes that, while avoiding parallel redundant xDS +// requests for X. Each of those subscriptions is viewed as a "watch" on X, while behind the scenes +// there is just a single real subscription to that resource name. +// +// This class maintains the watches<-->subscription mapping: it +// 1) delivers updates to all interested watches, and +// 2) tracks which resource names should be {added to,removed from} the subscription when the +// {first,last} watch on a resource name is {added,removed}. +// +// #1 is accomplished by WatchMap's implementation of the SubscriptionCallbacks interface. +// This interface allows the xDS client to just throw each xDS update message it receives directly +// into WatchMap::onConfigUpdate, rather than having to track the various watches' callbacks. +// +// The information for #2 is returned by updateWatchInterest(); the caller should use it to +// update the subscription accordingly. +// +// A WatchMap is assumed to be dedicated to a single type_url type of resource (EDS, CDS, etc). +class WatchMap : public SubscriptionCallbacks, public Logger::Loggable { +public: + WatchMap() = default; + + // Adds 'callbacks' to the WatchMap, with every possible resource being watched. + // (Use updateWatchInterest() to narrow it down to some specific names). + // Returns the newly added watch, to be used with updateWatchInterest and removeWatch. + Watch* addWatch(SubscriptionCallbacks& callbacks); + + // Updates the set of resource names that the given watch should watch. + // Returns any resource name additions/removals that are unique across all watches. That is: + // 1) if 'resources' contains X and no other watch cares about X, X will be in added_. + // 2) if 'resources' does not contain Y, and this watch was the only one that cared about Y, + // Y will be in removed_. + AddedRemoved updateWatchInterest(Watch* watch, + const std::set& update_to_these_names); + + // Expects that the watch to be removed has already had all of its resource names removed via + // updateWatchInterest(). + void removeWatch(Watch* watch); + + // SubscriptionCallbacks + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; + + void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; + + std::string resourceName(const ProtobufWkt::Any&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + WatchMap(const WatchMap&) = delete; + WatchMap& operator=(const WatchMap&) = delete; + +private: + // Given a list of names that are new to an individual watch, returns those names that are in fact + // new to the entire subscription. + std::set findAdditions(const std::vector& newly_added_to_watch, + Watch* watch); + + // Given a list of names that an individual watch no longer cares about, returns those names that + // in fact the entire subscription no longer cares about. + std::set findRemovals(const std::vector& newly_removed_from_watch, + Watch* watch); + + // Returns the union of watch_interest_[resource_name] and wildcard_watches_. + absl::flat_hash_set watchesInterestedIn(const std::string& resource_name); + + absl::flat_hash_set> watches_; + + // Watches whose interest set is currently empty, which is interpreted as "everything". + absl::flat_hash_set wildcard_watches_; + + // Maps a resource name to the set of watches interested in that resource. Has two purposes: + // 1) Acts as a reference count; no watches care anymore ==> the resource can be removed. + // 2) Enables efficient lookup of all interested watches when a resource has been updated. + absl::flat_hash_map> watch_interest_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/docs/xDS_code_diagram.png b/source/docs/xDS_code_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..ef4df79cc1e30f59f865d5c29abc91ca05b19f72 GIT binary patch literal 97291 zcmY&=Wk6JIwDru;ozkszmxLfgcPNN-NDb00Js^!B-7%Daga|`7iZl#JND0!?-SHjY zd+(3$-*9Hm%sJ=T&)#dVwKmZ@TFL~tG`JuTh(J|E;Ux%!)&&AV=&&Kckp>wZ9uNo) zQdN-EdpEb+PUQM(CU;YHBtdZA7VGVB?OUSNlJ+L0>fWj2l+S6OOMXmd@#;FT6}bsJ z*VD59xW&N@<4}rAl@%#YwC7EX4pjsxQ@O4gFmFc@V>~gAk4cmLq?i!BOM%?|5hphU zChN5@613>N%ee@oqu=-U^SeID%PT&qc>QJn*~RTuuE$vGg-6hZ&uAv`@k(zjCBIqQ zyQ7s}^&X*av5^1!3pmVxX#aQoe?I`)`@dJfAuQPco)nut&i(pJ#Q9$Ak^S3ar(~XH zH>-EYl{zY(v&VN<~w1HLi1t`;X7BS2uBf0?}FK6^6o_h)H$d7a&uUoD-C`xdIeF3)%4`fTU^ zX|^BO6Pj=TQ%=|}>-|770XA;7g5tTbk2Csb(BmLm+pPX%w#-!tkKv=q^2Cf<#%}@C z)QQ7a8;&AKLwH2{0?{%Dv0gJ7yOPi*=nN`9xc)d$C$%mobPv&}80)M{D~-b*KUuT< z%qrq)_BVqbr`Scq+UzU|B;pNcPlT>HVpiXBn#V(5(N&G0@P{m;!=0jbef0Cg5_Ag! zYv*~L%w(O+lW_rf)`U<@CDOtf!IT~Lm7q}FQS%-g(u215Y2%d`lRWMU+*|kkDF*5- zRP8mM2b5O|>Q7xd@$=`&ggQU3dB^R|(!>2%t3F9d$y;u(2Q_JGh+LIPb8SgUuX*AT z+tSsj50~kDR=OTe0v1(BG4^YVv+XF}y5c9HA9Hgzc0(~X*gx2Sjid<3VjH1Q>scWW z69ZU0M(vyLb4DIg!9n>T`b+FybQFugI})z?OAfV)vZ#!F-qM#CBYyC6{MRiq_-bY<^X*I7DH;sNjcboBaz&;|Q%yAyBx?u#izbY6HyCjkOj5C7x{D|Av$b4n zBJ*tFQ+ri8+GAP4=Q^U}rwmMN(Bvu_odi>T+v^Vz*1-P%@a*sYe$YmyUB^ZHftQ!p zRbN|hNuFN$moE?Jjh%b^-Qp=Hn`6QXodH+1o$H?rpw>}X=a58GO2YBaUc&H9bOQ=R z)w8e+4Jg~%6ghPELx=!%s?C#qJcR02kaeA&b1bDY3*{rlH-oM6#2>Nu3$(5O^vn-e z$SJ+Gs{qXzIaDB)9z7}m!=*T$^yml?A!((mxZod1MG3zVHM;Ez@>)nsI^qr0_2`q4 zl)hB9aV9aTryy`}gnxc(mN4@))L#*@eYP%8bVO?P0ou#zK4aJ{#(FN_jWRo<8k`FWo z{Tei9JD;t4Y(@votLy|PVk}ZvDthNZ=_#}W54qUa+(m|+KAYaNu%>3 zkfQH?2Dahw%%>b@9;&DD9Pvy3sA@Z#Hl}^*c<8f8uI}GjuvYOp?#TUU#f-KJ^x`>b zf$?)H$y6q2tJV)OXR2y<_B8@}1V(+?no&SW_8D(O9uDcOpJEt3ZK0yCkyVYYYUnB! z(Ye0))K?A(CisR`BypXsVg!dfcA`~8#;cAYC#NHjQKd`VmO_K=Y7kh`!a~V=uVv&r zVU#WjUkjb$T2Dy-jwB(JERK7${u4G$oRa?ge>bsT+|+crDf7@AjQl9?3*_)dr}?y% z`*XYCbGwf9zABR@SIfT>CHK|YYG13{g}i%m(+by+0=~5&T3c$mOW%J2k4aYcyDDWh z7te=xdKVa|CvGy3l}k`nd04I6TnTenMwupI8LvwHJ55pt>M8Orn17o|vOTV`!!i0> zQHK5z(MK~|a#PeC(6LObE}SJ%v%K&{1*F2wd3;%^mG@U9rkL3Jt{- zg^81)?5NGbwOu8ySI{Ze%xRdXfeCR%72}u zz~SK-09UdW@-a=e(Wn=dbD|nu^w+Nb=4y?USa=$hmaXHyLlk%HPV<$H$B3Q-I{#D_ zOa1G}2+8yeRz!SeGajm-rmumX2vK({wt|K$h}>>!Fe8-(5_{ z!`gFcjazy+p?bg`>Uq?9wYG{ zzSXX;xg67rPVtNVPEKrgAi)j733M*s<;QEdYoGF~Z$c=#qr6pWL}Sdt3>@5k&AJDQ zV)IZ5OK2)IwKC*?wy^qPPd8LQ_@ssq^NM4^w2mdAd3xQozOcYEWMN@J4o%`P`7;I| z>^4T1BTki+aB^8qeVopLZiS)s>br^492ye1G&8A@%lnO9;AxLDV-eBJU{EVtg__pd zQ2BuArdH;={y~;J+koJc6`=ZU5$`;+Y!?F3|35&07WeP`fb^4<679d+LO~dUrj%{( z{{MIW2l4+0>{8k z{z(5U&5u7BHEJ00Gyq2H1SPGU2AI%hImyLMtxkY^Bv->MCbVo&2Tk1%GMQ|+lS>NG% zHqG`V;H&G=y@{eZ4-H?uIQRq)jP_Q?3HYQXdb1bDH~yRDX#NR#-dvMo=B%2)?8Ksz zH#)NQ57YX(l00#~ZZcc%FC$>NDES|wp#Y2+aI@Fw`=m7Ks*4hLZ+G|f@)-lj@8*23 zXBys4eT;@UXumL%v=lPh7*}Qt&m&@dzKh9c-a%(Wiah(2qFN1|imZwUT6({< z2KK{8^E`bX>Q*?CODiaoo|-BBfZRFDK;Kfxa;Ex{==6}sP-oug9e^Kx;fZ_}@-GZ* ztU)x^vkgx3?}?s^|4Gxxe=g0>&)=VlXfn+SoB#+<25GTWM^6O7?bAw)p%P! zV^)N?ySZR@-bH`&vU05q1Urnc_L?#E>D@KbVOm7g)=m`uX|91*vuo!)S<^5OUuF++ zX**If@1V$8|J)oHT>-Vj`;IR1FBT#h{8VJd2hGH3Lx?lWc%dg`#G7`WyyA+@6TB_# zWQBDwDk|u)=9FU|AM~fMXTa_+K^6lSg7QcyzS-{`8RB0Z&&Yqv8+*fArOlkGIMaz}S}2^44I74wOQO0W!dtyF zbz$p9wXY0Y?HS8=wf$p&Qc5xw`9n!}ed`v7o#tA{ZRhbH2+}BNfr~qi#=5K}L4bQm zVQAp5lk_@U9dbnT8b!_lbr+?aeaVq%)quuGSX|26-ylPMoNSw@_c}YjI?yi+eElPi zc71()LE*fYbpK6NRh6bU_o+zQ_V#x5B)&#rOe@L$^S9qFRV7J1MURs-;FvnKb#=a& zOa2G#0KWa*5qJZ9!I>KUG|u670o;CPyzrdklM}{w%V0SM|J|?OrNL48QUq9bewF3r znc{9Ns{gZ|n9N8eSy|cPx&>mB?MIYGGWHy<`wPt;8^gXU(d@ejJT?mB`GR;l;U}kj z4(@DhmAezAyps&vb{9O9xx}WLUTAtPH~X!rGSt=PoxhbZ?@Bc?SK{SN-}}6J^@zE8 z10U(wfREg7JJBNSms;)tU(4KIu6}%eU8Xs-q`S<0R*4#c1g65S`TO+VIU8K=j9Xc4 z?}8;YQND8`@8M}QXuCUa^0jPi&s$!)@_3@jvulYkul?{*QgY!UIV@I9)Ky3}t{bu5 z7^af=l~V+6DP&pcWe#aheOMc47i7i&uY8~;vFX>@Rng=e_rm^~Sy9_nqA-4{TWUM@ zbg9lZGohzvUWmCiEd^8Zm8Qs=#dzU6-srhGToP%2Kj)LziGl{skDfMOojk^`S}10= z$H7OwrxU(}$7CIAWnIv%t*!MmdHuT4 z{^zIC1?Qw{A>f%R&G)`EFmW^Jd0+Yi$4%^zd7y*54RQop%$GXuf9!`rCy=o>TNmld zFt6%94-&PYpLr4idCAtT{4QozMAj{FsMQr6{aT~VZS4`kAHgy*4-fq9kltG9_r(eb z%JLmZtS1+7g5ZY_-)p@qK1MI9YNgvAQfN-#KtyV-2S{f>W7N0$9Ha?S*&trQ83Oer zSr(c6V{D_!NFam6h7@i#u^T$a4K)gABKFiO*iBbfJ>4RT8hA)Nln5kFURH!@)jCPt za;Nsqd|gXR)>`p|HWBJC6O*1{CYV3!v4z7nB?A1jnh(tY^+;y@JC9m&cWpUmXpb7u zVY+oCkZ4Hgh`8qDyTo7Z?~3HGw~-k}VWej35OIW^Tt+>vsv=(|lTDIIEo2IMDi&#E z#1hZgGg8QsTw2-2q9`I8d47)lesHtN9KMmAGmx)D_Pq$CxVAy#j^SP4v8`pwffu-J zS~$>8znD~}EL_<910P8%19pyASJxC*&}v8t%}XiNef6rn*$oF38TR=LQ=Z@736-fW zZ`8W3^V?u_dep)rqf~6p47Fb}P>F4D0?&! zzw6K{w*@7e!k^im9UmE_2jA=8%0~Tuj)so@1Km97q<@o#$bM+ceZi#N&kOmys~bqH zm9pc3v%CjUuvNZ(x%(8-JGEThLjpV1N5BiDH`=%xg925%BfbK89F^6S@}DuxJXz zA{*oq+moHfd9h{6FAy{fmMm3Y2)tN4Ux7?l4-q@wttD&Et&pO_b`ARoE}UU4_)Nx|?w4>rqb{Rz}$A&0g`cu!4w3(R26VZfTRTe-&TQEB?A`fs%x z(a?^<$}0Tf42GT&hs!Ds&UD1k`NrE?bbb@6ulmPd;g7{^vT^i$4iSR0qghisK0DXY z#s;O`X{qh2{^z`8m4$9GQZ|+0%T*h2HfB5u1=4CWdP_gOBxZ{1;!lXG;cu?R%fFqW z9Ax}%q+r+%jqKw?GUor=mv zNB(Hx8jxk==9>CCxZA3#4pf`AMq%pzZgMNCs^Vp5$8ue>lb4rALZt3JJUpV8H%!dT zMzt}H4cfhU<0Nv3)f6^qU+k0<=T9`t*xK5@JfCh6j^xE4(o8FNmfv1Iw9}=xhYho=APN3m1f9pa;{q9-h zkAJVzRmBQIfe)ONQl}H>un;B2P_mb^ol?9Jqw)b2t6)E>@KQ!?MPiEKTE5ZuAejH> zYCcG;)E)*(pUH@}uB=~xn@U+D#7hMFP1ap6A=8i%#JAit{+pruP1h-fs3OO#W?9`D zC;DV$7%%USOB$W2@UtNBTwYSX60%#D$zgB(x<8vmOZ^-JHNl&Zjr)p}Pd3Lp!9!T8 z%(f9P19myi53Ptof?!nph@F{5EGQX-(ET+;<4)h8ED@a3H6?r)X)@Fx%|FoRont5= zKrXo0+slg8F52{o6GG9tpFAPkenq@be<-+q%)4GRvbUw$ob%Sj#T=k8*9MXcJhi#W_wo3?oV&G)62KVIh#zRF&@-7vqKGQaC)yam(oeDl!MByc@B{6=J4TWSoT zPm^VCEz;U3W&}6N`{DjB>+BH-R`O~zTg;_l+Vh_0y4h}wA(PgW$eHnOIjeVeEwMkDtC{~(#@|d$ZcgG@G8h9i(^%?q%QQEx%NOC!t zw5h}U@SB&lfa;ezwR10eD|cTI0*wr95{KHM7Yjz6Vbwx4iycC70+L?Y!Y4g@HZn-V z&+3lp`gN?QqOvoVGW*tM16g-%Z9iSx$0AJqJmfD3$}Ofi)W214@NJNf#z?Pz^yeF8 z!_e_RX$1l++>k(cbik#dDLu3OtaJGC(_rdoFEs|rTHTL?ty)Fw?neucMBMLW`RuJzP+3PihywEG> zJ2WId&K*Q#Oe(gGX6akFPNisWs7=g?o`Lh(>V#w8Y<8bh*Q2n=EquFi$~-{}4g-6M zn~4&2E-jRXqBC*-k3T5Y(5yDD~h5XAx@WUNg#$9~a<|rc>`;1|!p5y1q z0f62d>{16LYP`S0170Sp2dBRY=9rkjELV4JUWXN&m6GKb(tx(0Mlm%g zAPKA=(H)v3nMeK*U_ks-AY>4E@}x9WLE)($ys&`zjU9mnqxOrM`dXo$bRqoEk96PI z1oHbe%Ga{NkGse@n3U~gvjf;wdO}#ee#K`9Sq%MtBpppG^(f_J4eLOJ=wq#p!BZ(lGD?K&!8v;D1DQ8@c4Mr9k zrk2?bR|tHNi>}bGdSkF6>oGiZb+Y+z{pI0$$o&4;r}M^iM*8~KS1Y&aP-6L}nnypw zziF~)o3vAWL_MzGwPdEwD6s0DDrh#X? z)A3f(L{oYKW=00_Ya=Y7ASof zFg>u0qX=s-x#;p*W9%3X;Pke8|ACnOZYFy%e%MJSI4%bu`7{6V<$&=YceSTIa_<%qG+b6FC;NJ*%BC>~3$<EL|yTJEEYzq5Qa6?(Wa*6p4bIS zEd+2xxdRf(FIZS|f!8lr&Ya6-dk`^4ylZ*!OLFk~-q(4ZHRyfFrzq-bx=FUd*#>PU z52P2ZY%>Op)Tiffu_iF*Liz>gyLnZdx2EHu)k4$RHeukomQ6x(L^Fq}N zdz4Ru`Matv{JuR(NHs=8=&HYSA!&UenH^)xU4FCD)}uw(P=d*WB|wm8&QS`4DxP{O zufW7MXdGLwZ?cg+F&?%Ca6otp|Equ)%$Q(TjD#xm~iX`>2135A@U z-DL|3R5(&qoUtMS#*8@E7sYBt{W--cY12kQ;efj`UqeZWLv*9>9!zt|UGIE_28LZp z%cnIrH-ol5Pz!(W=7SC$_|75i0V#EKa4_^SKgJJ-Kf@tLEY`NRqxmw76N-$rtmuo_ zA)-4y&R5YO5wSiDOIgNd51qmow&#Y5r9j^)YMSJF`a&NJx7CC9tnETXQVW$t2wmG; z|HzTF0>my6JzP|3#pyyvzhW`GivW7BeC?${|hKHAJZAXJVnRsDx z5kgKsz_258i@y%=A95OV;8U5Yh@hwZdJ>N(^(n5A&)!6Km|`}1N&UjQ6nY2ZwnQm2 z28t{{Fx|LyO?;Ua_ui!<7!_OsKcF{@CD7V+YdWZVrO{J;2S) zZK)X)BkrXUR%P7yoIDMP46JIlQWX_<;(R{Jz%lep%9dW>X4={oMC1C_Ib1Dc3UcdEHEV&S|7{n*~wnGJ&Rf^$YT)O{cafBp?G{{Y&jv*|gB$+@}7 zf$n&gesI+51_k|9VqL79sR_ngJ<v#c*ZuhHOf?b8D5bE!kcH+|}gHo7T<~ z4vFXMXEWxTs$U59E;&yM@B~qHx7~Y+&6o5B;0MJe?W{8fEQ3n)`Q=Fj zaPd=yln0*_@cgl>eBtsta^mt*~*HokG8j;mn(Z zPR7H)%RiBp=2cd_4c>|AB=l>VDq32j=^dvff}N1%3HFDz$esJ9n{d_AhTEGFU!sqD zgU|0yfBqxoTD%+r1Enbe0v~J3T17eXEACi^B}(E|a$ws#Y8<$;y}cnU9qUv`s;Fe6 z*_-tTNvdAfU|VrEfZ6^4W!rr@{2t(I8G>)!Kz?`EXXo{21@e(h8vv6|FB23nk|mn` z)ai3Yg@cO=I1lGsMz{X~lPPX6WP!dJHmRw{i#=vU{Ca4c`JMV$>`U7!L)P|G8b^r#J#ei=}we9g?vx@J8!2Kx@$kN%EzUS4&y`o(WN z?B|bt80&bUtNU$TOFJTnm*-CD=X??nxEq_BGQj-?$(N8M_AwZDUj*N{_&yxTFyx7p z--KX&epu6vXn8RM;9D`rU#`ntA%HM{v)S~YqyhC;HHSxJ|0l`E?_oC2>H8Mh?%Sna zm^}Mha`R?W{D2{nkRBq9HXr|i;;V}~V20?soliohzBX^ajtst>ecf?RjxZ0IIX zS^2`Xoy&Kps*#I}D~;dW%+S#H?%L}?)B`$G&-d)q~^1=}sg= zv-?KE0qHorHsIawDrfUnG%vrpVuu!*ln1ne$(z13|Bjr@jPD2_w^y zXj`|px2nl(yntt&=b#w!ba-kR$Lv5S-f zpWw&d2j9AqroOSP$@m^yXSr=TZGq?12SET^vUPd1YE`-K9lv~F?(d#6OhT6v@Nlyf ze6^mg9NS=Z?=%1_*yIoB<{5kzc)j_szZ7`y^02o#aE|sS;Q?@|0OpvV?{m{V!CMm( z696CDjQ0es7SL-xG+Voe0`>$n=^4qJh0V3K?@K@ZslbvQzxG}vJ(>G7-|G5~+7vWm zG2t8`hPgKA|3z^F`{|<(^Ae%aL&h%cXCp2_9!S`;h^4d-EPfp)qhk->f-iSt&2K$V zHiFB`o89+Kd~gSWl5TbSW5BHK0~OTH+FD^=4QKo*>($j2z_%M38vg6Sot#ESMtC_n z2Zo1pgze(jSAp$IK|w(ZYh2#n-^XOB4@7v8lvh_1hWjWolhKZT&EWvSCPs6gr3c{R z;_i;l52f*Di#kw9xXd9{f#lmeLKq8)R(k#cJqq}R=+U{ok7!)A7TpjbcE;vJn8Pg4 zlE2_%<@(ehQ=GL{Lfwn48nKche>X;CGy+dzhpkXe35tKG}#KcsDPZrY1hpc zLOw%7@s1Mp1Y#4C2f(H>OH*-is48Y9=%`RJHVVr0Xk2Oi5b5;?1%gk!cy;5Y{>lke zl}pa9m7@~a?X%aWSwODb{5rRuZ-8u6|K38{zJ$xj4bj_Bj$zo1LVn0j6U1y59A>g( z%3cNK{*$%9E&PHW=-STjvt8~lyS3-eZzMuhx7q(PS?xMSr+;4ug5CUPZbs^tuPPp# zf^6OpK*zd`=#u2`L;ao#LG2s+F|8$7t?JG$VED1|A&$r3fM@~#bH{OSI(S3fz8*1_ z4nK->^Y7@W)?lbOMs3UhxRUb={*Kii1nK1#Q4lT9R*e^l0gI_ScaK>v<&x7dHvf#6 zP2lKalHJ8K#yV$z&E*i$$SXhoGD+i;W;R72@4F`kMbE=OH<6%k0g#8`dB8hmgC>!3 zL!N2gl&|PeZwOrc{3EiO(S@oSD{Nrcvayp{-E1tZ#D zSmbna9a(I$@`7mY@sy^1dk;J333MN68UvhGC02Zn%%MCP_)JT=? zGFhY$FJY{s147@?gBzkbkMKcT@o^hoDu_VmS4^r@sC1?_8XRW$Y9U3Nqq(#rO%#YC zVHQUu8x8yPXlP}cXNLMK6~0QqJBZ(zV1 z_f29}+}zk0jCY`HTV>Co!9v(5r>6_M85tWF0S*nWH2tZmDZn7+USu*6np$1ah==+d zZrZ7hn-HVX&vh*qH58&!AVnbl5_I>=>SMt+8j)0u|HXbXyLQLj`8=>3ez!+`IOufA z$W2I!s?~2mASE9Z4BubsWTfZtoNsXYi;42}^%YSJO)O8L&0GbXHGXi07!%)v`knt4 z>z~)~p9uRMfY9ymPog`xDe1ib+B#6o0z1NZCF-zCb9wCAtKT70N?lFOF|W6`S7Fso zSNA8t`eF20v-((qk!O#>O_(i0#F=ya+VDoK(+M{Xb^=RTo*HL&8ei0n=e zxawuuo52SvrH?o$i+`+gD&4@~VC@G65bV{Umh=-|oPVZC$W|)%>y5RwU(D!|AF!bZ zzrLc~F$SKF!*o8?OJFRstu~Ym|EEK{e+4>8cc2$2rG*lyKtQ>9NI@e+u}}Q+lGB+7 z`ajzuU=J9I0wbwtn$Qp44Uc6AqIuRi=%a(O#DUhAsdl$}#VRVf!r%_JUI$Lpsao{u--%D7kabui~h;_pu^0kECQ(vvA{SJrek zxJ*uhDn=mSa*Y1Fg7-SrejmUVFv9+PD4`>I7-3t`-L*1b?w zKw#)St=@yVm6PsQ7_Syx<2f<1*5J3V=&0SH0X>WLoLw01Y$%#QQJDh^gB0GSeVPe< zJ7yPkbv%it0UOf|T8+l~$7H8IVhY9Njak+X+HM-7hi+Z8#^ok9WwCnoRcdU3PQg(^ zIy57!>Q_2xD-3kd_jv*w*cmF;(321>h9*EMUjOo81I2*#fqPjokwYfrMYc7-ECX1e zZKt}G17fCUEr{eGc2FkNc95B}uh3CN6J`VvK=2>}n0ZXQ~ z@yjxF=ODoq6FITpfQE|wd`b#>6TjW7P6PI2A+72E)QQeU60c2-`eqAITTDiwC+Fyi zHB9)|qt6bYUXbV5V@9fYxVYFM*;O)_iFXGLo!DG4UsW&KAO9 z>>yS{IAcVJS!p`|H1T40wkQ$J8vm7XUIbi%JB_tqN|y@*G)Irar2YJ=xWcY0{2N5% zm6F(8*o0TiaI{9$ZxDxF)KUkr4jF3)Nadn;l)(HdScwywEfGqfK1|mg!Aj@vse;7^ zhyrZEaRB!DuZRKwc}t-LXvx2`WWZ32?%rKwy5i<+DhQYBue(gP$GsF-H3B$YuF48wiYRh3g>RQUp_HlqN_88RNSQ z=}q~#0#)thB^3qqUynbs(VHNaB=y2ULsY^%+YHTNYx5dWqIprP$_GkSDMj4(xe1 zyQwnQZEs$}F2c06_EZjig|Rs#ymvDKOd-dBG)w4Sp7xOsDDpC+@xsapd0iP?IxO!+ zZsrQi|27gtz97w?Qi~mz**oSb*K>!O)Y|`KlPvzv2=!lu0bf6Lcmo(TEs^0W{&ZNZ zP9mCQA8=48wC{8$DK7_{bFrb6mt zjw+n*68IGSI{9M@(}uzZ_FF2Y`?nN+T7ameIf5Du#7STS(Ps$IC_|((^S!6-uU!X# zF6woakr&|6eu0ZJqDxjEyL)0Y9cauU{bx8cgtz(E__!d?7tiZd0Da-oEZoO)>Q4NR z8*bv-(gUbS@`{0rTV&uZ_^OcW<3`+ORfAyjojNe9j)_H!CCF#`i#cE$QoQAZe(Q5~ zXD|ixW51xRsSmhvH|AI}GM7rw*Pd120jO1h>u*FdbO0S)MS_Z(uK5NJfruR6%;e#> zUt)l+>llZ(I(}tJ!AD0&BMkQflwSkfz_)Vr{PL{FRI|#5?K7l*^YQ&5Atoc=o!ZOA z_6Z?~pFeA`eB67L7o|0&73%y=3y9dgqI#V63L+VB9hJ`jl)>7O5I>Anek7_E;Er)2 z>8x}9JYNW~!}(KV08UJy8VMDaiOI>ynHAD;()M?XG3(n?6@X!EwY9Xgk56f;~g1stn3CPp{-oxSzLD=9+z+*0&dlJ}d$kFb`?b0Zz|7z+jE z;Efc=!?0ibYI$@XHuV|=_FkzZE$tDm*4)ym**88QGka;$LgCETZ+i8m!w(8NGK^)S zUzdX1_vY$L)YCEb_vjv7i0pYm$Nt24gfvYDcm&uPm{6Agc|;)8{QfAoK*#w_7P@2P z^j|d8mQtjKt2jv$|MT zAf%EgjwJi9dW`xl=2tg!`BJXp5AoG5+Tu-rx(_-JXN+E=u74BD{*Y_I;zek^u{1SZ zNmOPi_;$K(Paj9`tp1sxnZ-mOx1`ad;Htc$$V{!Xka2FJN;f%8u9Qze%&w%=>?4Qq znP7*g+f#vby=pd36+Yy|+w>6_AQi&ySg#)Ia14~xu*CQwJDkm z_>{z)a0~;+@On#lMd|1i4D+QttoazU`gio*4j!UDD%&(+w7&32z@s!bWN82CqBVml z$49Z&-9?vgWZ-&+x}+gC`V574z^0jB^CqJJ2Qmc}d?ysA2{_x}1?*2vD z!oeY3=t~7219SBbSM2dE{#j;!PuIVmzzixcQs}O)@=2Qj<=!bwLA((%mk({Ru}Bi3 z%wibo%&4ojdhe_WY)zb$vhavqQ^7g)yeIm`XV&ef@V2YR=xsbm17bx7B(>`<+n`p3**B)xcA?)-=Sgkx+NOo(IZcd=UR)!pB7G0r}Rt?65 zvsR+~X6qFXqYL zc*rg)t+E_DPLB$azMQoMLJm!R1VI{ZRQfnVFvP)<{@PMkEbRG@M~xu*8gV6KL=pX| zsO(ch=#Zfd%2ZV2Ra|k~eBRp-u%=<`lD1a<^N|r>k>5X0NqI;jDXzftf2gg?V_EWp zEIGcB3Ly!GLL`~#m1r55Y5xB8qDJF#W^`YB<{Z`_A4s6Osa~C7$i_TN`$()x-8KC+r9*&27lrXG?VDUtgNhje8ha`7Vo1}$F0@# z0RAeWv7v{e32o$1#DFINt9JF3e-C;qJ(H2DJRt`mZsdU}o`@VG_81VcRx8@w&g4#f zD1SExZ6%6FY!*Y1l^Y>_d5U(@l;IeiH=DxQ4?AJp>po2$=#O{-+u2hKAB-*qGnDY5 zPV{t21~wGylML=lQH#}X{u(Hmo;#SMW7DTgR)z+Fwlm*9W+p|Bh2ULiWx;wvqHTT< ziaE)?dXWsU=`FOC@rC|bI{Zh`p2#Owk9bMH#IB5vEJMk_XJn3Er4#P5AzH1*$x^l7 zqiW5RpHZ_zP~Pv^uM4E0DIb4F8gQeP=;rL@A1l-bzdYmE$@=2x3rWg3fWf_+^&dicTN7TA<%0m1&I2Q4;~!y3)W6kW2tr8M2R#F5}j^^(r-M@{E2$5eMJjI&vRB zmgi^4P9d-Q-Vm!pZN0i(OCUc9+WbxEL*!V@VSOK6w zJ-9mtUbu`ZK86sjc10u_wQ27j15FF&dEi+WD$5yn)|t{&-WC5 zo{zq-umoz)AaHBv*9E$tG;mNJg$$g9%ZjrRAW)!Ob_WE9{DD}{Jjx*ziC~YIXs@>nSH~S%eND9 ziL|MlBQqPDk~6(=ncmSA?5KOD>U1L!D3(;+B8jazsOmyx{ISOU_;$YF%-avb7U!Tn z)++GaTr_yrW+?5bZ$tYLBf@8IE;TjP2F+Ip1C*svSuXwwXuCsuf&3qU=gawKd$Ro8 z(brc(($ze{mvj8>Gmxo{4vHUI1VSAEVus`pd#N|fXGZ~lOcev>aOZAkSvbXBdSUV~ z&iJqq9^82Pkaf7;8$3tJ(jPuoU%@H%2VpUj8(wZ7J{JiYZU^WKG+005)o=d;j8=?} zVBYsRDY+0aVq0cnSk&QzDj)nLZ~D|i7uiE<=f{#Ry77{hM%1v? zs}n@uR2zbQ8*PFoh9))Q8rNCvaf&Sr$nI-qP>gutG1>Tg%WIPlw7dSgR`&0o|5nlG< zL#Mu*n_wTn*?CjMN@=eQKSnw~S%YHSV8E0xz~)Xsw>+WjM=lZqTiN7%vAW`4iJc=dPsZArFoSc&OKZjY0 z3nqXlNyC$5sTppWcoK0YckCU4RvArob>&IcDzW*Xx-8m$m#c?=zFBKU^!QZyML5F= zk0dH2th2r;(^d5;kA^uewxs62IRZQhE{6lj?ux&sZ^(X6u$}0|+$3ewYF53Pu-Rtw zwU~`71_+W*Sw$ClpB{m3dKLi7;o%Q?%6RUrec!bf1JubLDYqay|8HQdg@PKm(G2m! z&e-NthHJoQshIu#xXWv^)X4WS4={aa@t&PP?Tn_FAD-r5bE?#Eo4q#k2N<`?fWQ#G z=ex6V8pPhf1ovHwD@PAN+cwK~l~ZW*LLFX-k8W&igf@>m%-1(Io(M3-=VU}{k6IvO z8?4pcJ6=E$z!;d6)11gwz<8lziWyMkO3kwPGm?{q0UXo;TvSgL zb)DKW=6{GGP(MTuxlT|3A;4~}Z<(?7bL6n6Oi?-lPGfI354Ymh0p*>`#*PCbi7>GE z$++`^xIb>&K`1S3x~?eP8cpO|HbDKy{r0DcLwF&I()~b6Q=gB(bAg6O^M`9vbLNZq zE$gbAdFw2=mR>l8jcF-9wYl4r{pM!GU`E)5lg7Ukz{Q*l^4pdPpQHJRaqk1G_Obwe z)Tfq5bm}wx-8Gb~ulj1s&yG{G_0MM%$2te4)rj-&a*^3gu@*Cg01EBe@gh`@}rZzgl$%w1aGJ?KnuJUxr&Euy^D%Xxa6x~Ux_}UM^ z-_syJL=W$jpF|@P2_yL^yNNjth@ODuW+?(2p*yRD&roRtm>1H&TE1&|+Q6!whVLQ+ zDAL3RE@@z#F%r5k{c(z)M@(oK;pFNXd?6LP#}7r)*- zDx{>D%bKTuKO0ztd{NQLATdzSQOg}-nkGK&{n`7-ex{m-5!RlS0D02*UeZUGIaSL# zb|Ysh%O|S=HdCWcj0mxSE=|vQbt~IFV(1FDGE9>f|@(}!degE%YOBAD>V%s>M`SnEl3eW0+O(jhp z>unwIZvZm?gd`o($s${>zTH&cx%*!zo4%hOL^~}5Z2=N6z!R9YdOHnMVf;U)zA~!H zuI+ZyASHryN=tWlmy~pulyrx*bO}hObV_$4-Q5yWN=i51!sk6>od0{=d*zjL&IM@% zjJRV&WND&CmP{Tecvt7Ox0>twdwUB2BX+dfV&d6rK~8ZRdBfSmFe713=ctt^TP?Qi zZQAg7SUy^4Y^#v}kQSSqw!B<(B*|||(3YE1-r_`FQn zOa5?py|Zvcy}jvfVCN4aW4u5Rq+tyL$pYV-VFS;%L>j^7bQc#F&(!^3M9NhhzEQz| zuU9KYDP9+=B{|k#lag=*rY%O0)zHey({>=J`w+!nH|5&$5x%SsmgU63I@`}H#{2r%!Vo-C{^{2ArfZ9cq)ocm6 z;B_Z}wpf5$`uv5M=X%O(I`vlICW$eomQGgc*W6#D$8RjQw>{j>cc3AmLd$lV@kyP4 zQsNyl`%lO%Fp_|`Ej{j~Jp{Q0e9flaOs{&-P^|Fmw*KBGSq?M2@Z8b>TkM{YD-|Pn zHo~?$o^_QF8$1Pip#Yu2-5PAZ_1xlp_@i++#TzCz`1LY%U+B}q#Qoo>z%=qbGZe@< zXx1gIw?wd@{2M3eRhhNQxwvqk2>j$+BxVTS&CI>&v2d5v*4ED6HMrlz3A>8zRowD9 zy|iVMs{Hc^inO>xgd$A28FGxejj>5d;A6=D&H4FTk;^$C*i_cnN7K7;;hE|^xd-Q` z^=ip}dvSL5c@6D+@0J0?kpI)I^EZKC(??1=i`$33%(_mnZIQf4^lpuA$3Fr0KJT!6 zufVEyd_ zj-WwDre?_A3l5y1OGRXkE$nTE^PUn1I`gq~trpjEUI|6TanS#1X=(Xix9++4{Y&L> z10jdx{(H{8#0G)-&_-%O=lp!J*n9?$3*ne=OlCVZWWMhqatlOi=~8Y3WkmGPDQM z9^6lMmwDBidmghTYM`acD#W`l8iIq%@h%B}H%g~_u~x5fNUqupRD)fI)|*}Se*)5y z^P|(GGESkiGMx6;??4@UAeWwkKtb^>aKl%EerLCRY$@LY2@!Kuej|y%6 zBRo?gIhNDCXBnJe%Yu38cjx4n+M2dC`S>W(H!5~^)qcXD^!xtijD?(7!1J77?K;@t z$X<645Z*x*+f2Y4@IUbJ@u%*Yd`{N8aP-}~3|bs$07i=!RQG*{{*kEV9Jdain9irR zCgh%+_Kw}6tb5L=Gx~Zxo@ytUun7uxbrRB(lIG`Mb%2I7b8UlI3?`4q=@w9LneFTA z8?X)^P6DTGa(X&`=jj;mIe@Ci%GK0gs_e5=4(lmb7m0t%gZF^FM`9ZE)04B-Z3t13 zgixf1tr3Nt&k0uV1|$TaJ!0Tn+jLlDpi zvo#{aeR(FnyU{*cMQ3SHI?q4fvPX>Ff1MMVm!UV*i#e*jJJ&-X-~Kgi+w+~=*!`#s zd3Fq4Z47Y)BeH(O22_-%3lvy1=%yV*_UFG}4HHGnXjPjcCAGJ=|I|B*HGyn*e@4#T z?s&{4c|Z)lblv+)in8-N9k{k&-yi>5u{^T**!_W_X9y-Y+idXexNUN=|E$)|v@)K6 z6mtQnk(X)LSry4KdQai~O?-L zoU%`2>m*>}I|BbhDPV*#F@3MEceUv2yd&Sc-|TpC*fM;p6Z3}0<ZyIJ3xTeGyfm)kez=3Mb4=+! z^1WH-ss2}wGW~sg7P!aaIhvE5KQlb~20Z4v=^IEzpAbhpZ%)k?sz5P|xuzetBApNS zcF_05;iEml-0!LI{b{v%9 z3VRzWSyS&vC=a-k^N=`m|+d?RZH-QsBeF=E$KT!0K6vs^$D&k;c z17DNk@Xo}<;wQgv{}m830ESBphoPkl(@mVBH`qM9w)6s5*w>6Nn)o*zzy!Nrocw>c zeIH{T!;OS=ZO5lY&VpXqX>dTU#x;z(h6a;f^LeK~3?ahU&Th!i(5ulGp2Up4F=JyB zBPoC1VBtu~{|aABiPM406IAl@*3qG1o4#UwHc63Acwfx8R6e-R5sIvT-DGUYuLv;m za=JJ((5~C+$S8Jq=eMt+1Hk|{xINd!$CV61-A|ibqaSWWc@H<>cF<3!Oaq5e9Ppqr z`7HR;+J3y2v;|o|=;-sg9kqgn-UxS7Q%*?<24v3%E>@wH;Ejea$*Q7YGFN4LU0p5bQ>qccT|-z!jyCgQ+YWOzF?%XBO<~$`e=(bnqpGyT`7TUPubsg2IeHn9`;)XP9kp?@A_AJtDoaBu4aDh8F}HBtNI)O825&w3*FoEzfZJMS$xDxS6h3v~oPd2LQftF75kAPZ z6i`x~)YD?2uTQM}+GqP9ihycxrK!t`QRjoERs$h?%~Wjdk5&hAl|)HIXp2A4(Gi)- zX`49dN^uSAv z))D0}Ee%Gv$!Ygp=FzVp+AkUgpDw;?=;NVt+^^3rH78wDKePJOvbXOn}QTNpUzND1A-dx52BH-m}m;MrjS5Z0*MF9bb!;&!U2YkLhcA5G4ebV%5hu-42TuV97 zq}Xd>J@2)))!&Op?p3&P+|X+p?k&D6J2>L36Qc`;W$A@xmscEf6(w7gwfQgL&yCwO z&=K&7_EcDKWaS* z-fk`u*kPC!;Rbqd8JLdYEbu;2MquM*Qy?sAK{=e@ya>V9_#TZL$AInTe2z3RU2xGy zPtQJv&H|NMjm>xv6{c=z`v#&Q?cgQh*PbCqnJw}|8f}zH9gt!bV z{f<@~xy;F2?xPI{j}D#S%~{RrxbwwxO0cqmKn~vOEA>3OJ&^-vIsxo!!QzuFK5y zXhlB;4(-QKx^@$5_o8qOoW*rx@gQMCorHI`h>EZ-^U&4SAXUK1=g*lx$O_~Kz(im? zIODzhA_x9fn_<8Z!s%ezR zR!R^RzI?%`Qfq-IWS^#Rl+GVjctrh+d04S2&4z6UPFAjGPIP{;OTA3Ap_4meHq>mo z;e6ZT0aPbELGD0AL}b#^Qm)tH%0A07Moj7KS0o@imd+yv_n<&&CTakng_yb3_4Oj{ zw%V=StVrmRu)O|4elf>$bBH)gpE=aOAHi8{3Rh@p?UcolXLCYQGi;z4BCNYmduM+9 z0JZM#NL)PL=-n!bpswI9f|5qvgrozCxtqqh#EtYV)l_f+>sY)t+NN-?7)Cc3O{B`V z2mECzsH=5~UT0_7TQ2;x#eWHR9Xpqn_3VYgq&S9cDF#n~_jq9JsTLaK#O~Sxkfp|+ zWCGgIw}j)0`c4^wpFDfRUxQTMq_Llf$I%oNaV$%84P##Qc`fDUk#Thi$QCUvw0r+1 zC-J!^D<4fh+ndY{9D_Z8iC)DKIx#Tp(@~5;YQo>Pg%^q&+7qPJuWP}L9$IpwTemtA}DG1Gn=J{Vt<}X*Bw2JH6b7q$s)#mZU{>v*BCFbzU&d&OU zr3D15Z|ia%loAZ;tkfJeh^o92*vsWdi7DSxGcnn{j9+M_py`hx86O|_eZ1KXu{KP} zB3a~l)(3sY@yf`@H6Qd<5@e1Op@0apdI!;G%Ch=FrrxM5cUZK;R(jHh4ru^zhtMsE zfdb`0x&5;U3;>>?=(sNBZaEh>1+ZMv$UJ@ zxRfj2IxV1+4){50rRr@@k9Q^o1VbAeAUk{bF-=T~lGW{%fQyh8F4J(REb99%P(W5? zBY|}+2;i~5L5oHvpLya(V5@kafnXTpj zGK;)Y3cJOqHTE=0#}Hfh$HpG_;-d_!cA+EOvG9mAgH(QPz_ToTm!S>R3N)AcVw9nR zum}TLYWJ;U;;u?i7^zLJhRB8ov(JW^I?^L46>L|SBk-1c)5`fT60|}U z)&eo7mcAyNWJYCDN#DaEzghclWhn)+7OkI5mjycMRZEiv*$qh4H$|WT|6vQbItSt$HMIE#p8+pEkzcE z$s#IVCRwJ4;?(Uy@WxUqPv87a#_0xBK~?W)!)#fhd4Dt!2*QAf7~!X14Zzkh9^GH* zeLWk=W(0R{N`1=HI(d@(xc!ci$O~lKGOI<^ZL8dNu=TQEI@_b7zSv36rxxKb9$X~u zKw8@!$log&?vTdKZVi6pAf zFKqAl0gwJ&vuYER&wrO111ozQ211%a>~c5vh1Y1E8m!k~2bn!a^IiU{ASSFnHi$`~VY!Fy@-f8vx{4Zp%bp_=E;&&mcj5uap z(18PQ0|rzwS3>1&)zpIP)ff>O4REUSEuQB3VVv=igS1%&>07*VCi&0S-wU&$?WM2I zqhFvQ2IK!?q4NQmW#H$uG1CG21$;9EG`aZr4$lG zAksQMUmv#*3mf~EdUHLmrRA;{$H0``eQV$g*12h{^+tD)v1qz+S0I8I4-dTe@-nK0 zXl#xSfid!TxQWeo$P*_OIxiiz^xE939-c01jYN^u?aB5l zunvV2MrwfeJX(5c1h!t86uQ)bQKu^m--%HeS*J#o_jvqV$x_i;GVk299T>X|jZS=(_e?buU zH6bC-^E00t=x&*JhXjfoDytY67;su-O|!>>bIFNjHXHo+Es*^&zeeEgxcyUFym;T5 z-wV;&WO}Hnjx=)FmW9cVz)rEN)V!acHKQ-U!-GG-rVoXhU|G#r?SxDMfeCCpo`S{{ zHZP&s**;ffPsb6h+Vb+KAI^8V>l|pp``JtxCW!#|!S!{vDenhvH(pRSE&_J8jf1Ty z@&qPzA7e%B3<}EDwX)jwlK{ZZgt+9-Xfkfo8r{f1b7!u)5peCV&uaB?4o+j0@i+`K zPsdtIf8l1f-Y=i^y}&7A77ftkKdFg<=q34MHT3I3x^+dgk1KfrD@TQ^s;TY;dVNsy07{$no3b&Zw^lMak1c_1Yx^?tf!}Ec9AkK%rTJL zrV--YaW1HyX5AVX9$v08ks8vy+1SBjdlQmAXL6KRgmraBLCPj`aYN$Q@?g>n=hldA;;%GgbH(H0<1-d@jY#9;f)~!M;Xu*> zyRgh8+|x+$vH8N0g_>H~FF09Gkg61j-DU-+re=Nq3}4;08;r20>`=PTQVZK`X^mo- zZ{SB&tv}Qgz;De1Pr_4a_(J^+e;iUm#R0_-umaQTMkqS z>-ZJWUuzbq8wcc2b4k4a9Qavl6fRgntie0rE-q_yU#}dx_Cfgr5PQ}$NJZo*Gw=TL zxsSE*TX75i%QqTm&=#-BT8Q!-O^xg(M>lXjvd+>ma41Zq^G6QbqAZ<^xKJZm*>4nG zWZV47OuI`2VUeGMR17guaO(jvW#oHR5X69p0-cbX8DA%Ja+vOW>!#N_rXqc0#L{DZ{zJH?xK|LQ&!3?9-_yA1L7L}seZ%ot)7MB9uaKx=S zCgG&SRQM!gTL+-z^Ai1UM2w|MIJt*R7!p=vZXP5xOMLJ5U*BgrIL2&-@v>Gj_PZ zA0D5e*wOXqIvaL~hTh-nt9`E;@?@4fe@_);Z+%q_Y4c5W(x79#$HL4$q#fkj4$jFh z;qZEDCKIiI)G7%^cV1< zt?TG6LJ>>R5xoQ#DD2_anA*dh7Z)?cw;kB(+Eabdw~jK7Qq|a)N=}25(B;6UXLdY; z$b~+tXvi-O`e>Jj%fR50+|@N+(ipv5o+qYEC*sK!w0Q3IP0p@^vkbZ(rN{0ML!i$9 z3gFMJQXmR5gC3vFzIWJX&lHO|J6KH+TT$^-xVMSd`{}KMK1#A)-GC&i5(`XSy%Wh@ zxrXp=^7bVi1>Op)Q<%my;Nq}jXMQhluw#CKF+|ptKUH6?@0S=D1p9)5huf?RBYlef zN9VaX@kY|}EfDsDdgUlTKM7ILVjKhJmD}brhG9~3HWgG(l18e%%i!H^(QVB+ud@ct zB+X`L+O^^oTHM&pO?_YZfBM4ja_%2025N;mL(!$J+Ppko^+Z%sZSZFT#OMvp*~y%) zaCXZ3ROx;bpr~eVh;yzaZ}FYUZ^zG`;fGP-TGwA1NTIeLc5wLD!6yW~FN3};Q20D* zAItJA|!IIGGt>&vl*;`aMpu2D=J5Jm%a`ahsUn43TN%u55w*L4NXZU84cyS)N93Lb#B zgfW>@J>^dC>~d4O0W_8OFff{Kpc7)>Ak3w_p4g)HpU~IQGBv>r-b zS~M7n{b;(Vh8EyR{AAOBT?;J?R~(CHio}=;g8+pU+<>LpDI0de=;ZuMcugQw zvNdfOEGs!ZKlrC(O`MIHltcfV9DMow@av$C@s}3zU|aiMJx8|y7rNexfNbp^>nI!- zx>d}pEs$e_K0{AUx8A84D9cDi6Uz7spcC<)Z$>cP{`@lp>H?s;n={5)ja-uOHD!C3 zU*I{@_FNhM>eS{#q`x_kowQmRyn9_So1LXXj)SpeAkMyGIlzf|MxRi08Ep)-KOFmJ zd!B9uqY~URNSTC@OGW#Qfv%x3?*XNjlHndtaSi z!#TF-rumk%5*&J}pjT_Sf<9Zp@YDDXK<9TLbtvcLkISA+a1!IisdM_)aN_69vQ$hb z4g10F&lnkeyd&C^T(;L8U!I{9l1lIm2a<1s%{TpaUY^9Tj&9>^&Y1x=6K5L!C8a-+CeT`~ z9d&gj&JF!^cP}*fk9bHKsNB=g6$uC2)j4k?JFJ;D}TSe@YmU!BXu#@(FXSxPk;i~( zwc17ZVc}rK@U2b!Q4`Z0y{^}3mqXUzcGK$fdEhTVz^ktGcfCY`Dk+SfD|!0{D2<5h{<`_Z5IvWM4v-=d zX(82?a;Xv6dhK;mVNG+d zv;-{Drj7@#Ee#(ICkF?r=tgEHF_xUK(5We6PkEAoPs*S9nVGy8DB?Vry-G+>*5aXcXaPZuWS8k5l z8<>JQ2ITb7(WnG7s8m^6osu4eh1Z1ST3UTRD?f=hK2(3xk7c*q9i4XPgKqpmoo=Bn zPzR%?U)f0{mzGRZxi=@-Ly>g4*C%3_-m`r+Tnsdfr>9nJl~|XG;CUOdr4WBfr8IcB zy1vI>s#MM8hOp$~U@J8$SX_~!gu1?Q?qbC7!-l}!E69kdo9;o^>i=jNE1sm=v&^!= zNq&cw8uK-kr|YmQo*G8`d2w&%QNJ+o_;quuF81+rP4nC>KYgbiPu~eu}A85=!duW% zLr~PW!MEcFrO-+MbgZhZT=!VdAq;;}9HjOpFi;N3B-G`9) zBnxT}Y5|{~jN?^+e)?rT=zb-pZtM|`*fb* z$nvokS3Ev0HTCZydE4(9s_4Qs{dV~JJvMXLZu?;^7_kxjUH(o0G+Qh{1eqvX?GmhD z9n@>!wl@0AG9xU!<)cG{#&(kD_QzI%fyRqhpW}<`G}2zus+ij3B;7d)(5J7jr1OJ6 z)ee6%Qpoi?3{qPciV2OX`f_E^zwvBEBwiQv09TI)+Hqi8uq|vp7}4UTqI~|uU(`-F zNj4b!BWX3e33v7-kD>cx?eYvXY@z3RcX;Pck`D7VRgP0edV4 z>N1_Vs05r8n<jRrJg_GevBuXu_JzrM0cl=15{y3^t_@r4mKEjXPxAua|lp z!eWN1>4;=}W5^WEmeWC#OXK5_X$}q_NGyzvOJZzT<$h`Iz1BlLpXlj9`A*Z%=>jtv zEP&TO;mgEy#)%=M@Av>Mo%Hd~g0CB)YHqX{nyrEvUs zm}j@&tJYpxJKXGy_v>~(0jk{!~E&3G*$~tyGCp6hU0}3EVcq| zRAJZ$;3?zmI(gNhYnR@jRXh}^vuoz2(7vXxRk8=g3Lhe!Dy!T0wnEz}abSf)LcMrII~5sNS%5L4HRU6|=+(}bpj zY^w*GA+MXfBp@x&zZk~+)6w#~_>(!jg)qNt-fq=FX95KE{Tm%tZPhfr!Bsr=#cX^x zwKi5{@jR}(o+S{b=P z{1GQW`8E1gA;uma3Tj+t4q7(0JyLLVL~U5=`e`t;_B_Z5!@w?v&aiSW&bc*n6a?zXH&$z*5d z4^ujj4Bvipr4e#hnsiJwg10DCyF5jTWko|~ylFPp_Nzdvi4+kTBPM0Dk6dDFWPI0X zq=`O}omIn#Sq8xjj>XUk}bD|*P% zHyCb4G+8t<>{1j2zAVwfVmz{A z9t^h8uXbNDvj_?bGBDV{tI*TY!HfaLLug)bR9&djQ(CDMq1iof;{!vovep3DZU)l! z?sW|*FnkNm3i;3A*OgX}8YC*9d3iY_KMOrk2(#y+*e{NVf|BP-1>>AQ#EiHiCB&kr z`QcS^OLjIX7uS54kmtH10Qb7PbC1mg|3$T-j3v=BO?o%L7I6{qjy!zR+>;gYtfppM zMoSJayDL?gWu4Rz!x>H!wp|`!qV^A(yjT^hANO_AeQ;MnobzAXinK-PkCei6R2`-h zJWE}ZmKNqb8np6!EQYOzXxPf(yowOJk91M$&&SdN+x>1gdic-_ifC=?)UC4yW1`0#_ z8yl176t{Fu_JFW?jiW1Az`AJky{L!=dW}KRTo=-!12b_nl{C%oes{p}z%9TgOn?Ia z2Y%0>bar#K<0h@6$?Z6^#%B{~dSGB+{QmvhwN6E;q)`($Q&lxdeF` z<#JVdg-~hY=V}{;hS&`45TMOy!&R6B6IjA9x{x@0EL@dMc*62|(n3D9v-&cMHCAYm$)L0Jh2Do@C%ynlZYQAhPfHFW9JNJ#ItC}d$%(%w+&_e zI9_~*3;*e3;`jwNYJ*)lT5m#Ng3}X?RSbQS8y6n|x@D?Sjil()N zjsV+1@X`stG4ppKagA9^vxD?BS}7z=-(W(@UnckGDR%xv-90_jEuEkUi#b961gG2F z*ocCHG6I?S`ST}KiQ4UtTvcfH-Vf>_$DLK`<=V3g3!1vRkqxHok=50#`=7FhGlNjA zZEYp(X5TG?U-D>b@P}1G;#{4a?f{Tnr|>@8LBRr!x|h0&3lF9Hcy9Tw_<(>OI|3^8 zg8y57P@WE609F=lwy9Wms~ZY`$t4oh;& z6hkKziI4`q%{ z%pdA_ATgQ10MTagaL8s|62Aj*&Hl^hM`RNcu$1T}dZcA#yY&}@$<@L2lQWSz_$)b4 zi(;S0>O8T_?`fwV8j*d1(z9+ zw&wWS?o+;jJh6oNk=nboT-Z-u6MY6wEmIQ{FV|laS}HM+mH^|60K)!QcTN|I68ZLQxw5p?{ zL$~ZpB|JYrZ-R3WsyFjUwg>M{6dK8xb7pl*0-hGrU{Mt2vOfn*8%+}G<;Txw0!};f z?(Xg$9!;MzaRl>|aWT}tl|>$l{+py!6k4)aoLYzifz6UA7PBeTxkC9OMn3K0W!K~f z#1=Bt@pzrNucwa!j%&}v zx7MpC#9vpRpN3l{Uyd|uuJAAL*hIf#K392;KdfwR(a_Q7nWo~hGBY#5iqPhk)YaC$ ze@EWR;pPW#2F-Q&*7xGTlpWte;e-r4Nwev>WKJzIf`I;_hpn){qzX4%tE|bgxsrg( zH-D>4$+3gUo9sxJ*@^AOk{-gPn3S6**~257u9jVKeeGVZ-a=CqtE!?>;Jg4NkbiPU zonUWcqz^l=m_r;m403;PfupP@kQOYrl-@yS#zZqQ{0|w+^!$sw7yg zll@ZYc!~bC8j}CJ5mD{SKGkFW>oJWJlL$h}#o=D7j$u9?sX>+Ly}dn1H#_Y{|MD><{b26|-orC7$z64=&X@SXe|e#n39DsPrHL{@KUc=Va{yoe zbl5>_?qkq&B4%`yfo@wrpnQ5ny^ z2!jLY4lo&XZ{Hrc!fl;YN)H_#F`f#=I5ZMHAB*19Qu=ON?|co}M>`{zu^W@^@~55; zD`jKK0i2YKPSLOCBAdOgY{4^NV7Eo%7C{4OdU_h?lpU-ZvyOdhNdLqHXv>L;iaPUV ztX5zJy?rbGrn${4Ylvgi_}^|U$_B4OTI75w3J2=0n570^^B{ziMr?+X0s&=R{-XZ8( z+t}zO_iS%%(QAq~ef#zeNWAh)n1hR#J!uh5B4j&G2j}J0%PQjL_N6d&I=Aw9@e>(l zRGExBzaG9a;(7!-VE0{_@xfR2XI7{o&Nw@YYIK|q-t-R5M)q*o%_K;OfzmjWL3=Bt z9~wVk9;eeTsJdanrjZMI6<`2aSXdYtE1H@XZTXON;n_jYft;t4Eut4PYCNh(4iD9S zZ2L-bGyKyKxc7wFT`eRZF)0M{GjLz?4u2R@W*vC{Co3Bp4r%x5WUdqe zKK{-f4UiD9w6whR-3EL(us=OLJ*6cj2Rs_aMmhEih=@u{+>103y~chYM%DCYyG2nh z%NMN2k~MM6(OQ&qQvhnQz#JSW7r0#-BZg*4DXE(3YGV@0z5pe8ta&mYx#xjZO=;&UPko@&jUvk0zZz^RQ`cVD5ZN10@Y`fs;i(x{_ ztc*=UN_sDKGc(FqTYZKflDRDW{NDO7$_;gO5>isjhZ23S8@+E>E1cfFi_Z9tg@X7p zc(+@$olW!KJ!8CW7R60-S`FH`Nz-m8rr(_g{1LE!0KvxPOALp@s5>w^n$e*OdYJ~e zNk>{m1qG71lX-c0^xlkVyP~_z&*wxMLmv`>BZwEZ3 z$hlzCpEpA*Fyjq}OirQZ+0`QL*r!qLRI864QBYAc(5Gx{ZKI>2GSH6@&fT1y7Y5-D zj*ppWXgWFQ85ru8zXn8({WGf^Tv0J$-abig;WT*jWe z;8-{n*Htt$k{~FxFhEFmRCsq~Mc0u1e`Dauy3 zS#J~6a(e1mu5P_tK}AIskc0COt_Rc)-@bie&b7t@g_rT46=Z=`di3m{mNi`v9bLGdGa9}6g+0xN{WMKpN0-0uJ`vuNk#R4!`g2G!{S%;SW|Sjp6NvzM+Nh$1A85lW{TONPp}jj5*+-| zdfX>^u(1o`ijIof1|GyFs=y*7z)Vr9|ESDqqjGx6T>&-<&~%+35I=~FxX633X#z>N zslCme-fmTrl@H%|200^oJ%7p+O}YbQX=rFDAn%N{ap-t309I{bVIk8=K7?`KnRf&V zyhnY&e9Ha-Q@ecgck##Ng1w9TQL|?QU$%?82`^6AeB~`J-y95!krmx*JgNn_K;Vjj zi|zEWU)_OihrFPFq$U)Y`E0hLRp4Ma4BiFnO8` zvyYA+sy@>BnSA{C5flb>@3B9=YPtcvMF$55WYY`n>DmXZ=-5tW^m_HB~p z@bI4U--zhxIE-H|)~~Hnu-b7^M$oU&V_W;M^Yf>tq#&gp{QjNdLhdw9l$@Bzspx^q z;0&Votu50i$?1s+LvxZK72UGy+ZQ2GTc-Nt|6RMe(o*eWlvh-{;cY{0vfu{Kbr-d^ z^0Tv_g8|hnuNpE31qF>Ph2S1U=Z;_f5pZS&Pf76e6H>P{emSL|SuFn18@7PI+(5^h zp~(t9`D~c!*jGwdU2u zj;Hi*R_f~Ofl(aa%+Aft1)FkK8;Kt854f=>CnsfPW$o>PElp!@>Vq2^Lgn55V1?wH z{5L|58OrpuHLb$EdPKKf4A(1@ssmBdF<`igii$u)A$Z&lHlxNKy29VA#NLo(L>!(R?fB!Ob3pdgalxHha=P_I}-RT#X!GMiep+k+0cv&5f|KGgx zyWU}RH%+M0{(UWyA%$=qSE+ef26UpKgvZl)Odgg59sArF6~n-(76-B_mCM|`vzJA?Fc7}p_gZYB#rRo2+1dOj3GYsZzRpM1w`-kW8qAbvIJeLEdk2P1HR(fUKqLa(T)DGtyW27GTOG^VqvPW&xf4Fx+(}J+{daV5s1f)= zDaJf@YmCs=L7$Ogo<9ffuNGSZDxCoqou@A_+fm9WgehUYj_zQGRf71e-%?Xk1BKVs z)YSa^d_Xl%Fv&d(%&uL>J>NeKxM0+`R~0S>8H(&px7Y8uWcS6PDwY~FIU zL>ZdA^vlfWJPQN^)uNE?@w0@hhoA;Yz4Vh6u=(#iz7s@&#lw&Kuv28O9_Jt?7RZq< zkl+0OxqEE0{Pj*h6-}DRQCnMC!)(oGpZ9!I_Gjkl=iqi)A~*sLae6&Pup_n)N{Wlg z95ZsFIok0+4BSw&&9YlFSXbA}kBW{0=2l0v38TH>^9Q z@=Fqut{9*5+^nbjU2t+h=2{;jT}7YRW9ja$SvuR-->-;s;jwe=43vIB98mda&%wcg zA4Nq;3Afc2Td1W|YQY{}Sh)F%+&g2jH>KB@MVKizgNla-uQ4$x$qhW^4H^`LiciOy z5pTU=85kI@xleNL0h2ij#O5F(F=3D3K!*APPFkUE&-~mR{8k29d--_7)Y6jY`JQs@ zEpTTz2d&!L3FK1ZSmv#9arTV&$~Dik`>Xi4O)%mU%0`7JXP{~2kPEDsc8q2>y8z*eYy`t%8awG?Dzss^4k)YLwA4_*M( zICC`q`L(sIv@|*RP&iFmO&vZXJ9sV)jWemC=Oc_G_RMJp&xTLenEVF7xz0wxk%&K9 zWxGj}7O}l)>AiA3xIf6p{Za{oeip3CYOZ0^54-&)(jI z1v}XHTIMV8-aVWO39oag@p|j!lJ%D_U)tK*u;;d)>ju5OL;{|*b#*R+o)Ul4U|`_5 z?rvy{i$_ME1E{63gp7;_pzOyQ$y@ZeZSa`??OVVx9kw-EEpK{aB0N4L6H^vzf~mdz z2{?x2r+Yx#LSI()MRH3;#n_I(??10t(rAyvI)wUjzy{Fs!S7|=Z&3qwh4CvFrH3Q8 zjx@(bOb{V3f~*4;9v)y^O0P=`c7foEl(_iwP=O4>cfj<*#K*TfFAW)jYLR$;e+DiO$>Rn|B@q@@LTaj2>DPdsp78%;>MNtF>Y}v| zozl{clypd^beDAZ0VJhCLb{QbJb-|R3P>Z}A#iAt6%WEIN?rCi(=DI_7(64kTK>p;)Sir#=jP%u-cVoullX&^sPS>Ik( z1}E@50R0SQVpyFlzsAZ17`$w5ZmzDb{u7ScOWu+_0C2?r;OpDlt2Mfq_S!FQAV%g_ zd9Fyw>lM!Uk%x!F#eb#BC^|fo(^Ld3xiY=#H%qMnf*c%2BZ4cNRd9CZj851y=D389 z&up{fL+d+t!EFPsS7fY)gOefEDCv}tE&ymPR2Rzt<~}2#+7*y@_jh+7NQ64C00qv|pa4#7%Fk^-ldl_#VUV{2#-8|p441T+(Un34Evt$MBD0s+vu9N^*XCl2m!qgcq-aVYrQ8~ecI zLh^o30~9L%C`8>+xHRJa{30Uc1>dQbv8AP@9T79|uqD884gMI)Ly-+w%^?;fBqW+I zgjIF6_8{5ua7#-|@b4@LeqkRojj|+L&5|lR-;LM2I36en^q`?!ocv8Sa9xrqj{u|H z{%SmuKLRYzV$usQE(+@fEQj9y-5R&1ZS-{JCCq{=4J0VB z=as2~YZnUL1)QEHZgXMoe@ip}fz^EVdthv?m~c2IOhby#*K_vU`}A9XyYTTM+g$=nWWeM}$hXh_n8uyoe%Igv3u?3=jJ|5?g7nnh#nYZ09c2SkrDW% z)#}>S8GdSPB!#L}pvL%vlo~|M3@Olz?X9gzFN4MzF!~DHu={zaq>81nW*znQXKiBX zTK;(&=^7Y7D_ma%J&!QL=GT$uN#Q9zJ)U4D&Rv(AN15g7j@W*GVl5jHLuX?Od|)zuZO?>BGw zl5YM3B(_MkMPG*`0XtDDQqoTk@Y|akKuiP8F96u6DJj2C7&J3pTS2^eVsf*<0U6qB0d2a)mVgt1Py+^}h=+#G*w+W2k7o=T#kRhmmCV&O*03mZX1QjNK_XE!d?>~P=lR1(pcu~*r&^#-!l;q*F;OCz61?1>2~B^sI}uY1ZT$UbA`R#uiJgk_Gz2D`79 zb7G&plSGw$LE$aT8+_z!6vwc1EksOay*|2*x>|#O%ao9iW@y=E&LVfoVclSDEveq+ z1uy?t1L$y%ms^LCB>&xH0tXH4_eVLdZDdqbPxhPLHzH*=Z4EF-DL^^X?0qy3q9vU= zEOtRc3#m3? zK$kFUUw&Nzjz1{|8%5NCB1*19AzLVxC-zB7B|08ug0Y_y440Kv=7&#Gvh`ph6=ILO zJoX6TP5n-%z86wMT-3h0ShGL$W8jM7FE(q&pbK7NS4u7uHZv5bw8`O9|GOFG=REl< zs$TLEi;P##R|HK;{0u-kVt-7S;2Vjdq{k{D9iDSXFhHr>L?BNCw>5SxqTp|sbps&o zl$Khh@bz10OPvDB3y1);x3dGq@~J0yry%Xg)YKHXHLr|$gI@nr@Z+EIOx&xBXt=9m z16mq{b8B6Url4Bi3_ndN*xcVCZK(R~`6k|9L8SLv$6PI^S<#dc;Wbe>;{Ip7T6K^7 zzv6Aho`^u*P8kk=_+MT}K9N69sMogeITaj1w(Bsr5Wl9DP1NaN-Czkh|M^qe0mXcs z_rep0pX8l8jV^LhMvIVHt(owFRiJ%0bZ+iAq*aACot>k<(Gn7bD#d+^lew>iFv=&?jKmRqHZX&p~q!znuUXHE3pTyF@ zn|2j9Bv?1(#v= zJOhJdvKmk_`ccJVGWjUy0HP>jAyT-Q-ufRQ``+DKE_^oJ+TCLx{-2Zh3-j|oUEq+d zHoL6@OaL^L@FQ;{unyLzKc6pSSmdPxD4zeti z`?I0Jbp<`t%l`(?fGjw)_gaM~U;kTRJk?;jpF}R>z|Y*QYrMdAyAh!+OJR2_bmlI1 zA?Emz74OT2w2<`vxS1r9FMhA^v909S-u+D>tyEOA!O;|68 zwjJHPYxMpqSBgP50>F(G;{7UQbZ}5fO>GNda4lTp2xvmEPEGfZkH1w^q&DFKWQ>oG zkF~LXcb8@5PC&~z2N6Ap;6~%MuOBNlwX)%drk?I%K$t+c=}hyrB2ClV5EEMb+AYC==i3pF9$Mo}4T~{46x9ZSKX~d>hfU zzMf;*s+=lne;#YR8hOv%mp@kr|9iC80foE)cd(rfBUf9b1bORqBnslw>#8Tbx>*pEo z(=lKi*xf|diR%uZ?m0j4qU?4Y^G1Ra9QQS8jGyuvUYQ0~KQ?A|Xz|I)BRZ8-=G;#) zBoWEa*umgHy6e4K_wKWE&{V(%SgB?5H*9TB~ zP;h?ziWu#4Ncwi`spV@_`(cA&UloF@I}F+U0!ef$;3}xKDrzQo2?-m1=y=H9ON}+Hw|u&LpY4Ki0k+^j4!NnDHQP`eORNBL zonLR1s|pM0RL58v6+S7kva*6YU!82xdPlw2N@Q~R+te&u?G&LDcAu9I$uSvf*r}_8 zmvS0Xd235rJa+Zh*|_s+9!74s3F4tyn~8&qp3MYd2Tzl8pnYsU-CJ$}7a}vf4js4O zZEqR4T4=ijtiuj}{C3_d>F|U2UyRJ^Z$29T&|`l<;wetmi43+AUpQTy#BibJG68ph zqw8B^p^$obm_W_j8*!R7$*&9acV&`eI@lc8^k4*c(S^Xj^T!)%c^<}^T)g-U;7njV z0q{6cU__{>s0hN}^(v+hF2TuUHlVukyDTdUgL%8_W$&cU_3MzV)>agrhFbL08a=jU z6q#a89zVI+nb=kVXWo-W+c;mJ1%Wet^7`(sG|6kEH>6-Bkbn&L^1=l*%Ftb~)J~=u zd7T8-ASDWYyJtB~6jEKW%BmOM9a|-OWqe9XP(9C>>AkrLf=E)@36lJRcljRyTym_uE%BAv=D-xT8=6=?U5H_wNxS3R}e1S`b?L%n#!{z6(wo zvmj`wxF%nDTZWreceT{3mr)qh?E?cLq4l|RwtHP&TwiLw}!nYVs#N@9>gL4}kVNJ;eVq=ACW^2r*!LolXa()S!7UMHApxP1ro7E#^ zZ#{$Dr_l|jIkW&W4}hvUIdQ}i+QB(RwAigK;=mm}`<}tduRxEDk^>ft**RnZT|K`# zdgn>^nh>w}p6LyN^oSmplpAYVX262^w7&i5)M z6^$gW*z`iq?Wh0uw9a&x$I>7A00!5JD&OhTO?=nxYdsj7R{vjT|>L4hng-{s+(K0Y*8_3EwZBV!1!O-M16{6+-Q}1ji zRj`&=UpT+;1fU89VgSE{Gf1}c82yfVa+i}rk7!$^()Vs}Ex~s~lwzAJW>q`z98wyv zA5o^@7vlC?=!Ao}srKpf`=g(n|GOfPYA0iP6Lj2?qNCCOd*&U6Mg_wUZhgt;r+7}c zw63nz^+~dR8qq1X*iQy#XAi%wZ83IaoBZJ84JSdv7hnSAZYypvymh1O){4J)1#o4TozE@tbLDdw^kljtx25d2P{2BB(A4%)wQ1^79-*x%bLuc!bl6A^GCxVcq<`MYCXnnR#>r#LFG zD^O0YlDEE^fsm<`Vf7N#2n!psUm`i{*oH~>EYcplaBObI%*1td)nXQ1IGnfa7WG%2 z^w3RLq$j0$#0ykadeHky@S!>^jE0d3)1=g3oIPN2btLz!s}9F9zEKAQ zJj1%i8~%SsFW4&x{;Z#hi^JXp^z`>nTJ->hnWh`=0bfF;v3)%a8&i@^?AO9c3?D2j zkGo!Pxrn*MIKsBkfIqRVRch`6w3?Tcg?`(j^If$q-bX^cDGPN@vc+V;(&}{R>6NKB z_GPg97N|PVjs&ji-@XD=9+pWVls~)sd_uHe$8`R~j3cF(1xc3MD0kYJ(X0B|E(l&PFW`flrntLdfs z^G&U=02BR;?yG3IaL5*ncI_5{)4X1Fl@~^~8}DJkZ;{>QX%MVXL%Hm?A!!V%Ws2tc z>%oleoMgjIyBsW*wCe7%5|f3cixSb zZX#yklxQ=Zb1V8qj>I1q#Z#sY046Z-uqDUE!AVR^90mC(8kOUF$Lgc}>f4*^R+a)Y zP+3|WbMrNywRQS;;z-${qsT+)=|$)4i;~xRUJ?mut-N!&V;a=VH@X!nr;NR4Za6jk z>N^+TF>cs#0i?(V_(RL=AOGB^ad~;a#oU!_UP`tI(xoCQz)|m9lOTZwTTk?Sff)|} z+qCPJKM|(p=iMIyZMC&s3=HlnDp5~XX#TW6%zijpU<+?U{aT=Sm1{j4Kd(o#71$3ED4fCIfd=oY)g7HXf9{gQ20yo)^1k z8$Lm2ap&RT>*veRv?D+I-}1$Cne(kGS6#hE05lY1|x#wR~OAjZf107 zLU0)pbhk!l$dH7ubqyH*{tQ?63uV!1mTh&-2}dB^659RBO!o2AB|(0wXYDgib{VnI zOkGFWw*|C?p9mVqy0Omu+UsmH?kR32{O=B8&YOj_r1NpY&vVLw$RqI^=UsEx;`MEi z@~zx3brI!GL`|ycjW)A{!jL5^v)rd73C*{;5kCo%fs^Gb=*NQIe$ROWi?ejODXZu)Nq)recbs9ZZUtW2vB@cNvxpWxus zmm-lyj&Iw%%QUi^_^ew2Bu7l`KkP=xTMl-1`n$6TKQ+w$1Y@*qMRH5aE;~?W=caV_ zfBR-9yZl3+G$l19ykAZ(!l2JS%ROJCQ=Db8QNFlP^4MOf{pk@idcQco#A9r^KW9nu@h(sUY{3Ufz|#AT#5 zu;J>5UDcM|k<%6W6@VV5%*|Bj29llj&vSDr(*TMjQKJ@b4-B>I=%i$J(|e7DS2*c`5ek-`&UqCgGaa)(j|r z&|hGnM+a*G#nl)@Jnt*X?|tCmPRGQAx{zfDsCQdCJFbh3_7k*7bW&${U9`+#ILL%`1vl!U5(b_ULoKo$7(aa?qtj`mf|d7 znT!M-!TmTIWu8b9uD&<**R|*4TSGp<`yS)3d)7S)Q*HM88&Hmd7cQpCLR?@kP@&AI zyI|5BF9q6_@6UL72^hA(XqtVfeA}iy@h|Ewuu-r=UTP0+?=@aqTayYzg^Kb}gn6>A zQ!vh4{UMR)r?uACN9doMs~vnTCgH33YSM9p?P~vEho?Q}#s0xT_n*7PrZ-^xfQx|< z6B|3>Sm)pR0On?6=?;rA2+d_>Xnf8jB$WmD8=1Yrg>kXd%JN< zo$E>)V2zIT6IyIexb&Nf4nUl(wTd$KxkF8MT97+JOST6)kvPJ-rxI&yaDedWb8^vU z+#~|OqWgCg@)pGz!&;=pk2;Mlzsv1PexQHIn2suT>~Qjr`!XI@>h-L{%2S)uM%H$p zKCrP`=Q}6$*6>VhU|yM79pN+@F?m!*+$5GzSBA zbqlbe{S>$hr!)KjX$EvA35opWX(&IKgzm2qMu>VJR-LYQTmeH2cxmsc5}N;BNv39W zdlu@zj2n?BMzFi~xd8}+J`>SmEZu>o`NFoM;LE{2F`J{EF?RhAQeBu^L$fju@HdY zT>>sFi?*w;7soGbI=;rXgP9A01+WC+`-rJ)CWFMt3)aq)Kvabe3{Ra6R>ZC!nNh=s~{f{3?f6C$6t2T~?6U!(9M`c-pb zF#3`xXtjTGB|g&3#rr<198eX3g=Toq7{9*sIZTINy`5lj2=6jTm=56_p7GB*6j{U zHS>#Vv2{>T8+ar4{#fLCxCPffB}!0Cs^4d4bVvfNU^W{Y48~jWqpLuhw%Z3>_`g<{ zFF2ASsXc5mcd`61n6bj+Ad7ckrB)OaHq<=n46}>EQe4@ z5XFhK9r3i)6t>r;0a?szw4`BCl;%)|oVt4VQ`(2yW~-ub%BT5Qu>OAJH!97H^vG$& zZQ;T&Ndy`{0+mn&_r}T#M6!}Sq9^Yp96D*jKgcA=&&8Asa~Ww?7BFxfu_9!*lt`B_ zwh=}ULlQQug6LbL#sW0vK0lp&_tJx@)0}LDd`FIx%@e^^SdpB0!qCoJne5dQqn(zY zBkCA%&nBC4|u+Z>`HL;r0v*PS9MjU**>+kM;K(al{d(`LT zRD<&$uhu&RP>l>jF%d4HhSbqe>Zec)Nrc*FW|mGViC@At5|P!F?Zg7}CvP=E_P&?5 z#dmkVbLlMIM3*+h%MnoXl{RSUX!IS<&Rop<(z|J|Y7k%C)BB+{VX&}EdZZ7l7$AXQ zw5C~l1|&>BQU4D-0F<{pQkXl|#_|Ubx)V?fEF?pL#e^?83~!A1@9oo(4}0Irb};hk zCdi-U%VT|FLLfgf@O6g=F05m4^GQvbU6bs_JMb}0NcoAxPJuRZJcT50L&rMAEIFzx zTM+o8Jgd(lEtxRFEmpaEHAik4*`$|5HpbepT_2u4(Dyl30OAHhL{ob~o$y5-8 zn4a-m9Sxpx!Xrng*|!EMuI(XQ0=lxCWQ1cOwKQuVPsq+o$df)fVgJNINN;MXQtxe0 z5#lERVDBQ0_?6a3O>T7=>NMg~lGy)oP8Gr%$hpfOhvf#Xz8g4Yc*b9m&&(~gpb2;+ zfChC1fs-eohG8~|gh(^qrgTQa$Ye6Z9Uv^r+6_{#pWjPOLDY~w%Sk2O-35M^96v1; z;7B!86kt?Ph!32I3)5Ta@9#IpE+DNbZ_tLKJ%x+^Qp)|(+!e>whR)v*iM`Dm|CEG{ zO%FZ%{(hz4T?mF-CYRvtOJvmN3W?~M7_nSDRHH?Bu7nxH+DIBycdFPHOSb~oV*27=U}t?-H|-lqOd884!d#wL*)5-s|6YIU-7 zLLHBWAFXCSPBLSbMw^=_Jbyk@ib)&cE_4T)PAy;~D@=(vkoc04=$_f$yw=U(M~Gb7 z!F>UyWLXyR)wD$3+0!hn>r#2Q-bC)Pa>`lmBhW=_}%sbuJ~yC`Kw z*-pz1!>fx=8L;8M3&E%`BZD{q0;N|;W)mXlPVwBKJ;Cu<=%8g@5=OVIo&|1XkQrb^ zeGWo7K@);S8?;UT5{o#qnBhYQMqRnuc~k9aDC)ZPD|J;>Qyb1o|C-1{Kp4Uh=93cx<_;d)sYPPI|1g*ZBON7HsQ8-7%+O>OK(_ zKyJC)I^87StE;u-U3uUf2AA& zsyg6}BZUE_uxJf_43fI?gs2IuHX+r{$H_eSJMjb-J@RKD?T)|c5a*spggFF6;>_XY zY2=a`wff(2J*;okm$hy*H(%2YL5>vZ`#%rT2~GX^LJo5ZV46;2j@t z?{ywIGfK$uKlj0N!VdC~zCs5x4jQO-w=TIt(!E$17Mh$)J4o%&)64MOm3*^}t*j~W zx?AY|mzcO^wmSr9dd{->H3zmg=TgO)+Sk*ej*_h6%(%CR{NaNso@#~IyrV^OsJnnF z5gv{lmjVkg}j*4xoRpmrO42C9$406paQ81 z(9lS_gmAP9?amtif_({@ksA91Gh0pKnLvbbXyy+{QT^$T+-(7)Q{8E#D_NTmZ_ja8 z31Yp?9YiP=Ma@kVtE$gdph0%S+0Vl1R>u~32B(=XbSW8ZoKs2YV%b1Iz0S-e#>lN?l%%AA~>9Rj?(7t>n;P0Hq2 zMDjxf=4*Xb5Ur9UH81z|3)Y)+u^3)_PQ6vEpNXRZIkQAS6ak|5#$0)k~V+rQj88;o#q$w@}bRs;VI_h+OiDsCfBk^ z7}PVSWtHCl2}y{G^v{mHCGz3%vqXqHi`GQzF7)FfYnc{&;pYzb#}*#bm!xMgiWbqN z@-n8fNu}lz;^+1UrS1fmZIy6vqwFG+4~bpwD(>}9S;X>S zgwb$7y0|096HC!NrjS1tAr})p3FU7?{LWKliB}Tr@GOWkq)$^}bmn~47>fqC1&fwR zMYzxUd?nc_{gdrJb8|vi+uix_xv8;9_rq7?gtj*8xvXLCKC6&=nYBpzevZWlX7yYa z(3*2{_I;HdjOV5)H4iF9f;KycS~Xtoe#>;?sA8Zd$?9Jp?L}gil%yG>r0ngAkQwdc z2rQjWiU!_y69j!gSX(Qf#aY0;dC&fb@r0M`m-WSc=JCo8P$bR2Tv}Rcj0Xg5MJ1)e z@Vwe*_z{4TwQ;+Ix?e6B<%g=OAB9nsUv|gNo)L!VX<$I=Zft1yq1RZx_^jYzufPU7 z76@~1=)8tBD`A#i(ak5SPMbVtL(djSrQI<}e|Hq0|1l(QfPN_=LVbm3DHGY{%NdeE zO~Ol^A%{ZlZ`-e_F7K56RC5?=3DOo`A(VY_apa7RAu|KVz{Q?)4z4vA2aDp{wFBbt z1uVI1Dg1|lQ6>=UgVs!^%AUYQ#%q^iv8>iVG(3C1AZx0|M@(Duo#@*kW{BN5KN}NB z1xvTuEYv3oMrzevC>yjVPpZ7lD-cqfv*FLCK7kX^2%uX}hZ0k9sNz6;ax?@dSTw+3Tk-2R*(V|J=m1p-BDJ zYl9NA{VB`^mMw=q_p7{S^7MST7*H*pog-05`v6EB$F}%2>*Y=kzc4oCGNVx)1Zrh+ z7#$UPQw?i7b8rYZaiTjhe{rrJWMv?R(8kyMOIFkd3&xDPKN%^vt!2oPf%~8!f>e81 zM4HIUbQ6k%w!X0ec$PB4qJB#2>@Vh6^$PdyQIzx#w@#_Pl;FU@LPv%J$08lQe{^aC zgE`=*NJ0Y3Gt=Ru&;5Zz69llS0qoF^WH2&VX5%xdh4Qu|GA<+6-969m!)?hL)J7@j z{-`pODYBX`sO!OMsvQ}N+L?5#L^p7E_(2QsZ~UQdL&PG$ZZD7lh$3L;pfxxB`0?lJ zN?kAqjK$|0TnpqJpQ1u<(RQD*Q8&~tH>6s=X052141F1Bjy>9j+oCn7i^qMz>IIV9ckkW-A~paR668}eGW39sVQvn(x~M+J^MVY| z8%l-~7Nl9sG3cf#>nDV`fi;u5W{MrR^mRWO1)LPV2LDh9hvj&_jQkcDi zJ(<88N3Xio+3&pXwJ0m6UcI-VtPDewyZX=?FtXQwc7C2cpnbf~`~DY5%;Sw}N*e1i z(`#e#X};N7vveuYmRsLCa{dU{Io@GdV@bt8}LDRXr8@z{wMf_+ zw2nN!Lvsg1ttn5SNN}1>3_CRUT|2wE++g|D;JSIb7PYr0BeDdrTU;>z3!I%xI)1c2 z@O^N!$9z1*)5r%Z&d5kqqZCq5X%Y~mWMvs>YDyXQgSE(~*j%G(lVCPPS7NAw1}g;l zyMVWhX*ZiF?fZP~7mx4{cBI<-gNc%d>5Bdr*_IAIuSC6s5TwF%7>P-UpJxF}C~GU^ z7eFsz`7}eS#UYhtvBthjs$v>*ro~sQR@3}3@Ed+2eVlP!&1A@+u98x)-8L=6AMgzR zaSmJhJe_)D`vH9$P_u=-59vzHV2FA+M(Q@>G($_u{cUdPzHlrlf?s71TY7Crn!E>jl4}viABqGMV+w_6~E%+=$v-d`Ty%4J9$t zfDnVsRA#9aj-u@w09*{e+2{+qyOR!#Fbj0|+#2d%-2=H*N=js(KU>{ih&i@cBotO8 zAk62#5E1bQ>m5jD07nssu_jVvheVd=<_5qUWpYq{bL~uMPLPob@zl{FMBv|^)Y0Q6 zPXM$Z;M)qGRr6>`^rh2i+v9z|O?drZZh0FM)32{yp-)UZj>H{7OQ@i8lRBP|Kx`pY zwn-zwz>SpD5drJNIAUMcmV3yBO=#KVWYpD3aIJf>JNWW2N7X@c^5fqeb~57EpIr`D z0`o2pbA2y?23i3aV`^!=|0$B6lHviRq+nbDtOkRK)sp$Svz4;9C?QY&USC6CVN(2) z=zk=s(E!MS4aX_dkZ5|03jXBl8~^yLAmMTOg!S<{DJfq#Ud=39-jXFH1(`T10^{oP zkKjT)D?Rf2>dH#LL8`9}!=nO~wZ&4e*%i>Bbzm0(>Tk#%VA2A&)1-JxPL3n{nv2C3 zQ*P&AgpLYTAVF1wfi(#}cA6Ju@{=jEeOx9$bNlyX$dFd&1p$aBVzRwI&~tH#iQRo| zSO%3W;Eb`?)h~|?l40WnZEo4#rAp$?Y#K!nkv zx#(qbwGL;Cbx4pIV85<{jc(i!+Z$ck!i$))o#Qe?K7HbQ+N%p-T|!$bV3j@%X3So9 zAT|h4AXLN;8lUa?%m6Zi{tL-k!$f2_#45ITeDMyOCRq6^Vmuh6L$h35Ng!(KB=x9d@xbOpNtg33DkSUYj3^s3DO=rwlA_IK56Uc_rzb6j=_rKm!;raI$ zUTQEoWOPILKNOg%c10#7C8?L?6b30Mpds<&TC1p8Thp(arg8*19PSg~zsx0om3%YV zTWN5$0c-R+&be*l%d1BceRGjS}Rsw zdE?@!7yV$S!hYwzFpmu^Wl4x}Z0vA`bPKQlfEVgpo*8Skk*aY#CQ`cg4goBC9TO81 zAWi|o>de70pcZBaDWF?}o6{EDst;gTMC3-61?^{j6f?o%QV9$79G3x42Shh;`LN-97zXu#?8;UdAJTe_ znk!w*$R%<8-LWlc;+F$_4|RQm1~I$*8LtOvYA!gZ{v1)fk05}ZePi!212+cHnE>K9 zlSiJ_1}N};zR%yQt>K0w;3uR|gcJ%o&u?yT%S#(oQxOxF)>8tb?JH1*1LqMy1_Jge zmXwRuDWL08O~E513=^3{p#V~z-&a>=+~!zAB0=Ch1~xQ9B8f55z!W28KxBEAy-V?+ zo?EHr5Fa)Tr0PtWP;|mG)&p4uh2WdLGO>B#PXG*mpOy9LLF&&TesoJ<)5HQBsHN$vN^k?RO zoBsV4a8wG@*=TP6CLqLBe5U>x(Pe8@dT~MbGcho*56%nE#RM0LvNHR7e*{Q#Yb(_h z+iXu#LIRLwxOAF+T|E{>Jbmk-fQIk|p;IG2G~jA|55_X5C4Pk;m->W)f#IR#9NgNg zz8p)rSe#*<@VPlf!+zka@GdND{0J0efQN2xmU0bCP|CKQg*@52^hbo^;wF77f5&b? z9ls}v>x=UlTm?QWZ~v5Yt@BC;gox2{&1D8nfBA&>1ywVZ7}|1cbMaxWPYC66cl#XG z!d!#}zJ+pVZ|BwpBchV=-RHt(k`qL-7#HbJAr6;qv<~wm8?=rDnng(d_cZ|#H@lKX zeNC%)*Cb!HFv5}E<74(0sOP!cHe9R3qaPolViJ z-K;Kz<~mQ%W^EVShIrU*!!BkZx11e&l%iL7SvVg-dt>4sa<{Y+!5;tD@W@9CI6)E6 zf|$q`0t3$7laqX`tTC&=Fo@JGUOE9XPlekVlT%ILHLunOf!Bj*Moq2o*LaF@2F2E1(b^l26`j6sb1|UCd6{6GvfFo$4KmfRIFwDh; zkjYY$O73kR*4E|a7%G%rZa!la!-L|<&EJ@eK?5fQxQ96=ty(|I3xB_umox=sS19%G z*h|hX2k z2~mC`H$}0ZtY%rD|7>X@zr!eHP2z}&SBYw6;&u^^yjmKEi z`drv?YZ+bR1V2)rF&P<|1r&&)r0R2WC>nvG%)5t@rIK_2H05Mx2flslV1uj*R04^H ze^uaJn7Qdnvd!#=YfG}dq^*`(P0>%W_%e(6q)!dcQ(IDx!tG+%t!I={QTw)UbY zhLD)1fs#uqb=CAPoolEC7@lcF+FjOn(q2E6VZuv39C8fFxO7K{e-z`q*LJMt!ud>1 zHi)$Oq4njk4J6~`V7|CUUTJT2zL0!23VMdPegOBRfB_lgtIN^49N&9DzIC1b=|Pm#<8p>Y0y^z@iMDiWAlkSy-ZH>#wh;DLsZQ*%Eg?!T4@ zysj$S`2l#EJUrbg<|L-qFV`y)1EiNVH-AJB7|W&ghoF$_Wdh}qB`t&CZWtOaFi$b9 zn7+woDn~#-0QxfWggzR+Yvwy~Dk;e`!h@f~!`+E28ujOyFuC+`=*KS`K&&Vv`oXK^ za$d{PVPrvu4YnATQ6FA#Uu(;3)0$vE1?sVLdPWwom5x7ZuIQgz z(*>V&SP(ecpI0Y)MAIN$OwVeZh>d8fC@Mmtq`~hAZMHWy2x~2dL!CbZvj$+MsWsznlmZUIJWD0|RDPt|JoGUf zOq{P7CiU?-rvOFhXU|ZOP$=912=mIrRYR&IRM-q6)9JS=9MAOdV0Z20NU%sr$gq5Q z&TGxKTsMJubil^VBWld=gpZ?P+Fl}O!O}=vTpktm%kIl^rsKNl1s#LT>9>Xk?sWo_ z6ge`a(@nF%4Cz`Z*&|}7xHw0WoSnmE^;DPf1RCI>Saa2e%|M;$KUBCaNnP=CZtNor zl@Lb1Mo>Lmw?cKw8$HYdzNp1aUvqlsJ{H{YJtZZbTUc;vSQ-4r*$vo;0I~|%V+;yd zK7n!*yVgGCr;)gGv!u)`P#KGVWH@TMuHQKB6Hhd@ChlAZ8&6zJjD|R z7RS?(x^PDhCS_-|y`K%$V}s!plOLg|rpbR`ZHS6W?|S(rj97bCU-bZSoR?G!wNvwi zO%&X+lq};kWG^vWi%LRuu40YkQ7%j#R8{+5it^E@C`VQ!1rBFSFF?`66>|2?i^gf` zBpin(^-lmO?MctVtix}VeC9f}5VAcOm1QV-GoA!O01oel$q4ycg6@4ym%>T{qktYL zF|cu%N$qI<#k)+Z`Af-%>y3ZL3VK0a?f?qxrIPOS-ZiV@b);lU0Gkwf2Fuf_l|WBKi$rGF~Vg2%rF?bqvAZCiFJx4-X=k8Jxzaf~>3i0dW7@0f z+1cc`kKghoI1uGlz>RnW$8|~+zY}slKBAPkgU|Z>49d-)?t;8ePk(X1^+q*5%lVxb z4_SSBH~%m>1omaaLy7)j{f%r)7-+kn<(+(cMfIxfo5jB)8p~48NLTltl9%cE^XJ^$ zrzeKqv|@WfcN;Vt<$w0d3eHCaeC96CwDOTd>k<$V!@-txC?9!cWR#tkcZ7%hA?q66 zZ~uDmMkdK*UL0PO^W%44B2}^X0q*Z%^9O>-rl!DxC#Y*v_(c5Gctzz(?zG`=6008o zwH_!?(!5MKm8K3Z!F=2^o-sys0&qEBF^qXfcUzNRU*8;5MumspZAd;anDfn~=z2;Re_8Bh32-)f85Y#AsBNmF)l}LX98xi{N0(j6M^7(HtfJacW)=`}1G_ zehoa_C}|I_fokT9?(!eL)C+z8-hmZ5&}r%E*QNYnR*tFE{!O^0l9#2d&Iso-v_zN; z95GNa1o~Ux-GI|+aC8(2grvWJXPO1>0wP`9%-GW1>oNb6)<4^W4vh`^>;jrm`MJ4D zX>}GrGlUA*nZQ8~SVH~REHtbz#-w~)@zDK-^8AS8!^ceIe>6d#&oTqq`~m`z9$1-M ztXLd-wG~btiHwKuUSjDpJ36~}4LmBOc+AKNZxn4du>$iqqKCgWHT+C%Kqo8-L@z+8 z_&4B6b;L(v{u?Jx*NDzCa@{Fo-L?swhOO9O?$TtUxu(N{n1VtDZxZe%$@k`TU-#DZ zo4)R)#q{TXUiT$}@vtk3;u5a05|qGFfZ81Wx}zi}wtL_1kROi!b67=o19+Oge`65$ z>ER)0NaWkJ;|ua9|24^1SzeUS?tih$om6f%vpx+-uYYOl86qjmj0{FRj$9y;)zwu) zLxYBfW~eElvb3Op1{3t6JL~Hnn@ToIAeGhM*B3_q`~>vV-!~q=+YW0UJEy#UGOf7_ zEEf(A;@U~n)|@*>pn(hZ&U3co8Fay?kFg)t_iU9%F}L=t3N&$??lvb+870P_UN!7Fd~JgH-oOP`97VztA=ivKZs{ z19|V;Bl4p0T>`=X1PZUnBr=Kb?)oMsqYkf-Yier#GXgw@F*bQbaN3<5u3F*(ubgJ! z{Q%^>tOr8ABAlGZz;AwQYYVV*ylbDiH|jeK>T#YBJu{`It&a-$yYYq~r0);5lrYM_FYE6v&qCbcWDc&2S?HNU>p-L>V87EG-qPDzRhBQFl9-o5-%RKV}8pp{VM- z)A>m`$T(miCKWHQ{>G#I-d_32L+;OPR~}Mj{>@&#SGkQFZ13L{f%mRoCUgYR$L ze|W8Z`urKln2i>(Cq(jo>H1p+VOkrq{sW?cEG{M~sqD)}mQyG=+77OFL#kwgYZa1f zm(82f>Ap;{N1^V<#$tjI*`9Q0B{_^(2+ZU;73G6n|gDPXM$cLMbkC5fv|8HrKc2d#)r41v@c#Nz;~-96UWGEG#S# znbqcBORn~sUjY-uR2Z{3V%t0EfAYgIa(5^sN3XHy@Jw z{+QJ%9BLxQeJuKjr8CY^7GXn5ss?sb4Uyx`w`GSI2>i!JcX63=+Lo=$fiug?^eMmw za|29$GOyY1?kH)Ul_=CRKXL{5QKz?>TD56ur(jHGx&2Sp_f#IX!ignkbUo z@(<|cK#B}Xd6R7GQ}kuy-`VzM12Dj|{K4c!YvxSOECotQ+%}#rZPmLLAVHs~V9pn8 zgM;yJ60!eU3Aye%?j%>F=@UD$v#XB7$w%`^+wY9O$OS^ije#NDiDFaU?ozLQ@NJI( zyh)y+?N>owHs`aE)_^}i+8zX4>K?Y$FyLwa*{itb^f%Pb3rqr%U+0Hf@;FGMde#K*}*P-rEVp3qwS;)T^z!E>LP%-1^vC}Z*a z%Rv+0Ak-J>PfsJ(IYeTSXAEj83NakM)JJ6y@iST@u*{4jiJ)lkH;0|3{SQr76%bV$ zZD)X?yBnlKx_gk44rvex1*8N-I);)4k&tEp=@ulVQ>784I~Al;;(q@3-e(_>!#O+F zULlj-g~B8EaMIt{sbFKRPrC-(gen#IHvzNXu$sPc)VsfdpzCPl?ce*^U;f+SM<}NY zb8`;4qcRJ=9^2=~2`A@efBv|9`v+Pw{fA;HeV4@VY%=2XUI)Z!3BE#6@C}KBjWy9& z5_jt7&({Ckih;p1IPE&jxi~tOXV*_)i~RomJ9nl^@m#(yx%o8Np~1q$M711uABJSd z5$e9U?cF=feEH%1S4Fk>54taNpmPIuOlI^P>E0A2?5hOxohtlQH_^c&yRGou>R=CV zC6IHk zNxH=QFtqw@l9!f;XH7#OWQ}JQ11UXMPmfGt=R~b7C@!{VI2(UBX3oZ353%=I~E%&wFcaxKK02~>R;Lh6n z2pI-8r?s`U0F$np-=_cLC2fp?N@iqKm=u)ig$4{wz=<&Z4egME$pD3&>RO|wnnH-g z$J5X7%@}J7j~xg^MpSfbbCZmUDhe;Os|HkC8MeqYVx(@@@GuY9=ehn~UfOTWVH*qs zkIaxk-L9WM#n3ArXuMZtJP=KCBOq<}zdH6z+M_@YKM8xZ=}bU0?dDq@l*Q@NF)^Wc zzVi-(_bVl48fpfS!Wss{9MHu+qOVy(xC3mxWZ+4g zG>e@N8k_`jN&nEC;vGh>Qf<3H$MhqL(9+yXM(vN=M{&ly)o+a72U)d1I4ESmfFO^+$4Xp!@4iLp z@cmMnQp|X)&t!ChF|o-p&C{a96+K2&_L#3W3;i!#Q{wR!%j14*GVzG@_w*ZUAtb~$ z5i=|~{b8=ST5PL&v?T%(#Lr)6s;`Aa`d1kfVp2Y^*tWpLZSDL%aGffq zKWu%v{3Q9yG5*_MeJ`F1s}tb=nz!!Ohw-vKTw~|O$GLvM!BC#(80PqI&&)m%}7AKSGiOsOSAdt>cI%8Ad&O1k)Y1gqT!;e9tQlI9hLL_ zY=P4qx%<()mV5fyhWkT;xS;!spQ2ymAsJ8skjmEzUUKq)4!e5QYTPmk=e&;5eGf_b zQLf+Ox$kzquX1*1DG*a^b3XVE$BO$dX*157>i8tUI(5kdLbqPT!gBhmBQ5y$|JO-)q$pV|r*KUpU4jKC^VSZ@f# zAhPDREH87j4*90|b%vINQFlwrMxxV^UUtxieD+oD{xU*~{;M6Ss}O6rhT5eirwzxq z0aW7CD0dyqza_3;InY+(({Xz(9S^zvB2IEz%!MOsK0Y_#5y$6<+n=5xSF(EjBStwN zei*=rBYa^{gLOa@kB*Px$Di4tq&|be#y)^MW{k%ph8NfUZGomsj6pdmB|sh1xj(9w z%3L2BdBkxmgxw8?Emr>qCX^}}p+7E42 zY$P%U@~E1kK&qM(47X2%H2=JBaLk^}5MEbH+J~|*9(YQ~znm6>_7rmnn#QFr!KOoG zI;e1If{&p|lZ0u-$sm>-yHy*P<%I`}zYH_(rI)}vxNxN$_l_5^mVVgp+{{|?=yTY` zBRb(paiyVN{OM`m?ezp9i6JMfJnqePH}VHD>mT)UT~SAXDT;9UAw_v@ zP5vd;Pew^D@gF-e`1s$BX1akt3i^Lf3zQfTptxGG4z{!#;7|u-NY74BPkZmw?CON8 zPQ@yGEq80#f_L{($98hBwip*=yLbIcGo&S5r_m30Tw51`R7r`z0C%Db27f-uhp4C* z{&C8IkjR~hVj7>b_WSakj1IV7>&EIq95y=q+VqgmJOzvtUNn7sr6cip`Kfm=UD+|{|-2|&^wSpU* zc_`8=Y^Gq{$(Ar79&{z(CI7gjb;ZGjP0YSLMVjgn7G`_|o}sAn24U1ABh9#3J2rpR zU|C5a>pVxg>Kc<*vpdr)b9#D#6XMmW4f<$xK(^9$cl{d>^G0R=?g@=bpL~2w+E)_+ z#@~@)Ur*}#qzQDuD+B)>v%^QHcE54kW zg)h44{pc5~kf$ZAe%fmI^l8B5p;Z>uahUOwq$E=(ZB_HowN1_mY1C$MDedPdyfi9= z%KVXsGqpCTsHnBewXI1T-9+uz*M^(lh@8V}AR=N4Kl{a0Wc=!{SM)<*v zcYLwHVe2n|T8?`N(H_n4YDcE_B+JKi`VP^sT-28kT6^6iYaK1n=9w4U!Ke`4i z_;0aH)##DSw<*WyurXlr`XB*#ZnQhQ99iPvlSxZ!)PJ+=(v1^%YW!Gea$+#($L>cF zfj9|TOsZ|Vz%^9mF6}W@Eqq`XLhGNo#*`0FVSS=maQ@2yCF9hC53jLdLi-0WS4I`S zM%ZK7=5GvXw6PlZ@>r$y>-pDYgngJx&B(~l3rqBwUiCL?hP`)vFWxg$=Xk+!?k6&> z7^UA$!P{6#c4Nh^u7tE(m2Z)~U+lP@@5mN+d-T}Fad)KnVEJlo;(X=<;POTE5EV~K ziIVRCcjGbsR;PkWh&_S!kC4n+Rq7R-xg4-}Wo2n-XgtTy$<1w?b&3mq7K{P4M+%90 zHJi_4BhQLG7z&rsRF|`H>#-AF#K%ZDF#i}s`t7YcscB6Y1iY|I+W*+TlsQUHlI3Y; zf`qa*H=jCn_CV~QB_Oz}>871HHh;q7?_BYeuqc{B0h&&A-y>y_thyDJx3A!$xuL_u zOHMPatSzj1CP-b8&g^mQOhu|=jry12b*!K4J+f3)u(oqvd1kJ(xUQlYhZVd1F0Za0 za3=JU^KWcu;8ns^{808;du}s#QI|h1lg!1f>36>f&vzio`&VG?>|m;msaxhF80-v7 z^tdvTlBQHi(#A)q_6x1h;iowm>HztQ8xX>eIj3AI0%ImL5*&RVlB?fGgZo$=RBaWt z{!_1$0%BU4o{nMaNeiPt1}5Le2?d*w$7q5id~E`l8X6u84nk?zP{dmLth5`p)!(_!!p_b)HXqZk2`sJylvAp zGEFZzoWLr-(zzs^=|s`I^}6Hkta>z8j!(LU?9AEkS0g+996j#s=>5%TZqVJ;7R;W& zN1uY6d}MTVz2VR4m&}T@!6tWrwPzN3QR$P4Czul=3{^WC+2M7dpTx38szhv2S$hM* zxLwmhs91U@xsAZI@jLRYeT(t#c4hgrbw%k+7VSPhR|lu85-ir-P8`iG3M#!2DK z;nbw0b-}PM8d}=kI4^#|*K1O(_0>J})Uz`seZv*Zb7?tNhK3I#XqG@E6Ua3j8a308 zdD>e4fjD@49+3y2=S{CAA3LDHh2VPuOE1HPS+Y6mI3{4p?9vFJ8#`+22*9e3jYl8q zUz(kLiM9i*PqSfTEqTaBMMf4PH!vhB@ewZ1{3L^_?sVeUGMbt7x$u%pcu z+MqH*G@h+jQK5$#5=W*Vq?rN)$(5S9>ABiPE{AcFs&4s55^0(0F~;`ASjgbTX!f@e ztdd=xMuKef2wW~rwwQldP{r%g; zgFtqc7NyO*j?KBFuvhc#b$I>~AY5{^C}J2kwz3YaN69o$cYq)6bOQ=A+>G%ca$uaG zQd?8=MfHmzr*r8be!R3Q(dJU>FlQ1Hl8A^1a7a0H(ixeQ!a?R?sFP-|Kj%{Vg}E}m z>e&i-q5rspJS{QaqAV=$0^+TNB}~M|p(3B^b`#%Xh{98kD2j0*6x`YxJIwcR{4_{1 z8isvVthcl!UGiIEQh0G;q7t4^rJ*3nCzUr=58Q|iZ4muJX=WiW|7+)oaktv7R=*d2`Sah=G{4xV~?fNjzTpaj|VaQrXJK z`1RjOR1)!LX9w`1zkdI2)_hhETpmlMr2cHde^pdG9IBsnV&+z`h4iLj#rEF-59t+~ zXI$LeNzz<=d;=ubGbG_;efZU3Hn0o~HxiiCpKVj`fC-bIPF&Z@QaDejYy@i`^t|XPjzK)PCdZ&<^Xu^;{u5oBXAOuzBkY4Blb%XV0cV{J>#vbN6|FiL1 z2u+o|o2_k8e*UAAFbTn0!B$rfB}CxGwtWTccMN??+ZQX=VtoG*TL(V&_YtRXti)3X zZV^dTqK@AKun=`+iR{(K)G^({Bs8e3yv9!^sLbutLfH{+Y(;L;N2}>grn>ll*o{dx z*S9(^$Te2j|M2^h(fA$gpxT%X=?9RIhL|>?8Q`dO`d59wm<^Zc_rfvH&gkT&`E8&Q zn(2&Y@6v2a)wd?wBZ^Y>qrSt9_>XsU^K%alUi`md_fyq@;k;lioTgJLX*=~Iki7fb zJs4w_k@Fxc0*t3eO>pb@QGU4x&B#o&EtNbbw zGFB_-5#>1|G5M2Xi49?P_6g4$jTh^~V^@iT<&In)3M?!vaM%X^R!zg?xhBiL{+Xzm%hj;N_07U@a{Am7}#&O6^=8%{-gJEtMH>V4Cq~3Hg@Ek_eU6 z?4lOMP+>I=l#Z#%0R<)yptz_o{p}v zv=pR72}lY@;Ew^HsQo*Z(UV>sG_>sOT{+&#@Et*5e6Ev_;g50%b$O{SUUv5BNHEW`aZ-Shw|ME-9R^xtvqf_MnW)%rS(}^)M(4o> z&(2y&?i#)uQiD(dVeFTmxvT?(skVB^Jaxi5aqi0Sw#)c;41eBuZy(Qs=U}0st^LoW zdgcoRual39>zA?2DYBzOuGc8qa?>>E{k8``Pt(r+uU*{z$q{%th~B-#7bT8T91mq= zw7o5a%o*WFFcWis_mJ~(yd;y{{!n(yE7J1yt>bsJs1UZU_;2CNm~d_}$A$jR4<)25 zc)qA}A>z&7SKP;$lAfW3RyceFB(xb7#)s4L+p`qN?JfKGqb0137x!;|R6VatSD{6R z+S|14s0GX7U>m;a(Fq?9N8L^nur5NeKcCs^ioZlk1?T?GDy5mqg9Z+~;9ReVoSu(` z#xpEjo!(ZqQ^^0xq9GW|$4hd2r=cj*T=YQp4%hNLQ|#3@C`nX41qYOo(r}1|&_b6; zfCN!p%>GE_3hZo|H=h<7VbA8=sZ1IbUA_$Xt)I0Vu{=3O_WBnLg>)<@ATN$Sv1OkiXE&*Pm=&tUdB zHm9~*VFk)UHHirdQ2X_ay~;e(7jtg(O2gx(HN|X*Ew1R8Gx0|^QH&+(xCW+b+oG)* zE815dg7P{HAijP()j?&gL95#+V<=v7(Ac^%9*9sqZ8K$A?i zVIx-J?0jZyb52b?xG?iWQD0bkFTjf-73>b+7c+eP!^FTK<^B8jSy|?$rjo=|?}H|^ zY|u=a{8qL-I_{d+aW@{22V=zK+x1nap|geH2&m|CUW#*m#Tj#WF{FAF`(|(NhZT|I z+T#$x@i7AM#k%h}$A%|^mB9`(#Cs95Io-MQS| zfFtkEEXa=NVc6aIR7AtI_#>#ju1+nWBl`kC0J!HfoIH+CeN0-|sbKYjc=eKJg+-La z$UupB8sw2GtZV_^?aj>%&_L|%?Ex>vEBl?Oo+-}YH1^i|bfSRkoS;z%{!hwaWfiQN z&L+5?zYR6z@E@w3I7?~`FhEs2q~?!E7%pP=lMi<38(^1vhM~jAb}aV?{rjuc>cJ)1kay%z(sopzLE56hJT0cexOTXUDwu^n9)! z4{Fv+D3C*2uCz=VmYVtz3cW@4Q{k@SJ7Z*I|UQ8!UJhr+@$`o^a+?0fhdvbeNF3V@+c-U_<#f6tAI`_0{itt93tGO zmF49$3=9|op*cx_v;AK#5j72sA8=1$V`G}d8#_r+J{UdmnJfQF+V&^OlUja|L>A?i zhVDVPo(e`VCqL;-@kAubfEzy#d7D|c3VPbiwDLyu71B=hx1iwhrTUoA1>NDW+uBry z@o*YFoK?@N19Pm{?REbP&t}6YeI3l`N9FtsL0Qr7U*cwbtD0ehMoM@~r@Bw8R((MI zn)$1bsMD=egH|u)F$^t%ZdJrBEB)AQ3(_jUU>`qRD_b2}_%=D2#wkKMVzuK|Y(MD! z)&AA?=lin`d~L00n6K0hSP12IE`uSU0N{^>vhJ@N9`8_=ePpZ-7Py|Z%LQ?RL*3mW zv_gp-D1{%xx_F&j4f-V-Mj;`yUL>$sJGC z0%SDk>0WJ*&CfvGC9`ket0=NyKa*zS|G`_y{*pC7* zI<7Sa?B|o?V;}%z!h+!;azxGVep!rXYD;9M0?X%fsD1p~TC%C1+jzN+B)`ZIu$Oi5gZ_PRW^=B;xkf z`F-@6&=1ZO5)=e?jG}xjl$((%1IX~wGX_$}^M}9u)?fE)r^Y8IA3nN_@t}A0!tW)y{(Y^X zBIC;RbfJav(vn*nu*%1zA;%|ewKE~*)d}X}$|{_x={-bNOk@xbr1+oc;vViJkA5!1 z-N$U)(H?^qqB&hfwqM}}dCEma^_YV6{bu%4GsZJ!dFnk^nunFd*BJR$38ah0~{Bo5MK<3!0vYca{;J5bSInVdCvmCthAgvZ%P205InE>P9VEJvH&u9zWMpiOAWu))WzY zVu}YMsx>?t$h&6bU}G+gtql#Hqd795up5?(fIq&KQm9q%)e$lQfuSxOfB1WKhQ9vF zN5)Ph;<$61zopBv==!Upbp+{oo)<)P^XgmLFZY+byQEWS>a~NES@V&xGlXWVp;#Rd zx{s`C#u2Z5%LB1reGlj6Zki+iy`HdD89V(gx-88%?&5ShlmLMjui$)T;R4lC+YfeY@3U=K0#)ob+8m#zK4v zpG0k++h@Hb8g#5nY@@n;mrN?d$B2)m)@ew~G`1tvbyX{(4Ss^54*ZVYIDHvC4MI&A zrBsI6tx6Jy67>m3YA!dc!j7uF9U8dI@jQS`O(=|_l2%;)pLZMrOtOTig`eUImj7&2ZqMc68GqMo*MSEGdJ$ptdNt2Nltn? zrbNN-Q1M3z9Yg1Xm|t>DfTc5F?8zI>e0DUFp=nVVUsRP@N9x9$2s7l)+a%k-8pHVv zeSAVOna}_9#iMrUQy)&~Ed~-e=sRIlI5~Yp#R+k8p0b<%G3Kvqkpg+oQz4CaX+weE z0g3a~&9L@3v{sE`02tX-kAc%G#3#-XM*K?O)v@WKP&QG;{hM6(c2yF%4ay?AtXb#S z-boK+Sk!y3s4qWMkgW+kn^2MBLEG+rro&n+q??|V=4Mxjp&NnX=iu^_K5J%px$n61 z*n|AskG#wnTg?Rwy0@s_+3LP`ER&I|kXRw{`pErKw$s)JA^qv$;QX|lx zg}@9%)62NBr*zmL6v9zUFbL@$<_xTGmE$M_B7TF$8G@qeKa@k<1SE6VUi99r{n{KodA-Wz`>QAx#} zSCls8mho7~h=|fsuFrV!^lA!!dr<7BocUc|{Ni5oP|}9r=|nsQ<>}Y|_VBJ}F2n!I ze~bqC7Ctrw-H!s)Q*#rOE6_y0f?^yU9U1$Rl#zxK4S^adEJi{jA_4*o^SB3-EMLCl z1mIT!lt&QJE~g?C8c!XIj)aCY+HQ%SS*_}b)AoKqJbuw6Qi)|7hl_L@s+B0~XL@HX zFAqn&Jrsi`(@@)sJj+Y$>kIqK`0_ZhTN_;e?XA^$N>{+c8%3m>n3Q``{T5QC zxqb`?&JsmC;or_AJP?NJwp(~42$w$SYW`>p=pRIHdxaW%-R|V#Q^Ye2M zkOVLlpcVm#IxxM)>$v8;8Q5hXXXL#a-J9!kKMri4*ueogst=kwugR_nR8^M*UTYQq z#Ku9OrlBy*cG*}z5-Z>;vysJzAyd``%{;q4tjLYkxi9ux5ul7g6zkE3e*OJ<$RRUD zwcSZq5%JT5`LkGk321 z>EpQWkf1JkK|DSlh}*oWn89kQd1suepg=iJ5)r=0ZY*t>z*CyG6-tYVl=l`-OG95l zQTFqV9O=qPLr3JdhG2DaU=Khs5I zD%TNV7C%QBOJK`OLcIKwWJy$}j8iMv?&HLo5#08K)w1Y-@e!>mEraL$;MZ~WmDB^f%4T^9;ywDVe~V%cVBIAi%u62LWw7+}yvx-N&o2sp^P70I00~MLV(Qc&{~Y z=d^n4zF(da4gCMz;8ra;f$B2qF<2WCD-B%^oXIy8i(7`=4J{vJ2 z{R{up#G?IMC`?YiWQq5NR-R_M7lY&ro8*Ij9paTo4cs>EQ)N#q6;R3Ib1N&u-tXnb zE2Vu^3f7E?Ijg(@J~W|Rxt32bkY#Fo7#gF&^0D(}H=GV3Q0mV%Bn1XE>Rr8QJzWAn z+#pOR8vf<9G8}OVt}8=FnYYh94do2&X-rGrJ}4VOFFTA&s*LV#H2oz8!_{)fpA2RW zHC)x8x8QDAorc-BF5cZ91Tj3FJP5pK1y1oh%n8y!8XcLtGFMt!DsB+%c%>qDO=NT( z%5Z}KAW87JLw31QHOkIU=%zxVA97;x883u~7-oP;Pxt-%ccAm(Yro8=yI<{`_Eg(H#pOqv*IBpR}}} zGyE~)k161$Hqi2qkLc>@mDoTFqO2ONMVxd5T!9hCVNLaU7!EjYXoE1CZ{O53HQ$6& zmH_yqgMNC?!~*B^6Zlppwbr>h+r>9$=5ZYyn3%acm{CDTbNG8$L{xbqo`@`~=-o1yPztOjJ@>_`=c>0rbR^z%z2e!Nu{7QlW@*`k2*~%m?Pd7@!9_;ZLt z4#Kg7mL>uO7HQuSqqg)>FVGs;*$}J_CNl4FqXZoIE&Kf{E!L-Lf}ax72i>@x4(YO; zb}}pc&jQEAiMwZ z&7m9K1XIEb$zJ)Se7dSi%rVBfG0iWrgO+M=F5k-7AIoPuK$j?-;Ny+jyl{-jz34hzbK9fqpnp867mtx1WvzH*A z--{WV|L(Se8AVqdpr>LLPRYmR;o-2Tkq2{FT%1R5S#UninetR%QUyQ1h%7#W(4TBr zzjSu)3w55h5#QWcdXN$xbCF%6AgMyquLU-ybXZm&&|S}>W9)zzRZ4jV=4ideUHv70 zt0o|$O%^^?REX3Y^0~jgm=(qSu{9nY+lN?*kH;_2oZIN``r_#E9evIkzh%q;{~Q*# z@dkdFW;i&4@~MEp@}UmZimpiJ1erj=ZnYMU((oU zw5A->Q2y)pZd;Hm)FURQC%&1876~j4F7{^EJzH^baD=3HC4oKsVc2VN+6O6Ts#1Mr zLf>T6wMO?k+(eTWBmh8|-|Ff=!F&JNo(5Hzqy`H~Vu)j2zV%PYQZx*0@0U1e|I15& z*X!Zw+3?6<8eHfi7y@_Pn`7NXYXm7TT9G>a;DhhPaXuFHEjmB!((_y2ZXh@~}!_RxuvSsWE`2KvY zsFiPy*7^Xt*>O6do`w2|WlqcC)z6-MnC>$MrISsh(@hj^(ciy+4T#5pC=z5O2abt) zqr3-i!K^~U!+o5wi|xU1n+jF0)ytu(R@b4}cQ^yL%{vKGRZmalZ6wSyt~a)yg!C5{ zTF%GE`^GIZGcScbY%i-gMK|g zr66*~JhpBxI5Avb`#YW|*;3&MpELNkqP6?oiVaK#(fL|{vK6iG-DJVR=yk4Tu0=)sog|MYK?>c5czOLyR8*I{4z^S+k9b|LB!aAagh;MH(N^#v!>5<_28JHba6FnI0o!%K7{%@P zAIyaQeR-rPGNF0$6JDsYvmhe9FhLx8JRB4y;i4_f}}_jk__iHmP8Q zS;)FzfDhY!NO2yH<+!LO@1129?;M?G``5b3sPpntoW9--ZRfV8aAe+RL^th;*l&aCr2~r)mG3!RHmTHq>Mq!vZJ$FM?LY$O65ajHtk(Yr$>bC- z@hvR2e`9MRYeMfAKgVA$yJD@c?U1d|eT1Gib*ck52JjXx!UlvWEIxSdjUYT=OHy`GqN?8FG!<^iA(s(b6IXk_tvR za|sD0n0J9iLhTQi#`40ZcLLkMDmW=AiNNRiLh3BV5FS#j#kGXk-{*roi1V1(*#r%i zKnU$*3+$pIsrO)B z*Bqpz)A!Ok=v1OtO^&`s~i)4tPBVcT_mg0KD?XIHj>3V{@- z!LSFo|45e-zqKK$*(1@G4ptK&39FQrsp+;!kTJBd`18SQ+`EAT$f_vtyf;&f#5aVn z2PO#QZU_ryz$XCA>+B)aS2Zfe!90J6P)`?leugFy08Z5Ve})l<&bEgDWGl-Oc;8MVJ1f2q$sO)R)J&KB;v#hQtv=3ifJ|Uqs-Y zS65qjPo9XZ)5VAdBP!|WTHa*QRn%g7mw7_l@V9J*m3^WRCc2DiWE!}}c7leGIA%m4 z#C|gh$|$N#G9Y}8B8-bLeB>7b5gt)UE>(jEAzBq?o@DjR^zIO~qeAB>Jxs#8A%h`~ zUD^F_;HkKu*cSZnuC{#?-qnBQUI#&;NC#WD3RFF@>*z0F$SYEV{sAmRW!byg(ZIJ? z#u!R%VBRSiAr+>E0izI^(=i3}!GZ+YZ>K{jdas{LT6o-Cx(&%c7}{xv<$o>;PH6WV zb|=uktL)JDS2MB?S?HaSlyuA@T{4nLu$EwoMGkjpKN*i;B5{_2ilpJ7V!cDlP*rE1 z=oag^PJjbvfyKsG&uDqZUd`6WY$gj8isc$oJ``zoRs1`Mg$?LaEd=P~R&zTK@oP?S z#`fCYr4%MPMuJ-rVp4=lx%Wlf@OW+4w@YRA0Ze53zR|r@s~J*Q*gG&sJgrr)gbv}j z0P{Q$-4`+J@z<#ic~gLcp))TYLdYK{Ba%8h-SDf%{;hsuAe36D_Mo45?4{MDh$qyb z;Fj^FXzGD6LX@d`xt=XJe}&Vp@+)fZ&umHW!Uz5N_!g1RwEuK?6EmTklGe7Tw_iK| zz1VVT{x?vhMaxKP+3Bzyb^N&_T2coQMqZ!x$o8M}KG7u#_LZ9;i3WDrr(k4GqwWY~`MDknM+ z>Y=#kCguuraZ*leYMOjcPXi+^KK4+7YdQPaqhZunkwN4;vtuAr34>55vrKKa!FkS+ z(AiBOz1-kZK8Zm1+SHy$>IR9|4`*#6u=)U<{dTWdV7y|1Fyt<9sG|715?MMP=F~iZ zH9J?9n+l$U6Am0uR_8VjYYOUTPA0>@cYtF@)EaU(P;Hp-M}ZAsijZC`vO@>U?nDj)eF1_0{e^4+P7i`z@cT=M%jYNN{dwZvqYl z(tuKGjGo}9O|YU{GCX3X`W_!#pufI5}I%zLdLJf&7EP<6ozf`J{uF%Ow-@&Tc#u6RSfb;vg09a(Q z1pjYUi$QxRrT_*z!8y^F7*Fc~T{g9T;fu|a~DxK@vwgNWvrq{6Csw;GjePAI*yqC0TQylHHs3{7V49{29r{2 zkh%Ii31wwv{p06}gB&`K4;U!Y_xUWBFujV=a(70liglD3CG*tH02U)5bS`0mqI2%m1Et&3My|esbdx@>} z!?iD)Ht}4pI~qdcvpw1cbWb6YkvTzkJ{@;mQY!x~IQJs}lT>sg$r%HeE4tme;_NOX zrXK@(MNjcZCV>D-XGewDpGD8jI3ckSA%Ch<*Dd1=(p z*XHKhhfg4;^}7Fu!e4KdJO20o=%c4&e-i5F0aoqdv{r8gv}6pK#>!ZFQ&Us5$B!u& zoH!9mG{rgezNqb6l6ZKh4U1`|E|buDU5Nc>6i5qC6nmFHQAEC#%7*td$(<@BxyZv; zlVx>D7UsPT?6Q(AobQvsMN{K;_~`VRoHPfRBr0PT>gM2BJ|5yy)8-ezaUJYlznA*~ zBD%NdbVkCGotnDk*(yKrkG`|Nzed^CGqv2|>9LVWbi6QA7vU5(&OZsj7G8@lP4o~M zz4fQ7OQb&FR*5o}-`OH`wHo?l`Tlz)6?nt#+fL8;!!rY;OiWCib{^oy zjMxLS#KGtf`Ja?WM@J9lKI5YspZxkYVJmVM(z!}*oj+c)QoZ~!v`&UELz}3UR zun!2eZ_J}96BW%g>*~rsbzNbPImOaj=n-Ev)q=F|w$@guct~9NL8$fqrXu!2{O=IuGC8k%wx0uu0DY}U5)w#S! zh39^>bZquw0cHJVd=jrhl1oD36jd&tOtvz=aMkQOR`Qyk+5xnJ>Qu6+sL1dI>MUU? zy_*7RUx1M3oAFWIuH#2PH$MKMaEZ={Aiy`m4Rs1*UDw3?oAyCva|XP!x%EpnZ%N9jcS=I0sobhsq=5jKkz;{^o;c-|ZD zM;(K}>bF%JxxKjw^f4eS)s}#1b88FW^xr$$CX>@Bhq!G5bMjriN9eG7iA3`&ylU~|w|<@}@^d2gd%u;j#*mHPDyzmM+$kM+=P(%to3;0YYs)4NeZ6Hz z=0jfHS!&~8@9c+t|D4$R^(pLgW)`Y{9|t@*-{!7C10d+%WsJ`nFD|e_rKwG-3+?_^ z*7W8{!1HPB{3uaDCDc|_KaYhbw}OU-bI_5rieEqV>e~gG)Ov|~02p)JoR1S ze^U0vwFOyT5}&k2EdL;NXm(Qx3?$1fsfR9H*>dqxgqG4 zi>=zfA*`<;g$DpOSp{8qQmEqhJDXg6vuIVH2c1EdC;KrwFJfzU6>_|p`V?l!OM`6qR*(-+0YKbV?;-uTw?Cwdtw_DsAdtHi!3=5 zMhgm+D!Y6U*o}?;6N_GwQfup29{+QO@I-(_6Q$zcQ*?H|zT>hANJ%78x1otKg_wHW z%jFn1-I)C=I1bqwtQhvgLV9C^V+4=tx@%Q1WQ821euL2CN*Yx!4-a5B2P6fJzBx9a zvUR=~6JkS*kmN)Jfgvw2P&V{)7UsM$vXk(3DOd`T8TmRfHX-5X)KuR%N&u5mtGW7? z-NQB!wH$qhsexY{!Sl)YqVM9bv`=Eo(4tVRf{Pv4O2%y`<;hiIH5*oR`SUqRIEsI6 zktLJBI!Pl-zi7hSXv=A0VtHvxVE33!sl&gp!Q~JOp1v~9zZu+fqM}@)%5}>kD$wFu zlbBOVfsFM{AE}KJT(S}4p{MX5EkC5o#_k&iGj9$4%w~%Gp*@xs%Lvs73i)em*L4?i zip;jRuAREcg8QgPHZ;PIA-)m6|MWI*j;MembAHJiX?cdE-Z6jKgplddV(}sh1%R+F+19TAIiyzUm>fY@Ah^Jc{>s#$Uj23rM z0Rv7_J39W^@h5UicbMi;a}u zFL`Vbjm{y&BMR*eHJK9Gr$CN#L8_R%Kg5vdZJINBlE}GHTZ+_cayG%JHLV#u?p7cn zIpa>k?KSVVO2laT7ki;jX1DkWnZQrqx_lx^%3Q$G!C+8(dDb=y~ z@jhA{F6`1N&4r`LWiDhM+q^=TBrtdn{kuNRIQ<(qK^f`m)5`dIxVZ^@I=2NX6CVIT z^N`~dDY(@D_56BmsHIE!RM{4w;scrvrH^Sn4~$j-y-WxzvY_Jq4U|GuMClO^=x5?> zHq&(?yOUZiEGA83NV5Vqgf07eB-He&22xvxE}q7c^hcz6*e(ge{$^-|7;|f*BI70X z+sLJuBN%~qBF16f^CvHZAsNMWpGs7})Ym7&7n&;bJ%v-|OP{0Ivy!6pzvytj`Dv`L zUtcy9h2`)idd&LaH$YI}6kjq%@v}yn>FGUEm!By4QJ*+BO{XMil+ePiB6aRUes zd0sQF&#kS2Jj~~DbW~K(<6}q{AP)iB5X5cAM0d|qDsBD{jfW?i@nNccIp~|%It8Do zkL?0;vZ$CC$1hQYC9A60o}5ZrL~A{)`uzQKIUiwl54;IwoQtDZ-ttrL=Wum2mf^_K zw^(Z{hYyBFpKS&s)@}1;X#H0T*8Hl%_h%hG(zQyWLG3+(RPDnDQPKBik;IrOeqCRv zEhSm|e#aAIBF8C72OBJ$Fk1}Jy>HMe)`j6Tls<8BVo+flu`uR@x13f9t`4gVy4hlH z{pkOec#;e`FQ}b}*Oyp&I#DNx%e4M`b+&Aa+5^t=UbVNdf$>d=T{?|UNtg{w`$cEH zROo%J-;J$5@j#p=Xz=U;@V z<{*3@T)vBn*or4j%*-;gvZOap?5TlrhDrayEOw&|JTysv18<$3Iw2=bhTw*cBFnzO zW&cxVs`Z+6^RDw~#!HIOL3?M|6d46c+v;_vu-~5$pk(9X&n*^72!hoRFL!FvKDR8v zLeA1qvW8OQVB5o#wa;%?P;-(Bn3>6g1Ki}-*Mv}njnI-e+7wW0s8jRx=;sDUW;xK{ zqjZp=jhK1sJQ3X(a1tgIn(JvK+2hLyo6dLDcrAy zQaf8Z${3>e+9;5Qlkq2odRxMTGv{M%x?4?5cK0o7T#b|E>81(``!3xH5#;hgenAJ% zFmg%$jWl_GUvHRw9kpSz+`PR}B1muVoHf2qjq}ew(5!qqr^6bjgZpb-KvWda<*wFZ z_&{(Y8lQGErASA~Jld(P6H!bukP3PQ(o_(O5Ht^{S%53PuUhfcY^zdwLu=+X|BpLo% zA1eAH;d#MJ~4B%GzRDt*SD$WvLH}qCXBQ^}SYQhB+&|MNsHWA0_*=v4A0tAwnf~7FY$cti9_w-OjAn($ zrKLw1ouIf2{K)=Fz>wvC$=7HrT7$>?!4Xi$z<{w|7_(_{U?*KXYFs~#z6kM@4qo70 zu6eKNJA4%0IKaXF-ISgVL<0N|OIHugrzWT$^c3?vEIx$&MgT^%p`-d`OOao@Rw@O9@-21M$^HCpUWk}^z) zRH!o<2W~Er*$VLyt_Lp9Y(|?#b*f!Nl`-UO%b*31roNfAyT%CH=%=r8^}G7B5a|eu z7l6l-Hq^-H<^)R$Owu#DhqvE@K~0OBO9HyOockL+w;Tx+dK93)5SZa#Hu5cR_!6sR zxP6Gi5WXSQeapVUZS=v&$S5!3P)ZeMFuEfZC;?^OPgHjjeifvmf*#jZQ{xd35D*qF zeJGP&B+A_1*NFbyrA06ji71RY>z#%pAf7QqXV0W7D;Yl>`&gLX^sDMS2d#c-w>~jL zU1PFh~*8Fqp)zF%Mt^DG^Gw%>>XmX7Ju}csf?#jq~~BN zse!D!evZ=RL3p>L#)%lws;M+y$~`MJyNOlHU!hk__tdNYmPWA+2(?2oDS^q`Rn||h z$e!$nZ$P&t-gJN_!pZp!Xa?}UV|`v+FU zX#&o@kZ2%KMVtooW3a_Y9E$(>b7>tJ7ffh+RwuY%m3)v2y4O|rq)Y=NGpHKv#_HSFwc z=s#RwX{XQ=$m-yTtcX>AM;{hZ!1YM*CGQ+MRu&p)A^=paKM}1($HO3GWu6YsS=r;k1w%HER><$5y+d>)bQ^$`Mg5t8vojT<;LWLlhNUS zn+z!-#*m~e3TP7~VOj_2O&+&ro8Sng>1~LcP3rJQM00(TCnR^t^7|>MUQ#Hh6aIk) z=XL_nV}=F?yV14bzMRk0+8ca<*qK{BmVlS$sfW!6y8xo`GEnR*Ai~Fpm{9daIT+lt zj!vzUEeZ6!~R ziY2cW2%!GXNGmmwVZdZ;m_|fkIeMS`NB;KrfWBCQ!{%a@JQ~cN+rwq#RO!c#liTHM zyB(o?Iza^I=D!{uG#~p0$3q}h379wZ?xKMPttTaY@f(t+ee@*X%8Ur9JQX2IKkgk@ zEZ+(JA>6F^s>b*5Ec;l?kLe8Rna;v1_7yx{4>8{*JbxQqZsA-P06#IXnZc!3VrFJe zA^-`ibgy3*8j@r#f;;#2*3*v%i3lqC7<^j?8cRhbB{we^#Dj&}$lWEo`cx;s$U1a! zkT}yNC9bBXx>?p9?gi}NJ)m7cEcpow&Rd|y6|gG?G?0MU?B=P%PW=FkDMq}eQ29s%j9!IGlY3J8Bla!1`b$FEYdj{Fu1xca4ESE`cj zJPa~b94xmjUo=?&1oX}^9WqXJq)$|6!R7JDX0gaj@zAA zQs!*`FG{?r^Q5wV4H6pWE0$Egw(D@bv!nL!Pn%MIMbWJg#Y^x5!bAl9Rsh%eA!H7o zb{mXarlz_1`CA(s5DH4nuNU3#Dzy|})6vP)`HEYJSN9l02hh{fUfz!2P%T`*;NF{$ ztB{brzOOmu5>jy6>F>mUsX0LO6D6HHI4o)%pab?F8 z;F)`TatQeWr0N$JE&~%RP`2$zgQ7{v^82}YaS9jtP&*wT_AF9@(u*0#M;zkAs}mEl z&sGTzxrHuX-Xd-7yHK%*w~7-&TA;D3_+0s>JjCLsC9uZ*a(`Gqx7tkr&h)3uKU2h} zEWLc9&bSwz0u%SbPm^zxN|J6^NFTcq3m+{l3Trb#^F*9vHyqw`%lyD zd(WK|$`>}oC`T?4kuN~f{&7n}HxC#G5=5E-!+el#3Ca@-i!Csd2zH+Bm#?8=Vp14I zf`+JDm&{p$)e=sT6l~`D`Z{>t9FU7mZLwrgN9mOh#swC-QW^8RoLVBZc<`o0kEKM4Hexc z)L$J7DgDYeayA>-s&sdkkehLH|1d@^T3ffxVYrawQdv^M1imP6(W9rgFg6ZgTLA9{ z7$J~jQBYAgUH%A=5KjbLBw1PWn0UkwzB`NfmkWgs=?xCIJ1iNv_`*38n;M!=OJ{c& z7Gqp6BC(bex*L1u3V{cfJEYr_ZBP4ctzgaQ6UzQ_K^Bf= zMcQb*iASa<0iw?_H*=J2VJ-;;Ni_0DY0Sm|zaIb5pWd3Dwri))3GXox0`h2htP>Hu z5aJLocgBT}N3IVSLz~+S#(?9yZj6kEmev4*ctn&APkc5%^G1sz`lu4l_EfTigN@vWLuyqrSCHg;Sf>Df zJNUV}ub@(;H?3df<4+6ELPEBfqT$8?woATi#P7s_o=`scBfoSaYWHSk`*X0hhjt*2 z3@OA6%H1#)!%qPaP(fUhcb7CFK0af`4rqP1NQdDBF@a-!bhLkE3GKz45eTx^1Sryj(}K|#MQw>Z|{aZVfb1hp!`;} z-dX7!;!xCjj4%JgxW<5nObA{H#q*FTZ&LYQaS0a}bYiX{oTL$-irzqX*hGU?F(1PF z22%rDl1Z<4HfJT-#Xf?1J$+QO=TWrD=Mf}t6-*_^#i{D+gBB|$ytlU(v`M+SxdZrI zwJPdJZ5M-BHM94UoBE(nlStn1Li|6Xef8NU*bdYQOWy(A<|8M$%xmd=0@b( zNa<|X+3z64#P|8hL-xC-{c7)8{qlcAQ}0hDr%$gw`Rd1WU}YwkP3)|!6;5U|u7AYt?C9ju2;=S0iOl9$D;p0UB-_^lu)Y{c_-!PAY_^Z5U~7bS)0k%iNl~ znEg&O@SCznHUc`Tr;o|F?NrR%3;Wo)c_}!a4r{I1!K%2z{ zsv8jixP0Q0$|9H3Zq6aSw%bhrH!_?7e7MiDW34+r#U@bs{l zp=e8T*BIb>ej%f;JcY!!>IF{wt!(DxJiO(Y9}+`N;PtezIRFCB{2xp}?T|WRZgP$G z^XKQsE#+a?|RiDj1k{K$g(~s=>A;%=8}Q_AeEQ(r%|<~r24CN&l)&Zit_TwX=y!lZq0g15w;+Lo!-;&q@YSw zLAl`5zloi8e^@@dr7%=5Lc4m8!}n~44`C(7Dtr7)r(h8?c=_ZpZ&+CY`}}&*zU|OD zbN;OJqidOW`^y3k3feZb#3e-E0h=$-E?NWcRg&z}-!!!KcU|f`3otNHgAG!u`Df;C zc$O_pW8NyYDEY8S80m&&dC;56`Q^zGOREl`|MG=E%MeaWbxp&-Ku{YkiplPN39h*xtbW$6GmJsh%}`5con&CkHzyrjHb)cdA#z}znxtmS(%fSLfx8WaO? z8)RfGFDwARYg;r|#GCYQQ3H!GnI^114NWFOiF=e#=}moW+J-va|{*vV)_52x)dZNXVD_LzlEY^A(_qt5nPP)u)vN;fnwf< zw`b~GkeLFkivcC=-PL1MjEtt9g2GUHyLe4>FFxQ#Da&GOplq#o4IrI*S|mR7P*2=h zdwc%|a=6b&iz<2Uh5wO&;>&rbxJZ{1t#=2a?b%t=H?5$i=DIGJEn-oxTze;R8MzNE zM^;1HVZ_^|NTtKKhy;H+No%EX_;=pgUOJ6qAs3=9W?x6``<7`6o0yoiX2L?utzAG= z_Pcix92hR(^6&4L12W4VV|fLIb?LOUwE6jYrvMxCtibO{L=B6ZlXv%z-qbNr6taH3 z!6))U(HGLo_!&i_d#1gNN4avZQQp)qg4guukN_oxhwp9dirF=RZ7Z0ZzYp-pC9FJ& z+?|NFyT~>rI-aW7D`fAZ+eF_U?4xZ+vt3ENEmU)WaTr;rHa>U?7=j+Z&My>H>Ct|h zY59dBz3^Aq{S>g2_UWfFxr+}EM@}3H7NTJgSZ2pVT|!ImeVCuu`s(84C0KyicZXAI zg1V#(yR5RkJs@TKAayI#aippXwQyV%bCew`Bg-QX{Tio72SESK=`@SZ%cI-U-+VN| z+o^qD^@X}8vcM*K3#r!QBrx~%plwo447#~q!iO-%>5ne@7H@AedC>&yzF)C{Qm*B6 zOmc_qq5E(Tx|rR+gymWw{u`mT@rY zobjS3sG4?pe$#xWiK~@%>pJ#1_@aYbuE{s{SvAmlPXTd0c0Ty-yIX+yFmxpBNTag1 zeT3D+pRZ`}KhQD}xk-xzxnnWnE3t6FngcKa0m=nd>|i4VjuJRhz$Q*wS{hiV(a_KU zlUQM?;awLJ6Zfz>2658hqYrlFykeAUdLEf;RV0Bj_cSI=WG8Qr@kA6a=qDU6ufH-e zP6)HfNxgI@U|{^CdyuN);k8Ef+1iDvP7mN&(%t!-TiS-KM2S-_%9ox6H2Y6$d+?H( zC&-wn7h4sBBOw_-{mnY0TPghx>TOJKMNK%ip%4Wz6Fyj4dq%F=d7P|tfQ}*nrsXrc zu<+8qf8{o4;N=9{bK$ha#J+!%uXcf~3gjrr>om+jYv-JlZ$ObpBOoBW3_|eZKuqli z?L8j2DgNC~%&0`*b%xyQh!S9rW*0SD^Ot);JXYxi|JQGtEbN0el*NeY@wZfBLINvu z%+Hk-Ox`ACD9%ItwE3z9hg`1mPr0 z{9C3Txu@Br=Br)EEYiku+Q(Ap1%?JbISV2yO>Sx6+HQ~q=)10@!dOsVw9td3sla#u zdsH&e(Bx!gL9?f&q(mU=`+pI8EqXBv12LhK@cuap+5ex-QzW7I`X^a;yhlkF@(D)i zIn`{3Fe%ak;2{L$yta!+6n320rsOXdX(w*xEmw2>VuWQKZb|Oa$1hSdB8}W5TYH!h zk}PD>jAt?eerKOQawFp3>i-i*@y$a14fGiZK(l;E3xaIXO+G|TOsE>Vyni1(W2L99 z4J>lEkZWPuo0iw1$ALzc)a)H(q6w---qv;^|Lp1Ee9YureU44MO+yYUS^=#8Y7gn&8ahBI%L(pEz+=--jRb%+1%Am7)PJ*O4~nW0;VGSe@`Mgq^YOP!Lg*iLwbU8m|+}@8AkJS9i$bfGh&!7)(;2sRK0G z79i+JLFV783jBO^z{tj?ajwToh{@3{_3j-CLvc=yiBe~{t(~76f6b2%H-a8?fE5ooOBNNsi=0G6jFtS!0zUW!g#d=&=!3j6l#>h0V>GLr;`D3j ze27p{NeSvW@uQ~gEB>3>77^Z7WdIUcnR@PY#?+L-4R*K$tYXyeLfx&;vVRjm0;P-t zOODXPi}#~)9n2qFg-JW_Cz0sK05Qydk`uj5?+JAr`6~(l^rd1WkCLRTRI!8<}QZ0 z#+F(>^R12W35l;?1#GXklyTR|f-)8#jQFg>`{?eM$>o% z(f+VR>BtgJt>B(i{uGqJAMxAaLA51gTsZTj-Wf7dG3OvkN<_r(w9yY{(c5icV`qnT6YvBcBX$(8wm@4`T3QN%MM}{Y5H#7s($W)4V zUCCol&7fCkU@}%RGs`2<7zrjVnG%j6ev@nsLM!vDM327yA#j&7(StH56}H?VG**(A zN3IA*AdQq}DI`IE;dOo3yzxgyP(P3>qnb4_FrXw1)?=ql?3sYr*mN6hbZzcudf%L^ zzR+*qOZ)duZ1WpOhd}tx&(Age-_u)FR1JABWQtf}z;a_SKDB~bx~3W&u0Ulv;XXFo zJjfRSI+QUhd3CamdbG##st_h*Ty&}Vd=Oh%`-CmriBp+W=E!hvLzg9B!2rkSyE~pm zJnTpQCC8IW?ZSO?K+Bj3p_1?D$btb#?y)7if8i z4oPzB1%ZszuuVp0V{AFAGmTzAfLP=Tpy^ne#>Uw*R%RgxnmOMwjbVRQ#S$mHVbiZ0 zHDXOv!rFFjH2o1p!g~(lRSf|iV0F8>uhS) z_cM8&WXax{{s+4JyxS4uO8P-Z&_3GC1k9a!i9@Sirz7oyYA1K%w!fW(UCK2N@MWDq|G)j>K=#CqTA5emb8Yax(x`c103OavzxiK|5wV=1-?7 zM2&Zf?(N_uJs0o8_4XW++6PNaTSsWqG0pziwRLr4^-bA{?+X3aP5jWUHmb0lf?db9 zfm|v>Gv?&{ch^`PZDuqi2q~~>@dd#C^`{{B*oyJ+&(xDZx<7Dys6PmRdm{*yWnp3I zjSFU8+M3P@E}P)tF;(wNBX@WH2oc7hW3d?Qg>kVho1NQ+hX@A%%at3L3)#zQ6XA4m zO2twEef65IAR*-@*};p&sVP{bbv_eAPWJL&2if(QCp~1mXk6>XwyJpR=79R*{=`A) z@G~*707^T))N|5djsD$7GT@<IWOWmTpTWK(;cr{=5FSfK$#GCSwehEbZ~R_^uC zW;8>dqjh{R8R)u{LE}_wF+p}>-3_Bz*+Tr_zvp-SvvZuMuvsfG%71S7w1M&G7*`@x zCBja|2pa7z>d_o1!L8@>B!^c>^~by!OM?%UaZ{!y&?J8hYroCRn@^n_=v2u|=K-P_ z$ee{J?8;rQyNBVD{_SKVpXQO4#eNijxF8IctT6%M38AEsjNDQtCNx{?=I~8w$?965 zDnz^@W^G)we{Gnv91!;^`D$8-kDbzoBvjqmBuQNo8nW~zI>r{Xm3wrSaF7~;sou={ zq%D`qKOj-XatH8h>ztT*d0^9^KUsBup9{{`zhmY8SSO<<(Z;+REw`5K#p*ZzpqE zL=B({%%dz)09R$l`*R^QN2!0m?7781mul50sm0HN-7ZO|CQQvT16hMU|A_!>&6d0& z_Xa&u`$6fJLw`maF1$Jh-cdAq{UDI8M10THUjq86(8x9HKL4Pl?4;Tj-gwiW#X zyJs;ju8#%=W1^&)wxzKP6VR2~WqbWh$9pSKwt~5>z8wfinB!B#9XYKpjKh5=Mwpn6 zeDNSIg1>{P5Cr)W_u2LxE`sOOirsMdfAE*a&R@BU8xDvWVcsDy1t!gpj@&M`Pl?~Q zmy_tnVi>+#nE}Y?|GOO(bp(4O@{xFK!R91a$V=APK^;0x7z(Y^-|^*vc8HvWB!W=< ze2CbmCC3wC=nfWgAg5Tmo6SPM#nLvxl}!F+Ii$+h*%ZoF;-4$*&a139Ky3*ui>f&J z`RVBkz)DFPgw@ zzB2b#%H3hr%|!RX@1Y?-)WRcV_Q>=o^%S*#ANxXuMT}_iBhB88F7~1V{#)r_6D`DV zmt)m{yj)9*JR}HpvGR(-@Pd(bmXphJWQR+xuBHa0(ssjVI#Oj?++4Clb1RwroZR_) z%Y|BI=}u^?!BKMwJUj~o$YQ`kmrfY>*?$iz`1ba8A9K86T^eEhz};xUvH9zB^i9!`LR zD6Kg7q6)_Y7(+r!9|mAXu7VL zD@i~Tc(S}#R=`HyOkj){(LIU8h>0X79K4q~c-BH2c>+2taF( zkeTV}+`K%-W}ZI7Zy#y?LD}M@=T6A3w&!poc#g&(dZ6eQ=f8#Fx4eZ{@;jeBoc;C< zTzlbb8v4Bl6vCxn`(^;Q9e|J|k5L_IRy+E-&?Ip7PL*2|(b!Nm_UaOwX=}}4&fN$w zkP}uPlJjb)9|8f08bOP{CK5U(c%x1)Z<3r(er(S$f44WEjk&#q*$6c@U^qf`*T)daW+rywX!LJS4 zAbgAfkzAmkZ>ab;H`f=(dSd~;Ka0GQ5<&;8+h03Ajc7L+sPP8R{M=ly`J|8u5PF@@ zfp|{rP&IVG<}ZElC5tSwQJx_=axImGcl-WudKxb#T~Igb2ISdzdspJ+8L^pbEYmrR zy~O_3#%HanSX=K8nrU{%P|Be2QP5Igg-dqp^QDpiUjd9AZJ-;&vGwC;Mf6*7Vril3 zL`>WPGmx$#)4K*Cc{z?x#GcZIkV>`&GS@5=6J3Et#e%ohg9{c=dDvq)0btt(Qk5W_h@DBWGG$ z*MbCN^`vCtRDXA1bA0z_N@$n4WPOVeVJPR>_kDg}W9D|Tbp!j&Y;1GVwu9((aQ$Eq;S z&bPd&w$-4r6X#Gat*gGwiDG&TC;DH-p?PAkyjR#af=%3&G18#zh#06QzhIfuiS;_R zBkoA89{)=9NP!hwtiJdWbOXodTFqw#3@d7MA|qQH^`j>~$jZn#EN=dO11`Cw^t7~1 z6$)=Adn+p#i=hD=XTPI^+zrN^)PsdQe`<%Q9Lg~PcJ^^FBz*RhS;i#quY2v~c{a-G zl{Q#@n>5u%KkW~lO`cO##j&J#;MQ!vg0!U1PRVbL0vGfdLwtlkpZL*jw5oVZ(X)FB zO6B9K<~$_rlI3d&g8Es=jru;Rf;tnE!$v~V+K7Ka6(K+?^4SOeuno&fNA*Mes)#j- zYocvu)RWr(5~V`LSbWi!$Wlv+&C{e;5#BQCivc;Z*lxE-NyC)+IYDPtEca^n6d#t` z3~6{K2qqV`!C(rZF>@(_;s!0uluI+h0Pgy_x;Dx`bP^NRSNAYd?PRaa$Im|pOHm2S zsS^M0OHse|xWVUz9)#ULCFkZv3T*-&W;`*RrRC+ovu~^a8K5BtmZ3V{0OMF#q|CB*S!s9irvY_x846Xc&NQ^hQ{g=3A@7e}()($ExaITtWotrnH@y*FK!FnBlimH4 zeeF;2pSN}-hqwW=!RLqjy;0Nm(dHiZYyc2qD45zM{b%>ts^Q&O;j5R!tMC8bu`IwP z+3fUtj_mrqQ(1cGri8!Jq`hWWYcVw0fgiN4oxD0aC9s zG&a}N#2qQEpbY*rR#^vBXb(9bW@tgYvMV|(yIHi`aA^Pl);2UW0Eb?0`)9qL@7Fow z=~4|ejs(CbEe>J=H@u1D?ziha@4~YNLs3L!2yeisyodDB ztyx`@;kiw&Mya^GycMh&$q*})EXKds{CyMwmnt=-o-JL3TK;mn?q6R?TEd@4vA<7- zYP-7`f}9&F?=QuX1w0(0$?Rf7DFWuIei^)z$sjG$9?F4@a^2{U12qP4V5LXMNJ*U= zb^R>OZ0u}pc@S)SlutLqSj3;757T@VKUq8-zh~v6T|SH6u!`dN_ud3z+*TD%_wd%~ zJ$F$f&qPwM2kBy-&)Xo3fG~VKR08es53`hEg@u`gpRx?I@ZeGKi>PsQ3#AGFXaxu3 z=rV8K_m~Y3@Gf1i2FCVrb+NafSbMZ_Qdx|~ZvPY*_DrAi+;SGhOz`r6IF1z+X|u+3 z+$UNxj^Eg=>nnvc^d(~TMwuhyx`rhB3b6Rk`Sm1?1bq!(d$>IZ*NreH9z-Ia z@_BEH`t54InW(mgOY#1={j8Nkhkr-t^Q(Ey0#2Lh-4>rB^1)-rS!vuK=~sj~7yT4Q6&CDwBOE8HO_{~!Ao#=f4}_sxW%!)_ekCRsn(FejdBD>|At}^ zw)XLX1dFo=iiTo)Pabka81=p*5apoWIi=-4zL%q zHK$!o-jB=s)0y6&qG>Sesyj)nfB2N5L(apUZ>Z5iKw&FjPW2`cyF%WEiFODnm@Qeg zqOLI)v$`OmG$(*#6bRY?#rU)NaxtUO67tHAh)CS60zqGsCBZfc_`}KCwf#y&{|mJq+*@+$Icw`- z0e%&4#lqzuQj!xJ11mWOpRI8B2$gEsJebCw|7dk8u#6ifWj}dPyl!jI>$Cm6>CIT+ zy%GElDQ7;t6k)tHjD%r0{!GqgM1bOBra^%1XXTtx^ zIyb9xe{WAhLIQm9f-SHw2YKZ$W)?QQb08ea(^CuMC@neJQh=GD${>@8sdkvOM8ld+ zPbCw5^YE)n39$KCC%@UMfz|dORz%FLr5kk>;Er#JqH6r*e|65KcCTa8JKV5RCGeC@5!dWFTz~#V)EVt06R{(H zUjvqGH#awWsAeZjl`o7?S0Q2B&0$nY?O$OooD0FE**!jHV`e5BZ2_4%6Cq~i z=Giea?+;g3p}!~ep@2d|f{!1JFvdR2EL32N%Ke)dMo*phLc3Xgzn1*@h6Whttbd_c z#)fI}s)g+hrW&Om$HvCq9TRZb83)o;T8(@kB$1_TldY|hRQtb8!MBCZ4qR;R$E9i^ z>wwfbJT_Kei}1$xrd-riBAd-H;tmPXGh9?#G=e^@84~-NRYHPChPk7-ZcxSeZvE{l zq^vZ7?l2#z2IY%-XsAF^LurZarEE+r;}_*K391%?!D-mNSqRDC{@QgTb@~))& z@yZpKsyYu3qXW%0G83eF=u%fS63sSw{M~0 zz4$+Ts9A0P;vJ$-L7}}@Us!m01Ok$;R=Yw!vW_4B&Y`WQ>3%hCMsTfGD9CO=d%L^8 zAIOx$$x)gB;>u?#ImdF)-_0?8Tq4%&m$|a3ttozuxJZFL;-E>PU8` z_&WJ5NHNcHDx7^>cPxk&;4IyBfI-h3(tIuN?@$qAsaCRcy!yr4`va|WG>^(zo$aD} zh5k!gJYa%bu=TC1Z;A2q6Dg>27IS`?35s{zXjn7==4b<^AU&!htFFF{yTM#l72CYH z0QID?GXFp!qfAbeF6K2C7p^N!afkdq=kd-C5(>(%S_Bfh8p27aETeCTC{*3PC1iW8 zkDp6sXT1-LSiT(*mE1AK;;iLqlSv{s>eKvq^-0!}fHiooPY;j&9REUA0!(TgOUcYUcP;dp8t+@DPp<&>d9_I2 zO+qUdlXflu$&Er^>nY`)a`&5Y0}sOBrtk%^0^ndwMoDHNC6k3HtM^gfx#O z-bEk~Gt>~?CisiIG zQo=}(5s6Lgtn*-km6Lae+7e$=vjqDGO)4J;hgaTYyRS5QqpVcypPl|pezf!8cgiNd zoumJBkNK^Z4mQNfqw*LOGdhASX_ZH|Smzo_4fIE5i27^iYUq1~@oT}ieO2;m-3mJB zXiu7N@@5KJA^qBsHItuAEEve#kr)c->=?>IwS$aR`3bVP4@UQrI+DUe@R^dhYRW3h z$g5&NYfA^EdmeJHfgDcpQ8V7vp^rH)%k=Smqkb*btgk)!+dfqcyKx~|x%`Ro@k-xk zpROrONb=IsQf}nte6__{4m}BrySsZ$7{q=Yl10T#(uO}Oo{}>D1Eb~Xa2f5>!;^KjznY7LYKe}nX^Kri;vy~|W>5G|C%P_qrdnO3&tmt*FH^I+C%av`=0 z%Rs%buKRwhNYrprq1)JLzk(+LE{`Q~TXa=*(7J0T!G_v)={3NGuA0@FE`2C@IOlEu zD_}R6wa{?;R8VH{qKgd-2pIh-Fa>5Vd&_ZcDQW4f$3HcexZK)JG2{Pi{{1HAEYnUI zcwlr+Sy|YGrwv0a2xw&sTvT(& zrY+xKKY`H|AWP_Z@_7YtUXOTRE&=+4LB`*1{|Z!pV%G6D^Gof9KYH?l>$o^#<3)%vu|8o>B3$Pukw?wq;ceVd|m7Wd)?@17#DoRVEA|n;vzXzrh(zsy=XysA7@~*QW!C&9@)GXLdG7>R8G_laYwerj%kI{eDJx`eqjJrNQJ`V^VGLe%9YU0s@!`>>Q z&RNHbUeqWbQ24UWWS2jSwtf{NM(vFWi}CN6XhWxCsS3e4L())Erfdohi-`F4M9M7z zdA`uef|fK97>En_4#9zaemaINFNb9l%dZ$wR+=@=nW8tS#QOQ@N=($r>3!#e?kF59>v!gJFR9MUi*Kgf?l>NdU`C- zuTyy~9bL{8v!b!AN?<@A&ln2LC|j1tdkq2~sZKvKjl%}+#sbFV4Q;MCE@zj2NT=+j z-UP0!?{VbldiiPe9Bp9T66bu(4lw3NMnMsL$vPO>+28*do-y%?(LdpB8*L`mx!Nn5 zEkjX~T+r}t@mo%=RI-;LF6==Wjqgl;V0lJ>BN06;q#}1z2g|Cy2FJT+5}Y0Tr!js^ zkm&B8M5GQ~jWK}5rp4%BxelYjH{69><02**vsosbS8qN-B8f8$tT3{&PfkuMml%(_ zzy^Zk(h0}4l-Zy7Lo3xh*&FIhs2Vl@5Dq{rPgSJz4g$1K9x-Zan7rMLsdx%^{B@36 zn6#uM4-kp!gUL7^^mSrts{9d7Y}C|K#R7?&yGeAWR~upijyDcSAP{guB*_KnNr7_4 zyJ2Gpgkhl?bYweeB(a2Cw2MqNIzB9}qA4i|OG|XFu{Q=$dYKiE1|#WX53&CA_`kpP zL;TkzWWY4Qw!6JS&j_h0tTcMv`Qry3Lxrhd$kQIzti1C1FSDUarTJG$?ScsurGA~A zE}IAxm)~B)#${R&Pa`WhCdSZoyp@Q&q?%KQCn;R*zDzCyib54U6ui)0qtBl|?@#@E zUnaheru*9?&OjQ4o61zNMu;KLLAG`_Fh(P^jEW;&*mH7f$|T8if3|FRSn&ab*dz*^ zt;9Tqe&RO7p$&I)DNfhPfJ3M@c~4<$YYUVm6zjb_j<6Qb+wFJ!+`qMi z|NU~BI!66g1}t5z-~rc;3aAMUda!qnXFmLWFRn`qx{+-Al!J&JELQ-<@9pfUIFI|W z=>OK_%*{rZoSK}T4l=g{t?#5@_Msy<_ls^rs_S8jVqTx=UETSgffm7J>81i+8}lM? z9Yf$-x0Jqcb!7fqv^P-#I7>@FPI|#vN9p4M5@0PYi(E8ve0IopB|2LxNz{AGnpaQJ zXuE$+R~r20$<3Scr7j61RxUpGRtXQ(sO1{fs;aQgs z^B##`$Xv=LcF_Ms>@Fkh)Fnq@jXeTWQcs` zJrSNSo@u2l@EXowq6*pk_lB1>rke-xR>tZJfD`X*74Gk~+X`WD^8GV7`<;frO# zB{n&1V4MA3-aPn8O*ujLs5l;*-u7^&U}$Ivklev%SGVy$e{SxVpTRPRFvANi)%|f& z^es$Ty9PB-;{b)Q3q*oU2{_o3Eeyh#MQ<(;yonc=JRF#*cmAMi z&~&-2HF6KKWRAx_2%$Y$*YeHU&UqdSNA#$_{LKBI;{~zHF8u|*TX&{aF=orHUqC5& zc1`eCegskyd&qBu9SWFaoG=#Oj%+Z+v7)_e!djoFTu1hn_wN zso-b5SLm;3$iMh7S71dF!GnnKQ8*mVt`OX69&aTH zs1il&tDX6-d~jR3`&@#R+IAh4eDJY^h}x^58?#-NUOU^?8}3{;DgO{^rrbUa9B)-G)C)luqwvZI> zqNn`%a^`7#2d1r7N&nS-gy~2}uOgH1_I`ou>37CoWO5)8!ocDsA!q9O72rzIN}^JA z`vWe&!LrfmH*5V_TC^Ei#zI0XCyy+oY?8!J*(9$+R`sJ)gH?aM&CKhY!_vyas;I3T zOI(wg9;!?cp&~;G-)yIwEGS3dnW_jMi@Z`ef_p94Q!H-5QtM>H5c*#kY7?0fd;?yg zJNhRciFeUQM@LK=<%Pc|K-YA~(Lxp(8#@N5p)%7AFXrjM$r~_Y6~{S5I-bn{i68t3 zg+)cHGcx^Q;QCNSN3q6(gk?(nQhX@;@KN#dgX;}jt#)iFoP37s1od@p;8lVp1im!f ziqg30H8HUzOaplzbcDJan|G5(b}Ptgkx^WtFI-rg=xWKVEprjrJkalk#w zLOCFYI70M`?59t7lp?RYc?4@#f&m$K83 z?IBf4#U*ttZm;9;Lx>=rN~$gfKlP)u(jU68-#s_k&|=~l2m5blkt6sYl}@?6Ii3GQ z{!WIf!MXpE)BHA599N<@<_k8td9~;B6YwzF{KzmKODUFai!5j^sJQ$#Ohnl8ta}jh zE`b^qIUy5%wQ6w1Y75jf$T{D6LdWKeb#~vR%OIr4{>8{Vcqa%?61jp&5QxTYX*_KsXu+9{2)O zcTYB;6X@p1+$)Ni+}pelwUKoxT>W(+AV~K3#=qo_@H;23h}n@I%s3 zgzq%48{E%~nM?Hnc`Bud7q#Z(RF_oXidHk z1Vz|2x#V7415>D^ku4y21$bO&AU7Xh$qyiH0e>SWM-72xvZ&Ey0^@}%K>o=29Sni1 zMxjxGdS){906KBPl?0{{wa3vjN4BA2mC{a;S-iKE#rvx;4D`nI1}_guYf7#F5WV2xC$a|6U}g*@B_+Y+20Yw=C_E#B zvX~8+&9Ugz8N&Edr2(Y{Fz2{FUe>@jcX#J|aOL+8@qKBYB)nJJp?9_G&Zz@X3UYIA z?(ga8>FeuBzduTPv*|YY<(-&A>K}XFfFCf1T!;V*3o1H2Tv`03%zY04r2#fj^$WP0 z-=4Dzz>MHcC=CPCz!C0AQyGu*7Y#I}=)wN}{b?WNi_*L|nzRrWy)C4}_=PwE%mbpL zqLPw`O34TbL+ioa2@LLOgcpBp06%WvmB_@)>o$z}VVwvVtHG%PKN`G|;5c94nVAIJ zB6GX`TMig$fk=9{<9{OS0fH}m)xes;7j3r~)bm=GM_wAiaVG!{WeZe^Vc6UJ_u0mP zw2c_^eLf4orohtxdmV=*K*XxAsfpUNNJ!= z+l=&>m03v?GT!6r_q^{P@8|P-{(C-m?)$pF;~d9%oX2UDR8{2-IZ&EC4?mRJY1!Ba z;?Agr0bAeq_8*jq-W)o#Op2tcBcn0(@lH93I|sQXHpKZp3Y7 zABCHm>GG+F`7s{hLy!7G%f;afXylCuax`O3zjf!1GMm*RasW)a`@6ravGmLa+mB=M zm$$Rgll0ys@72eKu-ZP|=PXb?a@bZwD=;w7#wIIvhXSK)>36Ho7d4wUd>yfQMhqm! z97Q%rU0Rg!AG!n>8tcLL&K*}XKVMk<Sgh6=6}b7cc#G>n6{i>tDCVt^%x zg@M6c`@vx2F%a~d%pUiJw6E%ipzoE z>E69`dN`O&7i&E}wSXhve)iGfA<|_}G$%PbPrVWdj5)N9ETOxn=fz8#w{fvLm9l5@ zS7K_d(az$w`RWf{#JH@77EcbToVm-@VaO>xJMf6@Nui%FiVhz-JMl9dPNQbNetsjh ztV4J$h&`;K_&BE9y@750^DWoA+1bjA!@+3OW#BJhpquHVjEjsQv^BJv zMuY0?-dEW3Z|_ttWf8&Yq}K1kpo~wuG`&H^o7R4;w~z|Fzo^{IvPcQ`E(3Y?W*pW zFx%b|^yY~Ohm-LTx{1Y1AG*1_gHZeC?vZGJXu0Zs3F(2Rzz~Kn+l;>F<%m1e=c9-~ z+OQlQtHvx4jZ-(=??0DD3rfcHLhPM(UiBfwjTCk zb^1+AG~v(Bds$%ez)cb~@Pg0Q%drOspSMY81Yz8p5xF_U=**c|u^T@XJzuQ2VL-AVMv z0>3B{Y2gcaudpMKr@=O(H6C_Lbe>u*zOzJ_MB<^ldK`y6T;XoX<>=tV9O3=Yhw3U5 zWO?4&dVa9R$Bg@ybwHZI)A~y2N%e`!$#MC0+@W%bUdZ{yuNA#HC-tSb;lq)SWcz%# z`W?c|j(_nM4F1ix#q{|LJE7D9yXTC>b!$&ZYH5)SOX7XSYbfT~i<@X`(MhOl{w*J* zj#G`JMmVU3zo$-AgfqFO?Y?VwmzHN#-uvt6h%?YHGq-v$t+I0@dRf6hlfHC|lS`bM zoJ-$q6R+#=K3!we*0wfi5fbvuf)Qp~u2Uu^Ced8`l2C^}TuOiG0jUmM)bft3^uk{3 z>^m#^L})mUs5%b?{5qqX&3cN)=1ht83yhq(rXI@5OA0fr=G%KIyvH(N!Q|6Zc{x5g zu6?JSUcrGi3*nsg z>0yg98r!EQsf6+GahjZ-urM3lcrPtXnK0(FyN$oegtOIe-XBVY3o%D_lD-P?+)UZ# zqD3FZ)GJOdIrC?pv@MK?nQ47x zoXv7;lh>=y413XS(=5Ncr#?2P&w#%~DP+U+^V{yTMm%fB<{#I3BiWoG_}4pkUYnYf zg`5JkYlwZa7kB9|bEuB9q021|Ee%s%p1t_`bG%0s2%t9~0z(k;n*eyR&d>^w5EF^u}SrcPksx|TqI z*RQWyBNy_4-C}!{LtVA zwY>4QG@S#4d0w|)H7n~0`MXSeUH7&snR)Yw>V;Bc+udR8PAG@9%Af5?z5sGLK9T0F z?n=#djpZnhrsP~Q5&N=MPNXK)v5+EDoz6?p3F(@V6eGfQW z?REa+E33t*4aqjomUEc6#0^>^AV9N)oU~;R^^MT%=smoZ zJnAt)_-r+Vl}A{guG<^{?Ka;A=(EYtCGLloH>1(YoB?!O?_BNNl8XV z#)vOjntj~9nAS-W*R32|=eD9e4pOK)ObOv}xeu(at^#EPPQbpKl0@@frnOAz`Cq*c zwi-C)sFKl@I{#wZ%?;mXCMJZ@!uXTHHFfXOj)7vTd{=%~o79*8)>FuSHyGtdh;pvKxm>=pic3 z3EYgI8``hAbjWAMpLu$zqP+Y+JBQ>$-E*Hnk#~ej^e&KoCi!`c=->K{#UA|l#lrPj zg;U2)PIYnpxk*}{TVf6zk`MFqpY!>M8m^b`8Y8U^U$WpL=_cDBsXik^E$RR8?LN7S1n3hTlqElpXeLiNK#LsfJ4q?KBd zvkLndJ$FPb$8dEXjyYaJpBqt_;;eqO^aiJxsdZv;v1INZ=8VJC%#yE!`l9c>W#7EE z;8R56V_-3pr1b6L;ChC zX*@%dV!q`|mTX(f^^snWIze|H`MpysDlP6IVT0!p?S-RfBQ!j`>p&*d_EJViZIQu; zm=XPHYc!cHt*wuqsQn2JNA}+FN6{el*TyA`@KX+>;jcf;OIMXV-Z$(fWvDK5Kv;y2 zL#H&o-z97*lG>K1h@mG;##W?P+FnLaM<>|(Sn2G!fHI~C$;!r^+dd}l()##F#Z3Cd zo!8V|H+N;R?4+*2%lD>jWcD@y&n9slxc}x>SDm+0o{{B%;J_0HcKYD zTi9D%1rT>6fN$xEs@q>49`?sG*fTUUs9u-O4~-Sp)hX+Rl62%_J+8PBcWb7~kx|wV zf*fWWdDzB>F$E8r`ox_m9sDJ})PdsUOyFl8(}Q@U{XA?=mG?QE?v8`ZvfLKmkoG)t zefQ?+*&dGP$F-xAOcu2}?G;qh@A(h_?TPLbL(` z0&Ek5gO2C=?;O^8Zx`k_&ZeQ5Z&)WgdeX<_()#bq%6c|%I_~=h*)$~)G$RTHO~O15T-P21j2}j zrp7sA@HUa#BdUJjuF~wX17JI#>Zvy5Y$>vE4 zZj%=k6T=3y>oi4z6c&CMm5bD&pQkt;;;HZ~lHhjkW%7%hO2Y6$LFWyAq(a)cKZ z6>ax~KDjjh*5UlehSSe{7f>&$vCOy$iDU=~>d$Xw@P3b)ovCXe;@d37bRny`-rnA3 zADJx+Uw(h9Q~q@4or~Nzq^T8EpEk1`bR9&3{l`m`bocf~sc30v#5_&_#&p!lx7nC# zGO^qN-+mn;Xvxjpot-irLpjX6aw_S0c`C(brM4a{XVd9rzjBFtCW#^$=%-Lrur z-{*N7aAEhiLfcxbF;d~mN;UHhj~!Co=fo(i}GYQ+df!>m^g4KaBMP%*rE^{Q*_QrMG8`{6*IZQJ^w2`<)nY7&&ECl}9K zSPW{ zdieSMj%43kvIbM|Fg5I%wMZ|LJ0ruoJ}$bj`ee0kHb!AO&-om{Nt-?AT}?`tgHw`9 zvvD@WpMjg5qoQNDc&JdqpId@Jqy>9bS0x_H6IwKeZ$lAi&L@Yzsv4;pySgEo9&*?u+S6 zB|=1r>fo+j^_oip{Gp$3hHcd8fjU!3%+lcSvBaZ zb0NY{yG?5~_uH{oHm$OsTWjx`=?-5}St4=K($VQOBVd)Gmu2~D8aw&2b0LrKgDL8x zCx}Da_to)+s?^BjWMuF8ww#4uzo1fvy_k4qCb*x?%4!Hq^5i9OG)X-p))42g>#osJHe9j1zE?1ZxQ z9?IqD9`ao0xk4KA8E%1NESH~9eFLwINJ1kxAvXNk>}pt$V_3o|YQ+VZ+Y@^z=qv7V z47xmBSY=%#!~7`3YUTNVmkms z?VKkswZeJ*)#}FF9M|98!4uJ;Lv|t@ry0tVE(dIym#uPS<>z0W9Z~;}+me~c(dqgAq`!M>!OzFcJQr&p_t?_T)lrsb@zP>XezrdpGwsd$Or&fWicb3H z$^@A2gHo{DYu2HV1a4J5y*IFnxwi=WLp@gzpU^ZP#>`8u%;t3JzahAh$27K6FGJpD z!Fu=nDH)g7^?^h@b~P66ufW3~x4O(mJ;epTG_@hLX6f@SM2W{tOnSW7oj&a4Q}Vw_ zT;ptVak0MU$qqVoyR!iShxJ(m-)QOUzs1$X7&9SNvdtYNwL8g`wB{if$C_eCDSZtn zyLzU~#T_G~aFbJIb}3foU749mX z(VGhvWK|IPqNAfTgZR*^25>1qe6K1Yk%)i|M1HU|g3hWOHrU3+m710|iTVmqxcBc{ zlGP)AfbeB4lLp%+m1Q2*qT$TR;8(JbB1-iGrlI6LpbnRCn?5e5ZK9QpB!v0T{f~0+ z^YJ}2P0dkpXbfANx+9aacJ`U)badlR4yJ+Bdl#1#DPRZXBv$TCsCSXO13(I6d}AUb z?-v$kRtKK2v&*ilTZa(iX5yU3q?eV+9Azn_J1+iqlc3&Fz=h?L$m!V~&r*(a3mJs* zys%m=5!s?(Jz>3ul&eH3hW)O{*zHYpay>oAaN)4WNBW<8 zzPE3Q36fcelqsS26lZ^%WyZHXDwJ+IT&f}6%*sP;I{w7Es{Mq)Uz?I zetGfxSmfftOqkb}>te}23egatmXUy&<{;ukkr010H8jjGEPO9#9eoI-aHg+x3~*DI z+yNTHs-57`ykve0e2?G41Didw0@B={3v>wGfKbs#Jr*d;r~Vxca5j?)K6Ce)?VtEA zh#v4wKLM#xK0g1o6(23m2wx>d#hQEf>^o~+i9g4kV?aro^?pJ^0-_C!&?%TI&nYb2 zCo0-vs6K+m)qwAfTa%ZL6Aw!-*l<)>Obl`uqLPwz{!22tfR2;}1xMfmVTjl{ymgv% z{rnE`EknX=Oq_0g2wBVOyTH}01VBqKXzxX!kc8A~in(e!4io030Y1bO7WAVW77(~I zJUo2mzm_KZ%>&Sh<4@6cMm6qexdU_UbgH#`1% zU6*TVSccS`??Gy68S%(J^7|rec)^r8`{0!(kDS^-*fvORh-PO;{ z$|@CRsVl>eP??xg`K2xM+A?3|7ZCe1cb%Xm!e*b8)YK3*680FlbVp^~cy4&4G-&y({f5v&g(Tcu2K0e+u+Cj+0 zuYzULT8_Rv>Pl*-6jiKct^eaI^$_}~NMRlI2Y4>;Z_Of;X7Aw8uYK-aXJ-v8cw6Lsrf`0Ha>$hpR<78c_QXYiWG_TUg{C-cy;w6WCih}$B!L*S$BEa|5INE ztC~3d~{5hn7l7uyubsbI^xmj^mJiWo6e4#ks%5dGUlEdzwBv{@UuUM?$8fMDuGv0m!g(bY z#puG}X#_eqBdHfw`$JeAcM1#BA8hWXq_rZ~GH6Ch5;)hqS42bvynPB4K~gb=IO*Vl&`&v8N|^~k6wkeZbKwFPtF zOtEa*WH0kH?KS8Rx|Sja!(T=il2O4P^h*$B%6fC3%VGy$>4+ zaSi=1(r6E8eD#Vr^FIvD%(emH2he3cshjl&fQR(>AGpM9&}TeTB(R%@uEcl2c}k;DcK;q^$kR$~ASXdNjvj@oqdit1g6Jn8>w#ZM|N}+h{81@%MUcgSU+u%A@ zIH55Dw%T9-afRRI%PMZA=9-#e;zumx1K0eK@vSU;|DMQ3SD1ABc9DQPYgdvm4||^* zNW`#;D5l8h$nxA!F7k$oCsqv8WlW5X(M@80K;cop|LXEWNC?eJ&O1>MM2pTml~WBR zcxEJ$@7cqLw27BuE}(mopHDCMVPGKoT>gFOpbUPNya70U%L`7GUm1UmyVc zF+Z<}cAafBJX104fFP0EwTZ%7U=XV}-T{0|RF7*RXI$;KxpzdFUX-hE{~c9X8ey=G z^))_f+S=Npaw|)P#iB1gNk;gzqa7aVCW>rvQBhlhet!RMC?;uD%cmCO+go3sJ)_|C zrKYB)X$NIW#1)qQQ3g^-7XI1K6bc179nYg6Ko79ohbr5k6qoObS#)g`fGPy-TWEMR zG&Ee^mrueW;*VhPyG;cIW3S^{-m~znh{MvytCD-jLwJ^mAdo9b<$Ln?!TIo;@O(9K zyS;&$8d6@4%Uc1M(x=G;6%`kUlTbc>2>ST_LL33}{`&e|s3ar%2DIxeR9S#d>JG*! z+V}14vG|)(x^6kQ2^4y}Zke23L>r31k@fj5{jvC*rb$e5(bd(RnVIpr=F0Tv@IH?> z-1Weo$6y>zKQ}j5v?6p4n(CW3DY+A8Akw@^*T;yrD`66mO2yhjzXZGTqBO;?#T7yFUFKtn1DrlF&A zeP7Nn5Ou#bB>!>j4pz0-E}xOLYV7d@g`hdnc*e=MDHbXDsHIxYYpGl9Qj-|AhEn~^ zK$RqLF)d3YBkG*LU0h^0UyH;@vTdG_CqP~Q^5yE{;%`8XK*7Md%AD~7*cqa`2aP4Z zHx4LjXwwTO>uYJfA{Y+*{D)7!IYgIq{W>QboAf_?Po6ysfCYoPN2#I$+#zBsQ3;7! zOg+Qs*cH&M6 z;$c()y`$=c^QEHl{QKA|@blL2u!fW8(AbLWDS>_E<$a1=2nwY>*9^Xl>IW{ff7bs% z>EE?|5Oc1vZpZwH5QK;VlOP!Y-7%m+pLY9eU)b55nE$(8RWlgX%n z%?TX)*D>5j%PZl;Mn*=6i1XeX#CVVjMNM|Aq~*-y#UB=i*<>iTNT%?ti z6*RYM-Twir3D7U=4M;^`F%SFv@5^(7O`_~OKLLaqDxbiBq|YY94!p$$$DLth#5me4 z5&<}1>oU?{WdJ;3VD|^Ku?Yzg;U7u`fEL*m`!xMMA*g(B39NjFo;4VAh~n)_8;R^hx@^Dpkv{?>Wu5GoA}?5f8FqZ|A4Rzf4=rV z3oJ<7ls{kfUqASd{|;gO|369p-#+;N&+>oY{9ix#_gU;W&aG4VYUwObIZJu}h5wEo L(bFhWv%dB}cK|?= literal 0 HcmV?d00001 diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 509d423919a9..60101fa4aa22 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -274,6 +274,17 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "watch_map_test", + srcs = ["watch_map_test.cc"], + deps = [ + "//source/common/config:watch_map_lib", + "//test/mocks/config:config_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:eds_cc", + ], +) + envoy_cc_test( name = "filter_json_test", srcs = ["filter_json_test.cc"], diff --git a/test/common/config/watch_map_test.cc b/test/common/config/watch_map_test.cc new file mode 100644 index 000000000000..543298557fab --- /dev/null +++ b/test/common/config/watch_map_test.cc @@ -0,0 +1,397 @@ +#include + +#include "envoy/api/v2/eds.pb.h" +#include "envoy/common/exception.h" +#include "envoy/stats/scope.h" + +#include "common/config/watch_map.h" + +#include "test/mocks/config/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::_; +using ::testing::Invoke; + +namespace Envoy { +namespace Config { +namespace { + +class NamedMockSubscriptionCallbacks + : public MockSubscriptionCallbacks { +public: + std::string resourceName(const ProtobufWkt::Any& resource) override { + return TestUtility::anyConvert(resource).cluster_name(); + } +}; + +// expectDeltaAndSotwUpdate() EXPECTs two birds with one function call: we want to cover both SotW +// and delta, which, while mechanically different, can behave identically for our testing purposes. +// Specifically, as a simplification for these tests, every still-present resource is updated in +// every update. Therefore, a resource can never show up in the SotW update but not the delta +// update. We can therefore use the same expected_resources for both. +void expectDeltaAndSotwUpdate( + NamedMockSubscriptionCallbacks& callbacks, + const std::vector& expected_resources, + const std::vector& expected_removals, const std::string& version) { + EXPECT_CALL(callbacks, onConfigUpdate(_, version)) + .WillOnce(Invoke( + [expected_resources](const Protobuf::RepeatedPtrField& gotten_resources, + const std::string&) { + EXPECT_EQ(expected_resources.size(), gotten_resources.size()); + for (size_t i = 0; i < expected_resources.size(); i++) { + envoy::api::v2::ClusterLoadAssignment cur_gotten_resource; + gotten_resources[i].UnpackTo(&cur_gotten_resource); + EXPECT_TRUE(TestUtility::protoEqual(cur_gotten_resource, expected_resources[i])); + } + })); + EXPECT_CALL(callbacks, onConfigUpdate(_, _, _)) + .WillOnce( + Invoke([expected_resources, expected_removals, version]( + const Protobuf::RepeatedPtrField& gotten_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string&) { + EXPECT_EQ(expected_resources.size(), gotten_resources.size()); + for (size_t i = 0; i < expected_resources.size(); i++) { + EXPECT_EQ(gotten_resources[i].version(), version); + envoy::api::v2::ClusterLoadAssignment cur_gotten_resource; + gotten_resources[i].resource().UnpackTo(&cur_gotten_resource); + EXPECT_TRUE(TestUtility::protoEqual(cur_gotten_resource, expected_resources[i])); + } + EXPECT_EQ(expected_removals.size(), removed_resources.size()); + for (size_t i = 0; i < expected_removals.size(); i++) { + EXPECT_EQ(expected_removals[i], removed_resources[i]); + } + })); +} + +// Sometimes we want to verify that a delta onConfigUpdate simply doesn't happen. However, for SotW, +// every update triggers all onConfigUpdate()s, so we should still expect empty calls for that. +void expectNoDeltaUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string& version) { + EXPECT_CALL(callbacks, onConfigUpdate(_, version)) + .WillOnce(Invoke([](const Protobuf::RepeatedPtrField& gotten_resources, + const std::string&) { EXPECT_EQ(0, gotten_resources.size()); })); + EXPECT_CALL(callbacks, onConfigUpdate(_, _, _)).Times(0); +} + +Protobuf::RepeatedPtrField +wrapInResource(const Protobuf::RepeatedPtrField& anys, + const std::string& version) { + Protobuf::RepeatedPtrField ret; + for (const auto& a : anys) { + envoy::api::v2::ClusterLoadAssignment cur_endpoint; + a.UnpackTo(&cur_endpoint); + auto* cur_resource = ret.Add(); + cur_resource->set_name(cur_endpoint.cluster_name()); + cur_resource->mutable_resource()->CopyFrom(a); + cur_resource->set_version(version); + } + return ret; +} + +// Similar to expectDeltaAndSotwUpdate(), but making the onConfigUpdate() happen, rather than +// EXPECTing it. +void doDeltaAndSotwUpdate(SubscriptionCallbacks& watch_map, + const Protobuf::RepeatedPtrField& sotw_resources, + const std::vector& removed_names, + const std::string& version) { + watch_map.onConfigUpdate(sotw_resources, version); + + Protobuf::RepeatedPtrField delta_resources = + wrapInResource(sotw_resources, version); + Protobuf::RepeatedPtrField removed_names_proto; + for (const auto& n : removed_names) { + *removed_names_proto.Add() = n; + } + watch_map.onConfigUpdate(delta_resources, removed_names_proto, "version1"); +} + +// Tests the simple case of a single watch. Checks that the watch will not be told of updates to +// resources it doesn't care about. Checks that the watch can later decide it does care about them, +// and then receive subsequent updates to them. +TEST(WatchMapTest, Basic) { + NamedMockSubscriptionCallbacks callbacks; + WatchMap watch_map; + Watch* watch = watch_map.addWatch(callbacks); + + { + // The watch is interested in Alice and Bob... + std::set update_to({"alice", "bob"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch, update_to); + EXPECT_EQ(update_to, added_removed.added_); + EXPECT_TRUE(added_removed.removed_.empty()); + + // ...the update is going to contain Bob and Carol... + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment bob; + bob.set_cluster_name("bob"); + updated_resources.Add()->PackFrom(bob); + envoy::api::v2::ClusterLoadAssignment carol; + carol.set_cluster_name("carol"); + updated_resources.Add()->PackFrom(carol); + + // ...so the watch should receive only Bob. + std::vector expected_resources; + expected_resources.push_back(bob); + + expectDeltaAndSotwUpdate(callbacks, expected_resources, {}, "version1"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); + } + { + // The watch is now interested in Bob, Carol, Dave, Eve... + std::set update_to({"bob", "carol", "dave", "eve"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch, update_to); + EXPECT_EQ(std::set({"carol", "dave", "eve"}), added_removed.added_); + EXPECT_EQ(std::set({"alice"}), added_removed.removed_); + + // ...the update is going to contain Alice, Carol, Dave... + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + updated_resources.Add()->PackFrom(alice); + envoy::api::v2::ClusterLoadAssignment carol; + carol.set_cluster_name("carol"); + updated_resources.Add()->PackFrom(carol); + envoy::api::v2::ClusterLoadAssignment dave; + dave.set_cluster_name("dave"); + updated_resources.Add()->PackFrom(dave); + + // ...so the watch should receive only Carol and Dave. + std::vector expected_resources; + expected_resources.push_back(carol); + expected_resources.push_back(dave); + + expectDeltaAndSotwUpdate(callbacks, expected_resources, {"bob"}, "version2"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {"bob"}, "version2"); + } +} + +// Checks the following: +// First watch on a resource name ==> updateWatchInterest() returns "add it to subscription" +// Second watch on that name ==> updateWatchInterest() returns nothing about that name +// Original watch loses interest ==> nothing +// Second watch also loses interest ==> "remove it from subscription" +// NOTE: we need the resource name "dummy" to keep either watch from ever having no names watched, +// which is treated as interest in all names. +TEST(WatchMapTest, Overlap) { + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + WatchMap watch_map; + Watch* watch1 = watch_map.addWatch(callbacks1); + Watch* watch2 = watch_map.addWatch(callbacks2); + + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + updated_resources.Add()->PackFrom(alice); + + // First watch becomes interested. + { + std::set update_to({"alice", "dummy"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch1, update_to); + EXPECT_EQ(update_to, added_removed.added_); // add to subscription + EXPECT_TRUE(added_removed.removed_.empty()); + watch_map.updateWatchInterest(watch2, {"dummy"}); + + // First watch receives update. + expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version1"); + expectNoDeltaUpdate(callbacks2, "version1"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); + } + // Second watch becomes interested. + { + std::set update_to({"alice", "dummy"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch2, update_to); + EXPECT_TRUE(added_removed.added_.empty()); // nothing happens + EXPECT_TRUE(added_removed.removed_.empty()); + + // Both watches receive update. + expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version2"); + expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version2"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version2"); + } + // First watch loses interest. + { + AddedRemoved added_removed = watch_map.updateWatchInterest(watch1, {"dummy"}); + EXPECT_TRUE(added_removed.added_.empty()); // nothing happens + EXPECT_TRUE(added_removed.removed_.empty()); + + // *Only* second watch receives update. + expectNoDeltaUpdate(callbacks1, "version3"); + expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version3"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version3"); + } + // Second watch loses interest. + { + AddedRemoved added_removed = watch_map.updateWatchInterest(watch2, {"dummy"}); + EXPECT_TRUE(added_removed.added_.empty()); + EXPECT_EQ(std::set({"alice"}), added_removed.removed_); // remove from subscription + } +} + +// Checks the following: +// First watch on a resource name ==> updateWatchInterest() returns "add it to subscription" +// Watch loses interest ==> "remove it from subscription" +// Second watch on that name ==> "add it to subscription" +// NOTE: we need the resource name "dummy" to keep either watch from ever having no names watched, +// which is treated as interest in all names. +TEST(WatchMapTest, AddRemoveAdd) { + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + WatchMap watch_map; + Watch* watch1 = watch_map.addWatch(callbacks1); + Watch* watch2 = watch_map.addWatch(callbacks2); + + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + updated_resources.Add()->PackFrom(alice); + + // First watch becomes interested. + { + std::set update_to({"alice", "dummy"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch1, update_to); + EXPECT_EQ(update_to, added_removed.added_); // add to subscription + EXPECT_TRUE(added_removed.removed_.empty()); + watch_map.updateWatchInterest(watch2, {"dummy"}); + + // First watch receives update. + expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version1"); + expectNoDeltaUpdate(callbacks2, "version1"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); + } + // First watch loses interest. + { + AddedRemoved added_removed = watch_map.updateWatchInterest(watch1, {"dummy"}); + EXPECT_TRUE(added_removed.added_.empty()); + EXPECT_EQ(std::set({"alice"}), added_removed.removed_); // remove from subscription + + // (The xDS client should have responded to updateWatchInterest()'s return value by removing + // Alice from the subscription, so onConfigUpdate() calls should be impossible right now.) + } + // Second watch becomes interested. + { + std::set update_to({"alice", "dummy"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch2, update_to); + EXPECT_EQ(std::set({"alice"}), added_removed.added_); // add to subscription + EXPECT_TRUE(added_removed.removed_.empty()); + + // *Only* second watch receives update. + expectNoDeltaUpdate(callbacks1, "version2"); + expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version2"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version2"); + } +} + +// Tests that nothing breaks if an update arrives that we entirely do not care about. +TEST(WatchMapTest, UninterestingUpdate) { + NamedMockSubscriptionCallbacks callbacks; + WatchMap watch_map; + Watch* watch = watch_map.addWatch(callbacks); + watch_map.updateWatchInterest(watch, {"alice"}); + + Protobuf::RepeatedPtrField alice_update; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + alice_update.Add()->PackFrom(alice); + + Protobuf::RepeatedPtrField bob_update; + envoy::api::v2::ClusterLoadAssignment bob; + bob.set_cluster_name("bob"); + bob_update.Add()->PackFrom(bob); + + expectNoDeltaUpdate(callbacks, "version1"); + doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version1"); + + expectDeltaAndSotwUpdate(callbacks, {alice}, {}, "version2"); + doDeltaAndSotwUpdate(watch_map, alice_update, {}, "version2"); + + expectNoDeltaUpdate(callbacks, "version3"); + doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version3"); + + // Clean removal of the watch: first update to "interested in nothing", then remove. + watch_map.updateWatchInterest(watch, {}); + watch_map.removeWatch(watch); + + // Finally, test that calling onConfigUpdate on a map with no watches doesn't break. + doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version4"); +} + +// Tests that a watch that specifies no particular resource interest is treated as interested in +// everything. +TEST(WatchMapTest, WatchingEverything) { + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + WatchMap watch_map; + /*Watch* watch1 = */ watch_map.addWatch(callbacks1); + Watch* watch2 = watch_map.addWatch(callbacks2); + // watch1 never specifies any names, and so is treated as interested in everything. + watch_map.updateWatchInterest(watch2, {"alice"}); + + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + updated_resources.Add()->PackFrom(alice); + envoy::api::v2::ClusterLoadAssignment bob; + bob.set_cluster_name("bob"); + updated_resources.Add()->PackFrom(bob); + + std::vector expected_resources1; + expected_resources1.push_back(alice); + expected_resources1.push_back(bob); + std::vector expected_resources2; + expected_resources2.push_back(alice); + + expectDeltaAndSotwUpdate(callbacks1, expected_resources1, {}, "version1"); + expectDeltaAndSotwUpdate(callbacks2, expected_resources2, {}, "version1"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); +} + +// Delta onConfigUpdate has some slightly subtle details with how it handles the three cases where a +// watch receives {only updates, updates+removals, only removals} to its resources. This test +// exercise those cases. Also, the removal-only case tests that SotW does call a watch's +// onConfigUpdate even if none of the watch's interested resources are among the updated resources. +// (Which ensures we deliver empty config updates when a resource is dropped.) +TEST(WatchMapTest, DeltaOnConfigUpdate) { + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + NamedMockSubscriptionCallbacks callbacks3; + WatchMap watch_map; + Watch* watch1 = watch_map.addWatch(callbacks1); + Watch* watch2 = watch_map.addWatch(callbacks2); + Watch* watch3 = watch_map.addWatch(callbacks3); + watch_map.updateWatchInterest(watch1, {"updated"}); + watch_map.updateWatchInterest(watch2, {"updated", "removed"}); + watch_map.updateWatchInterest(watch3, {"removed"}); + + Protobuf::RepeatedPtrField update; + envoy::api::v2::ClusterLoadAssignment updated; + updated.set_cluster_name("updated"); + update.Add()->PackFrom(updated); + + expectDeltaAndSotwUpdate(callbacks1, {updated}, {}, "version1"); // only update + expectDeltaAndSotwUpdate(callbacks2, {updated}, {"removed"}, "version1"); // update+remove + expectDeltaAndSotwUpdate(callbacks3, {}, {"removed"}, "version1"); // only remove + doDeltaAndSotwUpdate(watch_map, update, {"removed"}, "version1"); +} + +TEST(WatchMapTest, OnConfigUpdateFailed) { + WatchMap watch_map; + // calling on empty map doesn't break + watch_map.onConfigUpdateFailed(ConfigUpdateFailureReason::UpdateRejected, nullptr); + + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + watch_map.addWatch(callbacks1); + watch_map.addWatch(callbacks2); + + EXPECT_CALL(callbacks1, onConfigUpdateFailed(ConfigUpdateFailureReason::UpdateRejected, nullptr)); + EXPECT_CALL(callbacks2, onConfigUpdateFailed(ConfigUpdateFailureReason::UpdateRejected, nullptr)); + watch_map.onConfigUpdateFailed(ConfigUpdateFailureReason::UpdateRejected, nullptr); +} + +} // namespace +} // namespace Config +} // namespace Envoy diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 50abfff89e48..3f935e236cef 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -85,6 +85,7 @@ EVAL EVLOOP EVP EWOULDBLOCK +EXPECTs EXPR FAQ FDs @@ -468,6 +469,7 @@ evthread evwatch exe execlp +expectDeltaAndSotwUpdate facto favicon fd @@ -624,6 +626,7 @@ params paren parentid parsers +passthroughs pcall pcap pclose From 261d4438c0690f127c9f4a7690da3e8377a51fd6 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 13 Aug 2019 22:23:50 -0700 Subject: [PATCH 384/542] grpc access logger: refactor common gRPC ALS out (#7879) Description: In preparation to implement TCP gRPC Access Logger. Risk Level: Low (refactoring only) Testing: CI Docs Changes: N/A Release Notes: N/A Signed-off-by: Lizan Zhou --- .../access_loggers/{http_grpc => grpc}/BUILD | 34 +- .../grpc/grpc_access_log_impl.cc | 114 +++++ .../grpc_access_log_impl.h | 43 +- .../grpc_access_log_proto_descriptors.cc | 2 +- .../grpc_access_log_proto_descriptors.h | 0 .../grpc/grpc_access_log_utils.cc | 232 +++++++++ .../grpc/grpc_access_log_utils.h | 25 + .../config.cc => grpc/http_config.cc} | 12 +- .../config.h => grpc/http_config.h} | 2 +- .../grpc/http_grpc_access_log_impl.cc | 156 ++++++ .../grpc/http_grpc_access_log_impl.h | 63 +++ .../http_grpc/grpc_access_log_impl.cc | 462 ------------------ source/extensions/extensions_build_config.bzl | 4 +- test/extensions/access_loggers/grpc/BUILD | 79 +++ .../grpc/grpc_access_log_impl_test.cc | 295 +++++++++++ .../grpc/grpc_access_log_utils_test.cc | 51 ++ .../http_config_test.cc} | 2 +- .../http_grpc_access_log_impl_test.cc} | 303 +----------- .../http_grpc_access_log_integration_test.cc} | 0 .../extensions/access_loggers/http_grpc/BUILD | 52 -- tools/check_format.py | 2 +- 21 files changed, 1062 insertions(+), 871 deletions(-) rename source/extensions/access_loggers/{http_grpc => grpc}/BUILD (67%) create mode 100644 source/extensions/access_loggers/grpc/grpc_access_log_impl.cc rename source/extensions/access_loggers/{http_grpc => grpc}/grpc_access_log_impl.h (75%) rename source/extensions/access_loggers/{http_grpc => grpc}/grpc_access_log_proto_descriptors.cc (87%) rename source/extensions/access_loggers/{http_grpc => grpc}/grpc_access_log_proto_descriptors.h (100%) create mode 100644 source/extensions/access_loggers/grpc/grpc_access_log_utils.cc create mode 100644 source/extensions/access_loggers/grpc/grpc_access_log_utils.h rename source/extensions/access_loggers/{http_grpc/config.cc => grpc/http_config.cc} (82%) rename source/extensions/access_loggers/{http_grpc/config.h => grpc/http_config.h} (89%) create mode 100644 source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc create mode 100644 source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h delete mode 100644 source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc create mode 100644 test/extensions/access_loggers/grpc/BUILD create mode 100644 test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc create mode 100644 test/extensions/access_loggers/grpc/grpc_access_log_utils_test.cc rename test/extensions/access_loggers/{http_grpc/config_test.cc => grpc/http_config_test.cc} (96%) rename test/extensions/access_loggers/{http_grpc/grpc_access_log_impl_test.cc => grpc/http_grpc_access_log_impl_test.cc} (61%) rename test/extensions/access_loggers/{http_grpc/grpc_access_log_integration_test.cc => grpc/http_grpc_access_log_integration_test.cc} (100%) delete mode 100644 test/extensions/access_loggers/http_grpc/BUILD diff --git a/source/extensions/access_loggers/http_grpc/BUILD b/source/extensions/access_loggers/grpc/BUILD similarity index 67% rename from source/extensions/access_loggers/http_grpc/BUILD rename to source/extensions/access_loggers/grpc/BUILD index 941d950ff9c4..e0e705952699 100644 --- a/source/extensions/access_loggers/http_grpc/BUILD +++ b/source/extensions/access_loggers/grpc/BUILD @@ -24,7 +24,6 @@ envoy_cc_library( "//include/envoy/upstream:upstream_interface", "//source/common/grpc:async_client_lib", "//source/common/grpc:typed_async_client_lib", - "//source/common/network:utility_lib", "//source/extensions/access_loggers/common:access_log_base", "@envoy_api//envoy/config/accesslog/v2:als_cc", "@envoy_api//envoy/config/filter/accesslog/v2:accesslog_cc", @@ -32,6 +31,29 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "grpc_access_log_utils", + srcs = ["grpc_access_log_utils.cc"], + hdrs = ["grpc_access_log_utils.h"], + deps = [ + "//include/envoy/upstream:upstream_interface", + "//source/common/network:utility_lib", + "//source/common/stream_info:stream_info_lib", + "//source/common/stream_info:utility_lib", + "@envoy_api//envoy/data/accesslog/v2:accesslog_cc", + ], +) + +envoy_cc_library( + name = "http_grpc_access_log_lib", + srcs = ["http_grpc_access_log_impl.cc"], + hdrs = ["http_grpc_access_log_impl.h"], + deps = [ + ":grpc_access_log_lib", + ":grpc_access_log_utils", + ], +) + envoy_cc_library( name = "grpc_access_log_proto_descriptors_lib", srcs = ["grpc_access_log_proto_descriptors.cc"], @@ -44,16 +66,16 @@ envoy_cc_library( ) envoy_cc_library( - name = "config", - srcs = ["config.cc"], - hdrs = ["config.h"], + name = "http_config", + srcs = ["http_config.cc"], + hdrs = ["http_config.h"], deps = [ "//include/envoy/registry", "//include/envoy/server:access_log_config_interface", "//source/common/common:assert_lib", "//source/common/protobuf", "//source/extensions/access_loggers:well_known_names", - "//source/extensions/access_loggers/http_grpc:grpc_access_log_lib", - "//source/extensions/access_loggers/http_grpc:grpc_access_log_proto_descriptors_lib", + "//source/extensions/access_loggers/grpc:grpc_access_log_proto_descriptors_lib", + "//source/extensions/access_loggers/grpc:http_grpc_access_log_lib", ], ) diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc new file mode 100644 index 000000000000..38020326b7da --- /dev/null +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -0,0 +1,114 @@ +#include "extensions/access_loggers/grpc/grpc_access_log_impl.h" + +#include "envoy/upstream/upstream.h" + +#include "common/common/assert.h" +#include "common/network/utility.h" +#include "common/stream_info/utility.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace GrpcCommon { + +void GrpcAccessLoggerImpl::LocalStream::onRemoteClose(Grpc::Status::GrpcStatus, + const std::string&) { + ASSERT(parent_.stream_ != absl::nullopt); + if (parent_.stream_->stream_ != nullptr) { + // Only reset if we have a stream. Otherwise we had an inline failure and we will clear the + // stream data in send(). + parent_.stream_.reset(); + } +} + +GrpcAccessLoggerImpl::GrpcAccessLoggerImpl(Grpc::RawAsyncClientPtr&& client, std::string log_name, + std::chrono::milliseconds buffer_flush_interval_msec, + uint64_t buffer_size_bytes, + Event::Dispatcher& dispatcher, + const LocalInfo::LocalInfo& local_info) + : client_(std::move(client)), log_name_(log_name), + buffer_flush_interval_msec_(buffer_flush_interval_msec), + flush_timer_(dispatcher.createTimer([this]() { + flush(); + flush_timer_->enableTimer(buffer_flush_interval_msec_); + })), + buffer_size_bytes_(buffer_size_bytes), local_info_(local_info) { + flush_timer_->enableTimer(buffer_flush_interval_msec_); +} + +void GrpcAccessLoggerImpl::log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) { + approximate_message_size_bytes_ += entry.ByteSizeLong(); + message_.mutable_http_logs()->add_log_entry()->Swap(&entry); + if (approximate_message_size_bytes_ >= buffer_size_bytes_) { + flush(); + } +} + +void GrpcAccessLoggerImpl::flush() { + if (!message_.has_http_logs()) { + // Nothing to flush. + return; + } + + if (stream_ == absl::nullopt) { + stream_.emplace(*this); + } + + if (stream_->stream_ == nullptr) { + stream_->stream_ = + client_->start(*Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs"), + *stream_); + + auto* identifier = message_.mutable_identifier(); + *identifier->mutable_node() = local_info_.node(); + identifier->set_log_name(log_name_); + } + + if (stream_->stream_ != nullptr) { + stream_->stream_->sendMessage(message_, false); + } else { + // Clear out the stream data due to stream creation failure. + stream_.reset(); + } + + // Clear the message regardless of the success. + approximate_message_size_bytes_ = 0; + message_.Clear(); +} + +GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, + Stats::Scope& scope, + ThreadLocal::SlotAllocator& tls, + const LocalInfo::LocalInfo& local_info) + : async_client_manager_(async_client_manager), scope_(scope), tls_slot_(tls.allocateSlot()), + local_info_(local_info) { + tls_slot_->set( + [](Event::Dispatcher& dispatcher) { return std::make_shared(dispatcher); }); +} + +GrpcAccessLoggerSharedPtr GrpcAccessLoggerCacheImpl::getOrCreateLogger( + const envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) { + // TODO(euroelessar): Consider cleaning up loggers. + auto& cache = tls_slot_->getTyped(); + // TODO(lizan): Include logger type in the hash + const std::size_t cache_key = MessageUtil::hash(config); + const auto it = cache.access_loggers_.find(cache_key); + if (it != cache.access_loggers_.end()) { + return it->second; + } + const Grpc::AsyncClientFactoryPtr factory = + async_client_manager_.factoryForGrpcService(config.grpc_service(), scope_, false); + const GrpcAccessLoggerSharedPtr logger = std::make_shared( + factory->create(), config.log_name(), + std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, buffer_flush_interval, 1000)), + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, buffer_size_bytes, 16384), cache.dispatcher_, + local_info_); + cache.access_loggers_.emplace(cache_key, logger); + return logger; +} + +} // namespace GrpcCommon +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h similarity index 75% rename from source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h rename to source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 5fb8e152108c..71745adc54d1 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -19,7 +19,7 @@ namespace Envoy { namespace Extensions { namespace AccessLoggers { -namespace HttpGrpc { +namespace GrpcCommon { // TODO(mattklein123): Stats @@ -127,44 +127,7 @@ class GrpcAccessLoggerCacheImpl : public Singleton::Instance, public GrpcAccessL const LocalInfo::LocalInfo& local_info_; }; -/** - * Access log Instance that streams HTTP logs over gRPC. - */ -class HttpGrpcAccessLog : public Common::ImplBase { -public: - HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, - envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config, - ThreadLocal::SlotAllocator& tls, - GrpcAccessLoggerCacheSharedPtr access_logger_cache); - - static void responseFlagsToAccessLogResponseFlags( - envoy::data::accesslog::v2::AccessLogCommon& common_access_log, - const StreamInfo::StreamInfo& stream_info); - -private: - /** - * Per-thread cached logger. - */ - struct ThreadLocalLogger : public ThreadLocal::ThreadLocalObject { - ThreadLocalLogger(GrpcAccessLoggerSharedPtr logger); - - const GrpcAccessLoggerSharedPtr logger_; - }; - - // Common::ImplBase - void emitLog(const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers, - const Http::HeaderMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) override; - - const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config_; - const ThreadLocal::SlotPtr tls_slot_; - const GrpcAccessLoggerCacheSharedPtr access_logger_cache_; - std::vector request_headers_to_log_; - std::vector response_headers_to_log_; - std::vector response_trailers_to_log_; -}; - -} // namespace HttpGrpc +} // namespace GrpcCommon } // namespace AccessLoggers } // namespace Extensions -} // namespace Envoy +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_proto_descriptors.cc b/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.cc similarity index 87% rename from source/extensions/access_loggers/http_grpc/grpc_access_log_proto_descriptors.cc rename to source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.cc index 4abb54f024cb..3b038045ee89 100644 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_proto_descriptors.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.cc @@ -1,4 +1,4 @@ -#include "extensions/access_loggers/http_grpc/grpc_access_log_proto_descriptors.h" +#include "extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h" #include "envoy/service/accesslog/v2/als.pb.h" diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_proto_descriptors.h b/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h similarity index 100% rename from source/extensions/access_loggers/http_grpc/grpc_access_log_proto_descriptors.h rename to source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc new file mode 100644 index 000000000000..6bab3fd1e274 --- /dev/null +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc @@ -0,0 +1,232 @@ +#include "extensions/access_loggers/grpc/grpc_access_log_utils.h" + +#include "envoy/upstream/upstream.h" + +#include "common/network/utility.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace GrpcCommon { + +namespace { + +using namespace envoy::data::accesslog::v2; + +// Helper function to convert from a BoringSSL textual representation of the +// TLS version to the corresponding enum value used in gRPC access logs. +TLSProperties_TLSVersion tlsVersionStringToEnum(const std::string& tls_version) { + if (tls_version == "TLSv1") { + return TLSProperties_TLSVersion_TLSv1; + } else if (tls_version == "TLSv1.1") { + return TLSProperties_TLSVersion_TLSv1_1; + } else if (tls_version == "TLSv1.2") { + return TLSProperties_TLSVersion_TLSv1_2; + } else if (tls_version == "TLSv1.3") { + return TLSProperties_TLSVersion_TLSv1_3; + } + + return TLSProperties_TLSVersion_VERSION_UNSPECIFIED; +} + +} // namespace + +void Utility::responseFlagsToAccessLogResponseFlags( + envoy::data::accesslog::v2::AccessLogCommon& common_access_log, + const StreamInfo::StreamInfo& stream_info) { + + static_assert(StreamInfo::ResponseFlag::LastFlag == 0x20000, + "A flag has been added. Fix this code."); + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::FailedLocalHealthCheck)) { + common_access_log.mutable_response_flags()->set_failed_local_healthcheck(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::NoHealthyUpstream)) { + common_access_log.mutable_response_flags()->set_no_healthy_upstream(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamRequestTimeout)) { + common_access_log.mutable_response_flags()->set_upstream_request_timeout(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::LocalReset)) { + common_access_log.mutable_response_flags()->set_local_reset(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamRemoteReset)) { + common_access_log.mutable_response_flags()->set_upstream_remote_reset(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamConnectionFailure)) { + common_access_log.mutable_response_flags()->set_upstream_connection_failure(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamConnectionTermination)) { + common_access_log.mutable_response_flags()->set_upstream_connection_termination(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamOverflow)) { + common_access_log.mutable_response_flags()->set_upstream_overflow(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::NoRouteFound)) { + common_access_log.mutable_response_flags()->set_no_route_found(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::DelayInjected)) { + common_access_log.mutable_response_flags()->set_delay_injected(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::FaultInjected)) { + common_access_log.mutable_response_flags()->set_fault_injected(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::RateLimited)) { + common_access_log.mutable_response_flags()->set_rate_limited(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UnauthorizedExternalService)) { + common_access_log.mutable_response_flags()->mutable_unauthorized_details()->set_reason( + envoy::data::accesslog::v2::ResponseFlags_Unauthorized_Reason:: + ResponseFlags_Unauthorized_Reason_EXTERNAL_SERVICE); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::RateLimitServiceError)) { + common_access_log.mutable_response_flags()->set_rate_limit_service_error(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::DownstreamConnectionTermination)) { + common_access_log.mutable_response_flags()->set_downstream_connection_termination(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamRetryLimitExceeded)) { + common_access_log.mutable_response_flags()->set_upstream_retry_limit_exceeded(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout)) { + common_access_log.mutable_response_flags()->set_stream_idle_timeout(true); + } + + if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::InvalidEnvoyRequestHeaders)) { + common_access_log.mutable_response_flags()->set_invalid_envoy_request_headers(true); + } +} + +void Utility::extractCommonAccessLogProperties( + envoy::data::accesslog::v2::AccessLogCommon& common_access_log, + const StreamInfo::StreamInfo& stream_info) { + if (stream_info.downstreamRemoteAddress() != nullptr) { + Network::Utility::addressToProtobufAddress( + *stream_info.downstreamRemoteAddress(), + *common_access_log.mutable_downstream_remote_address()); + } + if (stream_info.downstreamLocalAddress() != nullptr) { + Network::Utility::addressToProtobufAddress( + *stream_info.downstreamLocalAddress(), + *common_access_log.mutable_downstream_local_address()); + } + if (stream_info.downstreamSslConnection() != nullptr) { + auto* tls_properties = common_access_log.mutable_tls_properties(); + const auto* downstream_ssl_connection = stream_info.downstreamSslConnection(); + + tls_properties->set_tls_sni_hostname(stream_info.requestedServerName()); + + auto* local_properties = tls_properties->mutable_local_certificate_properties(); + for (const auto& uri_san : downstream_ssl_connection->uriSanLocalCertificate()) { + auto* local_san = local_properties->add_subject_alt_name(); + local_san->set_uri(uri_san); + } + local_properties->set_subject(downstream_ssl_connection->subjectLocalCertificate()); + + auto* peer_properties = tls_properties->mutable_peer_certificate_properties(); + for (const auto& uri_san : downstream_ssl_connection->uriSanPeerCertificate()) { + auto* peer_san = peer_properties->add_subject_alt_name(); + peer_san->set_uri(uri_san); + } + + peer_properties->set_subject(downstream_ssl_connection->subjectPeerCertificate()); + tls_properties->set_tls_session_id(downstream_ssl_connection->sessionId()); + tls_properties->set_tls_version( + tlsVersionStringToEnum(downstream_ssl_connection->tlsVersion())); + + auto* local_tls_cipher_suite = tls_properties->mutable_tls_cipher_suite(); + local_tls_cipher_suite->set_value(downstream_ssl_connection->ciphersuiteId()); + } + common_access_log.mutable_start_time()->MergeFrom( + Protobuf::util::TimeUtil::NanosecondsToTimestamp( + std::chrono::duration_cast( + stream_info.startTime().time_since_epoch()) + .count())); + + absl::optional dur = stream_info.lastDownstreamRxByteReceived(); + if (dur) { + common_access_log.mutable_time_to_last_rx_byte()->MergeFrom( + Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); + } + + dur = stream_info.firstUpstreamTxByteSent(); + if (dur) { + common_access_log.mutable_time_to_first_upstream_tx_byte()->MergeFrom( + Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); + } + + dur = stream_info.lastUpstreamTxByteSent(); + if (dur) { + common_access_log.mutable_time_to_last_upstream_tx_byte()->MergeFrom( + Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); + } + + dur = stream_info.firstUpstreamRxByteReceived(); + if (dur) { + common_access_log.mutable_time_to_first_upstream_rx_byte()->MergeFrom( + Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); + } + + dur = stream_info.lastUpstreamRxByteReceived(); + if (dur) { + common_access_log.mutable_time_to_last_upstream_rx_byte()->MergeFrom( + Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); + } + + dur = stream_info.firstDownstreamTxByteSent(); + if (dur) { + common_access_log.mutable_time_to_first_downstream_tx_byte()->MergeFrom( + Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); + } + + dur = stream_info.lastDownstreamTxByteSent(); + if (dur) { + common_access_log.mutable_time_to_last_downstream_tx_byte()->MergeFrom( + Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); + } + + if (stream_info.upstreamHost() != nullptr) { + Network::Utility::addressToProtobufAddress( + *stream_info.upstreamHost()->address(), + *common_access_log.mutable_upstream_remote_address()); + common_access_log.set_upstream_cluster(stream_info.upstreamHost()->cluster().name()); + } + + if (!stream_info.getRouteName().empty()) { + common_access_log.set_route_name(stream_info.getRouteName()); + } + + if (stream_info.upstreamLocalAddress() != nullptr) { + Network::Utility::addressToProtobufAddress(*stream_info.upstreamLocalAddress(), + *common_access_log.mutable_upstream_local_address()); + } + responseFlagsToAccessLogResponseFlags(common_access_log, stream_info); + if (!stream_info.upstreamTransportFailureReason().empty()) { + common_access_log.set_upstream_transport_failure_reason( + stream_info.upstreamTransportFailureReason()); + } + if (stream_info.dynamicMetadata().filter_metadata_size() > 0) { + common_access_log.mutable_metadata()->MergeFrom(stream_info.dynamicMetadata()); + } +} + +} // namespace GrpcCommon +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.h b/source/extensions/access_loggers/grpc/grpc_access_log_utils.h new file mode 100644 index 000000000000..1ba23cd6d169 --- /dev/null +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.h @@ -0,0 +1,25 @@ +#pragma once + +#include "envoy/data/accesslog/v2/accesslog.pb.h" +#include "envoy/stream_info/stream_info.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace GrpcCommon { + +class Utility { +public: + static void + extractCommonAccessLogProperties(envoy::data::accesslog::v2::AccessLogCommon& common_access_log, + const StreamInfo::StreamInfo& stream_info); + + static void responseFlagsToAccessLogResponseFlags( + envoy::data::accesslog::v2::AccessLogCommon& common_access_log, + const StreamInfo::StreamInfo& stream_info); +}; + +} // namespace GrpcCommon +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/access_loggers/http_grpc/config.cc b/source/extensions/access_loggers/grpc/http_config.cc similarity index 82% rename from source/extensions/access_loggers/http_grpc/config.cc rename to source/extensions/access_loggers/grpc/http_config.cc index a58327004da5..2e3f19e87927 100644 --- a/source/extensions/access_loggers/http_grpc/config.cc +++ b/source/extensions/access_loggers/grpc/http_config.cc @@ -1,4 +1,4 @@ -#include "extensions/access_loggers/http_grpc/config.h" +#include "extensions/access_loggers/grpc/http_config.h" #include "envoy/config/accesslog/v2/als.pb.validate.h" #include "envoy/config/filter/accesslog/v2/accesslog.pb.validate.h" @@ -10,8 +10,8 @@ #include "common/grpc/async_client_impl.h" #include "common/protobuf/protobuf.h" -#include "extensions/access_loggers/http_grpc/grpc_access_log_impl.h" -#include "extensions/access_loggers/http_grpc/grpc_access_log_proto_descriptors.h" +#include "extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h" +#include "extensions/access_loggers/grpc/http_grpc_access_log_impl.h" #include "extensions/access_loggers/well_known_names.h" namespace Envoy { @@ -30,10 +30,10 @@ HttpGrpcAccessLogFactory::createAccessLogInstance(const Protobuf::Message& confi const auto& proto_config = MessageUtil::downcastAndValidate< const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig&>(config); - std::shared_ptr grpc_access_logger_cache = - context.singletonManager().getTyped( + std::shared_ptr grpc_access_logger_cache = + context.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(grpc_access_logger_cache), [&context] { - return std::make_shared( + return std::make_shared( context.clusterManager().grpcAsyncClientManager(), context.scope(), context.threadLocal(), context.localInfo()); }); diff --git a/source/extensions/access_loggers/http_grpc/config.h b/source/extensions/access_loggers/grpc/http_config.h similarity index 89% rename from source/extensions/access_loggers/http_grpc/config.h rename to source/extensions/access_loggers/grpc/http_config.h index df13bcdb96d1..9e046ac39218 100644 --- a/source/extensions/access_loggers/http_grpc/config.h +++ b/source/extensions/access_loggers/grpc/http_config.h @@ -23,7 +23,7 @@ class HttpGrpcAccessLogFactory : public Server::Configuration::AccessLogInstance std::string name() const override; }; -// TODO(mattklein123): Add TCP access log and refactor into base/concrete gRPC access logs. +// TODO(mattklein123): Add TCP access log. } // namespace HttpGrpc } // namespace AccessLoggers diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc new file mode 100644 index 000000000000..07b91a0a894a --- /dev/null +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc @@ -0,0 +1,156 @@ +#include "extensions/access_loggers/grpc/http_grpc_access_log_impl.h" + +#include "common/common/assert.h" +#include "common/network/utility.h" +#include "common/stream_info/utility.h" + +#include "extensions/access_loggers/grpc/grpc_access_log_utils.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace HttpGrpc { + +HttpGrpcAccessLog::ThreadLocalLogger::ThreadLocalLogger( + GrpcCommon::GrpcAccessLoggerSharedPtr logger) + : logger_(std::move(logger)) {} + +HttpGrpcAccessLog::HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, + envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config, + ThreadLocal::SlotAllocator& tls, + GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache) + : Common::ImplBase(std::move(filter)), config_(std::move(config)), + tls_slot_(tls.allocateSlot()), access_logger_cache_(std::move(access_logger_cache)) { + for (const auto& header : config_.additional_request_headers_to_log()) { + request_headers_to_log_.emplace_back(header); + } + + for (const auto& header : config_.additional_response_headers_to_log()) { + response_headers_to_log_.emplace_back(header); + } + + for (const auto& header : config_.additional_response_trailers_to_log()) { + response_trailers_to_log_.emplace_back(header); + } + + tls_slot_->set([this](Event::Dispatcher&) { + return std::make_shared( + access_logger_cache_->getOrCreateLogger(config_.common_config())); + }); +} + +void HttpGrpcAccessLog::emitLog(const Http::HeaderMap& request_headers, + const Http::HeaderMap& response_headers, + const Http::HeaderMap& response_trailers, + const StreamInfo::StreamInfo& stream_info) { + // Common log properties. + // TODO(mattklein123): Populate sample_rate field. + envoy::data::accesslog::v2::HTTPAccessLogEntry log_entry; + GrpcCommon::Utility::extractCommonAccessLogProperties(*log_entry.mutable_common_properties(), + stream_info); + + if (stream_info.protocol()) { + switch (stream_info.protocol().value()) { + case Http::Protocol::Http10: + log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP10); + break; + case Http::Protocol::Http11: + log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP11); + break; + case Http::Protocol::Http2: + log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP2); + break; + } + } + + // HTTP request properties. + // TODO(mattklein123): Populate port field. + auto* request_properties = log_entry.mutable_request(); + if (request_headers.Scheme() != nullptr) { + request_properties->set_scheme(std::string(request_headers.Scheme()->value().getStringView())); + } + if (request_headers.Host() != nullptr) { + request_properties->set_authority(std::string(request_headers.Host()->value().getStringView())); + } + if (request_headers.Path() != nullptr) { + request_properties->set_path(std::string(request_headers.Path()->value().getStringView())); + } + if (request_headers.UserAgent() != nullptr) { + request_properties->set_user_agent( + std::string(request_headers.UserAgent()->value().getStringView())); + } + if (request_headers.Referer() != nullptr) { + request_properties->set_referer( + std::string(request_headers.Referer()->value().getStringView())); + } + if (request_headers.ForwardedFor() != nullptr) { + request_properties->set_forwarded_for( + std::string(request_headers.ForwardedFor()->value().getStringView())); + } + if (request_headers.RequestId() != nullptr) { + request_properties->set_request_id( + std::string(request_headers.RequestId()->value().getStringView())); + } + if (request_headers.EnvoyOriginalPath() != nullptr) { + request_properties->set_original_path( + std::string(request_headers.EnvoyOriginalPath()->value().getStringView())); + } + request_properties->set_request_headers_bytes(request_headers.byteSize()); + request_properties->set_request_body_bytes(stream_info.bytesReceived()); + if (request_headers.Method() != nullptr) { + envoy::api::v2::core::RequestMethod method = + envoy::api::v2::core::RequestMethod::METHOD_UNSPECIFIED; + envoy::api::v2::core::RequestMethod_Parse( + std::string(request_headers.Method()->value().getStringView()), &method); + request_properties->set_request_method(method); + } + if (!request_headers_to_log_.empty()) { + auto* logged_headers = request_properties->mutable_request_headers(); + + for (const auto& header : request_headers_to_log_) { + const Http::HeaderEntry* entry = request_headers.get(header); + if (entry != nullptr) { + logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); + } + } + } + + // HTTP response properties. + auto* response_properties = log_entry.mutable_response(); + if (stream_info.responseCode()) { + response_properties->mutable_response_code()->set_value(stream_info.responseCode().value()); + } + if (stream_info.responseCodeDetails()) { + response_properties->set_response_code_details(stream_info.responseCodeDetails().value()); + } + response_properties->set_response_headers_bytes(response_headers.byteSize()); + response_properties->set_response_body_bytes(stream_info.bytesSent()); + if (!response_headers_to_log_.empty()) { + auto* logged_headers = response_properties->mutable_response_headers(); + + for (const auto& header : response_headers_to_log_) { + const Http::HeaderEntry* entry = response_headers.get(header); + if (entry != nullptr) { + logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); + } + } + } + + if (!response_trailers_to_log_.empty()) { + auto* logged_headers = response_properties->mutable_response_trailers(); + + for (const auto& header : response_trailers_to_log_) { + const Http::HeaderEntry* entry = response_trailers.get(header); + if (entry != nullptr) { + logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); + } + } + } + + tls_slot_->getTyped().logger_->log(std::move(log_entry)); +} + +} // namespace HttpGrpc +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h new file mode 100644 index 000000000000..6fa2b505eac3 --- /dev/null +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +#include "envoy/config/accesslog/v2/als.pb.h" +#include "envoy/config/filter/accesslog/v2/accesslog.pb.h" +#include "envoy/grpc/async_client.h" +#include "envoy/grpc/async_client_manager.h" +#include "envoy/local_info/local_info.h" +#include "envoy/service/accesslog/v2/als.pb.h" +#include "envoy/singleton/instance.h" +#include "envoy/thread_local/thread_local.h" + +#include "common/grpc/typed_async_client.h" + +#include "extensions/access_loggers/common/access_log_base.h" +#include "extensions/access_loggers/grpc/grpc_access_log_impl.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace HttpGrpc { + +// TODO(mattklein123): Stats + +/** + * Access log Instance that streams HTTP logs over gRPC. + */ +class HttpGrpcAccessLog : public Common::ImplBase { +public: + HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, + envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config, + ThreadLocal::SlotAllocator& tls, + GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache); + +private: + /** + * Per-thread cached logger. + */ + struct ThreadLocalLogger : public ThreadLocal::ThreadLocalObject { + ThreadLocalLogger(GrpcCommon::GrpcAccessLoggerSharedPtr logger); + + const GrpcCommon::GrpcAccessLoggerSharedPtr logger_; + }; + + // Common::ImplBase + void emitLog(const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers, + const Http::HeaderMap& response_trailers, + const StreamInfo::StreamInfo& stream_info) override; + + const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config_; + const ThreadLocal::SlotPtr tls_slot_; + const GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache_; + std::vector request_headers_to_log_; + std::vector response_headers_to_log_; + std::vector response_trailers_to_log_; +}; + +} // namespace HttpGrpc +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc deleted file mode 100644 index 17a443881c5c..000000000000 --- a/source/extensions/access_loggers/http_grpc/grpc_access_log_impl.cc +++ /dev/null @@ -1,462 +0,0 @@ -#include "extensions/access_loggers/http_grpc/grpc_access_log_impl.h" - -#include "envoy/upstream/upstream.h" - -#include "common/common/assert.h" -#include "common/network/utility.h" -#include "common/stream_info/utility.h" - -namespace Envoy { -namespace Extensions { -namespace AccessLoggers { -namespace HttpGrpc { - -namespace { - -using namespace envoy::data::accesslog::v2; - -// Helper function to convert from a BoringSSL textual representation of the -// TLS version to the corresponding enum value used in gRPC access logs. -TLSProperties_TLSVersion tlsVersionStringToEnum(const std::string& tls_version) { - if (tls_version == "TLSv1") { - return TLSProperties_TLSVersion_TLSv1; - } else if (tls_version == "TLSv1.1") { - return TLSProperties_TLSVersion_TLSv1_1; - } else if (tls_version == "TLSv1.2") { - return TLSProperties_TLSVersion_TLSv1_2; - } else if (tls_version == "TLSv1.3") { - return TLSProperties_TLSVersion_TLSv1_3; - } - - return TLSProperties_TLSVersion_VERSION_UNSPECIFIED; -} -}; // namespace - -void GrpcAccessLoggerImpl::LocalStream::onRemoteClose(Grpc::Status::GrpcStatus, - const std::string&) { - ASSERT(parent_.stream_ != absl::nullopt); - if (parent_.stream_->stream_ != nullptr) { - // Only reset if we have a stream. Otherwise we had an inline failure and we will clear the - // stream data in send(). - parent_.stream_.reset(); - } -} - -GrpcAccessLoggerImpl::GrpcAccessLoggerImpl(Grpc::RawAsyncClientPtr&& client, std::string log_name, - std::chrono::milliseconds buffer_flush_interval_msec, - uint64_t buffer_size_bytes, - Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info) - : client_(std::move(client)), log_name_(log_name), - buffer_flush_interval_msec_(buffer_flush_interval_msec), - flush_timer_(dispatcher.createTimer([this]() { - flush(); - flush_timer_->enableTimer(buffer_flush_interval_msec_); - })), - buffer_size_bytes_(buffer_size_bytes), local_info_(local_info) { - flush_timer_->enableTimer(buffer_flush_interval_msec_); -} - -void GrpcAccessLoggerImpl::log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) { - approximate_message_size_bytes_ += entry.ByteSizeLong(); - message_.mutable_http_logs()->add_log_entry()->Swap(&entry); - if (approximate_message_size_bytes_ >= buffer_size_bytes_) { - flush(); - } -} - -void GrpcAccessLoggerImpl::flush() { - if (!message_.has_http_logs()) { - // Nothing to flush. - return; - } - - if (stream_ == absl::nullopt) { - stream_.emplace(*this); - } - - if (stream_->stream_ == nullptr) { - stream_->stream_ = - client_->start(*Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs"), - *stream_); - - auto* identifier = message_.mutable_identifier(); - *identifier->mutable_node() = local_info_.node(); - identifier->set_log_name(log_name_); - } - - if (stream_->stream_ != nullptr) { - stream_->stream_->sendMessage(message_, false); - } else { - // Clear out the stream data due to stream creation failure. - stream_.reset(); - } - - // Clear the message regardless of the success. - approximate_message_size_bytes_ = 0; - message_.Clear(); -} - -GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& async_client_manager, - Stats::Scope& scope, - ThreadLocal::SlotAllocator& tls, - const LocalInfo::LocalInfo& local_info) - : async_client_manager_(async_client_manager), scope_(scope), tls_slot_(tls.allocateSlot()), - local_info_(local_info) { - tls_slot_->set( - [](Event::Dispatcher& dispatcher) { return std::make_shared(dispatcher); }); -} - -GrpcAccessLoggerSharedPtr GrpcAccessLoggerCacheImpl::getOrCreateLogger( - const envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) { - // TODO(euroelessar): Consider cleaning up loggers. - auto& cache = tls_slot_->getTyped(); - const std::size_t cache_key = MessageUtil::hash(config); - const auto it = cache.access_loggers_.find(cache_key); - if (it != cache.access_loggers_.end()) { - return it->second; - } - const Grpc::AsyncClientFactoryPtr factory = - async_client_manager_.factoryForGrpcService(config.grpc_service(), scope_, false); - const GrpcAccessLoggerSharedPtr logger = std::make_shared( - factory->create(), config.log_name(), - std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, buffer_flush_interval, 1000)), - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, buffer_size_bytes, 16384), cache.dispatcher_, - local_info_); - cache.access_loggers_.emplace(cache_key, logger); - return logger; -} - -HttpGrpcAccessLog::ThreadLocalLogger::ThreadLocalLogger(GrpcAccessLoggerSharedPtr logger) - : logger_(std::move(logger)) {} - -HttpGrpcAccessLog::HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, - envoy::config::accesslog::v2::HttpGrpcAccessLogConfig config, - ThreadLocal::SlotAllocator& tls, - GrpcAccessLoggerCacheSharedPtr access_logger_cache) - : Common::ImplBase(std::move(filter)), config_(std::move(config)), - tls_slot_(tls.allocateSlot()), access_logger_cache_(std::move(access_logger_cache)) { - for (const auto& header : config_.additional_request_headers_to_log()) { - request_headers_to_log_.emplace_back(header); - } - - for (const auto& header : config_.additional_response_headers_to_log()) { - response_headers_to_log_.emplace_back(header); - } - - for (const auto& header : config_.additional_response_trailers_to_log()) { - response_trailers_to_log_.emplace_back(header); - } - - tls_slot_->set([this](Event::Dispatcher&) { - return std::make_shared( - access_logger_cache_->getOrCreateLogger(config_.common_config())); - }); -} - -void HttpGrpcAccessLog::responseFlagsToAccessLogResponseFlags( - envoy::data::accesslog::v2::AccessLogCommon& common_access_log, - const StreamInfo::StreamInfo& stream_info) { - - static_assert(StreamInfo::ResponseFlag::LastFlag == 0x20000, - "A flag has been added. Fix this code."); - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::FailedLocalHealthCheck)) { - common_access_log.mutable_response_flags()->set_failed_local_healthcheck(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::NoHealthyUpstream)) { - common_access_log.mutable_response_flags()->set_no_healthy_upstream(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamRequestTimeout)) { - common_access_log.mutable_response_flags()->set_upstream_request_timeout(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::LocalReset)) { - common_access_log.mutable_response_flags()->set_local_reset(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamRemoteReset)) { - common_access_log.mutable_response_flags()->set_upstream_remote_reset(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamConnectionFailure)) { - common_access_log.mutable_response_flags()->set_upstream_connection_failure(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamConnectionTermination)) { - common_access_log.mutable_response_flags()->set_upstream_connection_termination(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamOverflow)) { - common_access_log.mutable_response_flags()->set_upstream_overflow(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::NoRouteFound)) { - common_access_log.mutable_response_flags()->set_no_route_found(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::DelayInjected)) { - common_access_log.mutable_response_flags()->set_delay_injected(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::FaultInjected)) { - common_access_log.mutable_response_flags()->set_fault_injected(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::RateLimited)) { - common_access_log.mutable_response_flags()->set_rate_limited(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UnauthorizedExternalService)) { - common_access_log.mutable_response_flags()->mutable_unauthorized_details()->set_reason( - envoy::data::accesslog::v2::ResponseFlags_Unauthorized_Reason:: - ResponseFlags_Unauthorized_Reason_EXTERNAL_SERVICE); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::RateLimitServiceError)) { - common_access_log.mutable_response_flags()->set_rate_limit_service_error(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::DownstreamConnectionTermination)) { - common_access_log.mutable_response_flags()->set_downstream_connection_termination(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::UpstreamRetryLimitExceeded)) { - common_access_log.mutable_response_flags()->set_upstream_retry_limit_exceeded(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::StreamIdleTimeout)) { - common_access_log.mutable_response_flags()->set_stream_idle_timeout(true); - } - - if (stream_info.hasResponseFlag(StreamInfo::ResponseFlag::InvalidEnvoyRequestHeaders)) { - common_access_log.mutable_response_flags()->set_invalid_envoy_request_headers(true); - } -} - -void HttpGrpcAccessLog::emitLog(const Http::HeaderMap& request_headers, - const Http::HeaderMap& response_headers, - const Http::HeaderMap& response_trailers, - const StreamInfo::StreamInfo& stream_info) { - // Common log properties. - // TODO(mattklein123): Populate sample_rate field. - envoy::data::accesslog::v2::HTTPAccessLogEntry log_entry; - auto* common_properties = log_entry.mutable_common_properties(); - - if (stream_info.downstreamRemoteAddress() != nullptr) { - Network::Utility::addressToProtobufAddress( - *stream_info.downstreamRemoteAddress(), - *common_properties->mutable_downstream_remote_address()); - } - if (stream_info.downstreamLocalAddress() != nullptr) { - Network::Utility::addressToProtobufAddress( - *stream_info.downstreamLocalAddress(), - *common_properties->mutable_downstream_local_address()); - } - if (stream_info.downstreamSslConnection() != nullptr) { - auto* tls_properties = common_properties->mutable_tls_properties(); - const auto* downstream_ssl_connection = stream_info.downstreamSslConnection(); - - tls_properties->set_tls_sni_hostname(stream_info.requestedServerName()); - - auto* local_properties = tls_properties->mutable_local_certificate_properties(); - for (const auto& uri_san : downstream_ssl_connection->uriSanLocalCertificate()) { - auto* local_san = local_properties->add_subject_alt_name(); - local_san->set_uri(uri_san); - } - local_properties->set_subject(downstream_ssl_connection->subjectLocalCertificate()); - - auto* peer_properties = tls_properties->mutable_peer_certificate_properties(); - for (const auto& uri_san : downstream_ssl_connection->uriSanPeerCertificate()) { - auto* peer_san = peer_properties->add_subject_alt_name(); - peer_san->set_uri(uri_san); - } - - peer_properties->set_subject(downstream_ssl_connection->subjectPeerCertificate()); - tls_properties->set_tls_session_id(downstream_ssl_connection->sessionId()); - tls_properties->set_tls_version( - tlsVersionStringToEnum(downstream_ssl_connection->tlsVersion())); - - auto* local_tls_cipher_suite = tls_properties->mutable_tls_cipher_suite(); - local_tls_cipher_suite->set_value(downstream_ssl_connection->ciphersuiteId()); - } - common_properties->mutable_start_time()->MergeFrom( - Protobuf::util::TimeUtil::NanosecondsToTimestamp( - std::chrono::duration_cast( - stream_info.startTime().time_since_epoch()) - .count())); - - absl::optional dur = stream_info.lastDownstreamRxByteReceived(); - if (dur) { - common_properties->mutable_time_to_last_rx_byte()->MergeFrom( - Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); - } - - dur = stream_info.firstUpstreamTxByteSent(); - if (dur) { - common_properties->mutable_time_to_first_upstream_tx_byte()->MergeFrom( - Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); - } - - dur = stream_info.lastUpstreamTxByteSent(); - if (dur) { - common_properties->mutable_time_to_last_upstream_tx_byte()->MergeFrom( - Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); - } - - dur = stream_info.firstUpstreamRxByteReceived(); - if (dur) { - common_properties->mutable_time_to_first_upstream_rx_byte()->MergeFrom( - Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); - } - - dur = stream_info.lastUpstreamRxByteReceived(); - if (dur) { - common_properties->mutable_time_to_last_upstream_rx_byte()->MergeFrom( - Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); - } - - dur = stream_info.firstDownstreamTxByteSent(); - if (dur) { - common_properties->mutable_time_to_first_downstream_tx_byte()->MergeFrom( - Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); - } - - dur = stream_info.lastDownstreamTxByteSent(); - if (dur) { - common_properties->mutable_time_to_last_downstream_tx_byte()->MergeFrom( - Protobuf::util::TimeUtil::NanosecondsToDuration(dur.value().count())); - } - - if (stream_info.upstreamHost() != nullptr) { - Network::Utility::addressToProtobufAddress( - *stream_info.upstreamHost()->address(), - *common_properties->mutable_upstream_remote_address()); - common_properties->set_upstream_cluster(stream_info.upstreamHost()->cluster().name()); - } - - if (!stream_info.getRouteName().empty()) { - common_properties->set_route_name(stream_info.getRouteName()); - } - - if (stream_info.upstreamLocalAddress() != nullptr) { - Network::Utility::addressToProtobufAddress( - *stream_info.upstreamLocalAddress(), *common_properties->mutable_upstream_local_address()); - } - responseFlagsToAccessLogResponseFlags(*common_properties, stream_info); - if (!stream_info.upstreamTransportFailureReason().empty()) { - common_properties->set_upstream_transport_failure_reason( - stream_info.upstreamTransportFailureReason()); - } - if (stream_info.dynamicMetadata().filter_metadata_size() > 0) { - common_properties->mutable_metadata()->MergeFrom(stream_info.dynamicMetadata()); - } - - if (stream_info.protocol()) { - switch (stream_info.protocol().value()) { - case Http::Protocol::Http10: - log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP10); - break; - case Http::Protocol::Http11: - log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP11); - break; - case Http::Protocol::Http2: - log_entry.set_protocol_version(envoy::data::accesslog::v2::HTTPAccessLogEntry::HTTP2); - break; - } - } - - // HTTP request properties. - // TODO(mattklein123): Populate port field. - auto* request_properties = log_entry.mutable_request(); - if (request_headers.Scheme() != nullptr) { - request_properties->set_scheme(std::string(request_headers.Scheme()->value().getStringView())); - } - if (request_headers.Host() != nullptr) { - request_properties->set_authority(std::string(request_headers.Host()->value().getStringView())); - } - if (request_headers.Path() != nullptr) { - request_properties->set_path(std::string(request_headers.Path()->value().getStringView())); - } - if (request_headers.UserAgent() != nullptr) { - request_properties->set_user_agent( - std::string(request_headers.UserAgent()->value().getStringView())); - } - if (request_headers.Referer() != nullptr) { - request_properties->set_referer( - std::string(request_headers.Referer()->value().getStringView())); - } - if (request_headers.ForwardedFor() != nullptr) { - request_properties->set_forwarded_for( - std::string(request_headers.ForwardedFor()->value().getStringView())); - } - if (request_headers.RequestId() != nullptr) { - request_properties->set_request_id( - std::string(request_headers.RequestId()->value().getStringView())); - } - if (request_headers.EnvoyOriginalPath() != nullptr) { - request_properties->set_original_path( - std::string(request_headers.EnvoyOriginalPath()->value().getStringView())); - } - request_properties->set_request_headers_bytes(request_headers.byteSize()); - request_properties->set_request_body_bytes(stream_info.bytesReceived()); - if (request_headers.Method() != nullptr) { - envoy::api::v2::core::RequestMethod method = - envoy::api::v2::core::RequestMethod::METHOD_UNSPECIFIED; - envoy::api::v2::core::RequestMethod_Parse( - std::string(request_headers.Method()->value().getStringView()), &method); - request_properties->set_request_method(method); - } - if (!request_headers_to_log_.empty()) { - auto* logged_headers = request_properties->mutable_request_headers(); - - for (const auto& header : request_headers_to_log_) { - const Http::HeaderEntry* entry = request_headers.get(header); - if (entry != nullptr) { - logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); - } - } - } - - // HTTP response properties. - auto* response_properties = log_entry.mutable_response(); - if (stream_info.responseCode()) { - response_properties->mutable_response_code()->set_value(stream_info.responseCode().value()); - } - if (stream_info.responseCodeDetails()) { - response_properties->set_response_code_details(stream_info.responseCodeDetails().value()); - } - response_properties->set_response_headers_bytes(response_headers.byteSize()); - response_properties->set_response_body_bytes(stream_info.bytesSent()); - if (!response_headers_to_log_.empty()) { - auto* logged_headers = response_properties->mutable_response_headers(); - - for (const auto& header : response_headers_to_log_) { - const Http::HeaderEntry* entry = response_headers.get(header); - if (entry != nullptr) { - logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); - } - } - } - - if (!response_trailers_to_log_.empty()) { - auto* logged_headers = response_properties->mutable_response_trailers(); - - for (const auto& header : response_trailers_to_log_) { - const Http::HeaderEntry* entry = response_trailers.get(header); - if (entry != nullptr) { - logged_headers->insert({header.get(), std::string(entry->value().getStringView())}); - } - } - } - - tls_slot_->getTyped().logger_->log(std::move(log_entry)); -} - -} // namespace HttpGrpc -} // namespace AccessLoggers -} // namespace Extensions -} // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 95627187f702..f8b1ccc5158f 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -5,7 +5,7 @@ EXTENSIONS = { # "envoy.access_loggers.file": "//source/extensions/access_loggers/file:config", - "envoy.access_loggers.http_grpc": "//source/extensions/access_loggers/http_grpc:config", + "envoy.access_loggers.http_grpc": "//source/extensions/access_loggers/grpc:http_config", # # Clusters @@ -148,7 +148,7 @@ WINDOWS_EXTENSIONS = { # "envoy.access_loggers.file": "//source/extensions/access_loggers/file:config", - #"envoy.access_loggers.http_grpc": "//source/extensions/access_loggers/http_grpc:config", + #"envoy.access_loggers.http_grpc": "//source/extensions/access_loggers/grpc:http_config", # # gRPC Credentials Plugins diff --git a/test/extensions/access_loggers/grpc/BUILD b/test/extensions/access_loggers/grpc/BUILD new file mode 100644 index 000000000000..9e2c7b46240f --- /dev/null +++ b/test/extensions/access_loggers/grpc/BUILD @@ -0,0 +1,79 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "grpc_access_log_impl_test", + srcs = ["grpc_access_log_impl_test.cc"], + extension_name = "envoy.access_loggers.http_grpc", + deps = [ + "//source/extensions/access_loggers/grpc:http_grpc_access_log_lib", + "//test/mocks/access_log:access_log_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/ssl:ssl_mocks", + "//test/mocks/stream_info:stream_info_mocks", + "//test/mocks/thread_local:thread_local_mocks", + ], +) + +envoy_extension_cc_test( + name = "grpc_access_log_utils_test", + srcs = ["grpc_access_log_utils_test.cc"], + extension_name = "envoy.access_loggers.http_grpc", + deps = [ + "//source/extensions/access_loggers/grpc:grpc_access_log_utils", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/ssl:ssl_mocks", + "//test/mocks/stream_info:stream_info_mocks", + ], +) + +envoy_extension_cc_test( + name = "http_grpc_access_log_impl_test", + srcs = ["http_grpc_access_log_impl_test.cc"], + extension_name = "envoy.access_loggers.http_grpc", + deps = [ + "//source/extensions/access_loggers/grpc:http_grpc_access_log_lib", + "//test/mocks/access_log:access_log_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/ssl:ssl_mocks", + "//test/mocks/stream_info:stream_info_mocks", + "//test/mocks/thread_local:thread_local_mocks", + ], +) + +envoy_extension_cc_test( + name = "http_config_test", + srcs = ["http_config_test.cc"], + extension_name = "envoy.access_loggers.http_grpc", + deps = [ + "//source/extensions/access_loggers/grpc:http_config", + "//test/mocks/server:server_mocks", + ], +) + +envoy_extension_cc_test( + name = "http_grpc_access_log_integration_test", + srcs = ["http_grpc_access_log_integration_test.cc"], + extension_name = "envoy.access_loggers.http_grpc", + deps = [ + "//source/common/buffer:zero_copy_input_stream_lib", + "//source/common/grpc:codec_lib", + "//source/common/grpc:common_lib", + "//source/extensions/access_loggers/grpc:http_config", + "//test/common/grpc:grpc_client_integration_lib", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc new file mode 100644 index 000000000000..571d29fe48dd --- /dev/null +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -0,0 +1,295 @@ +#include + +#include "common/buffer/zero_copy_input_stream_impl.h" +#include "common/network/address_impl.h" + +#include "extensions/access_loggers/grpc/http_grpc_access_log_impl.h" + +#include "test/mocks/access_log/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/ssl/mocks.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/mocks/thread_local/mocks.h" + +using testing::_; +using testing::InSequence; +using testing::Invoke; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace GrpcCommon { +namespace { + +constexpr std::chrono::milliseconds FlushInterval(10); + +class GrpcAccessLoggerImplTest : public testing::Test { +public: + using MockAccessLogStream = Grpc::MockAsyncStream; + using AccessLogCallbacks = + Grpc::AsyncStreamCallbacks; + + void initLogger(std::chrono::milliseconds buffer_flush_interval_msec, size_t buffer_size_bytes) { + timer_ = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*timer_, enableTimer(buffer_flush_interval_msec)); + logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, + log_name_, buffer_flush_interval_msec, + buffer_size_bytes, dispatcher_, local_info_); + } + + void expectStreamStart(MockAccessLogStream& stream, AccessLogCallbacks** callbacks_to_set) { + EXPECT_CALL(*async_client_, startRaw(_, _, _)) + .WillOnce(Invoke([&stream, callbacks_to_set](absl::string_view, absl::string_view, + Grpc::RawAsyncStreamCallbacks& callbacks) { + *callbacks_to_set = dynamic_cast(&callbacks); + return &stream; + })); + } + + void expectStreamMessage(MockAccessLogStream& stream, const std::string& expected_message_yaml) { + envoy::service::accesslog::v2::StreamAccessLogsMessage expected_message; + TestUtility::loadFromYaml(expected_message_yaml, expected_message); + EXPECT_CALL(stream, sendMessageRaw_(_, false)) + .WillOnce(Invoke([expected_message](Buffer::InstancePtr& request, bool) { + envoy::service::accesslog::v2::StreamAccessLogsMessage message; + Buffer::ZeroCopyInputStreamImpl request_stream(std::move(request)); + EXPECT_TRUE(message.ParseFromZeroCopyStream(&request_stream)); + EXPECT_EQ(message.DebugString(), expected_message.DebugString()); + })); + } + + std::string log_name_ = "test_log_name"; + LocalInfo::MockLocalInfo local_info_; + Event::MockTimer* timer_ = nullptr; + Event::MockDispatcher dispatcher_; + Grpc::MockAsyncClient* async_client_{new Grpc::MockAsyncClient}; + std::unique_ptr logger_; +}; + +// Test basic stream logging flow. +TEST_F(GrpcAccessLoggerImplTest, BasicFlow) { + InSequence s; + initLogger(FlushInterval, 0); + + // Start a stream for the first log. + MockAccessLogStream stream; + AccessLogCallbacks* callbacks; + expectStreamStart(stream, &callbacks); + EXPECT_CALL(local_info_, node()); + expectStreamMessage(stream, R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name +http_logs: + log_entry: + request: + path: /test/path1 +)EOF"); + envoy::data::accesslog::v2::HTTPAccessLogEntry entry; + entry.mutable_request()->set_path("/test/path1"); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + + expectStreamMessage(stream, R"EOF( +http_logs: + log_entry: + request: + path: /test/path2 +)EOF"); + entry.mutable_request()->set_path("/test/path2"); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + + // Verify that sending an empty response message doesn't do anything bad. + callbacks->onReceiveMessage( + std::make_unique()); + + // Close the stream and make sure we make a new one. + callbacks->onRemoteClose(Grpc::Status::Internal, "bad"); + expectStreamStart(stream, &callbacks); + EXPECT_CALL(local_info_, node()); + expectStreamMessage(stream, R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name +http_logs: + log_entry: + request: + path: /test/path3 +)EOF"); + entry.mutable_request()->set_path("/test/path3"); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); +} + +// Test that stream failure is handled correctly. +TEST_F(GrpcAccessLoggerImplTest, StreamFailure) { + InSequence s; + initLogger(FlushInterval, 0); + + EXPECT_CALL(*async_client_, startRaw(_, _, _)) + .WillOnce(Invoke( + [](absl::string_view, absl::string_view, Grpc::RawAsyncStreamCallbacks& callbacks) { + callbacks.onRemoteClose(Grpc::Status::Internal, "bad"); + return nullptr; + })); + EXPECT_CALL(local_info_, node()); + envoy::data::accesslog::v2::HTTPAccessLogEntry entry; + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); +} + +// Test that log entries are batched. +TEST_F(GrpcAccessLoggerImplTest, Batching) { + InSequence s; + initLogger(FlushInterval, 100); + + MockAccessLogStream stream; + AccessLogCallbacks* callbacks; + expectStreamStart(stream, &callbacks); + EXPECT_CALL(local_info_, node()); + const std::string path1(30, '1'); + const std::string path2(30, '2'); + const std::string path3(80, '3'); + expectStreamMessage(stream, fmt::format(R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name +http_logs: + log_entry: + - request: + path: "{}" + - request: + path: "{}" + - request: + path: "{}" +)EOF", + path1, path2, path3)); + envoy::data::accesslog::v2::HTTPAccessLogEntry entry; + entry.mutable_request()->set_path(path1); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + entry.mutable_request()->set_path(path2); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + entry.mutable_request()->set_path(path3); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + + const std::string path4(120, '4'); + expectStreamMessage(stream, fmt::format(R"EOF( +http_logs: + log_entry: + request: + path: "{}" +)EOF", + path4)); + entry.mutable_request()->set_path(path4); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); +} + +// Test that log entries are flushed periodically. +TEST_F(GrpcAccessLoggerImplTest, Flushing) { + InSequence s; + initLogger(FlushInterval, 100); + + // Nothing to do yet. + EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + timer_->invokeCallback(); + + envoy::data::accesslog::v2::HTTPAccessLogEntry entry; + // Not enough data yet to trigger flush on batch size. + entry.mutable_request()->set_path("/test/path1"); + logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); + + MockAccessLogStream stream; + AccessLogCallbacks* callbacks; + expectStreamStart(stream, &callbacks); + EXPECT_CALL(local_info_, node()); + expectStreamMessage(stream, fmt::format(R"EOF( + identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + log_name: test_log_name + http_logs: + log_entry: + - request: + path: /test/path1 + )EOF")); + EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + timer_->invokeCallback(); + + // Flush on empty message does nothing. + EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + timer_->invokeCallback(); +} + +class GrpcAccessLoggerCacheImplTest : public testing::Test { +public: + GrpcAccessLoggerCacheImplTest() { + logger_cache_ = std::make_unique(async_client_manager_, scope_, tls_, + local_info_); + } + + void expectClientCreation() { + factory_ = new Grpc::MockAsyncClientFactory; + async_client_ = new Grpc::MockAsyncClient; + EXPECT_CALL(async_client_manager_, factoryForGrpcService(_, _, false)) + .WillOnce(Invoke([this](const envoy::api::v2::core::GrpcService&, Stats::Scope&, bool) { + EXPECT_CALL(*factory_, create()).WillOnce(Invoke([this] { + return Grpc::RawAsyncClientPtr{async_client_}; + })); + return Grpc::AsyncClientFactoryPtr{factory_}; + })); + } + + LocalInfo::MockLocalInfo local_info_; + NiceMock tls_; + Grpc::MockAsyncClientManager async_client_manager_; + Grpc::MockAsyncClient* async_client_ = nullptr; + Grpc::MockAsyncClientFactory* factory_ = nullptr; + std::unique_ptr logger_cache_; + NiceMock scope_; +}; + +TEST_F(GrpcAccessLoggerCacheImplTest, Deduplication) { + InSequence s; + + ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig config; + config.set_log_name("log-1"); + config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("cluster-1"); + + expectClientCreation(); + GrpcAccessLoggerSharedPtr logger1 = logger_cache_->getOrCreateLogger(config); + EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config)); + + // Changing log name leads to another logger. + config.set_log_name("log-2"); + expectClientCreation(); + EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config)); + + config.set_log_name("log-1"); + EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config)); + + // Changing cluster name leads to another logger. + config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("cluster-2"); + expectClientCreation(); + EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config)); +} + +} // namespace +} // namespace GrpcCommon +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_utils_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_utils_test.cc new file mode 100644 index 000000000000..a03e329f5705 --- /dev/null +++ b/test/extensions/access_loggers/grpc/grpc_access_log_utils_test.cc @@ -0,0 +1,51 @@ +#include "envoy/data/accesslog/v2/accesslog.pb.h" + +#include "extensions/access_loggers/grpc/grpc_access_log_utils.h" + +#include "test/mocks/stream_info/mocks.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace GrpcCommon { +namespace { + +using testing::_; +using testing::Return; + +TEST(UtilityResponseFlagsToAccessLogResponseFlagsTest, All) { + NiceMock stream_info; + ON_CALL(stream_info, hasResponseFlag(_)).WillByDefault(Return(true)); + envoy::data::accesslog::v2::AccessLogCommon common_access_log; + Utility::responseFlagsToAccessLogResponseFlags(common_access_log, stream_info); + + envoy::data::accesslog::v2::AccessLogCommon common_access_log_expected; + common_access_log_expected.mutable_response_flags()->set_failed_local_healthcheck(true); + common_access_log_expected.mutable_response_flags()->set_no_healthy_upstream(true); + common_access_log_expected.mutable_response_flags()->set_upstream_request_timeout(true); + common_access_log_expected.mutable_response_flags()->set_local_reset(true); + common_access_log_expected.mutable_response_flags()->set_upstream_remote_reset(true); + common_access_log_expected.mutable_response_flags()->set_upstream_connection_failure(true); + common_access_log_expected.mutable_response_flags()->set_upstream_connection_termination(true); + common_access_log_expected.mutable_response_flags()->set_upstream_overflow(true); + common_access_log_expected.mutable_response_flags()->set_no_route_found(true); + common_access_log_expected.mutable_response_flags()->set_delay_injected(true); + common_access_log_expected.mutable_response_flags()->set_fault_injected(true); + common_access_log_expected.mutable_response_flags()->set_rate_limited(true); + common_access_log_expected.mutable_response_flags()->mutable_unauthorized_details()->set_reason( + envoy::data::accesslog::v2::ResponseFlags_Unauthorized_Reason:: + ResponseFlags_Unauthorized_Reason_EXTERNAL_SERVICE); + common_access_log_expected.mutable_response_flags()->set_rate_limit_service_error(true); + common_access_log_expected.mutable_response_flags()->set_downstream_connection_termination(true); + common_access_log_expected.mutable_response_flags()->set_upstream_retry_limit_exceeded(true); + common_access_log_expected.mutable_response_flags()->set_stream_idle_timeout(true); + common_access_log_expected.mutable_response_flags()->set_invalid_envoy_request_headers(true); + + EXPECT_EQ(common_access_log_expected.DebugString(), common_access_log.DebugString()); +} + +} // namespace +} // namespace GrpcCommon +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/access_loggers/http_grpc/config_test.cc b/test/extensions/access_loggers/grpc/http_config_test.cc similarity index 96% rename from test/extensions/access_loggers/http_grpc/config_test.cc rename to test/extensions/access_loggers/grpc/http_config_test.cc index 6cc54924f80a..7cd5bbc00f97 100644 --- a/test/extensions/access_loggers/http_grpc/config_test.cc +++ b/test/extensions/access_loggers/grpc/http_config_test.cc @@ -2,7 +2,7 @@ #include "envoy/server/access_log_config.h" #include "envoy/stats/scope.h" -#include "extensions/access_loggers/http_grpc/grpc_access_log_impl.h" +#include "extensions/access_loggers/grpc/http_grpc_access_log_impl.h" #include "extensions/access_loggers/well_known_names.h" #include "test/mocks/server/mocks.h" diff --git a/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc similarity index 61% rename from test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc rename to test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index ecf1e048fd5d..22e1f7cd59c0 100644 --- a/test/extensions/access_loggers/http_grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -3,7 +3,7 @@ #include "common/buffer/zero_copy_input_stream_impl.h" #include "common/network/address_impl.h" -#include "extensions/access_loggers/http_grpc/grpc_access_log_impl.h" +#include "extensions/access_loggers/grpc/http_grpc_access_log_impl.h" #include "test/mocks/access_log/mocks.h" #include "test/mocks/grpc/mocks.h" @@ -25,281 +25,17 @@ namespace AccessLoggers { namespace HttpGrpc { namespace { -constexpr std::chrono::milliseconds FlushInterval(10); - -class GrpcAccessLoggerImplTest : public testing::Test { -public: - using MockAccessLogStream = Grpc::MockAsyncStream; - using AccessLogCallbacks = - Grpc::AsyncStreamCallbacks; - - void initLogger(std::chrono::milliseconds buffer_flush_interval_msec, size_t buffer_size_bytes) { - timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*timer_, enableTimer(buffer_flush_interval_msec)); - logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, - log_name_, buffer_flush_interval_msec, - buffer_size_bytes, dispatcher_, local_info_); - } - - void expectStreamStart(MockAccessLogStream& stream, AccessLogCallbacks** callbacks_to_set) { - EXPECT_CALL(*async_client_, startRaw(_, _, _)) - .WillOnce(Invoke([&stream, callbacks_to_set](absl::string_view, absl::string_view, - Grpc::RawAsyncStreamCallbacks& callbacks) { - *callbacks_to_set = dynamic_cast(&callbacks); - return &stream; - })); - } - - void expectStreamMessage(MockAccessLogStream& stream, const std::string& expected_message_yaml) { - envoy::service::accesslog::v2::StreamAccessLogsMessage expected_message; - TestUtility::loadFromYaml(expected_message_yaml, expected_message); - EXPECT_CALL(stream, sendMessageRaw_(_, false)) - .WillOnce(Invoke([expected_message](Buffer::InstancePtr& request, bool) { - envoy::service::accesslog::v2::StreamAccessLogsMessage message; - Buffer::ZeroCopyInputStreamImpl request_stream(std::move(request)); - EXPECT_TRUE(message.ParseFromZeroCopyStream(&request_stream)); - EXPECT_EQ(message.DebugString(), expected_message.DebugString()); - })); - } - - std::string log_name_ = "test_log_name"; - LocalInfo::MockLocalInfo local_info_; - Event::MockTimer* timer_ = nullptr; - Event::MockDispatcher dispatcher_; - Grpc::MockAsyncClient* async_client_{new Grpc::MockAsyncClient}; - std::unique_ptr logger_; -}; - -// Test basic stream logging flow. -TEST_F(GrpcAccessLoggerImplTest, BasicFlow) { - InSequence s; - initLogger(FlushInterval, 0); - - // Start a stream for the first log. - MockAccessLogStream stream; - AccessLogCallbacks* callbacks; - expectStreamStart(stream, &callbacks); - EXPECT_CALL(local_info_, node()); - expectStreamMessage(stream, R"EOF( -identifier: - node: - id: node_name - cluster: cluster_name - locality: - zone: zone_name - log_name: test_log_name -http_logs: - log_entry: - request: - path: /test/path1 -)EOF"); - envoy::data::accesslog::v2::HTTPAccessLogEntry entry; - entry.mutable_request()->set_path("/test/path1"); - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); - - expectStreamMessage(stream, R"EOF( -http_logs: - log_entry: - request: - path: /test/path2 -)EOF"); - entry.mutable_request()->set_path("/test/path2"); - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); - - // Verify that sending an empty response message doesn't do anything bad. - callbacks->onReceiveMessage( - std::make_unique()); - - // Close the stream and make sure we make a new one. - callbacks->onRemoteClose(Grpc::Status::Internal, "bad"); - expectStreamStart(stream, &callbacks); - EXPECT_CALL(local_info_, node()); - expectStreamMessage(stream, R"EOF( -identifier: - node: - id: node_name - cluster: cluster_name - locality: - zone: zone_name - log_name: test_log_name -http_logs: - log_entry: - request: - path: /test/path3 -)EOF"); - entry.mutable_request()->set_path("/test/path3"); - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); -} - -// Test that stream failure is handled correctly. -TEST_F(GrpcAccessLoggerImplTest, StreamFailure) { - InSequence s; - initLogger(FlushInterval, 0); - - EXPECT_CALL(*async_client_, startRaw(_, _, _)) - .WillOnce(Invoke( - [](absl::string_view, absl::string_view, Grpc::RawAsyncStreamCallbacks& callbacks) { - callbacks.onRemoteClose(Grpc::Status::Internal, "bad"); - return nullptr; - })); - EXPECT_CALL(local_info_, node()); - envoy::data::accesslog::v2::HTTPAccessLogEntry entry; - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); -} - -// Test that log entries are batched. -TEST_F(GrpcAccessLoggerImplTest, Batching) { - InSequence s; - initLogger(FlushInterval, 100); - - MockAccessLogStream stream; - AccessLogCallbacks* callbacks; - expectStreamStart(stream, &callbacks); - EXPECT_CALL(local_info_, node()); - const std::string path1(30, '1'); - const std::string path2(30, '2'); - const std::string path3(80, '3'); - expectStreamMessage(stream, fmt::format(R"EOF( -identifier: - node: - id: node_name - cluster: cluster_name - locality: - zone: zone_name - log_name: test_log_name -http_logs: - log_entry: - - request: - path: "{}" - - request: - path: "{}" - - request: - path: "{}" -)EOF", - path1, path2, path3)); - envoy::data::accesslog::v2::HTTPAccessLogEntry entry; - entry.mutable_request()->set_path(path1); - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); - entry.mutable_request()->set_path(path2); - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); - entry.mutable_request()->set_path(path3); - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); - - const std::string path4(120, '4'); - expectStreamMessage(stream, fmt::format(R"EOF( -http_logs: - log_entry: - request: - path: "{}" -)EOF", - path4)); - entry.mutable_request()->set_path(path4); - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); -} - -// Test that log entries are flushed periodically. -TEST_F(GrpcAccessLoggerImplTest, Flushing) { - InSequence s; - initLogger(FlushInterval, 100); - - // Nothing to do yet. - EXPECT_CALL(*timer_, enableTimer(FlushInterval)); - timer_->invokeCallback(); - - envoy::data::accesslog::v2::HTTPAccessLogEntry entry; - // Not enough data yet to trigger flush on batch size. - entry.mutable_request()->set_path("/test/path1"); - logger_->log(envoy::data::accesslog::v2::HTTPAccessLogEntry(entry)); - - MockAccessLogStream stream; - AccessLogCallbacks* callbacks; - expectStreamStart(stream, &callbacks); - EXPECT_CALL(local_info_, node()); - expectStreamMessage(stream, fmt::format(R"EOF( - identifier: - node: - id: node_name - cluster: cluster_name - locality: - zone: zone_name - log_name: test_log_name - http_logs: - log_entry: - - request: - path: /test/path1 - )EOF")); - EXPECT_CALL(*timer_, enableTimer(FlushInterval)); - timer_->invokeCallback(); - - // Flush on empty message does nothing. - EXPECT_CALL(*timer_, enableTimer(FlushInterval)); - timer_->invokeCallback(); -} - -class GrpcAccessLoggerCacheImplTest : public testing::Test { -public: - GrpcAccessLoggerCacheImplTest() { - logger_cache_ = std::make_unique(async_client_manager_, scope_, tls_, - local_info_); - } - - void expectClientCreation() { - factory_ = new Grpc::MockAsyncClientFactory; - async_client_ = new Grpc::MockAsyncClient; - EXPECT_CALL(async_client_manager_, factoryForGrpcService(_, _, false)) - .WillOnce(Invoke([this](const envoy::api::v2::core::GrpcService&, Stats::Scope&, bool) { - EXPECT_CALL(*factory_, create()).WillOnce(Invoke([this] { - return Grpc::RawAsyncClientPtr{async_client_}; - })); - return Grpc::AsyncClientFactoryPtr{factory_}; - })); - } - - LocalInfo::MockLocalInfo local_info_; - NiceMock tls_; - Grpc::MockAsyncClientManager async_client_manager_; - Grpc::MockAsyncClient* async_client_ = nullptr; - Grpc::MockAsyncClientFactory* factory_ = nullptr; - std::unique_ptr logger_cache_; - NiceMock scope_; -}; - -TEST_F(GrpcAccessLoggerCacheImplTest, Deduplication) { - InSequence s; - - ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig config; - config.set_log_name("log-1"); - config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("cluster-1"); - - expectClientCreation(); - GrpcAccessLoggerSharedPtr logger1 = logger_cache_->getOrCreateLogger(config); - EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config)); - - // Changing log name leads to another logger. - config.set_log_name("log-2"); - expectClientCreation(); - EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config)); - - config.set_log_name("log-1"); - EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config)); - - // Changing cluster name leads to another logger. - config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("cluster-2"); - expectClientCreation(); - EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config)); -} - -class MockGrpcAccessLogger : public GrpcAccessLogger { +class MockGrpcAccessLogger : public GrpcCommon::GrpcAccessLogger { public: // GrpcAccessLogger MOCK_METHOD1(log, void(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry)); }; -class MockGrpcAccessLoggerCache : public GrpcAccessLoggerCache { +class MockGrpcAccessLoggerCache : public GrpcCommon::GrpcAccessLoggerCache { public: // GrpcAccessLoggerCache MOCK_METHOD1(getOrCreateLogger, - GrpcAccessLoggerSharedPtr( + GrpcCommon::GrpcAccessLoggerSharedPtr( const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config)); }; @@ -863,37 +599,6 @@ TEST_F(HttpGrpcAccessLogTest, MarshallingAdditionalHeaders) { } } -TEST(responseFlagsToAccessLogResponseFlagsTest, All) { - NiceMock stream_info; - ON_CALL(stream_info, hasResponseFlag(_)).WillByDefault(Return(true)); - envoy::data::accesslog::v2::AccessLogCommon common_access_log; - HttpGrpcAccessLog::responseFlagsToAccessLogResponseFlags(common_access_log, stream_info); - - envoy::data::accesslog::v2::AccessLogCommon common_access_log_expected; - common_access_log_expected.mutable_response_flags()->set_failed_local_healthcheck(true); - common_access_log_expected.mutable_response_flags()->set_no_healthy_upstream(true); - common_access_log_expected.mutable_response_flags()->set_upstream_request_timeout(true); - common_access_log_expected.mutable_response_flags()->set_local_reset(true); - common_access_log_expected.mutable_response_flags()->set_upstream_remote_reset(true); - common_access_log_expected.mutable_response_flags()->set_upstream_connection_failure(true); - common_access_log_expected.mutable_response_flags()->set_upstream_connection_termination(true); - common_access_log_expected.mutable_response_flags()->set_upstream_overflow(true); - common_access_log_expected.mutable_response_flags()->set_no_route_found(true); - common_access_log_expected.mutable_response_flags()->set_delay_injected(true); - common_access_log_expected.mutable_response_flags()->set_fault_injected(true); - common_access_log_expected.mutable_response_flags()->set_rate_limited(true); - common_access_log_expected.mutable_response_flags()->mutable_unauthorized_details()->set_reason( - envoy::data::accesslog::v2::ResponseFlags_Unauthorized_Reason:: - ResponseFlags_Unauthorized_Reason_EXTERNAL_SERVICE); - common_access_log_expected.mutable_response_flags()->set_rate_limit_service_error(true); - common_access_log_expected.mutable_response_flags()->set_downstream_connection_termination(true); - common_access_log_expected.mutable_response_flags()->set_upstream_retry_limit_exceeded(true); - common_access_log_expected.mutable_response_flags()->set_stream_idle_timeout(true); - common_access_log_expected.mutable_response_flags()->set_invalid_envoy_request_headers(true); - - EXPECT_EQ(common_access_log_expected.DebugString(), common_access_log.DebugString()); -} - TEST_F(HttpGrpcAccessLogTest, LogWithRequestMethod) { InSequence s; expectLogRequestMethod("GET"); diff --git a/test/extensions/access_loggers/http_grpc/grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc similarity index 100% rename from test/extensions/access_loggers/http_grpc/grpc_access_log_integration_test.cc rename to test/extensions/access_loggers/grpc/http_grpc_access_log_integration_test.cc diff --git a/test/extensions/access_loggers/http_grpc/BUILD b/test/extensions/access_loggers/http_grpc/BUILD deleted file mode 100644 index cc3cb25cb2e8..000000000000 --- a/test/extensions/access_loggers/http_grpc/BUILD +++ /dev/null @@ -1,52 +0,0 @@ -licenses(["notice"]) # Apache 2 - -load( - "//bazel:envoy_build_system.bzl", - "envoy_package", -) -load( - "//test/extensions:extensions_build_system.bzl", - "envoy_extension_cc_test", -) - -envoy_package() - -envoy_extension_cc_test( - name = "grpc_access_log_impl_test", - srcs = ["grpc_access_log_impl_test.cc"], - extension_name = "envoy.access_loggers.http_grpc", - deps = [ - "//source/extensions/access_loggers/http_grpc:grpc_access_log_lib", - "//test/mocks/access_log:access_log_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/ssl:ssl_mocks", - "//test/mocks/stream_info:stream_info_mocks", - "//test/mocks/thread_local:thread_local_mocks", - ], -) - -envoy_extension_cc_test( - name = "config_test", - srcs = ["config_test.cc"], - extension_name = "envoy.access_loggers.http_grpc", - deps = [ - "//source/extensions/access_loggers/http_grpc:config", - "//test/mocks/server:server_mocks", - ], -) - -envoy_extension_cc_test( - name = "grpc_access_log_integration_test", - srcs = ["grpc_access_log_integration_test.cc"], - extension_name = "envoy.access_loggers.http_grpc", - deps = [ - "//source/common/buffer:zero_copy_input_stream_lib", - "//source/common/grpc:codec_lib", - "//source/common/grpc:common_lib", - "//source/extensions/access_loggers/http_grpc:config", - "//test/common/grpc:grpc_client_integration_lib", - "//test/integration:http_integration_lib", - "//test/test_common:utility_lib", - ], -) diff --git a/tools/check_format.py b/tools/check_format.py index 54eda668bc4f..09485526fdf5 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -136,7 +136,7 @@ "extensions/stat_sinks/common", "extensions/stat_sinks/common/statsd", "extensions/health_checkers/redis", - "extensions/access_loggers/http_grpc", + "extensions/access_loggers/grpc", "extensions/access_loggers/file", "extensions/common/tap", "extensions/transport_sockets/raw_buffer", From 58526530c72f48d4dcb66d8b5e20a51e3a96914f Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Wed, 14 Aug 2019 23:26:45 +0700 Subject: [PATCH 385/542] lua: Allow to set header entry as table to httpCall and respond APIs (#7851) Signed-off-by: Dhi Aurrahman --- .../configuration/http_filters/lua_filter.rst | 26 ++++++++++--------- docs/root/intro/version_history.rst | 1 + .../extensions/filters/http/lua/lua_filter.cc | 16 +++++++++--- .../filters/http/lua/lua_filter_test.cc | 22 +++++++++++----- 4 files changed, 43 insertions(+), 22 deletions(-) diff --git a/docs/root/configuration/http_filters/lua_filter.rst b/docs/root/configuration/http_filters/lua_filter.rst index 23f4d2a65ad6..202bf74340e6 100644 --- a/docs/root/configuration/http_filters/lua_filter.rst +++ b/docs/root/configuration/http_filters/lua_filter.rst @@ -109,7 +109,8 @@ more details on the supported API. { [":method"] = "POST", [":path"] = "/", - [":authority"] = "lua_cluster" + [":authority"] = "lua_cluster", + ["set-cookie"] = { "lang=lua; Path=/", "type=binding; Path=/" } }, "hello world", 5000) @@ -236,9 +237,9 @@ httpCall() Makes an HTTP call to an upstream host. Envoy will yield the script until the call completes or has an error. *cluster* is a string which maps to a configured cluster manager cluster. *headers* -is a table of key/value pairs to send. Note that the *:method*, *:path*, and *:authority* headers -must be set. *body* is an optional string of body data to send. *timeout* is an integer that -specifies the call timeout in milliseconds. +is a table of key/value pairs to send (the value can be a string or table of strings). Note that +the *:method*, *:path*, and *:authority* headers must be set. *body* is an optional string of body +data to send. *timeout* is an integer that specifies the call timeout in milliseconds. Returns *headers* which is a table of response headers. Returns *body* which is the string response body. May be nil if there is no body. @@ -264,8 +265,9 @@ passed to subsequent filters. Meaning, the following Lua code is invalid: end end -*headers* is a table of key/value pairs to send. Note that the *:status* header -must be set. *body* is a string and supplies the optional response body. May be nil. +*headers* is a table of key/value pairs to send (the value can be a string or table of strings). +Note that the *:status* header must be set. *body* is a string and supplies the optional response +body. May be nil. metadata() ^^^^^^^^^^ @@ -316,10 +318,10 @@ importPublicKey() ^^^^^^^^^^^^^^^^^ .. code-block:: lua - + pubkey = handle:importPublicKey(keyder, keyderLength) -Returns public key which is used by :ref:`verifySignature ` to verify digital signature. +Returns public key which is used by :ref:`verifySignature ` to verify digital signature. .. _verify_signature: @@ -330,13 +332,13 @@ verifySignature() ok, error = verifySignature(hashFunction, pubkey, signature, signatureLength, data, dataLength) -Verify signature using provided parameters. *hashFunction* is the variable for hash function which be used -for verifying signature. *SHA1*, *SHA224*, *SHA256*, *SHA384* and *SHA512* are supported. -*pubkey* is the public key. *signature* is the signature to be verified. *signatureLength* is +Verify signature using provided parameters. *hashFunction* is the variable for hash function which be used +for verifying signature. *SHA1*, *SHA224*, *SHA256*, *SHA384* and *SHA512* are supported. +*pubkey* is the public key. *signature* is the signature to be verified. *signatureLength* is the length of the signature. *data* is the content which will be hashed. *dataLength* is the length of data. The function returns a pair. If the first element is *true*, the second element will be empty -which means signature is verified; otherwise, the second element will store the error message. +which means signature is verified; otherwise, the second element will store the error message. .. _config_http_filters_lua_header_wrapper: diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 8be96d2a9f19..4bd05d66bf13 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -21,6 +21,7 @@ Version history * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. +* lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * router check tool: add coverage reporting & enforcement. * tls: added verification of IP address SAN fields in certificates against configured SANs in the diff --git a/source/extensions/filters/http/lua/lua_filter.cc b/source/extensions/filters/http/lua/lua_filter.cc index 1dd5edff1fad..2f163162df06 100644 --- a/source/extensions/filters/http/lua/lua_filter.cc +++ b/source/extensions/filters/http/lua/lua_filter.cc @@ -146,9 +146,19 @@ Http::HeaderMapPtr StreamHandleWrapper::buildHeadersFromTable(lua_State* state, while (lua_next(state, table_index) != 0) { // Uses 'key' (at index -2) and 'value' (at index -1). const char* key = luaL_checkstring(state, -2); - const char* value = luaL_checkstring(state, -1); - headers->addCopy(Http::LowerCaseString(key), value); - + // Check if the current value is a table, we iterate through the table and add each element of + // it as a header entry value for the current key. + if (lua_istable(state, -1)) { + lua_pushnil(state); + while (lua_next(state, -2) != 0) { + const char* value = luaL_checkstring(state, -1); + headers->addCopy(Http::LowerCaseString(key), value); + lua_pop(state, 1); + } + } else { + const char* value = luaL_checkstring(state, -1); + headers->addCopy(Http::LowerCaseString(key), value); + } // Removes 'value'; keeps 'key' for next iteration. This is the input for lua_next() so that // it can push the next key/value pair onto the stack. lua_pop(state, 1); diff --git a/test/extensions/filters/http/lua/lua_filter_test.cc b/test/extensions/filters/http/lua/lua_filter_test.cc index 2d444fc6f61a..ea09b1c38e41 100644 --- a/test/extensions/filters/http/lua/lua_filter_test.cc +++ b/test/extensions/filters/http/lua/lua_filter_test.cc @@ -744,7 +744,8 @@ TEST_F(LuaHttpFilterTest, HttpCall) { { [":method"] = "POST", [":path"] = "/", - [":authority"] = "foo" + [":authority"] = "foo", + ["set-cookie"] = { "flavor=chocolate; Path=/", "variant=chewy; Path=/" } }, "hello world", 5000) @@ -770,6 +771,8 @@ TEST_F(LuaHttpFilterTest, HttpCall) { EXPECT_EQ((Http::TestHeaderMapImpl{{":path", "/"}, {":method", "POST"}, {":authority", "foo"}, + {"set-cookie", "flavor=chocolate; Path=/"}, + {"set-cookie", "variant=chewy; Path=/"}, {"content-length", "11"}}), message->headers()); callbacks = &cb; @@ -959,7 +962,10 @@ TEST_F(LuaHttpFilterTest, HttpCallImmediateResponse) { nil, 5000) request_handle:respond( - {[":status"] = "403"}, + { + [":status"] = "403", + ["set-cookie"] = { "flavor=chocolate; Path=/", "variant=chewy; Path=/" } + }, nil) end )EOF"}; @@ -988,7 +994,9 @@ TEST_F(LuaHttpFilterTest, HttpCallImmediateResponse) { Http::MessagePtr response_message(new Http::ResponseMessageImpl( Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); - Http::TestHeaderMapImpl expected_headers{{":status", "403"}}; + Http::TestHeaderMapImpl expected_headers{{":status", "403"}, + {"set-cookie", "flavor=chocolate; Path=/"}, + {"set-cookie", "variant=chewy; Path=/"}}; EXPECT_CALL(decoder_callbacks_, encodeHeaders_(HeaderMapEqualRef(&expected_headers), true)); callbacks->onSuccess(std::move(response_message)); } @@ -1668,28 +1676,28 @@ TEST_F(LuaHttpFilterTest, SignatureVerify) { rawsig = signature:fromhex() - ok, error = request_handle:verifySignature(hashFunc, pubkey, rawsig, string.len(rawsig), data, string.len(data)) + ok, error = request_handle:verifySignature(hashFunc, pubkey, rawsig, string.len(rawsig), data, string.len(data)) if ok then request_handle:logTrace("signature is valid") else request_handle:logTrace(error) end - ok, error = request_handle:verifySignature("unknown", pubkey, rawsig, string.len(rawsig), data, string.len(data)) + ok, error = request_handle:verifySignature("unknown", pubkey, rawsig, string.len(rawsig), data, string.len(data)) if ok then request_handle:logTrace("signature is valid") else request_handle:logTrace(error) end - ok, error = request_handle:verifySignature(hashFunc, pubkey, "0000", 4, data, string.len(data)) + ok, error = request_handle:verifySignature(hashFunc, pubkey, "0000", 4, data, string.len(data)) if ok then request_handle:logTrace("signature is valid") else request_handle:logTrace(error) end - ok, error = request_handle:verifySignature(hashFunc, pubkey, rawsig, string.len(rawsig), "xxxx", 4) + ok, error = request_handle:verifySignature(hashFunc, pubkey, rawsig, string.len(rawsig), "xxxx", 4) if ok then request_handle:logTrace("signature is valid") else From 71a679d11c692c0d0751566679c8910cf2f9db3f Mon Sep 17 00:00:00 2001 From: Derek Date: Wed, 14 Aug 2019 09:28:34 -0700 Subject: [PATCH 386/542] tools: add comprehensive coverage reporting to router check (#7865) Signed-off-by: Derek Schaller --- .../root/configuration/tools/router_check.rst | 19 ++++- docs/root/intro/version_history.rst | 1 + test/tools/router_check/coverage.cc | 75 +++++++++++++++++-- test/tools/router_check/coverage.h | 41 +++++++++- test/tools/router_check/router.cc | 37 +++++++-- test/tools/router_check/router.h | 10 ++- test/tools/router_check/router_check.cc | 4 +- .../config/ComprehensiveRoutes.golden.json | 56 ++++++++++++++ .../test/config/ComprehensiveRoutes.yaml | 22 ++++++ test/tools/router_check/test/route_tests.sh | 6 ++ 10 files changed, 249 insertions(+), 22 deletions(-) create mode 100644 test/tools/router_check/test/config/ComprehensiveRoutes.golden.json create mode 100644 test/tools/router_check/test/config/ComprehensiveRoutes.yaml diff --git a/docs/root/configuration/tools/router_check.rst b/docs/root/configuration/tools/router_check.rst index 5a596521e95c..1d4dd2482757 100644 --- a/docs/root/configuration/tools/router_check.rst +++ b/docs/root/configuration/tools/router_check.rst @@ -167,6 +167,19 @@ run will fail. .. code:: bash - > bazel-bin/test/tools/router_check/router_check_tool --config-path ... --test-path ... --useproto --fail-under 0.08 - Current route coverage: 0.0744863 - Failed to meet coverage requirement: 0.08 + > bazel-bin/test/tools/router_check/router_check_tool --config-path ... --test-path ... --useproto --fail-under 8 + Current route coverage: 7.44863% + Failed to meet coverage requirement: 8% + + +By default the coverage report measures test coverage by checking that at least one field is +verified for every route. However, this can leave holes in the tests where fields +aren't validated and later changed. For more comprehensive coverage you can add a flag, +`--covall`, which will calculate coverage taking into account all of the possible +fields that could be tested. + +.. code:: bash + + > bazel-bin/test/tools/router_check/router_check_tool --config-path ... --test-path ... --useproto --f 7 --covall + Current route coverage: 6.2948% + Failed to meet coverage requirement: 7% diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 4bd05d66bf13..81800919060f 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -24,6 +24,7 @@ Version history * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * router check tool: add coverage reporting & enforcement. +* router check tool: add comprehensive coverage reporting. * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. * upstream: added network filter chains to upstream connections, see :ref:`filters`. diff --git a/test/tools/router_check/coverage.cc b/test/tools/router_check/coverage.cc index 5f8c51a51351..3e81b105097f 100644 --- a/test/tools/router_check/coverage.cc +++ b/test/tools/router_check/coverage.cc @@ -5,19 +5,80 @@ #include "envoy/api/v2/core/base.pb.h" namespace Envoy { -void Coverage::markCovered(const Envoy::Router::RouteEntry& route) { - // n.b. If we reach the end of the seen routes without finding the specified - // route we add it as seen, otherwise it's a duplicate. - if (std::find(seen_routes_.begin(), seen_routes_.end(), &route) == seen_routes_.end()) { - seen_routes_.push_back(&route); +double RouteCoverage::report() { + uint64_t route_weight = 0; + for (const auto& covered_field : coverageFields()) { + if (covered_field) { + route_weight += 1; + } } + return static_cast(route_weight) / coverageFields().size(); +} + +void Coverage::markClusterCovered(const Envoy::Router::RouteEntry& route) { + coveredRoute(route).setClusterCovered(); +} + +void Coverage::markVirtualClusterCovered(const Envoy::Router::RouteEntry& route) { + coveredRoute(route).setVirtualClusterCovered(); +} + +void Coverage::markVirtualHostCovered(const Envoy::Router::RouteEntry& route) { + coveredRoute(route).setVirtualHostCovered(); +} + +void Coverage::markPathRewriteCovered(const Envoy::Router::RouteEntry& route) { + coveredRoute(route).setPathRewriteCovered(); +} + +void Coverage::markHostRewriteCovered(const Envoy::Router::RouteEntry& route) { + coveredRoute(route).setHostRewriteCovered(); +} + +void Coverage::markRedirectPathCovered(const Envoy::Router::RouteEntry& route) { + coveredRoute(route).setRedirectPathCovered(); } double Coverage::report() { uint64_t num_routes = 0; for (const auto& host : route_config_.virtual_hosts()) { - num_routes += host.routes_size(); + for (const auto& route : host.routes()) { + if (route.route().has_weighted_clusters()) { + num_routes += route.route().weighted_clusters().clusters_size(); + } else { + num_routes += 1; + } + } + } + return 100 * static_cast(covered_routes_.size()) / num_routes; +} + +double Coverage::detailedReport() { + uint64_t num_routes = 0; + for (const auto& host : route_config_.virtual_hosts()) { + for (const auto& route : host.routes()) { + if (route.route().has_weighted_clusters()) { + num_routes += route.route().weighted_clusters().clusters_size(); + } else { + num_routes += 1; + } + } + } + double cumulative_coverage = 0; + for (auto& covered_route : covered_routes_) { + cumulative_coverage += covered_route->report(); } - return 100 * static_cast(seen_routes_.size()) / num_routes; + return 100 * cumulative_coverage / num_routes; } + +RouteCoverage& Coverage::coveredRoute(const Envoy::Router::RouteEntry& route) { + for (auto& route_coverage : covered_routes_) { + if (route_coverage->covers(&route)) { + return *route_coverage; + } + } + std::unique_ptr new_coverage = std::make_unique(&route); + covered_routes_.push_back(std::move(new_coverage)); + return coveredRoute(route); +}; } // namespace Envoy diff --git a/test/tools/router_check/coverage.h b/test/tools/router_check/coverage.h index 778906a89fca..c2327449d89b 100644 --- a/test/tools/router_check/coverage.h +++ b/test/tools/router_check/coverage.h @@ -5,14 +5,51 @@ #include "test/mocks/server/mocks.h" namespace Envoy { +class RouteCoverage : Logger::Loggable { +public: + RouteCoverage(const Envoy::Router::RouteEntry* route) : route_(*route){}; + + double report(); + void setClusterCovered() { cluster_covered_ = true; } + void setVirtualClusterCovered() { virtual_cluster_covered_ = true; } + void setVirtualHostCovered() { virtual_host_covered_ = true; } + void setPathRewriteCovered() { path_rewrite_covered_ = true; } + void setHostRewriteCovered() { host_rewrite_covered_ = true; } + void setRedirectPathCovered() { redirect_path_covered_ = true; } + bool covers(const Envoy::Router::RouteEntry* route) { return &route_ == route; } + +private: + const Envoy::Router::RouteEntry& route_; + bool cluster_covered_{false}; + bool virtual_cluster_covered_{false}; + bool virtual_host_covered_{false}; + bool path_rewrite_covered_{false}; + bool host_rewrite_covered_{false}; + bool redirect_path_covered_{false}; + + std::vector coverageFields() { + return std::vector{cluster_covered_, virtual_cluster_covered_, + virtual_host_covered_, path_rewrite_covered_, + host_rewrite_covered_, redirect_path_covered_}; + } +}; + class Coverage : Logger::Loggable { public: Coverage(envoy::api::v2::RouteConfiguration config) : route_config_(config){}; - void markCovered(const Envoy::Router::RouteEntry& route); + void markClusterCovered(const Envoy::Router::RouteEntry& route); + void markVirtualClusterCovered(const Envoy::Router::RouteEntry& route); + void markVirtualHostCovered(const Envoy::Router::RouteEntry& route); + void markPathRewriteCovered(const Envoy::Router::RouteEntry& route); + void markHostRewriteCovered(const Envoy::Router::RouteEntry& route); + void markRedirectPathCovered(const Envoy::Router::RouteEntry& route); double report(); + double detailedReport(); private: - std::vector seen_routes_; + RouteCoverage& coveredRoute(const Envoy::Router::RouteEntry& route); + + std::vector> covered_routes_; const envoy::api::v2::RouteConfiguration route_config_; }; } // namespace Envoy diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index 70ed8c6fbd19..c75053792f6a 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -211,7 +211,11 @@ bool RouterCheckTool::compareCluster(ToolConfig& tool_config, const std::string& if (tool_config.route_->routeEntry() != nullptr) { actual = tool_config.route_->routeEntry()->clusterName(); } - return compareResults(actual, expected, "cluster_name"); + const bool matches = compareResults(actual, expected, "cluster_name"); + if (matches && tool_config.route_->routeEntry() != nullptr) { + coverage_.markClusterCovered(*tool_config.route_->routeEntry()); + } + return matches; } bool RouterCheckTool::compareCluster( @@ -234,7 +238,11 @@ bool RouterCheckTool::compareVirtualCluster(ToolConfig& tool_config, const std:: tool_config.route_->routeEntry()->virtualCluster(*tool_config.headers_)->statName(); actual = tool_config.symbolTable().toString(stat_name); } - return compareResults(actual, expected, "virtual_cluster_name"); + const bool matches = compareResults(actual, expected, "virtual_cluster_name"); + if (matches && tool_config.route_->routeEntry() != nullptr) { + coverage_.markVirtualClusterCovered(*tool_config.route_->routeEntry()); + } + return matches; } bool RouterCheckTool::compareVirtualCluster( @@ -254,7 +262,11 @@ bool RouterCheckTool::compareVirtualHost(ToolConfig& tool_config, const std::str Stats::StatName stat_name = tool_config.route_->routeEntry()->virtualHost().statName(); actual = tool_config.symbolTable().toString(stat_name); } - return compareResults(actual, expected, "virtual_host_name"); + const bool matches = compareResults(actual, expected, "virtual_host_name"); + if (matches && tool_config.route_->routeEntry() != nullptr) { + coverage_.markVirtualHostCovered(*tool_config.route_->routeEntry()); + } + return matches; } bool RouterCheckTool::compareVirtualHost( @@ -282,8 +294,8 @@ bool RouterCheckTool::compareRewritePath(ToolConfig& tool_config, const std::str actual = tool_config.headers_->get_(Http::Headers::get().Path); } const bool matches = compareResults(actual, expected, "path_rewrite"); - if (matches) { - coverage_.markCovered(*tool_config.route_->routeEntry()); + if (matches && tool_config.route_->routeEntry() != nullptr) { + coverage_.markPathRewriteCovered(*tool_config.route_->routeEntry()); } return matches; } @@ -312,7 +324,11 @@ bool RouterCheckTool::compareRewriteHost(ToolConfig& tool_config, const std::str actual = tool_config.headers_->get_(Http::Headers::get().Host); } - return compareResults(actual, expected, "host_rewrite"); + const bool matches = compareResults(actual, expected, "host_rewrite"); + if (matches && tool_config.route_->routeEntry() != nullptr) { + coverage_.markHostRewriteCovered(*tool_config.route_->routeEntry()); + } + return matches; } bool RouterCheckTool::compareRewriteHost( @@ -332,7 +348,11 @@ bool RouterCheckTool::compareRedirectPath(ToolConfig& tool_config, const std::st actual = tool_config.route_->directResponseEntry()->newPath(*tool_config.headers_); } - return compareResults(actual, expected, "path_redirect"); + const bool matches = compareResults(actual, expected, "path_redirect"); + if (matches && tool_config.route_->routeEntry() != nullptr) { + coverage_.markRedirectPathCovered(*tool_config.route_->routeEntry()); + } + return matches; } bool RouterCheckTool::compareRedirectPath( @@ -422,6 +442,8 @@ Options::Options(int argc, char** argv) { TCLAP::ValueArg fail_under("f", "fail-under", "Fail if test coverage is under a specified amount", false, 0.0, "float", cmd); + TCLAP::SwitchArg comprehensive_coverage( + "", "covall", "Measure coverage by checking all route fields", cmd, false); TCLAP::ValueArg config_path("c", "config-path", "Path to configuration file.", false, "", "string", cmd); TCLAP::ValueArg test_path("t", "test-path", "Path to test file.", false, "", @@ -438,6 +460,7 @@ Options::Options(int argc, char** argv) { is_proto_ = is_proto.getValue(); is_detailed_ = is_detailed.getValue(); fail_under_ = fail_under.getValue(); + comprehensive_coverage_ = comprehensive_coverage.getValue(); if (is_proto_) { config_path_ = config_path.getValue(); diff --git a/test/tools/router_check/router.h b/test/tools/router_check/router.h index f4af695dcf7a..a3395de2ad87 100644 --- a/test/tools/router_check/router.h +++ b/test/tools/router_check/router.h @@ -89,7 +89,9 @@ class RouterCheckTool : Logger::Loggable { */ void setShowDetails() { details_ = true; } - float coverage() { return coverage_.report(); } + float coverage(bool detailed) { + return detailed ? coverage_.detailedReport() : coverage_.report(); + } private: RouterCheckTool( @@ -181,6 +183,11 @@ class Options { */ double failUnder() const { return fail_under_; } + /** + * @return true if test coverage should be comprehensive. + */ + bool comprehensiveCoverage() const { return comprehensive_coverage_; } + /** * @return true if proto schema test is used. */ @@ -197,6 +204,7 @@ class Options { std::string unlabelled_test_path_; std::string unlabelled_config_path_; float fail_under_; + bool comprehensive_coverage_; bool is_proto_; bool is_detailed_; }; diff --git a/test/tools/router_check/router_check.cc b/test/tools/router_check/router_check.cc index d7b27d99f3a3..e17f3eecc827 100644 --- a/test/tools/router_check/router_check.cc +++ b/test/tools/router_check/router_check.cc @@ -25,8 +25,8 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } - const double current_coverage = checktool.coverage(); - std::cerr << "Current route coverage: " << current_coverage << "%" << std::endl; + const double current_coverage = checktool.coverage(options.comprehensiveCoverage()); + std::cout << "Current route coverage: " << current_coverage << "%" << std::endl; if (enforce_coverage) { if (current_coverage < options.failUnder()) { std::cerr << "Failed to meet coverage requirement: " << options.failUnder() << "%" diff --git a/test/tools/router_check/test/config/ComprehensiveRoutes.golden.json b/test/tools/router_check/test/config/ComprehensiveRoutes.golden.json new file mode 100644 index 000000000000..5a0e4d8725e4 --- /dev/null +++ b/test/tools/router_check/test/config/ComprehensiveRoutes.golden.json @@ -0,0 +1,56 @@ +[ + { + "test_name": "Test 1", + "input": { + ":authority": "www.lyft.com", + ":path": "/new_endpoint" + }, + "validate": { + "cluster_name": "www2", + "virtual_cluster_name": "other", + "virtual_host_name": "www2_host", + "path_rewrite": "/api/new_endpoint", + "host_rewrite": "www.lyft.com", + "path_redirect": "" + } + }, + { + "test_name": "Test 2", + "input": { + ":authority": "www.lyft.com", + ":path": "/" + }, + "validate": { + "cluster_name": "root_www2", + "virtual_cluster_name": "other", + "virtual_host_name": "www2_host", + "path_rewrite": "/", + "host_rewrite": "www.lyft.com", + "path_redirect": "" + } + }, + { + "test_name": "Test 3", + "input": { + ":authority": "www.lyft.com", + ":path": "/foobar" + }, + "validate": { + "cluster_name": "www2", + "virtual_cluster_name": "other", + "virtual_host_name": "www2_host", + "path_rewrite": "/foobar", + "host_rewrite": "www.lyft.com", + "path_redirect": "" + } + }, + { + "test_name": "Test 4", + "input": { + ":authority": "www.lyft.com", + ":path": "/users/123", + ":method": "PUT" + }, + "validate": {"virtual_cluster_name": "update_user"} + } +] diff --git a/test/tools/router_check/test/config/ComprehensiveRoutes.yaml b/test/tools/router_check/test/config/ComprehensiveRoutes.yaml new file mode 100644 index 000000000000..0613410256ca --- /dev/null +++ b/test/tools/router_check/test/config/ComprehensiveRoutes.yaml @@ -0,0 +1,22 @@ +virtual_hosts: + - name: www2_host + domains: + - www.lyft.com + routes: + - match: + prefix: /new_endpoint + route: + cluster: www2 + prefix_rewrite: /api/new_endpoint + - match: + path: / + route: + cluster: root_www2 + - match: + prefix: / + route: + cluster: www2 + virtual_clusters: + - pattern: ^/users/\d+$ + method: PUT + name: update_user diff --git a/test/tools/router_check/test/route_tests.sh b/test/tools/router_check/test/route_tests.sh index cba937d3f7e3..0b06cb7425bf 100755 --- a/test/tools/router_check/test/route_tests.sh +++ b/test/tools/router_check/test/route_tests.sh @@ -24,6 +24,12 @@ if [[ "${COVERAGE_OUTPUT}" != *"Current route coverage: "* ]] ; then exit 1 fi +COMP_COVERAGE_CMD="${PATH_BIN} ${PATH_CONFIG}/ComprehensiveRoutes.yaml ${PATH_CONFIG}/ComprehensiveRoutes.golden.json --details -f " +COVERAGE_OUTPUT=$($COMP_COVERAGE_CMD "100" "--covall" 2>&1) || echo "${COVERAGE_OUTPUT:-no-output}" +if [[ "${COVERAGE_OUTPUT}" != *"Current route coverage: 100%"* ]] ; then + exit 1 +fi + # Testing coverage flag fails COVERAGE_OUTPUT=$($COVERAGE_CMD "100" 2>&1) || echo "${COVERAGE_OUTPUT:-no-output}" if [[ "${COVERAGE_OUTPUT}" != *"Failed to meet coverage requirement: 100%"* ]] ; then From 8747862bcd8ec873e18939a17df8c7d8dfa61ea5 Mon Sep 17 00:00:00 2001 From: Xuyang Tao Date: Wed, 14 Aug 2019 09:32:34 -0700 Subject: [PATCH 387/542] grpc-json transcoder: add integration test (#7703) Signed-off-by: Xuyang Tao --- .../grpc_json_transcoder_integration_test.cc | 241 ++++++++++++++++++ test/proto/bookstore.proto | 15 +- 2 files changed, 255 insertions(+), 1 deletion(-) diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index 89459dd4f341..14f491375a8a 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -175,6 +175,117 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryPost) { R"({"id":"20","theme":"Children"})"); } +TEST_P(GrpcJsonTranscoderIntegrationTest, QueryParams) { + HttpIntegrationTest::initialize(); + // 1. Binding theme='Children' in CreateShelfRequest + // Using the following HTTP template: + // POST /shelves + // body: shelf + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/shelf?shelf.theme=Children"}, + {":authority", "host"}, + {"content-type", "application/json"}}, + "", {R"(shelf { theme: "Children" })"}, {R"(id: 20 theme: "Children" )"}, Status(), + Http::TestHeaderMapImpl{ + {":status", "200"}, + {"content-type", "application/json"}, + }, + R"({"id":"20","theme":"Children"})"); + + // 2. Binding theme='Children' and id='999' in CreateShelfRequest + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/shelf?shelf.id=999&shelf.theme=Children"}, + {":authority", "host"}, + {"content-type", "application/json"}}, + "", {R"(shelf { id: 999 theme: "Children" })"}, {R"(id: 999 theme: "Children" )"}, Status(), + Http::TestHeaderMapImpl{ + {":status", "200"}, + {"content-type", "application/json"}, + }, + R"({"id":"999","theme":"Children"})"); + + // 3. Binding shelf=1, book= and book.title='War and Peace' in CreateBookRequest + // Using the following HTTP template: + // POST /shelves/{shelf}/books + // body: book + testTranscoding( + Http::TestHeaderMapImpl{{":method", "PUT"}, + {":path", "/shelves/1/books?book.title=War%20and%20Peace"}, + {":authority", "host"}}, + R"({"author" : "Leo Tolstoy"})", + {R"(shelf: 1 book { author: "Leo Tolstoy" title: "War and Peace" })"}, + {R"(id: 3 author: "Leo Tolstoy" title: "War and Peace")"}, Status(), + Http::TestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, + R"({"id":"3","author":"Leo Tolstoy","title":"War and Peace"})"); + + // 4. Binding shelf=1, book.author='Leo Tolstoy' and book.title='War and Peace' in + // CreateBookRequest + // Using the following HTTP template: + // POST /shelves/{shelf}/books + // body: book + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "PUT"}, + {":path", "/shelves/1/books?book.author=Leo%20Tolstoy&book.title=War%20and%20Peace"}, + {":authority", "host"}}, + "", {R"(shelf: 1 book { author: "Leo Tolstoy" title: "War and Peace" })"}, + {R"(id: 3 author: "Leo Tolstoy" title: "War and Peace")"}, Status(), + Http::TestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, + R"({"id":"3","author":"Leo Tolstoy","title":"War and Peace"})"); + + // 5. Test URL decoding. + testTranscoding( + Http::TestHeaderMapImpl{{":method", "PUT"}, + {":path", "/shelves/1/books?book.title=War%20%26%20Peace"}, + {":authority", "host"}}, + R"({"author" : "Leo Tolstoy"})", + {R"(shelf: 1 book { author: "Leo Tolstoy" title: "War & Peace" })"}, + {R"(id: 3 author: "Leo Tolstoy" title: "War & Peace")"}, Status(), + Http::TestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, + R"({"id":"3","author":"Leo Tolstoy","title":"War & Peace"})"); + + // 6. Binding all book fields through query params. + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "PUT"}, + {":path", + "/shelves/1/books?book.id=999&book.author=Leo%20Tolstoy&book.title=War%20and%20Peace"}, + {":authority", "host"}}, + "", {R"(shelf: 1 book { id : 999 author: "Leo Tolstoy" title: "War and Peace" })"}, + {R"(id: 999 author: "Leo Tolstoy" title: "War and Peace")"}, Status(), + Http::TestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, + R"({"id":"999","author":"Leo Tolstoy","title":"War and Peace"})"); + + // 7. Binding shelf=3, book= and the repeated field book.quote with + // two values ("Winter is coming" and "Hold the door") in CreateBookRequest. + // These values should be added to the repeated field in addition to what is + // translated in the body. + // Using the following HTTP template: + // POST /shelves/{shelf}/books + // body: book + std::string reqBody = + R"({"id":"999","author":"George R.R. Martin","title":"A Game of Thrones",)" + R"("quotes":["A girl has no name","A very small man can cast a very large shadow"]})"; + std::string grpcResp = R"(id : 999 author: "George R.R. Martin" title: "A Game of Thrones" + quotes: "A girl has no name" quotes : "A very small man can cast a very large shadow" + quotes: "Winter is coming" quotes : "Hold the door")"; + std::string expectGrpcRequest = absl::StrCat("shelf: 1 book {", grpcResp, "}"); + std::string respBody = + R"({"id":"999","author":"George R.R. Martin","title":"A Game of Thrones","quotes":["A girl has no name")" + R"(,"A very small man can cast a very large shadow","Winter is coming","Hold the door"]})"; + + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "PUT"}, + {":path", + "/shelves/1/books?book.quotes=Winter%20is%20coming&book.quotes=Hold%20the%20door"}, + {":authority", "host"}}, + reqBody, {expectGrpcRequest}, {grpcResp}, Status(), + Http::TestHeaderMapImpl{{":status", "200"}, {"content-type", "application/json"}}, respBody); +} + TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryGet) { HttpIntegrationTest::initialize(); testTranscoding( @@ -363,6 +474,136 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, InvalidJson) { R"({ "theme" "Children" })", {}, {}, Status(), Http::TestHeaderMapImpl{{":status", "400"}, {"content-type", "text/plain"}}, "Expected : between key:value pair.\n", false); + + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, {":path", "/shelf"}, {":authority", "host"}}, + R"({ "theme" : "Children" }EXTRA)", {}, {}, Status(), + Http::TestHeaderMapImpl{{":status", "400"}, {"content-type", "text/plain"}}, + "Parsing terminated before end of input.\n", false); +} + +std::string createDeepJson(int level, bool valid) { + std::string begin = R"({"k":)"; + std::string deep_val = R"("v")"; + std::string end = R"(})"; + std::string json; + + for (int i = 0; i < level; ++i) { + absl::StrAppend(&json, begin); + } + if (valid) { + absl::StrAppend(&json, deep_val); + } + for (int i = 0; i < level; ++i) { + absl::StrAppend(&json, end); + } + return json; +} + +std::string jsonStrToPbStrucStr(std::string json) { + Envoy::ProtobufWkt::Struct message; + std::string structStr; + TestUtility::loadFromJson(json, message); + TextFormat::PrintToString(message, &structStr); + return structStr; +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, DeepStruct) { + HttpIntegrationTest::initialize(); + // Due to the limit of protobuf util, we can only compare to level 32. + std::string deepJson = createDeepJson(32, true); + std::string deepProto = "content {" + jsonStrToPbStrucStr(deepJson) + "}"; + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "POST"}, {":path", "/echoStruct"}, {":authority", "host"}}, + deepJson, {deepProto}, {deepProto}, Status(), + Http::TestHeaderMapImpl{ + {":status", "200"}, {"content-type", "application/json"}, {"grpc-status", "0"}}, + R"({"content":)" + deepJson + R"(})"); + + // The valid deep struct is parsed successfully. + // Since we didn't set the response, it return 503. + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "POST"}, {":path", "/echoStruct"}, {":authority", "host"}}, + createDeepJson(100, true), {}, {}, Status(), + Http::TestHeaderMapImpl{{":status", "503"}, {"content-type", "application/grpc"}}, ""); + + // The invalid deep struct is detected. + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "POST"}, {":path", "/echoStruct"}, {":authority", "host"}}, + createDeepJson(100, false), {}, {}, Status(), + Http::TestHeaderMapImpl{{":status", "400"}, {"content-type", "text/plain"}}, + "Unexpected token.\n", false); +} + +std::string createLargeJson(int level) { + std::shared_ptr cur = std::make_shared(); + for (int i = 0; i < level - 1; ++i) { + std::shared_ptr next = std::make_shared(); + ProtobufWkt::Value val = ProtobufWkt::Value(); + ProtobufWkt::Value left = ProtobufWkt::Value(*cur); + ProtobufWkt::Value right = ProtobufWkt::Value(*cur); + val.mutable_list_value()->add_values()->Swap(&left); + val.mutable_list_value()->add_values()->Swap(&right); + (*next->mutable_struct_value()->mutable_fields())["k"] = val; + cur = next; + } + return MessageUtil::getJsonStringFromMessage(*cur, false, false); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, LargeStruct) { + HttpIntegrationTest::initialize(); + // Create a 40kB json payload. + + std::string largeJson = createLargeJson(12); + std::string largeProto = "content {" + jsonStrToPbStrucStr(largeJson) + "}"; + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "POST"}, {":path", "/echoStruct"}, {":authority", "host"}}, + largeJson, {largeProto}, {largeProto}, Status(), + Http::TestHeaderMapImpl{ + {":status", "200"}, {"content-type", "application/json"}, {"grpc-status", "0"}}, + R"({"content":)" + largeJson + R"(})"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, UnknownField) { + HttpIntegrationTest::initialize(); + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/shelf"}, + {":authority", "host"}, + {"content-type", "application/json"}}, + R"({"theme": "Children", "unknown1": "a", "unknown2" : {"a" : "b"}, "unknown3" : ["a", "b", "c"]})", + {R"(shelf { theme: "Children" })"}, {R"(id: 20 theme: "Children" )"}, Status(), + Http::TestHeaderMapImpl{{":status", "200"}, + {"content-type", "application/json"}, + {"content-length", "30"}, + {"grpc-status", "0"}}, + R"({"id":"20","theme":"Children"})"); +} + +TEST_P(GrpcJsonTranscoderIntegrationTest, UTF8) { + HttpIntegrationTest::initialize(); + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/shelf"}, + {":authority", "host"}, + {"content-type", "application/json"}}, + "{\"id\":\"20\",\"theme\":\"\xC2\xAE\"}", {"shelf {id : 20 theme: \"®\" }"}, + {"id: 20 theme: \"\xC2\xAE\""}, Status(), + Http::TestHeaderMapImpl{ + {":status", "200"}, {"content-type", "application/json"}, {"grpc-status", "0"}}, + R"({"id":"20","theme":"®"})"); + + testTranscoding( + Http::TestHeaderMapImpl{{":method", "POST"}, + {":path", "/shelf"}, + {":authority", "host"}, + {"content-type", "application/json"}}, + "{\"id\":\"20\",\"theme\":\"\xC3\x28\"}", {}, {""}, Status(), + Http::TestHeaderMapImpl{{":status", "400"}}, R"(Encountered non UTF-8 code points)", false); } } // namespace diff --git a/test/proto/bookstore.proto b/test/proto/bookstore.proto index 6fb65b42d119..fc632de0c1ba 100644 --- a/test/proto/bookstore.proto +++ b/test/proto/bookstore.proto @@ -5,6 +5,7 @@ package bookstore; import "google/api/annotations.proto"; import "google/api/httpbody.proto"; import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; // A simple Bookstore API. // @@ -90,6 +91,12 @@ service Bookstore { get: "/index" }; } + rpc EchoStruct(EchoStructReqResp) returns (EchoStructReqResp) { + option (google.api.http) = { + post: "/echoStruct" + body: "content" + }; + } } // A shelf resource. @@ -192,4 +199,10 @@ message DeleteBookRequest { message GetAuthorRequest { // The ID of the author resource to retrieve. int64 author = 1; -} \ No newline at end of file +} + +// Request and Response message for EchoStructReqResp method. +message EchoStructReqResp { + // The content of request. + google.protobuf.Struct content = 1; +} From b2719f6c0a0b3a1dcbb5b33c74769f425b0e5a9f Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 14 Aug 2019 12:33:45 -0400 Subject: [PATCH 388/542] config: fully validating terminal filter ordering for L4 (#7904) Signed-off-by: Alyssa Wilk --- ci/build_setup.sh | 2 +- source/server/listener_manager_impl.cc | 7 ++----- test/server/listener_manager_impl_test.cc | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ci/build_setup.sh b/ci/build_setup.sh index 834ba5db9c9d..53379d9ab8da 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -93,7 +93,7 @@ if [ "$1" != "-nofetch" ]; then fi # This is the hash on https://github.com/envoyproxy/envoy-filter-example.git we pin to. - (cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" && git fetch origin && git checkout -f dcd3374baa9365ab7ab505018232994d6c8a8d81) + (cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" && git fetch origin && git checkout -f 1995c1e0eccea84bbb39f64e75ef3e9102d1ae82) sed -e "s|{ENVOY_SRCDIR}|${ENVOY_SRCDIR}|" "${ENVOY_SRCDIR}"/ci/WORKSPACE.filter.example > "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/WORKSPACE fi diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index da8c12b82b64..6db28bdd0f36 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -65,11 +65,8 @@ std::vector ProdListenerComponentFactory::createNetwor Config::Utility::getAndCheckFactory( string_name); - // TODO(alyssar) replace this block and reinstate TerminalNotLast test once echo2 is updated - if (factory.isTerminalFilter() && i != filters.size() - 1) { - throw EnvoyException( - fmt::format("Error: {} must be the terminal network filter.", filters[i].name())); - } + Config::Utility::validateTerminalFilters(filters[i].name(), "network", + factory.isTerminalFilter(), i == filters.size() - 1); Network::FilterFactoryCb callback; if (Config::Utility::allowDeprecatedV1Config(context.runtime(), *filter_config)) { diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 33f8fabede56..e3cdeffa44f1 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -468,7 +468,7 @@ class NonTerminalFilterFactory : public Configuration::NamedNetworkFilterConfigF std::string name() override { return "non_terminal"; } }; -TEST_F(ListenerManagerImplWithRealFiltersTest, DISABLED_TerminalNotLast) { +TEST_F(ListenerManagerImplWithRealFiltersTest, TerminalNotLast) { Registry::RegisterFactory registered; From 4d2ca4d31e2590cb34cdd694ab4a5eee2cd8dd0b Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 14 Aug 2019 09:34:05 -0700 Subject: [PATCH 389/542] coverage: publish report to GCS (#7909) Signed-off-by: Lizan Zhou --- .circleci/config.yml | 17 ++++++++++++++++- bazel/README.md | 2 +- ci/coverage_publish.sh | 24 ++++++++++++------------ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 87236193005f..22d9de1203af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -86,11 +86,24 @@ jobs: command: ci/do_circle_ci.sh bazel.coverage no_output_timeout: 60m - - run: ci/coverage_publish.sh + - persist_to_workspace: + root: /build/envoy/generated + paths: + - coverage - store_artifacts: path: /build/envoy/generated destination: / + coverage_publish: + docker: + - image: google/cloud-sdk + steps: + - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken + - checkout + - attach_workspace: + at: /build/envoy/generated + - run: ci/coverage_publish.sh + clang_tidy: executor: ubuntu-build steps: @@ -150,6 +163,8 @@ workflows: - api - filter_example_mirror - coverage + - coverage_publish: + requires: [coverage] - format - clang_tidy - build_image diff --git a/bazel/README.md b/bazel/README.md index b1be84925a72..2631fbec01f3 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -506,7 +506,7 @@ have seen some issues with seeing the artifacts tab. If you can't see it, log ou then log back in and it should start working. The latest coverage report for master is available -[here](https://s3.amazonaws.com/lyft-envoy/coverage/report-master/index.html). +[here](https://storage.googleapis.com/envoy-coverage/report-master/index.html). It's also possible to specialize the coverage build to a specified test or test dir. This is useful when doing things like exploring the coverage of a fuzzer over its corpus. This can be done by diff --git a/ci/coverage_publish.sh b/ci/coverage_publish.sh index ad4b05ba3019..c04eafff0323 100755 --- a/ci/coverage_publish.sh +++ b/ci/coverage_publish.sh @@ -8,26 +8,26 @@ if [ "${CIRCLECI}" != "true" ]; then exit 0 fi +[[ -z "${ENVOY_BUILD_DIR}" ]] && ENVOY_BUILD_DIR=/build +COVERAGE_FILE="${ENVOY_BUILD_DIR}/envoy/generated/coverage/index.html" + +if [ ! -f "${COVERAGE_FILE}" ]; then + echo "ERROR: Coverage file not found." + exit 1 +fi + # available for master builds if [ -z "$CIRCLE_PR_NUMBER" ] then echo "Uploading coverage report..." - [[ -z "${ENVOY_BUILD_DIR}" ]] && ENVOY_BUILD_DIR=/build - COVERAGE_FILE="${ENVOY_BUILD_DIR}/envoy/generated/coverage/index.html" - - if [ ! -f "${COVERAGE_FILE}" ]; then - echo "ERROR: Coverage file not found." - exit 1 - fi - BRANCH_NAME="${CIRCLE_BRANCH}" COVERAGE_DIR="$(dirname "${COVERAGE_FILE}")" - S3_LOCATION="lyft-envoy/coverage/report-${BRANCH_NAME}" + GCS_LOCATION="envoy-coverage/report-${BRANCH_NAME}" - pip install awscli --upgrade - aws s3 cp "${COVERAGE_DIR}" "s3://${S3_LOCATION}" --recursive --acl public-read --quiet --sse - echo "Coverage report for branch '${BRANCH_NAME}': https://s3.amazonaws.com/${S3_LOCATION}/index.html" + echo ${GCP_SERVICE_ACCOUNT_KEY} | base64 --decode | gcloud auth activate-service-account --key-file=- + gsutil -m rsync -dr ${COVERAGE_DIR} gs://${GCS_LOCATION} + echo "Coverage report for branch '${BRANCH_NAME}': https://storage.googleapis.com/${GCS_LOCATION}/index.html" else echo "Coverage report will not be uploaded for this build." fi From e1d713efc1342790208a86b7ba795d692b47f443 Mon Sep 17 00:00:00 2001 From: Guangming Wang Date: Thu, 15 Aug 2019 00:34:37 +0800 Subject: [PATCH 390/542] cleanup: conn_manager.cc err message word fix (#7901) Signed-off-by: Guangming Wang --- source/extensions/filters/network/dubbo_proxy/conn_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/extensions/filters/network/dubbo_proxy/conn_manager.cc b/source/extensions/filters/network/dubbo_proxy/conn_manager.cc index b9a5f27d278f..4546ed746348 100644 --- a/source/extensions/filters/network/dubbo_proxy/conn_manager.cc +++ b/source/extensions/filters/network/dubbo_proxy/conn_manager.cc @@ -79,7 +79,7 @@ void ConnectionManager::onBelowWriteBufferLowWatermark() { } StreamHandler& ConnectionManager::newStream() { - ENVOY_LOG(debug, "dubbo: create the new docoder event handler"); + ENVOY_LOG(debug, "dubbo: create the new decoder event handler"); ActiveMessagePtr new_message(std::make_unique(*this)); new_message->createFilterChain(); From 3dd84d3b41b6be378ccb71e81c0063a84bb956e3 Mon Sep 17 00:00:00 2001 From: Cynthia Coan Date: Wed, 14 Aug 2019 12:45:12 -0600 Subject: [PATCH 391/542] use new buffers by default (#7883) Signed-off-by: Cynthia Coan --- docs/root/intro/version_history.rst | 1 + source/server/options_impl.cc | 2 +- test/server/options_impl_test.cc | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 81800919060f..b7ac02962784 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -22,6 +22,7 @@ Version history * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. +* performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * router check tool: add coverage reporting & enforcement. * router check tool: add comprehensive coverage reporting. diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc index 9db443e9848f..4d1b444e5df8 100644 --- a/source/server/options_impl.cc +++ b/source/server/options_impl.cc @@ -104,7 +104,7 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv, TCLAP::ValueArg use_libevent_buffer("", "use-libevent-buffers", "Use the original libevent buffer implementation", - false, true, "bool", cmd); + false, false, "bool", cmd); cmd.setExceptionHandling(false); try { diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index 86ab2b378be7..78c3a8373429 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -94,7 +94,7 @@ TEST_F(OptionsImplTest, All) { EXPECT_EQ(std::chrono::seconds(60), options->drainTime()); EXPECT_EQ(std::chrono::seconds(90), options->parentShutdownTime()); EXPECT_EQ(true, options->hotRestartDisabled()); - EXPECT_EQ(true, options->libeventBufferEnabled()); + EXPECT_EQ(false, options->libeventBufferEnabled()); EXPECT_EQ(true, options->cpusetThreadsEnabled()); options = createOptionsImpl("envoy --mode init_only"); From aff9cafa7e45d7b3dcd2fb15df4c61dda6a09435 Mon Sep 17 00:00:00 2001 From: Cynthia Coan Date: Wed, 14 Aug 2019 18:44:28 -0600 Subject: [PATCH 392/542] upgrade yapf to the latest version (#7926) Description: Upgrade yapf to the latest version. We'd like to reuse it in our local repo, and some of the newer versions contain fixes we need. This required reformatting all the files for their new rules. Happy to change any rules, as necessary. Risk Level: Low Testing: Ensure py format checks can still pass. Docs Changes: N/A Release Notes: N/A fixes #7389 Signed-off-by: Cynthia Coan --- configs/configgen.py | 57 +++++-------- .../network/kafka/protocol/generator.py | 4 +- .../network/kafka/serialization/generator.py | 4 +- .../test_access_log_schema.py | 52 ++++++------ .../test_http_conn_network_filter_schema.py | 18 ++-- .../test_route_configuration_schema.py | 10 +-- .../network/thrift_proxy/driver/client.py | 39 +++++---- .../driver/fbthrift/THeaderTransport.py | 5 +- tools/check_format.py | 83 +++++++++---------- tools/check_spelling_pedantic.py | 57 +++++++------ .../deprecate_features/deprecate_features.py | 4 +- tools/envoy_collect/envoy_collect.py | 32 +++---- tools/find_related_envoy_files.py | 4 +- tools/format_python_tools.py | 15 ++-- tools/gen_compilation_database.py | 9 +- tools/gen_gdb_wrapper_script.py | 7 +- tools/header_order.py | 9 +- tools/protodoc/protodoc.py | 16 ++-- tools/requirements.txt | 2 +- tools/socket_passing.py | 29 +++---- 20 files changed, 218 insertions(+), 238 deletions(-) diff --git a/configs/configgen.py b/configs/configgen.py index 6c5ad0c08078..c255b0d4e2a1 100755 --- a/configs/configgen.py +++ b/configs/configgen.py @@ -42,14 +42,10 @@ # DynamoDB statistics filter, as well as generating a special access log which includes the # X-AMZN-RequestId response header. external_virtual_hosts = [{ - 'name': - 'dynamodb_iad', - 'address': - "127.0.0.1", - 'protocol': - "TCP", - 'port_value': - "9204", + 'name': 'dynamodb_iad', + 'address': "127.0.0.1", + 'protocol': "TCP", + 'port_value': "9204", 'hosts': [{ 'name': 'dynamodb_iad', 'domain': '*', @@ -59,10 +55,8 @@ 'verify_subject_alt_name': ['dynamodb.us-east-1.amazonaws.com'], 'ssl': True }], - 'is_amzn_service': - True, - 'cluster_type': - 'logical_dns' + 'is_amzn_service': True, + 'cluster_type': 'logical_dns' }] # This is the set of mongo clusters that local Envoys can talk to. Each database defines a set of @@ -72,12 +66,9 @@ # as it demonstrates how to setup TCP proxy and the network rate limit filter. mongos_servers = { 'somedb': { - 'address': - "127.0.0.1", - 'protocol': - "TCP", - 'port_value': - 27019, + 'address': "127.0.0.1", + 'protocol': "TCP", + 'port_value': 27019, 'hosts': [ { 'port_value': 27817, @@ -100,17 +91,15 @@ 'protocol': 'TCP' }, ], - 'ratelimit': - True + 'ratelimit': True } } def generate_config(template_path, template, output_file, **context): """ Generate a final config file based on a template and some context. """ - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(template_path, followlinks=True), - undefined=jinja2.StrictUndefined) + env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_path, followlinks=True), + undefined=jinja2.StrictUndefined) raw_output = env.get_template(template).render(**context) with open(output_file, 'w') as fh: fh.write(raw_output) @@ -118,11 +107,10 @@ def generate_config(template_path, template, output_file, **context): # Generate a demo config for the main front proxy. This sets up both HTTP and HTTPS listeners, # as well as a listener for the double proxy to connect to via SSL client authentication. -generate_config( - SCRIPT_DIR, - 'envoy_front_proxy_v2.template.yaml', - '{}/envoy_front_proxy.v2.yaml'.format(OUT_DIR), - clusters=front_envoy_clusters) +generate_config(SCRIPT_DIR, + 'envoy_front_proxy_v2.template.yaml', + '{}/envoy_front_proxy.v2.yaml'.format(OUT_DIR), + clusters=front_envoy_clusters) # Generate a demo config for the double proxy. This sets up both an HTTP and HTTPS listeners, # and backhauls the traffic to the main front proxy. @@ -137,13 +125,12 @@ def generate_config(template_path, template, output_file, **context): # optional external service ports: built from external_virtual_hosts above. Each external host # that Envoy proxies to listens on its own port. # optional mongo ports: built from mongos_servers above. -generate_config( - SCRIPT_DIR, - 'envoy_service_to_service_v2.template.yaml', - '{}/envoy_service_to_service.yaml'.format(OUT_DIR), - internal_virtual_hosts=service_to_service_envoy_clusters, - external_virtual_hosts=external_virtual_hosts, - mongos_servers=mongos_servers) +generate_config(SCRIPT_DIR, + 'envoy_service_to_service_v2.template.yaml', + '{}/envoy_service_to_service.yaml'.format(OUT_DIR), + internal_virtual_hosts=service_to_service_envoy_clusters, + external_virtual_hosts=external_virtual_hosts, + mongos_servers=mongos_servers) for google_ext in ['v2.yaml']: shutil.copy(os.path.join(SCRIPT_DIR, 'google_com_proxy.%s' % google_ext), OUT_DIR) diff --git a/source/extensions/filters/network/kafka/protocol/generator.py b/source/extensions/filters/network/kafka/protocol/generator.py index d407eca1ce61..f72e562b024f 100755 --- a/source/extensions/filters/network/kafka/protocol/generator.py +++ b/source/extensions/filters/network/kafka/protocol/generator.py @@ -512,6 +512,6 @@ def get_template(template): import sys # Templates are resolved relatively to main start script, due to main & test templates being # stored in different directories. - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(searchpath=os.path.dirname(os.path.abspath(sys.argv[0])))) + env = jinja2.Environment(loader=jinja2.FileSystemLoader( + searchpath=os.path.dirname(os.path.abspath(sys.argv[0])))) return env.get_template(template) diff --git a/source/extensions/filters/network/kafka/serialization/generator.py b/source/extensions/filters/network/kafka/serialization/generator.py index 574aa3d66de3..3eaafaf6d038 100755 --- a/source/extensions/filters/network/kafka/serialization/generator.py +++ b/source/extensions/filters/network/kafka/serialization/generator.py @@ -52,6 +52,6 @@ def get_template(template): import sys # Templates are resolved relatively to main start script, due to main & test templates being # stored in different directories. - env = jinja2.Environment( - loader=jinja2.FileSystemLoader(searchpath=os.path.dirname(os.path.abspath(sys.argv[0])))) + env = jinja2.Environment(loader=jinja2.FileSystemLoader( + searchpath=os.path.dirname(os.path.abspath(sys.argv[0])))) return env.get_template(template) diff --git a/test/common/json/config_schemas_test_data/test_access_log_schema.py b/test/common/json/config_schemas_test_data/test_access_log_schema.py index 9cf78f4e1059..09aae1658095 100644 --- a/test/common/json/config_schemas_test_data/test_access_log_schema.py +++ b/test/common/json/config_schemas_test_data/test_access_log_schema.py @@ -5,7 +5,7 @@ "access_log": [{ "filter": { "type": - "logical_and", + "logical_and", "filters": [{ "type": "not_healthcheck" }, { @@ -14,32 +14,30 @@ }] }, "path": "/var/log/envoy/access.log" - }, - { - "filter": { - "type": - "logical_or", - "filters": [{ - "runtime_key": "access_log.access_error.status", - "type": "status_code", - "value": 500, - "op": ">=" - }, { - "type": "status_code", - "value": 429, - "op": "=" - }, - { - "runtime_key": "access_log.access_error.duration", - "type": "duration", - "value": 1000, - "op": ">=" - }, { - "type": "traceable_request" - }] - }, - "path": "/var/log/envoy/access_error.log" - }] + }, { + "filter": { + "type": + "logical_or", + "filters": [{ + "runtime_key": "access_log.access_error.status", + "type": "status_code", + "value": 500, + "op": ">=" + }, { + "type": "status_code", + "value": 429, + "op": "=" + }, { + "runtime_key": "access_log.access_error.duration", + "type": "duration", + "value": 1000, + "op": ">=" + }, { + "type": "traceable_request" + }] + }, + "path": "/var/log/envoy/access_error.log" + }] } diff --git a/test/common/json/config_schemas_test_data/test_http_conn_network_filter_schema.py b/test/common/json/config_schemas_test_data/test_http_conn_network_filter_schema.py index 3c1fa0c6c8d1..e7566cdfcf6a 100644 --- a/test/common/json/config_schemas_test_data/test_http_conn_network_filter_schema.py +++ b/test/common/json/config_schemas_test_data/test_http_conn_network_filter_schema.py @@ -2,14 +2,10 @@ from util import true, false HTTP_CONN_NETWORK_FILTER_BLOB = { - "idle_timeout_s": - 300, - "stat_prefix": - "router", - "use_remote_address": - true, - "server_name": - "envoy-123", + "idle_timeout_s": 300, + "stat_prefix": "router", + "use_remote_address": true, + "server_name": "envoy-123", "access_log": [], "tracing": { "request_headers_for_tags": ["x-source"], @@ -26,10 +22,8 @@ "name": "router" }], "route_config": {}, - "add_user_agent": - true, - "codec_type": - "auto" + "add_user_agent": true, + "codec_type": "auto" } diff --git a/test/common/json/config_schemas_test_data/test_route_configuration_schema.py b/test/common/json/config_schemas_test_data/test_route_configuration_schema.py index 841fbb64805c..fb492747e1fb 100644 --- a/test/common/json/config_schemas_test_data/test_route_configuration_schema.py +++ b/test/common/json/config_schemas_test_data/test_route_configuration_schema.py @@ -5,12 +5,10 @@ "virtual_hosts": [{ "domains": ["production.example.com"], "require_ssl": "all", - "routes": [ - { - "host_redirect": "example.com", - "prefix": "/" - }, - ], + "routes": [{ + "host_redirect": "example.com", + "prefix": "/" + },], "name": "production_redirect" }], "internal_only_headers": ["x-role", "x-source"], diff --git a/test/extensions/filters/network/thrift_proxy/driver/client.py b/test/extensions/filters/network/thrift_proxy/driver/client.py index d9092a54a25f..54e30e70a62b 100755 --- a/test/extensions/filters/network/thrift_proxy/driver/client.py +++ b/test/extensions/filters/network/thrift_proxy/driver/client.py @@ -126,26 +126,25 @@ def main(cfg, reqhandle, resphandle): v = client.add(a, b) print("client: added {0} + {1} = {2}".format(a, b, v)) elif cfg.method == "execute": - param = Param( - return_fields=cfg.params, - the_works=TheWorks( - field_1=True, - field_2=0x7f, - field_3=0x7fff, - field_4=0x7fffffff, - field_5=0x7fffffffffffffff, - field_6=-1.5, - field_7=u"string is UTF-8: \U0001f60e", - field_8=b"binary is bytes: \x80\x7f\x00\x01", - field_9={ - 1: "one", - 2: "two", - 3: "three" - }, - field_10=[1, 2, 4, 8], - field_11=set(["a", "b", "c"]), - field_12=False, - )) + param = Param(return_fields=cfg.params, + the_works=TheWorks( + field_1=True, + field_2=0x7f, + field_3=0x7fff, + field_4=0x7fffffff, + field_5=0x7fffffffffffffff, + field_6=-1.5, + field_7=u"string is UTF-8: \U0001f60e", + field_8=b"binary is bytes: \x80\x7f\x00\x01", + field_9={ + 1: "one", + 2: "two", + 3: "three" + }, + field_10=[1, 2, 4, 8], + field_11=set(["a", "b", "c"]), + field_12=False, + )) try: result = client.execute(param) diff --git a/test/extensions/filters/network/thrift_proxy/driver/fbthrift/THeaderTransport.py b/test/extensions/filters/network/thrift_proxy/driver/fbthrift/THeaderTransport.py index 774c37b46a92..66d7af97b22b 100644 --- a/test/extensions/filters/network/thrift_proxy/driver/fbthrift/THeaderTransport.py +++ b/test/extensions/filters/network/thrift_proxy/driver/fbthrift/THeaderTransport.py @@ -521,8 +521,9 @@ def flushImpl(self, oneway): # We don't include the framing bytes as part of the frame size check frame_size = buf.tell() - (4 if wsz < MAX_FRAME_SIZE else 12) - _frame_size_check( - frame_size, self.__max_frame_size, header=self.__client_type == CLIENT_TYPE.HEADER) + _frame_size_check(frame_size, + self.__max_frame_size, + header=self.__client_type == CLIENT_TYPE.HEADER) self.getTransport().write(buf.getvalue()) if oneway: self.getTransport().onewayFlush() diff --git a/tools/check_format.py b/tools/check_format.py index 09485526fdf5..67292cea223b 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -65,8 +65,8 @@ CLANG_FORMAT_PATH = os.getenv("CLANG_FORMAT", "clang-format-8") BUILDIFIER_PATH = os.getenv("BUILDIFIER_BIN", "$GOPATH/bin/buildifier") -ENVOY_BUILD_FIXER_PATH = os.path.join( - os.path.dirname(os.path.abspath(sys.argv[0])), "envoy_build_fixer.py") +ENVOY_BUILD_FIXER_PATH = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), + "envoy_build_fixer.py") HEADER_ORDER_PATH = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "header_order.py") SUBDIR_SET = set(common.includeDirOrder()) INCLUDE_ANGLE = "#include <" @@ -502,8 +502,8 @@ def checkSourceLine(line, file_path, reportError): "should be %s" % (invalid_construct, valid_construct)) for invalid_construct, valid_construct in LIBCXX_REPLACEMENTS.items(): if invalid_construct in line: - reportError("term %s should be replaced with standard library term %s" % (invalid_construct, - valid_construct)) + reportError("term %s should be replaced with standard library term %s" % + (invalid_construct, valid_construct)) # Some errors cannot be fixed automatically, and actionable, consistent, # navigable messages should be emitted to make it easy to find and fix @@ -811,53 +811,48 @@ def checkErrorMessages(error_messages): if __name__ == "__main__": parser = argparse.ArgumentParser(description="Check or fix file format.") - parser.add_argument( - "operation_type", - type=str, - choices=["check", "fix"], - help="specify if the run should 'check' or 'fix' format.") + parser.add_argument("operation_type", + type=str, + choices=["check", "fix"], + help="specify if the run should 'check' or 'fix' format.") parser.add_argument( "target_path", type=str, nargs="?", default=".", help="specify the root directory for the script to recurse over. Default '.'.") - parser.add_argument( - "--add-excluded-prefixes", type=str, nargs="+", help="exclude additional prefixes.") - parser.add_argument( - "-j", - "--num-workers", - type=int, - default=multiprocessing.cpu_count(), - help="number of worker processes to use; defaults to one per core.") + parser.add_argument("--add-excluded-prefixes", + type=str, + nargs="+", + help="exclude additional prefixes.") + parser.add_argument("-j", + "--num-workers", + type=int, + default=multiprocessing.cpu_count(), + help="number of worker processes to use; defaults to one per core.") parser.add_argument("--api-prefix", type=str, default="./api/", help="path of the API tree.") - parser.add_argument( - "--skip_envoy_build_rule_check", - action="store_true", - help="skip checking for '@envoy//' prefix in build rules.") - parser.add_argument( - "--namespace_check", - type=str, - nargs="?", - default="Envoy", - help="specify namespace check string. Default 'Envoy'.") - parser.add_argument( - "--namespace_check_excluded_paths", - type=str, - nargs="+", - default=[], - help="exclude paths from the namespace_check.") - parser.add_argument( - "--build_fixer_check_excluded_paths", - type=str, - nargs="+", - default=[], - help="exclude paths from envoy_build_fixer check.") - parser.add_argument( - "--include_dir_order", - type=str, - default=",".join(common.includeDirOrder()), - help="specify the header block include directory order.") + parser.add_argument("--skip_envoy_build_rule_check", + action="store_true", + help="skip checking for '@envoy//' prefix in build rules.") + parser.add_argument("--namespace_check", + type=str, + nargs="?", + default="Envoy", + help="specify namespace check string. Default 'Envoy'.") + parser.add_argument("--namespace_check_excluded_paths", + type=str, + nargs="+", + default=[], + help="exclude paths from the namespace_check.") + parser.add_argument("--build_fixer_check_excluded_paths", + type=str, + nargs="+", + default=[], + help="exclude paths from envoy_build_fixer check.") + parser.add_argument("--include_dir_order", + type=str, + default=",".join(common.includeDirOrder()), + help="specify the header block include directory order.") args = parser.parse_args() operation_type = args.operation_type diff --git a/tools/check_spelling_pedantic.py b/tools/check_spelling_pedantic.py index 393674c65c21..d1bcae132d2f 100755 --- a/tools/check_spelling_pedantic.py +++ b/tools/check_spelling_pedantic.py @@ -108,13 +108,12 @@ def start(self): aspell_args = [ "aspell", "pipe", "--run-together", "--lang=en_US", "--encoding=utf-8", "--personal=" + pws ] - self.aspell = subprocess.Popen( - aspell_args, - bufsize=4096, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) + self.aspell = subprocess.Popen(aspell_args, + bufsize=4096, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) # Read the version line that aspell emits on startup. self.aspell.stdout.readline() @@ -501,8 +500,8 @@ def execute(files, dictionary_file, fix): checker.stop() - print("Checked %d file(s) and %d comment(s), found %d error(s)." % (total_files, total_comments, - total_errors)) + print("Checked %d file(s) and %d comment(s), found %d error(s)." % + (total_files, total_comments, total_errors)) return total_errors == 0 @@ -511,27 +510,27 @@ def execute(files, dictionary_file, fix): default_dictionary = os.path.join(TOOLS_DIR, 'spelling_dictionary.txt') parser = argparse.ArgumentParser(description="Check comment spelling.") - parser.add_argument( - 'operation_type', - type=str, - choices=['check', 'fix'], - help="specify if the run should 'check' or 'fix' spelling.") - parser.add_argument( - 'target_paths', type=str, nargs="*", help="specify the files for the script to process.") + parser.add_argument('operation_type', + type=str, + choices=['check', 'fix'], + help="specify if the run should 'check' or 'fix' spelling.") + parser.add_argument('target_paths', + type=str, + nargs="*", + help="specify the files for the script to process.") parser.add_argument('-d', '--debug', action='store_true', help="Debug spell checker subprocess.") - parser.add_argument( - '--mark', action='store_true', help="Emits extra output to mark misspelled words.") - parser.add_argument( - '--dictionary', - type=str, - default=default_dictionary, - help="specify a location for Envoy-specific dictionary words") - parser.add_argument( - '--color', - type=str, - choices=['on', 'off', 'auto'], - default="auto", - help="Controls colorized output. Auto limits color to TTY devices.") + parser.add_argument('--mark', + action='store_true', + help="Emits extra output to mark misspelled words.") + parser.add_argument('--dictionary', + type=str, + default=default_dictionary, + help="specify a location for Envoy-specific dictionary words") + parser.add_argument('--color', + type=str, + choices=['on', 'off', 'auto'], + default="auto", + help="Controls colorized output. Auto limits color to TTY devices.") args = parser.parse_args() COLOR = args.color == "on" or (args.color == "auto" and sys.stdout.isatty()) diff --git a/tools/deprecate_features/deprecate_features.py b/tools/deprecate_features/deprecate_features.py index 0f67d3e280b9..203d4d2a99db 100644 --- a/tools/deprecate_features/deprecate_features.py +++ b/tools/deprecate_features/deprecate_features.py @@ -49,8 +49,8 @@ def deprecate_proto(): # Sorts out the list of features which should be default enabled and returns a tuple of # email and code changes. def flip_runtime_features(): - grep_output = subprocess.check_output( - 'grep -r "envoy.reloadable_features\." source/*', shell=True) + grep_output = subprocess.check_output('grep -r "envoy.reloadable_features\." source/*', + shell=True) features_to_flip = set() diff --git a/tools/envoy_collect/envoy_collect.py b/tools/envoy_collect/envoy_collect.py index c22a526a37b7..60aa85e01525 100755 --- a/tools/envoy_collect/envoy_collect.py +++ b/tools/envoy_collect/envoy_collect.py @@ -127,8 +127,11 @@ def envoy_preexec_fn(): # Launch Envoy, register for SIGINT, and wait for the child process to exit. with open(envoy_log_path, 'w') as envoy_log: - envoy_proc = sp.Popen( - envoy_shcmd, stdin=sp.PIPE, stderr=envoy_log, preexec_fn=envoy_preexec_fn, shell=True) + envoy_proc = sp.Popen(envoy_shcmd, + stdin=sp.PIPE, + stderr=envoy_log, + preexec_fn=envoy_preexec_fn, + shell=True) def signal_handler(signum, frame): # The read is deferred until the signal so that the Envoy process gets a @@ -227,17 +230,18 @@ def envoy_collect(parse_result, unknown_args): # We either need to interpret or override these, so we declare them in # envoy_collect.py and always parse and present them again when invoking # Envoy. - parser.add_argument( - '--config-path', '-c', required=True, help='Path to Envoy configuration file.') - parser.add_argument( - '--log-level', '-l', help='Envoy log level. This will be overridden when invoking Envoy.') + parser.add_argument('--config-path', + '-c', + required=True, + help='Path to Envoy configuration file.') + parser.add_argument('--log-level', + '-l', + help='Envoy log level. This will be overridden when invoking Envoy.') # envoy_collect specific args. - parser.add_argument( - '--performance', - action='store_true', - help='Performance mode (collect perf trace, minimize log verbosity).') - parser.add_argument( - '--envoy-binary', - default=DEFAULT_ENVOY_PATH, - help='Path to Envoy binary (%s by default).' % DEFAULT_ENVOY_PATH) + parser.add_argument('--performance', + action='store_true', + help='Performance mode (collect perf trace, minimize log verbosity).') + parser.add_argument('--envoy-binary', + default=DEFAULT_ENVOY_PATH, + help='Path to Envoy binary (%s by default).' % DEFAULT_ENVOY_PATH) sys.exit(envoy_collect(*parser.parse_known_args(sys.argv))) diff --git a/tools/find_related_envoy_files.py b/tools/find_related_envoy_files.py index f1151af783f9..60acbcc30ab4 100755 --- a/tools/find_related_envoy_files.py +++ b/tools/find_related_envoy_files.py @@ -58,8 +58,8 @@ def emit(source_path, dest_path, source_ending, dest_ending): if fname.endswith(source_ending) and path.startswith(source_path + "/"): path_len = len(path) - len(source_path) - len(source_ending) - new_path = ( - absolute_location + dest_path + path[len(source_path):-len(source_ending)] + dest_ending) + new_path = (absolute_location + dest_path + path[len(source_path):-len(source_ending)] + + dest_ending) if os.path.isfile(new_path): print(new_path) diff --git a/tools/format_python_tools.py b/tools/format_python_tools.py index e81cb11a6a0c..8b2cacc321bb 100644 --- a/tools/format_python_tools.py +++ b/tools/format_python_tools.py @@ -23,7 +23,8 @@ def collectFiles(): for root, dirnames, filenames in os.walk(dirname): dirnames[:] = [d for d in dirnames if d not in EXCLUDE_LIST] for filename in fnmatch.filter(filenames, '*.py'): - matches.append(os.path.join(root, filename)) + if not filename.endswith('_pb2.py') and not filename.endswith('_pb2_grpc.py'): + matches.append(os.path.join(root, filename)) return matches @@ -37,8 +38,10 @@ def validateFormat(fix=False): failed_update_files = set() successful_update_files = set() for python_file in collectFiles(): - reformatted_source, encoding, changed = FormatFile( - python_file, style_config='.style.yapf', in_place=fix, print_diff=not fix) + reformatted_source, encoding, changed = FormatFile(python_file, + style_config='.style.yapf', + in_place=fix, + print_diff=not fix) if not fix: fixes_required = True if changed else fixes_required if reformatted_source: @@ -64,8 +67,10 @@ def displayFixResults(successful_files, failed_files): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Tool to format python files.') - parser.add_argument( - 'action', choices=['check', 'fix'], default='check', help='Fix invalid syntax in files.') + parser.add_argument('action', + choices=['check', 'fix'], + default='check', + help='Fix invalid syntax in files.') args = parser.parse_args() is_valid = validateFormat(args.action == 'fix') sys.exit(0 if is_valid else 1) diff --git a/tools/gen_compilation_database.py b/tools/gen_compilation_database.py index dadb9a1348c3..b987a403111f 100755 --- a/tools/gen_compilation_database.py +++ b/tools/gen_compilation_database.py @@ -12,8 +12,8 @@ def generateCompilationDatabase(args): ["bazel", "build", "--build_tag_filters=-manual", "--jobs=" + os.environ.get('NUM_CPUS')] + args.bazel_targets) - gen_compilation_database_sh = os.path.join( - os.path.realpath(os.path.dirname(__file__)), "../bazel/gen_compilation_database.sh") + gen_compilation_database_sh = os.path.join(os.path.realpath(os.path.dirname(__file__)), + "../bazel/gen_compilation_database.sh") subprocess.check_call([gen_compilation_database_sh] + args.bazel_targets) @@ -80,8 +80,9 @@ def fixCompilationDatabase(args): parser.add_argument('--include_genfiles', action='store_true') parser.add_argument('--include_headers', action='store_true') parser.add_argument('--vscode', action='store_true') - parser.add_argument( - 'bazel_targets', nargs='*', default=["//source/...", "//test/...", "//tools/..."]) + parser.add_argument('bazel_targets', + nargs='*', + default=["//source/...", "//test/...", "//tools/..."]) args = parser.parse_args() generateCompilationDatabase(args) fixCompilationDatabase(args) diff --git a/tools/gen_gdb_wrapper_script.py b/tools/gen_gdb_wrapper_script.py index a23d19fa6c57..03508a7b1dbe 100755 --- a/tools/gen_gdb_wrapper_script.py +++ b/tools/gen_gdb_wrapper_script.py @@ -30,10 +30,9 @@ test_args[0] = os.path.abspath(test_args[0]) with open(generated_path, 'w') as f: f.write( - GDB_RUNNER_SCRIPT.substitute( - b64env=str(dict(os.environ)), - gdb=gdb, - test_args=' '.join(pipes.quote(arg) for arg in test_args))) + GDB_RUNNER_SCRIPT.substitute(b64env=str(dict(os.environ)), + gdb=gdb, + test_args=' '.join(pipes.quote(arg) for arg in test_args))) # To make bazel consider the test a failure we exit non-zero. print('Test was not run, instead a gdb wrapper script was produced in %s' % generated_path) sys.exit(1) diff --git a/tools/header_order.py b/tools/header_order.py index 6949547bd9d4..9962d825a3f5 100755 --- a/tools/header_order.py +++ b/tools/header_order.py @@ -108,11 +108,10 @@ def regex_filter(regex): parser = argparse.ArgumentParser(description='Header reordering.') parser.add_argument('--path', type=str, help='specify the path to the header file') parser.add_argument('--rewrite', action='store_true', help='rewrite header file in-place') - parser.add_argument( - '--include_dir_order', - type=str, - default=','.join(common.includeDirOrder()), - help='specify the header block include directory order') + parser.add_argument('--include_dir_order', + type=str, + default=','.join(common.includeDirOrder()), + help='specify the header block include directory order') args = parser.parse_args() target_path = args.path include_dir_order = args.include_dir_order.split(',') diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 2a8775aef1b2..8835dce7e40a 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -552,8 +552,8 @@ def FormatFieldAsDefinitionListItem(outer_type_context, type_context, field): comment = '(%s) ' % ', '.join([FormatFieldType(type_context, field)] + annotations) + leading_comment - return anchor + field.name + '\n' + MapLines( - functools.partial(Indent, 2), comment + oneof_comment) + return anchor + field.name + '\n' + MapLines(functools.partial(Indent, 2), + comment + oneof_comment) def FormatMessageAsDefinitionList(type_context, msg): @@ -599,8 +599,8 @@ def FormatMessage(type_context, msg): # We need to do some extra work to recover the map type annotation from the # synthesized messages. type_context.map_typenames = { - '%s.%s' % (type_context.name, nested_msg.name): 'map<%s, %s>' % tuple( - map(functools.partial(FormatFieldType, type_context), nested_msg.field)) + '%s.%s' % (type_context.name, nested_msg.name): + 'map<%s, %s>' % tuple(map(functools.partial(FormatFieldType, type_context), nested_msg.field)) for nested_msg in msg.nested_type if nested_msg.options.map_entry } @@ -650,8 +650,8 @@ def FormatEnumAsDefinitionList(type_context, enum): RST formatted definition list item. """ return '\n'.join( - FormatEnumValueAsDefinitionListItem( - type_context.ExtendEnumValue(index, enum_value.name), enum_value) + FormatEnumValueAsDefinitionListItem(type_context.ExtendEnumValue(index, enum_value.name), + enum_value) for index, enum_value in enumerate(enum.value)) + '\n' @@ -720,8 +720,8 @@ def Main(): if cprofile_enabled: pr.disable() stats_stream = StringIO.StringIO() - ps = pstats.Stats( - pr, stream=stats_stream).sort_stats(os.getenv('CPROFILE_SORTBY', 'cumulative')) + ps = pstats.Stats(pr, + stream=stats_stream).sort_stats(os.getenv('CPROFILE_SORTBY', 'cumulative')) stats_file = response.file.add() stats_file.name = proto_file.name + '.rst.profile' ps.print_stats() diff --git a/tools/requirements.txt b/tools/requirements.txt index b6920fa69873..048ee9d5a6e5 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,2 +1,2 @@ flake8==3.7.7 -yapf==0.25.0 +yapf==0.28.0 diff --git a/tools/socket_passing.py b/tools/socket_passing.py index ee30db34ad01..85c9194d9d84 100755 --- a/tools/socket_passing.py +++ b/tools/socket_passing.py @@ -80,20 +80,21 @@ def GenerateNewConfig(original_yaml, admin_address, updated_json): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Replace listener addressses in json file.') - parser.add_argument( - '-o', - '--original_json', - type=str, - required=True, - help='Path of the original config json file') - parser.add_argument( - '-a', '--admin_address_path', type=str, required=True, help='Path of the admin address file') - parser.add_argument( - '-u', - '--updated_json', - type=str, - required=True, - help='Path to output updated json config file') + parser.add_argument('-o', + '--original_json', + type=str, + required=True, + help='Path of the original config json file') + parser.add_argument('-a', + '--admin_address_path', + type=str, + required=True, + help='Path of the admin address file') + parser.add_argument('-u', + '--updated_json', + type=str, + required=True, + help='Path to output updated json config file') args = parser.parse_args() admin_address_path = args.admin_address_path From 5c9f1f077923cf1f76c37d9b623bebb0c40b2ec9 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Wed, 14 Aug 2019 20:22:45 -0700 Subject: [PATCH 393/542] router: remote implicit case of lambda to bool (#7930) Per #7736, we're inadvertently casting a lambda expression to bool. Fortunately, the code is attempting to pass true, so it works. Risk Level: low Testing: n/a Docs Changes: n/a Release Notes: n/a Fixes: #7736 Signed-off-by: Stephan Zuercher --- source/common/router/retry_state_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/router/retry_state_impl.cc b/source/common/router/retry_state_impl.cc index e7674bd9ba93..72b6e1b8a563 100644 --- a/source/common/router/retry_state_impl.cc +++ b/source/common/router/retry_state_impl.cc @@ -225,7 +225,7 @@ RetryStatus RetryStateImpl::shouldHedgeRetryPerTryTimeout(DoRetryCallback callba // retries are associated with a stream reset which is analogous to a gateway // error. When hedging on per try timeout is enabled, however, there is no // stream reset. - return shouldRetry([]() -> bool { return true; }, callback); + return shouldRetry(true, callback); } bool RetryStateImpl::wouldRetryFromHeaders(const Http::HeaderMap& response_headers) { From 882a30677619856446f7e1b9d28c6ab319b21d1b Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Wed, 14 Aug 2019 23:51:59 -0700 Subject: [PATCH 394/542] add DNS SAN as principal (#7881) Description: Adds support for DNS SAN as Principal in RBAC filter. Risk Level: Low Testing: Added automated tests Docs Changes: Updated Release Notes: Added Fixes #7836 Signed-off-by: Rama Chavali --- api/envoy/config/rbac/v2/rbac.proto | 5 ++-- docs/root/intro/version_history.rst | 1 + .../filters/common/rbac/matchers.cc | 13 ++++++++--- .../filters/common/rbac/matchers_test.cc | 23 +++++++++++++++++++ 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/api/envoy/config/rbac/v2/rbac.proto b/api/envoy/config/rbac/v2/rbac.proto index 3cfe43a828fe..77e1aa687fa9 100644 --- a/api/envoy/config/rbac/v2/rbac.proto +++ b/api/envoy/config/rbac/v2/rbac.proto @@ -170,8 +170,9 @@ message Principal { reserved 1; reserved "name"; - // The name of the principal. If set, The URI SAN is used from the certificate, otherwise the - // subject field is used. If unset, it applies to any user that is authenticated. + // The name of the principal. If set, The URI SAN or DNS SAN in that order is used from the + // certificate, otherwise the subject field is used. If unset, it applies to any user that is + // authenticated. envoy.type.matcher.StringMatcher principal_name = 2; } diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b7ac02962784..fa2a9a70b4ef 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -21,6 +21,7 @@ Version history * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. +* rbac: added support for DNS SAN as :ref:`principal_name `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. diff --git a/source/extensions/filters/common/rbac/matchers.cc b/source/extensions/filters/common/rbac/matchers.cc index d718e8b34167..07472b49a003 100644 --- a/source/extensions/filters/common/rbac/matchers.cc +++ b/source/extensions/filters/common/rbac/matchers.cc @@ -142,10 +142,17 @@ bool AuthenticatedMatcher::matches(const Network::Connection& connection, const auto uriSans = ssl->uriSanPeerCertificate(); std::string principal; - if (uriSans.empty()) { - principal = ssl->subjectPeerCertificate(); - } else { + // If set, The URI SAN or DNS SAN in that order is used as Principal, otherwise the subject field + // is used. + if (!uriSans.empty()) { principal = uriSans[0]; + } else { + const auto dnsSans = ssl->dnsSansPeerCertificate(); + if (!dnsSans.empty()) { + principal = dnsSans[0]; + } else { + principal = ssl->subjectPeerCertificate(); + } } return matcher_.value().match(principal); diff --git a/test/extensions/filters/common/rbac/matchers_test.cc b/test/extensions/filters/common/rbac/matchers_test.cc index 424c1c56a74e..1262d8281550 100644 --- a/test/extensions/filters/common/rbac/matchers_test.cc +++ b/test/extensions/filters/common/rbac/matchers_test.cc @@ -205,12 +205,35 @@ TEST(AuthenticatedMatcher, uriSanPeerCertificate) { checkMatcher(AuthenticatedMatcher(auth), false, conn); } +TEST(AuthenticatedMatcher, dnsSanPeerCertificate) { + Envoy::Network::MockConnection conn; + Envoy::Ssl::MockConnectionInfo ssl; + + const std::vector uri_sans; + const std::vector dns_sans{"foo", "baz"}; + + EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(uri_sans)); + EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(&ssl)); + + EXPECT_CALL(ssl, dnsSansPeerCertificate()).WillRepeatedly(Return(dns_sans)); + EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(&ssl)); + + // We should get the first DNS SAN as URI SAN is not available. + envoy::config::rbac::v2::Principal_Authenticated auth; + auth.mutable_principal_name()->set_exact("foo"); + checkMatcher(AuthenticatedMatcher(auth), true, conn); + + auth.mutable_principal_name()->set_exact("bar"); + checkMatcher(AuthenticatedMatcher(auth), false, conn); +} + TEST(AuthenticatedMatcher, subjectPeerCertificate) { Envoy::Network::MockConnection conn; Envoy::Ssl::MockConnectionInfo ssl; const std::vector sans; EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(ssl, dnsSansPeerCertificate()).WillRepeatedly(Return(sans)); EXPECT_CALL(ssl, subjectPeerCertificate()).WillRepeatedly(Return("bar")); EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(&ssl)); From 15d54866d6a627accb2ca42a23367c66a83aaf18 Mon Sep 17 00:00:00 2001 From: Guangming Wang Date: Fri, 16 Aug 2019 03:23:28 +0800 Subject: [PATCH 395/542] fix misspelled words in client.py help message (#7933) Signed-off-by: Guangming Wang --- test/extensions/filters/network/thrift_proxy/driver/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/extensions/filters/network/thrift_proxy/driver/client.py b/test/extensions/filters/network/thrift_proxy/driver/client.py index 54e30e70a62b..f323cd7f3a49 100755 --- a/test/extensions/filters/network/thrift_proxy/driver/client.py +++ b/test/extensions/filters/network/thrift_proxy/driver/client.py @@ -233,7 +233,7 @@ def main(cfg, reqhandle, resphandle): "--headers", dest="headers", metavar="KEY=VALUE[,KEY=VALUE]", - help="list of comma-delimited, key value pairs to include as tranport headers.", + help="list of comma-delimited, key value pairs to include as transport headers.", ) cfg = parser.parse_args() From b73e5e68d784743a93930a83939916699360a317 Mon Sep 17 00:00:00 2001 From: antonio Date: Thu, 15 Aug 2019 15:35:27 -0400 Subject: [PATCH 396/542] LoadBalancer: Skip EdfScheduler creation in LoadBalancerBase if all host weights are equal. (#7877) Only create EdfScheduler for round-robing and least-request load balancers if hosts have different weights. This reduces CPU usage during load balancer creation and memory usage in the common case where all hosts have the same weight. Signed-off-by: Antonio Vicente --- .../load_balancing/load_balancers.rst | 24 +++---- docs/root/intro/version_history.rst | 1 + source/common/upstream/load_balancer_impl.cc | 43 +++++++++--- source/common/upstream/load_balancer_impl.h | 18 ++--- test/common/upstream/BUILD | 1 + .../upstream/load_balancer_benchmark.cc | 68 +++++++++++++++++-- .../upstream/load_balancer_impl_test.cc | 4 +- test/integration/stats_integration_test.cc | 3 +- 8 files changed, 119 insertions(+), 43 deletions(-) diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst b/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst index df762656fea6..5b6c4bb5c40f 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/load_balancers.rst @@ -28,10 +28,10 @@ effective weighting. Weighted least request ^^^^^^^^^^^^^^^^^^^^^^ -The least request load balancer uses different algorithms depending on whether any of the hosts have -weight greater than 1. +The least request load balancer uses different algorithms depending on whether hosts have the +same or different weights. -* *all weights 1*: An O(1) algorithm which selects N random available hosts as specified in the +* *all weights equal*: An O(1) algorithm which selects N random available hosts as specified in the :ref:`configuration ` (2 by default) and picks the host which has the fewest active requests (`Research `_ has shown that this @@ -39,17 +39,13 @@ weight greater than 1. choices). The P2C load balancer has the property that a host with the highest number of active requests in the cluster will never receive new requests. It will be allowed to drain until it is less than or equal to all of the other hosts. -* *not all weights 1*: If any host in the cluster has a load balancing weight greater than 1, the - load balancer shifts into a mode where it uses a weighted round robin schedule in which weights - are dynamically adjusted based on the host's request load at the time of selection (weight is - divided by the current active request count. For example, a host with weight 2 and an active - request count of 4 will have a synthetic weight of 2 / 4 = 0.5). This algorithm provides good - balance at steady state but may not adapt to load imbalance as quickly. Additionally, unlike P2C, - a host will never truly drain, though it will receive fewer requests over time. - - .. note:: - If all weights are not 1, but are the same (e.g., 42), Envoy will still use the weighted round - robin schedule instead of P2C. +* *all weights not equal*: If two or more hosts in the cluster have different load balancing + weights, the load balancer shifts into a mode where it uses a weighted round robin schedule in + which weights are dynamically adjusted based on the host's request load at the time of selection + (weight is divided by the current active request count. For example, a host with weight 2 and an + active request count of 4 will have a synthetic weight of 2 / 4 = 0.5). This algorithm provides + good balance at steady state but may not adapt to load imbalance as quickly. Additionally, unlike + P2C, a host will never truly drain, though it will receive fewer requests over time. .. _arch_overview_load_balancing_types_ring_hash: diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index fa2a9a70b4ef..0d309f18f022 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -30,6 +30,7 @@ Version history * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. * upstream: added network filter chains to upstream connections, see :ref:`filters`. +* upstream: use p2c to select hosts for least-requests load balancers if all host weights are the same, even in cases where weights are not equal to 1. * zookeeper: parse responses and emit latency stats. 1.11.1 (August 13, 2019) diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index e4b653d928dc..79bf87b79c44 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -50,6 +50,20 @@ std::pair distributeLoad(PriorityLoad& per_priority_load, return {first_available_priority, total_load}; } +// Returns true if the weights of all the hosts in the HostVector are equal. +bool hostWeightsAreEqual(const HostVector& hosts) { + if (hosts.size() <= 1) { + return true; + } + const uint32_t weight = hosts[0]->weight(); + for (size_t i = 1; i < hosts.size(); ++i) { + if (hosts[i]->weight() != weight) { + return false; + } + } + return true; +} + } // namespace std::pair @@ -629,6 +643,16 @@ void EdfLoadBalancerBase::refresh(uint32_t priority) { auto& scheduler = scheduler_[source] = Scheduler{}; refreshHostSource(source); + // Check if the original host weights are equal and skip EDF creation if they are. When all + // original weights are equal we can rely on unweighted host pick to do optimal round robin and + // least-loaded host selection with lower memory and CPU overhead. + if (hostWeightsAreEqual(hosts)) { + // Skip edf creation. + return; + } + + scheduler.edf_ = std::make_unique>(); + // Populate scheduler with host list. // TODO(mattklein123): We must build the EDF schedule even if all of the hosts are currently // weighted 1. This is because currently we don't refresh host sets if only weights change. @@ -639,7 +663,7 @@ void EdfLoadBalancerBase::refresh(uint32_t priority) { // notification, this will only be stale until this host is next picked, // at which point it is reinserted into the EdfScheduler with its new // weight in chooseHost(). - scheduler.edf_.add(hostWeight(*host), host); + scheduler.edf_->add(hostWeight(*host), host); } // Cycle through hosts to achieve the intended offset behavior. @@ -647,8 +671,8 @@ void EdfLoadBalancerBase::refresh(uint32_t priority) { // refreshes for the weighted case. if (!hosts.empty()) { for (uint32_t i = 0; i < seed_ % hosts.size(); ++i) { - auto host = scheduler.edf_.pick(); - scheduler.edf_.add(hostWeight(*host), host); + auto host = scheduler.edf_->pick(); + scheduler.edf_->add(hostWeight(*host), host); } } }; @@ -684,15 +708,12 @@ HostConstSharedPtr EdfLoadBalancerBase::chooseHostOnce(LoadBalancerContext* cont // As has been commented in both EdfLoadBalancerBase::refresh and // BaseDynamicClusterImpl::updateDynamicHostList, we must do a runtime pivot here to determine - // whether to use EDF or do unweighted (fast) selection. - // TODO(mattklein123): As commented elsewhere, this is wasteful, and we should just refresh the - // host set if any weights change. Additionally, it has the property that if all weights are - // the same but not 1 (like 42), we will use the EDF schedule not the unweighted pick. This is - // not optimal. If this is fixed, remove the note in the arch overview docs for the LR LB. - if (stats_.max_host_weight_.value() != 1) { - auto host = scheduler.edf_.pick(); + // whether to use EDF or do unweighted (fast) selection. EDF is non-null iff the original weights + // of 2 or more hosts differ. + if (scheduler.edf_ != nullptr) { + auto host = scheduler.edf_->pick(); if (host != nullptr) { - scheduler.edf_.add(hostWeight(*host), host); + scheduler.edf_->add(hostWeight(*host), host); } return host; } else { diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 518ea5071121..98fd0c75d7d1 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -346,8 +346,10 @@ class EdfLoadBalancerBase : public ZoneAwareLoadBalancerBase { protected: struct Scheduler { - // EdfScheduler for weighted LB. - EdfScheduler edf_; + // EdfScheduler for weighted LB. The edf_ is only created when the original + // host weights of 2 or more hosts differ. When not present, the + // implementation of chooseHostOnce falls back to unweightedHostPick. + std::unique_ptr> edf_; }; void initialize(); @@ -408,12 +410,12 @@ class RoundRobinLoadBalancer : public EdfLoadBalancerBase { /** * Weighted Least Request load balancer. * - * In a normal setup when all hosts have the same weight of 1 it randomly picks up N healthy hosts + * In a normal setup when all hosts have the same weight it randomly picks up N healthy hosts * (where N is specified in the LB configuration) and compares number of active requests. Technique * is based on http://www.eecs.harvard.edu/~michaelm/postscripts/mythesis.pdf and is known as P2C * (power of two choices). * - * When any hosts have a weight that is not 1, an RR EDF schedule is used. Host weight is scaled + * When hosts have different weights, an RR EDF schedule is used. Host weight is scaled * by the number of active requests at pick/insert time. Thus, hosts will never fully drain as * they would in normal P2C, though they will get picked less and less often. In the future, we * can consider two alternate algorithms: @@ -442,11 +444,9 @@ class LeastRequestLoadBalancer : public EdfLoadBalancerBase { void refreshHostSource(const HostsSource&) override {} double hostWeight(const Host& host) override { // Here we scale host weight by the number of active requests at the time we do the pick. We - // always add 1 to avoid division by 0. Note that if all weights are 1, the EDF schedule is - // unlikely to yield the same result as P2C given the lack of randomness as well as the fact - // that hosts are always picked, regardless of their current request load at the time of pick. - // It might be possible to do better by picking two hosts off of the schedule, and selecting - // the one with fewer active requests at the time of selection. + // always add 1 to avoid division by 0. It might be possible to do better by picking two hosts + // off of the schedule, and selecting the one with fewer active requests at the time of + // selection. // TODO(mattklein123): @htuch brings up the point that how we are scaling weight here might not // be the only/best way of doing this. Essentially, it makes weight and active requests equally // important. Are they equally important in practice? There is no right answer here and we might diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 566e06cf6012..4e5d8b4679ba 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -333,6 +333,7 @@ envoy_cc_binary( "benchmark", ], deps = [ + "//source/common/memory:stats_lib", "//source/common/upstream:maglev_lb_lib", "//source/common/upstream:ring_hash_lb_lib", "//source/common/upstream:upstream_lib", diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc index 9bd79569b52f..b1f736873346 100644 --- a/test/common/upstream/load_balancer_benchmark.cc +++ b/test/common/upstream/load_balancer_benchmark.cc @@ -2,6 +2,7 @@ #include +#include "common/memory/stats.h" #include "common/runtime/runtime_impl.h" #include "common/upstream/maglev_lb.h" #include "common/upstream/ring_hash_lb.h" @@ -27,15 +28,18 @@ class BaseTester { hosts.push_back(makeTestHost(info_, fmt::format("tcp://10.0.{}.{}:6379", i / 256, i % 256), should_weight ? weight : 1)); } - HostVectorConstSharedPtr updated_hosts{new HostVector(hosts)}; - priority_set_.updateHosts( - 0, - updateHostsParams(updated_hosts, nullptr, - std::make_shared(*updated_hosts), nullptr), - {}, hosts, {}, absl::nullopt); + + HostVectorConstSharedPtr updated_hosts = std::make_shared(hosts); + HostsPerLocalityConstSharedPtr hosts_per_locality = makeHostsPerLocality({hosts}); + priority_set_.updateHosts(0, HostSetImpl::partitionHosts(updated_hosts, hosts_per_locality), {}, + hosts, {}, absl::nullopt); + local_priority_set_.updateHosts(0, + HostSetImpl::partitionHosts(updated_hosts, hosts_per_locality), + {}, hosts, {}, absl::nullopt); } PrioritySetImpl priority_set_; + PrioritySetImpl local_priority_set_; Stats::IsolatedStoreImpl stats_store_; ClusterStats stats_{ClusterInfoImpl::generateStats(stats_store_)}; NiceMock runtime_; @@ -44,6 +48,58 @@ class BaseTester { std::shared_ptr info_{new NiceMock()}; }; +class RoundRobinTester : public BaseTester { +public: + RoundRobinTester(uint64_t num_hosts, uint32_t weighted_subset_percent = 0, uint32_t weight = 0) + : BaseTester(num_hosts, weighted_subset_percent, weight) {} + + void initialize() { + lb_ = std::make_unique(priority_set_, &local_priority_set_, stats_, + runtime_, random_, common_config_); + } + + std::unique_ptr lb_; +}; + +void BM_RoundRobinLoadBalancerBuild(benchmark::State& state) { + for (auto _ : state) { + state.PauseTiming(); + const uint64_t num_hosts = state.range(0); + const uint64_t weighted_subset_percent = state.range(1); + const uint64_t weight = state.range(2); + + RoundRobinTester tester(num_hosts, weighted_subset_percent, weight); + const size_t start_mem = Memory::Stats::totalCurrentlyAllocated(); + + // We are only interested in timing the initial build. + state.ResumeTiming(); + tester.initialize(); + state.PauseTiming(); + const size_t end_mem = Memory::Stats::totalCurrentlyAllocated(); + state.counters["memory"] = end_mem - start_mem; + state.counters["memory_per_host"] = (end_mem - start_mem) / num_hosts; + state.ResumeTiming(); + } +} +BENCHMARK(BM_RoundRobinLoadBalancerBuild) + ->Args({1, 0, 1}) + ->Args({500, 0, 1}) + ->Args({500, 50, 50}) + ->Args({500, 100, 50}) + ->Args({2500, 0, 1}) + ->Args({2500, 50, 50}) + ->Args({2500, 100, 50}) + ->Args({10000, 0, 1}) + ->Args({10000, 50, 50}) + ->Args({10000, 100, 50}) + ->Args({25000, 0, 1}) + ->Args({25000, 50, 50}) + ->Args({25000, 100, 50}) + ->Args({50000, 0, 1}) + ->Args({50000, 50, 50}) + ->Args({50000, 100, 50}) + ->Unit(benchmark::kMillisecond); + class RingHashTester : public BaseTester { public: RingHashTester(uint64_t num_hosts, uint64_t min_ring_size) : BaseTester(num_hosts) { diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 5a42f7bf5edf..e29dd9b0dcd8 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -1229,7 +1229,7 @@ TEST_P(LeastRequestLoadBalancerTest, SingleHost) { // Host weight is 100. { - EXPECT_CALL(random_, random()).WillOnce(Return(0)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(2)).WillOnce(Return(3)); stats_.max_host_weight_.set(100UL); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_.chooseHost(nullptr)); } @@ -1237,7 +1237,7 @@ TEST_P(LeastRequestLoadBalancerTest, SingleHost) { HostVector empty; { hostSet().runCallbacks(empty, empty); - EXPECT_CALL(random_, random()).WillOnce(Return(0)); + EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(2)).WillOnce(Return(3)); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_.chooseHost(nullptr)); } diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index cc016ffbea1a..ceadf07bacd7 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -212,6 +212,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // 2019/07/06 7477 42742 43000 fork gauge representation to drop pending_increment_ // 2019/07/15 7555 42806 43000 static link libstdc++ in tests // 2019/07/24 7503 43030 44000 add upstream filters to clusters + // 2019/08/13 7877 42838 44000 skip EdfScheduler creation if all host weights equal // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -221,7 +222,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 43030); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 42838); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 44000); } From 6ab335198b12f46797d2d734f58401990487452f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Thu, 15 Aug 2019 15:36:20 -0400 Subject: [PATCH 397/542] Make iterators const (#7924) Signed-off-by: Raul Gutierrez Segales --- source/common/config/watch_map.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index 718231009101..f351dd962a2f 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -42,7 +42,7 @@ AddedRemoved WatchMap::updateWatchInterest(Watch* watch, absl::flat_hash_set WatchMap::watchesInterestedIn(const std::string& resource_name) { absl::flat_hash_set ret = wildcard_watches_; - auto watches_interested = watch_interest_.find(resource_name); + const auto watches_interested = watch_interest_.find(resource_name); if (watches_interested != watch_interest_.end()) { for (const auto& watch : watches_interested->second) { ret.insert(watch); @@ -73,7 +73,7 @@ void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField // We just bundled up the updates into nice per-watch packages. Now, deliver them. for (auto& watch : watches_) { - auto this_watch_updates = per_watch_updates.find(watch); + const auto this_watch_updates = per_watch_updates.find(watch); if (this_watch_updates == per_watch_updates.end()) { // This update included no resources this watch cares about - so we do an empty // onConfigUpdate(), to notify the watch that its resources - if they existed before this - @@ -110,7 +110,7 @@ void WatchMap::onConfigUpdate( // We just bundled up the updates into nice per-watch packages. Now, deliver them. for (const auto& added : per_watch_added) { const Watch* cur_watch = added.first; - auto removed = per_watch_removed.find(cur_watch); + const auto removed = per_watch_removed.find(cur_watch); if (removed == per_watch_removed.end()) { // additions only, no removals cur_watch->callbacks_.onConfigUpdate(added.second, {}, system_version_info); From d4186f2c636d768e01b78ddadb0718a2ec1c0abf Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Thu, 15 Aug 2019 15:14:55 -0700 Subject: [PATCH 398/542] bazel: set strict action env (#7940) We pass all environment variables we care about explicitly and should not pass any other not set variables. Signed-off-by: Matt Klein --- .bazelrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.bazelrc b/.bazelrc index 552052098379..a74e3e350c02 100644 --- a/.bazelrc +++ b/.bazelrc @@ -13,6 +13,7 @@ startup --host_jvm_args=-Xmx2g build --workspace_status_command=bazel/get_workspace_status build --experimental_remap_main_repo build --experimental_local_memory_estimate +build --experimental_strict_action_env=true build --host_force_python=PY2 build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a build --action_env=BAZEL_LINKOPTS=-lm:-static-libgcc From 4f0fc2dd5522669950c19b8529f528d47a84c8b2 Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 15 Aug 2019 18:15:24 -0400 Subject: [PATCH 399/542] runtime: add static layer case to loader impl (#7932) Signed-off-by: Asra Ali --- source/common/runtime/runtime_impl.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 2b67fdeafb78..3e86fcba3329 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -474,6 +474,9 @@ LoaderImpl::LoaderImpl(Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator throw EnvoyException(absl::StrCat("Duplicate layer name: ", layer.name())); } switch (layer.layer_specifier_case()) { + case envoy::config::bootstrap::v2::RuntimeLayer::kStaticLayer: + // Nothing needs to be done here. + break; case envoy::config::bootstrap::v2::RuntimeLayer::kAdminLayer: if (admin_layer_ != nullptr) { throw EnvoyException( @@ -494,8 +497,7 @@ LoaderImpl::LoaderImpl(Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator init_manager.add(subscriptions_.back()->init_target_); break; default: - ENVOY_LOG(warn, "Skipping unsupported runtime layer: {}", layer.DebugString()); - break; + NOT_REACHED_GCOVR_EXCL_LINE; } } From e64361bcc3ed77f0721dee83915810b7f6c3ec0d Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 15 Aug 2019 18:16:22 -0400 Subject: [PATCH 400/542] test: mock cleanup (#7922) Signed-off-by: Alyssa Wilk --- .../access_log_manager_impl_test.cc | 14 +- .../config/delta_subscription_test_harness.h | 2 +- .../config/grpc_subscription_test_harness.h | 2 +- .../config/http_subscription_test_harness.h | 2 +- test/common/http/async_client_impl_test.cc | 10 +- test/common/http/conn_manager_impl_test.cc | 24 +-- test/common/http/date_provider_impl_test.cc | 2 +- test/common/http/http1/conn_pool_test.cc | 6 +- test/common/http/http2/conn_pool_test.cc | 2 +- test/common/network/connection_impl_test.cc | 9 +- test/common/router/retry_state_impl_test.cc | 60 +++--- test/common/router/router_test.cc | 44 ++--- .../common/router/router_upstream_log_test.cc | 2 +- test/common/tcp/conn_pool_test.cc | 6 +- test/common/tcp_proxy/tcp_proxy_test.cc | 6 +- .../upstream/cluster_manager_impl_test.cc | 14 +- .../upstream/health_checker_impl_test.cc | 183 +++++++++--------- .../upstream/logical_dns_cluster_test.cc | 8 +- .../upstream/original_dst_cluster_test.cc | 8 +- .../upstream/outlier_detection_impl_test.cc | 42 ++-- test/common/upstream/upstream_impl_test.cc | 36 ++-- .../clusters/redis/redis_cluster_test.cc | 14 +- .../http/health_check/health_check_test.cc | 4 +- .../filters/http/squash/squash_filter_test.cc | 11 +- .../client_ssl_auth/client_ssl_auth_test.cc | 8 +- .../network/common/redis/client_impl_test.cc | 4 +- .../filters/network/mongo_proxy/proxy_test.cc | 4 +- .../health_checkers/redis/redis_test.cc | 36 ++-- .../datadog/datadog_tracer_impl_test.cc | 2 +- .../lightstep/lightstep_tracer_impl_test.cc | 2 +- .../tracers/zipkin/zipkin_tracer_impl_test.cc | 2 +- test/mocks/event/mocks.h | 6 +- test/server/connection_handler_test.cc | 4 +- test/server/drain_manager_impl_test.cc | 4 +- 34 files changed, 290 insertions(+), 293 deletions(-) diff --git a/test/common/access_log/access_log_manager_impl_test.cc b/test/common/access_log/access_log_manager_impl_test.cc index 4681e9453a7c..151ac7544b47 100644 --- a/test/common/access_log/access_log_manager_impl_test.cc +++ b/test/common/access_log/access_log_manager_impl_test.cc @@ -84,7 +84,7 @@ TEST_F(AccessLogManagerImplTest, flushToLogFilePeriodically) { // make sure timer is re-enabled on callback call log_file->write("test2"); EXPECT_CALL(*timer, enableTimer(timeout_40ms_)); - timer->callback_(); + timer->invokeCallback(); { Thread::LockGuard lock(file_->write_mutex_); @@ -147,7 +147,7 @@ TEST_F(AccessLogManagerImplTest, flushToLogFileOnDemand) { // make sure timer is re-enabled on callback call log_file->write("test2"); EXPECT_CALL(*timer, enableTimer(timeout_40ms_)); - timer->callback_(); + timer->invokeCallback(); expected_writes++; { @@ -176,7 +176,7 @@ TEST_F(AccessLogManagerImplTest, reopenFile) { })); log_file->write("before"); - timer->callback_(); + timer->invokeCallback(); { Thread::LockGuard lock(file_->write_mutex_); @@ -205,7 +205,7 @@ TEST_F(AccessLogManagerImplTest, reopenFile) { log_file->reopen(); log_file->write("reopened"); - timer->callback_(); + timer->invokeCallback(); { Thread::LockGuard lock(file_->write_mutex_); @@ -237,7 +237,7 @@ TEST_F(AccessLogManagerImplTest, reopenThrows) { .WillOnce(Return(ByMove(Filesystem::resultFailure(false, 0)))); log_file->write("test write"); - timer->callback_(); + timer->invokeCallback(); { Thread::LockGuard lock(file_->write_mutex_); while (file_->num_writes_ != 1) { @@ -247,7 +247,7 @@ TEST_F(AccessLogManagerImplTest, reopenThrows) { log_file->reopen(); log_file->write("this is to force reopen"); - timer->callback_(); + timer->invokeCallback(); { Thread::LockGuard lock(file_->open_mutex_); @@ -258,7 +258,7 @@ TEST_F(AccessLogManagerImplTest, reopenThrows) { // write call should not cause any exceptions log_file->write("random data"); - timer->callback_(); + timer->invokeCallback(); } TEST_F(AccessLogManagerImplTest, bigDataChunkShouldBeFlushedWithoutTimer) { diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index da7b82d8007a..cae4cebbf298 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -164,7 +164,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(*init_timeout_timer_, disableTimer()); } - void callInitFetchTimeoutCb() override { init_timeout_timer_->callback_(); } + void callInitFetchTimeoutCb() override { init_timeout_timer_->invokeCallback(); } const Protobuf::MethodDescriptor* method_descriptor_; Grpc::MockAsyncClient* async_client_; diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 7c58a62bcb00..3031c2e947ef 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -149,7 +149,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(*init_timeout_timer_, disableTimer()); } - void callInitFetchTimeoutCb() override { init_timeout_timer_->callback_(); } + void callInitFetchTimeoutCb() override { init_timeout_timer_->invokeCallback(); } std::string version_; const Protobuf::MethodDescriptor* method_descriptor_; diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index 4178c82301c6..95d1123be5b5 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -166,7 +166,7 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(*init_timeout_timer_, disableTimer()); } - void callInitFetchTimeoutCb() override { init_timeout_timer_->callback_(); } + void callInitFetchTimeoutCb() override { init_timeout_timer_->invokeCallback(); } void timerTick() { expectSendMessage(cluster_names_, version_); diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index b1e30d65d5fd..ae5b7cdbb6fb 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -192,7 +192,7 @@ TEST_F(AsyncClientImplTest, Retry) { EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(&message_copy->headers()), false)); EXPECT_CALL(stream_encoder_, encodeData(BufferEqual(&data), true)); - timer_->callback_(); + timer_->invokeCallback(); // Normal response. expectSuccess(200); @@ -240,7 +240,7 @@ TEST_F(AsyncClientImplTest, RetryWithStream) { EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(&headers), false)); EXPECT_CALL(stream_encoder_, encodeData(BufferEqual(body.get()), true)); - timer_->callback_(); + timer_->invokeCallback(); // Normal response. expectResponseHeaders(stream_callbacks_, 200, true); @@ -731,7 +731,7 @@ TEST_F(AsyncClientImplTest, StreamTimeout) { AsyncClient::Stream* stream = client_.start( stream_callbacks_, AsyncClient::StreamOptions().setTimeout(std::chrono::milliseconds(40))); stream->sendHeaders(message_->headers(), true); - timer_->callback_(); + timer_->invokeCallback(); EXPECT_EQ(1UL, cm_.thread_local_cluster_.cluster_.info_->stats_store_.counter("upstream_rq_timeout") @@ -765,7 +765,7 @@ TEST_F(AsyncClientImplTest, StreamTimeoutHeadReply) { AsyncClient::Stream* stream = client_.start( stream_callbacks_, AsyncClient::StreamOptions().setTimeout(std::chrono::milliseconds(40))); stream->sendHeaders(message->headers(), true); - timer_->callback_(); + timer_->invokeCallback(); } TEST_F(AsyncClientImplTest, RequestTimeout) { @@ -783,7 +783,7 @@ TEST_F(AsyncClientImplTest, RequestTimeout) { EXPECT_CALL(stream_encoder_.stream_, resetStream(_)); client_.send(std::move(message_), callbacks_, AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds(40))); - timer_->callback_(); + timer_->invokeCallback(); EXPECT_EQ(1UL, cm_.thread_local_cluster_.cluster_.info_->stats_store_.counter("upstream_rq_timeout") diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 2ae4d2483eed..083bbbe73753 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -1410,7 +1410,7 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutGlobal) { // encodeHeaders()/encodeData(). EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); })); // 408 direct response after timeout. @@ -1449,7 +1449,7 @@ TEST_F(HttpConnectionManagerImplTest, AccessEncoderRouteBeforeHeadersArriveOnIdl EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); // Simulate and idle timeout so that the filter chain gets created. - idle_timer->callback_(); + idle_timer->invokeCallback(); })); // This should not be called as we don't have request headers. @@ -1489,7 +1489,7 @@ TEST_F(HttpConnectionManagerImplTest, TestStreamIdleAccessLog) { // encodeHeaders()/encodeData(). EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); })); std::shared_ptr filter(new NiceMock()); @@ -1597,7 +1597,7 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterDownstreamHeaders // encodeHeaders()/encodeData(). EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); data.drain(4); })); @@ -1669,7 +1669,7 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterDownstreamHeaders // encodeHeaders()/encodeData(). EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); data.drain(4); })); @@ -1720,7 +1720,7 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterUpstreamHeaders) filter->callbacks_->encodeHeaders(std::move(response_headers), false); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); data.drain(4); })); @@ -1784,7 +1784,7 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterBidiData) { filter->callbacks_->encodeData(fake_response, false); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); data.drain(4); })); @@ -1865,7 +1865,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutCallbackDisarmsAndReturns408 EXPECT_CALL(response_encoder_, encodeData(_, true)).WillOnce(AddBufferToString(&response_body)); conn_manager_->newStream(response_encoder_); - request_timer->callback_(); + request_timer->invokeCallback(); })); Buffer::OwnedImpl fake_input("1234"); @@ -2148,7 +2148,7 @@ TEST_F(HttpConnectionManagerImplTest, DrainClose) { EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWriteAndDelay)); EXPECT_CALL(*drain_timer, disableTimer()); - drain_timer->callback_(); + drain_timer->invokeCallback(); EXPECT_EQ(1U, stats_.named_.downstream_cx_drain_close_.value()); EXPECT_EQ(1U, stats_.named_.downstream_rq_3xx_.value()); @@ -2381,7 +2381,7 @@ TEST_F(HttpConnectionManagerImplTest, IdleTimeoutNoCodec) { EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); EXPECT_EQ(1U, stats_.named_.downstream_cx_idle_timeout_.value()); } @@ -2426,14 +2426,14 @@ TEST_F(HttpConnectionManagerImplTest, IdleTimeout) { Event::MockTimer* drain_timer = setUpTimer(); EXPECT_CALL(*drain_timer, enableTimer(_)); - idle_timer->callback_(); + idle_timer->invokeCallback(); EXPECT_CALL(*codec_, goAway()); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWriteAndDelay)); EXPECT_CALL(*idle_timer, disableTimer()); EXPECT_CALL(*drain_timer, disableTimer()); - drain_timer->callback_(); + drain_timer->invokeCallback(); EXPECT_EQ(1U, stats_.named_.downstream_cx_idle_timeout_.value()); } diff --git a/test/common/http/date_provider_impl_test.cc b/test/common/http/date_provider_impl_test.cc index ea037ee0f8d9..619a5f9a96e3 100644 --- a/test/common/http/date_provider_impl_test.cc +++ b/test/common/http/date_provider_impl_test.cc @@ -27,7 +27,7 @@ TEST(DateProviderImplTest, All) { EXPECT_NE(nullptr, headers.Date()); EXPECT_CALL(*timer, enableTimer(std::chrono::milliseconds(500))); - timer->callback_(); + timer->invokeCallback(); headers.removeDate(); provider.setDateHeader(headers); diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc index adc3e10472f9..d91c9cd862a7 100644 --- a/test/common/http/http1/conn_pool_test.cc +++ b/test/common/http/http1/conn_pool_test.cc @@ -109,7 +109,7 @@ class ConnPoolImplForTest : public ConnPoolImpl { void expectAndRunUpstreamReady() { EXPECT_TRUE(upstream_ready_enabled_); - mock_upstream_ready_timer_->callback_(); + mock_upstream_ready_timer_->invokeCallback(); EXPECT_FALSE(upstream_ready_enabled_); } @@ -400,10 +400,10 @@ TEST_F(Http1ConnPoolImplTest, ConnectTimeout) { EXPECT_NE(nullptr, conn_pool_.newStream(outer_decoder2, callbacks2)); })); - conn_pool_.test_clients_[0].connect_timer_->callback_(); + conn_pool_.test_clients_[0].connect_timer_->invokeCallback(); EXPECT_CALL(callbacks2.pool_failure_, ready()); - conn_pool_.test_clients_[1].connect_timer_->callback_(); + conn_pool_.test_clients_[1].connect_timer_->invokeCallback(); EXPECT_CALL(conn_pool_, onClientDestroy()).Times(2); dispatcher_.clearDeferredDeleteList(); diff --git a/test/common/http/http2/conn_pool_test.cc b/test/common/http/http2/conn_pool_test.cc index 9a17581301c7..6490da7337e1 100644 --- a/test/common/http/http2/conn_pool_test.cc +++ b/test/common/http/http2/conn_pool_test.cc @@ -622,7 +622,7 @@ TEST_F(Http2ConnPoolImplTest, ConnectTimeout) { expectClientCreate(); ActiveTestRequest r1(*this, 0, false); EXPECT_CALL(r1.callbacks_.pool_failure_, ready()); - test_clients_[0].connect_timer_->callback_(); + test_clients_[0].connect_timer_->invokeCallback(); EXPECT_CALL(*this, onClientDestroy()); dispatcher_.clearDeferredDeleteList(); diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index e4f280e96a17..734734f09bfc 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -1247,7 +1247,7 @@ TEST_P(ConnectionImplTest, DelayedCloseTimerResetWithPendingWriteBufferFlushes) (*mocks.file_ready_cb_)(Event::FileReadyType::Write); // Force the delayed close timeout to trigger so the connection is cleaned up. - mocks.timer_->callback_(); + mocks.timer_->invokeCallback(); } // Test that tearing down the connection will disable the delayed close timer. @@ -1321,16 +1321,9 @@ TEST_P(ConnectionImplTest, DelayedCloseTimeoutNullStats) { EXPECT_CALL(*mocks.timer_, enableTimer(_)).Times(1); server_connection->close(ConnectionCloseType::FlushWriteAndDelay); EXPECT_CALL(*mocks.timer_, disableTimer()).Times(1); - // Copy the callback since mocks.timer will be freed when closeSocket() is called. - Event::TimerCb callback = mocks.timer_->callback_; // The following close() will call closeSocket() and reset internal data structures such as // stats. server_connection->close(ConnectionCloseType::NoFlush); - // Verify the onDelayedCloseTimeout() callback is resilient to the post closeSocket(), pre - // destruction state. This should not actually happen due to the timeout disablement in - // closeSocket(), but there is enough complexity in connection handling codepaths that being - // extra defensive is valuable. - callback(); } class FakeReadFilter : public Network::ReadFilter { diff --git a/test/common/router/retry_state_impl_test.cc b/test/common/router/retry_state_impl_test.cc index 7650fd5a3e54..6120470db1cb 100644 --- a/test/common/router/retry_state_impl_test.cc +++ b/test/common/router/retry_state_impl_test.cc @@ -77,7 +77,7 @@ TEST_F(RouterRetryStateImplTest, PolicyRefusedStream) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(remote_refused_stream_reset_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryReset(remote_refused_stream_reset_, callback_)); @@ -98,7 +98,7 @@ TEST_F(RouterRetryStateImplTest, Policy5xxRemoteReset) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(remote_reset_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryReset(remote_reset_, callback_)); } @@ -112,7 +112,7 @@ TEST_F(RouterRetryStateImplTest, Policy5xxRemote503) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -146,7 +146,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGatewayErrorRemote502) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -161,7 +161,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGatewayErrorRemote503) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -176,7 +176,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGatewayErrorRemote504) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -197,7 +197,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGatewayErrorRemoteReset) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(remote_reset_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryReset(remote_reset_, callback_)); } @@ -211,7 +211,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGrpcCancelled) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -226,7 +226,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGrpcDeadlineExceeded) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -241,7 +241,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGrpcResourceExhausted) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -256,7 +256,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGrpcUnavilable) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -271,7 +271,7 @@ TEST_F(RouterRetryStateImplTest, PolicyGrpcInternal) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryHeaders(response_headers, callback_)); @@ -314,7 +314,7 @@ TEST_F(RouterRetryStateImplTest, PolicyConnectFailureResetConnectFailure) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); } TEST_F(RouterRetryStateImplTest, PolicyRetriable4xxRetry) { @@ -326,7 +326,7 @@ TEST_F(RouterRetryStateImplTest, PolicyRetriable4xxRetry) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryHeaders(response_headers, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); } TEST_F(RouterRetryStateImplTest, PolicyRetriable4xxNoRetry) { @@ -420,7 +420,7 @@ TEST_F(RouterRetryStateImplTest, PolicyResetRemoteReset) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(remote_reset_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, state_->shouldRetryReset(remote_reset_, callback_)); } @@ -435,7 +435,7 @@ TEST_F(RouterRetryStateImplTest, RouteConfigNoHeaderConfig) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); } TEST_F(RouterRetryStateImplTest, NoAvailableRetries) { @@ -464,17 +464,17 @@ TEST_F(RouterRetryStateImplTest, MaxRetriesHeader) { expectTimerCreateAndEnable(); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(*retry_timer_, enableTimer(_)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(*retry_timer_, enableTimer(_)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_EQ(1UL, cluster_.circuit_breakers_stats_.rq_retry_open_.value()); EXPECT_EQ(RetryStatus::NoRetryLimitExceeded, @@ -496,19 +496,19 @@ TEST_F(RouterRetryStateImplTest, Backoff) { EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(24))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(149)); EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(74))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(349)); EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(174))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); Http::TestHeaderMapImpl response_headers{{":status", "200"}}; EXPECT_EQ(RetryStatus::No, state_->shouldRetryHeaders(response_headers, callback_)); @@ -533,25 +533,25 @@ TEST_F(RouterRetryStateImplTest, CustomBackOffInterval) { EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(49))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(350)); EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(50))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(751)); EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(51))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(1499)); EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(1200))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); } // Test the default maximum retry back-off interval. @@ -568,25 +568,25 @@ TEST_F(RouterRetryStateImplTest, CustomBackOffIntervalDefaultMax) { EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(49))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(350)); EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(50))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(751)); EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(51))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(1499)); EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(1000))); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); - retry_timer_->callback_(); + retry_timer_->invokeCallback(); } TEST_F(RouterRetryStateImplTest, HostSelectionAttempts) { diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index b4d39e204140..770ce8bef6ae 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -1142,7 +1142,7 @@ TEST_F(RouterTest, UpstreamTimeout) { EXPECT_CALL(*router_.retry_state_, shouldRetryReset(_, _)).Times(0); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); - response_timeout_->callback_(); + response_timeout_->invokeCallback(); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_.counter("upstream_rq_timeout") @@ -1367,7 +1367,7 @@ TEST_F(RouterTest, UpstreamTimeoutWithAltResponse) { EXPECT_CALL( cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(204))); - response_timeout_->callback_(); + response_timeout_->invokeCallback(); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_.counter("upstream_rq_timeout") @@ -1415,7 +1415,7 @@ TEST_F(RouterTest, UpstreamPerTryTimeout) { EXPECT_CALL( cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ .counter("upstream_rq_per_try_timeout") @@ -1466,7 +1466,7 @@ TEST_F(RouterTest, UpstreamPerTryTimeoutDelayedPoolReady) { EXPECT_CALL(callbacks_, encodeData(_, true)); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ .counter("upstream_rq_per_try_timeout") @@ -1521,7 +1521,7 @@ TEST_F(RouterTest, UpstreamPerTryTimeoutExcludesNewStream) { {":status", "504"}, {"content-length", "24"}, {"content-type", "text/plain"}}; EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), false)); EXPECT_CALL(callbacks_, encodeData(_, true)); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ .counter("upstream_rq_per_try_timeout") @@ -1564,7 +1564,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { NiceMock encoder2; Http::StreamDecoder* response_decoder2 = nullptr; router_.retry_state_->expectHedgedPerTryTimeoutRetry(); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) @@ -1637,7 +1637,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutResetsOnBadHeaders) { NiceMock encoder2; Http::StreamDecoder* response_decoder2 = nullptr; router_.retry_state_->expectHedgedPerTryTimeoutRetry(); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) @@ -1750,7 +1750,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutThirdRequestSucceeds) { })); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); expectPerTryTimerCreate(); router_.retry_state_->callback_(); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); @@ -1807,7 +1807,7 @@ TEST_F(RouterTest, RetryOnlyOnceForSameUpstreamRequest) { cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); NiceMock encoder2; Http::StreamDecoder* response_decoder2 = nullptr; @@ -1834,7 +1834,7 @@ TEST_F(RouterTest, RetryOnlyOnceForSameUpstreamRequest) { cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); - response_timeout_->callback_(); + response_timeout_->invokeCallback(); } // Sequence: upstream request hits soft per try timeout and is retried, and @@ -1871,7 +1871,7 @@ TEST_F(RouterTest, BadHeadersDroppedIfPreviousRetryScheduled) { cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); expectPerTryTimerCreate(); @@ -1969,7 +1969,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutGlobalTimeout) { EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); NiceMock encoder2; Http::StreamDecoder* response_decoder2 = nullptr; @@ -1997,7 +1997,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutGlobalTimeout) { .WillOnce(Invoke([&](Http::HeaderMap& headers, bool) -> void { EXPECT_EQ(headers.Status()->value(), "504"); })); - response_timeout_->callback_(); + response_timeout_->invokeCallback(); EXPECT_TRUE(verifyHostUpstreamStats(0, 2)); EXPECT_EQ(2, cm_.conn_pool_.host_->stats_store_.counter("rq_timeout").value()); // TODO: Verify hedge stats here once they are implemented. @@ -2035,7 +2035,7 @@ TEST_F(RouterTest, HedgingRetriesExhaustedBadResponse) { EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); NiceMock encoder2; Http::StreamDecoder* response_decoder2 = nullptr; @@ -2119,7 +2119,7 @@ TEST_F(RouterTest, HedgingRetriesProceedAfterReset) { EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); router_.retry_state_->expectHedgedPerTryTimeoutRetry(); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); NiceMock encoder2; Http::StreamDecoder* response_decoder2 = nullptr; @@ -2199,7 +2199,7 @@ TEST_F(RouterTest, HedgingRetryImmediatelyReset) { putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, absl::optional(504))); EXPECT_CALL(encoder.stream_, resetStream(_)).Times(0); EXPECT_CALL(callbacks_, encodeHeaders_(_, _)).Times(0); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); NiceMock encoder2; EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) @@ -2342,7 +2342,7 @@ TEST_F(RouterTest, RetryUpstreamPerTryTimeout) { router_.retry_state_->expectResetRetry(); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); // We expect this reset to kick off a new request. @@ -2438,7 +2438,7 @@ TEST_F(RouterTest, DontResetStartedResponseOnUpstreamPerTryTimeout) { Buffer::OwnedImpl body("test body"); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(200)); response_decoder->decodeHeaders(std::move(response_headers), false); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); EXPECT_CALL(callbacks_, encodeData(_, true)); response_decoder->decodeData(body, true); EXPECT_TRUE(verifyHostUpstreamStats(1, 0)); @@ -2586,7 +2586,7 @@ TEST_F(RouterTest, RetryTimeoutDuringRetryDelay) { {":status", "504"}, {"content-length", "24"}, {"content-type", "text/plain"}}; EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), false)); EXPECT_CALL(callbacks_, encodeData(_, true)); - response_timeout_->callback_(); + response_timeout_->invokeCallback(); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); } @@ -2632,7 +2632,7 @@ TEST_F(RouterTest, RetryTimeoutDuringRetryDelayWithUpstreamRequestNoHost) { {":status", "504"}, {"content-length", "24"}, {"content-type", "text/plain"}}; EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), false)); EXPECT_CALL(callbacks_, encodeData(_, true)); - response_timeout_->callback_(); + response_timeout_->invokeCallback(); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); } @@ -2679,7 +2679,7 @@ TEST_F(RouterTest, RetryTimeoutDuringRetryDelayWithUpstreamRequestNoHostAltRespo EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putResponseTime(_)).Times(0); Http::TestHeaderMapImpl response_headers{{":status", "204"}}; EXPECT_CALL(callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); - response_timeout_->callback_(); + response_timeout_->invokeCallback(); EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); } @@ -3414,7 +3414,7 @@ TEST_F(RouterTest, UpstreamTimingTimeout) { response_decoder->decodeHeaders(std::move(response_headers), false); test_time_.sleep(std::chrono::milliseconds(99)); - response_timeout_->callback_(); + response_timeout_->invokeCallback(); EXPECT_TRUE(stream_info.firstUpstreamTxByteSent().has_value()); EXPECT_TRUE(stream_info.lastUpstreamTxByteSent().has_value()); diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index ce4574fe9358..581db1087f3c 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -176,7 +176,7 @@ class RouterUpstreamLogTest : public testing::Test { router_->retry_state_->expectResetRetry(); EXPECT_CALL(context_.cluster_manager_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_TIMEOUT, _)); - per_try_timeout_->callback_(); + per_try_timeout_->invokeCallback(); // We expect this reset to kick off a new request. NiceMock encoder2; diff --git a/test/common/tcp/conn_pool_test.cc b/test/common/tcp/conn_pool_test.cc index 5bcb19698b39..ec42cf60e602 100644 --- a/test/common/tcp/conn_pool_test.cc +++ b/test/common/tcp/conn_pool_test.cc @@ -116,7 +116,7 @@ class ConnPoolImplForTest : public ConnPoolImpl { void expectAndRunUpstreamReady() { EXPECT_TRUE(upstream_ready_enabled_); - mock_upstream_ready_timer_->callback_(); + mock_upstream_ready_timer_->invokeCallback(); EXPECT_FALSE(upstream_ready_enabled_); } @@ -576,10 +576,10 @@ TEST_F(TcpConnPoolImplTest, ConnectTimeout) { EXPECT_NE(nullptr, conn_pool_.newConnection(callbacks2)); })); - conn_pool_.test_conns_[0].connect_timer_->callback_(); + conn_pool_.test_conns_[0].connect_timer_->invokeCallback(); EXPECT_CALL(callbacks2.pool_failure_, ready()); - conn_pool_.test_conns_[1].connect_timer_->callback_(); + conn_pool_.test_conns_[1].connect_timer_->invokeCallback(); EXPECT_CALL(conn_pool_, onConnDestroyedForTest()).Times(2); dispatcher_.clearDeferredDeleteList(); diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index dbfb840d619c..903249d9a06e 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -841,7 +841,7 @@ TEST_F(TcpProxyTest, IdleTimeout) { EXPECT_CALL(*upstream_connections_.at(0), close(Network::ConnectionCloseType::NoFlush)); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); } // Tests that the idle timer is disabled when the downstream connection is closed. @@ -918,7 +918,7 @@ TEST_F(TcpProxyTest, IdleTimeoutWithOutstandingDataFlushed) { EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); EXPECT_CALL(*idle_timer, disableTimer()); - idle_timer->callback_(); + idle_timer->invokeCallback(); } // Test that access log fields %UPSTREAM_HOST% and %UPSTREAM_CLUSTER% are correctly logged. @@ -1089,7 +1089,7 @@ TEST_F(TcpProxyTest, UpstreamFlushTimeoutExpired) { EXPECT_EQ(1U, config_->stats().upstream_flush_active_.value()); EXPECT_CALL(*upstream_connections_.at(0), close(Network::ConnectionCloseType::NoFlush)); - idle_timer->callback_(); + idle_timer->invokeCallback(); EXPECT_EQ(1U, config_->stats().upstream_flush_total_.value()); EXPECT_EQ(0U, config_->stats().upstream_flush_active_.value()); EXPECT_EQ(1U, config_->stats().idle_timeout_.value()); diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 7e3fff828ad9..5ed984b29124 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1793,7 +1793,7 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemove) { EXPECT_CALL(*tcp1_high, addDrainedCallback(_)).WillOnce(SaveArg<0>(&tcp_drained_cb_high)); // Remove the first host, this should lead to the first cp being drained. - dns_timer_->callback_(); + dns_timer_->invokeCallback(); dns_callback(TestUtility::makeDnsResponse({"127.0.0.2"})); drained_cb(); drained_cb = nullptr; @@ -1826,7 +1826,7 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemove) { // Now add and remove a host that we never have a conn pool to. This should not lead to any // drain callbacks, etc. - dns_timer_->callback_(); + dns_timer_->invokeCallback(); dns_callback(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.3"})); factory_.tls_.shutdownThread(); } @@ -2009,7 +2009,7 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemoveWithTls) { EXPECT_CALL(*tcp1_ibm_com, addDrainedCallback(_)).WillOnce(SaveArg<0>(&tcp_drained_cb_ibm_com)); // Remove the first host, this should lead to the first cp being drained. - dns_timer_->callback_(); + dns_timer_->invokeCallback(); dns_callback(TestUtility::makeDnsResponse({"127.0.0.2"})); drained_cb(); drained_cb = nullptr; @@ -2056,7 +2056,7 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemoveWithTls) { // Now add and remove a host that we never have a conn pool to. This should not lead to any // drain callbacks, etc. - dns_timer_->callback_(); + dns_timer_->invokeCallback(); dns_callback(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.3"})); factory_.tls_.shutdownThread(); } @@ -2123,7 +2123,7 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemoveDefaultPriority) { // Remove the first host, this should lead to the cp being drained, without // crash. - dns_timer_->callback_(); + dns_timer_->invokeCallback(); dns_callback(TestUtility::makeDnsResponse({})); factory_.tls_.shutdownThread(); @@ -2199,7 +2199,7 @@ TEST_F(ClusterManagerImplTest, ConnPoolDestroyWithDraining) { EXPECT_CALL(*cp, addDrainedCallback(_)).WillOnce(SaveArg<0>(&drained_cb)); Tcp::ConnectionPool::Instance::DrainedCb tcp_drained_cb; EXPECT_CALL(*tcp, addDrainedCallback(_)).WillOnce(SaveArg<0>(&tcp_drained_cb)); - dns_timer_->callback_(); + dns_timer_->invokeCallback(); dns_callback(TestUtility::makeDnsResponse({})); // The drained callback might get called when the CP is being destroyed. @@ -2327,7 +2327,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); // Ensure the merged updates were applied. - timer->callback_(); + timer->invokeCallback(); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index f48d8d2286ca..7c8d7cfb5701 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -559,7 +559,7 @@ class HttpHealthCheckerImplTest : public testing::Test { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); @@ -572,7 +572,7 @@ class HttpHealthCheckerImplTest : public testing::Test { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); @@ -646,7 +646,7 @@ TEST_F(HttpHealthCheckerImplTest, Degraded) { expectStreamCreate(0); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); EXPECT_CALL(*event_logger_, logNoLongerDegraded(_, _)); respond(0, "200", false, false, true, false, {}, false); @@ -673,7 +673,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitter) { EXPECT_CALL(random_, random()).WillOnce(Return(i)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // the jitter is 1000ms here EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000 + i % 1000))); @@ -693,7 +693,7 @@ TEST_F(HttpHealthCheckerImplTest, InitialJitterNoTraffic) { EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); health_checker_->start(); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); @@ -704,7 +704,7 @@ TEST_F(HttpHealthCheckerImplTest, InitialJitterNoTraffic) { EXPECT_CALL(random_, random()).WillOnce(Return(i)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // the jitter is 40% of 5000, so should be 2000 EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000 + i % 2000))); @@ -733,7 +733,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitterPercentNoTraffic) { EXPECT_CALL(random_, random()).WillOnce(Return(i)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // the jitter is 40% of 5000, so should be 2000 EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000 + i % 2000))); @@ -763,7 +763,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitterPercent) { EXPECT_CALL(random_, random()).WillOnce(Return(i)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // the jitter is 40% of 1000, so should be 400 EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000 + i % 400))); @@ -1087,7 +1087,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithAdditionalHeaders) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); } TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithoutUserAgent) { @@ -1127,7 +1127,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithoutUserAgent) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); } TEST_F(HttpHealthCheckerImplTest, ServiceDoesNotMatchFail) { @@ -1339,7 +1339,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFail) { Host::ActiveHealthFailureType::UNHEALTHY); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); @@ -1352,7 +1352,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFail) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); @@ -1386,7 +1386,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailLogError) { Host::ActiveHealthFailureType::UNHEALTHY); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // logUnhealthy is called with first_check == false EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -1403,7 +1403,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailLogError) { Host::ActiveHealthFailureType::UNHEALTHY); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); @@ -1416,7 +1416,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailLogError) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); @@ -1446,7 +1446,7 @@ TEST_F(HttpHealthCheckerImplTest, Disconnect) { expectClientCreate(0); expectStreamCreate(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(cluster_->prioritySet().getMockHostSet(0)->hosts_[0], HealthTransition::Changed)); @@ -1475,7 +1475,7 @@ TEST_F(HttpHealthCheckerImplTest, Timeout) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_EQ(Host::Health::Unhealthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1504,13 +1504,13 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutThenSuccess) { EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); expectClientCreate(0); expectStreamCreate(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); @@ -1533,13 +1533,13 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutThenRemoteClose) { EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); expectClientCreate(0); expectStreamCreate(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); @@ -1562,7 +1562,7 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutAfterDisconnect) { expectSessionCreate(); expectStreamCreate(0); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)).Times(2); health_checker_->start(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)).Times(1); @@ -1576,7 +1576,8 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutAfterDisconnect) { EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->enableTimer(std::chrono::seconds(10)); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_EQ(Host::Health::Unhealthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); } @@ -1618,7 +1619,7 @@ TEST_F(HttpHealthCheckerImplTest, ConnectionClose) { expectClientCreate(0); expectStreamCreate(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); } TEST_F(HttpHealthCheckerImplTest, ProxyConnectionClose) { @@ -1640,7 +1641,7 @@ TEST_F(HttpHealthCheckerImplTest, ProxyConnectionClose) { expectClientCreate(0); expectStreamCreate(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); } TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { @@ -1662,7 +1663,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Follow up successful checks should respect interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -1673,7 +1674,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Follow up successful checks should respect interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -1684,7 +1685,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // A logical failure is not considered a network failure, therefore the unhealthy threshold is // ignored and health state changes immediately. Since the threshold is ignored, next health @@ -1698,7 +1699,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -1709,7 +1710,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -1720,7 +1721,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // When transitioning to a successful state, checks should respect healthy_edge_interval. Health // state should be delayed pending healthy threshold. @@ -1732,7 +1733,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); @@ -1742,7 +1743,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // After the healthy threshold is reached, health state should change while checks should respect // the default interval. @@ -1755,7 +1756,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -1766,7 +1767,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // First failed check after a run o successful ones should respect unhealthy_edge_interval. A // timeout, being a network type failure, should respect unhealthy threshold before changing the @@ -1774,26 +1775,26 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000))); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a network timeout. expectClientCreate(0); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000))); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a network timeout. expectClientCreate(0); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. As the unhealthy threshold is // reached, health state should also change. @@ -1801,27 +1802,27 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a network timeout. expectClientCreate(0); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Remaining failing checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a network timeout. expectClientCreate(0); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // When transitioning to a successful state, checks should respect healthy_edge_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); @@ -1832,7 +1833,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); @@ -1842,7 +1843,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // After the healthy threshold is reached, health state should change while checks should respect // the default interval. @@ -1855,7 +1856,7 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -1885,7 +1886,7 @@ TEST_F(HttpHealthCheckerImplTest, RemoteCloseBetweenChecks) { expectClientCreate(0); expectStreamCreate(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); @@ -1916,7 +1917,7 @@ TEST_F(HttpHealthCheckerImplTest, DontReuseConnectionBetweenChecks) { expectClientCreate(0); expectStreamCreate(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); @@ -2506,7 +2507,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutThenRemoteClose) { EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*timeout_timer_, disableTimer()); EXPECT_CALL(*interval_timer_, enableTimer(_)); - timeout_timer_->callback_(); + timeout_timer_->invokeCallback(); EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::TIMEOUT); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -2514,7 +2515,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutThenRemoteClose) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); EXPECT_CALL(*timeout_timer_, enableTimer(_)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2530,7 +2531,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutThenRemoteClose) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); EXPECT_CALL(*timeout_timer_, enableTimer(_)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2567,7 +2568,7 @@ TEST_F(TcpHealthCheckerImplTest, Timeout) { EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*timeout_timer_, disableTimer()); EXPECT_CALL(*interval_timer_, enableTimer(_)); - timeout_timer_->callback_(); + timeout_timer_->invokeCallback(); EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::TIMEOUT); EXPECT_EQ(Host::Health::Unhealthy, @@ -2600,7 +2601,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*timeout_timer_, disableTimer()); EXPECT_CALL(*interval_timer_, enableTimer(_)); - timeout_timer_->callback_(); + timeout_timer_->invokeCallback(); EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::TIMEOUT); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -2608,7 +2609,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); EXPECT_CALL(*timeout_timer_, enableTimer(_)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2616,7 +2617,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*timeout_timer_, disableTimer()); EXPECT_CALL(*interval_timer_, enableTimer(_)); - timeout_timer_->callback_(); + timeout_timer_->invokeCallback(); EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::TIMEOUT); EXPECT_EQ(Host::Health::Unhealthy, @@ -2625,7 +2626,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); EXPECT_CALL(*timeout_timer_, enableTimer(_)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2666,7 +2667,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutWithoutReusingConnection) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); EXPECT_CALL(*timeout_timer_, enableTimer(_)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2687,7 +2688,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutWithoutReusingConnection) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); EXPECT_CALL(*timeout_timer_, enableTimer(_)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2726,7 +2727,7 @@ TEST_F(TcpHealthCheckerImplTest, NoData) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(0); EXPECT_CALL(*timeout_timer_, enableTimer(_)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); } TEST_F(TcpHealthCheckerImplTest, PassiveFailure) { @@ -3389,7 +3390,7 @@ TEST_F(GrpcHealthCheckerImplTest, SuccessStartFailedFailFirst) { // Next successful healthcheck does not move host int healthy state (because we configured // healthchecker this way). expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); // Host still unhealthy, need yet another healthcheck. @@ -3399,7 +3400,7 @@ TEST_F(GrpcHealthCheckerImplTest, SuccessStartFailedFailFirst) { // 2nd successful healthcheck renders host healthy. expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); @@ -3428,7 +3429,7 @@ TEST_F(GrpcHealthCheckerImplTest, GrpcHealthFail) { // Next, we need 2 successful checks for host to become available again. expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); // Host still considered unhealthy. @@ -3437,7 +3438,7 @@ TEST_F(GrpcHealthCheckerImplTest, GrpcHealthFail) { expectHostHealthy(false); expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); // Host should has become healthy. @@ -3466,7 +3467,7 @@ TEST_F(GrpcHealthCheckerImplTest, Disconnect) { expectClientCreate(0); expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); // Now, host should be unhealthy. @@ -3490,7 +3491,7 @@ TEST_F(GrpcHealthCheckerImplTest, Timeout) { // Unhealthy threshold is 1 so first timeout causes unhealthy EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); expectHostHealthy(false); } @@ -3508,11 +3509,11 @@ TEST_F(GrpcHealthCheckerImplTest, DoubleTimeout) { expectHealthcheckStop(0); // Timeouts are considered network failures and make host unhealthy also after 2nd event. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); expectHostHealthy(true); expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); @@ -3560,7 +3561,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Follow up successful checks should respect interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -3571,7 +3572,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Follow up successful checks should respect interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -3582,7 +3583,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // A logical failure is not considered a network failure, therefore the unhealthy threshold is // ignored and health state changes immediately. Since the threshold is ignored, next health @@ -3596,7 +3597,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -3607,7 +3608,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -3618,7 +3619,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // When transitioning to a successful state, checks should respect healthy_edge_interval. Health // state should be delayed pending healthy threshold. @@ -3630,7 +3631,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); @@ -3640,7 +3641,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // After the healthy threshold is reached, health state should change while checks should respect // the default interval. @@ -3653,7 +3654,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -3664,7 +3665,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // First failed check after a run o successful ones should respect unhealthy_edge_interval. A // timeout, being a network type failure, should respect unhealthy threshold before changing the @@ -3672,22 +3673,22 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000))); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000))); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. As the unhealthy threshold is // reached, health state should also change. @@ -3695,23 +3696,23 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Remaining failing checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->callback_(); + test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // When transitioning to a successful state, checks should respect healthy_edge_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); @@ -3722,7 +3723,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); @@ -3732,7 +3733,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // After the healthy threshold is reached, health state should change while checks should respect // the default interval. @@ -3745,7 +3746,7 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); // Needed after a response is sent. expectStreamCreate(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); @@ -3774,7 +3775,7 @@ TEST_F(GrpcHealthCheckerImplTest, RemoteCloseBetweenChecks) { expectClientCreate(0); expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); // Test host state haven't changed. @@ -3803,7 +3804,7 @@ TEST_F(GrpcHealthCheckerImplTest, DontReuseConnectionBetweenChecks) { // closes the connection. expectClientCreate(0); expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); // Test host state haven't changed. @@ -3863,7 +3864,7 @@ TEST_F(GrpcHealthCheckerImplTest, GoAwayBetweenChecks) { expectClientCreate(0); expectHealthcheckStart(0); - test_sessions_[0]->interval_timer_->callback_(); + test_sessions_[0]->interval_timer_->invokeCallback(); expectHealthcheckStop(0); // Test host state haven't changed. diff --git a/test/common/upstream/logical_dns_cluster_test.cc b/test/common/upstream/logical_dns_cluster_test.cc index 4433db2931f1..e09890e76ebe 100644 --- a/test/common/upstream/logical_dns_cluster_test.cc +++ b/test/common/upstream/logical_dns_cluster_test.cc @@ -101,7 +101,7 @@ class LogicalDnsClusterTest : public testing::Test { logical_host->outlierDetector().putHttpResponseCode(200); expectResolve(Network::DnsLookupFamily::V4Only, expected_address); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); // Should not cause any changes. EXPECT_CALL(*resolve_timer_, enableTimer(_)); @@ -133,7 +133,7 @@ class LogicalDnsClusterTest : public testing::Test { data.host_description_->healthChecker().setUnhealthy(); expectResolve(Network::DnsLookupFamily::V4Only, expected_address); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); // Should cause a change. EXPECT_CALL(*resolve_timer_, enableTimer(_)); @@ -151,7 +151,7 @@ class LogicalDnsClusterTest : public testing::Test { logical_host->createConnection(dispatcher_, nullptr, nullptr); expectResolve(Network::DnsLookupFamily::V4Only, expected_address); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); // Empty should not cause any change. EXPECT_CALL(*resolve_timer_, enableTimer(_)); @@ -167,7 +167,7 @@ class LogicalDnsClusterTest : public testing::Test { // Make sure we cancel. EXPECT_CALL(active_dns_query_, cancel()); expectResolve(Network::DnsLookupFamily::V4Only, expected_address); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); tls_.shutdownThread(); } diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index dd0a2c1babf3..eccedcc9437c 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -290,7 +290,7 @@ TEST_F(OriginalDstClusterTest, Membership) { ASSERT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(true, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->used()); EXPECT_CALL(*cleanup_timer_, enableTimer(_)); - cleanup_timer_->callback_(); + cleanup_timer_->invokeCallback(); EXPECT_EQ( cluster_hosts, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()); // hosts vector remains the same @@ -301,7 +301,7 @@ TEST_F(OriginalDstClusterTest, Membership) { EXPECT_CALL(*cleanup_timer_, enableTimer(_)); EXPECT_CALL(membership_updated_, ready()); - cleanup_timer_->callback_(); + cleanup_timer_->invokeCallback(); EXPECT_NE(cluster_hosts, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()); // hosts vector changes @@ -392,7 +392,7 @@ TEST_F(OriginalDstClusterTest, Membership2) { EXPECT_EQ(true, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->used()); EXPECT_EQ(true, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[1]->used()); EXPECT_CALL(*cleanup_timer_, enableTimer(_)); - cleanup_timer_->callback_(); + cleanup_timer_->invokeCallback(); EXPECT_EQ( cluster_hosts, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()); // hosts vector remains the same @@ -404,7 +404,7 @@ TEST_F(OriginalDstClusterTest, Membership2) { EXPECT_CALL(*cleanup_timer_, enableTimer(_)); EXPECT_CALL(membership_updated_, ready()); - cleanup_timer_->callback_(); + cleanup_timer_->invokeCallback(); EXPECT_NE(cluster_hosts, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()); // hosts vector changes diff --git a/test/common/upstream/outlier_detection_impl_test.cc b/test/common/upstream/outlier_detection_impl_test.cc index c585a5509855..6de146f062a7 100644 --- a/test/common/upstream/outlier_detection_impl_test.cc +++ b/test/common/upstream/outlier_detection_impl_test.cc @@ -232,7 +232,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodes) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); // Interval that does bring the host back in. @@ -241,7 +241,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodes) { EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -359,7 +359,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaNonHttpCodes) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); // Interval that does bring the host back in. @@ -368,7 +368,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaNonHttpCodes) { EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -450,7 +450,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailure) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); // Interval that does bring the host back in. @@ -459,7 +459,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailure) { EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -537,7 +537,7 @@ TEST_F(OutlierDetectorImplTest, TimeoutWithHttpCode) { EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); // Report several LOCAL_ORIGIN_TIMEOUT with HTTP code other that 500. Node should not be ejected. @@ -611,7 +611,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowLocalOriginFailure) { // Wait short time - not enough to be unejected time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); // Interval that does bring the host back in. @@ -620,7 +620,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowLocalOriginFailure) { EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); @@ -699,7 +699,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailureAnd5xx) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); // Interval that does bring the host back in. @@ -708,7 +708,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailureAnd5xx) { EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -848,7 +848,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_stdev_factor", 1900)) .WillByDefault(Return(1900)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_EQ(50, hosts_[4]->outlierDetector().successRate( DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); EXPECT_EQ(90, detector->successRateAverage( @@ -868,7 +868,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(19999)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); @@ -878,7 +878,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[4]))); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); @@ -901,7 +901,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); @@ -982,7 +982,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_stdev_factor", 1900)) .WillByDefault(Return(1900)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_EQ(50, hosts_[4]->outlierDetector().successRate( DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); EXPECT_EQ(90, @@ -1002,7 +1002,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(19999)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); @@ -1012,7 +1012,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[4]))); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); @@ -1031,7 +1031,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); @@ -1053,7 +1053,7 @@ TEST_F(OutlierDetectorImplTest, EmptySuccessRate) { EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_minimum_hosts", 5)) .WillByDefault(Return(0)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); } TEST_F(OutlierDetectorImplTest, RemoveWhileEjected) { @@ -1083,7 +1083,7 @@ TEST_F(OutlierDetectorImplTest, RemoveWhileEjected) { time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); } TEST_F(OutlierDetectorImplTest, Overflow) { diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index eb0e898f28b5..f55f2884097a 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -311,7 +311,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { EXPECT_EQ("localhost1", cluster.prioritySet().hostSetsPerPriority()[0]->hosts()[1]->hostname()); resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( @@ -319,14 +319,14 @@ TEST_F(StrictDnsClusterImplTest, Basic) { ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.3"})); @@ -353,9 +353,9 @@ TEST_F(StrictDnsClusterImplTest, Basic) { // Make sure we cancel. resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); resolver2.expectResolve(*dns_resolver_); - resolver2.timer_->callback_(); + resolver2.timer_->invokeCallback(); EXPECT_CALL(resolver1.active_dns_query_, cancel()); EXPECT_CALL(resolver2.active_dns_query_, cancel()); @@ -610,7 +610,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { EXPECT_EQ(0UL, stats_.counter("cluster.name.update_no_rebuild").value()); resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( @@ -622,7 +622,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { EXPECT_EQ(1UL, stats_.counter("cluster.name.update_no_rebuild").value()); resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( @@ -641,14 +641,14 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { EXPECT_EQ(2UL, stats_.counter("cluster.name.update_no_rebuild").value()); resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); // We again received the same set as before for localhost1. No rebuild this time. EXPECT_EQ(3UL, stats_.counter("cluster.name.update_no_rebuild").value()); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.3"})); @@ -717,11 +717,11 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { // Make sure we cancel. resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); resolver2.expectResolve(*dns_resolver_); - resolver2.timer_->callback_(); + resolver2.timer_->invokeCallback(); resolver3.expectResolve(*dns_resolver_); - resolver3.timer_->callback_(); + resolver3.timer_->invokeCallback(); EXPECT_CALL(resolver1.active_dns_query_, cancel()); EXPECT_CALL(resolver2.active_dns_query_, cancel()); @@ -799,7 +799,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { EXPECT_EQ("localhost1", cluster.prioritySet().hostSetsPerPriority()[0]->hosts()[1]->hostname()); resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( @@ -807,14 +807,14 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.3"})); @@ -850,11 +850,11 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { // Make sure we cancel. resolver1.expectResolve(*dns_resolver_); - resolver1.timer_->callback_(); + resolver1.timer_->invokeCallback(); resolver2.expectResolve(*dns_resolver_); - resolver2.timer_->callback_(); + resolver2.timer_->invokeCallback(); resolver3.expectResolve(*dns_resolver_); - resolver3.timer_->callback_(); + resolver3.timer_->invokeCallback(); EXPECT_CALL(resolver1.active_dns_query_, cancel()); EXPECT_CALL(resolver2.active_dns_query_, cancel()); diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 4d60412503ff..cb3a1b5568d1 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -375,14 +375,14 @@ class RedisClusterTest : public testing::Test, // Add new host. expectRedisResolve(); EXPECT_CALL(membership_updated_, ready()); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); expectClusterSlotResponse(twoSlotsMasters()); expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); // No change. expectRedisResolve(); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1).WillOnce(Return(false)); expectClusterSlotResponse(twoSlotsMasters()); expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); @@ -390,7 +390,7 @@ class RedisClusterTest : public testing::Test, // Remove host. expectRedisResolve(); EXPECT_CALL(membership_updated_, ready()); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); expectClusterSlotResponse(singleSlotMasterSlave("127.0.0.1", "127.0.0.2", 22120)); expectHealthyHosts(std::list({"127.0.0.1:22120"})); @@ -607,7 +607,7 @@ TEST_F(RedisClusterTest, RedisResolveFailure) { EXPECT_EQ(1U, cluster_->info()->stats().update_failure_.value()); expectRedisResolve(true); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); @@ -616,7 +616,7 @@ TEST_F(RedisClusterTest, RedisResolveFailure) { // Expect no change if resolve failed. expectRedisResolve(); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); expectClusterSlotFailure(); expectHealthyHosts(std::list({"127.0.0.1:22120"})); EXPECT_EQ(3U, cluster_->info()->stats().update_attempt_.value()); @@ -675,7 +675,7 @@ TEST_F(RedisClusterTest, RedisErrorResponse) { EXPECT_EQ(1U, cluster_->info()->stats().update_failure_.value()); expectRedisResolve(); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); @@ -689,7 +689,7 @@ TEST_F(RedisClusterTest, RedisErrorResponse) { for (uint64_t i = 0; i < (1 << 10); i++) { std::bitset<10> flags(i); expectRedisResolve(); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); if (flags.all()) { EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1).WillOnce(Return(false)); } diff --git a/test/extensions/filters/http/health_check/health_check_test.cc b/test/extensions/filters/http/health_check/health_check_test.cc index 6a2d4b5de94b..85f2d242c39d 100644 --- a/test/extensions/filters/http/health_check/health_check_test.cc +++ b/test/extensions/filters/http/health_check/health_check_test.cc @@ -360,7 +360,7 @@ TEST_F(HealthCheckFilterCachingTest, All) { // Fire the timer, this should result in the next request going through. EXPECT_CALL(*cache_timer_, enableTimer(_)); - cache_timer_->callback_(); + cache_timer_->invokeCallback(); prepareFilter(true); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, true)); } @@ -394,7 +394,7 @@ TEST_F(HealthCheckFilterCachingTest, DegradedHeader) { // Fire the timer, this should result in the next request going through. EXPECT_CALL(*cache_timer_, enableTimer(_)); - cache_timer_->callback_(); + cache_timer_->invokeCallback(); prepareFilter(true); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, true)); } diff --git a/test/extensions/filters/http/squash/squash_filter_test.cc b/test/extensions/filters/http/squash/squash_filter_test.cc index 534b59b87a74..0800a51499cb 100644 --- a/test/extensions/filters/http/squash/squash_filter_test.cc +++ b/test/extensions/filters/http/squash/squash_filter_test.cc @@ -330,7 +330,7 @@ TEST_F(SquashFilterTest, Timeout) { EXPECT_CALL(request_, cancel()); EXPECT_CALL(filter_callbacks_, continueDecoding()); - attachmentTimeout_timer_->callback_(); + attachmentTimeout_timer_->invokeCallback(); EXPECT_EQ(Envoy::Http::FilterDataStatus::Continue, filter_->decodeData(buffer, false)); } @@ -359,7 +359,7 @@ TEST_F(SquashFilterTest, CheckRetryPollingAttachment) { // Expect the second get attachment request expectAsyncClientSend(); - retry_timer->callback_(); + retry_timer->invokeCallback(); EXPECT_CALL(filter_callbacks_, continueDecoding()); completeGetStatusRequest("attached"); } @@ -378,7 +378,7 @@ TEST_F(SquashFilterTest, CheckRetryPollingAttachmentOnFailure) { // Expect the second get attachment request expectAsyncClientSend(); - retry_timer->callback_(); + retry_timer->invokeCallback(); EXPECT_CALL(filter_callbacks_, continueDecoding()); completeGetStatusRequest("attached"); @@ -430,10 +430,13 @@ TEST_F(SquashFilterTest, TimerExpiresInline) { initFilter(); attachmentTimeout_timer_ = new NiceMock(&filter_callbacks_.dispatcher_); + // TODO: this is a really synthetic test as the callback can't actually be called under the stack + // of enableTimer. It'd be good to clean this up. EXPECT_CALL(*attachmentTimeout_timer_, enableTimer(config_->attachmentTimeout())) .WillOnce(Invoke([&](const std::chrono::milliseconds&) { + attachmentTimeout_timer_->enabled_ = true; // timer expires inline - attachmentTimeout_timer_->callback_(); + attachmentTimeout_timer_->invokeCallback(); })); EXPECT_CALL(cm_.async_client_, send_(_, _, _)) diff --git a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc index 5321d370a189..23b7f9acf6d8 100644 --- a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc +++ b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc @@ -221,7 +221,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { // Interval timer fires. setupRequest(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Error response. EXPECT_CALL(*interval_timer_, enableTimer(_)); @@ -231,7 +231,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { // Interval timer fires. setupRequest(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Parsing error EXPECT_CALL(*interval_timer_, enableTimer(_)); @@ -242,7 +242,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { // Interval timer fires. setupRequest(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // No response failure. EXPECT_CALL(*interval_timer_, enableTimer(_)); @@ -259,7 +259,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { return nullptr; })); EXPECT_CALL(*interval_timer_, enableTimer(_)); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); EXPECT_EQ(4U, stats_store_.counter("auth.clientssl.vpn.update_failure").value()); } diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 064c740dacc6..d9be95058d6b 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -510,7 +510,7 @@ TEST_F(RedisClientImplTest, ConnectTimeout) { EXPECT_CALL(*upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); EXPECT_CALL(callbacks1, onFailure()); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - connect_or_op_timer_->callback_(); + connect_or_op_timer_->invokeCallback(); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_cx_connect_timeout_.value()); EXPECT_EQ(1UL, host_->stats_.cx_connect_fail_.value()); @@ -553,7 +553,7 @@ TEST_F(RedisClientImplTest, OpTimeout) { EXPECT_CALL(*upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); EXPECT_CALL(callbacks1, onFailure()); EXPECT_CALL(*connect_or_op_timer_, disableTimer()); - connect_or_op_timer_->callback_(); + connect_or_op_timer_->invokeCallback(); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_rq_timeout_.value()); EXPECT_EQ(1UL, host_->stats_.rq_timeout_.value()); diff --git a/test/extensions/filters/network/mongo_proxy/proxy_test.cc b/test/extensions/filters/network/mongo_proxy/proxy_test.cc index 60860f974845..2fb9b66221db 100644 --- a/test/extensions/filters/network/mongo_proxy/proxy_test.cc +++ b/test/extensions/filters/network/mongo_proxy/proxy_test.cc @@ -178,7 +178,7 @@ TEST_F(MongoProxyFilterTest, DelayFaults) { EXPECT_EQ(1U, store_.counter("test.op_kill_cursors").value()); EXPECT_CALL(read_filter_callbacks_, continueReading()); - delay_timer->callback_(); + delay_timer->invokeCallback(); EXPECT_EQ(1U, store_.counter("test.delays_injected").value()); } @@ -558,7 +558,7 @@ TEST_F(MongoProxyFilterTest, ConcurrentQueryWithDrainClose) { EXPECT_CALL(read_filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); EXPECT_CALL(*drain_timer, disableTimer()); - drain_timer->callback_(); + drain_timer->invokeCallback(); EXPECT_EQ(0U, store_.gauge("test.op_query_active", Stats::Gauge::ImportMode::Accumulate).value()); EXPECT_EQ(1U, store_.counter("test.cx_drain_close").value()); diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index d5741a158eb4..f8be7912d129 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -212,7 +212,7 @@ TEST_F(RedisHealthCheckerTest, PingAndVariousFailures) { pool_callbacks_->onResponse(std::move(response)); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Failure EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); @@ -222,7 +222,7 @@ TEST_F(RedisHealthCheckerTest, PingAndVariousFailures) { pool_callbacks_->onResponse(std::move(response)); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Redis failure via disconnect EXPECT_CALL(*timeout_timer_, disableTimer()); @@ -232,18 +232,18 @@ TEST_F(RedisHealthCheckerTest, PingAndVariousFailures) { expectClientCreate(); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Timeout EXPECT_CALL(pool_request_, cancel()); EXPECT_CALL(*client_, close()); EXPECT_CALL(*timeout_timer_, disableTimer()); EXPECT_CALL(*interval_timer_, enableTimer(_)); - timeout_timer_->callback_(); + timeout_timer_->invokeCallback(); expectClientCreate(); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Shutdown with active request. EXPECT_CALL(pool_request_, cancel()); @@ -280,7 +280,7 @@ TEST_F(RedisHealthCheckerTest, FailuresLogging) { pool_callbacks_->onResponse(std::move(response)); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Failure EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); @@ -291,7 +291,7 @@ TEST_F(RedisHealthCheckerTest, FailuresLogging) { pool_callbacks_->onResponse(std::move(response)); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Fail again EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, false)); @@ -301,7 +301,7 @@ TEST_F(RedisHealthCheckerTest, FailuresLogging) { pool_callbacks_->onResponse(std::move(response)); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Shutdown with active request. EXPECT_CALL(pool_request_, cancel()); @@ -338,7 +338,7 @@ TEST_F(RedisHealthCheckerTest, LogInitialFailure) { expectClientCreate(); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Success EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); @@ -351,7 +351,7 @@ TEST_F(RedisHealthCheckerTest, LogInitialFailure) { pool_callbacks_->onResponse(std::move(response)); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Shutdown with active request. EXPECT_CALL(pool_request_, cancel()); @@ -388,7 +388,7 @@ TEST_F(RedisHealthCheckerTest, Exists) { pool_callbacks_->onResponse(std::move(response)); expectExistsRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Failure, exists EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); @@ -400,7 +400,7 @@ TEST_F(RedisHealthCheckerTest, Exists) { pool_callbacks_->onResponse(std::move(response)); expectExistsRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Failure, no value EXPECT_CALL(*timeout_timer_, disableTimer()); @@ -439,7 +439,7 @@ TEST_F(RedisHealthCheckerTest, ExistsRedirected) { pool_callbacks_->onRedirection(moved_response); expectExistsRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Success with ask redirection EXPECT_CALL(*timeout_timer_, disableTimer()); @@ -481,7 +481,7 @@ TEST_F(RedisHealthCheckerTest, NoConnectionReuse) { expectClientCreate(); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // The connection will close on failure. EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); @@ -493,7 +493,7 @@ TEST_F(RedisHealthCheckerTest, NoConnectionReuse) { expectClientCreate(); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Redis failure via disconnect, the connection was closed by the other end. EXPECT_CALL(*timeout_timer_, disableTimer()); @@ -503,18 +503,18 @@ TEST_F(RedisHealthCheckerTest, NoConnectionReuse) { expectClientCreate(); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Timeout, the connection will be closed. EXPECT_CALL(pool_request_, cancel()); EXPECT_CALL(*client_, close()); EXPECT_CALL(*timeout_timer_, disableTimer()); EXPECT_CALL(*interval_timer_, enableTimer(_)); - timeout_timer_->callback_(); + timeout_timer_->invokeCallback(); expectClientCreate(); expectPingRequestCreate(); - interval_timer_->callback_(); + interval_timer_->invokeCallback(); // Shutdown with active request. EXPECT_CALL(pool_request_, cancel()); diff --git a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc index 1870d31c4a12..50e2722f7cfc 100644 --- a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc +++ b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc @@ -148,7 +148,7 @@ TEST_F(DatadogDriverTest, FlushSpansTimer) { // Timer should be re-enabled. EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900))); - timer_->callback_(); + timer_->invokeCallback(); EXPECT_EQ(1U, stats_.counter("tracing.datadog.timer_flushed").value()); EXPECT_EQ(1U, stats_.counter("tracing.datadog.traces_sent").value()); diff --git a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc index c6c2b72f753a..046bd9596c4b 100644 --- a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc +++ b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc @@ -342,7 +342,7 @@ TEST_F(LightStepDriverTest, FlushSpansTimer) { EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.flush_interval_ms", 1000U)) .WillOnce(Return(1000U)); - timer_->callback_(); + timer_->invokeCallback(); EXPECT_EQ(1U, stats_.counter("tracing.lightstep.timer_flushed").value()); EXPECT_EQ(1U, stats_.counter("tracing.lightstep.spans_sent").value()); diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index 69f46e25cf75..eceef5245fee 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -246,7 +246,7 @@ TEST_F(ZipkinDriverTest, FlushSpansTimer) { EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.flush_interval_ms", 5000U)) .WillOnce(Return(5000U)); - timer_->callback_(); + timer_->invokeCallback(); EXPECT_EQ(1U, stats_.counter("tracing.zipkin.timer_flushed").value()); EXPECT_EQ(1U, stats_.counter("tracing.zipkin.spans_sent").value()); diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 4ff1eda3140c..0e65635e4b08 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -140,9 +140,9 @@ class MockTimer : public Timer { MOCK_METHOD0(enabled, bool()); bool enabled_{}; - Event::TimerCb callback_; // TODO(mattklein123): This should be private and only called via - // invoke callback to clear enabled_, but that will break too many - // tests and can be done later. + +private: + Event::TimerCb callback_; }; class MockSignalEvent : public SignalEvent { diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 850d55a348e7..7e4e2e03237c 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -609,7 +609,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeout) { EXPECT_EQ(1UL, downstream_pre_cx_active.value()); EXPECT_CALL(*timeout, disableTimer()); - timeout->callback_(); + timeout->invokeCallback(); dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0UL, downstream_pre_cx_active.value()); EXPECT_EQ(1UL, stats_store_.counter("downstream_pre_cx_timeout").value()); @@ -658,7 +658,7 @@ TEST_F(ConnectionHandlerTest, ContinueOnListenerFilterTimeout) { EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(nullptr)); EXPECT_CALL(*timeout, disableTimer()); - timeout->callback_(); + timeout->invokeCallback(); dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0UL, downstream_pre_cx_active.value()); EXPECT_EQ(1UL, stats_store_.counter("downstream_pre_cx_timeout").value()); diff --git a/test/server/drain_manager_impl_test.cc b/test/server/drain_manager_impl_test.cc index ba69e4dffdb1..58919df1ce7e 100644 --- a/test/server/drain_manager_impl_test.cc +++ b/test/server/drain_manager_impl_test.cc @@ -37,7 +37,7 @@ TEST_F(DrainManagerImplTest, Default) { drain_manager.startParentShutdownSequence(); EXPECT_CALL(server_.hot_restart_, sendParentTerminateRequest()); - shutdown_timer->callback_(); + shutdown_timer->invokeCallback(); // Verify basic drain close. EXPECT_CALL(server_, healthCheckFailed()).WillOnce(Return(false)); @@ -58,7 +58,7 @@ TEST_F(DrainManagerImplTest, Default) { } else { EXPECT_CALL(drain_complete, ready()); } - drain_timer->callback_(); + drain_timer->invokeCallback(); } EXPECT_CALL(server_, healthCheckFailed()).WillOnce(Return(false)); From 560d09f51546b9c363b90c871df5979472976204 Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Thu, 15 Aug 2019 15:58:10 -0700 Subject: [PATCH 401/542] Redis cluster read replica (#7496) Signed-off-by: Henry Yang --- .../network/redis_proxy/v2/redis_proxy.proto | 25 +- .../arch_overview/other_protocols/redis.rst | 1 + docs/root/intro/version_history.rst | 1 + source/common/upstream/upstream_impl.h | 3 +- source/extensions/clusters/redis/BUILD | 3 + .../clusters/redis/redis_cluster.cc | 50 ++- .../extensions/clusters/redis/redis_cluster.h | 9 +- .../clusters/redis/redis_cluster_lb.cc | 167 ++++++++-- .../clusters/redis/redis_cluster_lb.h | 147 ++++++--- .../filters/network/common/redis/client.h | 10 + .../network/common/redis/client_impl.cc | 27 +- .../network/common/redis/client_impl.h | 2 + .../network/common/redis/supported_commands.h | 38 ++- .../filters/network/common/redis/utility.cc | 13 + .../filters/network/common/redis/utility.h | 6 + .../network/redis_proxy/conn_pool_impl.cc | 13 +- .../network/redis_proxy/router_impl.cc | 3 +- .../extensions/health_checkers/redis/redis.h | 3 + test/extensions/clusters/redis/BUILD | 4 + test/extensions/clusters/redis/mocks.cc | 8 +- test/extensions/clusters/redis/mocks.h | 3 +- .../redis/redis_cluster_integration_test.cc | 22 +- .../clusters/redis/redis_cluster_lb_test.cc | 300 ++++++++++++++++-- .../clusters/redis/redis_cluster_test.cc | 233 ++++++++++++-- .../network/common/redis/client_impl_test.cc | 5 +- .../filters/network/common/redis/test_utils.h | 11 +- .../redis_proxy/conn_pool_impl_test.cc | 65 +++- 27 files changed, 988 insertions(+), 184 deletions(-) diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index 9f714dd66cd0..175e564dec3f 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -90,6 +90,29 @@ message RedisProxy { // this limit, then redirection will fail and the original redirection error will be passed // downstream unchanged. This limit defaults to 100. google.protobuf.UInt32Value max_upstream_unknown_connections = 6; + + // ReadPolicy controls how Envoy routes read commands to Redis nodes. This is currently + // supported for Redis Cluster. All ReadPolicy settings except MASTER may return stale data + // because replication is asynchronous and requires some delay. You need to ensure that your + // application can tolerate stale data. + enum ReadPolicy { + // Default mode. Read from the current master node. + MASTER = 0; + // Read from the master, but if it is unavailable, read from replica nodes. + PREFER_MASTER = 1; + // Read from replica nodes. If multiple replica nodes are present within a shard, a random + // node is selected. Healthy nodes have precedent over unhealthy nodes. + REPLICA = 2; + // Read from the replica nodes (similar to REPLICA), but if all replicas are unavailable (not + // present or unhealthy), read from the master. + PREFER_REPLICA = 3; + // Read from any node of the cluster. A random node is selected among the master and replicas, + // healthy nodes have precedent over unhealthy nodes. + ANY = 4; + } + + // Read policy. The default is to read from the master. + ReadPolicy read_policy = 7 [(validate.rules).enum.defined_only = true]; } // Network settings for the connection pool to the upstream clusters. @@ -210,4 +233,4 @@ message RedisProtocolOptions { // Upstream server password as defined by the `requirepass directive // `_ in the server's configuration file. envoy.api.v2.core.DataSource auth_password = 1; -} \ No newline at end of file +} diff --git a/docs/root/intro/arch_overview/other_protocols/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst index 22650a4264ad..47e55418c679 100644 --- a/docs/root/intro/arch_overview/other_protocols/redis.rst +++ b/docs/root/intro/arch_overview/other_protocols/redis.rst @@ -27,6 +27,7 @@ The Redis project offers a thorough reference on partitioning as it relates to R * Prefix routing. * Separate downstream client and upstream server authentication. * Request mirroring for all requests or write requests only. +* Control :ref:`read requests routing`. This only works with Redis Cluster. **Planned future enhancements**: diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 0d309f18f022..72e4c3b12f0e 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -21,6 +21,7 @@ Version history * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. +* redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. * rbac: added support for DNS SAN as :ref:`principal_name `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 110b08bc955e..07469e36d0f3 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -715,6 +715,8 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable initialization_complete_callback_; diff --git a/source/extensions/clusters/redis/BUILD b/source/extensions/clusters/redis/BUILD index d6238dc9fd3d..7608fa092f9a 100644 --- a/source/extensions/clusters/redis/BUILD +++ b/source/extensions/clusters/redis/BUILD @@ -29,6 +29,9 @@ envoy_cc_library( "//source/common/upstream:upstream_includes", "//source/common/upstream:upstream_lib", "//source/extensions/clusters:well_known_names", + "//source/extensions/filters/network/common/redis:client_interface", + "//source/extensions/filters/network/common/redis:codec_interface", + "//source/extensions/filters/network/common/redis:supported_commands_lib", ], ) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index bf5160805944..9e70ceaf7cf3 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -76,11 +76,14 @@ void RedisCluster::updateAllHosts(const Upstream::HostVector& hosts_added, hosts_added, hosts_removed, absl::nullopt); } -void RedisCluster::onClusterSlotUpdate(const std::vector& slots) { +void RedisCluster::onClusterSlotUpdate(ClusterSlotsPtr&& slots) { Upstream::HostVector new_hosts; - for (const ClusterSlot& slot : slots) { + for (const ClusterSlot& slot : *slots) { new_hosts.emplace_back(new RedisHost(info(), "", slot.master(), *this, true)); + for (auto const& replica : slot.replicas()) { + new_hosts.emplace_back(new RedisHost(info(), "", replica, *this, false)); + } } std::unordered_map updated_hosts; @@ -89,7 +92,7 @@ void RedisCluster::onClusterSlotUpdate(const std::vector& slots) { const bool host_updated = updateDynamicHostList(new_hosts, hosts_, hosts_added, hosts_removed, updated_hosts, all_hosts_); const bool slot_updated = - lb_factory_ ? lb_factory_->onClusterSlotUpdate(slots, updated_hosts) : false; + lb_factory_ ? lb_factory_->onClusterSlotUpdate(std::move(slots), updated_hosts) : false; // If slot is updated, call updateAllHosts regardless of if there's new hosts to force // update of the thread local load balancers. @@ -111,6 +114,13 @@ void RedisCluster::onClusterSlotUpdate(const std::vector& slots) { onPreInitComplete(); } +void RedisCluster::reloadHealthyHostsHelper(const Upstream::HostSharedPtr& host) { + if (lb_factory_) { + lb_factory_->onHostHealthUpdate(); + } + ClusterImplBase::reloadHealthyHostsHelper(host); +} + // DnsDiscoveryResolveTarget RedisCluster::DnsDiscoveryResolveTarget::DnsDiscoveryResolveTarget(RedisCluster& parent, const std::string& dns_address, @@ -259,13 +269,18 @@ void RedisCluster::RedisDiscoverySession::onResponse( NetworkFilters::Common::Redis::RespValuePtr&& value) { current_request_ = nullptr; + const uint32_t SlotRangeStart = 0; + const uint32_t SlotRangeEnd = 1; + const uint32_t SlotMaster = 2; + const uint32_t SlotSlaveStart = 3; + // Do nothing if the cluster is empty. if (value->type() != NetworkFilters::Common::Redis::RespType::Array || value->asArray().empty()) { onUnexpectedResponse(value); return; } - std::vector slots_; + auto slots = std::make_unique>(); // Loop through the cluster slot response and error checks for each field. for (const NetworkFilters::Common::Redis::RespValue& part : value->asArray()) { @@ -275,27 +290,36 @@ void RedisCluster::RedisDiscoverySession::onResponse( } const std::vector& slot_range = part.asArray(); if (slot_range.size() < 3 || - slot_range[0].type() != + slot_range[SlotRangeStart].type() != NetworkFilters::Common::Redis::RespType::Integer || // Start slot range is an integer. - slot_range[1].type() != + slot_range[SlotRangeEnd].type() != NetworkFilters::Common::Redis::RespType::Integer) { // End slot range is an integer. onUnexpectedResponse(value); return; } // Field 2: Master address for slot range - // TODO(hyang): For now we're only adding the master node for each slot. When we're ready to - // send requests to replica nodes, we need to add subsequent address in the response as - // replica nodes. - auto master_address = ProcessCluster(slot_range[2]); + auto master_address = ProcessCluster(slot_range[SlotMaster]); if (!master_address) { onUnexpectedResponse(value); return; } - slots_.emplace_back(slot_range[0].asInteger(), slot_range[1].asInteger(), master_address); + + slots->emplace_back(slot_range[SlotRangeStart].asInteger(), + slot_range[SlotRangeEnd].asInteger(), master_address); + + for (auto replica = std::next(slot_range.begin(), SlotSlaveStart); replica != slot_range.end(); + ++replica) { + auto replica_address = ProcessCluster(*replica); + if (!replica_address) { + onUnexpectedResponse(value); + return; + } + slots->back().addReplica(std::move(replica_address)); + } } - parent_.onClusterSlotUpdate(slots_); + parent_.onClusterSlotUpdate(std::move(slots)); resolve_timer_->enableTimer(parent_.cluster_refresh_rate_); } @@ -340,7 +364,7 @@ RedisClusterFactory::createClusterWithConfig( std::move(stats_scope), context.addedViaApi(), nullptr), nullptr); } - auto lb_factory = std::make_shared(); + auto lb_factory = std::make_shared(context.random()); return std::make_pair(std::make_shared( cluster, proto_config, NetworkFilters::Common::Redis::Client::ClientFactoryImpl::instance_, diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index d7b776353671..9117fa7daccc 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -122,7 +122,9 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { void updateAllHosts(const Upstream::HostVector& hosts_added, const Upstream::HostVector& hosts_removed, uint32_t priority); - void onClusterSlotUpdate(const std::vector&); + void onClusterSlotUpdate(ClusterSlotsPtr&&); + + void reloadHealthyHostsHelper(const Upstream::HostSharedPtr& host) override; const envoy::api::v2::endpoint::LocalityLbEndpoints& localityLbEndpoint() const { // Always use the first endpoint. @@ -212,6 +214,11 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { uint32_t maxBufferSizeBeforeFlush() const override { return 0; } std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + // This is effectively not in used for making the "Cluster Slots" calls. + // since we call cluster slots on both the master and slaves, ANY is more appropriate here. + Extensions::NetworkFilters::Common::Redis::Client::ReadPolicy readPolicy() const override { + return Extensions::NetworkFilters::Common::Redis::Client::ReadPolicy::Any; + } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; diff --git a/source/extensions/clusters/redis/redis_cluster_lb.cc b/source/extensions/clusters/redis/redis_cluster_lb.cc index 9c08e4951a11..fefd17fff2fc 100644 --- a/source/extensions/clusters/redis/redis_cluster_lb.cc +++ b/source/extensions/clusters/redis/redis_cluster_lb.cc @@ -5,44 +5,116 @@ namespace Extensions { namespace Clusters { namespace Redis { +bool ClusterSlot::operator==(const Envoy::Extensions::Clusters::Redis::ClusterSlot& rhs) const { + return start_ == rhs.start_ && end_ == rhs.end_ && master_ == rhs.master_ && + replicas_ == rhs.replicas_; +} + // RedisClusterLoadBalancerFactory -bool RedisClusterLoadBalancerFactory::onClusterSlotUpdate( - const std::vector& slots, - Envoy::Upstream::HostMap all_hosts) { +bool RedisClusterLoadBalancerFactory::onClusterSlotUpdate(ClusterSlotsPtr&& slots, + Envoy::Upstream::HostMap all_hosts) { + // The slots is sorted, allowing for a quick comparison to make sure we need to update the slot + // array sort based on start and end to enable efficient comparison + std::sort( + slots->begin(), slots->end(), [](const ClusterSlot& lhs, const ClusterSlot& rhs) -> bool { + return lhs.start() < rhs.start() || (!(lhs.start() < rhs.start()) && lhs.end() < rhs.end()); + }); - SlotArraySharedPtr current; - { - absl::ReaderMutexLock lock(&mutex_); - current = slot_array_; + if (current_cluster_slot_ && *current_cluster_slot_ == *slots) { + return false; } - bool should_update = !current; auto updated_slots = std::make_shared(); - for (const ClusterSlot& slot : slots) { - auto host = all_hosts.find(slot.master()->asString()); - ASSERT(host != all_hosts.end(), "we expect all address to be found in the updated_hosts"); - for (auto i = slot.start(); i <= slot.end(); ++i) { - updated_slots->at(i) = host->second; - if (current && current->at(i)->address()->asString() != host->second->address()->asString()) { - should_update = true; + auto shard_vector = std::make_shared>(); + absl::flat_hash_map shards; + + for (const ClusterSlot& slot : *slots) { + // look in the updated map + const std::string master_address = slot.master()->asString(); + + auto result = shards.try_emplace(master_address, shard_vector->size()); + if (result.second) { + auto master_host = all_hosts.find(master_address); + ASSERT(master_host != all_hosts.end(), + "we expect all address to be found in the updated_hosts"); + + Upstream::HostVectorSharedPtr master_and_replicas = std::make_shared(); + Upstream::HostVectorSharedPtr replicas = std::make_shared(); + master_and_replicas->push_back(master_host->second); + + for (auto const& replica : slot.replicas()) { + auto replica_host = all_hosts.find(replica->asString()); + ASSERT(replica_host != all_hosts.end(), + "we expect all address to be found in the updated_hosts"); + replicas->push_back(replica_host->second); + master_and_replicas->push_back(replica_host->second); } + + shard_vector->emplace_back( + std::make_shared(master_host->second, replicas, master_and_replicas)); + } + + for (auto i = slot.start(); i <= slot.end(); ++i) { + updated_slots->at(i) = result.first->second; } } - if (should_update) { + { absl::WriterMutexLock lock(&mutex_); - slot_array_ = updated_slots; + current_cluster_slot_ = std::move(slots); + slot_array_ = std::move(updated_slots); + shard_vector_ = std::move(shard_vector); + } + return true; +} + +void RedisClusterLoadBalancerFactory::onHostHealthUpdate() { + ShardVectorSharedPtr current_shard_vector; + { + absl::ReaderMutexLock lock(&mutex_); + current_shard_vector = shard_vector_; + } + + auto shard_vector = std::make_shared>(); + + for (auto const& shard : *current_shard_vector) { + shard_vector->emplace_back(std::make_shared( + shard->master(), shard->replicas().hostsPtr(), shard->allHosts().hostsPtr())); + } + + { + absl::WriterMutexLock lock(&mutex_); + shard_vector_ = std::move(shard_vector); } - return should_update; } Upstream::LoadBalancerPtr RedisClusterLoadBalancerFactory::create() { absl::ReaderMutexLock lock(&mutex_); - return std::make_unique(slot_array_); + return std::make_unique(slot_array_, shard_vector_, random_); } -Upstream::HostConstSharedPtr -RedisClusterLoadBalancer::chooseHost(Envoy::Upstream::LoadBalancerContext* context) { +namespace { +Upstream::HostConstSharedPtr chooseRandomHost(const Upstream::HostSetImpl& host_set, + Runtime::RandomGenerator& random) { + auto hosts = host_set.healthyHosts(); + if (hosts.empty()) { + hosts = host_set.degradedHosts(); + } + + if (hosts.empty()) { + hosts = host_set.hosts(); + } + + if (!hosts.empty()) { + return hosts[random.random() % hosts.size()]; + } else { + return nullptr; + } +} +} // namespace + +Upstream::HostConstSharedPtr RedisClusterLoadBalancerFactory::RedisClusterLoadBalancer::chooseHost( + Envoy::Upstream::LoadBalancerContext* context) { if (!slot_array_) { return nullptr; } @@ -55,17 +127,60 @@ RedisClusterLoadBalancer::chooseHost(Envoy::Upstream::LoadBalancerContext* conte return nullptr; } - return slot_array_->at(hash.value() % Envoy::Extensions::Clusters::Redis::MaxSlot); + auto shard = shard_vector_->at( + slot_array_->at(hash.value() % Envoy::Extensions::Clusters::Redis::MaxSlot)); + + auto redis_context = dynamic_cast(context); + if (redis_context && redis_context->isReadCommand()) { + switch (redis_context->readPolicy()) { + case NetworkFilters::Common::Redis::Client::ReadPolicy::Master: + return shard->master(); + case NetworkFilters::Common::Redis::Client::ReadPolicy::PreferMaster: + if (shard->master()->health() == Upstream::Host::Health::Healthy) { + return shard->master(); + } else { + return chooseRandomHost(shard->allHosts(), random_); + } + case NetworkFilters::Common::Redis::Client::ReadPolicy::Replica: + return chooseRandomHost(shard->replicas(), random_); + case NetworkFilters::Common::Redis::Client::ReadPolicy::PreferReplica: + if (!shard->replicas().healthyHosts().empty()) { + return chooseRandomHost(shard->replicas(), random_); + } else { + return chooseRandomHost(shard->allHosts(), random_); + } + case NetworkFilters::Common::Redis::Client::ReadPolicy::Any: + return chooseRandomHost(shard->allHosts(), random_); + } + } + return shard->master(); +} + +namespace { +bool isReadRequest(const NetworkFilters::Common::Redis::RespValue& request) { + if (request.type() != NetworkFilters::Common::Redis::RespType::Array) { + return false; + } + auto first = request.asArray()[0]; + if (first.type() != NetworkFilters::Common::Redis::RespType::SimpleString && + first.type() != NetworkFilters::Common::Redis::RespType::BulkString) { + return false; + } + return NetworkFilters::Common::Redis::SupportedCommands::isReadCommand(first.asString()); } +} // namespace -RedisLoadBalancerContext::RedisLoadBalancerContext(const std::string& key, bool enabled_hashtagging, - bool use_crc16) +RedisLoadBalancerContextImpl::RedisLoadBalancerContextImpl( + const std::string& key, bool enabled_hashtagging, bool use_crc16, + const NetworkFilters::Common::Redis::RespValue& request, + NetworkFilters::Common::Redis::Client::ReadPolicy read_policy) : hash_key_(use_crc16 ? Crc16::crc16(hashtag(key, enabled_hashtagging)) - : MurmurHash::murmurHash2_64(hashtag(key, enabled_hashtagging))) {} + : MurmurHash::murmurHash2_64(hashtag(key, enabled_hashtagging))), + is_read_(isReadRequest(request)), read_policy_(read_policy) {} // Inspired by the redis-cluster hashtagging algorithm // https://redis.io/topics/cluster-spec#keys-hash-tags -absl::string_view RedisLoadBalancerContext::hashtag(absl::string_view v, bool enabled) { +absl::string_view RedisLoadBalancerContextImpl::hashtag(absl::string_view v, bool enabled) { if (!enabled) { return v; } diff --git a/source/extensions/clusters/redis/redis_cluster_lb.h b/source/extensions/clusters/redis/redis_cluster_lb.h index c34112352bfa..3ad30cf76074 100644 --- a/source/extensions/clusters/redis/redis_cluster_lb.h +++ b/source/extensions/clusters/redis/redis_cluster_lb.h @@ -14,7 +14,12 @@ #include "source/extensions/clusters/redis/crc16.h" #include "extensions/clusters/well_known_names.h" +#include "extensions/filters/network/common/redis/client.h" +#include "extensions/filters/network/common/redis/codec.h" +#include "extensions/filters/network/common/redis/supported_commands.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/synchronization/mutex.h" namespace Envoy { @@ -24,10 +29,6 @@ namespace Redis { static const uint64_t MaxSlot = 16384; -using SlotArray = std::array; - -using SlotArraySharedPtr = std::shared_ptr; - class ClusterSlot { public: ClusterSlot(int64_t start, int64_t end, Network::Address::InstanceConstSharedPtr master) @@ -35,48 +36,57 @@ class ClusterSlot { int64_t start() const { return start_; } int64_t end() const { return end_; } - Network::Address::InstanceConstSharedPtr master() const { return master_; }; + Network::Address::InstanceConstSharedPtr master() const { return master_; } + const absl::flat_hash_set& replicas() const { + return replicas_; + } + void addReplica(Network::Address::InstanceConstSharedPtr replica_address) { + replicas_.insert(std::move(replica_address)); + } + + bool operator==(const ClusterSlot& rhs) const; private: - const int64_t start_; - const int64_t end_; + int64_t start_; + int64_t end_; Network::Address::InstanceConstSharedPtr master_; + absl::flat_hash_set replicas_; +}; + +using ClusterSlotsPtr = std::unique_ptr>; +using ClusterSlotsSharedPtr = std::shared_ptr>; + +class RedisLoadBalancerContext { +public: + virtual ~RedisLoadBalancerContext() = default; + + virtual bool isReadCommand() const PURE; + virtual NetworkFilters::Common::Redis::Client::ReadPolicy readPolicy() const PURE; }; -class RedisLoadBalancerContext : public Upstream::LoadBalancerContextBase { +class RedisLoadBalancerContextImpl : public RedisLoadBalancerContext, + public Upstream::LoadBalancerContextBase { public: - RedisLoadBalancerContext(const std::string& key, bool enabled_hashtagging, bool use_crc16); + RedisLoadBalancerContextImpl(const std::string& key, bool enabled_hashtagging, bool use_crc16, + const NetworkFilters::Common::Redis::RespValue& request, + NetworkFilters::Common::Redis::Client::ReadPolicy read_policy = + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); // Upstream::LoadBalancerContextBase absl::optional computeHashKey() override { return hash_key_; } + bool isReadCommand() const override { return is_read_; } + + NetworkFilters::Common::Redis::Client::ReadPolicy readPolicy() const override { + return read_policy_; + } + private: absl::string_view hashtag(absl::string_view v, bool enabled); const absl::optional hash_key_; -}; - -/* - * This class implements load balancing according to `Redis Cluster - * `_. This load balancer is thread local and created through - * the RedisClusterLoadBalancerFactory by the cluster manager. - * - * The topology is stored in cluster_slots_map_. According to the - * `Redis Cluster Spec & slots, - Upstream::HostMap all_hosts) PURE; + virtual bool onClusterSlotUpdate(ClusterSlotsPtr&& slots, Upstream::HostMap all_hosts) PURE; + + /** + * Callback when a host's health status is updated + */ + virtual void onHostHealthUpdate() PURE; }; using ClusterSlotUpdateCallBackSharedPtr = std::shared_ptr; @@ -102,16 +116,77 @@ using ClusterSlotUpdateCallBackSharedPtr = std::shared_ptr& slots, - Upstream::HostMap all_hosts) override; + bool onClusterSlotUpdate(ClusterSlotsPtr&& slots, Upstream::HostMap all_hosts) override; + + void onHostHealthUpdate() override; // Upstream::LoadBalancerFactory Upstream::LoadBalancerPtr create() override; private: + class RedisShard { + public: + RedisShard(Upstream::HostConstSharedPtr master, Upstream::HostVectorConstSharedPtr replicas, + Upstream::HostVectorConstSharedPtr all_hosts) + : master_(std::move(master)) { + replicas_.updateHosts(Upstream::HostSetImpl::partitionHosts( + std::move(replicas), Upstream::HostsPerLocalityImpl::empty()), + nullptr, {}, {}); + all_hosts_.updateHosts(Upstream::HostSetImpl::partitionHosts( + std::move(all_hosts), Upstream::HostsPerLocalityImpl::empty()), + nullptr, {}, {}); + } + const Upstream::HostConstSharedPtr master() const { return master_; } + const Upstream::HostSetImpl& replicas() const { return replicas_; } + const Upstream::HostSetImpl& allHosts() const { return all_hosts_; } + + private: + const Upstream::HostConstSharedPtr master_; + Upstream::HostSetImpl replicas_{0, absl::nullopt}; + Upstream::HostSetImpl all_hosts_{0, absl::nullopt}; + }; + + using RedisShardSharedPtr = std::shared_ptr; + using ShardVectorSharedPtr = std::shared_ptr>; + using SlotArray = std::array; + using SlotArraySharedPtr = std::shared_ptr; + + /* + * This class implements load balancing according to `Redis Cluster + * `_. This load balancer is thread local and created + * through the RedisClusterLoadBalancerFactory by the cluster manager. + * + * The topology is stored in slot_array_ and shard_vector_. According to the + * `Redis Cluster Spec ; +/** + * Read policy to use for Redis cluster. + */ +enum class ReadPolicy { Master, PreferMaster, Replica, PreferReplica, Any }; + /** * Configuration for a redis connection pool. */ @@ -157,6 +162,11 @@ class Config { * minimize the need for a large maxUpstreamUnknownConnections() value. */ virtual uint32_t maxUpstreamUnknownConnections() const PURE; + + /** + * @return the read policy the proxy should use. + */ + virtual ReadPolicy readPolicy() const PURE; }; /** diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 192177dca7a1..a2610417a873 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -19,7 +19,32 @@ ConfigImpl::ConfigImpl( 3)), // Default timeout is 3ms. If max_buffer_size_before_flush is zero, this is not used // as the buffer is flushed on each request immediately. max_upstream_unknown_connections_( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)) {} + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)) { + switch (config.read_policy()) { + case envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_MASTER: + read_policy_ = ReadPolicy::Master; + break; + case envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_MASTER: + read_policy_ = ReadPolicy::PreferMaster; + break; + case envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_REPLICA: + read_policy_ = ReadPolicy::PreferMaster; + break; + case envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_REPLICA: + read_policy_ = ReadPolicy::PreferMaster; + break; + case envoy::config::filter::network::redis_proxy::v2::RedisProxy_ConnPoolSettings_ReadPolicy_ANY: + read_policy_ = ReadPolicy::PreferMaster; + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + break; + } +} ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index ffeb0ee458d8..9522482fb022 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -49,6 +49,7 @@ class ConfigImpl : public Config { uint32_t maxUpstreamUnknownConnections() const override { return max_upstream_unknown_connections_; } + ReadPolicy readPolicy() const override { return read_policy_; } private: const std::chrono::milliseconds op_timeout_; @@ -57,6 +58,7 @@ class ConfigImpl : public Config { const uint32_t max_buffer_size_before_flush_; const std::chrono::milliseconds buffer_flush_timeout_; const uint32_t max_upstream_unknown_connections_; + ReadPolicy read_policy_; }; class ClientImpl : public Client, public DecoderCallbacks, public Network::ConnectionCallbacks { diff --git a/source/extensions/filters/network/common/redis/supported_commands.h b/source/extensions/filters/network/common/redis/supported_commands.h index a315a02deeec..89e93868fc8d 100644 --- a/source/extensions/filters/network/common/redis/supported_commands.h +++ b/source/extensions/filters/network/common/redis/supported_commands.h @@ -18,34 +18,34 @@ struct SupportedCommands { /** * @return commands which hash to a single server */ - static const std::vector& simpleCommands() { + static const absl::flat_hash_set& simpleCommands() { CONSTRUCT_ON_FIRST_USE( - std::vector, "append", "bitcount", "bitfield", "bitpos", "decr", "decrby", - "dump", "expire", "expireat", "geoadd", "geodist", "geohash", "geopos", "georadius_ro", - "georadiusbymember_ro", "get", "getbit", "getrange", "getset", "hdel", "hexists", "hget", - "hgetall", "hincrby", "hincrbyfloat", "hkeys", "hlen", "hmget", "hmset", "hscan", "hset", - "hsetnx", "hstrlen", "hvals", "incr", "incrby", "incrbyfloat", "lindex", "linsert", "llen", - "lpop", "lpush", "lpushx", "lrange", "lrem", "lset", "ltrim", "persist", "pexpire", - "pexpireat", "psetex", "pttl", "restore", "rpop", "rpush", "rpushx", "sadd", "scard", "set", - "setbit", "setex", "setnx", "setrange", "sismember", "smembers", "spop", "srandmember", - "srem", "sscan", "strlen", "ttl", "type", "zadd", "zcard", "zcount", "zincrby", "zlexcount", - "zpopmin", "zpopmax", "zrange", "zrangebylex", "zrangebyscore", "zrank", "zrem", - "zremrangebylex", "zremrangebyrank", "zremrangebyscore", "zrevrange", "zrevrangebylex", - "zrevrangebyscore", "zrevrank", "zscan", "zscore"); + absl::flat_hash_set, "append", "bitcount", "bitfield", "bitpos", "decr", + "decrby", "dump", "expire", "expireat", "geoadd", "geodist", "geohash", "geopos", + "georadius_ro", "georadiusbymember_ro", "get", "getbit", "getrange", "getset", "hdel", + "hexists", "hget", "hgetall", "hincrby", "hincrbyfloat", "hkeys", "hlen", "hmget", "hmset", + "hscan", "hset", "hsetnx", "hstrlen", "hvals", "incr", "incrby", "incrbyfloat", "lindex", + "linsert", "llen", "lpop", "lpush", "lpushx", "lrange", "lrem", "lset", "ltrim", "persist", + "pexpire", "pexpireat", "psetex", "pttl", "restore", "rpop", "rpush", "rpushx", "sadd", + "scard", "set", "setbit", "setex", "setnx", "setrange", "sismember", "smembers", "spop", + "srandmember", "srem", "sscan", "strlen", "ttl", "type", "zadd", "zcard", "zcount", + "zincrby", "zlexcount", "zpopmin", "zpopmax", "zrange", "zrangebylex", "zrangebyscore", + "zrank", "zrem", "zremrangebylex", "zremrangebyrank", "zremrangebyscore", "zrevrange", + "zrevrangebylex", "zrevrangebyscore", "zrevrank", "zscan", "zscore"); } /** * @return commands which hash on the fourth argument */ - static const std::vector& evalCommands() { - CONSTRUCT_ON_FIRST_USE(std::vector, "eval", "evalsha"); + static const absl::flat_hash_set& evalCommands() { + CONSTRUCT_ON_FIRST_USE(absl::flat_hash_set, "eval", "evalsha"); } /** * @return commands which are sent to multiple servers and coalesced by summing the responses */ - static const std::vector& hashMultipleSumResultCommands() { - CONSTRUCT_ON_FIRST_USE(std::vector, "del", "exists", "touch", "unlink"); + static const absl::flat_hash_set& hashMultipleSumResultCommands() { + CONSTRUCT_ON_FIRST_USE(absl::flat_hash_set, "del", "exists", "touch", "unlink"); } /** @@ -81,6 +81,10 @@ struct SupportedCommands { "zadd", "zincrby", "touch", "zpopmin", "zpopmax", "zrem", "zremrangebylex", "zremrangebyrank", "zremrangebyscore", "unlink"); } + + static bool isReadCommand(const std::string& command) { + return !writeCommands().contains(command); + } }; } // namespace Redis diff --git a/source/extensions/filters/network/common/redis/utility.cc b/source/extensions/filters/network/common/redis/utility.cc index 81f69e20b0ad..c13b9c4fb4e8 100644 --- a/source/extensions/filters/network/common/redis/utility.cc +++ b/source/extensions/filters/network/common/redis/utility.cc @@ -18,6 +18,19 @@ Redis::RespValue makeAuthCommand(const std::string& password) { return auth_command; } +ReadOnlyRequest::ReadOnlyRequest() { + std::vector values(1); + values[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + values[0].asString() = "readonly"; + type(NetworkFilters::Common::Redis::RespType::Array); + asArray().swap(values); +} + +const ReadOnlyRequest& ReadOnlyRequest::instance() { + static const ReadOnlyRequest* instance = new ReadOnlyRequest{}; + return *instance; +} + } // namespace Utility } // namespace Redis } // namespace Common diff --git a/source/extensions/filters/network/common/redis/utility.h b/source/extensions/filters/network/common/redis/utility.h index 619c470b359e..fc8369acef45 100644 --- a/source/extensions/filters/network/common/redis/utility.h +++ b/source/extensions/filters/network/common/redis/utility.h @@ -13,6 +13,12 @@ namespace Utility { Redis::RespValue makeAuthCommand(const std::string& password); +class ReadOnlyRequest : public Redis::RespValue { +public: + ReadOnlyRequest(); + static const ReadOnlyRequest& instance(); +}; + } // namespace Utility } // namespace Redis } // namespace Common diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index 44f2634aa89a..2e1414e0252c 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -9,7 +9,6 @@ #include "common/stats/utility.h" #include "extensions/filters/network/redis_proxy/config.h" -#include "extensions/filters/network/well_known_names.h" namespace Envoy { namespace Extensions { @@ -198,11 +197,19 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared client->host_ = host; client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_); client->redis_client_->addConnectionCallbacks(*client); + // TODO(hyang): should the auth command and readonly command be moved to the factory method? if (!auth_password_.empty()) { // Send an AUTH command to the upstream server. client->redis_client_->makeRequest(Common::Redis::Utility::makeAuthCommand(auth_password_), null_pool_callbacks); } + // Any connection to replica requires the READONLY command in order to perform read. + // Also the READONLY command is a no-opt for the master. + // We only need to send the READONLY command iff it's possible that the host is a replica. + if (parent_.config_.readPolicy() != Common::Redis::Client::ReadPolicy::Master) { + client->redis_client_->makeRequest(Common::Redis::Utility::ReadOnlyRequest::instance(), + null_pool_callbacks); + } } return client; } @@ -218,8 +225,8 @@ InstanceImpl::ThreadLocalPool::makeRequest(const std::string& key, } const bool use_crc16 = is_redis_cluster_; - Clusters::Redis::RedisLoadBalancerContext lb_context(key, parent_.config_.enableHashtagging(), - use_crc16); + Clusters::Redis::RedisLoadBalancerContextImpl lb_context(key, parent_.config_.enableHashtagging(), + use_crc16, request); Upstream::HostConstSharedPtr host = cluster_->loadBalancer().chooseHost(&lb_context); if (!host) { return nullptr; diff --git a/source/extensions/filters/network/redis_proxy/router_impl.cc b/source/extensions/filters/network/redis_proxy/router_impl.cc index 7fd51f52005a..6cfa59022041 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.cc +++ b/source/extensions/filters/network/redis_proxy/router_impl.cc @@ -19,8 +19,7 @@ bool MirrorPolicyImpl::shouldMirror(const std::string& command) const { return false; } - if (exclude_read_commands_ && Common::Redis::SupportedCommands::writeCommands().find(command) == - Common::Redis::SupportedCommands::writeCommands().end()) { + if (exclude_read_commands_ && Common::Redis::SupportedCommands::isReadCommand(command)) { return false; } diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index 346b27e9b951..4c7cc320e01f 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -68,6 +68,9 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { bool enableRedirection() const override { return true; } // Redirection errors are treated as check successes. + NetworkFilters::Common::Redis::Client::ReadPolicy readPolicy() const override { + return NetworkFilters::Common::Redis::Client::ReadPolicy::Master; + } // Batching unsigned int maxBufferSizeBeforeFlush() const override { diff --git a/test/extensions/clusters/redis/BUILD b/test/extensions/clusters/redis/BUILD index b990f1df93fe..25495a9d7d95 100644 --- a/test/extensions/clusters/redis/BUILD +++ b/test/extensions/clusters/redis/BUILD @@ -49,6 +49,7 @@ envoy_extension_cc_test( srcs = ["redis_cluster_lb_test.cc"], extension_name = "envoy.clusters.redis", deps = [ + "//include/envoy/upstream:cluster_manager_interface", "//source/common/event:dispatcher_lib", "//source/common/network:utility_lib", "//source/common/upstream:cluster_factory_lib", @@ -56,6 +57,9 @@ envoy_extension_cc_test( "//source/common/upstream:upstream_lib", "//source/extensions/clusters/redis:redis_cluster", "//source/extensions/clusters/redis:redis_cluster_lb", + "//source/extensions/filters/network/common/redis:client_interface", + "//source/extensions/filters/network/common/redis:codec_lib", + "//source/extensions/filters/network/common/redis:supported_commands_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", "//test/common/upstream:utility_lib", diff --git a/test/extensions/clusters/redis/mocks.cc b/test/extensions/clusters/redis/mocks.cc index e10eb2705333..b66ddd7c0505 100644 --- a/test/extensions/clusters/redis/mocks.cc +++ b/test/extensions/clusters/redis/mocks.cc @@ -12,13 +12,7 @@ namespace Clusters { namespace Redis { MockClusterSlotUpdateCallBack::MockClusterSlotUpdateCallBack() { - ON_CALL(*this, onClusterSlotUpdate(_, _)) - .WillByDefault( - Invoke([&](const std::vector& slots, Upstream::HostMap all_hosts) -> bool { - EXPECT_FALSE(slots.empty()); - EXPECT_FALSE(all_hosts.empty()); - return true; - })); + ON_CALL(*this, onClusterSlotUpdate(_, _)).WillByDefault(Return(true)); } } // namespace Redis diff --git a/test/extensions/clusters/redis/mocks.h b/test/extensions/clusters/redis/mocks.h index 7269548b1b2c..c9cdb9841aa9 100644 --- a/test/extensions/clusters/redis/mocks.h +++ b/test/extensions/clusters/redis/mocks.h @@ -17,7 +17,8 @@ class MockClusterSlotUpdateCallBack : public ClusterSlotUpdateCallBack { MockClusterSlotUpdateCallBack(); ~MockClusterSlotUpdateCallBack() override = default; - MOCK_METHOD2(onClusterSlotUpdate, bool(const std::vector&, Upstream::HostMap)); + MOCK_METHOD2(onClusterSlotUpdate, bool(ClusterSlotsPtr&&, Upstream::HostMap)); + MOCK_METHOD0(onHostHealthUpdate, void()); }; } // namespace Redis diff --git a/test/extensions/clusters/redis/redis_cluster_integration_test.cc b/test/extensions/clusters/redis/redis_cluster_integration_test.cc index 077dc3c9d92e..91aecc412a3d 100644 --- a/test/extensions/clusters/redis/redis_cluster_integration_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_integration_test.cc @@ -217,13 +217,13 @@ class RedisClusterIntegrationTest : public testing::TestWithParamaddressAsString(), master->port()) - << makeIp(slave->addressAsString(), slave->port()); + << makeIp(replica->addressAsString(), replica->port()); return resp.str(); } @@ -286,16 +286,16 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, RedisClusterWithAuthIntegrationTest, // This test sends a simple "get foo" command from a fake // downstream client through the proxy to a fake upstream -// Redis cluster with a single slot with master and slave. +// Redis cluster with a single slot with master and replica. // The fake server sends a valid response back to the client. // The request and response should make it through the envoy // proxy server code unchanged. -TEST_P(RedisClusterIntegrationTest, SingleSlotMasterSlave) { +TEST_P(RedisClusterIntegrationTest, SingleSlotMasterReplica) { random_index_ = 0; on_server_init_function_ = [this]() { - std::string cluster_slot_response = singleSlotMasterSlave( + std::string cluster_slot_response = singleSlotMasterReplica( fake_upstreams_[0]->localAddress()->ip(), fake_upstreams_[1]->localAddress()->ip()); expectCallClusterSlot(random_index_, cluster_slot_response); }; @@ -333,7 +333,7 @@ TEST_P(RedisClusterIntegrationTest, TwoSlot) { // This test sends a simple "get foo" command from a fake // downstream client through the proxy to a fake upstream -// Redis cluster with a single slot with master and slave. +// Redis cluster with a single slot with master and replica. // The fake server sends a valid response back to the client. // The request and response should make it through the envoy // proxy server code unchanged. @@ -343,11 +343,11 @@ TEST_P(RedisClusterIntegrationTest, TwoSlot) { // "cluster slots" command), and one to authenticate the connection // that carries the "get foo" request. -TEST_P(RedisClusterWithAuthIntegrationTest, SingleSlotMasterSlave) { +TEST_P(RedisClusterWithAuthIntegrationTest, SingleSlotMasterReplica) { random_index_ = 0; on_server_init_function_ = [this]() { - std::string cluster_slot_response = singleSlotMasterSlave( + std::string cluster_slot_response = singleSlotMasterReplica( fake_upstreams_[0]->localAddress()->ip(), fake_upstreams_[1]->localAddress()->ip()); expectCallClusterSlot(0, cluster_slot_response, "somepassword"); }; diff --git a/test/extensions/clusters/redis/redis_cluster_lb_test.cc b/test/extensions/clusters/redis/redis_cluster_lb_test.cc index 5fe696a10491..dedcf8e51ec8 100644 --- a/test/extensions/clusters/redis/redis_cluster_lb_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_lb_test.cc @@ -2,24 +2,38 @@ #include "source/extensions/clusters/redis/redis_cluster_lb.h" +#include "extensions/filters/network/common/redis/client.h" + #include "test/common/upstream/utility.h" #include "test/mocks/upstream/mocks.h" +using testing::Return; + namespace Envoy { namespace Extensions { namespace Clusters { namespace Redis { -class TestLoadBalancerContext : public Upstream::LoadBalancerContextBase { +class TestLoadBalancerContext : public RedisLoadBalancerContext, + public Upstream::LoadBalancerContextBase { public: - TestLoadBalancerContext(uint64_t hash_key) : hash_key_(hash_key) {} + TestLoadBalancerContext(uint64_t hash_key, bool is_read, + NetworkFilters::Common::Redis::Client::ReadPolicy read_policy) + : hash_key_(hash_key), is_read_(is_read), read_policy_(read_policy) {} TestLoadBalancerContext(absl::optional hash) : hash_key_(hash) {} // Upstream::LoadBalancerContext absl::optional computeHashKey() override { return hash_key_; } + bool isReadCommand() const override { return is_read_; }; + NetworkFilters::Common::Redis::Client::ReadPolicy readPolicy() const override { + return read_policy_; + }; + absl::optional hash_key_; + bool is_read_; + NetworkFilters::Common::Redis::Client::ReadPolicy read_policy_; }; class RedisClusterLoadBalancerTest : public testing::Test { @@ -27,28 +41,44 @@ class RedisClusterLoadBalancerTest : public testing::Test { RedisClusterLoadBalancerTest() = default; void init() { - factory_ = std::make_shared(); + factory_ = std::make_shared(random_); lb_ = std::make_unique(factory_); lb_->initialize(); } void validateAssignment(Upstream::HostVector& hosts, - const std::vector> expected_assignments) { + const std::vector>& expected_assignments, + bool read_command = false, + NetworkFilters::Common::Redis::Client::ReadPolicy read_policy = + NetworkFilters::Common::Redis::Client::ReadPolicy::Master) { Upstream::LoadBalancerPtr lb = lb_->factory()->create(); for (auto& assignment : expected_assignments) { - TestLoadBalancerContext context(assignment.first); - EXPECT_EQ(hosts[assignment.second]->address()->asString(), - lb->chooseHost(&context)->address()->asString()); + TestLoadBalancerContext context(assignment.first, read_command, read_policy); + auto host = lb->chooseHost(&context); + EXPECT_FALSE(host == nullptr); + EXPECT_EQ(hosts[assignment.second]->address()->asString(), host->address()->asString()); } } + static std::pair makePair(Upstream::HostSharedPtr host) { + return std::make_pair(host->address()->asString(), std::move(host)); + } + + Upstream::HostMap generateHostMap(Upstream::HostVector& hosts) { + Upstream::HostMap map; + std::transform(hosts.begin(), hosts.end(), std::inserter(map, map.end()), makePair); + return map; + } + std::shared_ptr factory_; - SlotArraySharedPtr slot_array_; std::unique_ptr lb_; std::shared_ptr info_{new NiceMock()}; + NiceMock random_; }; +class RedisLoadBalancerContextImplTest : public testing::Test {}; + // Works correctly without any hosts. TEST_F(RedisClusterLoadBalancerTest, NoHost) { init(); @@ -61,18 +91,18 @@ TEST_F(RedisClusterLoadBalancerTest, NoHash) { Upstream::makeTestHost(info_, "tcp://127.0.0.1:91"), Upstream::makeTestHost(info_, "tcp://127.0.0.1:92")}; - const std::vector slots{ + ClusterSlotsPtr slots = std::make_unique>(std::vector{ ClusterSlot(0, 1000, hosts[0]->address()), ClusterSlot(1001, 2000, hosts[1]->address()), ClusterSlot(2001, 16383, hosts[2]->address()), - }; + }); Upstream::HostMap all_hosts{ {hosts[0]->address()->asString(), hosts[0]}, {hosts[1]->address()->asString(), hosts[1]}, {hosts[2]->address()->asString(), hosts[2]}, }; init(); - factory_->onClusterSlotUpdate(slots, all_hosts); + factory_->onClusterSlotUpdate(std::move(slots), all_hosts); TestLoadBalancerContext context(absl::nullopt); EXPECT_EQ(nullptr, lb_->factory()->create()->chooseHost(&context)); }; @@ -82,18 +112,18 @@ TEST_F(RedisClusterLoadBalancerTest, Basic) { Upstream::makeTestHost(info_, "tcp://127.0.0.1:91"), Upstream::makeTestHost(info_, "tcp://127.0.0.1:92")}; - const std::vector slots{ + ClusterSlotsPtr slots = std::make_unique>(std::vector{ ClusterSlot(0, 1000, hosts[0]->address()), ClusterSlot(1001, 2000, hosts[1]->address()), ClusterSlot(2001, 16383, hosts[2]->address()), - }; + }); Upstream::HostMap all_hosts{ {hosts[0]->address()->asString(), hosts[0]}, {hosts[1]->address()->asString(), hosts[1]}, {hosts[2]->address()->asString(), hosts[2]}, }; init(); - factory_->onClusterSlotUpdate(slots, all_hosts); + factory_->onClusterSlotUpdate(std::move(slots), all_hosts); // A list of (hash: host_index) pair const std::vector> expected_assignments = { @@ -102,15 +132,183 @@ TEST_F(RedisClusterLoadBalancerTest, Basic) { validateAssignment(hosts, expected_assignments); } +TEST_F(RedisClusterLoadBalancerTest, ReadStrategiesHealthy) { + Upstream::HostVector hosts{ + Upstream::makeTestHost(info_, "tcp://127.0.0.1:90"), + Upstream::makeTestHost(info_, "tcp://127.0.0.1:91"), + Upstream::makeTestHost(info_, "tcp://127.0.0.2:90"), + Upstream::makeTestHost(info_, "tcp://127.0.0.2:91"), + }; + + ClusterSlotsPtr slots = std::make_unique>(std::vector{ + ClusterSlot(0, 2000, hosts[0]->address()), + ClusterSlot(2001, 16383, hosts[1]->address()), + }); + slots->at(0).addReplica(hosts[2]->address()); + slots->at(1).addReplica(hosts[3]->address()); + Upstream::HostMap all_hosts; + std::transform(hosts.begin(), hosts.end(), std::inserter(all_hosts, all_hosts.end()), makePair); + init(); + factory_->onClusterSlotUpdate(std::move(slots), all_hosts); + + // A list of (hash: host_index) pair + const std::vector> replica_assignments = { + {0, 2}, {1100, 2}, {2000, 2}, {18382, 2}, {2001, 3}, {2100, 3}, {16383, 3}, {19382, 3}}; + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Replica); + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferReplica); + + const std::vector> master_assignments = { + {0, 0}, {1100, 0}, {2000, 0}, {18382, 0}, {2001, 1}, {2100, 1}, {16383, 1}, {19382, 1}}; + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferMaster); + + ON_CALL(random_, random()).WillByDefault(Return(0)); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Any); + ON_CALL(random_, random()).WillByDefault(Return(1)); + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Any); +} + +TEST_F(RedisClusterLoadBalancerTest, ReadStrategiesUnhealthyMaster) { + Upstream::HostVector hosts{ + Upstream::makeTestHost(info_, "tcp://127.0.0.1:90"), + Upstream::makeTestHost(info_, "tcp://127.0.0.1:91"), + Upstream::makeTestHost(info_, "tcp://127.0.0.2:90"), + Upstream::makeTestHost(info_, "tcp://127.0.0.2:91"), + }; + + ClusterSlotsPtr slots = std::make_unique>(std::vector{ + ClusterSlot(0, 2000, hosts[0]->address()), + ClusterSlot(2001, 16383, hosts[1]->address()), + }); + slots->at(0).addReplica(hosts[2]->address()); + slots->at(1).addReplica(hosts[3]->address()); + Upstream::HostMap all_hosts; + std::transform(hosts.begin(), hosts.end(), std::inserter(all_hosts, all_hosts.end()), makePair); + init(); + factory_->onClusterSlotUpdate(std::move(slots), all_hosts); + + hosts[0]->healthFlagSet(Upstream::Host::HealthFlag::FAILED_ACTIVE_HC); + hosts[1]->healthFlagSet(Upstream::Host::HealthFlag::FAILED_ACTIVE_HC); + + factory_->onHostHealthUpdate(); + + // A list of (hash: host_index) pair + const std::vector> replica_assignments = { + {0, 2}, {1100, 2}, {2000, 2}, {18382, 2}, {2001, 3}, {2100, 3}, {16383, 3}, {19382, 3}}; + const std::vector> master_assignments = { + {0, 0}, {1100, 0}, {2000, 0}, {18382, 0}, {2001, 1}, {2100, 1}, {16383, 1}, {19382, 1}}; + + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Replica); + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferReplica); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferMaster); + + ON_CALL(random_, random()).WillByDefault(Return(0)); + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Any); + ON_CALL(random_, random()).WillByDefault(Return(1)); + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Any); +} + +TEST_F(RedisClusterLoadBalancerTest, ReadStrategiesUnhealthyReplica) { + Upstream::HostVector hosts{ + Upstream::makeTestHost(info_, "tcp://127.0.0.1:90"), + Upstream::makeTestHost(info_, "tcp://127.0.0.1:91"), + Upstream::makeTestHost(info_, "tcp://127.0.0.2:90"), + Upstream::makeTestHost(info_, "tcp://127.0.0.2:91"), + }; + + ClusterSlotsPtr slots = std::make_unique>(std::vector{ + ClusterSlot(0, 2000, hosts[0]->address()), + ClusterSlot(2001, 16383, hosts[1]->address()), + }); + slots->at(0).addReplica(hosts[2]->address()); + slots->at(1).addReplica(hosts[3]->address()); + Upstream::HostMap all_hosts; + std::transform(hosts.begin(), hosts.end(), std::inserter(all_hosts, all_hosts.end()), makePair); + init(); + factory_->onClusterSlotUpdate(std::move(slots), all_hosts); + + hosts[2]->healthFlagSet(Upstream::Host::HealthFlag::FAILED_ACTIVE_HC); + hosts[3]->healthFlagSet(Upstream::Host::HealthFlag::FAILED_ACTIVE_HC); + + factory_->onHostHealthUpdate(); + + // A list of (hash: host_index) pair + const std::vector> replica_assignments = { + {0, 2}, {1100, 2}, {2000, 2}, {18382, 2}, {2001, 3}, {2100, 3}, {16383, 3}, {19382, 3}}; + const std::vector> master_assignments = { + {0, 0}, {1100, 0}, {2000, 0}, {18382, 0}, {2001, 1}, {2100, 1}, {16383, 1}, {19382, 1}}; + + validateAssignment(hosts, replica_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Replica); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferReplica); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferMaster); + + ON_CALL(random_, random()).WillByDefault(Return(0)); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Any); + ON_CALL(random_, random()).WillByDefault(Return(1)); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Any); +} + +TEST_F(RedisClusterLoadBalancerTest, ReadStrategiesNoReplica) { + Upstream::HostVector hosts{Upstream::makeTestHost(info_, "tcp://127.0.0.1:90"), + Upstream::makeTestHost(info_, "tcp://127.0.0.1:91")}; + + ClusterSlotsPtr slots = std::make_unique>(std::vector{ + ClusterSlot(0, 2000, hosts[0]->address()), + ClusterSlot(2001, 16383, hosts[1]->address()), + }); + Upstream::HostMap all_hosts; + std::transform(hosts.begin(), hosts.end(), std::inserter(all_hosts, all_hosts.end()), makePair); + init(); + factory_->onClusterSlotUpdate(std::move(slots), all_hosts); + + // A list of (hash: host_index) pair + const std::vector> master_assignments = { + {0, 0}, {1100, 0}, {2000, 0}, {18382, 0}, {2001, 1}, {2100, 1}, {16383, 1}, {19382, 1}}; + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferMaster); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Any); + validateAssignment(hosts, master_assignments, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferReplica); + + Upstream::LoadBalancerPtr lb = lb_->factory()->create(); + TestLoadBalancerContext context(1100, true, + NetworkFilters::Common::Redis::Client::ReadPolicy::Replica); + auto host = lb->chooseHost(&context); + EXPECT_TRUE(host == nullptr); +} + TEST_F(RedisClusterLoadBalancerTest, ClusterSlotUpdate) { Upstream::HostVector hosts{Upstream::makeTestHost(info_, "tcp://127.0.0.1:90"), Upstream::makeTestHost(info_, "tcp://127.0.0.1:91")}; - const std::vector slots{ClusterSlot(0, 1000, hosts[0]->address()), - ClusterSlot(1001, 16383, hosts[1]->address())}; + ClusterSlotsPtr slots = std::make_unique>(std::vector{ + ClusterSlot(0, 1000, hosts[0]->address()), ClusterSlot(1001, 16383, hosts[1]->address())}); Upstream::HostMap all_hosts{{hosts[0]->address()->asString(), hosts[0]}, {hosts[1]->address()->asString(), hosts[1]}}; init(); - EXPECT_EQ(true, factory_->onClusterSlotUpdate(slots, all_hosts)); + EXPECT_EQ(true, factory_->onClusterSlotUpdate(std::move(slots), all_hosts)); // A list of initial (hash: host_index) pair const std::vector> original_assignments = { @@ -124,7 +322,8 @@ TEST_F(RedisClusterLoadBalancerTest, ClusterSlotUpdate) { ClusterSlot(1001, 2000, hosts[1]->address()), ClusterSlot(2001, 16383, hosts[0]->address()), }; - EXPECT_EQ(true, factory_->onClusterSlotUpdate(updated_slot, all_hosts)); + EXPECT_EQ(true, factory_->onClusterSlotUpdate( + std::make_unique>(updated_slot), all_hosts)); // A list of updated (hash: host_index) pair. const std::vector> updated_assignments = { @@ -137,11 +336,11 @@ TEST_F(RedisClusterLoadBalancerTest, ClusterSlotNoUpdate) { Upstream::makeTestHost(info_, "tcp://127.0.0.1:91"), Upstream::makeTestHost(info_, "tcp://127.0.0.1:92")}; - const std::vector slots{ + ClusterSlotsPtr slots = std::make_unique>(std::vector{ ClusterSlot(0, 1000, hosts[0]->address()), ClusterSlot(1001, 2000, hosts[1]->address()), ClusterSlot(2001, 16383, hosts[2]->address()), - }; + }); Upstream::HostMap all_hosts{ {hosts[0]->address()->asString(), hosts[0]}, {hosts[1]->address()->asString(), hosts[1]}, @@ -153,7 +352,7 @@ TEST_F(RedisClusterLoadBalancerTest, ClusterSlotNoUpdate) { {100, 0}, {1100, 1}, {2100, 2}}; init(); - EXPECT_EQ(true, factory_->onClusterSlotUpdate(slots, all_hosts)); + EXPECT_EQ(true, factory_->onClusterSlotUpdate(std::move(slots), all_hosts)); validateAssignment(hosts, expected_assignments); // Calling cluster slot update without change should not change assignment. @@ -162,10 +361,67 @@ TEST_F(RedisClusterLoadBalancerTest, ClusterSlotNoUpdate) { ClusterSlot(1001, 2000, hosts[1]->address()), ClusterSlot(2001, 16383, hosts[2]->address()), }; - EXPECT_EQ(false, factory_->onClusterSlotUpdate(updated_slot, all_hosts)); + EXPECT_EQ(false, factory_->onClusterSlotUpdate( + std::make_unique>(updated_slot), all_hosts)); validateAssignment(hosts, expected_assignments); } +TEST_F(RedisLoadBalancerContextImplTest, Basic) { + // Simple read command + std::vector get_foo(2); + get_foo[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + get_foo[0].asString() = "get"; + get_foo[1].type(NetworkFilters::Common::Redis::RespType::BulkString); + get_foo[1].asString() = "foo"; + + NetworkFilters::Common::Redis::RespValue get_request; + get_request.type(NetworkFilters::Common::Redis::RespType::Array); + get_request.asArray().swap(get_foo); + + RedisLoadBalancerContextImpl context1("foo", true, true, get_request, + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); + + EXPECT_EQ(absl::optional(44950), context1.computeHashKey()); + EXPECT_EQ(true, context1.isReadCommand()); + EXPECT_EQ(NetworkFilters::Common::Redis::Client::ReadPolicy::Master, context1.readPolicy()); + + // Simple write command + std::vector set_foo(3); + set_foo[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + set_foo[0].asString() = "set"; + set_foo[1].type(NetworkFilters::Common::Redis::RespType::BulkString); + set_foo[1].asString() = "foo"; + set_foo[2].type(NetworkFilters::Common::Redis::RespType::BulkString); + set_foo[2].asString() = "bar"; + + NetworkFilters::Common::Redis::RespValue set_request; + set_request.type(NetworkFilters::Common::Redis::RespType::Array); + set_request.asArray().swap(set_foo); + + RedisLoadBalancerContextImpl context2("foo", true, true, set_request, + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); + + EXPECT_EQ(absl::optional(44950), context2.computeHashKey()); + EXPECT_EQ(false, context2.isReadCommand()); + EXPECT_EQ(NetworkFilters::Common::Redis::Client::ReadPolicy::Master, context2.readPolicy()); +} + +TEST_F(RedisLoadBalancerContextImplTest, UnsupportedCommand) { + std::vector unknown(1); + unknown[0].type(NetworkFilters::Common::Redis::RespType::Integer); + unknown[0].asInteger() = 1; + NetworkFilters::Common::Redis::RespValue unknown_request; + unknown_request.type(NetworkFilters::Common::Redis::RespType::Array); + unknown_request.asArray().swap(unknown); + + RedisLoadBalancerContextImpl context3("foo", true, true, unknown_request, + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); + + EXPECT_EQ(absl::optional(44950), context3.computeHashKey()); + EXPECT_EQ(false, context3.isReadCommand()); + EXPECT_EQ(NetworkFilters::Common::Redis::Client::ReadPolicy::Master, context3.readPolicy()); +} + } // namespace Redis } // namespace Clusters } // namespace Extensions diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index cb3a1b5568d1..d3af8ce09dcc 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -177,19 +177,20 @@ class RedisClusterTest : public testing::Test, pool_callbacks_->onFailure(); } - NetworkFilters::Common::Redis::RespValuePtr - singleSlotMasterSlave(const std::string& master, const std::string& slave, int64_t port) const { + NetworkFilters::Common::Redis::RespValuePtr singleSlotMasterReplica(const std::string& master, + const std::string& replica, + int64_t port) const { std::vector master_1(2); master_1[0].type(NetworkFilters::Common::Redis::RespType::BulkString); master_1[0].asString() = master; master_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); master_1[1].asInteger() = port; - std::vector slave_1(2); - slave_1[0].type(NetworkFilters::Common::Redis::RespType::BulkString); - slave_1[0].asString() = slave; - slave_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); - slave_1[1].asInteger() = port; + std::vector replica_1(2); + replica_1[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + replica_1[0].asString() = replica; + replica_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); + replica_1[1].asInteger() = port; std::vector slot_1(4); slot_1[0].type(NetworkFilters::Common::Redis::RespType::Integer); @@ -199,7 +200,7 @@ class RedisClusterTest : public testing::Test, slot_1[2].type(NetworkFilters::Common::Redis::RespType::Array); slot_1[2].asArray().swap(master_1); slot_1[3].type(NetworkFilters::Common::Redis::RespType::Array); - slot_1[3].asArray().swap(slave_1); + slot_1[3].asArray().swap(replica_1); std::vector slots(1); slots[0].type(NetworkFilters::Common::Redis::RespType::Array); @@ -254,6 +255,64 @@ class RedisClusterTest : public testing::Test, return response; } + NetworkFilters::Common::Redis::RespValuePtr twoSlotsMastersWithReplica() const { + std::vector master_1(2); + master_1[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + master_1[0].asString() = "127.0.0.1"; + master_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); + master_1[1].asInteger() = 22120; + + std::vector master_2(2); + master_2[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + master_2[0].asString() = "127.0.0.2"; + master_2[1].type(NetworkFilters::Common::Redis::RespType::Integer); + master_2[1].asInteger() = 22120; + + std::vector replica_1(2); + replica_1[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + replica_1[0].asString() = "127.0.0.3"; + replica_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); + replica_1[1].asInteger() = 22120; + + std::vector replica_2(2); + replica_2[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + replica_2[0].asString() = "127.0.0.4"; + replica_2[1].type(NetworkFilters::Common::Redis::RespType::Integer); + replica_2[1].asInteger() = 22120; + + std::vector slot_1(4); + slot_1[0].type(NetworkFilters::Common::Redis::RespType::Integer); + slot_1[0].asInteger() = 0; + slot_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); + slot_1[1].asInteger() = 9999; + slot_1[2].type(NetworkFilters::Common::Redis::RespType::Array); + slot_1[2].asArray().swap(master_1); + slot_1[3].type(NetworkFilters::Common::Redis::RespType::Array); + slot_1[3].asArray().swap(replica_1); + + std::vector slot_2(4); + slot_2[0].type(NetworkFilters::Common::Redis::RespType::Integer); + slot_2[0].asInteger() = 10000; + slot_2[1].type(NetworkFilters::Common::Redis::RespType::Integer); + slot_2[1].asInteger() = 16383; + slot_2[2].type(NetworkFilters::Common::Redis::RespType::Array); + slot_2[2].asArray().swap(master_2); + slot_2[3].type(NetworkFilters::Common::Redis::RespType::Array); + slot_2[3].asArray().swap(replica_2); + + std::vector slots(2); + slots[0].type(NetworkFilters::Common::Redis::RespType::Array); + slots[0].asArray().swap(slot_1); + slots[1].type(NetworkFilters::Common::Redis::RespType::Array); + slots[1].asArray().swap(slot_2); + + NetworkFilters::Common::Redis::RespValuePtr response( + new NetworkFilters::Common::Redis::RespValue()); + response->type(NetworkFilters::Common::Redis::RespType::Array); + response->asArray().swap(slots); + return response; + } + NetworkFilters::Common::Redis::RespValue createStringField(bool is_correct_type, const std::string& correct_value) const { NetworkFilters::Common::Redis::RespValue respValue; @@ -296,7 +355,8 @@ class RedisClusterTest : public testing::Test, // Create a redis cluster slot response. If a bit is set in the bitset, then that part of // of the response is correct, otherwise it's incorrect. - NetworkFilters::Common::Redis::RespValuePtr createResponse(std::bitset<10> flags) const { + NetworkFilters::Common::Redis::RespValuePtr createResponse(std::bitset<10> flags, + std::bitset<3> replica_flags) const { int64_t idx(0); int64_t slots_type = idx++; int64_t slots_size = idx++; @@ -308,6 +368,10 @@ class RedisClusterTest : public testing::Test, int64_t master_size = idx++; int64_t master_ip_type = idx++; int64_t master_port_type = idx++; + idx = 0; + int64_t replica_size = idx++; + int64_t replica_ip_type = idx++; + int64_t replica_port_type = idx++; std::vector master_1_array; if (flags.test(master_size)) { @@ -317,11 +381,23 @@ class RedisClusterTest : public testing::Test, master_1_array.push_back(createIntegerField(flags.test(master_port_type), 22120)); } + std::vector replica_1_array; + if (replica_flags.any()) { + // Ip field. + replica_1_array.push_back( + createStringField(replica_flags.test(replica_ip_type), "127.0.0.2")); + // Port field. + replica_1_array.push_back(createIntegerField(replica_flags.test(replica_port_type), 22120)); + } + std::vector slot_1_array; if (flags.test(slot1_size)) { slot_1_array.push_back(createIntegerField(flags.test(slot1_range_start_type), 0)); slot_1_array.push_back(createIntegerField(flags.test(slot1_range_end_type), 16383)); slot_1_array.push_back(createArrayField(flags.test(master_type), master_1_array)); + if (replica_flags.any()) { + slot_1_array.push_back(createArrayField(replica_flags.test(replica_size), replica_1_array)); + } } std::vector slots_array; @@ -368,11 +444,10 @@ class RedisClusterTest : public testing::Test, cluster_->initialize([&]() -> void { initialized_.ready(); }); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); - expectClusterSlotResponse(singleSlotMasterSlave("127.0.0.1", "127.0.0.2", 22120)); - // TODO(hyang): this will change once we register slaves as well - expectHealthyHosts(std::list({"127.0.0.1:22120"})); + expectClusterSlotResponse(singleSlotMasterReplica("127.0.0.1", "127.0.0.2", 22120)); + expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); - // Add new host. + // Promote replica to master expectRedisResolve(); EXPECT_CALL(membership_updated_, ready()); resolve_timer_->invokeCallback(); @@ -387,13 +462,30 @@ class RedisClusterTest : public testing::Test, expectClusterSlotResponse(twoSlotsMasters()); expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); - // Remove host. + // Add replicas to masters expectRedisResolve(); EXPECT_CALL(membership_updated_, ready()); resolve_timer_->invokeCallback(); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); - expectClusterSlotResponse(singleSlotMasterSlave("127.0.0.1", "127.0.0.2", 22120)); - expectHealthyHosts(std::list({"127.0.0.1:22120"})); + expectClusterSlotResponse(twoSlotsMastersWithReplica()); + expectHealthyHosts(std::list( + {"127.0.0.1:22120", "127.0.0.3:22120", "127.0.0.2:22120", "127.0.0.4:22120"})); + + // No change. + expectRedisResolve(); + resolve_timer_->callback_(); + EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1).WillOnce(Return(false)); + expectClusterSlotResponse(twoSlotsMastersWithReplica()); + expectHealthyHosts(std::list( + {"127.0.0.1:22120", "127.0.0.3:22120", "127.0.0.2:22120", "127.0.0.4:22120"})); + + // Remove 2nd shard. + expectRedisResolve(); + EXPECT_CALL(membership_updated_, ready()); + resolve_timer_->callback_(); + EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); + expectClusterSlotResponse(singleSlotMasterReplica("127.0.0.1", "127.0.0.2", 22120)); + expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); } void exerciseStubs() { @@ -461,28 +553,28 @@ std::vector generateRedisDnsParams() { std::string family_yaml(""); Network::DnsLookupFamily family(Network::DnsLookupFamily::Auto); std::list dns_response{"127.0.0.1", "127.0.0.2"}; - std::list resolved_host{"127.0.0.1:22120"}; + std::list resolved_host{"127.0.0.1:22120", "127.0.0.2:22120"}; dns_config.push_back(std::make_tuple(family_yaml, family, dns_response, resolved_host)); } { std::string family_yaml(R"EOF(dns_lookup_family: V4_ONLY)EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::V4Only); std::list dns_response{"127.0.0.1", "127.0.0.2"}; - std::list resolved_host{"127.0.0.1:22120"}; + std::list resolved_host{"127.0.0.1:22120", "127.0.0.2:22120"}; dns_config.push_back(std::make_tuple(family_yaml, family, dns_response, resolved_host)); } { std::string family_yaml(R"EOF(dns_lookup_family: V6_ONLY)EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::V6Only); - std::list dns_response{"::1", "::2"}; - std::list resolved_host{"[::1]:22120"}; + std::list dns_response{"::1", "2001:0db8:85a3:0000:0000:8a2e:0370:7334"}; + std::list resolved_host{"[::1]:22120", "[2001:db8:85a3::8a2e:370:7334]:22120"}; dns_config.push_back(std::make_tuple(family_yaml, family, dns_response, resolved_host)); } { std::string family_yaml(R"EOF(dns_lookup_family: AUTO)EOF"); Network::DnsLookupFamily family(Network::DnsLookupFamily::Auto); - std::list dns_response{"::1", "::2"}; - std::list resolved_host{"[::1]:22120"}; + std::list dns_response{"::1", "2001:0db8:85a3:0000:0000:8a2e:0370:7334"}; + std::list resolved_host{"[::1]:22120", "[2001:db8:85a3::8a2e:370:7334]:22120"}; dns_config.push_back(std::make_tuple(family_yaml, family, dns_response, resolved_host)); } return dns_config; @@ -525,7 +617,7 @@ TEST_P(RedisDnsParamTest, ImmediateResolveDns) { cb(TestUtility::makeDnsResponse(address_pair)); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); expectClusterSlotResponse( - singleSlotMasterSlave(address_pair.front(), address_pair.back(), 22120)); + singleSlotMasterReplica(address_pair.front(), address_pair.back(), 22120)); return nullptr; })); @@ -611,14 +703,14 @@ TEST_F(RedisClusterTest, RedisResolveFailure) { EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); - expectClusterSlotResponse(singleSlotMasterSlave("127.0.0.1", "127.0.0.2", 22120)); - expectHealthyHosts(std::list({"127.0.0.1:22120"})); + expectClusterSlotResponse(singleSlotMasterReplica("127.0.0.1", "127.0.0.2", 22120)); + expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); // Expect no change if resolve failed. expectRedisResolve(); resolve_timer_->invokeCallback(); expectClusterSlotFailure(); - expectHealthyHosts(std::list({"127.0.0.1:22120"})); + expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); EXPECT_EQ(3U, cluster_->info()->stats().update_attempt_.value()); EXPECT_EQ(2U, cluster_->info()->stats().update_failure_.value()); } @@ -679,7 +771,9 @@ TEST_F(RedisClusterTest, RedisErrorResponse) { EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); - expectClusterSlotResponse(singleSlotMasterSlave("127.0.0.1", "127.0.0.2", 22120)); + std::bitset<10> single_slot_master(0x7ff); + std::bitset<3> no_replica(0); + expectClusterSlotResponse(createResponse(single_slot_master, no_replica)); expectHealthyHosts(std::list({"127.0.0.1:22120"})); // Expect no change if resolve failed. @@ -693,7 +787,7 @@ TEST_F(RedisClusterTest, RedisErrorResponse) { if (flags.all()) { EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1).WillOnce(Return(false)); } - expectClusterSlotResponse(createResponse(flags)); + expectClusterSlotResponse(createResponse(flags, no_replica)); expectHealthyHosts(std::list({"127.0.0.1:22120"})); EXPECT_EQ(++update_attempt, cluster_->info()->stats().update_attempt_.value()); if (!flags.all()) { @@ -702,6 +796,43 @@ TEST_F(RedisClusterTest, RedisErrorResponse) { } } +TEST_F(RedisClusterTest, RedisReplicaErrorResponse) { + setupFromV2Yaml(BasicConfig); + const std::list resolved_addresses{"127.0.0.1", "127.0.0.2"}; + expectResolveDiscovery(Network::DnsLookupFamily::V4Only, "foo.bar.com", resolved_addresses); + expectRedisResolve(true); + + cluster_->initialize([&]() -> void { initialized_.ready(); }); + + EXPECT_CALL(membership_updated_, ready()); + EXPECT_CALL(initialized_, ready()); + EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); + std::bitset<10> single_slot_master(0x7ff); + std::bitset<3> no_replica(0); + expectClusterSlotResponse(createResponse(single_slot_master, no_replica)); + expectHealthyHosts(std::list({"127.0.0.1:22120"})); + + // Expect no change if resolve failed. + uint64_t update_attempt = 1; + uint64_t update_failure = 0; + // Test every combination the replica error response. + for (uint64_t i = 1; i < (1 << 3); i++) { + std::bitset<3> replica_flags(i); + expectRedisResolve(); + resolve_timer_->callback_(); + if (replica_flags.all()) { + EXPECT_CALL(membership_updated_, ready()); + EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1).WillOnce(Return(false)); + } + expectHealthyHosts(std::list({"127.0.0.1:22120"})); + expectClusterSlotResponse(createResponse(single_slot_master, replica_flags)); + EXPECT_EQ(++update_attempt, cluster_->info()->stats().update_attempt_.value()); + if (!(replica_flags.all() || replica_flags.none())) { + EXPECT_EQ(++update_failure, cluster_->info()->stats().update_failure_.value()); + } + } +} + TEST_F(RedisClusterTest, DnsDiscoveryResolverBasic) { setupFromV2Yaml(BasicConfig); testDnsResolve("foo.bar.com", 22120); @@ -756,6 +887,52 @@ TEST_F(RedisClusterTest, MultipleDnsDiscovery) { EXPECT_CALL(pool_request_, cancel()); } +TEST_F(RedisClusterTest, HostRemovalAfterHcFail) { + setupFromV2Yaml(BasicConfig); + auto health_checker = std::make_shared(); + EXPECT_CALL(*health_checker, start()); + EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)).Times(2); + cluster_->setHealthChecker(health_checker); + + const std::list resolved_addresses{"127.0.0.1", "127.0.0.2"}; + expectResolveDiscovery(Network::DnsLookupFamily::V4Only, "foo.bar.com", resolved_addresses); + expectRedisResolve(true); + + EXPECT_CALL(membership_updated_, ready()); + EXPECT_CALL(initialized_, ready()); + cluster_->initialize([&]() -> void { initialized_.ready(); }); + + EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); + expectClusterSlotResponse(singleSlotMasterReplica("127.0.0.1", "127.0.0.2", 22120)); + + // Verify that both hosts are initially marked with FAILED_ACTIVE_HC, then + // clear the flag to simulate that these hosts have been successfully health + // checked. + { + EXPECT_CALL(membership_updated_, ready()); + const auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts(); + EXPECT_EQ(2UL, hosts.size()); + + for (size_t i = 0; i < 2; ++i) { + EXPECT_TRUE(hosts[i]->healthFlagGet(Upstream::Host::HealthFlag::FAILED_ACTIVE_HC)); + hosts[i]->healthFlagClear(Upstream::Host::HealthFlag::FAILED_ACTIVE_HC); + hosts[i]->healthFlagClear(Upstream::Host::HealthFlag::PENDING_ACTIVE_HC); + health_checker->runCallbacks(hosts[i], Upstream::HealthTransition::Changed); + } + expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); + } + + // Failed HC + EXPECT_CALL(membership_updated_, ready()); + EXPECT_CALL(*cluster_callback_, onHostHealthUpdate()); + const auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts(); + hosts[1]->healthFlagSet(Upstream::Host::HealthFlag::FAILED_ACTIVE_HC); + health_checker->runCallbacks(hosts[1], Upstream::HealthTransition::Changed); + + EXPECT_THAT(2U, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); + EXPECT_THAT(1U, cluster_->prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); +} + } // namespace Redis } // namespace Clusters } // namespace Extensions diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index d9be95058d6b..e9d28235610d 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -1,7 +1,6 @@ #include #include "common/buffer/buffer_impl.h" -#include "common/common/assert.h" #include "common/network/utility.h" #include "common/upstream/upstream_impl.h" @@ -10,9 +9,7 @@ #include "test/extensions/filters/network/common/redis/mocks.h" #include "test/extensions/filters/network/common/redis/test_utils.h" #include "test/mocks/network/mocks.h" -#include "test/mocks/thread_local/mocks.h" #include "test/mocks/upstream/mocks.h" -#include "test/test_common/printers.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -158,6 +155,7 @@ class ConfigBufferSizeGTSingleRequest : public Config { return std::chrono::milliseconds(1); } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + ReadPolicy readPolicy() const override { return ReadPolicy::Master; } }; TEST_F(RedisClientImplTest, BatchWithTimerFiring) { @@ -469,6 +467,7 @@ class ConfigOutlierDisabled : public Config { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return std::chrono::milliseconds(0); } + ReadPolicy readPolicy() const override { return ReadPolicy::Master; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } }; diff --git a/test/extensions/filters/network/common/redis/test_utils.h b/test/extensions/filters/network/common/redis/test_utils.h index e720394b83f2..bc26dfbf0210 100644 --- a/test/extensions/filters/network/common/redis/test_utils.h +++ b/test/extensions/filters/network/common/redis/test_utils.h @@ -8,6 +8,8 @@ #include "common/protobuf/utility.h" +#include "external/envoy_api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.pb.h" + namespace Envoy { namespace Extensions { namespace NetworkFilters { @@ -16,13 +18,18 @@ namespace Redis { namespace Client { inline envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings -createConnPoolSettings(int64_t millis = 20, bool hashtagging = true, - bool redirection_support = true, uint32_t max_unknown_conns = 100) { +createConnPoolSettings( + int64_t millis = 20, bool hashtagging = true, bool redirection_support = true, + uint32_t max_unknown_conns = 100, + envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings::ReadPolicy + read_policy = envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_MASTER) { envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings setting{}; setting.mutable_op_timeout()->CopyFrom(Protobuf::util::TimeUtil::MillisecondsToDuration(millis)); setting.set_enable_hashtagging(hashtagging); setting.set_enable_redirection(redirection_support); setting.mutable_max_upstream_unknown_connections()->set_value(max_unknown_conns); + setting.set_read_policy(read_policy); return setting; } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index ee16cfaa5015..22161d0e00fb 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -9,13 +9,8 @@ #include "test/extensions/clusters/redis/mocks.h" #include "test/extensions/filters/network/common/redis/mocks.h" #include "test/extensions/filters/network/common/redis/test_utils.h" -#include "test/extensions/filters/network/redis_proxy/mocks.h" #include "test/mocks/api/mocks.h" -#include "test/mocks/network/mocks.h" #include "test/mocks/thread_local/mocks.h" -#include "test/mocks/upstream/mocks.h" -#include "test/test_common/global.h" -#include "test/test_common/printers.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -71,10 +66,11 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client max_upstream_unknown_connections_reached_.value_++; })); - std::unique_ptr conn_pool_impl = std::make_unique( - cluster_name_, cm_, *this, tls_, - Common::Redis::Client::createConnPoolSettings(20, hashtagging, true, max_unknown_conns), - api_, std::move(store)); + std::unique_ptr conn_pool_impl = + std::make_unique(cluster_name_, cm_, *this, tls_, + Common::Redis::Client::createConnPoolSettings( + 20, hashtagging, true, max_unknown_conns, read_policy_), + api_, std::move(store)); // Set the authentication password for this connection pool. conn_pool_impl->tls_->getTyped().auth_password_ = auth_password_; conn_pool_ = std::move(conn_pool_impl); @@ -163,6 +159,43 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client return Common::Redis::Client::ClientPtr{create_(host)}; } + void testReadPolicy( + envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings::ReadPolicy + read_policy) { + InSequence s; + + read_policy_ = read_policy; + setup(); + + Common::Redis::RespValue value; + Common::Redis::Client::MockPoolRequest auth_request, active_request, readonly_request; + Common::Redis::Client::MockPoolCallbacks callbacks; + Common::Redis::Client::MockClient* client = new NiceMock(); + + EXPECT_CALL(cm_.thread_local_cluster_.lb_, chooseHost(_)) + .WillOnce( + Invoke([&](Upstream::LoadBalancerContext* context) -> Upstream::HostConstSharedPtr { + EXPECT_EQ(context->computeHashKey().value(), MurmurHash::murmurHash2_64("hash_key")); + EXPECT_EQ(context->metadataMatchCriteria(), nullptr); + EXPECT_EQ(context->downstreamConnection(), nullptr); + return cm_.thread_local_cluster_.lb_.host_; + })); + EXPECT_CALL(*this, create_(_)).WillOnce(Return(client)); + EXPECT_CALL( + *client, + makeRequest(Eq(NetworkFilters::Common::Redis::Utility::ReadOnlyRequest::instance()), _)) + .WillOnce(Return(&readonly_request)); + EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, address()) + .WillRepeatedly(Return(test_address_)); + EXPECT_CALL(*client, makeRequest(Ref(value), Ref(callbacks))).WillOnce(Return(&active_request)); + Common::Redis::Client::PoolRequest* request = + conn_pool_->makeRequest("hash_key", value, callbacks); + EXPECT_EQ(&active_request, request); + + EXPECT_CALL(*client, close()); + tls_.shutdownThread(); + } + MOCK_METHOD1(create_, Common::Redis::Client::Client*(Upstream::HostConstSharedPtr host)); const std::string cluster_name_{"fake_cluster"}; @@ -174,6 +207,9 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client Network::Address::InstanceConstSharedPtr test_address_; std::string auth_password_; NiceMock api_; + envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings::ReadPolicy + read_policy_ = envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_MASTER; NiceMock upstream_cx_drained_; NiceMock max_upstream_unknown_connections_reached_; }; @@ -241,6 +277,17 @@ TEST_F(RedisConnPoolImplTest, BasicWithAuthPassword) { tls_.shutdownThread(); }; +TEST_F(RedisConnPoolImplTest, BasicWithReadPolicy) { + testReadPolicy(envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_MASTER); + testReadPolicy(envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_REPLICA); + testReadPolicy(envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_REPLICA); + testReadPolicy( + envoy::config::filter::network::redis_proxy::v2::RedisProxy_ConnPoolSettings_ReadPolicy_ANY); +}; + TEST_F(RedisConnPoolImplTest, Hashtagging) { InSequence s; From b9fe0b93371094827827b4932aaad09f89b14928 Mon Sep 17 00:00:00 2001 From: Nicolas Meessen Date: Fri, 16 Aug 2019 10:02:18 +1000 Subject: [PATCH 402/542] docs: 0 disables route timeout (#7931) Calling out that it's possible to use the value 0 to disable route timeout (as it is for idle timeout). Signed-off-by: nmeessen --- api/envoy/api/v2/route/route.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/envoy/api/v2/route/route.proto b/api/envoy/api/v2/route/route.proto index e14e56583853..93d021ddb216 100644 --- a/api/envoy/api/v2/route/route.proto +++ b/api/envoy/api/v2/route/route.proto @@ -562,7 +562,8 @@ message RouteAction { // Specifies the upstream timeout for the route. If not specified, the default is 15s. This // spans between the point at which the entire downstream request (i.e. end-of-stream) has been - // processed and when the upstream response has been completely processed. + // processed and when the upstream response has been completely processed. A value of 0 will + // disable the route's timeout. // // .. note:: // From 5a7bab734f8758fedbd9f080dc7553edb2ae4421 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 15 Aug 2019 17:05:02 -0700 Subject: [PATCH 403/542] redis: fix merge race (#7943) Signed-off-by: Lizan Zhou --- test/extensions/clusters/redis/redis_cluster_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index d3af8ce09dcc..7d6a4967cc8a 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -473,7 +473,7 @@ class RedisClusterTest : public testing::Test, // No change. expectRedisResolve(); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1).WillOnce(Return(false)); expectClusterSlotResponse(twoSlotsMastersWithReplica()); expectHealthyHosts(std::list( @@ -482,7 +482,7 @@ class RedisClusterTest : public testing::Test, // Remove 2nd shard. expectRedisResolve(); EXPECT_CALL(membership_updated_, ready()); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); expectClusterSlotResponse(singleSlotMasterReplica("127.0.0.1", "127.0.0.2", 22120)); expectHealthyHosts(std::list({"127.0.0.1:22120", "127.0.0.2:22120"})); @@ -819,7 +819,7 @@ TEST_F(RedisClusterTest, RedisReplicaErrorResponse) { for (uint64_t i = 1; i < (1 << 3); i++) { std::bitset<3> replica_flags(i); expectRedisResolve(); - resolve_timer_->callback_(); + resolve_timer_->invokeCallback(); if (replica_flags.all()) { EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1).WillOnce(Return(false)); From cb15cc3d2010aff77d6e022ddf6d723fa8becbc0 Mon Sep 17 00:00:00 2001 From: moderation Date: Fri, 16 Aug 2019 02:59:46 -0700 Subject: [PATCH 404/542] Dependency: Update rules_go, Go, Python, curl, bazel-toolchains, bazelisk (#7934) Description: Update dependencies: - rules_go 0.19.2 ([release](https://github.com/bazelbuild/rules_go/releases/tag/0.19.2)), [0.19.0 release notes](https://github.com/bazelbuild/rules_go/releases/tag/0.19.0). The big change is that `org_golang_google_grpc` is no longer declared in `go_rules_dependencies`. In order for the API to build, changes to `WORKSPACE` outlined [here](https://github.com/bazelbuild/rules_go/blob/0.19.0/go/workspace.rst#grpc-dependencies) are required. This pollutes `WORKSPACE` with some dependencies. Open on ways to improve this. - Go 1.12.8 (to match rules_go) - various Python updates - curl 7.65.3 ([changelog](https://curl.haxx.se/changes.html#7_65_3)) - bazel-toolchains switches to 0.28.5 release and address @lizan TODO ([changes](https://github.com/bazelbuild/bazel-toolchains/compare/5a8611ee011d0d68498b16bf42a9c69d139bc708...0.28.5)) - bazelisk 1.0 ([release notes](https://github.com/bazelbuild/bazelisk/releases/tag/v1.0)) Risk Level: Low Testing: `./docs/build.sh`, `bazel build @envoy_api//envoy/...`, `bazel test //test/...` Docs Changes: None required Release Notes: None required Signed-off-by: Michael Payne --- bazel/dependency_imports.bzl | 26 +++++++++++++++++++- bazel/repository_locations.bzl | 21 ++++++---------- ci/build_container/build_container_common.sh | 4 +-- ci/verify_examples.sh | 4 +-- docs/requirements.txt | 26 ++++++++++---------- examples/grpc-bridge/client/requirements.txt | 4 +-- tools/deprecate_features/requirements.txt | 4 +-- tools/deprecate_version/requirements.txt | 4 +-- tools/requirements.txt | 2 +- 9 files changed, 57 insertions(+), 38 deletions(-) diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index 3915eb1b553f..c7eb3e1cb702 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -1,12 +1,36 @@ load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") load("@envoy//bazel/toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config") +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") # go version for rules_go -GO_VERSION = "1.12.5" +GO_VERSION = "1.12.8" def envoy_dependency_imports(go_version = GO_VERSION): rules_foreign_cc_dependencies() go_rules_dependencies() go_register_toolchains(go_version) rbe_toolchains_config() + gazelle_dependencies() + + go_repository( + name = "org_golang_google_grpc", + build_file_proto_mode = "disable", + importpath = "google.golang.org/grpc", + sum = "h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=", + version = "v1.23.0", + ) + + go_repository( + name = "org_golang_x_net", + importpath = "golang.org/x/net", + sum = "h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=", + version = "v0.0.0-20190813141303-74dc4d7220e7", + ) + + go_repository( + name = "org_golang_x_text", + importpath = "golang.org/x/text", + sum = "h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=", + version = "v0.3.0", + ) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 133e57d5731b..0c0002c0c657 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -4,14 +4,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.18.1/bazel-gazelle-0.18.1.tar.gz"], ), bazel_toolchains = dict( - sha256 = "0710ec5a88201c4c3038ea458f7e9078cc3ad7ad61736ab287c115438eb91b1d", - strip_prefix = "bazel-toolchains-5a8611ee011d0d68498b16bf42a9c69d139bc708", - # 2019-08-01 - # Need: - # - https://github.com/bazelbuild/bazel-toolchains/pull/644 to select correct toolchain from same image - # - https://github.com/bazelbuild/bazel-toolchains/pull/650 to support no java config - # TODO(lizan): Update to release when new version is released. - urls = ["https://github.com/bazelbuild/bazel-toolchains/archive/5a8611ee011d0d68498b16bf42a9c69d139bc708.tar.gz"], + sha256 = "b72e7a911436b2900b05759a1fcd735070edbd4442f0a3506ef021fdcd6e15b3", + strip_prefix = "bazel-toolchains-0.28.5", + urls = ["https://github.com/bazelbuild/bazel-toolchains/archive/0.28.5.tar.gz"], ), boringssl = dict( # Use commits from branch "chromium-stable-with-bazel" @@ -219,8 +214,8 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/archive/64d6ac985360b624d8e95105701b64a3814794cd.tar.gz"], ), io_bazel_rules_go = dict( - sha256 = "a82a352bffae6bee4e95f68a8d80a70e87f42c4741e6a448bec11998fcc82329", - urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.18.5/rules_go-0.18.5.tar.gz"], + sha256 = "96b1f81de5acc7658e1f5a86d7dc9e1b89bc935d83799b711363a748652c471a", + urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.19.2/rules_go-0.19.2.tar.gz"], ), rules_foreign_cc = dict( sha256 = "c957e6663094a1478c43330c1bbfa71afeaf1ab86b7565233783301240c7a0ab", @@ -239,9 +234,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/cad0d03ff3474cf14389fc249e16847ab7b6895f.tar.gz"], ), com_github_curl = dict( - sha256 = "821aeb78421375f70e55381c9ad2474bf279fc454b791b7e95fc83562951c690", - strip_prefix = "curl-7.65.1", - urls = ["https://github.com/curl/curl/releases/download/curl-7_65_1/curl-7.65.1.tar.gz"], + sha256 = "4376ac72b95572fb6c4fbffefb97c7ea0dd083e1974c0e44cd7e49396f454839", + strip_prefix = "curl-7.65.3", + urls = ["https://github.com/curl/curl/releases/download/curl-7_65_3/curl-7.65.3.tar.gz"], ), com_googlesource_quiche = dict( # Static snapshot of https://quiche.googlesource.com/quiche/+archive/2a930469533c3b541443488a629fe25cd8ff53d0.tar.gz diff --git a/ci/build_container/build_container_common.sh b/ci/build_container/build_container_common.sh index a80685ee6ba5..97bb0c2268de 100755 --- a/ci/build_container/build_container_common.sh +++ b/ci/build_container/build_container_common.sh @@ -9,8 +9,8 @@ if [[ "$(uname -m)" == "x86_64" ]]; then && chmod +x /usr/local/bin/buildifier # bazelisk - VERSION=0.0.8 - SHA256=5fced4fec06bf24beb631837fa9497b6698f34041463d9188610dfa7b91f4f8d + VERSION=1.0 + SHA256=820f1432bb729cf1d51697a64ce57c0cff7ea4013acaf871b8c24b6388174d0d curl --location --output /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v${VERSION}/bazelisk-linux-amd64 \ && echo "$SHA256 /usr/local/bin/bazel" | sha256sum --check \ && chmod +x /usr/local/bin/bazel diff --git a/ci/verify_examples.sh b/ci/verify_examples.sh index d66e63ebe648..9d99be9ebd87 100755 --- a/ci/verify_examples.sh +++ b/ci/verify_examples.sh @@ -23,8 +23,8 @@ cd ../ # Test grpc bridge example # install go -curl -O https://storage.googleapis.com/golang/go1.7.1.linux-amd64.tar.gz -tar -xf go1.7.1.linux-amd64.tar.gz +curl -O https://storage.googleapis.com/golang/go1.12.8.linux-amd64.tar.gz +tar -xf go1.12.8.linux-amd64.tar.gz sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin export GOPATH=$HOME/go diff --git a/docs/requirements.txt b/docs/requirements.txt index 79ee5cd0ad46..6ec6e3c52108 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,17 +1,17 @@ -GitPython==2.0.8 +GitPython==3.0.0 Jinja2==2.10.1 MarkupSafe==1.1.1 -Pygments==2.2.0 -alabaster==0.7.10 -babel==2.4.0 -docutils==0.14 +Pygments==2.4.2 +alabaster==0.7.12 +babel==2.7.0 +docutils==0.15.2 gitdb==0.6.4 -imagesize==0.7.1 -pytz==2017.2 -requests>=2.20.0 -six==1.10.0 +imagesize==1.1.0 +pytz==2019.2 +requests>=2.22.0 +six==1.12.0 smmap==0.9.0 -snowballstemmer==1.2.1 -sphinx==1.8.1 -sphinxcontrib-httpdomain==1.6.1 -sphinx_rtd_theme==0.4.2 +snowballstemmer==1.9.0 +sphinx==2.1.2 +sphinxcontrib-httpdomain==1.7.0 +sphinx_rtd_theme==0.4.3 diff --git a/examples/grpc-bridge/client/requirements.txt b/examples/grpc-bridge/client/requirements.txt index a86d6229e8b9..8ef4a95ccde0 100644 --- a/examples/grpc-bridge/client/requirements.txt +++ b/examples/grpc-bridge/client/requirements.txt @@ -1,3 +1,3 @@ -requests>=2.20.0 +requests>=2.22.0 grpcio -protobuf==3.7.1 +protobuf==3.8.0 diff --git a/tools/deprecate_features/requirements.txt b/tools/deprecate_features/requirements.txt index 78d6a21318da..dc2a917a768e 100644 --- a/tools/deprecate_features/requirements.txt +++ b/tools/deprecate_features/requirements.txt @@ -1,2 +1,2 @@ -GitPython==2.1.9 -PyGithub==1.38 +GitPython==3.0.0 +PyGithub==1.43.8 diff --git a/tools/deprecate_version/requirements.txt b/tools/deprecate_version/requirements.txt index 78d6a21318da..dc2a917a768e 100644 --- a/tools/deprecate_version/requirements.txt +++ b/tools/deprecate_version/requirements.txt @@ -1,2 +1,2 @@ -GitPython==2.1.9 -PyGithub==1.38 +GitPython==3.0.0 +PyGithub==1.43.8 diff --git a/tools/requirements.txt b/tools/requirements.txt index 048ee9d5a6e5..4ab3842b87d9 100644 --- a/tools/requirements.txt +++ b/tools/requirements.txt @@ -1,2 +1,2 @@ -flake8==3.7.7 +flake8==3.7.8 yapf==0.28.0 From 1c27f545c1aeeac938cf3f99a25d2e21cd2b524e Mon Sep 17 00:00:00 2001 From: Andrew Jenkins Date: Fri, 16 Aug 2019 16:12:28 -0600 Subject: [PATCH 405/542] network: Conns with the same local/remote address are local (#7840) Signed-off-by: Andrew Jenkins --- source/common/network/utility.cc | 44 +++++++---------------------- test/common/network/utility_test.cc | 20 ++++++++++++- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index aeef34dd5c0b..638547cd9d14 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -244,45 +244,21 @@ Address::InstanceConstSharedPtr Utility::getLocalAddress(const Address::IpVersio } bool Utility::isLocalConnection(const Network::ConnectionSocket& socket) { + // These are local: + // - Pipes + // - Sockets to a loopback address + // - Sockets where the local and remote address (ignoring port) are the same const auto& remote_address = socket.remoteAddress(); - // Before calling getifaddrs, verify the obvious checks. - // Note that there are corner cases, where remote and local address will be the same - // while the client is not actually local. Example could be an iptables intercepted - // connection. However, this is a rare exception and such assumption results in big - // performance optimization. if (remote_address->type() == Envoy::Network::Address::Type::Pipe || - remote_address == socket.localAddress() || isLoopbackAddress(*remote_address)) { + isLoopbackAddress(*remote_address)) { return true; } - - struct ifaddrs* ifaddr; - const int rc = getifaddrs(&ifaddr); - Cleanup ifaddr_cleanup([ifaddr] { - if (ifaddr) { - freeifaddrs(ifaddr); - } - }); - RELEASE_ASSERT(rc == 0, ""); - - const auto af_look_up = - (remote_address->ip()->version() == Address::IpVersion::v4) ? AF_INET : AF_INET6; - - for (struct ifaddrs* ifa = ifaddr; ifa != nullptr; ifa = ifa->ifa_next) { - if (ifa->ifa_addr == nullptr) { - continue; - } - - if (ifa->ifa_addr->sa_family == af_look_up) { - const auto* addr = reinterpret_cast(ifa->ifa_addr); - const auto local_address = Address::addressFromSockAddr( - *addr, (af_look_up == AF_INET) ? sizeof(sockaddr_in) : sizeof(sockaddr_in6)); - - if (remote_address == local_address) { - return true; - } - } + const auto local_ip = socket.localAddress()->ip(); + const auto remote_ip = remote_address->ip(); + if (remote_ip != nullptr && local_ip != nullptr && + remote_ip->addressAsString() == local_ip->addressAsString()) { + return true; } - return false; } diff --git a/test/common/network/utility_test.cc b/test/common/network/utility_test.cc index 335271422d67..3d9206548146 100644 --- a/test/common/network/utility_test.cc +++ b/test/common/network/utility_test.cc @@ -175,7 +175,7 @@ TEST(NetworkUtility, LocalConnection) { testing::NiceMock socket; - EXPECT_CALL(socket, remoteAddress()).WillRepeatedly(testing::ReturnRef(local_addr)); + EXPECT_CALL(socket, localAddress()).WillRepeatedly(testing::ReturnRef(local_addr)); EXPECT_CALL(socket, remoteAddress()).WillRepeatedly(testing::ReturnRef(remote_addr)); local_addr.reset(new Network::Address::Ipv4Instance("127.0.0.1")); @@ -197,6 +197,14 @@ TEST(NetworkUtility, LocalConnection) { remote_addr.reset(new Network::Address::Ipv4Instance("8.8.8.8")); EXPECT_FALSE(Utility::isLocalConnection(socket)); + local_addr.reset(new Network::Address::Ipv4Instance("4.4.4.4")); + remote_addr.reset(new Network::Address::Ipv4Instance("4.4.4.4")); + EXPECT_TRUE(Utility::isLocalConnection(socket)); + + local_addr.reset(new Network::Address::Ipv4Instance("4.4.4.4", 1234)); + remote_addr.reset(new Network::Address::Ipv4Instance("4.4.4.4", 4321)); + EXPECT_TRUE(Utility::isLocalConnection(socket)); + local_addr.reset(new Network::Address::Ipv6Instance("::1")); remote_addr.reset(new Network::Address::Ipv6Instance("::1")); EXPECT_TRUE(Utility::isLocalConnection(socket)); @@ -205,6 +213,16 @@ TEST(NetworkUtility, LocalConnection) { remote_addr.reset(new Network::Address::Ipv6Instance("::1")); EXPECT_TRUE(Utility::isLocalConnection(socket)); + remote_addr.reset(new Network::Address::Ipv6Instance("::3")); + EXPECT_FALSE(Utility::isLocalConnection(socket)); + + remote_addr.reset(new Network::Address::Ipv6Instance("::2")); + EXPECT_TRUE(Utility::isLocalConnection(socket)); + + remote_addr.reset(new Network::Address::Ipv6Instance("::2", 4321)); + local_addr.reset(new Network::Address::Ipv6Instance("::2", 1234)); + EXPECT_TRUE(Utility::isLocalConnection(socket)); + remote_addr.reset(new Network::Address::Ipv6Instance("fd00::")); EXPECT_FALSE(Utility::isLocalConnection(socket)); } From 817b2e36689cbc9e32892fc53f253f8e6361d5dd Mon Sep 17 00:00:00 2001 From: Guangming Wang Date: Sat, 17 Aug 2019 06:13:24 +0800 Subject: [PATCH 406/542] cleanup: fix words in assertion message. (#7950) Signed-off-by: Guangming Wang --- test/integration/fake_upstream.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index fd970b6de8f1..f55806361987 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -482,7 +482,7 @@ FakeUpstream::waitForHttpConnection(Event::Dispatcher& client_dispatcher, std::vector>& upstreams, FakeHttpConnectionPtr& connection, milliseconds timeout) { if (upstreams.empty()) { - return AssertionFailure() << "No upstreams confgured."; + return AssertionFailure() << "No upstreams configured."; } Event::TestTimeSystem& time_system = upstreams[0]->timeSystem(); auto end_time = time_system.monotonicTime() + timeout; From 0418a855d9f9e37ec70b4c6d1942688fc8bb5751 Mon Sep 17 00:00:00 2001 From: htuch Date: Mon, 19 Aug 2019 14:24:39 -0400 Subject: [PATCH 407/542] config: distinct CLI options for strict/permissive checking of static/dynamic config. (#7857) As per #6651, this PR plumbs in CLI options to allow independent control over static/dynamic unknown field validation. The defaults are the same for static as today (strict) and for dynamic we are by default permissive. This permits easy rollout of new API minor versions, including those related to security fixes. Fixes a regression that occurred in #7200 where strict/permissive checking CLI options were inverted. As per #6818, added stats/warning for any unknown fields encountered. Risk level: Low (strictly more permissive by default) Testing: additional unit and integration tests added, exercising both permissive/strict checking over various parts of the API (bootstrap, listeners, clusters, xDS, network filters, etc). Fixes #6651 Fixed #6818 Signed-off-by: Harvey Tuch --- api/envoy/admin/v2alpha/server_info.proto | 7 +- docs/root/configuration/statistics.rst | 4 + docs/root/intro/deprecated.rst | 1 + docs/root/intro/version_history.rst | 6 + docs/root/operations/admin.rst | 2 +- docs/root/operations/cli.rst | 21 +- docs/root/start/sandboxes/front_proxy.rst | 2 +- include/envoy/protobuf/message_validator.h | 15 ++ include/envoy/server/instance.h | 4 +- include/envoy/server/options.h | 9 +- .../config/filesystem_subscription_impl.cc | 7 +- source/common/protobuf/BUILD | 2 + .../common/protobuf/message_validator_impl.cc | 24 +++ .../common/protobuf/message_validator_impl.h | 59 ++++++ .../common/upstream/cluster_manager_impl.cc | 16 +- source/common/upstream/cluster_manager_impl.h | 8 +- .../config_validation/cluster_manager.cc | 6 +- .../config_validation/cluster_manager.h | 6 +- source/server/config_validation/server.cc | 15 +- source/server/config_validation/server.h | 9 +- source/server/configuration_impl.cc | 4 +- source/server/listener_manager_impl.cc | 20 +- source/server/listener_manager_impl.h | 25 +-- source/server/options_impl.cc | 20 +- source/server/options_impl.h | 12 +- source/server/server.cc | 27 ++- source/server/server.h | 8 +- .../filesystem_subscription_impl_test.cc | 2 +- test/common/protobuf/BUILD | 10 + .../protobuf/message_validator_impl_test.cc | 54 ++++++ .../upstream/cluster_manager_impl_test.cc | 23 ++- test/config/integration/BUILD | 4 + test/config/integration/server.yaml | 17 +- .../integration/server_unix_listener.yaml | 6 +- .../server_xds.cds.with_unknown_field.yaml | 15 ++ .../server_xds.eds.with_unknown_field.yaml | 12 ++ .../server_xds.lds.with_unknown_field.yaml | 20 ++ .../server_xds.rds.with_unknown_field.yaml | 11 ++ test/config_test/config_test.cc | 6 +- test/integration/BUILD | 10 + .../dynamic_validation_integration_test.cc | 181 ++++++++++++++++++ test/integration/integration.cc | 45 +++-- test/integration/integration.h | 8 +- test/integration/server.cc | 30 ++- test/integration/server.h | 13 +- test/integration/xds_integration_test.cc | 6 +- test/mocks/protobuf/mocks.cc | 9 + test/mocks/protobuf/mocks.h | 12 ++ test/mocks/server/BUILD | 1 + test/mocks/server/mocks.cc | 9 +- test/mocks/server/mocks.h | 10 +- test/server/BUILD | 6 + .../config_validation/cluster_manager_test.cc | 4 +- test/server/configuration_impl_test.cc | 2 +- test/server/listener_manager_impl_test.cc | 58 +++--- test/server/options_impl_test.cc | 50 ++++- test/server/server_test.cc | 129 ++++++++++++- .../bootstrap_unknown_field.yaml | 1 + .../cluster_unknown_field.yaml | 5 + .../listener_unknown_field.yaml | 10 + .../network_filter_unknown_field.yaml | 15 ++ 61 files changed, 944 insertions(+), 189 deletions(-) create mode 100644 test/common/protobuf/message_validator_impl_test.cc create mode 100644 test/config/integration/server_xds.cds.with_unknown_field.yaml create mode 100644 test/config/integration/server_xds.eds.with_unknown_field.yaml create mode 100644 test/config/integration/server_xds.lds.with_unknown_field.yaml create mode 100644 test/config/integration/server_xds.rds.with_unknown_field.yaml create mode 100644 test/integration/dynamic_validation_integration_test.cc create mode 100644 test/server/test_data/static_validation/bootstrap_unknown_field.yaml create mode 100644 test/server/test_data/static_validation/cluster_unknown_field.yaml create mode 100644 test/server/test_data/static_validation/listener_unknown_field.yaml create mode 100644 test/server/test_data/static_validation/network_filter_unknown_field.yaml diff --git a/api/envoy/admin/v2alpha/server_info.proto b/api/envoy/admin/v2alpha/server_info.proto index 0a4506f1676b..7389af5b0860 100644 --- a/api/envoy/admin/v2alpha/server_info.proto +++ b/api/envoy/admin/v2alpha/server_info.proto @@ -53,8 +53,11 @@ message CommandLineOptions { // See :option:`--config-yaml` for details. string config_yaml = 4; - // See :option:`--allow-unknown-fields` for details. - bool allow_unknown_fields = 5; + // See :option:`--allow-unknown-static-fields` for details. + bool allow_unknown_static_fields = 5; + + // See :option:`--reject-unknown-dynamic-fields` for details. + bool reject_unknown_dynamic_fields = 26; // See :option:`--admin-address-path` for details. string admin_address_path = 6; diff --git a/docs/root/configuration/statistics.rst b/docs/root/configuration/statistics.rst index 1e79f39a0bf9..0042051e0acb 100644 --- a/docs/root/configuration/statistics.rst +++ b/docs/root/configuration/statistics.rst @@ -3,6 +3,8 @@ Statistics ========== +.. _server_statistics: + Server ------ @@ -25,6 +27,8 @@ Server related statistics are rooted at *server.* with following statistics: hot_restart_epoch, Gauge, Current hot restart epoch initialization_time_ms, Histogram, Total time taken for Envoy initialization in milliseconds. This is the time from server start-up until the worker threads are ready to accept new connections debug_assertion_failures, Counter, Number of debug assertion failures detected in a release build if compiled with `--define log_debug_assert_in_release=enabled` or zero otherwise + static_unknown_fields, Counter, Number of messages in static configuration with unknown fields + dynamic_unknown_fields, Counter, Number of messages in dynamic configuration with unknown fields File system ----------- diff --git a/docs/root/intro/deprecated.rst b/docs/root/intro/deprecated.rst index c8399d9ab087..6ca0b034b759 100644 --- a/docs/root/intro/deprecated.rst +++ b/docs/root/intro/deprecated.rst @@ -13,6 +13,7 @@ Deprecated items below are listed in chronological order. Version 1.12.0 (pending) ======================== * The ORIGINAL_DST_LB :ref:`load balancing policy ` is deprecated, use CLUSTER_PROVIDED policy instead when configuring an :ref:`original destination cluster `. +* The :option:`--allow-unknown-fields` command-line option, use :option:`--allow-unknown-static-fields` instead. Version 1.11.0 (July 11, 2019) ============================== diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 72e4c3b12f0e..ca60b5387630 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -10,6 +10,12 @@ Version history * config: enforcing that terminal filters (e.g. HttpConnectionManager for L4, router for L7) be the last in their respective filter chains. * buffer filter: the buffer filter populates content-length header if not present, behavior can be disabled using the runtime feature `envoy.reloadable_features.buffer_filter_populate_content_length`. * config: added access log :ref:`extension filter`. +* config: added support for :option:`--reject-unknown-dynamic-fields`, providing independent control + over whether unknown fields are rejected in static and dynamic configuration. By default, unknown + fields in static configuration are rejected and are allowed in dynamic configuration. Warnings + are logged for the first use of any unknown field and these occurrences are counted in the + :ref:`server.static_unknown_fields ` and :ref:`server.dynamic_unknown_fields + ` statistics. * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * config: added stat :ref:`init_fetch_timeout `. diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index f0d1f1d73bc2..c681deb55951 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -219,7 +219,7 @@ modify different aspects of the server: "concurrency": 8, "config_path": "config.yaml", "config_yaml": "", - "allow_unknown_fields": false, + "allow_unknown_static_fields": false, "admin_address_path": "", "local_address_ip_version": "v4", "log_level": "info", diff --git a/docs/root/operations/cli.rst b/docs/root/operations/cli.rst index 3f6494e64138..4ebc9bb4b6c7 100644 --- a/docs/root/operations/cli.rst +++ b/docs/root/operations/cli.rst @@ -228,10 +228,25 @@ following are the command line options that Envoy supports. .. option:: --allow-unknown-fields - *(optional)* This flag disables validation of protobuf configurations for unknown fields. By default, the + *(optional)* Deprecated alias for :option:`--allow-unknown-static-fields`. + +.. option:: --allow-unknown-static-fields + + *(optional)* This flag disables validation of protobuf configurations for unknown fields. By default, the validation is enabled. For most deployments, the default should be used which ensures configuration errors - are caught upfront and Envoy is configured as intended. However in cases where Envoy needs to accept configuration - produced by newer control planes, effectively ignoring new features it does not know about yet, this can be disabled. + are caught upfront and Envoy is configured as intended. Warnings are logged for the first use of + any unknown field and these occurrences are counted in the :ref:`server.static_unknown_fields + ` statistic. + +.. option:: --reject-unknown-dynamic-fields + + *(optional)* This flag disables validation of protobuf configuration for unknown fields in + dynamic configuration. By default, this flag is set false, disabling validation for fields beyond + bootstrap. This allows newer xDS configurations to be delivered to older Envoys. This can be set + true for strict dynamic checking when this behavior is not wanted but the default should be + desirable for most Envoy deployments. Warnings are logged for the first use of any unknown field + and these occurrences are counted in the :ref:`server.dynamic_unknown_fields ` + statistic. .. option:: --version diff --git a/docs/root/start/sandboxes/front_proxy.rst b/docs/root/start/sandboxes/front_proxy.rst index e14d938cadfd..ee607db133dc 100644 --- a/docs/root/start/sandboxes/front_proxy.rst +++ b/docs/root/start/sandboxes/front_proxy.rst @@ -204,7 +204,7 @@ statistics. For example inside ``frontenvoy`` we can get:: "concurrency": 4, "config_path": "/etc/front-envoy.yaml", "config_yaml": "", - "allow_unknown_fields": false, + "allow_unknown_static_fields": false, "admin_address_path": "", "local_address_ip_version": "v4", "log_level": "info", diff --git a/include/envoy/protobuf/message_validator.h b/include/envoy/protobuf/message_validator.h index 1459aee4b8be..8c2ac4bc8c4e 100644 --- a/include/envoy/protobuf/message_validator.h +++ b/include/envoy/protobuf/message_validator.h @@ -25,5 +25,20 @@ class ValidationVisitor { virtual void onUnknownField(absl::string_view description) PURE; }; +class ValidationContext { +public: + virtual ~ValidationContext() = default; + + /** + * @return ValidationVisitor& the validation visitor for static configuration. + */ + virtual ValidationVisitor& staticValidationVisitor() PURE; + + /** + * @return ValidationVisitor& the validation visitor for dynamic configuration. + */ + virtual ValidationVisitor& dynamicValidationVisitor() PURE; +}; + } // namespace ProtobufMessage } // namespace Envoy diff --git a/include/envoy/server/instance.h b/include/envoy/server/instance.h index dbb964eb3df0..d87e72f84482 100644 --- a/include/envoy/server/instance.h +++ b/include/envoy/server/instance.h @@ -220,10 +220,10 @@ class Instance { virtual std::chrono::milliseconds statsFlushInterval() const PURE; /** - * @return ProtobufMessage::ValidationVisitor& validation visitor for configuration + * @return ProtobufMessage::ValidationContext& validation context for configuration * messages. */ - virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; + virtual ProtobufMessage::ValidationContext& messageValidationContext() PURE; }; } // namespace Server diff --git a/include/envoy/server/options.h b/include/envoy/server/options.h index 8904925f28e9..8ecc14f0c555 100644 --- a/include/envoy/server/options.h +++ b/include/envoy/server/options.h @@ -86,9 +86,14 @@ class Options { virtual const envoy::config::bootstrap::v2::Bootstrap& configProto() const PURE; /** - * @return bool allow unknown fields in the configuration? + * @return bool allow unknown fields in the static configuration? */ - virtual bool allowUnknownFields() const PURE; + virtual bool allowUnknownStaticFields() const PURE; + + /** + * @return bool allow unknown fields in the dynamic configuration? + */ + virtual bool rejectUnknownDynamicFields() const PURE; /** * @return const std::string& the admin address output file. diff --git a/source/common/config/filesystem_subscription_impl.cc b/source/common/config/filesystem_subscription_impl.cc index f14e39533075..1bef8eafe8f6 100644 --- a/source/common/config/filesystem_subscription_impl.cc +++ b/source/common/config/filesystem_subscription_impl.cc @@ -53,10 +53,9 @@ void FilesystemSubscriptionImpl::refresh() { } else { ENVOY_LOG(warn, "Filesystem config update failure: {}", e.what()); stats_.update_failure_.inc(); - // ConnectionFailure is not a meaningful error code for file system but it has been chosen so - // that the behaviour is uniform across all subscription types. - callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, - &e); + // This could happen due to filesystem issues or a bad configuration (e.g. proto validation). + // Since the latter is more likely, for now we will treat it as rejection. + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } } } diff --git a/source/common/protobuf/BUILD b/source/common/protobuf/BUILD index e6fa19e96d0a..4feeec55c982 100644 --- a/source/common/protobuf/BUILD +++ b/source/common/protobuf/BUILD @@ -31,6 +31,8 @@ envoy_cc_library( external_deps = ["protobuf"], deps = [ "//include/envoy/protobuf:message_validator_interface", + "//include/envoy/stats:stats_interface", + "//source/common/common:hash_lib", "//source/common/common:logger_lib", "//source/common/common:macros", ], diff --git a/source/common/protobuf/message_validator_impl.cc b/source/common/protobuf/message_validator_impl.cc index ba89b89bdb7b..4e921a0f0215 100644 --- a/source/common/protobuf/message_validator_impl.cc +++ b/source/common/protobuf/message_validator_impl.cc @@ -2,6 +2,8 @@ #include "envoy/common/exception.h" +#include "common/common/assert.h" +#include "common/common/hash.h" #include "common/common/logger.h" #include "common/common/macros.h" @@ -10,6 +12,28 @@ namespace Envoy { namespace ProtobufMessage { +void WarningValidationVisitorImpl::setCounter(Stats::Counter& counter) { + ASSERT(counter_ == nullptr); + counter_ = &counter; + counter.add(prestats_count_); +} + +void WarningValidationVisitorImpl::onUnknownField(absl::string_view description) { + const uint64_t hash = HashUtil::xxHash64(description); + auto it = descriptions_.insert(hash); + // If we've seen this before, skip. + if (!it.second) { + return; + } + // It's a new field, log and bump stat. + ENVOY_LOG(warn, "Unknown field: {}", description); + if (counter_ == nullptr) { + ++prestats_count_; + } else { + counter_->inc(); + } +} + void StrictValidationVisitorImpl::onUnknownField(absl::string_view description) { throw EnvoyException(absl::StrCat("Protobuf message (", description, ") has unknown fields")); } diff --git a/source/common/protobuf/message_validator_impl.h b/source/common/protobuf/message_validator_impl.h index cf42cb4e26a8..2d5b3d41af0f 100644 --- a/source/common/protobuf/message_validator_impl.h +++ b/source/common/protobuf/message_validator_impl.h @@ -1,6 +1,9 @@ #pragma once #include "envoy/protobuf/message_validator.h" +#include "envoy/stats/stats.h" + +#include "absl/container/flat_hash_set.h" namespace Envoy { namespace ProtobufMessage { @@ -13,6 +16,24 @@ class NullValidationVisitorImpl : public ValidationVisitor { ValidationVisitor& getNullValidationVisitor(); +class WarningValidationVisitorImpl : public ValidationVisitor, + public Logger::Loggable { +public: + void setCounter(Stats::Counter& counter); + + // Envoy::ProtobufMessage::ValidationVisitor + void onUnknownField(absl::string_view description) override; + +private: + // Track hashes of descriptions we've seen, to avoid log spam. A hash is used here to avoid + // wasting memory with unused strings. + absl::flat_hash_set descriptions_; + // This can be late initialized via setCounter(), enabling the server bootstrap loading which + // occurs prior to the initialization of the stats subsystem. + Stats::Counter* counter_{}; + uint64_t prestats_count_{}; +}; + class StrictValidationVisitorImpl : public ValidationVisitor { public: // Envoy::ProtobufMessage::ValidationVisitor @@ -21,5 +42,43 @@ class StrictValidationVisitorImpl : public ValidationVisitor { ValidationVisitor& getStrictValidationVisitor(); +class ValidationContextImpl : public ValidationContext { +public: + ValidationContextImpl(ValidationVisitor& static_validation_visitor, + ValidationVisitor& dynamic_validation_visitor) + : static_validation_visitor_(static_validation_visitor), + dynamic_validation_visitor_(dynamic_validation_visitor) {} + + // Envoy::ProtobufMessage::ValidationContext + ValidationVisitor& staticValidationVisitor() override { return static_validation_visitor_; } + ValidationVisitor& dynamicValidationVisitor() override { return dynamic_validation_visitor_; } + +private: + ValidationVisitor& static_validation_visitor_; + ValidationVisitor& dynamic_validation_visitor_; +}; + +class ProdValidationContextImpl : public ValidationContextImpl { +public: + ProdValidationContextImpl(bool allow_unknown_static_fields, bool allow_unknown_dynamic_fields) + : ValidationContextImpl(allow_unknown_static_fields ? static_warning_validation_visitor_ + : getStrictValidationVisitor(), + allow_unknown_dynamic_fields + ? dynamic_warning_validation_visitor_ + : ProtobufMessage::getStrictValidationVisitor()) {} + + ProtobufMessage::WarningValidationVisitorImpl& static_warning_validation_visitor() { + return static_warning_validation_visitor_; + } + + ProtobufMessage::WarningValidationVisitorImpl& dynamic_warning_validation_visitor() { + return dynamic_warning_validation_visitor_; + } + +private: + ProtobufMessage::WarningValidationVisitorImpl static_warning_validation_visitor_; + ProtobufMessage::WarningValidationVisitorImpl dynamic_warning_validation_visitor_; +}; + } // namespace ProtobufMessage } // namespace Envoy diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 69dae7cef8bb..7c4ac14f1bee 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -183,7 +183,7 @@ ClusterManagerImpl::ClusterManagerImpl( Stats::Store& stats, ThreadLocal::Instance& tls, Runtime::Loader& runtime, Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, AccessLog::AccessLogManager& log_manager, Event::Dispatcher& main_thread_dispatcher, - Server::Admin& admin, ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api, + Server::Admin& admin, ProtobufMessage::ValidationContext& validation_context, Api::Api& api, Http::Context& http_context) : factory_(factory), runtime_(runtime), stats_(stats), tls_(tls.allocateSlot()), random_(random), bind_config_(bootstrap.cluster_manager().upstream_bind_config()), @@ -192,8 +192,9 @@ ClusterManagerImpl::ClusterManagerImpl( config_tracker_entry_( admin.getConfigTracker().add("clusters", [this] { return dumpClusterConfigs(); })), time_source_(main_thread_dispatcher.timeSource()), dispatcher_(main_thread_dispatcher), - http_context_(http_context), subscription_factory_(local_info, main_thread_dispatcher, *this, - random, validation_visitor, api) { + http_context_(http_context), + subscription_factory_(local_info, main_thread_dispatcher, *this, random, + validation_context.dynamicValidationVisitor(), api) { async_client_manager_ = std::make_unique(*this, tls, time_source_, api); const auto& cm_config = bootstrap.cluster_manager(); @@ -1231,7 +1232,7 @@ ClusterManagerPtr ProdClusterManagerFactory::clusterManagerFromProto( const envoy::config::bootstrap::v2::Bootstrap& bootstrap) { return ClusterManagerPtr{new ClusterManagerImpl( bootstrap, *this, stats_, tls_, runtime_, random_, local_info_, log_manager_, - main_thread_dispatcher_, admin_, validation_visitor_, api_, http_context_)}; + main_thread_dispatcher_, admin_, validation_context_, api_, http_context_)}; } Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( @@ -1261,13 +1262,16 @@ std::pair ProdClusterManagerFactor return ClusterFactoryImplBase::create( cluster, cm, stats_, tls_, dns_resolver_, ssl_context_manager_, runtime_, random_, main_thread_dispatcher_, log_manager_, local_info_, admin_, singleton_manager_, - outlier_event_logger, added_via_api, validation_visitor_, api_); + outlier_event_logger, added_via_api, + added_via_api ? validation_context_.dynamicValidationVisitor() + : validation_context_.staticValidationVisitor(), + api_); } CdsApiPtr ProdClusterManagerFactory::createCds(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm) { // TODO(htuch): Differentiate static vs. dynamic validation visitors. - return CdsApiImpl::create(cds_config, cm, stats_, validation_visitor_); + return CdsApiImpl::create(cds_config, cm, stats_, validation_context_.dynamicValidationVisitor()); } } // namespace Upstream diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 055cb9fd2cce..ae0ecb2dbd40 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -43,10 +43,10 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { Event::Dispatcher& main_thread_dispatcher, const LocalInfo::LocalInfo& local_info, Secret::SecretManager& secret_manager, - ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api, + ProtobufMessage::ValidationContext& validation_context, Api::Api& api, Http::Context& http_context, AccessLog::AccessLogManager& log_manager, Singleton::Manager& singleton_manager) - : main_thread_dispatcher_(main_thread_dispatcher), validation_visitor_(validation_visitor), + : main_thread_dispatcher_(main_thread_dispatcher), validation_context_(validation_context), api_(api), http_context_(http_context), admin_(admin), runtime_(runtime), stats_(stats), tls_(tls), random_(random), dns_resolver_(dns_resolver), ssl_context_manager_(ssl_context_manager), local_info_(local_info), @@ -74,7 +74,7 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { protected: Event::Dispatcher& main_thread_dispatcher_; - ProtobufMessage::ValidationVisitor& validation_visitor_; + ProtobufMessage::ValidationContext& validation_context_; Api::Api& api_; Http::Context& http_context_; Server::Admin& admin_; @@ -173,7 +173,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable( bootstrap, *this, stats_, tls_, runtime_, random_, local_info_, log_manager_, - main_thread_dispatcher_, admin_, validation_visitor_, api_, http_context_, time_system_); + main_thread_dispatcher_, admin_, validation_context_, api_, http_context_, time_system_); } CdsApiPtr @@ -26,10 +26,10 @@ ValidationClusterManager::ValidationClusterManager( Stats::Store& stats, ThreadLocal::Instance& tls, Runtime::Loader& runtime, Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, AccessLog::AccessLogManager& log_manager, Event::Dispatcher& main_thread_dispatcher, - Server::Admin& admin, ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api, + Server::Admin& admin, ProtobufMessage::ValidationContext& validation_context, Api::Api& api, Http::Context& http_context, Event::TimeSystem& time_system) : ClusterManagerImpl(bootstrap, factory, stats, tls, runtime, random, local_info, log_manager, - main_thread_dispatcher, admin, validation_visitor, api, http_context), + main_thread_dispatcher, admin, validation_context, api, http_context), async_client_(api, time_system) {} Http::ConnectionPool::Instance* diff --git a/source/server/config_validation/cluster_manager.h b/source/server/config_validation/cluster_manager.h index f4b4ce40bd9b..5a7f29955475 100644 --- a/source/server/config_validation/cluster_manager.h +++ b/source/server/config_validation/cluster_manager.h @@ -24,12 +24,12 @@ class ValidationClusterManagerFactory : public ProdClusterManagerFactory { ThreadLocal::Instance& tls, Runtime::RandomGenerator& random, Network::DnsResolverSharedPtr dns_resolver, Ssl::ContextManager& ssl_context_manager, Event::Dispatcher& main_thread_dispatcher, const LocalInfo::LocalInfo& local_info, - Secret::SecretManager& secret_manager, ProtobufMessage::ValidationVisitor& validation_visitor, + Secret::SecretManager& secret_manager, ProtobufMessage::ValidationContext& validation_context, Api::Api& api, Http::Context& http_context, AccessLog::AccessLogManager& log_manager, Singleton::Manager& singleton_manager, Event::TimeSystem& time_system) : ProdClusterManagerFactory(admin, runtime, stats, tls, random, dns_resolver, ssl_context_manager, main_thread_dispatcher, local_info, - secret_manager, validation_visitor, api, http_context, + secret_manager, validation_context, api, http_context, log_manager, singleton_manager), time_system_(time_system) {} @@ -56,7 +56,7 @@ class ValidationClusterManager : public ClusterManagerImpl { Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, AccessLog::AccessLogManager& log_manager, Event::Dispatcher& dispatcher, Server::Admin& admin, - ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api, + ProtobufMessage::ValidationContext& validation_context, Api::Api& api, Http::Context& http_context, Event::TimeSystem& time_system); Http::ConnectionPool::Instance* httpConnPoolForCluster(const std::string&, ResourcePriority, diff --git a/source/server/config_validation/server.cc b/source/server/config_validation/server.cc index 9e32616d7124..f0f63045d604 100644 --- a/source/server/config_validation/server.cc +++ b/source/server/config_validation/server.cc @@ -43,7 +43,9 @@ ValidationInstance::ValidationInstance(const Options& options, Event::TimeSystem ComponentFactory& component_factory, Thread::ThreadFactory& thread_factory, Filesystem::Instance& file_system) - : options_(options), stats_store_(store), + : options_(options), validation_context_(options_.allowUnknownStaticFields(), + !options.rejectUnknownDynamicFields()), + stats_store_(store), api_(new Api::ValidationImpl(thread_factory, store, time_system, file_system)), dispatcher_(api_->allocateDispatcher()), singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), @@ -75,7 +77,8 @@ void ValidationInstance::initialize(const Options& options, // be ready to serve, then the config has passed validation. // Handle configuration that needs to take place prior to the main configuration load. envoy::config::bootstrap::v2::Bootstrap bootstrap; - InstanceUtil::loadBootstrapConfig(bootstrap, options, messageValidationVisitor(), *api_); + InstanceUtil::loadBootstrapConfig(bootstrap, options, + messageValidationContext().staticValidationVisitor(), *api_); Config::Utility::createTagProducer(bootstrap); @@ -86,9 +89,9 @@ void ValidationInstance::initialize(const Options& options, options.serviceNodeName()); Configuration::InitialImpl initial_config(bootstrap); - overload_manager_ = std::make_unique(dispatcher(), stats(), threadLocal(), - bootstrap.overload_manager(), - messageValidationVisitor(), *api_); + overload_manager_ = std::make_unique( + dispatcher(), stats(), threadLocal(), bootstrap.overload_manager(), + messageValidationContext().staticValidationVisitor(), *api_); listener_manager_ = std::make_unique(*this, *this, *this, false); thread_local_.registerThread(*dispatcher_, true); runtime_loader_ = component_factory.createRuntime(*this, initial_config); @@ -97,7 +100,7 @@ void ValidationInstance::initialize(const Options& options, createContextManager(Ssl::ContextManagerFactory::name(), api_->timeSource()); cluster_manager_factory_ = std::make_unique( admin(), runtime(), stats(), threadLocal(), random(), dnsResolver(), sslContextManager(), - dispatcher(), localInfo(), *secret_manager_, messageValidationVisitor(), *api_, http_context_, + dispatcher(), localInfo(), *secret_manager_, messageValidationContext(), *api_, http_context_, accessLogManager(), singletonManager(), time_system_); config_.initialize(bootstrap, *this, *cluster_manager_factory_); http_context_.setTracer(config_.httpTracer()); diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index 2a710cd23675..bbd350b325f1 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -104,15 +104,15 @@ class ValidationInstance : Logger::Loggable, return config_.statsFlushInterval(); } - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return options_.allowUnknownFields() ? ProtobufMessage::getStrictValidationVisitor() - : ProtobufMessage::getNullValidationVisitor(); + ProtobufMessage::ValidationContext& messageValidationContext() override { + return validation_context_; } // Server::ListenerComponentFactory LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { return std::make_unique(lds_config, clusterManager(), initManager(), stats(), - listenerManager(), messageValidationVisitor()); + listenerManager(), + messageValidationContext().dynamicValidationVisitor()); } std::vector createNetworkFilterFactoryList( const Protobuf::RepeatedPtrField& filters, @@ -173,6 +173,7 @@ class ValidationInstance : Logger::Loggable, // - There may be active connections referencing it. std::unique_ptr secret_manager_; const Options& options_; + ProtobufMessage::ProdValidationContextImpl validation_context_; Stats::IsolatedStoreImpl& stats_store_; ThreadLocal::InstanceImpl thread_local_; Api::ApiPtr api_; diff --git a/source/server/configuration_impl.cc b/source/server/configuration_impl.cc index 20c87a41e039..da83272b5392 100644 --- a/source/server/configuration_impl.cc +++ b/source/server/configuration_impl.cc @@ -108,7 +108,7 @@ void MainImpl::initializeTracers(const envoy::config::trace::v2::Tracing& config // Now see if there is a factory that will accept the config. auto& factory = Config::Utility::getAndCheckFactory(type); ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( - configuration.http(), server.messageValidationVisitor(), factory); + configuration.http(), server.messageValidationContext().staticValidationVisitor(), factory); http_tracer_ = factory.createHttpTracer(*message, server); } @@ -120,7 +120,7 @@ void MainImpl::initializeStatsSinks(const envoy::config::bootstrap::v2::Bootstra // Generate factory and translate stats sink custom config auto& factory = Config::Utility::getAndCheckFactory(sink_object.name()); ProtobufTypes::MessagePtr message = Config::Utility::translateToFactoryConfig( - sink_object, server.messageValidationVisitor(), factory); + sink_object, server.messageValidationContext().staticValidationVisitor(), factory); stats_sinks_.emplace_back(factory.createStatsSink(*message, server)); } diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 6db28bdd0f36..0f2ec74ee62b 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -187,8 +187,9 @@ ProdListenerComponentFactory::createDrainManager(envoy::api::v2::Listener::Drain } ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, const std::string& version_info, - ListenerManagerImpl& parent, const std::string& name, bool modifiable, - bool workers_started, uint64_t hash) + ListenerManagerImpl& parent, const std::string& name, bool added_via_api, + bool workers_started, uint64_t hash, + ProtobufMessage::ValidationVisitor& validation_visitor) : parent_(parent), address_(Network::Address::resolveProtoAddress(config.address())), filter_chain_manager_(address_), socket_type_(Network::Utility::protobufAddressSocketType(config.address())), @@ -200,8 +201,8 @@ ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, const std::st PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, use_original_dst, false)), per_connection_buffer_limit_bytes_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, per_connection_buffer_limit_bytes, 1024 * 1024)), - listener_tag_(parent_.factory_.nextListenerTag()), name_(name), modifiable_(modifiable), - workers_started_(workers_started), hash_(hash), + listener_tag_(parent_.factory_.nextListenerTag()), name_(name), added_via_api_(added_via_api), + workers_started_(workers_started), hash_(hash), validation_visitor_(validation_visitor), dynamic_init_manager_(fmt::format("Listener {}", name)), init_watcher_(std::make_unique( "ListenerImpl", [this] { parent_.onListenerWarmed(*this); })), @@ -284,8 +285,7 @@ ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, const std::st parent_.server_.admin(), parent_.server_.sslContextManager(), *listener_scope_, parent_.server_.clusterManager(), parent_.server_.localInfo(), parent_.server_.dispatcher(), parent_.server_.random(), parent_.server_.stats(), parent_.server_.singletonManager(), - parent_.server_.threadLocal(), parent_.server_.messageValidationVisitor(), - parent_.server_.api()); + parent_.server_.threadLocal(), validation_visitor, parent_.server_.api()); factory_context.setInitManager(initManager()); ListenerFilterChainFactoryBuilder builder(*this, factory_context); filter_chain_manager_.addFilterChain(config.filter_chains(), builder); @@ -488,7 +488,7 @@ ListenerManagerStats ListenerManagerImpl::generateStats(Stats::Scope& scope) { } bool ListenerManagerImpl::addOrUpdateListener(const envoy::api::v2::Listener& config, - const std::string& version_info, bool modifiable) { + const std::string& version_info, bool added_via_api) { std::string name; if (!config.name().empty()) { name = config.name(); @@ -511,8 +511,10 @@ bool ListenerManagerImpl::addOrUpdateListener(const envoy::api::v2::Listener& co return false; } - ListenerImplPtr new_listener( - new ListenerImpl(config, version_info, *this, name, modifiable, workers_started_, hash)); + ListenerImplPtr new_listener(new ListenerImpl( + config, version_info, *this, name, added_via_api, workers_started_, hash, + added_via_api ? server_.messageValidationContext().dynamicValidationVisitor() + : server_.messageValidationContext().staticValidationVisitor())); ListenerImpl& new_listener_ref = *new_listener; // We mandate that a listener with the same name must have the same configured address. This diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 9177a0ec03af..1197ac82a9be 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -60,9 +60,9 @@ class ProdListenerComponentFactory : public ListenerComponentFactory, // Server::ListenerComponentFactory LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { - return std::make_unique(lds_config, server_.clusterManager(), server_.initManager(), - server_.stats(), server_.listenerManager(), - server_.messageValidationVisitor()); + return std::make_unique( + lds_config, server_.clusterManager(), server_.initManager(), server_.stats(), + server_.listenerManager(), server_.messageValidationContext().dynamicValidationVisitor()); } std::vector createNetworkFilterFactoryList( const Protobuf::RepeatedPtrField& filters, @@ -126,7 +126,7 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable admin_address_path("", "admin-address-path", "Admin address path", false, "", "string", cmd); @@ -181,7 +188,13 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv, config_path_ = config_path.getValue(); config_yaml_ = config_yaml.getValue(); - allow_unknown_fields_ = allow_unknown_fields.getValue(); + if (allow_unknown_fields.getValue()) { + ENVOY_LOG(warn, + "--allow-unknown-fields is deprecated, use --allow-unknown-static-fields instead."); + } + allow_unknown_static_fields_ = + allow_unknown_static_fields.getValue() || allow_unknown_fields.getValue(); + reject_unknown_dynamic_fields_ = reject_unknown_dynamic_fields.getValue(); admin_address_path_ = admin_address_path.getValue(); log_path_ = log_path.getValue(); restart_epoch_ = restart_epoch.getValue(); @@ -241,7 +254,8 @@ Server::CommandLineOptionsPtr OptionsImpl::toCommandLineOptions() const { command_line_options->set_concurrency(concurrency()); command_line_options->set_config_path(configPath()); command_line_options->set_config_yaml(configYaml()); - command_line_options->set_allow_unknown_fields(allow_unknown_fields_); + command_line_options->set_allow_unknown_static_fields(allow_unknown_static_fields_); + command_line_options->set_reject_unknown_dynamic_fields(reject_unknown_dynamic_fields_); command_line_options->set_admin_address_path(adminAddressPath()); command_line_options->set_component_log_level(component_log_level_str_); command_line_options->set_log_level(spdlog::level::to_string_view(logLevel()).data(), diff --git a/source/server/options_impl.h b/source/server/options_impl.h index e9663953aa99..7fea3a2546a1 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -75,6 +75,12 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable process_context) - : workers_started_(false), shutdown_(false), options_(options), time_source_(time_system), - restarter_(restarter), start_time_(time(nullptr)), original_start_time_(start_time_), - stats_store_(store), thread_local_(tls), + : workers_started_(false), shutdown_(false), options_(options), + validation_context_(options_.allowUnknownStaticFields(), + !options.rejectUnknownDynamicFields()), + time_source_(time_system), restarter_(restarter), start_time_(time(nullptr)), + original_start_time_(start_time_), stats_store_(store), thread_local_(tls), api_(new Api::Impl(thread_factory, store, time_system, file_system)), dispatcher_(api_->allocateDispatcher()), singleton_manager_(new Singleton::ManagerImpl(api_->threadFactory())), @@ -269,7 +271,8 @@ void InstanceImpl::initialize(const Options& options, Buffer::OwnedImpl().usesOldImpl() ? "old (libevent)" : "new"); // Handle configuration that needs to take place prior to the main configuration load. - InstanceUtil::loadBootstrapConfig(bootstrap_, options, messageValidationVisitor(), *api_); + InstanceUtil::loadBootstrapConfig(bootstrap_, options, + messageValidationContext().staticValidationVisitor(), *api_); bootstrap_config_update_time_ = time_source_.systemTime(); // Immediate after the bootstrap has been loaded, override the header prefix, if configured to @@ -289,6 +292,10 @@ void InstanceImpl::initialize(const Options& options, ServerStats{ALL_SERVER_STATS(POOL_COUNTER_PREFIX(stats_store_, server_stats_prefix), POOL_GAUGE_PREFIX(stats_store_, server_stats_prefix), POOL_HISTOGRAM_PREFIX(stats_store_, server_stats_prefix))}); + validation_context_.static_warning_validation_visitor().setCounter( + server_stats_->static_unknown_fields_); + validation_context_.dynamic_warning_validation_visitor().setCounter( + server_stats_->dynamic_unknown_fields_); initialization_timer_ = std::make_unique(server_stats_->initialization_time_ms_, timeSource()); @@ -342,7 +349,7 @@ void InstanceImpl::initialize(const Options& options, // Initialize the overload manager early so other modules can register for actions. overload_manager_ = std::make_unique( *dispatcher_, stats_store_, thread_local_, bootstrap_.overload_manager(), - messageValidationVisitor(), *api_); + messageValidationContext().staticValidationVisitor(), *api_); heap_shrinker_ = std::make_unique(*dispatcher_, *overload_manager_, stats_store_); @@ -375,7 +382,7 @@ void InstanceImpl::initialize(const Options& options, cluster_manager_factory_ = std::make_unique( *admin_, Runtime::LoaderSingleton::get(), stats_store_, thread_local_, *random_generator_, dns_resolver_, *ssl_context_manager_, *dispatcher_, *local_info_, *secret_manager_, - messageValidationVisitor(), *api_, http_context_, access_log_manager_, *singleton_manager_); + messageValidationContext(), *api_, http_context_, access_log_manager_, *singleton_manager_); // Now the configuration gets parsed. The configuration may start setting // thread local data per above. See MainImpl::initialize() for why ConfigImpl @@ -405,8 +412,8 @@ void InstanceImpl::initialize(const Options& options, ->create(), *dispatcher_, Runtime::LoaderSingleton::get(), stats_store_, *ssl_context_manager_, *random_generator_, info_factory_, access_log_manager_, *config_.clusterManager(), - *local_info_, *admin_, *singleton_manager_, thread_local_, messageValidationVisitor(), - *api_); + *local_info_, *admin_, *singleton_manager_, thread_local_, + messageValidationContext().dynamicValidationVisitor(), *api_); } for (Stats::SinkPtr& sink : config_.statsSinks()) { @@ -438,8 +445,8 @@ Runtime::LoaderPtr InstanceUtil::createRuntime(Instance& server, ENVOY_LOG(info, "runtime: {}", MessageUtil::getYamlStringFromMessage(config.runtime())); return std::make_unique( server.dispatcher(), server.threadLocal(), config.runtime(), server.localInfo(), - server.initManager(), server.stats(), server.random(), server.messageValidationVisitor(), - server.api()); + server.initManager(), server.stats(), server.random(), + server.messageValidationContext().dynamicValidationVisitor(), server.api()); } void InstanceImpl::loadServerFlags(const absl::optional& flags_path) { diff --git a/source/server/server.h b/source/server/server.h index 90128e9b2deb..500f743f98a6 100644 --- a/source/server/server.h +++ b/source/server/server.h @@ -49,6 +49,8 @@ namespace Server { * All server wide stats. @see stats_macros.h */ #define ALL_SERVER_STATS(COUNTER, GAUGE, HISTOGRAM) \ + COUNTER(static_unknown_fields) \ + COUNTER(dynamic_unknown_fields) \ COUNTER(debug_assertion_failures) \ GAUGE(concurrency, NeverImport) \ GAUGE(days_until_first_cert_expiring, Accumulate) \ @@ -201,9 +203,8 @@ class InstanceImpl : Logger::Loggable, return config_.statsFlushInterval(); } - ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { - return options_.allowUnknownFields() ? ProtobufMessage::getStrictValidationVisitor() - : ProtobufMessage::getNullValidationVisitor(); + ProtobufMessage::ValidationContext& messageValidationContext() override { + return validation_context_; } // ServerLifecycleNotifier @@ -236,6 +237,7 @@ class InstanceImpl : Logger::Loggable, bool workers_started_; bool shutdown_; const Options& options_; + ProtobufMessage::ProdValidationContextImpl validation_context_; TimeSource& time_source_; HotRestart& restarter_; const time_t start_time_; diff --git a/test/common/config/filesystem_subscription_impl_test.cc b/test/common/config/filesystem_subscription_impl_test.cc index 840748a72800..524914346484 100644 --- a/test/common/config/filesystem_subscription_impl_test.cc +++ b/test/common/config/filesystem_subscription_impl_test.cc @@ -20,7 +20,7 @@ TEST_F(FilesystemSubscriptionImplTest, BadJsonRecovery) { startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); EXPECT_CALL(callbacks_, - onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); updateFile(";!@#badjso n"); EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); diff --git a/test/common/protobuf/BUILD b/test/common/protobuf/BUILD index 94fbcced56c7..81c30eb6a241 100644 --- a/test/common/protobuf/BUILD +++ b/test/common/protobuf/BUILD @@ -9,6 +9,16 @@ load( envoy_package() +envoy_cc_test( + name = "message_validator_impl_test", + srcs = ["message_validator_impl_test.cc"], + deps = [ + "//source/common/protobuf:message_validator_lib", + "//test/test_common:logging_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "utility_test", srcs = ["utility_test.cc"], diff --git a/test/common/protobuf/message_validator_impl_test.cc b/test/common/protobuf/message_validator_impl_test.cc new file mode 100644 index 000000000000..a2fa6f4b011a --- /dev/null +++ b/test/common/protobuf/message_validator_impl_test.cc @@ -0,0 +1,54 @@ +#include "envoy/common/exception.h" + +#include "common/protobuf/message_validator_impl.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/test_common/logging.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace ProtobufMessage { +namespace { + +// The null validation visitor doesn't do anything on unknown fields. +TEST(NullValidationVisitorImpl, UnknownField) { + NullValidationVisitorImpl null_validation_visitor; + EXPECT_NO_THROW(null_validation_visitor.onUnknownField("foo")); +} + +// The warning validation visitor logs and bumps stats on unknown fields +TEST(WarningValidationVisitorImpl, UnknownField) { + Stats::IsolatedStoreImpl stats; + Stats::Counter& counter = stats.counter("counter"); + WarningValidationVisitorImpl warning_validation_visitor; + // First time around we should log. + EXPECT_LOG_CONTAINS("warn", "Unknown field: foo", + warning_validation_visitor.onUnknownField("foo")); + // Duplicate descriptions don't generate a log the second time around. + EXPECT_LOG_NOT_CONTAINS("warn", "Unknown field: foo", + warning_validation_visitor.onUnknownField("foo")); + // Unrelated variable increments. + EXPECT_LOG_CONTAINS("warn", "Unknown field: bar", + warning_validation_visitor.onUnknownField("bar")); + // When we set the stats counter, the above increments are transferred. + EXPECT_EQ(0, counter.value()); + warning_validation_visitor.setCounter(counter); + EXPECT_EQ(2, counter.value()); + // A third unknown field is tracked in stats post-initialization. + EXPECT_LOG_CONTAINS("warn", "Unknown field: baz", + warning_validation_visitor.onUnknownField("baz")); + EXPECT_EQ(3, counter.value()); +} + +// The strict validation visitor throws on unknown fields. +TEST(StrictValidationVisitorImpl, UnknownField) { + StrictValidationVisitorImpl strict_validation_visitor; + EXPECT_THROW_WITH_MESSAGE(strict_validation_visitor.onUnknownField("foo"), EnvoyException, + "Protobuf message (foo) has unknown fields"); +} + +} // namespace +} // namespace ProtobufMessage +} // namespace Envoy diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 5ed984b29124..a7ada1599f80 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -162,9 +162,10 @@ class TestClusterManagerImpl : public ClusterManagerImpl { Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, AccessLog::AccessLogManager& log_manager, Event::Dispatcher& main_thread_dispatcher, Server::Admin& admin, - Api::Api& api, Http::Context& http_context) + ProtobufMessage::ValidationContext& validation_context, Api::Api& api, + Http::Context& http_context) : ClusterManagerImpl(bootstrap, factory, stats, tls, runtime, random, local_info, log_manager, - main_thread_dispatcher, admin, validation_visitor_, api, http_context) {} + main_thread_dispatcher, admin, validation_context, api, http_context) {} std::map> activeClusters() { std::map> clusters; @@ -173,8 +174,6 @@ class TestClusterManagerImpl : public ClusterManagerImpl { } return clusters; } - - NiceMock validation_visitor_; }; // Override postThreadLocalClusterUpdate so we can test that merged updates calls @@ -186,10 +185,12 @@ class MockedUpdatedClusterManagerImpl : public TestClusterManagerImpl { Stats::Store& stats, ThreadLocal::Instance& tls, Runtime::Loader& runtime, Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, AccessLog::AccessLogManager& log_manager, Event::Dispatcher& main_thread_dispatcher, - Server::Admin& admin, Api::Api& api, MockLocalClusterUpdate& local_cluster_update, - MockLocalHostsRemoved& local_hosts_removed, Http::Context& http_context) + Server::Admin& admin, ProtobufMessage::ValidationContext& validation_context, Api::Api& api, + MockLocalClusterUpdate& local_cluster_update, MockLocalHostsRemoved& local_hosts_removed, + Http::Context& http_context) : TestClusterManagerImpl(bootstrap, factory, stats, tls, runtime, random, local_info, - log_manager, main_thread_dispatcher, admin, api, http_context), + log_manager, main_thread_dispatcher, admin, validation_context, api, + http_context), local_cluster_update_(local_cluster_update), local_hosts_removed_(local_hosts_removed) {} protected: @@ -225,7 +226,8 @@ class ClusterManagerImplTest : public testing::Test { void create(const envoy::config::bootstrap::v2::Bootstrap& bootstrap) { cluster_manager_ = std::make_unique( bootstrap, factory_, factory_.stats_, factory_.tls_, factory_.runtime_, factory_.random_, - factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, *api_, http_context_); + factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, validation_context_, + *api_, http_context_); } void createWithLocalClusterUpdate(const bool enable_merge_window = true) { @@ -259,8 +261,8 @@ class ClusterManagerImplTest : public testing::Test { cluster_manager_ = std::make_unique( bootstrap, factory_, factory_.stats_, factory_.tls_, factory_.runtime_, factory_.random_, - factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, *api_, - local_cluster_update_, local_hosts_removed_, http_context_); + factory_.local_info_, log_manager_, factory_.dispatcher_, admin_, validation_context_, + *api_, local_cluster_update_, local_hosts_removed_, http_context_); } void checkStats(uint64_t added, uint64_t modified, uint64_t removed, uint64_t active, @@ -303,6 +305,7 @@ class ClusterManagerImplTest : public testing::Test { Event::SimulatedTimeSystem time_system_; Api::ApiPtr api_; NiceMock factory_; + NiceMock validation_context_; std::unique_ptr cluster_manager_; AccessLog::MockAccessLogManager log_manager_; NiceMock admin_; diff --git a/test/config/integration/BUILD b/test/config/integration/BUILD index d28d2d8011fa..a6c4442e3e0b 100644 --- a/test/config/integration/BUILD +++ b/test/config/integration/BUILD @@ -16,9 +16,13 @@ filegroup( name = "server_xds_files", srcs = [ "server_xds.bootstrap.yaml", + "server_xds.cds.with_unknown_field.yaml", "server_xds.cds.yaml", + "server_xds.eds.with_unknown_field.yaml", "server_xds.eds.yaml", + "server_xds.lds.with_unknown_field.yaml", "server_xds.lds.yaml", + "server_xds.rds.with_unknown_field.yaml", "server_xds.rds.yaml", ], ) diff --git a/test/config/integration/server.yaml b/test/config/integration/server.yaml index 455a17bc0592..88d34049619b 100644 --- a/test/config/integration/server.yaml +++ b/test/config/integration/server.yaml @@ -8,10 +8,10 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - drain_timeout_ms: 5000 + drain_timeout: 5s route_config: virtual_hosts: - - require_ssl: all + - require_tls: all routes: - route: { cluster: cluster_1 } match: { prefix: "/" } @@ -22,9 +22,6 @@ static_resources: - match: { prefix: "/" } route: cluster: cluster_1 - runtime: - key: some_key - default: 0 - match: { prefix: "/test/long/url" } route: rate_limits: @@ -42,15 +39,11 @@ static_resources: name: integration codec_type: http1 stat_prefix: router - filters: - - name: health_check + http_filters: + - name: envoy.health_check config: - endpoint: "/healthcheck" pass_through_mode: false - - name: rate_limit - config: - domain: foo - - name: router + - name: envoy.router config: {} access_log: - name: envoy.file_access_log diff --git a/test/config/integration/server_unix_listener.yaml b/test/config/integration/server_unix_listener.yaml index 0ba01442cb6d..bd0c6f090403 100644 --- a/test/config/integration/server_unix_listener.yaml +++ b/test/config/integration/server_unix_listener.yaml @@ -7,12 +7,12 @@ static_resources: - filters: - name: envoy.http_connection_manager config: - filters: - - name: router + http_filters: + - name: envoy.router config: {} codec_type: auto stat_prefix: router - drain_timeout_ms: 5000 + drain_timeout: 5s route_config: virtual_hosts: - domains: diff --git a/test/config/integration/server_xds.cds.with_unknown_field.yaml b/test/config/integration/server_xds.cds.with_unknown_field.yaml new file mode 100644 index 000000000000..01d794ab4032 --- /dev/null +++ b/test/config/integration/server_xds.cds.with_unknown_field.yaml @@ -0,0 +1,15 @@ +version_info: "0" +resources: +- "@type": type.googleapis.com/envoy.api.v2.Cluster + name: cluster_1 + connect_timeout: { seconds: 5 } + type: EDS + eds_cluster_config: + eds_config: { path: {{ eds_json_path }} } + lb_policy: ROUND_ROBIN + http2_protocol_options: {} + extension_protocol_options: + envoy.test.dynamic_validation: + stat_prefix: blah + cluster: blah + foo: bar diff --git a/test/config/integration/server_xds.eds.with_unknown_field.yaml b/test/config/integration/server_xds.eds.with_unknown_field.yaml new file mode 100644 index 000000000000..912be177993d --- /dev/null +++ b/test/config/integration/server_xds.eds.with_unknown_field.yaml @@ -0,0 +1,12 @@ +version_info: "0" +resources: +- "@type": type.googleapis.com/envoy.api.v2.ClusterLoadAssignment + cluster_name: cluster_1 + foo: bar + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ ntop_ip_loopback_address }} + port_value: {{ upstream_0 }} diff --git a/test/config/integration/server_xds.lds.with_unknown_field.yaml b/test/config/integration/server_xds.lds.with_unknown_field.yaml new file mode 100644 index 000000000000..e1fe53b7127b --- /dev/null +++ b/test/config/integration/server_xds.lds.with_unknown_field.yaml @@ -0,0 +1,20 @@ +version_info: "0" +resources: +- "@type": type.googleapis.com/envoy.api.v2.Listener + name: listener_0 + address: + socket_address: + address: {{ ntop_ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + codec_type: HTTP2 + drain_timeout: 5s + stat_prefix: router + rds: + route_config_name: route_config_0 + config_source: { path: {{ rds_json_path }} } + http_filters: [{ name: envoy.router }] + foo: bar diff --git a/test/config/integration/server_xds.rds.with_unknown_field.yaml b/test/config/integration/server_xds.rds.with_unknown_field.yaml new file mode 100644 index 000000000000..0fb40bd6a74a --- /dev/null +++ b/test/config/integration/server_xds.rds.with_unknown_field.yaml @@ -0,0 +1,11 @@ +version_info: "0" +resources: +- "@type": type.googleapis.com/envoy.api.v2.RouteConfiguration + name: route_config_0 + virtual_hosts: + - name: integration + domains: [ "*" ] + routes: + - match: { prefix: "/test/long/url" } + route: { cluster: cluster_1 } + foo: bar diff --git a/test/config_test/config_test.cc b/test/config_test/config_test.cc index b552cb9ec31b..d7311ec8c160 100644 --- a/test/config_test/config_test.cc +++ b/test/config_test/config_test.cc @@ -81,15 +81,15 @@ class ConfigTest { })); envoy::config::bootstrap::v2::Bootstrap bootstrap; - Server::InstanceUtil::loadBootstrapConfig(bootstrap, options_, - server_.messageValidationVisitor(), *api_); + Server::InstanceUtil::loadBootstrapConfig( + bootstrap, options_, server_.messageValidationContext().staticValidationVisitor(), *api_); Server::Configuration::InitialImpl initial_config(bootstrap); Server::Configuration::MainImpl main_config; cluster_manager_factory_ = std::make_unique( server_.admin(), server_.runtime(), server_.stats(), server_.threadLocal(), server_.random(), server_.dnsResolver(), ssl_context_manager_, server_.dispatcher(), - server_.localInfo(), server_.secretManager(), server_.messageValidationVisitor(), *api_, + server_.localInfo(), server_.secretManager(), server_.messageValidationContext(), *api_, server_.httpContext(), server_.accessLogManager(), server_.singletonManager(), time_system_); diff --git a/test/integration/BUILD b/test/integration/BUILD index f100db5066d8..393eadbfa2e8 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -772,6 +772,16 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "dynamic_validation_integration_test", + srcs = ["dynamic_validation_integration_test.cc"], + data = ["//test/config/integration:server_xds_files"], + deps = [ + ":http_integration_lib", + "//source/common/stats:stats_lib", + ], +) + envoy_cc_test( name = "xds_integration_test", srcs = ["xds_integration_test.cc"], diff --git a/test/integration/dynamic_validation_integration_test.cc b/test/integration/dynamic_validation_integration_test.cc new file mode 100644 index 000000000000..b4344312b85a --- /dev/null +++ b/test/integration/dynamic_validation_integration_test.cc @@ -0,0 +1,181 @@ +#include + +#include "envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.pb.validate.h" + +#include "extensions/filters/network/common/factory_base.h" + +#include "test/integration/http_integration.h" +#include "test/test_common/environment.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace { + +// This fake filter is used by CdsProtocolOptionsRejected. +class TestDynamicValidationNetworkFilter : public Network::WriteFilter { +public: + Network::FilterStatus onWrite(Buffer::Instance&, bool) override { + return Network::FilterStatus::Continue; + } +}; + +class TestDynamicValidationNetworkFilterConfigFactory + : public Extensions::NetworkFilters::Common::FactoryBase< + envoy::config::filter::network::tcp_proxy::v2::TcpProxy> { +public: + TestDynamicValidationNetworkFilterConfigFactory() + : Extensions::NetworkFilters::Common::FactoryBase< + envoy::config::filter::network::tcp_proxy::v2::TcpProxy>( + "envoy.test.dynamic_validation") {} + +private: + Network::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::config::filter::network::tcp_proxy::v2::TcpProxy& /*proto_config*/, + Server::Configuration::FactoryContext& /*context*/) override { + return Network::FilterFactoryCb(); + } + + Upstream::ProtocolOptionsConfigConstSharedPtr createProtocolOptionsTyped( + const envoy::config::filter::network::tcp_proxy::v2::TcpProxy&) override { + return nullptr; + } +}; + +REGISTER_FACTORY(TestDynamicValidationNetworkFilterConfigFactory, + Server::Configuration::NamedNetworkFilterConfigFactory); + +// Pretty-printing of parameterized test names. +std::string dynamicValidationTestParamsToString( + const ::testing::TestParamInfo>& params) { + return fmt::format( + "{}_{}", + TestUtility::ipTestParamsToString( + ::testing::TestParamInfo(std::get<0>(params.param), 0)), + std::get<1>(params.param) ? "with_reject_unknown_fields" : "without_reject_unknown_fields"); +} + +// Validate unknown field handling in dynamic configuration. +class DynamicValidationIntegrationTest + : public testing::TestWithParam>, + public HttpIntegrationTest { +public: + DynamicValidationIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, std::get<0>(GetParam())), + reject_unknown_dynamic_fields_(std::get<1>(GetParam())) { + setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + } + + void createEnvoy() override { + registerPort("upstream_0", fake_upstreams_.back()->localAddress()->ip()->port()); + createApiTestServer(api_filesystem_config_, {"http"}, reject_unknown_dynamic_fields_, + reject_unknown_dynamic_fields_, allow_lds_rejection_); + } + + ApiFilesystemConfig api_filesystem_config_; + const bool reject_unknown_dynamic_fields_; + bool allow_lds_rejection_{}; +}; + +INSTANTIATE_TEST_SUITE_P( + IpVersions, DynamicValidationIntegrationTest, + testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), testing::Bool()), + dynamicValidationTestParamsToString); + +// Protocol options in CDS with unknown fields are rejected if and only if strict. +TEST_P(DynamicValidationIntegrationTest, CdsProtocolOptionsRejected) { + api_filesystem_config_ = { + "test/config/integration/server_xds.bootstrap.yaml", + "test/config/integration/server_xds.cds.with_unknown_field.yaml", + "test/config/integration/server_xds.eds.yaml", + "test/config/integration/server_xds.lds.yaml", + "test/config/integration/server_xds.rds.yaml", + }; + initialize(); + if (reject_unknown_dynamic_fields_) { + EXPECT_EQ(0, test_server_->counter("cluster_manager.cds.update_success")->value()); + // CDS API parsing will reject due to unknown HCM field. + EXPECT_EQ(1, test_server_->counter("cluster_manager.cds.update_rejected")->value()); + EXPECT_EQ(0, test_server_->counter("server.dynamic_unknown_fields")->value()); + } else { + EXPECT_EQ(1, test_server_->counter("cluster_manager.cds.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("server.dynamic_unknown_fields")->value()); + } +} + +// Network filters in LDS with unknown fields are rejected if and only if strict. +TEST_P(DynamicValidationIntegrationTest, LdsFilterRejected) { + allow_lds_rejection_ = true; + api_filesystem_config_ = { + "test/config/integration/server_xds.bootstrap.yaml", + "test/config/integration/server_xds.cds.yaml", + "test/config/integration/server_xds.eds.yaml", + "test/config/integration/server_xds.lds.with_unknown_field.yaml", + "test/config/integration/server_xds.rds.yaml", + }; + initialize(); + if (reject_unknown_dynamic_fields_) { + EXPECT_EQ(0, test_server_->counter("listener_manager.lds.update_success")->value()); + // LDS API parsing will reject due to unknown HCM field. + EXPECT_EQ(1, test_server_->counter("listener_manager.lds.update_rejected")->value()); + EXPECT_EQ(nullptr, test_server_->counter("http.router.rds.route_config_0.update_success")); + EXPECT_EQ(0, test_server_->counter("server.dynamic_unknown_fields")->value()); + } else { + EXPECT_EQ(1, test_server_->counter("listener_manager.lds.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("http.router.rds.route_config_0.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("server.dynamic_unknown_fields")->value()); + } + EXPECT_EQ(1, test_server_->counter("cluster_manager.cds.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("cluster.cluster_1.update_success")->value()); +} + +// Unknown fields in RDS cause config load failure if and only if strict. +TEST_P(DynamicValidationIntegrationTest, RdsFailedBySubscription) { + api_filesystem_config_ = { + "test/config/integration/server_xds.bootstrap.yaml", + "test/config/integration/server_xds.cds.yaml", + "test/config/integration/server_xds.eds.yaml", + "test/config/integration/server_xds.lds.yaml", + "test/config/integration/server_xds.rds.with_unknown_field.yaml", + }; + initialize(); + EXPECT_EQ(1, test_server_->counter("listener_manager.lds.update_success")->value()); + if (reject_unknown_dynamic_fields_) { + EXPECT_EQ(0, test_server_->counter("http.router.rds.route_config_0.update_success")->value()); + // FilesystemSubscriptionImpl will reject early at the ingestion level + EXPECT_EQ(1, test_server_->counter("http.router.rds.route_config_0.update_failure")->value()); + EXPECT_EQ(0, test_server_->counter("server.dynamic_unknown_fields")->value()); + } else { + EXPECT_EQ(1, test_server_->counter("http.router.rds.route_config_0.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("server.dynamic_unknown_fields")->value()); + } + EXPECT_EQ(1, test_server_->counter("cluster_manager.cds.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("cluster.cluster_1.update_success")->value()); +} + +// Unknown fields in EDS cause config load failure if and only if strict. +TEST_P(DynamicValidationIntegrationTest, EdsFailedBySubscription) { + api_filesystem_config_ = { + "test/config/integration/server_xds.bootstrap.yaml", + "test/config/integration/server_xds.cds.yaml", + "test/config/integration/server_xds.eds.with_unknown_field.yaml", + "test/config/integration/server_xds.lds.yaml", + "test/config/integration/server_xds.rds.yaml", + }; + initialize(); + EXPECT_EQ(1, test_server_->counter("listener_manager.lds.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("http.router.rds.route_config_0.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("cluster_manager.cds.update_success")->value()); + if (reject_unknown_dynamic_fields_) { + EXPECT_EQ(0, test_server_->counter("cluster.cluster_1.update_success")->value()); + // FilesystemSubscriptionImpl will reject early at the ingestion level + EXPECT_EQ(1, test_server_->counter("cluster.cluster_1.update_failure")->value()); + EXPECT_EQ(0, test_server_->counter("server.dynamic_unknown_fields")->value()); + } else { + EXPECT_EQ(1, test_server_->counter("cluster.cluster_1.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("server.dynamic_unknown_fields")->value()); + } +} + +} // namespace +} // namespace Envoy diff --git a/test/integration/integration.cc b/test/integration/integration.cc index b7fd1d46696c..e0c124bd3c6b 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -354,7 +354,7 @@ void BaseIntegrationTest::createEnvoy() { for (int i = 0; i < static_resources.listeners_size(); ++i) { named_ports.push_back(static_resources.listeners(i).name()); } - createGeneratedApiTestServer(bootstrap_path, named_ports); + createGeneratedApiTestServer(bootstrap_path, named_ports, false, true, false); } void BaseIntegrationTest::setUpstreamProtocol(FakeHttpConnection::Type protocol) { @@ -416,10 +416,14 @@ void BaseIntegrationTest::registerTestServerPorts(const std::vector } void BaseIntegrationTest::createGeneratedApiTestServer(const std::string& bootstrap_path, - const std::vector& port_names) { - test_server_ = IntegrationTestServer::create(bootstrap_path, version_, on_server_init_function_, - deterministic_, timeSystem(), *api_, - defer_listener_finalization_, process_object_); + const std::vector& port_names, + bool allow_unknown_static_fields, + bool reject_unknown_dynamic_fields, + bool allow_lds_rejection) { + test_server_ = IntegrationTestServer::create( + bootstrap_path, version_, on_server_init_function_, deterministic_, timeSystem(), *api_, + defer_listener_finalization_, process_object_, allow_unknown_static_fields, + reject_unknown_dynamic_fields); if (config_helper_.bootstrap().static_resources().listeners_size() > 0 && !defer_listener_finalization_) { @@ -427,15 +431,19 @@ void BaseIntegrationTest::createGeneratedApiTestServer(const std::string& bootst // needs to know about the bound listener ports. auto end_time = time_system_.monotonicTime() + TestUtility::DefaultTimeout; const char* success = "listener_manager.listener_create_success"; - const char* failure = "listener_manager.lds.update_rejected"; - while (test_server_->counter(success) == nullptr || - test_server_->counter(success)->value() == 0) { + const char* rejected = "listener_manager.lds.update_rejected"; + while ((test_server_->counter(success) == nullptr || + test_server_->counter(success)->value() == 0) && + (!allow_lds_rejection || test_server_->counter(rejected) == nullptr || + test_server_->counter(rejected)->value() == 0)) { if (time_system_.monotonicTime() >= end_time) { RELEASE_ASSERT(0, "Timed out waiting for listeners."); } - RELEASE_ASSERT(test_server_->counter(failure) == nullptr || - test_server_->counter(failure)->value() == 0, - "Lds update failed"); + if (!allow_lds_rejection) { + RELEASE_ASSERT(test_server_->counter(rejected) == nullptr || + test_server_->counter(rejected)->value() == 0, + "Lds update failed"); + } time_system_.sleep(std::chrono::milliseconds(10)); } @@ -444,7 +452,10 @@ void BaseIntegrationTest::createGeneratedApiTestServer(const std::string& bootst } void BaseIntegrationTest::createApiTestServer(const ApiFilesystemConfig& api_filesystem_config, - const std::vector& port_names) { + const std::vector& port_names, + bool allow_unknown_static_fields, + bool reject_unknown_dynamic_fields, + bool allow_lds_rejection) { const std::string eds_path = TestEnvironment::temporaryFileSubstitute( api_filesystem_config.eds_path_, port_map_, version_); const std::string cds_path = TestEnvironment::temporaryFileSubstitute( @@ -453,11 +464,11 @@ void BaseIntegrationTest::createApiTestServer(const ApiFilesystemConfig& api_fil api_filesystem_config.rds_path_, port_map_, version_); const std::string lds_path = TestEnvironment::temporaryFileSubstitute( api_filesystem_config.lds_path_, {{"rds_json_path", rds_path}}, port_map_, version_); - createGeneratedApiTestServer(TestEnvironment::temporaryFileSubstitute( - api_filesystem_config.bootstrap_path_, - {{"cds_json_path", cds_path}, {"lds_json_path", lds_path}}, - port_map_, version_), - port_names); + createGeneratedApiTestServer( + TestEnvironment::temporaryFileSubstitute( + api_filesystem_config.bootstrap_path_, + {{"cds_json_path", cds_path}, {"lds_json_path", lds_path}}, port_map_, version_), + port_names, allow_unknown_static_fields, reject_unknown_dynamic_fields, allow_lds_rejection); } void BaseIntegrationTest::createTestServer(const std::string& json_path, diff --git a/test/integration/integration.h b/test/integration/integration.h index 4c09b7ecc6fa..db32e8a8cfc6 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -193,9 +193,13 @@ class BaseIntegrationTest : Logger::Loggable { void registerTestServerPorts(const std::vector& port_names); void createTestServer(const std::string& json_path, const std::vector& port_names); void createGeneratedApiTestServer(const std::string& bootstrap_path, - const std::vector& port_names); + const std::vector& port_names, + bool allow_unknown_static_fields, + bool reject_unknown_dynamic_fields, bool allow_lds_rejection); void createApiTestServer(const ApiFilesystemConfig& api_filesystem_config, - const std::vector& port_names); + const std::vector& port_names, + bool allow_unknown_static_fields, bool reject_unknown_dynamic_fields, + bool allow_lds_rejection); Event::TestTimeSystem& timeSystem() { return time_system_; } diff --git a/test/integration/server.cc b/test/integration/server.cc index 382332aa80c3..6161eeedf8b4 100644 --- a/test/integration/server.cc +++ b/test/integration/server.cc @@ -29,7 +29,9 @@ namespace Envoy { namespace Server { OptionsImpl createTestOptionsImpl(const std::string& config_path, const std::string& config_yaml, - Network::Address::IpVersion ip_version) { + Network::Address::IpVersion ip_version, + bool allow_unknown_static_fields, + bool reject_unknown_dynamic_fields) { OptionsImpl test_options("cluster_name", "node_name", "zone_name", spdlog::level::info); test_options.setConfigPath(config_path); @@ -38,6 +40,8 @@ OptionsImpl createTestOptionsImpl(const std::string& config_path, const std::str test_options.setFileFlushIntervalMsec(std::chrono::milliseconds(50)); test_options.setDrainTime(std::chrono::seconds(1)); test_options.setParentShutdownTime(std::chrono::seconds(2)); + test_options.setAllowUnkownFields(allow_unknown_static_fields); + test_options.setRejectUnknownFieldsDynamic(reject_unknown_dynamic_fields); return test_options; } @@ -48,11 +52,12 @@ IntegrationTestServerPtr IntegrationTestServer::create( const std::string& config_path, const Network::Address::IpVersion version, std::function on_server_init_function, bool deterministic, Event::TestTimeSystem& time_system, Api::Api& api, bool defer_listener_finalization, - absl::optional> process_object) { + absl::optional> process_object, + bool allow_unknown_static_fields, bool reject_unknown_dynamic_fields) { IntegrationTestServerPtr server{ std::make_unique(time_system, api, config_path)}; server->start(version, on_server_init_function, deterministic, defer_listener_finalization, - process_object); + process_object, allow_unknown_static_fields, reject_unknown_dynamic_fields); return server; } @@ -69,13 +74,16 @@ void IntegrationTestServer::waitUntilListenersReady() { void IntegrationTestServer::start( const Network::Address::IpVersion version, std::function on_server_init_function, bool deterministic, bool defer_listener_finalization, - absl::optional> process_object) { + absl::optional> process_object, + bool allow_unknown_static_fields, bool reject_unknown_dynamic_fields) { ENVOY_LOG(info, "starting integration test server"); ASSERT(!thread_); - thread_ = - api_.threadFactory().createThread([version, deterministic, process_object, this]() -> void { - threadRoutine(version, deterministic, process_object); - }); + thread_ = api_.threadFactory().createThread([version, deterministic, process_object, + allow_unknown_static_fields, + reject_unknown_dynamic_fields, this]() -> void { + threadRoutine(version, deterministic, process_object, allow_unknown_static_fields, + reject_unknown_dynamic_fields); + }); // If any steps need to be done prior to workers starting, do them now. E.g., xDS pre-init. // Note that there is no synchronization guaranteeing this happens either @@ -146,8 +154,10 @@ void IntegrationTestServer::serverReady() { void IntegrationTestServer::threadRoutine( const Network::Address::IpVersion version, bool deterministic, - absl::optional> process_object) { - OptionsImpl options(Server::createTestOptionsImpl(config_path_, "", version)); + absl::optional> process_object, + bool allow_unknown_static_fields, bool reject_unknown_dynamic_fields) { + OptionsImpl options(Server::createTestOptionsImpl( + config_path_, "", version, allow_unknown_static_fields, reject_unknown_dynamic_fields)); Thread::MutexBasicLockable lock; Runtime::RandomGeneratorPtr random_generator; diff --git a/test/integration/server.h b/test/integration/server.h index 55e3c11eab23..4489c436df19 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -32,7 +32,9 @@ namespace Server { // Create OptionsImpl structures suitable for tests. OptionsImpl createTestOptionsImpl(const std::string& config_path, const std::string& config_yaml, - Network::Address::IpVersion ip_version); + Network::Address::IpVersion ip_version, + bool allow_unknown_static_fields = false, + bool reject_unknown_dynamic_fields = false); class TestDrainManager : public DrainManager { public: @@ -233,7 +235,8 @@ class IntegrationTestServer : public Logger::Loggable, std::function on_server_init_function, bool deterministic, Event::TestTimeSystem& time_system, Api::Api& api, bool defer_listener_finalization = false, - absl::optional> process_object = absl::nullopt); + absl::optional> process_object = absl::nullopt, + bool allow_unknown_static_fields = false, bool reject_unknown_dynamic_fields = false); // Note that the derived class is responsible for tearing down the server in its // destructor. ~IntegrationTestServer() override; @@ -252,7 +255,8 @@ class IntegrationTestServer : public Logger::Loggable, void start(const Network::Address::IpVersion version, std::function on_server_init_function, bool deterministic, bool defer_listener_finalization, - absl::optional> process_object); + absl::optional> process_object, + bool allow_unknown_static_fields, bool reject_unknown_dynamic_fields); void waitForCounterEq(const std::string& name, uint64_t value) override { TestUtility::waitForCounterEq(stat_store(), name, value, time_system_); @@ -331,7 +335,8 @@ class IntegrationTestServer : public Logger::Loggable, * Runs the real server on a thread. */ void threadRoutine(const Network::Address::IpVersion version, bool deterministic, - absl::optional> process_object); + absl::optional> process_object, + bool allow_unknown_static_fields, bool reject_unknown_dynamic_fields); Event::TestTimeSystem& time_system_; Api::Api& api_; diff --git a/test/integration/xds_integration_test.cc b/test/integration/xds_integration_test.cc index 218ea948dc78..12e86df53292 100644 --- a/test/integration/xds_integration_test.cc +++ b/test/integration/xds_integration_test.cc @@ -28,7 +28,11 @@ class XdsIntegrationTest : public testing::TestWithParamcounter("listener_manager.lds.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("http.router.rds.route_config_0.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("cluster_manager.cds.update_success")->value()); + EXPECT_EQ(1, test_server_->counter("cluster.cluster_1.update_success")->value()); } }; diff --git a/test/mocks/protobuf/mocks.cc b/test/mocks/protobuf/mocks.cc index 908b08864fab..e425963e71dc 100644 --- a/test/mocks/protobuf/mocks.cc +++ b/test/mocks/protobuf/mocks.cc @@ -1,5 +1,7 @@ #include "test/mocks/protobuf/mocks.h" +using testing::ReturnRef; + namespace Envoy { namespace ProtobufMessage { @@ -7,5 +9,12 @@ MockValidationVisitor::MockValidationVisitor() = default; MockValidationVisitor::~MockValidationVisitor() = default; +MockValidationContext::MockValidationContext() { + ON_CALL(*this, staticValidationVisitor()).WillByDefault(ReturnRef(static_validation_visitor_)); + ON_CALL(*this, dynamicValidationVisitor()).WillByDefault(ReturnRef(dynamic_validation_visitor_)); +} + +MockValidationContext::~MockValidationContext() = default; + } // namespace ProtobufMessage } // namespace Envoy diff --git a/test/mocks/protobuf/mocks.h b/test/mocks/protobuf/mocks.h index 66e2f5d2b9c1..a099571d5e33 100644 --- a/test/mocks/protobuf/mocks.h +++ b/test/mocks/protobuf/mocks.h @@ -15,5 +15,17 @@ class MockValidationVisitor : public ValidationVisitor { MOCK_METHOD1(onUnknownField, void(absl::string_view)); }; +class MockValidationContext : public ValidationContext { +public: + MockValidationContext(); + ~MockValidationContext() override; + + MOCK_METHOD0(staticValidationVisitor, ValidationVisitor&()); + MOCK_METHOD0(dynamicValidationVisitor, ValidationVisitor&()); + + MockValidationVisitor static_validation_visitor_; + MockValidationVisitor dynamic_validation_visitor_; +}; + } // namespace ProtobufMessage } // namespace Envoy diff --git a/test/mocks/server/BUILD b/test/mocks/server/BUILD index 5d368f3c78ba..695f49be6c23 100644 --- a/test/mocks/server/BUILD +++ b/test/mocks/server/BUILD @@ -38,6 +38,7 @@ envoy_cc_mock( "//test/mocks/init:init_mocks", "//test/mocks/local_info:local_info_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/router:router_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/secret:secret_mocks", diff --git a/test/mocks/server/mocks.cc b/test/mocks/server/mocks.cc index e1006497c8ba..9d2d49f52750 100644 --- a/test/mocks/server/mocks.cc +++ b/test/mocks/server/mocks.cc @@ -23,6 +23,12 @@ MockOptions::MockOptions(const std::string& config_path) : config_path_(config_p ON_CALL(*this, configPath()).WillByDefault(ReturnRef(config_path_)); ON_CALL(*this, configProto()).WillByDefault(ReturnRef(config_proto_)); ON_CALL(*this, configYaml()).WillByDefault(ReturnRef(config_yaml_)); + ON_CALL(*this, allowUnknownStaticFields()).WillByDefault(Invoke([this] { + return allow_unknown_static_fields_; + })); + ON_CALL(*this, rejectUnknownDynamicFields()).WillByDefault(Invoke([this] { + return reject_unknown_dynamic_fields_; + })); ON_CALL(*this, adminAddressPath()).WillByDefault(ReturnRef(admin_address_path_)); ON_CALL(*this, serviceClusterName()).WillByDefault(ReturnRef(service_cluster_name_)); ON_CALL(*this, serviceNodeName()).WillByDefault(ReturnRef(service_node_name_)); @@ -152,8 +158,7 @@ MockInstance::MockInstance() ON_CALL(*this, mutexTracer()).WillByDefault(Return(nullptr)); ON_CALL(*this, singletonManager()).WillByDefault(ReturnRef(*singleton_manager_)); ON_CALL(*this, overloadManager()).WillByDefault(ReturnRef(overload_manager_)); - ON_CALL(*this, messageValidationVisitor()) - .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); + ON_CALL(*this, messageValidationContext()).WillByDefault(ReturnRef(validation_context_)); } MockInstance::~MockInstance() = default; diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 99fb3ebc1512..4ba0920f5695 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -7,6 +7,7 @@ #include "envoy/common/mutex_tracer.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.h" +#include "envoy/protobuf/message_validator.h" #include "envoy/server/admin.h" #include "envoy/server/configuration.h" #include "envoy/server/drain_manager.h" @@ -35,6 +36,7 @@ #include "test/mocks/init/mocks.h" #include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/protobuf/mocks.h" #include "test/mocks/router/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/secret/mocks.h" @@ -62,7 +64,8 @@ class MockOptions : public Options { MOCK_CONST_METHOD0(configPath, const std::string&()); MOCK_CONST_METHOD0(configProto, const envoy::config::bootstrap::v2::Bootstrap&()); MOCK_CONST_METHOD0(configYaml, const std::string&()); - MOCK_CONST_METHOD0(allowUnknownFields, bool()); + MOCK_CONST_METHOD0(allowUnknownStaticFields, bool()); + MOCK_CONST_METHOD0(rejectUnknownDynamicFields, bool()); MOCK_CONST_METHOD0(adminAddressPath, const std::string&()); MOCK_CONST_METHOD0(localAddressIpVersion, Network::Address::IpVersion()); MOCK_CONST_METHOD0(drainTime, std::chrono::seconds()); @@ -88,6 +91,8 @@ class MockOptions : public Options { std::string config_path_; envoy::config::bootstrap::v2::Bootstrap config_proto_; std::string config_yaml_; + bool allow_unknown_static_fields_{}; + bool reject_unknown_dynamic_fields_{}; std::string admin_address_path_; std::string service_cluster_name_; std::string service_node_name_; @@ -380,7 +385,7 @@ class MockInstance : public Instance { MOCK_METHOD0(threadLocal, ThreadLocal::Instance&()); MOCK_METHOD0(localInfo, const LocalInfo::LocalInfo&()); MOCK_CONST_METHOD0(statsFlushInterval, std::chrono::milliseconds()); - MOCK_METHOD0(messageValidationVisitor, ProtobufMessage::ValidationVisitor&()); + MOCK_METHOD0(messageValidationContext, ProtobufMessage::ValidationContext&()); TimeSource& timeSource() override { return time_system_; } @@ -410,6 +415,7 @@ class MockInstance : public Instance { Singleton::ManagerPtr singleton_manager_; Grpc::ContextImpl grpc_context_; Http::ContextImpl http_context_; + testing::NiceMock validation_context_; }; namespace Configuration { diff --git a/test/server/BUILD b/test/server/BUILD index fcd3adfbd670..6f2893539e87 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -243,6 +243,11 @@ filegroup( srcs = glob(["test_data/runtime/**"]), ) +filegroup( + name = "static_validation_test_data", + srcs = glob(["test_data/static_validation/**"]), +) + envoy_cc_test( name = "server_test", srcs = ["server_test.cc"], @@ -261,6 +266,7 @@ envoy_cc_test( ":node_bootstrap_without_access_log.yaml", ":runtime_bootstrap.yaml", ":runtime_test_data", + ":static_validation_test_data", ":stats_sink_bootstrap.yaml", ":zipkin_tracing.yaml", ], diff --git a/test/server/config_validation/cluster_manager_test.cc b/test/server/config_validation/cluster_manager_test.cc index ce734f717301..0d3639d4860b 100644 --- a/test/server/config_validation/cluster_manager_test.cc +++ b/test/server/config_validation/cluster_manager_test.cc @@ -30,7 +30,7 @@ namespace { TEST(ValidationClusterManagerTest, MockedMethods) { Stats::IsolatedStoreImpl stats_store; Event::SimulatedTimeSystem time_system; - NiceMock validation_visitor; + NiceMock validation_context; Api::ApiPtr api(Api::createApiForTest(stats_store, time_system)); NiceMock runtime; NiceMock tls; @@ -47,7 +47,7 @@ TEST(ValidationClusterManagerTest, MockedMethods) { ValidationClusterManagerFactory factory(admin, runtime, stats_store, tls, random, dns_resolver, ssl_context_manager, dispatcher, local_info, - secret_manager, validation_visitor, *api, http_context, + secret_manager, validation_context, *api, http_context, log_manager, singleton_manager, time_system); const envoy::config::bootstrap::v2::Bootstrap bootstrap; diff --git a/test/server/configuration_impl_test.cc b/test/server/configuration_impl_test.cc index c2c4247b1f87..dbc13a45d189 100644 --- a/test/server/configuration_impl_test.cc +++ b/test/server/configuration_impl_test.cc @@ -59,7 +59,7 @@ class ConfigurationImplTest : public testing::Test { server_.threadLocal(), server_.random(), server_.dnsResolver(), server_.sslContextManager(), server_.dispatcher(), server_.localInfo(), server_.secretManager(), - server_.messageValidationVisitor(), *api_, server_.httpContext(), + server_.messageValidationContext(), *api_, server_.httpContext(), server_.accessLogManager(), server_.singletonManager()) {} Api::ApiPtr api_; diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index e3cdeffa44f1..768468f8c96e 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -37,6 +37,7 @@ #include "gtest/gtest.h" using testing::_; +using testing::AtLeast; using testing::InSequence; using testing::Invoke; using testing::NiceMock; @@ -77,8 +78,15 @@ class ListenerManagerImplTest : public testing::Test { * 4) Creates a mock local drain manager for the listener. */ ListenerHandle* expectListenerCreate( - bool need_init, + bool need_init, bool added_via_api, envoy::api::v2::Listener::DrainType drain_type = envoy::api::v2::Listener_DrainType_DEFAULT) { + if (added_via_api) { + EXPECT_CALL(server_.validation_context_, staticValidationVisitor()).Times(0); + EXPECT_CALL(server_.validation_context_, dynamicValidationVisitor()); + } else { + EXPECT_CALL(server_.validation_context_, staticValidationVisitor()); + EXPECT_CALL(server_.validation_context_, dynamicValidationVisitor()).Times(0); + } ListenerHandle* raw_listener = new ListenerHandle(); EXPECT_CALL(listener_factory_, createDrainManager_(drain_type)) .WillOnce(Return(raw_listener->drain_manager_)); @@ -608,7 +616,7 @@ TEST_F(ListenerManagerImplTest, ModifyOnlyDrainType) { )EOF"; ListenerHandle* listener_foo = - expectListenerCreate(false, envoy::api::v2::Listener_DrainType_MODIFY_ONLY); + expectListenerCreate(false, true, envoy::api::v2::Listener_DrainType_MODIFY_ONLY); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); checkStats(1, 0, 0, 0, 1, 0); @@ -632,7 +640,7 @@ drain_type: default )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(false); + ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); checkStats(1, 0, 0, 0, 1, 0); @@ -650,7 +658,7 @@ drain_type: modify_only )EOF"; ListenerHandle* listener_foo_different_address = - expectListenerCreate(false, envoy::api::v2::Listener_DrainType_MODIFY_ONLY); + expectListenerCreate(false, true, envoy::api::v2::Listener_DrainType_MODIFY_ONLY); EXPECT_CALL(*listener_foo_different_address, onDestroy()); EXPECT_THROW_WITH_MESSAGE( manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_different_address_yaml), @@ -682,7 +690,7 @@ name: foo drain_type: default )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(false); + ListenerHandle* listener_foo = expectListenerCreate(false, true); ON_CALL(os_sys_calls, socket(AF_INET, _, 0)).WillByDefault(Return(Api::SysCallIntResult{5, 0})); ON_CALL(os_sys_calls, socket(AF_INET6, _, 0)).WillByDefault(Return(Api::SysCallIntResult{-1, 0})); @@ -714,7 +722,7 @@ name: foo drain_type: default )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(false); + ListenerHandle* listener_foo = expectListenerCreate(false, true); ON_CALL(os_sys_calls, socket(AF_INET, _, 0)).WillByDefault(Return(Api::SysCallIntResult{-1, 0})); ON_CALL(os_sys_calls, socket(AF_INET6, _, 0)).WillByDefault(Return(Api::SysCallIntResult{5, 0})); @@ -726,7 +734,7 @@ drain_type: default EXPECT_CALL(*listener_foo, onDestroy()); } -// Make sure that a listener that is not modifiable cannot be updated or removed. +// Make sure that a listener that is not added_via_api cannot be updated or removed. TEST_F(ListenerManagerImplTest, UpdateRemoveNotModifiableListener) { time_system_.setSystemTime(std::chrono::milliseconds(1001001001001)); @@ -743,7 +751,7 @@ name: foo - filters: [] )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(false); + ListenerHandle* listener_foo = expectListenerCreate(false, false); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", false)); checkStats(1, 0, 0, 0, 1, 0); @@ -816,7 +824,7 @@ name: "foo" filter_chains: {} )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(false); + ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_TRUE( manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "version1", true)); @@ -858,7 +866,7 @@ per_connection_buffer_limit_bytes: 10 time_system_.setSystemTime(std::chrono::milliseconds(2002002002002)); - ListenerHandle* listener_foo_update1 = expectListenerCreate(false); + ListenerHandle* listener_foo_update1 = expectListenerCreate(false, true); EXPECT_CALL(*listener_foo, onDestroy()); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_update1_yaml), "version2", true)); @@ -899,7 +907,7 @@ version_info: version2 // Update foo. Should go into warming, have an immediate warming callback, and start immediate // removal. - ListenerHandle* listener_foo_update2 = expectListenerCreate(false); + ListenerHandle* listener_foo_update2 = expectListenerCreate(false, true); EXPECT_CALL(*worker_, addListener(_, _)); EXPECT_CALL(*worker_, stopListener(_)); EXPECT_CALL(*listener_foo_update1->drain_manager_, startDrainSequence(_)); @@ -958,7 +966,7 @@ name: "bar" filter_chains: {} )EOF"; - ListenerHandle* listener_bar = expectListenerCreate(false); + ListenerHandle* listener_bar = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_CALL(*worker_, addListener(_, _)); EXPECT_TRUE( @@ -979,7 +987,7 @@ name: "baz" filter_chains: {} )EOF"; - ListenerHandle* listener_baz = expectListenerCreate(true); + ListenerHandle* listener_baz = expectListenerCreate(true, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_CALL(listener_baz->target_, initialize()); EXPECT_TRUE( @@ -1045,7 +1053,7 @@ name: baz config: {} )EOF"; - ListenerHandle* listener_baz_update1 = expectListenerCreate(true); + ListenerHandle* listener_baz_update1 = expectListenerCreate(true, true); EXPECT_CALL(*listener_baz, onDestroy()).WillOnce(Invoke([listener_baz]() -> void { // Call the initialize callback during destruction like RDS will. listener_baz->target_.ready(); @@ -1089,7 +1097,7 @@ name: foo new Network::Address::Ipv4Instance("127.0.0.1", 1234)); ON_CALL(*listener_factory_.socket_, localAddress()).WillByDefault(ReturnRef(local_address)); - ListenerHandle* listener_foo = expectListenerCreate(false); + ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_CALL(*worker_, addListener(_, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); @@ -1106,7 +1114,7 @@ name: foo checkStats(1, 0, 1, 0, 0, 1); // Add foo again. We should use the socket from draining. - ListenerHandle* listener_foo2 = expectListenerCreate(false); + ListenerHandle* listener_foo2 = expectListenerCreate(false, true); EXPECT_CALL(*worker_, addListener(_, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); worker_->callAddCompletion(true); @@ -1135,7 +1143,7 @@ name: foo - filters: [] )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(true); + ListenerHandle* listener_foo = expectListenerCreate(true, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)) .WillOnce(Throw(EnvoyException("can't bind"))); EXPECT_CALL(*listener_foo, onDestroy()); @@ -1159,7 +1167,7 @@ name: foo - filters: [] )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(false); + ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_CALL(*worker_, addListener(_, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); @@ -1213,7 +1221,7 @@ name: foo - filters: [] )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(true); + ListenerHandle* listener_foo = expectListenerCreate(true, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); @@ -1227,7 +1235,7 @@ name: foo checkStats(1, 0, 1, 0, 0, 0); // Add foo again and initialize it. - listener_foo = expectListenerCreate(true); + listener_foo = expectListenerCreate(true, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); @@ -1251,7 +1259,7 @@ name: foo config: {} )EOF"; - ListenerHandle* listener_foo_update1 = expectListenerCreate(true); + ListenerHandle* listener_foo_update1 = expectListenerCreate(true, true); EXPECT_CALL(listener_foo_update1->target_, initialize()); EXPECT_TRUE( manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_update1_yaml), "", true)); @@ -1290,7 +1298,7 @@ name: foo - filters: [] )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(false); + ListenerHandle* listener_foo = expectListenerCreate(false, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); EXPECT_CALL(*worker_, addListener(_, _)); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); @@ -1343,7 +1351,7 @@ name: foo - filters: [] )EOF"; - ListenerHandle* listener_foo = expectListenerCreate(true); + ListenerHandle* listener_foo = expectListenerCreate(true, true); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, false)); EXPECT_CALL(listener_foo->target_, initialize()); EXPECT_TRUE(manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_foo_yaml), "", true)); @@ -1361,7 +1369,7 @@ name: bar - filters: [] )EOF"; - ListenerHandle* listener_bar = expectListenerCreate(true); + ListenerHandle* listener_bar = expectListenerCreate(true, true); EXPECT_CALL(*listener_bar, onDestroy()); EXPECT_THROW_WITH_MESSAGE( manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_bar_yaml), "", true), @@ -1373,7 +1381,7 @@ name: bar listener_foo->target_.ready(); worker_->callAddCompletion(true); - listener_bar = expectListenerCreate(true); + listener_bar = expectListenerCreate(true, true); EXPECT_CALL(*listener_bar, onDestroy()); EXPECT_THROW_WITH_MESSAGE( manager_->addOrUpdateListener(parseListenerFromV2Yaml(listener_bar_yaml), "", true), diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index 78c3a8373429..71662bb36bd6 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -76,7 +76,8 @@ TEST_F(OptionsImplTest, All) { "--service-cluster cluster --service-node node --service-zone zone " "--file-flush-interval-msec 9000 " "--drain-time-s 60 --log-format [%v] --parent-shutdown-time-s 90 --log-path /foo/bar " - "--disable-hot-restart --cpuset-threads"); + "--disable-hot-restart --cpuset-threads --allow-unknown-static-fields " + "--reject-unknown-dynamic-fields"); EXPECT_EQ(Server::Mode::Validate, options->mode()); EXPECT_EQ(2U, options->concurrency()); EXPECT_EQ("hello", options->configPath()); @@ -93,14 +94,36 @@ TEST_F(OptionsImplTest, All) { EXPECT_EQ(std::chrono::milliseconds(9000), options->fileFlushIntervalMsec()); EXPECT_EQ(std::chrono::seconds(60), options->drainTime()); EXPECT_EQ(std::chrono::seconds(90), options->parentShutdownTime()); - EXPECT_EQ(true, options->hotRestartDisabled()); - EXPECT_EQ(false, options->libeventBufferEnabled()); - EXPECT_EQ(true, options->cpusetThreadsEnabled()); + EXPECT_TRUE(options->hotRestartDisabled()); + EXPECT_FALSE(options->libeventBufferEnabled()); + EXPECT_TRUE(options->cpusetThreadsEnabled()); + EXPECT_TRUE(options->allowUnknownStaticFields()); + EXPECT_TRUE(options->rejectUnknownDynamicFields()); options = createOptionsImpl("envoy --mode init_only"); EXPECT_EQ(Server::Mode::InitOnly, options->mode()); } +// Either variants of allow-unknown-[static-]-fields works. +TEST_F(OptionsImplTest, AllowUnknownFields) { + { + std::unique_ptr options = createOptionsImpl("envoy"); + EXPECT_FALSE(options->allowUnknownStaticFields()); + } + { + std::unique_ptr options; + EXPECT_LOG_CONTAINS( + "warning", + "--allow-unknown-fields is deprecated, use --allow-unknown-static-fields instead.", + options = createOptionsImpl("envoy --allow-unknown-fields")); + EXPECT_TRUE(options->allowUnknownStaticFields()); + } + { + std::unique_ptr options = createOptionsImpl("envoy --allow-unknown-static-fields"); + EXPECT_TRUE(options->allowUnknownStaticFields()); + } +} + TEST_F(OptionsImplTest, SetAll) { std::unique_ptr options = createOptionsImpl("envoy -c hello"); bool hot_restart_disabled = options->hotRestartDisabled(); @@ -130,6 +153,8 @@ TEST_F(OptionsImplTest, SetAll) { options->setHotRestartDisabled(!options->hotRestartDisabled()); options->setSignalHandling(!options->signalHandlingEnabled()); options->setCpusetThreads(!options->cpusetThreadsEnabled()); + options->setAllowUnkownFields(true); + options->setRejectUnknownFieldsDynamic(true); EXPECT_EQ(109876, options->baseId()); EXPECT_EQ(42U, options->concurrency()); @@ -154,6 +179,8 @@ TEST_F(OptionsImplTest, SetAll) { EXPECT_EQ(!hot_restart_disabled, options->hotRestartDisabled()); EXPECT_EQ(!signal_handling_enabled, options->signalHandlingEnabled()); EXPECT_EQ(!cpuset_threads_enabled, options->cpusetThreadsEnabled()); + EXPECT_TRUE(options->allowUnknownStaticFields()); + EXPECT_TRUE(options->rejectUnknownDynamicFields()); // Validate that CommandLineOptions is constructed correctly. Server::CommandLineOptionsPtr command_line_options = options->toCommandLineOptions(); @@ -190,8 +217,8 @@ TEST_F(OptionsImplTest, DefaultParams) { EXPECT_EQ("", options->adminAddressPath()); EXPECT_EQ(Network::Address::IpVersion::v4, options->localAddressIpVersion()); EXPECT_EQ(Server::Mode::Serve, options->mode()); - EXPECT_EQ(false, options->hotRestartDisabled()); - EXPECT_EQ(false, options->cpusetThreadsEnabled()); + EXPECT_FALSE(options->hotRestartDisabled()); + EXPECT_FALSE(options->cpusetThreadsEnabled()); // Validate that CommandLineOptions is constructed correctly with default params. Server::CommandLineOptionsPtr command_line_options = options->toCommandLineOptions(); @@ -202,8 +229,10 @@ TEST_F(OptionsImplTest, DefaultParams) { EXPECT_EQ(envoy::admin::v2alpha::CommandLineOptions::v4, command_line_options->local_address_ip_version()); EXPECT_EQ(envoy::admin::v2alpha::CommandLineOptions::Serve, command_line_options->mode()); - EXPECT_EQ(false, command_line_options->disable_hot_restart()); - EXPECT_EQ(false, command_line_options->cpuset_threads()); + EXPECT_FALSE(command_line_options->disable_hot_restart()); + EXPECT_FALSE(command_line_options->cpuset_threads()); + EXPECT_FALSE(command_line_options->allow_unknown_static_fields()); + EXPECT_FALSE(command_line_options->reject_unknown_dynamic_fields()); } // Validates that the server_info proto is in sync with the options. @@ -212,12 +241,13 @@ TEST_F(OptionsImplTest, OptionsAreInSyncWithProto) { Server::CommandLineOptionsPtr command_line_options = options->toCommandLineOptions(); // Failure of this condition indicates that the server_info proto is not in sync with the options. // If an option is added/removed, please update server_info proto as well to keep it in sync. - // Currently the following 4 options are not defined in proto, hence the count differs by 5. + // Currently the following 5 options are not defined in proto, hence the count differs by 5. // 1. version - default TCLAP argument. // 2. help - default TCLAP argument. // 3. ignore_rest - default TCLAP argument. // 4. use-libevent-buffers - short-term override for rollout of new buffer implementation. - EXPECT_EQ(options->count() - 4, command_line_options->GetDescriptor()->field_count()); + // 5. allow-unknown-fields - deprecated alias of allow-unknown-static-fields. + EXPECT_EQ(options->count() - 5, command_line_options->GetDescriptor()->field_count()); } TEST_F(OptionsImplTest, BadCliOption) { diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 2edb98e09540..ca2a755d26ac 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -165,10 +165,8 @@ class InitializingInstanceImpl : public InstanceImpl { }; // Class creates minimally viable server instance for testing. -class ServerInstanceImplTest : public testing::TestWithParam { +class ServerInstanceImplTestBase { protected: - ServerInstanceImplTest() : version_(GetParam()) {} - void initialize(const std::string& bootstrap_path) { initialize(bootstrap_path, false); } void initialize(const std::string& bootstrap_path, const bool use_intializing_instance) { @@ -258,6 +256,12 @@ class ServerInstanceImplTest : public testing::TestWithParam server_; }; +class ServerInstanceImplTest : public ServerInstanceImplTestBase, + public testing::TestWithParam { +protected: + ServerInstanceImplTest() { version_ = GetParam(); } +}; + // Custom StatsSink that just increments a counter when flush is called. class CustomStatsSink : public Stats::Sink { public: @@ -443,6 +447,66 @@ TEST_P(ServerInstanceImplTest, Stats) { #endif } +// Default validation mode +TEST_P(ServerInstanceImplTest, ValidationDefault) { + options_.service_cluster_name_ = "some_cluster_name"; + options_.service_node_name_ = "some_node_name"; + EXPECT_NO_THROW(initialize("test/server/empty_bootstrap.yaml")); + EXPECT_THAT_THROWS_MESSAGE( + server_->messageValidationContext().staticValidationVisitor().onUnknownField("foo"), + EnvoyException, "Protobuf message (foo) has unknown fields"); + EXPECT_EQ(0, TestUtility::findCounter(stats_store_, "server.static_unknown_fields")->value()); + EXPECT_NO_THROW( + server_->messageValidationContext().dynamicValidationVisitor().onUnknownField("bar")); + EXPECT_EQ(1, TestUtility::findCounter(stats_store_, "server.dynamic_unknown_fields")->value()); +} + +// Validation mode with --allow-unknown-static-fields +TEST_P(ServerInstanceImplTest, ValidationAllowStatic) { + options_.service_cluster_name_ = "some_cluster_name"; + options_.service_node_name_ = "some_node_name"; + options_.allow_unknown_static_fields_ = true; + EXPECT_NO_THROW(initialize("test/server/empty_bootstrap.yaml")); + EXPECT_NO_THROW( + server_->messageValidationContext().staticValidationVisitor().onUnknownField("foo")); + EXPECT_EQ(1, TestUtility::findCounter(stats_store_, "server.static_unknown_fields")->value()); + EXPECT_NO_THROW( + server_->messageValidationContext().dynamicValidationVisitor().onUnknownField("bar")); + EXPECT_EQ(1, TestUtility::findCounter(stats_store_, "server.dynamic_unknown_fields")->value()); +} + +// Validation mode with --reject-unknown-dynamic-fields +TEST_P(ServerInstanceImplTest, ValidationRejectDynamic) { + options_.service_cluster_name_ = "some_cluster_name"; + options_.service_node_name_ = "some_node_name"; + options_.reject_unknown_dynamic_fields_ = true; + EXPECT_NO_THROW(initialize("test/server/empty_bootstrap.yaml")); + EXPECT_THAT_THROWS_MESSAGE( + server_->messageValidationContext().staticValidationVisitor().onUnknownField("foo"), + EnvoyException, "Protobuf message (foo) has unknown fields"); + EXPECT_EQ(0, TestUtility::findCounter(stats_store_, "server.static_unknown_fields")->value()); + EXPECT_THAT_THROWS_MESSAGE( + server_->messageValidationContext().dynamicValidationVisitor().onUnknownField("bar"), + EnvoyException, "Protobuf message (bar) has unknown fields"); + EXPECT_EQ(0, TestUtility::findCounter(stats_store_, "server.dynamic_unknown_fields")->value()); +} + +// Validation mode with --allow-unknown-static-fields --reject-unknown-dynamic-fields +TEST_P(ServerInstanceImplTest, ValidationAllowStaticRejectDynamic) { + options_.service_cluster_name_ = "some_cluster_name"; + options_.service_node_name_ = "some_node_name"; + options_.allow_unknown_static_fields_ = true; + options_.reject_unknown_dynamic_fields_ = true; + EXPECT_NO_THROW(initialize("test/server/empty_bootstrap.yaml")); + EXPECT_NO_THROW( + server_->messageValidationContext().staticValidationVisitor().onUnknownField("foo")); + EXPECT_EQ(1, TestUtility::findCounter(stats_store_, "server.static_unknown_fields")->value()); + EXPECT_THAT_THROWS_MESSAGE( + server_->messageValidationContext().dynamicValidationVisitor().onUnknownField("bar"), + EnvoyException, "Protobuf message (bar) has unknown fields"); + EXPECT_EQ(0, TestUtility::findCounter(stats_store_, "server.dynamic_unknown_fields")->value()); +} + // Validate server localInfo() from bootstrap Node. TEST_P(ServerInstanceImplTest, BootstrapNode) { initialize("test/server/node_bootstrap.yaml"); @@ -781,6 +845,65 @@ TEST_P(ServerInstanceImplTest, WithProcessContext) { EXPECT_FALSE(object_from_context.boolean_flag_); } +// Static configuration validation. We test with both allow/reject settings various aspects of +// configuration from YAML. +class StaticValidationTest + : public ServerInstanceImplTestBase, + public testing::TestWithParam> { +protected: + StaticValidationTest() { + version_ = std::get<0>(GetParam()); + options_.service_cluster_name_ = "some_cluster_name"; + options_.service_node_name_ = "some_node_name"; + options_.allow_unknown_static_fields_ = std::get<1>(GetParam()); + // By inverting the static validation value, we can hopefully catch places we may have confused + // static/dynamic validation. + options_.reject_unknown_dynamic_fields_ = options_.allow_unknown_static_fields_; + } + + AssertionResult validate(absl::string_view yaml_filename) { + const std::string path = + absl::StrCat("test/server/test_data/static_validation/", yaml_filename); + try { + initialize(path); + } catch (EnvoyException&) { + return options_.allow_unknown_static_fields_ ? AssertionFailure() : AssertionSuccess(); + } + return options_.allow_unknown_static_fields_ ? AssertionSuccess() : AssertionFailure(); + } +}; + +std::string staticValidationTestParamsToString( + const ::testing::TestParamInfo>& params) { + return fmt::format( + "{}_{}", + TestUtility::ipTestParamsToString( + ::testing::TestParamInfo(std::get<0>(params.param), 0)), + std::get<1>(params.param) ? "with_allow_unknown_static_fields" + : "without_allow_unknown_static_fields"); +} + +INSTANTIATE_TEST_SUITE_P( + IpVersions, StaticValidationTest, + testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), testing::Bool()), + staticValidationTestParamsToString); + +TEST_P(StaticValidationTest, BootstrapUnknownField) { + EXPECT_TRUE(validate("bootstrap_unknown_field.yaml")); +} + +TEST_P(StaticValidationTest, ListenerUnknownField) { + EXPECT_TRUE(validate("listener_unknown_field.yaml")); +} + +TEST_P(StaticValidationTest, NetworkFilterUnknownField) { + EXPECT_TRUE(validate("network_filter_unknown_field.yaml")); +} + +TEST_P(StaticValidationTest, ClusterUnknownField) { + EXPECT_TRUE(validate("cluster_unknown_field.yaml")); +} + } // namespace } // namespace Server } // namespace Envoy diff --git a/test/server/test_data/static_validation/bootstrap_unknown_field.yaml b/test/server/test_data/static_validation/bootstrap_unknown_field.yaml new file mode 100644 index 000000000000..20e9ff3feaa8 --- /dev/null +++ b/test/server/test_data/static_validation/bootstrap_unknown_field.yaml @@ -0,0 +1 @@ +foo: bar diff --git a/test/server/test_data/static_validation/cluster_unknown_field.yaml b/test/server/test_data/static_validation/cluster_unknown_field.yaml new file mode 100644 index 000000000000..61675f55ee30 --- /dev/null +++ b/test/server/test_data/static_validation/cluster_unknown_field.yaml @@ -0,0 +1,5 @@ +static_resources: + clusters: + name: foo + connect_timeout: { seconds: 5 } + foo: bar diff --git a/test/server/test_data/static_validation/listener_unknown_field.yaml b/test/server/test_data/static_validation/listener_unknown_field.yaml new file mode 100644 index 000000000000..8dcf743e63ee --- /dev/null +++ b/test/server/test_data/static_validation/listener_unknown_field.yaml @@ -0,0 +1,10 @@ +static_resources: + listeners: + name: foo + address: + socket_address: + address: {{ ntop_ip_loopback_address }} + port_value: 0 + foo: bar + filter_chains: + - filters: diff --git a/test/server/test_data/static_validation/network_filter_unknown_field.yaml b/test/server/test_data/static_validation/network_filter_unknown_field.yaml new file mode 100644 index 000000000000..35450ef00657 --- /dev/null +++ b/test/server/test_data/static_validation/network_filter_unknown_field.yaml @@ -0,0 +1,15 @@ +static_resources: + listeners: + name: foo + address: + socket_address: + address: {{ ntop_ip_loopback_address }} + port_value: 0 + filter_chains: + - filters: + - name: envoy.http_connection_manager + config: + codec_type: HTTP2 + stat_prefix: blah + route_config: {} + foo: bar From 6858df7eef5867b3ae91c4eab6798ccd4490cf11 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Mon, 19 Aug 2019 12:42:28 -0700 Subject: [PATCH 408/542] tls: update BoringSSL to 265728de (3865). (#7952) Signed-off-by: Piotr Sikora --- bazel/repository_locations.bzl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 0c0002c0c657..005d26cd1f23 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -9,11 +9,15 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/bazel-toolchains/archive/0.28.5.tar.gz"], ), boringssl = dict( - # Use commits from branch "chromium-stable-with-bazel" - sha256 = "18edf961f8377e8d10fd8497bc8a331def9cb60a6c2a50a4c8eb322b045042d5", - strip_prefix = "boringssl-87d1c8f292e5184fd727efe84f458d89687d7742", - # chromium-76.0.3809.87 - urls = ["https://github.com/google/boringssl/archive/87d1c8f292e5184fd727efe84f458d89687d7742.tar.gz"], + sha256 = "c712766ddc844de2a38e686e1cdd7288795e9a6fe7f699c6636f1b76703db84e", + strip_prefix = "boringssl-265728decec4370cd02b941f72fba9f0735e2923", + # To update BoringSSL, which tracks Chromium releases: + # 1. Open https://omahaproxy.appspot.com/ and note of linux/beta release. + # 2. Open https://chromium.googlesource.com/chromium/src/+/refs/tags//DEPS and note . + # 3. Find a commit in BoringSSL's "master-with-bazel" branch that merges . + # + # chromium-77.0.3865.35 (BETA) + urls = ["https://github.com/google/boringssl/archive/265728decec4370cd02b941f72fba9f0735e2923.tar.gz"], ), boringssl_fips = dict( sha256 = "b12ad676ee533824f698741bd127f6fbc82c46344398a6d78d25e62c6c418c73", From 869981ff23e79f4a16fbdfd884324ccccf1572a7 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Mon, 19 Aug 2019 14:11:46 -0700 Subject: [PATCH 409/542] test: fix data races in FakeStream. (#7929) Fixes #7927. Risk Level: low (test only) Signed-off-by: Piotr Sikora --- test/integration/fake_upstream.cc | 42 +++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index f55806361987..b30deb3aee70 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -73,22 +73,30 @@ void FakeStream::decodeMetadata(Http::MetadataMapPtr&& metadata_map_ptr) { } void FakeStream::encode100ContinueHeaders(const Http::HeaderMapImpl& headers) { - std::shared_ptr headers_copy( + // TSan complains about thread-safety of std::shared_ptr when linked against libc++. + // See: https://github.com/envoyproxy/envoy/pull/7929 + std::unique_ptr headers_copy( new Http::HeaderMapImpl(static_cast(headers))); - parent_.connection().dispatcher().post( - [this, headers_copy]() -> void { encoder_.encode100ContinueHeaders(*headers_copy); }); + parent_.connection().dispatcher().post([this, headers = headers_copy.release()]() -> void { + encoder_.encode100ContinueHeaders(*headers); + delete headers; + }); } void FakeStream::encodeHeaders(const Http::HeaderMapImpl& headers, bool end_stream) { - std::shared_ptr headers_copy( + // TSan complains about thread-safety of std::shared_ptr when linked against libc++. + // See: https://github.com/envoyproxy/envoy/pull/7929 + std::unique_ptr headers_copy( new Http::HeaderMapImpl(static_cast(headers))); if (add_served_by_header_) { headers_copy->addCopy(Http::LowerCaseString("x-served-by"), parent_.connection().localAddress()->asString()); } - parent_.connection().dispatcher().post([this, headers_copy, end_stream]() -> void { - encoder_.encodeHeaders(*headers_copy, end_stream); - }); + parent_.connection().dispatcher().post( + [this, headers = headers_copy.release(), end_stream]() -> void { + encoder_.encodeHeaders(*headers, end_stream); + delete headers; + }); } void FakeStream::encodeData(absl::string_view data, bool end_stream) { @@ -106,16 +114,24 @@ void FakeStream::encodeData(uint64_t size, bool end_stream) { } void FakeStream::encodeData(Buffer::Instance& data, bool end_stream) { - std::shared_ptr data_copy(new Buffer::OwnedImpl(data)); - parent_.connection().dispatcher().post( - [this, data_copy, end_stream]() -> void { encoder_.encodeData(*data_copy, end_stream); }); + // TSan complains about thread-safety of std::shared_ptr when linked against libc++. + // See: https://github.com/envoyproxy/envoy/pull/7929 + std::unique_ptr data_copy(new Buffer::OwnedImpl(data)); + parent_.connection().dispatcher().post([this, data = data_copy.release(), end_stream]() -> void { + encoder_.encodeData(*data, end_stream); + delete data; + }); } void FakeStream::encodeTrailers(const Http::HeaderMapImpl& trailers) { - std::shared_ptr trailers_copy( + // TSan complains about thread-safety of std::shared_ptr when linked against libc++. + // See: https://github.com/envoyproxy/envoy/pull/7929 + std::unique_ptr trailers_copy( new Http::HeaderMapImpl(static_cast(trailers))); - parent_.connection().dispatcher().post( - [this, trailers_copy]() -> void { encoder_.encodeTrailers(*trailers_copy); }); + parent_.connection().dispatcher().post([this, trailers = trailers_copy.release()]() -> void { + encoder_.encodeTrailers(*trailers); + delete trailers; + }); } void FakeStream::encodeResetStream() { From f90e1b08ac5b4973c45a6529780ebdd211ff901f Mon Sep 17 00:00:00 2001 From: Kuat Date: Mon, 19 Aug 2019 14:28:53 -0700 Subject: [PATCH 410/542] filter: add conditions to access control filter (#7716) Introduces a generic expression-based admission filter using https://github.com/google/cel-cpp. This is a follow-up to discussion in https://github.com/envoyproxy/envoy/issues/6751. The advantage of this approach is: 1. Un-opinionated about the policy structure since the only config is an expression. This is friendly towards control planes which can bear the complexity of translation, analysis, and evolution of policies. 2. Multi-language, CEL supports go, java, and c++ runtimes. 3. Inter-operability with other filters using request `metadata`. Companion filters can populate metadata about requests and resources that affect policy decisions. 4. Generic utility, it can be used for custom metric labels, access log entries, etc. The dis-advantage of this approach is that its performance is lower than domain-optimized interpreters. On a fair example, the interpreter evaluates in around 1ms (see https://github.com/google/cel-cpp/blob/master/eval/tests/benchmark_test.cc#L591) vs ~150ns for hand-written C++ native code. There is space for improvement (especially if WASM can be used as a compilation target), but ultimately the generic expression form carries a cost. Conditions are added to support RBAC filter for complementing the existing principal/permission model. They add support for the extended checks (e.g. time of query, resource-bound), but add no cost unless used. Description: add expression-based admission filter Risk Level: low Testing: Docs Changes: Release Notes: Signed-off-by: Kuat Yessenov --- CODEOWNERS | 2 + api/bazel/api_build_system.bzl | 7 +- api/envoy/config/rbac/v2/BUILD | 10 + api/envoy/config/rbac/v2/rbac.proto | 8 +- bazel/repositories.bzl | 5 + bazel/repository_locations.bzl | 10 + docs/root/intro/version_history.rst | 1 + source/extensions/filters/common/expr/BUILD | 34 ++ .../extensions/filters/common/expr/context.cc | 164 ++++++++ .../extensions/filters/common/expr/context.h | 129 +++++++ .../filters/common/expr/evaluator.cc | 86 +++++ .../filters/common/expr/evaluator.h | 50 +++ source/extensions/filters/common/rbac/BUILD | 1 + .../extensions/filters/common/rbac/engine.h | 12 +- .../filters/common/rbac/engine_impl.cc | 18 +- .../filters/common/rbac/engine_impl.h | 13 +- .../filters/common/rbac/matchers.cc | 33 +- .../extensions/filters/common/rbac/matchers.h | 41 +- .../extensions/filters/common/rbac/utility.h | 12 +- .../filters/http/rbac/rbac_filter.cc | 17 +- .../filters/http/rbac/rbac_filter.h | 20 +- .../filters/network/rbac/rbac_filter.cc | 7 +- .../filters/network/rbac/rbac_filter.h | 9 +- test/extensions/filters/common/expr/BUILD | 25 ++ .../filters/common/expr/context_test.cc | 363 ++++++++++++++++++ test/extensions/filters/common/rbac/BUILD | 1 + .../filters/common/rbac/engine_impl_test.cc | 187 ++++++++- .../filters/common/rbac/matchers_test.cc | 6 +- test/extensions/filters/common/rbac/mocks.h | 7 +- 29 files changed, 1188 insertions(+), 90 deletions(-) create mode 100644 source/extensions/filters/common/expr/BUILD create mode 100644 source/extensions/filters/common/expr/context.cc create mode 100644 source/extensions/filters/common/expr/context.h create mode 100644 source/extensions/filters/common/expr/evaluator.cc create mode 100644 source/extensions/filters/common/expr/evaluator.h create mode 100644 test/extensions/filters/common/expr/BUILD create mode 100644 test/extensions/filters/common/expr/context_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index a6ef0e4aa0dc..72efd16d0562 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -51,3 +51,5 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/http/adaptive_concurrency @tonya11en @mattklein123 # http inspector /*/extensions/filters/listener/http_inspector @crazyxy @PiotrSikora @lizan +# attribute context +/*/extensions/filters/common/expr @kyessenov @yangminzhu diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index 9eb8e434a531..d9a3e2d943e6 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -23,13 +23,13 @@ def _LibrarySuffix(library_name, suffix): # TODO(htuch): Convert this to native py_proto_library once # https://github.com/bazelbuild/bazel/issues/3935 and/or # https://github.com/bazelbuild/bazel/issues/2626 are resolved. -def api_py_proto_library(name, srcs = [], deps = [], has_services = 0): +def api_py_proto_library(name, srcs = [], deps = [], external_py_proto_deps = [], has_services = 0): _py_proto_library( name = _Suffix(name, _PY_SUFFIX), srcs = srcs, default_runtime = "@com_google_protobuf//:protobuf_python", protoc = "@com_google_protobuf//:protoc", - deps = [_LibrarySuffix(d, _PY_SUFFIX) for d in deps] + [ + deps = [_LibrarySuffix(d, _PY_SUFFIX) for d in deps] + external_py_proto_deps + [ "@com_envoyproxy_protoc_gen_validate//validate:validate_py", "@com_google_googleapis//google/rpc:status_py_proto", "@com_google_googleapis//google/api:annotations_py_proto", @@ -116,6 +116,7 @@ def api_proto_library( deps = [], external_proto_deps = [], external_cc_proto_deps = [], + external_py_proto_deps = [], has_services = 0, linkstatic = None, require_py = 1): @@ -152,7 +153,7 @@ def api_proto_library( ) py_export_suffixes = [] if (require_py == 1): - api_py_proto_library(name, srcs, deps, has_services) + api_py_proto_library(name, srcs, deps, external_py_proto_deps, has_services) py_export_suffixes = ["_py", "_py_genproto"] # Allow unlimited visibility for consumers diff --git a/api/envoy/config/rbac/v2/BUILD b/api/envoy/config/rbac/v2/BUILD index c2059893912c..fac50eb66f9b 100644 --- a/api/envoy/config/rbac/v2/BUILD +++ b/api/envoy/config/rbac/v2/BUILD @@ -5,6 +5,15 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_prot api_proto_library_internal( name = "rbac", srcs = ["rbac.proto"], + external_cc_proto_deps = [ + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_cc_proto", + ], + external_proto_deps = [ + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", + ], + external_py_proto_deps = [ + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_py_proto", + ], visibility = ["//visibility:public"], deps = [ "//envoy/api/v2/core:address", @@ -22,5 +31,6 @@ api_go_proto_library( "//envoy/api/v2/route:route_go_proto", "//envoy/type/matcher:metadata_go_proto", "//envoy/type/matcher:string_go_proto", + "@com_google_googleapis//google/api/expr/v1alpha1:cel_go_proto", ], ) diff --git a/api/envoy/config/rbac/v2/rbac.proto b/api/envoy/config/rbac/v2/rbac.proto index 77e1aa687fa9..15554e561df4 100644 --- a/api/envoy/config/rbac/v2/rbac.proto +++ b/api/envoy/config/rbac/v2/rbac.proto @@ -7,6 +7,8 @@ import "envoy/api/v2/route/route.proto"; import "envoy/type/matcher/metadata.proto"; import "envoy/type/matcher/string.proto"; +import "google/api/expr/v1alpha1/syntax.proto"; + package envoy.config.rbac.v2; option java_outer_classname = "RbacProto"; @@ -81,7 +83,7 @@ message RBAC { // Policy specifies a role and the principals that are assigned/denied the role. A policy matches if // and only if at least one of its permissions match the action taking place AND at least one of its -// principals match the downstream. +// principals match the downstream AND the condition is true if specified. message Policy { // Required. The set of permissions that define a role. Each permission is matched with OR // semantics. To match all actions for this policy, a single Permission with the `any` field set @@ -92,6 +94,10 @@ message Policy { // principal is matched with OR semantics. To match all downstreams for this policy, a single // Principal with the `any` field set to true should be used. repeated Principal principals = 2 [(validate.rules).repeated .min_items = 1]; + + // An optional symbolic expression specifying an access control condition. + // The condition is combined with AND semantics. + google.api.expr.v1alpha1.Expr condition = 3; } // Permission defines an action (or actions) that a principal can take. diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index d0df39d541bd..319d374286bd 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -153,6 +153,8 @@ def envoy_dependencies(skip_targets = []): _com_lightstep_tracer_cpp() _io_opentracing_cpp() _net_zlib() + _repository_impl("com_googlesource_code_re2") + _com_google_cel_cpp() _repository_impl("bazel_toolchains") _python_deps() @@ -315,6 +317,9 @@ def _net_zlib(): actual = "@envoy//bazel/foreign_cc:zlib", ) +def _com_google_cel_cpp(): + _repository_impl("com_google_cel_cpp") + def _com_github_nghttp2_nghttp2(): location = REPOSITORY_LOCATIONS["com_github_nghttp2_nghttp2"] http_archive( diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 005d26cd1f23..ebb3e8569064 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -247,4 +247,14 @@ REPOSITORY_LOCATIONS = dict( sha256 = "fcdebf54c89d839ffa7eefae166c8e4b551c765559db13ff15bff98047f344fb", urls = ["https://storage.googleapis.com/quiche-envoy-integration/2a930469533c3b541443488a629fe25cd8ff53d0.tar.gz"], ), + com_google_cel_cpp = dict( + sha256 = "f027c551d57d38fb9f0b5e4f21a2b0b8663987119e23b1fd8dfcc7588e9a2350", + strip_prefix = "cel-cpp-d9d02b20ab85da2444dbdd03410bac6822141364", + urls = ["https://github.com/google/cel-cpp/archive/d9d02b20ab85da2444dbdd03410bac6822141364.tar.gz"], + ), + com_googlesource_code_re2 = dict( + sha256 = "f31db9cd224d018a7e4fe88ef84aaa874b0b3ed91d4d98ee5a1531101d3fdc64", + strip_prefix = "re2-87e2ad45e7b18738e1551474f7ee5886ff572059", + urls = ["https://github.com/google/re2/archive/87e2ad45e7b18738e1551474f7ee5886ff572059.tar.gz"], + ), ) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ca60b5387630..83998544fdf1 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -31,6 +31,7 @@ Version history * rbac: added support for DNS SAN as :ref:`principal_name `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). +* rbac: added conditions to the policy, see :ref:`condition `. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * router check tool: add coverage reporting & enforcement. * router check tool: add comprehensive coverage reporting. diff --git a/source/extensions/filters/common/expr/BUILD b/source/extensions/filters/common/expr/BUILD new file mode 100644 index 000000000000..7b2a6f140792 --- /dev/null +++ b/source/extensions/filters/common/expr/BUILD @@ -0,0 +1,34 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "evaluator_lib", + srcs = ["evaluator.cc"], + hdrs = ["evaluator.h"], + deps = [ + ":context_lib", + "//source/common/http:utility_lib", + "//source/common/protobuf", + "@com_google_cel_cpp//eval/public:builtin_func_registrar", + "@com_google_cel_cpp//eval/public:cel_expr_builder_factory", + "@com_google_cel_cpp//eval/public:cel_expression", + "@com_google_cel_cpp//eval/public:cel_value", + ], +) + +envoy_cc_library( + name = "context_lib", + srcs = ["context.cc"], + hdrs = ["context.h"], + deps = [ + "//source/common/http:utility_lib", + "@com_google_cel_cpp//eval/public:cel_value", + ], +) diff --git a/source/extensions/filters/common/expr/context.cc b/source/extensions/filters/common/expr/context.cc new file mode 100644 index 000000000000..5ee1e91e69b3 --- /dev/null +++ b/source/extensions/filters/common/expr/context.cc @@ -0,0 +1,164 @@ +#include "extensions/filters/common/expr/context.h" + +#include "absl/strings/numbers.h" +#include "absl/time/time.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Expr { + +namespace { + +absl::optional convertHeaderEntry(const Http::HeaderEntry* header) { + if (header == nullptr) { + return {}; + } + return CelValue::CreateString(header->value().getStringView()); +} + +} // namespace + +absl::optional HeadersWrapper::operator[](CelValue key) const { + if (value_ == nullptr || !key.IsString()) { + return {}; + } + auto out = value_->get(Http::LowerCaseString(std::string(key.StringOrDie().value()))); + return convertHeaderEntry(out); +} + +absl::optional RequestWrapper::operator[](CelValue key) const { + if (!key.IsString()) { + return {}; + } + auto value = key.StringOrDie().value(); + + if (value == Headers) { + return CelValue::CreateMap(&headers_); + } else if (value == Time) { + return CelValue::CreateTimestamp(absl::FromChrono(info_.startTime())); + } else if (value == Size) { + // it is important to make a choice whether to rely on content-length vs stream info + // (which is not available at the time of the request headers) + if (headers_.value_ != nullptr && headers_.value_->ContentLength() != nullptr) { + int64_t length; + if (absl::SimpleAtoi(headers_.value_->ContentLength()->value().getStringView(), &length)) { + return CelValue::CreateInt64(length); + } + } else { + return CelValue::CreateInt64(info_.bytesReceived()); + } + } else if (value == Duration) { + auto duration = info_.requestComplete(); + if (duration.has_value()) { + return CelValue::CreateDuration(absl::FromChrono(duration.value())); + } + } + + if (headers_.value_ != nullptr) { + if (value == Path) { + return convertHeaderEntry(headers_.value_->Path()); + } else if (value == UrlPath) { + absl::string_view path = headers_.value_->Path()->value().getStringView(); + size_t query_offset = path.find('?'); + if (query_offset == absl::string_view::npos) { + return CelValue::CreateString(path); + } + return CelValue::CreateString(path.substr(0, query_offset)); + } else if (value == Host) { + return convertHeaderEntry(headers_.value_->Host()); + } else if (value == Scheme) { + return convertHeaderEntry(headers_.value_->Scheme()); + } else if (value == Method) { + return convertHeaderEntry(headers_.value_->Method()); + } else if (value == Referer) { + return convertHeaderEntry(headers_.value_->Referer()); + } else if (value == ID) { + return convertHeaderEntry(headers_.value_->RequestId()); + } else if (value == UserAgent) { + return convertHeaderEntry(headers_.value_->UserAgent()); + } else if (value == TotalSize) { + return CelValue::CreateInt64(info_.bytesReceived() + headers_.value_->byteSize()); + } + } + return {}; +} + +absl::optional ResponseWrapper::operator[](CelValue key) const { + if (!key.IsString()) { + return {}; + } + auto value = key.StringOrDie().value(); + if (value == Code) { + auto code = info_.responseCode(); + if (code.has_value()) { + return CelValue::CreateInt64(code.value()); + } + } else if (value == Size) { + return CelValue::CreateInt64(info_.bytesSent()); + } else if (value == Headers) { + return CelValue::CreateMap(&headers_); + } else if (value == Trailers) { + return CelValue::CreateMap(&trailers_); + } + return {}; +} + +absl::optional ConnectionWrapper::operator[](CelValue key) const { + if (!key.IsString()) { + return {}; + } + auto value = key.StringOrDie().value(); + if (value == UpstreamAddress) { + auto upstream_host = info_.upstreamHost(); + if (upstream_host != nullptr && upstream_host->address() != nullptr) { + return CelValue::CreateString(upstream_host->address()->asStringView()); + } + } else if (value == UpstreamPort) { + auto upstream_host = info_.upstreamHost(); + if (upstream_host != nullptr && upstream_host->address() != nullptr && + upstream_host->address()->ip() != nullptr) { + return CelValue::CreateInt64(upstream_host->address()->ip()->port()); + } + } else if (value == MTLS) { + return CelValue::CreateBool(info_.downstreamSslConnection() != nullptr && + info_.downstreamSslConnection()->peerCertificatePresented()); + } else if (value == RequestedServerName) { + return CelValue::CreateString(info_.requestedServerName()); + } + + return {}; +} + +absl::optional PeerWrapper::operator[](CelValue key) const { + if (!key.IsString()) { + return {}; + } + auto value = key.StringOrDie().value(); + if (value == Address) { + if (local_) { + return CelValue::CreateString(info_.downstreamLocalAddress()->asStringView()); + } else { + return CelValue::CreateString(info_.downstreamRemoteAddress()->asStringView()); + } + } else if (value == Port) { + if (local_) { + if (info_.downstreamLocalAddress()->ip() != nullptr) { + return CelValue::CreateInt64(info_.downstreamLocalAddress()->ip()->port()); + } + } else { + if (info_.downstreamRemoteAddress()->ip() != nullptr) { + return CelValue::CreateInt64(info_.downstreamRemoteAddress()->ip()->port()); + } + } + } + + return {}; +} + +} // namespace Expr +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/common/expr/context.h b/source/extensions/filters/common/expr/context.h new file mode 100644 index 000000000000..7b59c41a5a10 --- /dev/null +++ b/source/extensions/filters/common/expr/context.h @@ -0,0 +1,129 @@ +#pragma once + +#include "envoy/stream_info/stream_info.h" + +#include "common/http/headers.h" + +#include "eval/public/cel_value.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Expr { + +using CelValue = google::api::expr::runtime::CelValue; + +// Symbols for traversing the request properties +constexpr absl::string_view Request = "request"; +constexpr absl::string_view Path = "path"; +constexpr absl::string_view UrlPath = "url_path"; +constexpr absl::string_view Host = "host"; +constexpr absl::string_view Scheme = "scheme"; +constexpr absl::string_view Method = "method"; +constexpr absl::string_view Referer = "referer"; +constexpr absl::string_view Headers = "headers"; +constexpr absl::string_view Time = "time"; +constexpr absl::string_view ID = "id"; +constexpr absl::string_view UserAgent = "useragent"; +constexpr absl::string_view Size = "size"; +constexpr absl::string_view TotalSize = "total_size"; +constexpr absl::string_view Duration = "duration"; + +// Symbols for traversing the response properties +constexpr absl::string_view Response = "response"; +constexpr absl::string_view Code = "code"; +constexpr absl::string_view Trailers = "trailers"; + +// Per-request or per-connection metadata +constexpr absl::string_view Metadata = "metadata"; + +// Connection properties +constexpr absl::string_view Connection = "connection"; +constexpr absl::string_view UpstreamAddress = "upstream_address"; +constexpr absl::string_view UpstreamPort = "upstream_port"; +constexpr absl::string_view MTLS = "mtls"; +constexpr absl::string_view RequestedServerName = "requested_server_name"; + +// Source properties +constexpr absl::string_view Source = "source"; +constexpr absl::string_view Address = "address"; +constexpr absl::string_view Port = "port"; + +// Destination properties +constexpr absl::string_view Destination = "destination"; + +class RequestWrapper; + +class HeadersWrapper : public google::api::expr::runtime::CelMap { +public: + HeadersWrapper(const Http::HeaderMap* value) : value_(value) {} + absl::optional operator[](CelValue key) const override; + int size() const override { return value_ == nullptr ? 0 : value_->size(); } + bool empty() const override { return value_ == nullptr ? true : value_->empty(); } + const google::api::expr::runtime::CelList* ListKeys() const override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + +private: + friend class RequestWrapper; + const Http::HeaderMap* value_; +}; + +class BaseWrapper : public google::api::expr::runtime::CelMap { +public: + int size() const override { return 0; } + bool empty() const override { return false; } + const google::api::expr::runtime::CelList* ListKeys() const override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } +}; + +class RequestWrapper : public BaseWrapper { +public: + RequestWrapper(const Http::HeaderMap* headers, const StreamInfo::StreamInfo& info) + : headers_(headers), info_(info) {} + absl::optional operator[](CelValue key) const override; + +private: + const HeadersWrapper headers_; + const StreamInfo::StreamInfo& info_; +}; + +class ResponseWrapper : public BaseWrapper { +public: + ResponseWrapper(const Http::HeaderMap* headers, const Http::HeaderMap* trailers, + const StreamInfo::StreamInfo& info) + : headers_(headers), trailers_(trailers), info_(info) {} + absl::optional operator[](CelValue key) const override; + +private: + const HeadersWrapper headers_; + const HeadersWrapper trailers_; + const StreamInfo::StreamInfo& info_; +}; + +class ConnectionWrapper : public BaseWrapper { +public: + ConnectionWrapper(const StreamInfo::StreamInfo& info) : info_(info) {} + absl::optional operator[](CelValue key) const override; + +private: + const StreamInfo::StreamInfo& info_; +}; + +class PeerWrapper : public BaseWrapper { +public: + PeerWrapper(const StreamInfo::StreamInfo& info, bool local) : info_(info), local_(local) {} + absl::optional operator[](CelValue key) const override; + +private: + const StreamInfo::StreamInfo& info_; + const bool local_; +}; + +} // namespace Expr +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/common/expr/evaluator.cc b/source/extensions/filters/common/expr/evaluator.cc new file mode 100644 index 000000000000..bd25da52975f --- /dev/null +++ b/source/extensions/filters/common/expr/evaluator.cc @@ -0,0 +1,86 @@ +#include "extensions/filters/common/expr/evaluator.h" + +#include "envoy/common/exception.h" + +#include "eval/public/builtin_func_registrar.h" +#include "eval/public/cel_expr_builder_factory.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Expr { + +BuilderPtr createBuilder(Protobuf::Arena* arena) { + google::api::expr::runtime::InterpreterOptions options; + + // Conformance with spec/go runtimes requires this setting + options.partial_string_match = true; + + if (arena != nullptr) { + options.constant_folding = true; + options.constant_arena = arena; + } + + auto builder = google::api::expr::runtime::CreateCelExpressionBuilder(options); + auto register_status = + google::api::expr::runtime::RegisterBuiltinFunctions(builder->GetRegistry()); + if (!register_status.ok()) { + throw EnvoyException( + absl::StrCat("failed to register built-in functions: ", register_status.message())); + } + return builder; +} + +ExpressionPtr createExpression(Builder& builder, const google::api::expr::v1alpha1::Expr& expr) { + google::api::expr::v1alpha1::SourceInfo source_info; + auto cel_expression_status = builder.CreateExpression(&expr, &source_info); + if (!cel_expression_status.ok()) { + throw EnvoyException( + absl::StrCat("failed to create an expression: ", cel_expression_status.status().message())); + } + return std::move(cel_expression_status.ValueOrDie()); +} + +absl::optional evaluate(const Expression& expr, Protobuf::Arena* arena, + const StreamInfo::StreamInfo& info, + const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers) { + google::api::expr::runtime::Activation activation; + const RequestWrapper request(request_headers, info); + const ResponseWrapper response(response_headers, response_trailers, info); + const ConnectionWrapper connection(info); + const PeerWrapper source(info, false); + const PeerWrapper destination(info, true); + activation.InsertValue(Request, CelValue::CreateMap(&request)); + activation.InsertValue(Response, CelValue::CreateMap(&response)); + activation.InsertValue(Metadata, CelValue::CreateMessage(&info.dynamicMetadata(), arena)); + activation.InsertValue(Connection, CelValue::CreateMap(&connection)); + activation.InsertValue(Source, CelValue::CreateMap(&source)); + activation.InsertValue(Destination, CelValue::CreateMap(&destination)); + + auto eval_status = expr.Evaluate(activation, arena); + if (!eval_status.ok()) { + return {}; + } + + return eval_status.ValueOrDie(); +} + +bool matches(const Expression& expr, const StreamInfo::StreamInfo& info, + const Http::HeaderMap& headers) { + Protobuf::Arena arena; + auto eval_status = Expr::evaluate(expr, &arena, info, &headers, nullptr, nullptr); + if (!eval_status.has_value()) { + return false; + } + auto result = eval_status.value(); + return result.IsBool() ? result.BoolOrDie() : false; +} + +} // namespace Expr +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/common/expr/evaluator.h b/source/extensions/filters/common/expr/evaluator.h new file mode 100644 index 000000000000..92ccea420d21 --- /dev/null +++ b/source/extensions/filters/common/expr/evaluator.h @@ -0,0 +1,50 @@ +#pragma once + +#include "envoy/stream_info/stream_info.h" + +#include "common/http/headers.h" +#include "common/protobuf/protobuf.h" + +#include "extensions/filters/common/expr/context.h" + +#include "eval/public/cel_expression.h" +#include "eval/public/cel_value.h" + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Expr { + +using Builder = google::api::expr::runtime::CelExpressionBuilder; +using BuilderPtr = std::unique_ptr; +using Expression = google::api::expr::runtime::CelExpression; +using ExpressionPtr = std::unique_ptr; + +// Creates an expression builder. The optional arena is used to enable constant folding +// for intermediate evaluation results. +// Throws an exception if fails to construct an expression builder. +BuilderPtr createBuilder(Protobuf::Arena* arena); + +// Creates an interpretable expression from a protobuf representation. +// Throws an exception if fails to construct a runtime expression. +ExpressionPtr createExpression(Builder& builder, const google::api::expr::v1alpha1::Expr& expr); + +// Evaluates an expression for a request. The arena is used to hold intermediate computational +// results and potentially the final value. +absl::optional evaluate(const Expression& expr, Protobuf::Arena* arena, + const StreamInfo::StreamInfo& info, + const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers); + +// Evaluates an expression and returns true if the expression evaluates to "true". +// Returns false if the expression fails to evaluate. +bool matches(const Expression& expr, const StreamInfo::StreamInfo& info, + const Http::HeaderMap& headers); + +} // namespace Expr +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/common/rbac/BUILD b/source/extensions/filters/common/rbac/BUILD index 6aa4fc4088bd..94398324482f 100644 --- a/source/extensions/filters/common/rbac/BUILD +++ b/source/extensions/filters/common/rbac/BUILD @@ -33,6 +33,7 @@ envoy_cc_library( "//source/common/common:matchers_lib", "//source/common/http:header_utility_lib", "//source/common/network:cidr_range_lib", + "//source/extensions/filters/common/expr:evaluator_lib", "@envoy_api//envoy/api/v2/core:base_cc", "@envoy_api//envoy/config/rbac/v2:rbac_cc", ], diff --git a/source/extensions/filters/common/rbac/engine.h b/source/extensions/filters/common/rbac/engine.h index 4a07c03d2a31..093eb72fb778 100644 --- a/source/extensions/filters/common/rbac/engine.h +++ b/source/extensions/filters/common/rbac/engine.h @@ -4,6 +4,7 @@ #include "envoy/http/filter.h" #include "envoy/http/header_map.h" #include "envoy/network/connection.h" +#include "envoy/stream_info/stream_info.h" namespace Envoy { namespace Extensions { @@ -24,24 +25,25 @@ class RoleBasedAccessControlEngine { * @param connection the downstream connection used to identify the action/principal. * @param headers the headers of the incoming request used to identify the action/principal. An * empty map should be used if there are no headers available. - * @param metadata the metadata with additional information about the action/principal. + * @param info the per-request or per-connection stream info with additional information + * about the action/principal. * @param effective_policy_id it will be filled by the matching policy's ID, * which is used to identity the source of the allow/deny. */ virtual bool allowed(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata, + const StreamInfo::StreamInfo& info, std::string* effective_policy_id) const PURE; /** * Returns whether or not the current action is permitted. * * @param connection the downstream connection used to identify the action/principal. - * @param metadata the metadata with additional information about the action/principal. + * @param info the per-request or per-connection stream info with additional information + * about the action/principal. * @param effective_policy_id it will be filled by the matching policy's ID, * which is used to identity the source of the allow/deny. */ - virtual bool allowed(const Network::Connection& connection, - const envoy::api::v2::core::Metadata& metadata, + virtual bool allowed(const Network::Connection& connection, const StreamInfo::StreamInfo& info, std::string* effective_policy_id) const PURE; }; diff --git a/source/extensions/filters/common/rbac/engine_impl.cc b/source/extensions/filters/common/rbac/engine_impl.cc index a35697d74d5f..4456dd6eafbb 100644 --- a/source/extensions/filters/common/rbac/engine_impl.cc +++ b/source/extensions/filters/common/rbac/engine_impl.cc @@ -12,19 +12,27 @@ RoleBasedAccessControlEngineImpl::RoleBasedAccessControlEngineImpl( const envoy::config::rbac::v2::RBAC& rules) : allowed_if_matched_(rules.action() == envoy::config::rbac::v2::RBAC_Action::RBAC_Action_ALLOW) { + // guard expression builder by presence of a condition in policies for (const auto& policy : rules.policies()) { - policies_.insert(std::make_pair(policy.first, policy.second)); + if (policy.second.has_condition()) { + builder_ = Expr::createBuilder(&constant_arena_); + break; + } + } + + for (const auto& policy : rules.policies()) { + policies_.emplace(policy.first, std::make_unique(policy.second, builder_.get())); } } bool RoleBasedAccessControlEngineImpl::allowed(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata, + const StreamInfo::StreamInfo& info, std::string* effective_policy_id) const { bool matched = false; for (const auto& policy : policies_) { - if (policy.second.matches(connection, headers, metadata)) { + if (policy.second->matches(connection, headers, info)) { matched = true; if (effective_policy_id != nullptr) { *effective_policy_id = policy.first; @@ -40,10 +48,10 @@ bool RoleBasedAccessControlEngineImpl::allowed(const Network::Connection& connec } bool RoleBasedAccessControlEngineImpl::allowed(const Network::Connection& connection, - const envoy::api::v2::core::Metadata& metadata, + const StreamInfo::StreamInfo& info, std::string* effective_policy_id) const { static const Http::HeaderMapImpl* empty_header = new Http::HeaderMapImpl(); - return allowed(connection, *empty_header, metadata, effective_policy_id); + return allowed(connection, *empty_header, info, effective_policy_id); } } // namespace RBAC diff --git a/source/extensions/filters/common/rbac/engine_impl.h b/source/extensions/filters/common/rbac/engine_impl.h index cabbeb31e30f..43b71fe5d8b0 100644 --- a/source/extensions/filters/common/rbac/engine_impl.h +++ b/source/extensions/filters/common/rbac/engine_impl.h @@ -11,22 +11,23 @@ namespace Filters { namespace Common { namespace RBAC { -class RoleBasedAccessControlEngineImpl : public RoleBasedAccessControlEngine { +class RoleBasedAccessControlEngineImpl : public RoleBasedAccessControlEngine, NonCopyable { public: RoleBasedAccessControlEngineImpl(const envoy::config::rbac::v2::RBAC& rules); bool allowed(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata, - std::string* effective_policy_id) const override; + const StreamInfo::StreamInfo& info, std::string* effective_policy_id) const override; - bool allowed(const Network::Connection& connection, - const envoy::api::v2::core::Metadata& metadata, + bool allowed(const Network::Connection& connection, const StreamInfo::StreamInfo& info, std::string* effective_policy_id) const override; private: const bool allowed_if_matched_; - std::map policies_; + std::map> policies_; + + Protobuf::Arena constant_arena_; + Expr::BuilderPtr builder_; }; } // namespace RBAC diff --git a/source/extensions/filters/common/rbac/matchers.cc b/source/extensions/filters/common/rbac/matchers.cc index 07472b49a003..d2742e2e9ea1 100644 --- a/source/extensions/filters/common/rbac/matchers.cc +++ b/source/extensions/filters/common/rbac/matchers.cc @@ -70,9 +70,9 @@ AndMatcher::AndMatcher(const envoy::config::rbac::v2::Principal_Set& set) { bool AndMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata) const { + const StreamInfo::StreamInfo& info) const { for (const auto& matcher : matchers_) { - if (!matcher->matches(connection, headers, metadata)) { + if (!matcher->matches(connection, headers, info)) { return false; } } @@ -95,9 +95,9 @@ OrMatcher::OrMatcher(const Protobuf::RepeatedPtrField<::envoy::config::rbac::v2: bool OrMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata) const { + const StreamInfo::StreamInfo& info) const { for (const auto& matcher : matchers_) { - if (matcher->matches(connection, headers, metadata)) { + if (matcher->matches(connection, headers, info)) { return true; } } @@ -107,17 +107,17 @@ bool OrMatcher::matches(const Network::Connection& connection, bool NotMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata) const { - return !matcher_->matches(connection, headers, metadata); + const StreamInfo::StreamInfo& info) const { + return !matcher_->matches(connection, headers, info); } bool HeaderMatcher::matches(const Network::Connection&, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const { + const StreamInfo::StreamInfo&) const { return Envoy::Http::HeaderUtility::matchHeaders(headers, header_); } bool IPMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap&, - const envoy::api::v2::core::Metadata&) const { + const StreamInfo::StreamInfo&) const { const Envoy::Network::Address::InstanceConstSharedPtr& ip = destination_ ? connection.localAddress() : connection.remoteAddress(); @@ -125,14 +125,14 @@ bool IPMatcher::matches(const Network::Connection& connection, const Envoy::Http } bool PortMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap&, - const envoy::api::v2::core::Metadata&) const { + const StreamInfo::StreamInfo&) const { const Envoy::Network::Address::Ip* ip = connection.localAddress().get()->ip(); return ip && ip->port() == port_; } bool AuthenticatedMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap&, - const envoy::api::v2::core::Metadata&) const { + const StreamInfo::StreamInfo&) const { const auto* ssl = connection.ssl(); if (!ssl) { // connection was not authenticated return false; @@ -159,20 +159,21 @@ bool AuthenticatedMatcher::matches(const Network::Connection& connection, } bool MetadataMatcher::matches(const Network::Connection&, const Envoy::Http::HeaderMap&, - const envoy::api::v2::core::Metadata& metadata) const { - return matcher_.match(metadata); + const StreamInfo::StreamInfo& info) const { + return matcher_.match(info.dynamicMetadata()); } bool PolicyMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata) const { - return permissions_.matches(connection, headers, metadata) && - principals_.matches(connection, headers, metadata); + const StreamInfo::StreamInfo& info) const { + return permissions_.matches(connection, headers, info) && + principals_.matches(connection, headers, info) && + (expr_ == nullptr ? true : Expr::matches(*expr_, info, headers)); } bool RequestedServerNameMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap&, - const envoy::api::v2::core::Metadata&) const { + const StreamInfo::StreamInfo&) const { return match(connection.requestedServerName()); } diff --git a/source/extensions/filters/common/rbac/matchers.h b/source/extensions/filters/common/rbac/matchers.h index 28b81846e114..98f369d49e20 100644 --- a/source/extensions/filters/common/rbac/matchers.h +++ b/source/extensions/filters/common/rbac/matchers.h @@ -11,6 +11,8 @@ #include "common/http/header_utility.h" #include "common/network/cidr_range.h" +#include "extensions/filters/common/expr/evaluator.h" + namespace Envoy { namespace Extensions { namespace Filters { @@ -36,7 +38,7 @@ class Matcher { * @param metadata the additional information about the action/principal. */ virtual bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata) const PURE; + const StreamInfo::StreamInfo& info) const PURE; /** * Creates a shared instance of a matcher based off the rules defined in the Permission config @@ -57,7 +59,7 @@ class Matcher { class AlwaysMatcher : public Matcher { public: bool matches(const Network::Connection&, const Envoy::Http::HeaderMap&, - const envoy::api::v2::core::Metadata&) const override { + const StreamInfo::StreamInfo&) const override { return true; } }; @@ -72,7 +74,7 @@ class AndMatcher : public Matcher { AndMatcher(const envoy::config::rbac::v2::Principal_Set& ids); bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; private: std::vector matchers_; @@ -90,7 +92,7 @@ class OrMatcher : public Matcher { OrMatcher(const Protobuf::RepeatedPtrField<::envoy::config::rbac::v2::Principal>& ids); bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; private: std::vector matchers_; @@ -104,7 +106,7 @@ class NotMatcher : public Matcher { : matcher_(Matcher::create(principal)) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; private: MatcherConstSharedPtr matcher_; @@ -119,7 +121,7 @@ class HeaderMatcher : public Matcher { HeaderMatcher(const envoy::api::v2::route::HeaderMatcher& matcher) : header_(matcher) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; private: const Envoy::Http::HeaderUtility::HeaderData header_; @@ -135,7 +137,7 @@ class IPMatcher : public Matcher { : range_(Network::Address::CidrRange::create(range)), destination_(destination) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; private: const Network::Address::CidrRange range_; @@ -150,7 +152,7 @@ class PortMatcher : public Matcher { PortMatcher(const uint32_t port) : port_(port) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; private: const uint32_t port_; @@ -168,7 +170,7 @@ class AuthenticatedMatcher : public Matcher { : absl::nullopt) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; private: const absl::optional matcher_; @@ -177,18 +179,27 @@ class AuthenticatedMatcher : public Matcher { /** * Matches a Policy which is a collection of permission and principal matchers. If any action * matches a permission, the principals are then checked for a match. + * The condition is a conjunction clause. */ -class PolicyMatcher : public Matcher { +class PolicyMatcher : public Matcher, NonCopyable { public: - PolicyMatcher(const envoy::config::rbac::v2::Policy& policy) - : permissions_(policy.permissions()), principals_(policy.principals()) {} + PolicyMatcher(const envoy::config::rbac::v2::Policy& policy, Expr::Builder* builder) + : permissions_(policy.permissions()), principals_(policy.principals()), + condition_(policy.condition()) { + if (policy.has_condition()) { + expr_ = Expr::createExpression(*builder, condition_); + } + } bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; private: const OrMatcher permissions_; const OrMatcher principals_; + + const google::api::expr::v1alpha1::Expr condition_; + Expr::ExpressionPtr expr_; }; class MetadataMatcher : public Matcher { @@ -196,7 +207,7 @@ class MetadataMatcher : public Matcher { MetadataMatcher(const Envoy::Matchers::MetadataMatcher& matcher) : matcher_(matcher) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata& metadata) const override; + const StreamInfo::StreamInfo& info) const override; private: const Envoy::Matchers::MetadataMatcher matcher_; @@ -212,7 +223,7 @@ class RequestedServerNameMatcher : public Matcher, Envoy::Matchers::StringMatche : Envoy::Matchers::StringMatcher(requested_server_name) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, - const envoy::api::v2::core::Metadata&) const override; + const StreamInfo::StreamInfo&) const override; }; } // namespace RBAC diff --git a/source/extensions/filters/common/rbac/utility.h b/source/extensions/filters/common/rbac/utility.h index bcf934f41feb..684c4204ecb6 100644 --- a/source/extensions/filters/common/rbac/utility.h +++ b/source/extensions/filters/common/rbac/utility.h @@ -47,16 +47,16 @@ RoleBasedAccessControlFilterStats generateStats(const std::string& prefix, Stats enum class EnforcementMode { Enforced, Shadow }; template -absl::optional createEngine(const ConfigType& config) { - return config.has_rules() ? absl::make_optional(config.rules()) - : absl::nullopt; +std::unique_ptr createEngine(const ConfigType& config) { + return config.has_rules() ? std::make_unique(config.rules()) + : nullptr; } template -absl::optional createShadowEngine(const ConfigType& config) { +std::unique_ptr createShadowEngine(const ConfigType& config) { return config.has_shadow_rules() - ? absl::make_optional(config.shadow_rules()) - : absl::nullopt; + ? std::make_unique(config.shadow_rules()) + : nullptr; } } // namespace RBAC diff --git a/source/extensions/filters/http/rbac/rbac_filter.cc b/source/extensions/filters/http/rbac/rbac_filter.cc index 965888fd3105..98e6db79b552 100644 --- a/source/extensions/filters/http/rbac/rbac_filter.cc +++ b/source/extensions/filters/http/rbac/rbac_filter.cc @@ -26,7 +26,7 @@ RoleBasedAccessControlFilterConfig::RoleBasedAccessControlFilterConfig( engine_(Filters::Common::RBAC::createEngine(proto_config)), shadow_engine_(Filters::Common::RBAC::createShadowEngine(proto_config)) {} -const absl::optional& +const Filters::Common::RBAC::RoleBasedAccessControlEngineImpl* RoleBasedAccessControlFilterConfig::engine(const Router::RouteConstSharedPtr route, Filters::Common::RBAC::EnforcementMode mode) const { if (!route || !route->routeEntry()) { @@ -70,14 +70,14 @@ Http::FilterHeadersStatus RoleBasedAccessControlFilter::decodeHeaders(Http::Head headers, callbacks_->streamInfo().dynamicMetadata().DebugString()); std::string effective_policy_id; - const auto& shadow_engine = + const auto shadow_engine = config_->engine(callbacks_->route(), Filters::Common::RBAC::EnforcementMode::Shadow); - if (shadow_engine.has_value()) { + if (shadow_engine != nullptr) { std::string shadow_resp_code = Filters::Common::RBAC::DynamicMetadataKeysSingleton::get().EngineResultAllowed; - if (shadow_engine->allowed(*callbacks_->connection(), headers, - callbacks_->streamInfo().dynamicMetadata(), &effective_policy_id)) { + if (shadow_engine->allowed(*callbacks_->connection(), headers, callbacks_->streamInfo(), + &effective_policy_id)) { ENVOY_LOG(debug, "shadow allowed"); config_->stats().shadow_allowed_.inc(); } else { @@ -102,11 +102,10 @@ Http::FilterHeadersStatus RoleBasedAccessControlFilter::decodeHeaders(Http::Head callbacks_->streamInfo().setDynamicMetadata(HttpFilterNames::get().Rbac, metrics); } - const auto& engine = + const auto engine = config_->engine(callbacks_->route(), Filters::Common::RBAC::EnforcementMode::Enforced); - if (engine.has_value()) { - if (engine->allowed(*callbacks_->connection(), headers, - callbacks_->streamInfo().dynamicMetadata(), nullptr)) { + if (engine != nullptr) { + if (engine->allowed(*callbacks_->connection(), headers, callbacks_->streamInfo(), nullptr)) { ENVOY_LOG(debug, "enforced allowed"); config_->stats().allowed_.inc(); return Http::FilterHeadersStatus::Continue; diff --git a/source/extensions/filters/http/rbac/rbac_filter.h b/source/extensions/filters/http/rbac/rbac_filter.h index 5397a3c31c58..e6b004458052 100644 --- a/source/extensions/filters/http/rbac/rbac_filter.h +++ b/source/extensions/filters/http/rbac/rbac_filter.h @@ -22,14 +22,15 @@ class RoleBasedAccessControlRouteSpecificFilterConfig : public Router::RouteSpec RoleBasedAccessControlRouteSpecificFilterConfig( const envoy::config::filter::http::rbac::v2::RBACPerRoute& per_route_config); - const absl::optional& + const Filters::Common::RBAC::RoleBasedAccessControlEngineImpl* engine(Filters::Common::RBAC::EnforcementMode mode) const { - return mode == Filters::Common::RBAC::EnforcementMode::Enforced ? engine_ : shadow_engine_; + return mode == Filters::Common::RBAC::EnforcementMode::Enforced ? engine_.get() + : shadow_engine_.get(); } private: - const absl::optional engine_; - const absl::optional shadow_engine_; + std::unique_ptr engine_; + std::unique_ptr shadow_engine_; }; /** @@ -43,20 +44,21 @@ class RoleBasedAccessControlFilterConfig { Filters::Common::RBAC::RoleBasedAccessControlFilterStats& stats() { return stats_; } - const absl::optional& + const Filters::Common::RBAC::RoleBasedAccessControlEngineImpl* engine(const Router::RouteConstSharedPtr route, Filters::Common::RBAC::EnforcementMode mode) const; private: - const absl::optional& + const Filters::Common::RBAC::RoleBasedAccessControlEngineImpl* engine(Filters::Common::RBAC::EnforcementMode mode) const { - return mode == Filters::Common::RBAC::EnforcementMode::Enforced ? engine_ : shadow_engine_; + return mode == Filters::Common::RBAC::EnforcementMode::Enforced ? engine_.get() + : shadow_engine_.get(); } Filters::Common::RBAC::RoleBasedAccessControlFilterStats stats_; - const absl::optional engine_; - const absl::optional shadow_engine_; + std::unique_ptr engine_; + std::unique_ptr shadow_engine_; }; using RoleBasedAccessControlFilterConfigSharedPtr = diff --git a/source/extensions/filters/network/rbac/rbac_filter.cc b/source/extensions/filters/network/rbac/rbac_filter.cc index 0c5008c88b1d..9bc369f246dc 100644 --- a/source/extensions/filters/network/rbac/rbac_filter.cc +++ b/source/extensions/filters/network/rbac/rbac_filter.cc @@ -77,11 +77,10 @@ void RoleBasedAccessControlFilter::setDynamicMetadata(std::string shadow_engine_ EngineResult RoleBasedAccessControlFilter::checkEngine(Filters::Common::RBAC::EnforcementMode mode) { - const auto& engine = config_->engine(mode); - if (engine.has_value()) { + const auto engine = config_->engine(mode); + if (engine != nullptr) { std::string effective_policy_id; - if (engine->allowed(callbacks_->connection(), - callbacks_->connection().streamInfo().dynamicMetadata(), + if (engine->allowed(callbacks_->connection(), callbacks_->connection().streamInfo(), &effective_policy_id)) { if (mode == Filters::Common::RBAC::EnforcementMode::Shadow) { ENVOY_LOG(debug, "shadow allowed"); diff --git a/source/extensions/filters/network/rbac/rbac_filter.h b/source/extensions/filters/network/rbac/rbac_filter.h index b548424c63de..a42214004c5c 100644 --- a/source/extensions/filters/network/rbac/rbac_filter.h +++ b/source/extensions/filters/network/rbac/rbac_filter.h @@ -26,9 +26,10 @@ class RoleBasedAccessControlFilterConfig { Filters::Common::RBAC::RoleBasedAccessControlFilterStats& stats() { return stats_; } - const absl::optional& + const Filters::Common::RBAC::RoleBasedAccessControlEngineImpl* engine(Filters::Common::RBAC::EnforcementMode mode) const { - return mode == Filters::Common::RBAC::EnforcementMode::Enforced ? engine_ : shadow_engine_; + return mode == Filters::Common::RBAC::EnforcementMode::Enforced ? engine_.get() + : shadow_engine_.get(); } envoy::config::filter::network::rbac::v2::RBAC::EnforcementType enforcementType() const { @@ -38,8 +39,8 @@ class RoleBasedAccessControlFilterConfig { private: Filters::Common::RBAC::RoleBasedAccessControlFilterStats stats_; - const absl::optional engine_; - const absl::optional shadow_engine_; + std::unique_ptr engine_; + std::unique_ptr shadow_engine_; const envoy::config::filter::network::rbac::v2::RBAC::EnforcementType enforcement_type_; }; diff --git a/test/extensions/filters/common/expr/BUILD b/test/extensions/filters/common/expr/BUILD new file mode 100644 index 000000000000..8ce5555328bf --- /dev/null +++ b/test/extensions/filters/common/expr/BUILD @@ -0,0 +1,25 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "context_test", + srcs = ["context_test.cc"], + extension_name = "envoy.filters.http.rbac", + deps = [ + "//source/extensions/filters/common/expr:context_lib", + "//test/mocks/ssl:ssl_mocks", + "//test/mocks/stream_info:stream_info_mocks", + "//test/mocks/upstream:upstream_mocks", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/common/expr/context_test.cc b/test/extensions/filters/common/expr/context_test.cc new file mode 100644 index 000000000000..0e79abe362ec --- /dev/null +++ b/test/extensions/filters/common/expr/context_test.cc @@ -0,0 +1,363 @@ +#include "common/network/utility.h" + +#include "extensions/filters/common/expr/context.h" + +#include "test/mocks/ssl/mocks.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/mocks/upstream/mocks.h" + +#include "absl/time/time.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Const; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Extensions { +namespace Filters { +namespace Common { +namespace Expr { +namespace { + +constexpr absl::string_view Undefined = "undefined"; + +TEST(Context, EmptyHeadersAttributes) { + HeadersWrapper headers(nullptr); + auto header = headers[CelValue::CreateString(Referer)]; + EXPECT_FALSE(header.has_value()); + EXPECT_EQ(0, headers.size()); + EXPECT_TRUE(headers.empty()); +} + +TEST(Context, RequestAttributes) { + NiceMock info; + Http::TestHeaderMapImpl header_map{ + {":method", "POST"}, {":scheme", "http"}, {":path", "/meow?yes=1"}, + {":authority", "kittens.com"}, {"referer", "dogs.com"}, {"user-agent", "envoy-mobile"}, + {"content-length", "10"}, {"x-request-id", "blah"}, + }; + RequestWrapper request(&header_map, info); + + EXPECT_CALL(info, bytesReceived()).WillRepeatedly(Return(10)); + // "2018-04-03T23:06:09.123Z". + const SystemTime start_time(std::chrono::milliseconds(1522796769123)); + EXPECT_CALL(info, startTime()).WillRepeatedly(Return(start_time)); + absl::optional dur = std::chrono::nanoseconds(15000000); + EXPECT_CALL(info, requestComplete()).WillRepeatedly(Return(dur)); + + // stub methods + EXPECT_EQ(0, request.size()); + EXPECT_FALSE(request.empty()); + + { + auto value = request[CelValue::CreateString(Undefined)]; + EXPECT_FALSE(value.has_value()); + } + + { + auto value = request[CelValue::CreateInt64(13)]; + EXPECT_FALSE(value.has_value()); + } + + { + auto value = request[CelValue::CreateString(Scheme)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("http", value.value().StringOrDie().value()); + } + { + auto value = request[CelValue::CreateString(Host)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("kittens.com", value.value().StringOrDie().value()); + } + + { + auto value = request[CelValue::CreateString(Path)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("/meow?yes=1", value.value().StringOrDie().value()); + } + + { + auto value = request[CelValue::CreateString(UrlPath)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("/meow", value.value().StringOrDie().value()); + } + + { + auto value = request[CelValue::CreateString(Method)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("POST", value.value().StringOrDie().value()); + } + + { + auto value = request[CelValue::CreateString(Referer)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("dogs.com", value.value().StringOrDie().value()); + } + + { + auto value = request[CelValue::CreateString(UserAgent)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("envoy-mobile", value.value().StringOrDie().value()); + } + + { + auto value = request[CelValue::CreateString(ID)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("blah", value.value().StringOrDie().value()); + } + + { + auto value = request[CelValue::CreateString(Size)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + EXPECT_EQ(10, value.value().Int64OrDie()); + } + + { + auto value = request[CelValue::CreateString(TotalSize)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + // this includes the headers size + EXPECT_EQ(138, value.value().Int64OrDie()); + } + + { + auto value = request[CelValue::CreateString(Time)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsTimestamp()); + EXPECT_EQ("2018-04-03T23:06:09.123+00:00", absl::FormatTime(value.value().TimestampOrDie())); + } + + { + auto value = request[CelValue::CreateString(Headers)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsMap()); + auto& map = *value.value().MapOrDie(); + EXPECT_FALSE(map.empty()); + EXPECT_EQ(8, map.size()); + + auto header = map[CelValue::CreateString(Referer)]; + EXPECT_TRUE(header.has_value()); + ASSERT_TRUE(header.value().IsString()); + EXPECT_EQ("dogs.com", header.value().StringOrDie().value()); + } + + { + auto value = request[CelValue::CreateString(Duration)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsDuration()); + EXPECT_EQ("15ms", absl::FormatDuration(value.value().DurationOrDie())); + } +} + +TEST(Context, RequestFallbackAttributes) { + NiceMock info; + Http::TestHeaderMapImpl header_map{ + {":method", "POST"}, + {":scheme", "http"}, + {":path", "/meow?yes=1"}, + }; + RequestWrapper request(&header_map, info); + + EXPECT_CALL(info, bytesReceived()).WillRepeatedly(Return(10)); + + { + auto value = request[CelValue::CreateString(Size)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + EXPECT_EQ(10, value.value().Int64OrDie()); + } + + { + auto value = request[CelValue::CreateString(UrlPath)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("/meow", value.value().StringOrDie().value()); + } +} + +TEST(Context, ResponseAttributes) { + NiceMock info; + const std::string header_name = "test-header"; + const std::string trailer_name = "test-trailer"; + Http::TestHeaderMapImpl header_map{{header_name, "a"}}; + Http::TestHeaderMapImpl trailer_map{{trailer_name, "b"}}; + ResponseWrapper response(&header_map, &trailer_map, info); + + EXPECT_CALL(info, responseCode()).WillRepeatedly(Return(404)); + EXPECT_CALL(info, bytesSent()).WillRepeatedly(Return(123)); + + { + auto value = response[CelValue::CreateString(Undefined)]; + EXPECT_FALSE(value.has_value()); + } + + { + auto value = response[CelValue::CreateInt64(13)]; + EXPECT_FALSE(value.has_value()); + } + + { + auto value = response[CelValue::CreateString(Size)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + EXPECT_EQ(123, value.value().Int64OrDie()); + } + + { + auto value = response[CelValue::CreateString(Code)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + EXPECT_EQ(404, value.value().Int64OrDie()); + } + + { + auto value = response[CelValue::CreateString(Headers)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsMap()); + auto& map = *value.value().MapOrDie(); + EXPECT_FALSE(map.empty()); + EXPECT_EQ(1, map.size()); + + auto header = map[CelValue::CreateString(header_name)]; + EXPECT_TRUE(header.has_value()); + ASSERT_TRUE(header.value().IsString()); + EXPECT_EQ("a", header.value().StringOrDie().value()); + + auto missing = map[CelValue::CreateString(Undefined)]; + EXPECT_FALSE(missing.has_value()); + } + + { + auto value = response[CelValue::CreateString(Trailers)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsMap()); + auto& map = *value.value().MapOrDie(); + EXPECT_FALSE(map.empty()); + EXPECT_EQ(1, map.size()); + + auto header = map[CelValue::CreateString(trailer_name)]; + EXPECT_TRUE(header.has_value()); + ASSERT_TRUE(header.value().IsString()); + EXPECT_EQ("b", header.value().StringOrDie().value()); + } +} + +TEST(Context, ConnectionAttributes) { + NiceMock info; + std::shared_ptr> host( + new NiceMock()); + NiceMock connection_info; + ConnectionWrapper connection(info); + PeerWrapper source(info, false); + PeerWrapper destination(info, true); + + Network::Address::InstanceConstSharedPtr local = + Network::Utility::parseInternetAddress("1.2.3.4", 123, false); + Network::Address::InstanceConstSharedPtr remote = + Network::Utility::parseInternetAddress("10.20.30.40", 456, false); + Network::Address::InstanceConstSharedPtr upstream = + Network::Utility::parseInternetAddress("10.1.2.3", 679, false); + const std::string sni_name = "kittens.com"; + EXPECT_CALL(info, downstreamLocalAddress()).WillRepeatedly(ReturnRef(local)); + EXPECT_CALL(info, downstreamRemoteAddress()).WillRepeatedly(ReturnRef(remote)); + EXPECT_CALL(info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(info, upstreamHost()).WillRepeatedly(Return(host)); + EXPECT_CALL(info, requestedServerName()).WillRepeatedly(ReturnRef(sni_name)); + EXPECT_CALL(connection_info, peerCertificatePresented()).WillRepeatedly(Return(true)); + EXPECT_CALL(*host, address()).WillRepeatedly(Return(upstream)); + + { + auto value = connection[CelValue::CreateString(Undefined)]; + EXPECT_FALSE(value.has_value()); + } + + { + auto value = connection[CelValue::CreateInt64(13)]; + EXPECT_FALSE(value.has_value()); + } + + { + auto value = source[CelValue::CreateString(Undefined)]; + EXPECT_FALSE(value.has_value()); + } + + { + auto value = source[CelValue::CreateInt64(13)]; + EXPECT_FALSE(value.has_value()); + } + + { + auto value = destination[CelValue::CreateString(Address)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("1.2.3.4:123", value.value().StringOrDie().value()); + } + + { + auto value = destination[CelValue::CreateString(Port)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + EXPECT_EQ(123, value.value().Int64OrDie()); + } + + { + auto value = source[CelValue::CreateString(Address)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("10.20.30.40:456", value.value().StringOrDie().value()); + } + + { + auto value = source[CelValue::CreateString(Port)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + EXPECT_EQ(456, value.value().Int64OrDie()); + } + + { + auto value = connection[CelValue::CreateString(UpstreamAddress)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ("10.1.2.3:679", value.value().StringOrDie().value()); + } + + { + auto value = connection[CelValue::CreateString(UpstreamPort)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsInt64()); + EXPECT_EQ(679, value.value().Int64OrDie()); + } + + { + auto value = connection[CelValue::CreateString(MTLS)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsBool()); + EXPECT_TRUE(value.value().BoolOrDie()); + } + + { + auto value = connection[CelValue::CreateString(RequestedServerName)]; + EXPECT_TRUE(value.has_value()); + ASSERT_TRUE(value.value().IsString()); + EXPECT_EQ(sni_name, value.value().StringOrDie().value()); + } +} + +} // namespace +} // namespace Expr +} // namespace Common +} // namespace Filters +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/common/rbac/BUILD b/test/extensions/filters/common/rbac/BUILD index d9b47acebf6a..1b0671adbcbc 100644 --- a/test/extensions/filters/common/rbac/BUILD +++ b/test/extensions/filters/common/rbac/BUILD @@ -32,6 +32,7 @@ envoy_extension_cc_test( "//source/extensions/filters/common/rbac:engine_lib", "//test/mocks/network:network_mocks", "//test/mocks/ssl:ssl_mocks", + "//test/mocks/stream_info:stream_info_mocks", "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/filters/common/rbac/engine_impl_test.cc b/test/extensions/filters/common/rbac/engine_impl_test.cc index bdca0c55ae5b..6d346dca62cf 100644 --- a/test/extensions/filters/common/rbac/engine_impl_test.cc +++ b/test/extensions/filters/common/rbac/engine_impl_test.cc @@ -4,6 +4,8 @@ #include "test/mocks/network/mocks.h" #include "test/mocks/ssl/mocks.h" +#include "test/mocks/stream_info/mocks.h" +#include "test/test_common/utility.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -25,7 +27,9 @@ void checkEngine(const RBAC::RoleBasedAccessControlEngineImpl& engine, bool expe const Envoy::Http::HeaderMap& headers = Envoy::Http::HeaderMapImpl(), const envoy::api::v2::core::Metadata& metadata = envoy::api::v2::core::Metadata(), std::string* policy_id = nullptr) { - EXPECT_EQ(expected, engine.allowed(connection, headers, metadata, policy_id)); + NiceMock info; + EXPECT_CALL(Const(info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); + EXPECT_EQ(expected, engine.allowed(connection, headers, info, policy_id)); } TEST(RoleBasedAccessControlEngineImpl, Disabled) { @@ -79,6 +83,187 @@ TEST(RoleBasedAccessControlEngineImpl, DeniedBlacklist) { checkEngine(engine, true, conn); } +TEST(RoleBasedAccessControlEngineImpl, BasicCondition) { + envoy::config::rbac::v2::Policy policy; + policy.add_permissions()->set_any(true); + policy.add_principals()->set_any(true); + policy.mutable_condition()->MergeFrom( + TestUtility::parseYaml(R"EOF( + const_expr: + bool_value: false + )EOF")); + + envoy::config::rbac::v2::RBAC rbac; + rbac.set_action(envoy::config::rbac::v2::RBAC_Action::RBAC_Action_ALLOW); + (*rbac.mutable_policies())["foo"] = policy; + RBAC::RoleBasedAccessControlEngineImpl engine(rbac); + checkEngine(engine, false); +} + +TEST(RoleBasedAccessControlEngineImpl, MalformedCondition) { + envoy::config::rbac::v2::Policy policy; + policy.add_permissions()->set_any(true); + policy.add_principals()->set_any(true); + policy.mutable_condition()->MergeFrom( + TestUtility::parseYaml(R"EOF( + call_expr: + function: undefined_extent + args: + - const_expr: + bool_value: false + )EOF")); + + envoy::config::rbac::v2::RBAC rbac; + rbac.set_action(envoy::config::rbac::v2::RBAC_Action::RBAC_Action_ALLOW); + (*rbac.mutable_policies())["foo"] = policy; + + EXPECT_THROW_WITH_REGEX(RBAC::RoleBasedAccessControlEngineImpl engine(rbac), EnvoyException, + "failed to create an expression: .*"); +} + +TEST(RoleBasedAccessControlEngineImpl, MistypedCondition) { + envoy::config::rbac::v2::Policy policy; + policy.add_permissions()->set_any(true); + policy.add_principals()->set_any(true); + policy.mutable_condition()->MergeFrom( + TestUtility::parseYaml(R"EOF( + const_expr: + int64_value: 13 + )EOF")); + + envoy::config::rbac::v2::RBAC rbac; + rbac.set_action(envoy::config::rbac::v2::RBAC_Action::RBAC_Action_ALLOW); + (*rbac.mutable_policies())["foo"] = policy; + RBAC::RoleBasedAccessControlEngineImpl engine(rbac); + checkEngine(engine, false); +} + +TEST(RoleBasedAccessControlEngineImpl, ErrorCondition) { + envoy::config::rbac::v2::Policy policy; + policy.add_permissions()->set_any(true); + policy.add_principals()->set_any(true); + policy.mutable_condition()->MergeFrom( + TestUtility::parseYaml(R"EOF( + call_expr: + function: _[_] + args: + - select_expr: + operand: + ident_expr: + name: request + field: undefined + - const_expr: + string_value: foo + )EOF")); + + envoy::config::rbac::v2::RBAC rbac; + rbac.set_action(envoy::config::rbac::v2::RBAC_Action::RBAC_Action_ALLOW); + (*rbac.mutable_policies())["foo"] = policy; + RBAC::RoleBasedAccessControlEngineImpl engine(rbac); + checkEngine(engine, false, Envoy::Network::MockConnection()); +} + +TEST(RoleBasedAccessControlEngineImpl, HeaderCondition) { + envoy::config::rbac::v2::Policy policy; + policy.add_permissions()->set_any(true); + policy.add_principals()->set_any(true); + policy.mutable_condition()->MergeFrom( + TestUtility::parseYaml(R"EOF( + call_expr: + function: _==_ + args: + - call_expr: + function: _[_] + args: + - select_expr: + operand: + ident_expr: + name: request + field: headers + - const_expr: + string_value: foo + - const_expr: + string_value: bar + )EOF")); + + envoy::config::rbac::v2::RBAC rbac; + rbac.set_action(envoy::config::rbac::v2::RBAC_Action::RBAC_Action_ALLOW); + (*rbac.mutable_policies())["foo"] = policy; + RBAC::RoleBasedAccessControlEngineImpl engine(rbac); + + Envoy::Http::HeaderMapImpl headers; + Envoy::Http::LowerCaseString key("foo"); + std::string value = "bar"; + headers.setReference(key, value); + + checkEngine(engine, true, Envoy::Network::MockConnection(), headers); +} + +TEST(RoleBasedAccessControlEngineImpl, MetadataCondition) { + envoy::config::rbac::v2::Policy policy; + policy.add_permissions()->set_any(true); + policy.add_principals()->set_any(true); + policy.mutable_condition()->MergeFrom( + TestUtility::parseYaml(R"EOF( + call_expr: + function: _==_ + args: + - call_expr: + function: _[_] + args: + - call_expr: + function: _[_] + args: + - select_expr: + operand: + ident_expr: + name: metadata + field: filter_metadata + - const_expr: + string_value: other + - const_expr: + string_value: label + - const_expr: + string_value: prod + )EOF")); + + envoy::config::rbac::v2::RBAC rbac; + rbac.set_action(envoy::config::rbac::v2::RBAC_Action::RBAC_Action_ALLOW); + (*rbac.mutable_policies())["foo"] = policy; + RBAC::RoleBasedAccessControlEngineImpl engine(rbac); + + Envoy::Http::HeaderMapImpl headers; + + auto label = MessageUtil::keyValueStruct("label", "prod"); + envoy::api::v2::core::Metadata metadata; + metadata.mutable_filter_metadata()->insert( + Protobuf::MapPair("other", label)); + + checkEngine(engine, true, Envoy::Network::MockConnection(), headers, metadata); +} + +TEST(RoleBasedAccessControlEngineImpl, ConjunctiveCondition) { + envoy::config::rbac::v2::Policy policy; + policy.add_permissions()->set_destination_port(123); + policy.add_principals()->set_any(true); + policy.mutable_condition()->MergeFrom( + TestUtility::parseYaml(R"EOF( + const_expr: + bool_value: false + )EOF")); + + envoy::config::rbac::v2::RBAC rbac; + rbac.set_action(envoy::config::rbac::v2::RBAC_Action::RBAC_Action_ALLOW); + (*rbac.mutable_policies())["foo"] = policy; + RBAC::RoleBasedAccessControlEngineImpl engine(rbac); + + Envoy::Network::MockConnection conn; + Envoy::Network::Address::InstanceConstSharedPtr addr = + Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 123, false); + EXPECT_CALL(conn, localAddress()).WillOnce(ReturnRef(addr)); + checkEngine(engine, false, conn); +} + } // namespace } // namespace RBAC } // namespace Common diff --git a/test/extensions/filters/common/rbac/matchers_test.cc b/test/extensions/filters/common/rbac/matchers_test.cc index 1262d8281550..43012da47eda 100644 --- a/test/extensions/filters/common/rbac/matchers_test.cc +++ b/test/extensions/filters/common/rbac/matchers_test.cc @@ -25,7 +25,9 @@ void checkMatcher( const Envoy::Network::Connection& connection = Envoy::Network::MockConnection(), const Envoy::Http::HeaderMap& headers = Envoy::Http::HeaderMapImpl(), const envoy::api::v2::core::Metadata& metadata = envoy::api::v2::core::Metadata()) { - EXPECT_EQ(expected, matcher.matches(connection, headers, metadata)); + NiceMock info; + EXPECT_CALL(Const(info), dynamicMetadata()).WillRepeatedly(ReturnRef(metadata)); + EXPECT_EQ(expected, matcher.matches(connection, headers, info)); } TEST(AlwaysMatcher, AlwaysMatches) { checkMatcher(RBAC::AlwaysMatcher(), true); } @@ -293,7 +295,7 @@ TEST(PolicyMatcher, PolicyMatcher) { policy.add_principals()->mutable_authenticated()->mutable_principal_name()->set_exact("foo"); policy.add_principals()->mutable_authenticated()->mutable_principal_name()->set_exact("bar"); - RBAC::PolicyMatcher matcher(policy); + RBAC::PolicyMatcher matcher(policy, nullptr); Envoy::Network::MockConnection conn; Envoy::Ssl::MockConnectionInfo ssl; diff --git a/test/extensions/filters/common/rbac/mocks.h b/test/extensions/filters/common/rbac/mocks.h index 6b14a834eec7..50555419dd4c 100644 --- a/test/extensions/filters/common/rbac/mocks.h +++ b/test/extensions/filters/common/rbac/mocks.h @@ -17,11 +17,10 @@ class MockEngine : public RoleBasedAccessControlEngineImpl { MOCK_CONST_METHOD4(allowed, bool(const Envoy::Network::Connection&, const Envoy::Http::HeaderMap&, - const envoy::api::v2::core::Metadata&, std::string* effective_policy_id)); + const StreamInfo::StreamInfo&, std::string* effective_policy_id)); - MOCK_CONST_METHOD3(allowed, - bool(const Envoy::Network::Connection&, const envoy::api::v2::core::Metadata&, - std::string* effective_policy_id)); + MOCK_CONST_METHOD3(allowed, bool(const Envoy::Network::Connection&, const StreamInfo::StreamInfo&, + std::string* effective_policy_id)); }; } // namespace RBAC From d2e2cd6fa3dd7669f406ae11736f2e179213b449 Mon Sep 17 00:00:00 2001 From: Ben Plotnick Date: Mon, 19 Aug 2019 20:08:25 -0700 Subject: [PATCH 411/542] ext_authz: add metadata_context to ext_authz filter (#7818) This adds the ability to specify dynamic metadata (by namespace) to send with the ext_authz check request. This allows one filter to specify information that can be then used in evaluating an authorization decision. Risk Level: Medium. Optional feature/extension of existing filter Testing: Unit testing Docs Changes: Inline in attribute_context.proto and ext_authz.proto Fixes #7699 Signed-off-by: Ben Plotnick --- .../filter/http/ext_authz/v2/ext_authz.proto | 14 +++++ .../service/auth/v2/attribute_context.proto | 4 ++ docs/root/intro/version_history.rst | 1 + .../common/ext_authz/check_request_utils.cc | 2 + .../common/ext_authz/check_request_utils.h | 1 + .../filters/http/ext_authz/ext_authz.cc | 15 ++++- .../filters/http/ext_authz/ext_authz.h | 9 +++ .../ext_authz/check_request_utils_test.cc | 23 +++++-- .../filters/http/ext_authz/ext_authz_test.cc | 62 +++++++++++++++++++ 9 files changed, 124 insertions(+), 7 deletions(-) diff --git a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto index 8e2ea7661e1f..de105eff3c80 100644 --- a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto +++ b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto @@ -72,6 +72,20 @@ message ExtAuthz { // Sets the HTTP status that is returned to the client when there is a network error between the // filter and the authorization server. The default status is HTTP 403 Forbidden. envoy.type.HttpStatus status_on_error = 7; + + // Specifies a list of metadata namespaces whose values, if present, will be passed to the + // ext_authz service as an opaque *protobuf::Struct*. + // + // For example, if the *jwt_authn* filter is used and :ref:`payload_in_metadata + // ` is set, + // then the following will pass the jwt payload to the authorization server. + // + // .. code-block:: yaml + // + // metadata_context_namespaces: + // - envoy.filters.http.jwt_authn + // + repeated string metadata_context_namespaces = 8; } // Configuration for buffering the request data. diff --git a/api/envoy/service/auth/v2/attribute_context.proto b/api/envoy/service/auth/v2/attribute_context.proto index f5b723e7b633..822e361dd81b 100644 --- a/api/envoy/service/auth/v2/attribute_context.proto +++ b/api/envoy/service/auth/v2/attribute_context.proto @@ -7,6 +7,7 @@ option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.auth.v2"; import "envoy/api/v2/core/address.proto"; +import "envoy/api/v2/core/base.proto"; import "google/protobuf/timestamp.proto"; import "gogoproto/gogo.proto"; @@ -135,6 +136,9 @@ message AttributeContext { // information to the auth server without modifying the proto definition. It maps to the // internal opaque context in the filter chain. map context_extensions = 10; + + // Dynamic metadata associated with the request. + envoy.api.v2.core.Metadata metadata_context = 11; } // The following items are left out of this proto diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 83998544fdf1..c40a3454e4b1 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -19,6 +19,7 @@ Version history * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * config: added stat :ref:`init_fetch_timeout `. +* ext_authz: added :ref:`configurable ability ` to send dynamic metadata to the `ext_authz` service. * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. * grpc: added :ref:`AWS IAM grpc credentials extension ` for AWS-managed xDS. * grpc-json: added support for :ref:`ignoring unknown query parameters`. diff --git a/source/extensions/filters/common/ext_authz/check_request_utils.cc b/source/extensions/filters/common/ext_authz/check_request_utils.cc index d243e04c5978..b98752cc5a14 100644 --- a/source/extensions/filters/common/ext_authz/check_request_utils.cc +++ b/source/extensions/filters/common/ext_authz/check_request_utils.cc @@ -143,6 +143,7 @@ void CheckRequestUtils::createHttpCheck( const Envoy::Http::StreamDecoderFilterCallbacks* callbacks, const Envoy::Http::HeaderMap& headers, Protobuf::Map&& context_extensions, + envoy::api::v2::core::Metadata&& metadata_context, envoy::service::auth::v2::CheckRequest& request, uint64_t max_request_bytes) { auto attrs = request.mutable_attributes(); @@ -158,6 +159,7 @@ void CheckRequestUtils::createHttpCheck( // Fill in the context extensions: (*attrs->mutable_context_extensions()) = std::move(context_extensions); + (*attrs->mutable_metadata_context()) = std::move(metadata_context); } void CheckRequestUtils::createTcpCheck(const Network::ReadFilterCallbacks* callbacks, diff --git a/source/extensions/filters/common/ext_authz/check_request_utils.h b/source/extensions/filters/common/ext_authz/check_request_utils.h index 5fa997c80a52..6f90d8d86b1a 100644 --- a/source/extensions/filters/common/ext_authz/check_request_utils.h +++ b/source/extensions/filters/common/ext_authz/check_request_utils.h @@ -48,6 +48,7 @@ class CheckRequestUtils { static void createHttpCheck(const Envoy::Http::StreamDecoderFilterCallbacks* callbacks, const Envoy::Http::HeaderMap& headers, Protobuf::Map&& context_extensions, + envoy::api::v2::core::Metadata&& metadata_context, envoy::service::auth::v2::CheckRequest& request, uint64_t max_request_bytes); diff --git a/source/extensions/filters/http/ext_authz/ext_authz.cc b/source/extensions/filters/http/ext_authz/ext_authz.cc index 87b566079927..d7685ab1a057 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.cc +++ b/source/extensions/filters/http/ext_authz/ext_authz.cc @@ -65,9 +65,20 @@ void Filter::initiateCall(const Http::HeaderMap& headers) { if (maybe_merged_per_route_config) { context_extensions = maybe_merged_per_route_config.value().takeContextExtensions(); } + + // If metadata_context_namespaces is specified, pass matching metadata to the ext_authz service + envoy::api::v2::core::Metadata metadata_context; + const auto& request_metadata = callbacks_->streamInfo().dynamicMetadata().filter_metadata(); + for (const auto& context_key : config_->metadataContextNamespaces()) { + const auto& metadata_it = request_metadata.find(context_key); + if (metadata_it != request_metadata.end()) { + (*metadata_context.mutable_filter_metadata())[metadata_it->first] = metadata_it->second; + } + } + Filters::Common::ExtAuthz::CheckRequestUtils::createHttpCheck( - callbacks_, headers, std::move(context_extensions), check_request_, - config_->maxRequestBytes()); + callbacks_, headers, std::move(context_extensions), std::move(metadata_context), + check_request_, config_->maxRequestBytes()); ENVOY_STREAM_LOG(trace, "ext_authz filter calling authorization server", *callbacks_); state_ = State::Calling; diff --git a/source/extensions/filters/http/ext_authz/ext_authz.h b/source/extensions/filters/http/ext_authz/ext_authz.h index 0b0528b53403..60ec7ca10f48 100644 --- a/source/extensions/filters/http/ext_authz/ext_authz.h +++ b/source/extensions/filters/http/ext_authz/ext_authz.h @@ -47,6 +47,8 @@ class FilterConfig { max_request_bytes_(config.with_request_body().max_request_bytes()), status_on_error_(toErrorCode(config.status_on_error().code())), local_info_(local_info), scope_(scope), runtime_(runtime), http_context_(http_context), pool_(scope.symbolTable()), + metadata_context_namespaces_(config.metadata_context_namespaces().begin(), + config.metadata_context_namespaces().end()), ext_authz_ok_(pool_.add("ext_authz.ok")), ext_authz_denied_(pool_.add("ext_authz.denied")), ext_authz_error_(pool_.add("ext_authz.error")), ext_authz_failure_mode_allowed_(pool_.add("ext_authz.failure_mode_allowed")) {} @@ -75,6 +77,10 @@ class FilterConfig { scope.counterFromStatName(name).inc(); } + const std::vector& metadataContextNamespaces() { + return metadata_context_namespaces_; + } + private: static Http::Code toErrorCode(uint64_t status) { const auto code = static_cast(status); @@ -93,8 +99,11 @@ class FilterConfig { Stats::Scope& scope_; Runtime::Loader& runtime_; Http::Context& http_context_; + Stats::StatNamePool pool_; + const std::vector metadata_context_namespaces_; + public: const Stats::StatName ext_authz_ok_; const Stats::StatName ext_authz_denied_; diff --git a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc index 7567a9a95023..fa925bf6b8bf 100644 --- a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc +++ b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc @@ -86,7 +86,8 @@ TEST_F(CheckRequestUtilsTest, BasicHttp) { ExpectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, request_headers, - Protobuf::Map(), request_, size); + Protobuf::Map(), + envoy::api::v2::core::Metadata(), request_, size); ASSERT_EQ(size, request_.attributes().request().http().body().size()); EXPECT_EQ(buffer_->toString().substr(0, size), request_.attributes().request().http().body()); EXPECT_EQ(request_.attributes().request().http().headers().end(), @@ -102,7 +103,8 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithPartialBody) { ExpectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, headers_, - Protobuf::Map(), request_, size); + Protobuf::Map(), + envoy::api::v2::core::Metadata(), request_, size); ASSERT_EQ(size, request_.attributes().request().http().body().size()); EXPECT_EQ(buffer_->toString().substr(0, size), request_.attributes().request().http().body()); EXPECT_EQ("true", request_.attributes().request().http().headers().at( @@ -116,8 +118,8 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithFullBody) { ExpectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, headers_, - Protobuf::Map(), request_, - buffer_->length()); + Protobuf::Map(), + envoy::api::v2::core::Metadata(), request_, buffer_->length()); ASSERT_EQ(buffer_->length(), request_.attributes().request().http().body().size()); EXPECT_EQ(buffer_->toString().substr(0, buffer_->length()), request_.attributes().request().http().body()); @@ -146,13 +148,24 @@ TEST_F(CheckRequestUtilsTest, CheckAttrContextPeer) { Protobuf::Map context_extensions; context_extensions["key"] = "value"; + envoy::api::v2::core::Metadata metadata_context; + auto metadata_val = MessageUtil::keyValueStruct("foo", "bar"); + (*metadata_context.mutable_filter_metadata())["meta.key"] = metadata_val; + CheckRequestUtils::createHttpCheck(&callbacks_, request_headers, std::move(context_extensions), - request, false); + std::move(metadata_context), request, false); EXPECT_EQ("source", request.attributes().source().principal()); EXPECT_EQ("destination", request.attributes().destination().principal()); EXPECT_EQ("foo", request.attributes().source().service()); EXPECT_EQ("value", request.attributes().context_extensions().at("key")); + EXPECT_EQ("bar", request.attributes() + .metadata_context() + .filter_metadata() + .at("meta.key") + .fields() + .at("foo") + .string_value()); } } // namespace diff --git a/test/extensions/filters/http/ext_authz/ext_authz_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_test.cc index d82354c0e917..3586d9d9b550 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_test.cc @@ -768,6 +768,68 @@ TEST_F(HttpFilterTest, NoClearCacheRouteDeniedResponse) { EXPECT_EQ("ext_authz_denied", filter_callbacks_.details_); } +// Verifies that specified metadata is passed along in the check request +TEST_F(HttpFilterTest, MetadataContext) { + initialize(R"EOF( + grpc_service: + envoy_grpc: + cluster_name: "ext_authz_server" + metadata_context_namespaces: + - jazz.sax + - rock.guitar + - hiphop.drums + )EOF"); + + const std::string yaml = R"EOF( + filter_metadata: + jazz.sax: + coltrane: john + parker: charlie + jazz.piano: + monk: thelonious + hancock: herbie + rock.guitar: + hendrix: jimi + richards: keith + )EOF"; + + envoy::api::v2::core::Metadata metadata; + TestUtility::loadFromYaml(yaml, metadata); + ON_CALL(filter_callbacks_.stream_info_, dynamicMetadata()).WillByDefault(ReturnRef(metadata)); + + prepareCheck(); + + envoy::service::auth::v2::CheckRequest check_request; + EXPECT_CALL(*client_, check(_, _, _)) + .WillOnce(WithArgs<1>(Invoke([&](const envoy::service::auth::v2::CheckRequest& check_param) + -> void { check_request = check_param; }))); + + filter_->decodeHeaders(request_headers_, false); + Http::MetadataMap metadata_map{{"metadata", "metadata"}}; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->decodeMetadata(metadata_map)); + + EXPECT_EQ("john", check_request.attributes() + .metadata_context() + .filter_metadata() + .at("jazz.sax") + .fields() + .at("coltrane") + .string_value()); + + EXPECT_EQ("jimi", check_request.attributes() + .metadata_context() + .filter_metadata() + .at("rock.guitar") + .fields() + .at("hendrix") + .string_value()); + + EXPECT_EQ(0, check_request.attributes().metadata_context().filter_metadata().count("jazz.piano")); + + EXPECT_EQ(0, + check_request.attributes().metadata_context().filter_metadata().count("hiphop.drums")); +} + // ------------------- // Parameterized Tests // ------------------- From 5aede462bb7fd8e39cd329fd270d3b74e7d7d39a Mon Sep 17 00:00:00 2001 From: asraa Date: Tue, 20 Aug 2019 13:13:50 -0400 Subject: [PATCH 412/542] fuzz: codec impl timeout fix + speed ups (#7963) Some speed-ups and validations for codec impl fuzz test: * validate actions aren't empty (another approach would be to scrub / clean these) * limit actions to 1024 * require oneofs Fixes OSS-Fuzz Issue: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=16481 Testing: local asan/libfuzzer exec/sec go from 25 to 50 Signed-off-by: Asra Ali --- ...case-codec_impl_fuzz_test-5687788200001536 | 11962 ++++++++++++++++ test/common/http/codec_impl_fuzz.proto | 6 +- test/common/http/codec_impl_fuzz_test.cc | 16 +- 3 files changed, 11979 insertions(+), 5 deletions(-) create mode 100644 test/common/http/codec_impl_corpus/clusterfuzz-testcase-codec_impl_fuzz_test-5687788200001536 diff --git a/test/common/http/codec_impl_corpus/clusterfuzz-testcase-codec_impl_fuzz_test-5687788200001536 b/test/common/http/codec_impl_corpus/clusterfuzz-testcase-codec_impl_fuzz_test-5687788200001536 new file mode 100644 index 000000000000..eee0fb76bd49 --- /dev/null +++ b/test/common/http/codec_impl_corpus/clusterfuzz-testcase-codec_impl_fuzz_test-5687788200001536 @@ -0,0 +1,11962 @@ +h2_settings { + client { + hpack_table_size: 35072 + initial_connection_window_size: 35072 + } + server { + hpack_table_size: 257 + initial_stream_window_size: 4294836216 + initial_connection_window_size: 4294835968 + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\000\000\000]" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\177H" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "YY" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\000\000\000]" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + request_headers { + headers { + value: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } + } + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "BB" + } + } + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "````````````````````````````````````````````````````````yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\000\225yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy````````````````````````````" + } + } + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 34 + value: 1545 + server: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "````````````````````````````````````````````````````````yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\000\225yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy````````````````````````````" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 34 + value: 1545 + server: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + quiesce_drain { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\000\000\000]" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + } + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + value: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\177\177\177" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "````````````````````````````````````````````````````````yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\000\225yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy````````````````````````````" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + request_headers { + headers { + key: "\177H" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\000\000\000]" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\000\000\000]" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "````````````````````````````````````````````````````````yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\000\225yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy````````````````````````````" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "````````````````````````````````````````````````````````yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\000\225yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy````````````````````````````" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + quiesce_drain { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\177H" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + server_drain { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "YY" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "YY" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + request_headers { + headers { + key: "\177H" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 34 + value: 1 + server: true + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 34 + value: 1545 + server: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "transfer-encodinG" + value: "YY" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 34 + value: 1545 + server: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + request_headers { + headers { + value: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + server_drain { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + end_stream: true + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + headers { + key: "\177\177\177" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + request_headers { + } + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { +} +actions { + new_stream { + request_headers { + headers { + value: "\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177\177" + } + } + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + new_stream { + } +} +actions { + mutate { + buffer: 1 + offset: 1 + value: 63 + } +} diff --git a/test/common/http/codec_impl_fuzz.proto b/test/common/http/codec_impl_fuzz.proto index 249e38eb642f..f5d39f9ded2f 100644 --- a/test/common/http/codec_impl_fuzz.proto +++ b/test/common/http/codec_impl_fuzz.proto @@ -4,17 +4,19 @@ package test.common.http; import "google/protobuf/empty.proto"; +import "validate/validate.proto"; import "test/fuzz/common.proto"; // Structured input for H2 codec_impl_fuzz_test. message NewStream { - test.fuzz.Headers request_headers = 1; + test.fuzz.Headers request_headers = 1 [(validate.rules).message.required = true]; bool end_stream = 2; } message DirectionalAction { oneof directional_action_selector { + option (validate.required) = true; test.fuzz.Headers continue_headers = 1; test.fuzz.Headers headers = 2; uint32 data = 3; @@ -30,6 +32,7 @@ message StreamAction { // Index into list of created streams (not HTTP/2 level stream ID). uint32 stream_id = 1; oneof stream_action_selector { + option (validate.required) = true; DirectionalAction request = 2; DirectionalAction response = 3; } @@ -56,6 +59,7 @@ message SwapBufferAction { message Action { oneof action_selector { + option (validate.required) = true; // Create new stream. NewStream new_stream = 1; // Perform an action on an existing stream. diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index e25f56194efc..28cb56d0e592 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -16,7 +16,7 @@ #include "common/http/http1/codec_impl.h" #include "common/http/http2/codec_impl.h" -#include "test/common/http/codec_impl_fuzz.pb.h" +#include "test/common/http/codec_impl_fuzz.pb.validate.h" #include "test/common/http/http2/codec_impl_test_util.h" #include "test/fuzz/fuzz_runner.h" #include "test/fuzz/utility.h" @@ -420,8 +420,10 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi } }; + constexpr auto max_actions = 1024; try { - for (const auto& action : input.actions()) { + for (int i = 0; i < std::min(max_actions, input.actions().size()); ++i) { + const auto& action = input.actions(i); ENVOY_LOG_MISC(trace, "action {} with {} streams", action.DebugString(), streams.size()); switch (action.action_selector_case()) { case test::common::http::Action::kNewStream: { @@ -502,8 +504,14 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi // Fuzz the H1/H2 codec implementations. DEFINE_PROTO_FUZZER(const test::common::http::CodecImplFuzzTestCase& input) { - codecFuzz(input, HttpVersion::Http1); - codecFuzz(input, HttpVersion::Http2); + try { + // Validate input early. + MessageUtil::validate(input); + codecFuzz(input, HttpVersion::Http1); + codecFuzz(input, HttpVersion::Http2); + } catch (const EnvoyException& e) { + ENVOY_LOG_MISC(debug, "EnvoyException: {}", e.what()); + } } } // namespace Http From c6b190ba830b035d1385791d3d648d4b0d904584 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 20 Aug 2019 13:51:47 -0400 Subject: [PATCH 413/542] docs: more detail about tracking down deprecated features (#7972) Risk Level: n/a (docs only) Testing: n/a Docs Changes: yes Release Notes: no #7945 Signed-off-by: Alyssa Wilk --- docs/root/configuration/runtime.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/configuration/runtime.rst b/docs/root/configuration/runtime.rst index ca1aa31a9481..6c9f81b84d47 100644 --- a/docs/root/configuration/runtime.rst +++ b/docs/root/configuration/runtime.rst @@ -257,7 +257,7 @@ The file system runtime provider emits some statistics in the *runtime.* namespa :widths: 1, 1, 2 admin_overrides_active, Gauge, 1 if any admin overrides are active otherwise 0 - deprecated_feature_use, Counter, Total number of times deprecated features were used + deprecated_feature_use, Counter, Total number of times deprecated features were used. Detailed information about the feature used will be logged to warning logs in the form "Using deprecated option 'X' from file Y". load_error, Counter, Total number of load attempts that resulted in an error in any layer load_success, Counter, Total number of load attempts that were successful at all layers num_keys, Gauge, Number of keys currently loaded From 9421bddb21f7626d0f3a9b8f25a5bd0245ff8b79 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 20 Aug 2019 11:14:38 -0700 Subject: [PATCH 414/542] Fix the alignement in optval of setsockopt when compiled with libc++. (#7958) Description: libc++ std::string may inline the data which results the memory is not aligned to `void*`. Use vector instead to store the optval. Detected by UBSAN with libc++ config. Preparation for #4251 Risk Level: Low Testing: unittest locally Docs Changes: N/A Release Notes: N/A Fixes #7968 Signed-off-by: Lizan Zhou --- source/common/network/socket_option_impl.cc | 12 ++++++------ source/common/network/socket_option_impl.h | 12 +++++++++--- .../network/socket_option_factory_test.cc | 17 +++++++++-------- test/common/network/socket_option_impl_test.cc | 3 ++- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/source/common/network/socket_option_impl.cc b/source/common/network/socket_option_impl.cc index 65b364810f21..dbf8aa153301 100644 --- a/source/common/network/socket_option_impl.cc +++ b/source/common/network/socket_option_impl.cc @@ -19,7 +19,7 @@ bool SocketOptionImpl::setOption(Socket& socket, } const Api::SysCallIntResult result = - SocketOptionImpl::setSocketOption(socket, optname_, value_); + SocketOptionImpl::setSocketOption(socket, optname_, value_.data(), value_.size()); if (result.rc_ != 0) { ENVOY_LOG(warn, "Setting {} option on socket failed: {}", optname_.name(), strerror(result.errno_)); @@ -39,22 +39,22 @@ SocketOptionImpl::getOptionDetails(const Socket&, Socket::Option::Details info; info.name_ = optname_; - info.value_ = value_; - return absl::optional(std::move(info)); + info.value_ = {value_.begin(), value_.end()}; + return absl::make_optional(std::move(info)); } bool SocketOptionImpl::isSupported() const { return optname_.has_value(); } Api::SysCallIntResult SocketOptionImpl::setSocketOption(Socket& socket, const Network::SocketOptionName& optname, - const absl::string_view value) { + const void* value, size_t size) { if (!optname.has_value()) { return {-1, ENOTSUP}; } auto& os_syscalls = Api::OsSysCallsSingleton::get(); - return os_syscalls.setsockopt(socket.ioHandle().fd(), optname.level(), optname.option(), - value.data(), value.size()); + return os_syscalls.setsockopt(socket.ioHandle().fd(), optname.level(), optname.option(), value, + size); } } // namespace Network diff --git a/source/common/network/socket_option_impl.h b/source/common/network/socket_option_impl.h index 59931506323c..1a13a67010cb 100644 --- a/source/common/network/socket_option_impl.h +++ b/source/common/network/socket_option_impl.h @@ -7,6 +7,7 @@ #include "envoy/api/os_sys_calls.h" #include "envoy/network/listen_socket.h" +#include "common/common/assert.h" #include "common/common/logger.h" namespace Envoy { @@ -103,7 +104,9 @@ class SocketOptionImpl : public Socket::Option, Logger::Loggable(value_.data()) % alignof(void*) == 0); + } // Socket::Option bool setOption(Socket& socket, @@ -123,17 +126,20 @@ class SocketOptionImpl : public Socket::Option, Logger::Loggable but not std::string because std::string might inline + // the buffer so its data() is not aligned in to alignof(void*). + const std::vector value_; }; } // namespace Network diff --git a/test/common/network/socket_option_factory_test.cc b/test/common/network/socket_option_factory_test.cc index 6ec4767ab372..ca5e25cc363f 100644 --- a/test/common/network/socket_option_factory_test.cc +++ b/test/common/network/socket_option_factory_test.cc @@ -154,20 +154,21 @@ TEST_F(SocketOptionFactoryTest, TestBuildLiteralOptions) { EXPECT_TRUE(option_details.has_value()); EXPECT_EQ(SOL_SOCKET, option_details->name_.level()); EXPECT_EQ(SO_LINGER, option_details->name_.option()); - EXPECT_EQ(sizeof(struct linger), option_details->value_.size()); - const struct linger* linger_ptr = - reinterpret_cast(option_details->value_.data()); - EXPECT_EQ(1, linger_ptr->l_onoff); - EXPECT_EQ(3456, linger_ptr->l_linger); + struct linger expected_linger; + expected_linger.l_onoff = 1; + expected_linger.l_linger = 3456; + absl::string_view linger_bstr{reinterpret_cast(&expected_linger), + sizeof(struct linger)}; + EXPECT_EQ(linger_bstr, option_details->value_); option_details = socket_options->at(1)->getOptionDetails( socket_mock_, envoy::api::v2::core::SocketOption::STATE_PREBIND); EXPECT_TRUE(option_details.has_value()); EXPECT_EQ(SOL_SOCKET, option_details->name_.level()); EXPECT_EQ(SO_KEEPALIVE, option_details->name_.option()); - EXPECT_EQ(sizeof(int), option_details->value_.size()); - const int* flag_ptr = reinterpret_cast(option_details->value_.data()); - EXPECT_EQ(1, *flag_ptr); + int value = 1; + absl::string_view value_bstr{reinterpret_cast(&value), sizeof(int)}; + EXPECT_EQ(value_bstr, option_details->value_); } } // namespace diff --git a/test/common/network/socket_option_impl_test.cc b/test/common/network/socket_option_impl_test.cc index 8d9e55ae2d19..d979d8a01098 100644 --- a/test/common/network/socket_option_impl_test.cc +++ b/test/common/network/socket_option_impl_test.cc @@ -8,7 +8,8 @@ class SocketOptionImplTest : public SocketOptionTest {}; TEST_F(SocketOptionImplTest, BadFd) { absl::string_view zero("\0\0\0\0", 4); - Api::SysCallIntResult result = SocketOptionImpl::setSocketOption(socket_, {}, zero); + Api::SysCallIntResult result = + SocketOptionImpl::setSocketOption(socket_, {}, zero.data(), zero.size()); EXPECT_EQ(-1, result.rc_); EXPECT_EQ(ENOTSUP, result.errno_); } From c5050773a959e907dd21a26c6221d11b17d7042d Mon Sep 17 00:00:00 2001 From: htuch Date: Tue, 20 Aug 2019 22:11:49 -0400 Subject: [PATCH 415/542] security: some intra-entity and 3rd party embargo clarifications. (#7977) * security: some intra-entity and 3rd party embargo clarifications. These came up in the last set of CVEs. Signed-off-by: Harvey Tuch --- SECURITY.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index e24c641f8861..883b3c3b067b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -217,11 +217,25 @@ issue fixed for your respective distribution's users. Before any information from the list is shared with respective members of your team required to fix said issue, they must agree to the same terms and only find out information on a need-to-know basis. +We typically expect a single point-of-contact (PoC) at any given legal entity. Within the +organization, it is the responsibility of the PoC to share CVE and related patches internally. This +should be performed on a strictly need-to-know basis with affected groups to the extent that this is +technically plausible. All teams should be aware of the embargo conditions and accept them. +Ultimately, if an organization breaks embargo transitively through such sharing, they will lose +the early disclosure privilege, so it's in their best interest to carefully share information internally, +following best practices and use their judgement in balancing the tradeoff between protecting users +and maintaining confidentiality. + The embargo applies to information shared, source code and binary images. **It is a violation of the embargo policy to share binary distributions of the security fixes before the public release date.** This includes, but is not limited to, Envoy binaries and Docker images. It is expected that distributors have a method to stage and validate new binaries without exposing them publicly. +If the information shared is under embargo from a third party, where Envoy is one of many projects +that a disclosure is shared with, it is critical to consider that the ramifications of any leak will +extend beyond the Envoy community and will leave us in a position in which we will be less likely to +receive embargoed reports in the future. + In the unfortunate event you share the information beyond what is allowed by this policy, you _must_ urgently inform the envoy-security@googlegroups.com mailing list of exactly what information leaked and to whom. A retrospective will take place after the leak so we can assess how to prevent making the From 7eed7332d513248a07e493bb8ec7bb3081a18b3e Mon Sep 17 00:00:00 2001 From: Andres Guedez <34292400+AndresGuedez@users.noreply.github.com> Date: Wed, 21 Aug 2019 10:36:43 -0400 Subject: [PATCH 416/542] protobuf: IWYU (#7989) Include What You Use fix for source/common/protobuf/message_validator_impl.h. Signed-off-by: Andres Guedez --- source/common/protobuf/message_validator_impl.cc | 1 - source/common/protobuf/message_validator_impl.h | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/common/protobuf/message_validator_impl.cc b/source/common/protobuf/message_validator_impl.cc index 4e921a0f0215..e07ae6ea8221 100644 --- a/source/common/protobuf/message_validator_impl.cc +++ b/source/common/protobuf/message_validator_impl.cc @@ -4,7 +4,6 @@ #include "common/common/assert.h" #include "common/common/hash.h" -#include "common/common/logger.h" #include "common/common/macros.h" #include "absl/strings/str_cat.h" diff --git a/source/common/protobuf/message_validator_impl.h b/source/common/protobuf/message_validator_impl.h index 2d5b3d41af0f..32d705fd44bf 100644 --- a/source/common/protobuf/message_validator_impl.h +++ b/source/common/protobuf/message_validator_impl.h @@ -3,6 +3,8 @@ #include "envoy/protobuf/message_validator.h" #include "envoy/stats/stats.h" +#include "common/common/logger.h" + #include "absl/container/flat_hash_set.h" namespace Envoy { From f04dccbcbaa428d3463cf041c76a585bf6f5bbf1 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 21 Aug 2019 09:36:52 -0700 Subject: [PATCH 417/542] api: add name into filter chain (#7966) Signed-off-by: Yuchen Dai --- api/envoy/api/v2/listener/listener.proto | 5 +++++ tools/spelling_dictionary.txt | 1 + 2 files changed, 6 insertions(+) diff --git a/api/envoy/api/v2/listener/listener.proto b/api/envoy/api/v2/listener/listener.proto index 77f753501fdd..293a4d2da221 100644 --- a/api/envoy/api/v2/listener/listener.proto +++ b/api/envoy/api/v2/listener/listener.proto @@ -188,6 +188,11 @@ message FilterChain { // See :ref:`base.TransportSocket` description. core.TransportSocket transport_socket = 6; + + // [#not-implemented-hide:] The unique name (or empty) by which this filter chain is known. If no + // name is provided, Envoy will allocate an internal UUID for the filter chain. If the filter + // chain is to be dynamically updated or removed via FCDS a unique name must be provided. + string name = 7; } message ListenerFilter { diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 3f935e236cef..98cfe52c85a8 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -88,6 +88,7 @@ EWOULDBLOCK EXPECTs EXPR FAQ +FCDS FDs FFFF FIN From 5d42b9b3171c8fa27a0f5eba96d02be78a4bbb8c Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 21 Aug 2019 10:00:31 -0700 Subject: [PATCH 418/542] rds: validate config in depth before update config dump (#7956) Route config need deep validation for virtual host duplication check, regex check, per filter config validation etc, which PGV wasn't enough. Risk Level: Low Testing: regression test Docs Changes: N/A Release Notes: N/A Fixes #7939 Signed-off-by: Lizan Zhou --- include/envoy/router/rds.h | 5 ++ .../router/route_config_update_receiver.h | 1 + source/common/router/rds_impl.cc | 11 ++++ source/common/router/rds_impl.h | 4 +- source/server/http/admin.h | 1 + test/common/http/conn_manager_impl_common.h | 1 + test/common/router/rds_impl_test.cc | 66 +++++++++++++++++++ 7 files changed, 88 insertions(+), 1 deletion(-) diff --git a/include/envoy/router/rds.h b/include/envoy/router/rds.h index 9dcd4c3f64e5..456e44922043 100644 --- a/include/envoy/router/rds.h +++ b/include/envoy/router/rds.h @@ -48,6 +48,11 @@ class RouteConfigProvider { * Callback used to notify RouteConfigProvider about configuration changes. */ virtual void onConfigUpdate() PURE; + + /** + * Validate if the route configuration can be applied to the context of the route config provider. + */ + virtual void validateConfig(const envoy::api::v2::RouteConfiguration& config) const PURE; }; using RouteConfigProviderPtr = std::unique_ptr; diff --git a/include/envoy/router/route_config_update_receiver.h b/include/envoy/router/route_config_update_receiver.h index 8ac284fae6d4..6e4d492b1676 100644 --- a/include/envoy/router/route_config_update_receiver.h +++ b/include/envoy/router/route_config_update_receiver.h @@ -28,6 +28,7 @@ class RouteConfigUpdateReceiver { */ virtual bool onRdsUpdate(const envoy::api::v2::RouteConfiguration& rc, const std::string& version_info) PURE; + /** * Called on updates via VHDS. * @param added_resources supplies Resources (each containing a VirtualHost) that have been diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index e316460e0f09..d94910325cdb 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -99,6 +99,11 @@ void RdsRouteConfigSubscription::onConfigUpdate( throw EnvoyException(fmt::format("Unexpected RDS configuration (expecting {}): {}", route_config_name_, route_config.name())); } + for (auto* provider : route_config_providers_) { + // This seems inefficient, though it is necessary to validate config in each context, + // especially when it comes with per_filter_config, + provider->validateConfig(route_config); + } if (config_update_info_->onRdsUpdate(route_config, version_info)) { stats_.config_reload_.inc(); @@ -198,6 +203,12 @@ void RdsRouteConfigProviderImpl::onConfigUpdate() { [this, new_config]() -> void { tls_->getTyped().config_ = new_config; }); } +void RdsRouteConfigProviderImpl::validateConfig( + const envoy::api::v2::RouteConfiguration& config) const { + // TODO(lizan): consider cache the config here until onConfigUpdate. + ConfigImpl validation_config(config, factory_context_, false); +} + RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& admin) { config_tracker_entry_ = admin.getConfigTracker().add("routes", [this] { return dumpRouteConfigs(); }); diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 6ec2c3e16047..2e8bbc61a5b4 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -66,6 +66,7 @@ class StaticRouteConfigProviderImpl : public RouteConfigProvider { } SystemTime lastUpdated() const override { return last_updated_; } void onConfigUpdate() override {} + void validateConfig(const envoy::api::v2::RouteConfiguration&) const override {} private: ConfigConstSharedPtr config_; @@ -159,7 +160,6 @@ class RdsRouteConfigProviderImpl : public RouteConfigProvider, ~RdsRouteConfigProviderImpl() override; RdsRouteConfigSubscription& subscription() { return *subscription_; } - void onConfigUpdate() override; // Router::RouteConfigProvider Router::ConfigConstSharedPtr config() override; @@ -167,6 +167,8 @@ class RdsRouteConfigProviderImpl : public RouteConfigProvider, return config_update_info_->configInfo(); } SystemTime lastUpdated() const override { return config_update_info_->lastUpdated(); } + void onConfigUpdate() override; + void validateConfig(const envoy::api::v2::RouteConfiguration& config) const override; private: struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { diff --git a/source/server/http/admin.h b/source/server/http/admin.h index e3806f58e188..09bf1ecedf4c 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -172,6 +172,7 @@ class AdminImpl : public Admin, absl::optional configInfo() const override { return {}; } SystemTime lastUpdated() const override { return time_source_.systemTime(); } void onConfigUpdate() override {} + void validateConfig(const envoy::api::v2::RouteConfiguration&) const override {} Router::ConfigConstSharedPtr config_; TimeSource& time_source_; diff --git a/test/common/http/conn_manager_impl_common.h b/test/common/http/conn_manager_impl_common.h index 1f68cc59412d..f7b8134dbb06 100644 --- a/test/common/http/conn_manager_impl_common.h +++ b/test/common/http/conn_manager_impl_common.h @@ -25,6 +25,7 @@ struct RouteConfigProvider : public Router::RouteConfigProvider { absl::optional configInfo() const override { return {}; } SystemTime lastUpdated() const override { return time_source_.systemTime(); } void onConfigUpdate() override {} + void validateConfig(const envoy::api::v2::RouteConfiguration&) const override {} TimeSource& time_source_; std::shared_ptr route_config_{new NiceMock()}; diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index e84bb925b367..1046b5827be0 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -493,6 +493,72 @@ TEST_F(RouteConfigProviderManagerImplTest, onConfigUpdateWrongSize) { EnvoyException, "Unexpected RDS resource length: 2"); } +// Regression test for https://github.com/envoyproxy/envoy/issues/7939 +TEST_F(RouteConfigProviderManagerImplTest, ConfigDumpAfterConfigRejected) { + auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["routes"](); + const auto& route_config_dump = + MessageUtil::downcastAndValidate( + *message_ptr); + + // No routes at all, no last_updated timestamp + envoy::admin::v2alpha::RoutesConfigDump expected_route_config_dump; + TestUtility::loadFromYaml(R"EOF( +static_route_configs: +dynamic_route_configs: +)EOF", + expected_route_config_dump); + EXPECT_EQ(expected_route_config_dump.DebugString(), route_config_dump.DebugString()); + + timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); + + // dynamic. + setup(); + EXPECT_CALL(*factory_context_.cluster_manager_.subscription_factory_.subscription_, start(_)); + factory_context_.init_manager_.initialize(init_watcher_); + + const std::string response1_yaml = R"EOF( +version_info: '1' +resources: +- "@type": type.googleapis.com/envoy.api.v2.RouteConfiguration + name: foo_route_config + virtual_hosts: + - name: integration + domains: + - "*" + routes: + - match: + prefix: "/foo" + route: + cluster_header: ":authority" + - name: duplicate + domains: + - "*" + routes: + - match: + prefix: "/foo" + route: + cluster_header: ":authority" +)EOF"; + auto response1 = TestUtility::parseYaml(response1_yaml); + + EXPECT_CALL(init_watcher_, ready()); + + EXPECT_THROW_WITH_MESSAGE( + rds_callbacks_->onConfigUpdate(response1.resources(), response1.version_info()), + EnvoyException, "Only a single wildcard domain is permitted"); + + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["routes"](); + const auto& route_config_dump3 = + MessageUtil::downcastAndValidate( + *message_ptr); + TestUtility::loadFromYaml(R"EOF( +static_route_configs: +dynamic_route_configs: +)EOF", + expected_route_config_dump); + EXPECT_EQ(expected_route_config_dump.DebugString(), route_config_dump3.DebugString()); +} + } // namespace } // namespace Router } // namespace Envoy From 87c38e1cd8bc7804ad07de276ffbb7c2d57c0404 Mon Sep 17 00:00:00 2001 From: Xin Date: Wed, 21 Aug 2019 16:12:12 -0400 Subject: [PATCH 419/542] =?UTF-8?q?tls:=20maintain=20a=20free=20slot=20ind?= =?UTF-8?q?ex=20set=20in=20TLS=20InstanceImpl=20to=20allocate=20in=20O(1?= =?UTF-8?q?=E2=80=A6=20(#7979)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Xin Zhuang --- .../common/thread_local/thread_local_impl.cc | 23 +++++++++++-------- .../common/thread_local/thread_local_impl.h | 3 +++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/source/common/thread_local/thread_local_impl.cc b/source/common/thread_local/thread_local_impl.cc index 4e8c32fed776..9781db07797b 100644 --- a/source/common/thread_local/thread_local_impl.cc +++ b/source/common/thread_local/thread_local_impl.cc @@ -1,5 +1,6 @@ #include "common/thread_local/thread_local_impl.h" +#include #include #include #include @@ -24,16 +25,16 @@ SlotPtr InstanceImpl::allocateSlot() { ASSERT(std::this_thread::get_id() == main_thread_id_); ASSERT(!shutdown_); - for (uint64_t i = 0; i < slots_.size(); i++) { - if (slots_[i] == nullptr) { - std::unique_ptr slot(new SlotImpl(*this, i)); - slots_[i] = slot.get(); - return slot; - } + if (free_slot_indexes_.empty()) { + std::unique_ptr slot(new SlotImpl(*this, slots_.size())); + slots_.push_back(slot.get()); + return slot; } - - std::unique_ptr slot(new SlotImpl(*this, slots_.size())); - slots_.push_back(slot.get()); + const uint32_t idx = free_slot_indexes_.front(); + free_slot_indexes_.pop_front(); + ASSERT(idx < slots_.size()); + std::unique_ptr slot(new SlotImpl(*this, idx)); + slots_[idx] = slot.get(); return slot; } @@ -73,6 +74,10 @@ void InstanceImpl::removeSlot(SlotImpl& slot) { const uint64_t index = slot.index_; slots_[index] = nullptr; + ASSERT(std::find(free_slot_indexes_.begin(), free_slot_indexes_.end(), index) == + free_slot_indexes_.end(), + fmt::format("slot index {} already in free slot set!", index)); + free_slot_indexes_.push_back(index); runOnAllThreads([index]() -> void { // This runs on each thread and clears the slot, making it available for a new allocations. // This is safe even if a new allocation comes in, because everything happens with post() and diff --git a/source/common/thread_local/thread_local_impl.h b/source/common/thread_local/thread_local_impl.h index 3e8e39c8fa89..39a0f12a3e4f 100644 --- a/source/common/thread_local/thread_local_impl.h +++ b/source/common/thread_local/thread_local_impl.h @@ -57,6 +57,9 @@ class InstanceImpl : Logger::Loggable, public Instance { static thread_local ThreadLocalData thread_local_data_; std::vector slots_; + // A list of index of freed slots. + std::list free_slot_indexes_; + std::list> registered_threads_; std::thread::id main_thread_id_; Event::Dispatcher* main_thread_dispatcher_{}; From 09466b586dda71980bbe998fc4f3dd3a40bf05bf Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Wed, 21 Aug 2019 13:12:45 -0700 Subject: [PATCH 420/542] redis: handle invalid ip address from cluster slots and added tests (#7984) Signed-off-by: Henry Yang --- .../clusters/redis/redis_cluster.cc | 15 +++--- .../extensions/clusters/redis/redis_cluster.h | 3 ++ .../clusters/redis/redis_cluster_test.cc | 48 ++++++++++++------- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index 9e70ceaf7cf3..dc41651b0ca1 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -170,11 +170,11 @@ RedisCluster::RedisDiscoverySession::RedisDiscoverySession( resolve_timer_(parent.dispatcher_.createTimer([this]() -> void { startResolveRedis(); })), client_factory_(client_factory), buffer_timeout_(0) {} -namespace { // Convert the cluster slot IP/Port response to and address, return null if the response does not // match the expected type. Network::Address::InstanceConstSharedPtr -ProcessCluster(const NetworkFilters::Common::Redis::RespValue& value) { +RedisCluster::RedisDiscoverySession::RedisDiscoverySession::ProcessCluster( + const NetworkFilters::Common::Redis::RespValue& value) { if (value.type() != NetworkFilters::Common::Redis::RespType::Array) { return nullptr; } @@ -185,14 +185,13 @@ ProcessCluster(const NetworkFilters::Common::Redis::RespValue& value) { return nullptr; } - std::string address = array[0].asString(); - bool ipv6 = (address.find(':') != std::string::npos); - if (ipv6) { - return std::make_shared(address, array[1].asInteger()); + try { + return Network::Utility::parseInternetAddress(array[0].asString(), array[1].asInteger(), false); + } catch (const EnvoyException& ex) { + ENVOY_LOG(debug, "Invalid ip address in CLUSTER SLOTS response: {}", ex.what()); + return nullptr; } - return std::make_shared(address, array[1].asInteger()); } -} // namespace RedisCluster::RedisDiscoverySession::~RedisDiscoverySession() { if (current_request_) { diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index 9117fa7daccc..f4038b7629f1 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -227,6 +227,9 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { bool onRedirection(const NetworkFilters::Common::Redis::RespValue&) override { return true; } void onUnexpectedResponse(const NetworkFilters::Common::Redis::RespValuePtr&); + Network::Address::InstanceConstSharedPtr + ProcessCluster(const NetworkFilters::Common::Redis::RespValue& value); + RedisCluster& parent_; Event::Dispatcher& dispatcher_; std::string current_host_address_; diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 7d6a4967cc8a..a3190debb640 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -55,6 +55,8 @@ const std::string BasicConfig = R"EOF( )EOF"; } +static const int ResponseFlagSize = 11; +static const int ResponseReplicaFlagSize = 4; class RedisClusterTest : public testing::Test, public Extensions::NetworkFilters::Common::Redis::Client::ClientFactory { public: @@ -192,7 +194,7 @@ class RedisClusterTest : public testing::Test, replica_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); replica_1[1].asInteger() = port; - std::vector slot_1(4); + std::vector slot_1(ResponseReplicaFlagSize); slot_1[0].type(NetworkFilters::Common::Redis::RespType::Integer); slot_1[0].asInteger() = 0; slot_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); @@ -280,7 +282,7 @@ class RedisClusterTest : public testing::Test, replica_2[1].type(NetworkFilters::Common::Redis::RespType::Integer); replica_2[1].asInteger() = 22120; - std::vector slot_1(4); + std::vector slot_1(ResponseReplicaFlagSize); slot_1[0].type(NetworkFilters::Common::Redis::RespType::Integer); slot_1[0].asInteger() = 0; slot_1[1].type(NetworkFilters::Common::Redis::RespType::Integer); @@ -290,7 +292,7 @@ class RedisClusterTest : public testing::Test, slot_1[3].type(NetworkFilters::Common::Redis::RespType::Array); slot_1[3].asArray().swap(replica_1); - std::vector slot_2(4); + std::vector slot_2(ResponseReplicaFlagSize); slot_2[0].type(NetworkFilters::Common::Redis::RespType::Integer); slot_2[0].asInteger() = 10000; slot_2[1].type(NetworkFilters::Common::Redis::RespType::Integer); @@ -321,7 +323,7 @@ class RedisClusterTest : public testing::Test, respValue.asString() = correct_value; } else { respValue.type(NetworkFilters::Common::Redis::RespType::Integer); - respValue.asInteger() = 10; + respValue.asInteger() = ResponseFlagSize; } return respValue; } @@ -355,8 +357,9 @@ class RedisClusterTest : public testing::Test, // Create a redis cluster slot response. If a bit is set in the bitset, then that part of // of the response is correct, otherwise it's incorrect. - NetworkFilters::Common::Redis::RespValuePtr createResponse(std::bitset<10> flags, - std::bitset<3> replica_flags) const { + NetworkFilters::Common::Redis::RespValuePtr + createResponse(std::bitset flags, + std::bitset replica_flags) const { int64_t idx(0); int64_t slots_type = idx++; int64_t slots_size = idx++; @@ -367,16 +370,22 @@ class RedisClusterTest : public testing::Test, int64_t master_type = idx++; int64_t master_size = idx++; int64_t master_ip_type = idx++; + int64_t master_ip_value = idx++; int64_t master_port_type = idx++; idx = 0; int64_t replica_size = idx++; int64_t replica_ip_type = idx++; + int64_t replica_ip_value = idx++; int64_t replica_port_type = idx++; std::vector master_1_array; if (flags.test(master_size)) { // Ip field. - master_1_array.push_back(createStringField(flags.test(master_ip_type), "127.0.0.1")); + if (flags.test(master_ip_value)) { + master_1_array.push_back(createStringField(flags.test(master_ip_type), "127.0.0.1")); + } else { + master_1_array.push_back(createStringField(flags.test(master_ip_type), "bad ip foo")); + } // Port field. master_1_array.push_back(createIntegerField(flags.test(master_port_type), 22120)); } @@ -384,8 +393,13 @@ class RedisClusterTest : public testing::Test, std::vector replica_1_array; if (replica_flags.any()) { // Ip field. - replica_1_array.push_back( - createStringField(replica_flags.test(replica_ip_type), "127.0.0.2")); + if (replica_flags.test(replica_ip_value)) { + replica_1_array.push_back( + createStringField(replica_flags.test(replica_ip_type), "127.0.0.2")); + } else { + replica_1_array.push_back( + createStringField(replica_flags.test(replica_ip_type), "bad ip bar")); + } // Port field. replica_1_array.push_back(createIntegerField(replica_flags.test(replica_port_type), 22120)); } @@ -771,8 +785,8 @@ TEST_F(RedisClusterTest, RedisErrorResponse) { EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); - std::bitset<10> single_slot_master(0x7ff); - std::bitset<3> no_replica(0); + std::bitset single_slot_master(0xfff); + std::bitset no_replica(0); expectClusterSlotResponse(createResponse(single_slot_master, no_replica)); expectHealthyHosts(std::list({"127.0.0.1:22120"})); @@ -780,8 +794,8 @@ TEST_F(RedisClusterTest, RedisErrorResponse) { uint64_t update_attempt = 2; uint64_t update_failure = 1; // Test every combination the cluster slots response. - for (uint64_t i = 0; i < (1 << 10); i++) { - std::bitset<10> flags(i); + for (uint64_t i = 0; i < (1 << ResponseFlagSize); i++) { + std::bitset flags(i); expectRedisResolve(); resolve_timer_->invokeCallback(); if (flags.all()) { @@ -807,8 +821,8 @@ TEST_F(RedisClusterTest, RedisReplicaErrorResponse) { EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); EXPECT_CALL(*cluster_callback_, onClusterSlotUpdate(_, _)).Times(1); - std::bitset<10> single_slot_master(0x7ff); - std::bitset<3> no_replica(0); + std::bitset single_slot_master(0xfff); + std::bitset no_replica(0); expectClusterSlotResponse(createResponse(single_slot_master, no_replica)); expectHealthyHosts(std::list({"127.0.0.1:22120"})); @@ -816,8 +830,8 @@ TEST_F(RedisClusterTest, RedisReplicaErrorResponse) { uint64_t update_attempt = 1; uint64_t update_failure = 0; // Test every combination the replica error response. - for (uint64_t i = 1; i < (1 << 3); i++) { - std::bitset<3> replica_flags(i); + for (uint64_t i = 1; i < (1 << ResponseReplicaFlagSize); i++) { + std::bitset replica_flags(i); expectRedisResolve(); resolve_timer_->invokeCallback(); if (replica_flags.all()) { From 6ab225d3c9e24ac22e93d5d12c44b9d4336d0ab4 Mon Sep 17 00:00:00 2001 From: htuch Date: Wed, 21 Aug 2019 17:08:36 -0400 Subject: [PATCH 421/542] protobuf: report field numbers for unknown fields. (#7978) Since binary proto won't have field names, report at least the field numbers, as per https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.unknown_field_set#UnknownField. Also fix minor typo encountered while doing this work. Risk level: Low Testing: Unit tests added/updated. Fixes #7937 Signed-off-by: Harvey Tuch --- source/common/protobuf/utility.cc | 15 +++++++++++++++ source/common/protobuf/utility.h | 6 +----- test/common/protobuf/utility_test.cc | 25 +++++++++++++++++++++---- test/server/server_test.cc | 2 +- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index 50a95b419974..33185a00fc5e 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -88,6 +88,21 @@ ProtoValidationException::ProtoValidationException(const std::string& validation ENVOY_LOG_MISC(debug, "Proto validation error; throwing {}", what()); } +void MessageUtil::checkUnknownFields(const Protobuf::Message& message, + ProtobufMessage::ValidationVisitor& validation_visitor) { + const auto& unknown_fields = message.GetReflection()->GetUnknownFields(message); + // If there are no unknown fields, we're done here. + if (unknown_fields.empty()) { + return; + } + std::string error_msg; + for (int n = 0; n < unknown_fields.field_count(); ++n) { + error_msg += absl::StrCat(n > 0 ? ", " : "", unknown_fields.field(n).number()); + } + validation_visitor.onUnknownField("type " + message.GetTypeName() + " with unknown field set {" + + error_msg + "}"); +} + void MessageUtil::loadFromJson(const std::string& json, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor) { Protobuf::util::JsonParseOptions options; diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index a05587338e26..2f2aff2a3f6e 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -207,11 +207,7 @@ class MessageUtil { } static void checkUnknownFields(const Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor) { - if (!message.GetReflection()->GetUnknownFields(message).empty()) { - validation_visitor.onUnknownField("type " + message.GetTypeName()); - } - } + ProtobufMessage::ValidationVisitor& validation_visitor); static void loadFromJson(const std::string& json, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor); diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 04fb200fc075..9d437982a0e0 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -147,15 +147,31 @@ TEST_F(ProtobufUtilityTest, LoadBinaryProtoFromFile) { EXPECT_TRUE(TestUtility::protoEqual(bootstrap, proto_from_file)); } +// An unknown field (or with wrong type) in a message is rejected. TEST_F(ProtobufUtilityTest, LoadBinaryProtoUnknownFieldFromFile) { ProtobufWkt::Duration source_duration; source_duration.set_seconds(42); const std::string filename = TestEnvironment::writeStringToFileForTest("proto.pb", source_duration.SerializeAsString()); envoy::config::bootstrap::v2::Bootstrap proto_from_file; - EXPECT_THROW_WITH_MESSAGE( - TestUtility::loadFromFile(filename, proto_from_file, *api_), EnvoyException, - "Protobuf message (type envoy.config.bootstrap.v2.Bootstrap) has unknown fields"); + EXPECT_THROW_WITH_MESSAGE(TestUtility::loadFromFile(filename, proto_from_file, *api_), + EnvoyException, + "Protobuf message (type envoy.config.bootstrap.v2.Bootstrap with " + "unknown field set {1}) has unknown fields"); +} + +// Multiple unknown fields (or with wrong type) in a message are rejected. +TEST_F(ProtobufUtilityTest, LoadBinaryProtoUnknownMultipleFieldsFromFile) { + ProtobufWkt::Duration source_duration; + source_duration.set_seconds(42); + source_duration.set_nanos(42); + const std::string filename = + TestEnvironment::writeStringToFileForTest("proto.pb", source_duration.SerializeAsString()); + envoy::config::bootstrap::v2::Bootstrap proto_from_file; + EXPECT_THROW_WITH_MESSAGE(TestUtility::loadFromFile(filename, proto_from_file, *api_), + EnvoyException, + "Protobuf message (type envoy.config.bootstrap.v2.Bootstrap with " + "unknown field set {1, 2}) has unknown fields"); } TEST_F(ProtobufUtilityTest, LoadTextProtoFromFile) { @@ -333,7 +349,8 @@ TEST_F(ProtobufUtilityTest, AnyConvertWrongFields) { source_any.set_type_url("type.google.com/google.protobuf.Timestamp"); EXPECT_THROW_WITH_MESSAGE(TestUtility::anyConvert(source_any), EnvoyException, - "Protobuf message (type google.protobuf.Timestamp) has unknown fields"); + "Protobuf message (type google.protobuf.Timestamp with unknown " + "field set {1}) has unknown fields"); } TEST_F(ProtobufUtilityTest, JsonConvertSuccess) { diff --git a/test/server/server_test.cc b/test/server/server_test.cc index ca2a755d26ac..1b9b0deeee19 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -696,7 +696,7 @@ TEST_P(ServerInstanceImplTest, EmptyBootstrap) { } // Custom header bootstrap succeeds. -TEST_P(ServerInstanceImplTest, CusomHeaderBoostrap) { +TEST_P(ServerInstanceImplTest, CustomHeaderBootstrap) { options_.config_path_ = TestEnvironment::writeStringToFileForTest( "custom.yaml", "header_prefix: \"x-envoy\"\nstatic_resources:\n"); options_.service_cluster_name_ = "some_cluster_name"; From 4549d12017d7583c63d7cee2c6954e3e2c26ec3e Mon Sep 17 00:00:00 2001 From: manish-404 <52198964+manish-404@users.noreply.github.com> Date: Thu, 22 Aug 2019 03:23:34 +0530 Subject: [PATCH 422/542] Content in envoy docs does not cover whole page (#7993) Signed-off-by: Manish Kumar --- docs/root/_static/css/envoy.css | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/root/_static/css/envoy.css b/docs/root/_static/css/envoy.css index 390a0fec1880..05569d4f134a 100644 --- a/docs/root/_static/css/envoy.css +++ b/docs/root/_static/css/envoy.css @@ -1,4 +1,9 @@ -@import "theme.css"; +@import url("theme.css"); + +/*Changing content max-width 100% ( 800px is default )*/ +.wy-nav-content { + max-width: 100% !important; +} /* Splits a long line descriptions in tables in to multiple lines */ .wy-table-responsive table td, .wy-table-responsive table th { @@ -8,4 +13,4 @@ /* align multi line csv table columns */ table.docutils div.line-block { margin-left: 0; -} \ No newline at end of file +} From 719245f988c779ac7d1edb8e8a49df26c5353e70 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Thu, 22 Aug 2019 07:21:31 +0900 Subject: [PATCH 423/542] stats: Add option to switch between fake and real symbol-tables on the command-line. (#7882) * Add option to switch between fake and real symbol-tables on the command-line. Signed-off-by: Joshua Marantz --- docs/root/intro/version_history.rst | 1 + include/envoy/server/options.h | 5 + include/envoy/stats/symbol_table.h | 2 +- source/common/stats/BUILD | 12 +++ source/common/stats/isolated_store_impl.cc | 4 +- source/common/stats/isolated_store_impl.h | 2 +- source/common/stats/symbol_table_creator.cc | 24 +++++ source/common/stats/symbol_table_creator.h | 57 +++++++++++ source/exe/BUILD | 2 +- source/exe/main_common.cc | 5 +- source/exe/main_common.h | 2 +- source/server/BUILD | 1 + source/server/options_impl.cc | 8 +- source/server/options_impl.h | 5 + test/common/grpc/context_impl_test.cc | 4 +- .../grpc_client_integration_test_harness.h | 2 +- test/common/http/BUILD | 1 + test/common/http/codes_test.cc | 8 +- .../http/conn_manager_impl_fuzz_test.cc | 5 +- test/common/router/config_impl_test.cc | 2 +- test/common/stats/BUILD | 1 + test/common/stats/stat_test_utility.h | 8 ++ test/common/stats/thread_local_store_test.cc | 97 +++++++++++++------ .../http1_bridge_filter_test.cc | 2 +- .../http/grpc_web/grpc_web_filter_test.cc | 2 +- .../lightstep/lightstep_tracer_impl_test.cc | 2 +- test/integration/BUILD | 2 + test/integration/integration.cc | 4 +- test/integration/integration.h | 2 + test/integration/integration_admin_test.cc | 7 +- test/integration/server.cc | 5 +- test/integration/stats_integration_test.cc | 51 +++++++++- test/mocks/router/BUILD | 1 + test/mocks/router/mocks.h | 5 +- test/mocks/server/mocks.cc | 2 +- test/mocks/server/mocks.h | 3 +- test/mocks/stats/BUILD | 1 + test/mocks/stats/mocks.cc | 5 +- test/mocks/stats/mocks.h | 35 +++++-- test/mocks/upstream/host.h | 4 +- .../config_validation/cluster_manager_test.cc | 1 - test/server/http/BUILD | 1 + test/server/http/admin_test.cc | 15 +-- test/server/options_impl_test.cc | 7 +- test/tools/router_check/router.h | 2 +- 45 files changed, 334 insertions(+), 83 deletions(-) create mode 100644 source/common/stats/symbol_table_creator.cc create mode 100644 source/common/stats/symbol_table_creator.h diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index c40a3454e4b1..2a049c36c740 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -28,6 +28,7 @@ Version history * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. +* performance: stats symbol table implementation (disabled by default; to test it, add "--use-fake-symbol-table 0" to the command-line arguments when starting Envoy). * redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. * rbac: added support for DNS SAN as :ref:`principal_name `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. diff --git a/include/envoy/server/options.h b/include/envoy/server/options.h index 8ecc14f0c555..32b277321790 100644 --- a/include/envoy/server/options.h +++ b/include/envoy/server/options.h @@ -184,6 +184,11 @@ class Options { */ virtual bool libeventBufferEnabled() const PURE; + /** + * @return whether to use the fake symbol table implementation. + */ + virtual bool fakeSymbolTableEnabled() const PURE; + /** * @return bool indicating whether cpuset size should determine the number of worker threads. */ diff --git a/include/envoy/stats/symbol_table.h b/include/envoy/stats/symbol_table.h index 8f15df4534da..a4346d69397b 100644 --- a/include/envoy/stats/symbol_table.h +++ b/include/envoy/stats/symbol_table.h @@ -185,7 +185,7 @@ class SymbolTable { virtual StoragePtr encode(absl::string_view name) PURE; }; -using SharedSymbolTable = std::shared_ptr; +using SymbolTablePtr = std::unique_ptr; } // namespace Stats } // namespace Envoy diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index cb88ed2dc976..7bc419766cdc 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -50,6 +50,7 @@ envoy_cc_library( ":scope_prefixer_lib", ":stats_lib", ":store_impl_lib", + ":symbol_table_creator_lib", "//include/envoy/stats:stats_macros", "//source/common/stats:allocator_lib", ], @@ -155,6 +156,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "symbol_table_creator_lib", + srcs = ["symbol_table_creator.cc"], + hdrs = ["symbol_table_creator.h"], + external_deps = ["abseil_base"], + deps = [ + ":fake_symbol_table_lib", + ":symbol_table_lib", + ], +) + envoy_cc_library( name = "fake_symbol_table_lib", hdrs = ["fake_symbol_table_impl.h"], diff --git a/source/common/stats/isolated_store_impl.cc b/source/common/stats/isolated_store_impl.cc index 6f2da0f899af..aa58676ae59a 100644 --- a/source/common/stats/isolated_store_impl.cc +++ b/source/common/stats/isolated_store_impl.cc @@ -8,13 +8,13 @@ #include "common/stats/fake_symbol_table_impl.h" #include "common/stats/histogram_impl.h" #include "common/stats/scope_prefixer.h" +#include "common/stats/symbol_table_creator.h" #include "common/stats/utility.h" namespace Envoy { namespace Stats { -IsolatedStoreImpl::IsolatedStoreImpl() - : IsolatedStoreImpl(std::make_unique()) {} +IsolatedStoreImpl::IsolatedStoreImpl() : IsolatedStoreImpl(SymbolTableCreator::makeSymbolTable()) {} IsolatedStoreImpl::IsolatedStoreImpl(std::unique_ptr&& symbol_table) : IsolatedStoreImpl(*symbol_table) { diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index 6a11ceec2a7c..cce741211df1 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -133,7 +133,7 @@ class IsolatedStoreImpl : public StoreImpl { private: IsolatedStoreImpl(std::unique_ptr&& symbol_table); - std::unique_ptr symbol_table_storage_; + SymbolTablePtr symbol_table_storage_; AllocatorImpl alloc_; IsolatedStatsCache counters_; IsolatedStatsCache gauges_; diff --git a/source/common/stats/symbol_table_creator.cc b/source/common/stats/symbol_table_creator.cc new file mode 100644 index 000000000000..8b29313130b5 --- /dev/null +++ b/source/common/stats/symbol_table_creator.cc @@ -0,0 +1,24 @@ +#include "common/stats/symbol_table_creator.h" + +namespace Envoy { +namespace Stats { + +bool SymbolTableCreator::initialized_ = false; +bool SymbolTableCreator::use_fake_symbol_tables_ = true; + +SymbolTablePtr SymbolTableCreator::initAndMakeSymbolTable(bool use_fake) { + ASSERT(!initialized_ || (use_fake_symbol_tables_ == use_fake)); + use_fake_symbol_tables_ = use_fake; + return makeSymbolTable(); +} + +SymbolTablePtr SymbolTableCreator::makeSymbolTable() { + initialized_ = true; + if (use_fake_symbol_tables_) { + return std::make_unique(); + } + return std::make_unique(); +} + +} // namespace Stats +} // namespace Envoy diff --git a/source/common/stats/symbol_table_creator.h b/source/common/stats/symbol_table_creator.h new file mode 100644 index 000000000000..4b51468890ce --- /dev/null +++ b/source/common/stats/symbol_table_creator.h @@ -0,0 +1,57 @@ +#pragma once + +#include "common/stats/fake_symbol_table_impl.h" +#include "common/stats/symbol_table_impl.h" + +namespace Envoy { +namespace Stats { + +namespace TestUtil { +class SymbolTableCreatorTestPeer; +} + +class SymbolTableCreator { +public: + /** + * Initializes the symbol-table creation system. Once this is called, it is a + * runtime assertion to call this again in production code, changing the + * use_fakes setting. However, tests can change the setting via + * TestUtil::SymbolTableCreatorTestPeer::setUseFakeSymbolTables(use_fakes). + * + * @param use_fakes Whether to use fake symbol tables; typically from a command-line option. + * @return a SymbolTable. + */ + static SymbolTablePtr initAndMakeSymbolTable(bool use_fakes); + + /** + * Factory method to create SymbolTables. This is needed to help make it + * possible to flag-flip use of real symbol tables, and ultimately should be + * removed. + * + * @return a SymbolTable. + */ + static SymbolTablePtr makeSymbolTable(); + + /** + * @return whether the system is initialized to use fake symbol tables. + */ + static bool useFakeSymbolTables() { return use_fake_symbol_tables_; } + +private: + friend class TestUtil::SymbolTableCreatorTestPeer; + + /** + * Sets whether fake or real symbol tables should be used. Tests that alter + * this should restore previous value at the end of the test. This must be + * called via TestUtil::SymbolTableCreatorTestPeer. + * + * *param use_fakes whether to use fake symbol tables. + */ + static void setUseFakeSymbolTables(bool use_fakes) { use_fake_symbol_tables_ = use_fakes; } + + static bool initialized_; + static bool use_fake_symbol_tables_; +}; + +} // namespace Stats +} // namespace Envoy diff --git a/source/exe/BUILD b/source/exe/BUILD index 4117a4bb83ca..d48f99ce33b8 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -70,7 +70,7 @@ envoy_cc_library( "//source/common/api:os_sys_calls_lib", "//source/common/common:compiler_requirements_lib", "//source/common/common:perf_annotation_lib", - "//source/common/stats:fake_symbol_table_lib", + "//source/common/stats:symbol_table_creator_lib", "//source/server:hot_restart_lib", "//source/server:hot_restart_nop_lib", "//source/server/config_validation:server_lib", diff --git a/source/exe/main_common.cc b/source/exe/main_common.cc index 84d57abfc5ac..ed76b4381847 100644 --- a/source/exe/main_common.cc +++ b/source/exe/main_common.cc @@ -7,6 +7,7 @@ #include "common/common/compiler_requirements.h" #include "common/common/perf_annotation.h" #include "common/network/utility.h" +#include "common/stats/symbol_table_creator.h" #include "common/stats/thread_local_store.h" #include "server/config_validation/server.h" @@ -45,7 +46,9 @@ MainCommonBase::MainCommonBase(const OptionsImpl& options, Event::TimeSystem& ti Filesystem::Instance& file_system, std::unique_ptr process_context) : options_(options), component_factory_(component_factory), thread_factory_(thread_factory), - file_system_(file_system), stats_allocator_(symbol_table_) { + file_system_(file_system), symbol_table_(Stats::SymbolTableCreator::initAndMakeSymbolTable( + options_.fakeSymbolTableEnabled())), + stats_allocator_(*symbol_table_) { switch (options_.mode()) { case Server::Mode::InitOnly: case Server::Mode::Serve: { diff --git a/source/exe/main_common.h b/source/exe/main_common.h index 5faf31b547cc..a0a4796de18e 100644 --- a/source/exe/main_common.h +++ b/source/exe/main_common.h @@ -67,10 +67,10 @@ class MainCommonBase { protected: ProcessWide process_wide_; // Process-wide state setup/teardown. const Envoy::OptionsImpl& options_; - Stats::FakeSymbolTableImpl symbol_table_; Server::ComponentFactory& component_factory_; Thread::ThreadFactory& thread_factory_; Filesystem::Instance& file_system_; + Stats::SymbolTablePtr symbol_table_; Stats::AllocatorImpl stats_allocator_; std::unique_ptr tls_; diff --git a/source/server/BUILD b/source/server/BUILD index 2e83ab4671c6..bd9c72b56e63 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -378,6 +378,7 @@ envoy_cc_library( "//source/common/runtime:runtime_lib", "//source/common/secret:secret_manager_impl_lib", "//source/common/singleton:manager_impl_lib", + "//source/common/stats:symbol_table_creator_lib", "//source/common/stats:thread_local_store_lib", "//source/common/upstream:cluster_manager_lib", "//source/common/upstream:health_discovery_service_lib", diff --git a/source/server/options_impl.cc b/source/server/options_impl.cc index c3fd28734149..2beca59d6f3c 100644 --- a/source/server/options_impl.cc +++ b/source/server/options_impl.cc @@ -112,7 +112,9 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv, TCLAP::ValueArg use_libevent_buffer("", "use-libevent-buffers", "Use the original libevent buffer implementation", false, false, "bool", cmd); - + TCLAP::ValueArg use_fake_symbol_table("", "use-fake-symbol-table", + "Use fake symbol table implementation", false, true, + "bool", cmd); cmd.setExceptionHandling(false); try { cmd.parse(argc, argv); @@ -136,6 +138,7 @@ OptionsImpl::OptionsImpl(int argc, const char* const* argv, mutex_tracing_enabled_ = enable_mutex_tracing.getValue(); libevent_buffer_enabled_ = use_libevent_buffer.getValue(); + fake_symbol_table_enabled_ = use_fake_symbol_table.getValue(); cpuset_threads_ = cpuset_threads.getValue(); log_level_ = default_log_level; @@ -300,6 +303,7 @@ OptionsImpl::OptionsImpl(const std::string& service_cluster, const std::string& service_cluster_(service_cluster), service_node_(service_node), service_zone_(service_zone), file_flush_interval_msec_(10000), drain_time_(600), parent_shutdown_time_(900), mode_(Server::Mode::Serve), hot_restart_disabled_(false), signal_handling_enabled_(true), - mutex_tracing_enabled_(false), cpuset_threads_(false), libevent_buffer_enabled_(false) {} + mutex_tracing_enabled_(false), cpuset_threads_(false), libevent_buffer_enabled_(false), + fake_symbol_table_enabled_(false) {} } // namespace Envoy diff --git a/source/server/options_impl.h b/source/server/options_impl.h index 7fea3a2546a1..a06365c64e6c 100644 --- a/source/server/options_impl.h +++ b/source/server/options_impl.h @@ -81,6 +81,9 @@ class OptionsImpl : public Server::Options, protected Logger::Loggable cluster; - Envoy::Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Stats::StatNamePool pool(*symbol_table_); const Stats::StatName service = pool.add("service"); const Stats::StatName method = pool.add("method"); @@ -58,7 +58,7 @@ TEST(GrpcContextTest, ResolveServiceAndMethod) { Http::HeaderMapImpl headers; Http::HeaderEntry& path = headers.insertPath(); path.value(std::string("/service_name/method_name")); - Envoy::Test::Global symbol_table; + Stats::TestSymbolTable symbol_table; ContextImpl context(*symbol_table); absl::optional request_names = context.resolveServiceAndMethod(&path); EXPECT_TRUE(request_names); diff --git a/test/common/grpc/grpc_client_integration_test_harness.h b/test/common/grpc/grpc_client_integration_test_harness.h index 997bae8b6bb8..1d02a8e81de6 100644 --- a/test/common/grpc/grpc_client_integration_test_harness.h +++ b/test/common/grpc/grpc_client_integration_test_harness.h @@ -416,7 +416,7 @@ class GrpcClientIntegrationTest : public GrpcClientIntegrationParamTest { FakeHttpConnectionPtr fake_connection_; std::vector fake_streams_; const Protobuf::MethodDescriptor* method_descriptor_; - Envoy::Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Stats::IsolatedStoreImpl* stats_store_ = new Stats::IsolatedStoreImpl(*symbol_table_); Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 56eefc2b25a6..8e13f1625db3 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -161,6 +161,7 @@ envoy_cc_fuzz_test( "//source/common/http:date_provider_lib", "//source/common/network:address_lib", "//source/common/network:utility_lib", + "//source/common/stats:symbol_table_creator_lib", "//test/fuzz:utility_lib", "//test/mocks/access_log:access_log_mocks", "//test/mocks/http:http_mocks", diff --git a/test/common/http/codes_test.cc b/test/common/http/codes_test.cc index 1bdd1f3202e4..de05c4178e94 100644 --- a/test/common/http/codes_test.cc +++ b/test/common/http/codes_test.cc @@ -8,6 +8,7 @@ #include "common/common/empty_string.h" #include "common/http/codes.h" #include "common/http/header_map_impl.h" +#include "common/stats/symbol_table_creator.h" #include "test/mocks/stats/mocks.h" #include "test/test_common/printers.h" @@ -45,7 +46,7 @@ class CodeUtilityTest : public testing::Test { code_stats_.chargeResponseStat(info); } - Envoy::Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Stats::IsolatedStoreImpl global_store_; Stats::IsolatedStoreImpl cluster_scope_; Http::CodeStatsImpl code_stats_; @@ -276,13 +277,14 @@ TEST_F(CodeUtilityTest, ResponseTimingTest) { class CodeStatsTest : public testing::Test { protected: - CodeStatsTest() : code_stats_(symbol_table_) {} + CodeStatsTest() + : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), code_stats_(*symbol_table_) {} absl::string_view stripTrailingDot(absl::string_view prefix) { return CodeStatsImpl::stripTrailingDot(prefix); } - Stats::FakeSymbolTableImpl symbol_table_; + Stats::SymbolTablePtr symbol_table_; CodeStatsImpl code_stats_; }; diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 5c2bf01fe1ab..47f151b5b1db 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -19,6 +19,7 @@ #include "common/http/exception.h" #include "common/network/address_impl.h" #include "common/network/utility.h" +#include "common/stats/symbol_table_creator.h" #include "test/common/http/conn_manager_impl_common.h" #include "test/common/http/conn_manager_impl_fuzz.pb.h" @@ -382,8 +383,8 @@ DEFINE_PROTO_FUZZER(const test::common::http::ConnManagerImplTestCase& input) { FuzzConfig config; NiceMock drain_close; NiceMock random; - Stats::FakeSymbolTableImpl symbol_table; - Http::ContextImpl http_context(symbol_table); + Stats::SymbolTablePtr symbol_table(Stats::SymbolTableCreator::makeSymbolTable()); + Http::ContextImpl http_context(*symbol_table); NiceMock runtime; NiceMock local_info; NiceMock cluster_manager; diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 5af81cbc1ff9..6b7c5743dc90 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -109,7 +109,7 @@ class ConfigImplTestBase { return factory_context_.scope().symbolTable().toString(name); } - Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Api::ApiPtr api_; NiceMock factory_context_; }; diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 6ab96a754d42..78157bdee706 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -61,6 +61,7 @@ envoy_cc_test_library( deps = [ "//source/common/common:assert_lib", "//source/common/memory:stats_lib", + "//source/common/stats:symbol_table_creator_lib", ], ) diff --git a/test/common/stats/stat_test_utility.h b/test/common/stats/stat_test_utility.h index afc97bb9c6cc..c10fe4032065 100644 --- a/test/common/stats/stat_test_utility.h +++ b/test/common/stats/stat_test_utility.h @@ -2,6 +2,7 @@ #include "common/common/logger.h" #include "common/memory/stats.h" +#include "common/stats/symbol_table_creator.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" @@ -104,6 +105,13 @@ class MemoryTest { } \ } while (false) +class SymbolTableCreatorTestPeer { +public: + static void setUseFakeSymbolTables(bool use_fakes) { + SymbolTableCreator::setUseFakeSymbolTables(use_fakes); + } +}; + } // namespace TestUtil } // namespace Stats } // namespace Envoy diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index 711d1c459e46..be0b87058ced 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -845,46 +845,85 @@ TEST_F(StatsThreadLocalStoreTest, NonHotRestartNoTruncation) { tls_.shutdownThread(); } -// Tests how much memory is consumed allocating 100k stats. -TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTls) { - MockSink sink; - Stats::FakeSymbolTableImpl symbol_table; - AllocatorImpl alloc(symbol_table); - ThreadLocalStoreImpl store(alloc); - store.addSink(sink); +class StatsThreadLocalStoreTestNoFixture : public testing::Test { +protected: + StatsThreadLocalStoreTestNoFixture() + : save_use_fakes_(SymbolTableCreator::useFakeSymbolTables()) {} + ~StatsThreadLocalStoreTestNoFixture() override { + TestUtil::SymbolTableCreatorTestPeer::setUseFakeSymbolTables(save_use_fakes_); + if (threading_enabled_) { + store_->shutdownThreading(); + tls_.shutdownThread(); + } + } - // Use a tag producer that will produce tags. - envoy::config::metrics::v2::StatsConfig stats_config; - store.setTagProducer(std::make_unique(stats_config)); + void init(bool use_fakes) { + TestUtil::SymbolTableCreatorTestPeer::setUseFakeSymbolTables(use_fakes); + symbol_table_ = SymbolTableCreator::makeSymbolTable(); + alloc_ = std::make_unique(*symbol_table_); + store_ = std::make_unique(*alloc_); + store_->addSink(sink_); + + // Use a tag producer that will produce tags. + envoy::config::metrics::v2::StatsConfig stats_config; + store_->setTagProducer(std::make_unique(stats_config)); + } + + void initThreading() { + threading_enabled_ = true; + store_->initializeThreading(main_thread_dispatcher_, tls_); + } + + static constexpr size_t million_ = 1000 * 1000; + + MockSink sink_; + SymbolTablePtr symbol_table_; + std::unique_ptr alloc_; + std::unique_ptr store_; + NiceMock main_thread_dispatcher_; + NiceMock tls_; + const bool save_use_fakes_; + bool threading_enabled_{false}; +}; +// Tests how much memory is consumed allocating 100k stats. +TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTlsFakeSymbolTable) { + init(true); TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat( - 1000, [&store](absl::string_view name) { store.counter(std::string(name)); }); - const size_t million = 1000 * 1000; + 1000, [this](absl::string_view name) { store_->counter(std::string(name)); }); EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 15268336); // June 30, 2019 - EXPECT_MEMORY_LE(memory_test.consumedBytes(), 16 * million); + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 16 * million_); } -TEST(StatsThreadLocalStoreTestNoFixture, MemoryWithTls) { - Stats::FakeSymbolTableImpl symbol_table; - AllocatorImpl alloc(symbol_table); - NiceMock main_thread_dispatcher; - NiceMock tls; - ThreadLocalStoreImpl store(alloc); +TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithTlsFakeSymbolTable) { + init(true); + initThreading(); + TestUtil::MemoryTest memory_test; + TestUtil::forEachSampleStat( + 1000, [this](absl::string_view name) { store_->counter(std::string(name)); }); + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 17496848); // June 30, 2019 + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 18 * million_); +} - // Use a tag producer that will produce tags. - envoy::config::metrics::v2::StatsConfig stats_config; - store.setTagProducer(std::make_unique(stats_config)); +// Tests how much memory is consumed allocating 100k stats. +TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithoutTlsRealSymbolTable) { + init(false); + TestUtil::MemoryTest memory_test; + TestUtil::forEachSampleStat( + 1000, [this](absl::string_view name) { store_->counter(std::string(name)); }); + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 9139120); // Aug 9, 2019 + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 10 * million_); +} - store.initializeThreading(main_thread_dispatcher, tls); +TEST_F(StatsThreadLocalStoreTestNoFixture, MemoryWithTlsRealSymbolTable) { + init(false); + initThreading(); TestUtil::MemoryTest memory_test; TestUtil::forEachSampleStat( - 1000, [&store](absl::string_view name) { store.counter(std::string(name)); }); - const size_t million = 1000 * 1000; - EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 17496848); // June 30, 2019 - EXPECT_MEMORY_LE(memory_test.consumedBytes(), 18 * million); - store.shutdownThreading(); - tls.shutdownThread(); + 1000, [this](absl::string_view name) { store_->counter(std::string(name)); }); + EXPECT_MEMORY_EQ(memory_test.consumedBytes(), 11367632); // Aug 9, 2019 + EXPECT_MEMORY_LE(memory_test.consumedBytes(), 12 * million_); } TEST_F(StatsThreadLocalStoreTest, ShuttingDown) { diff --git a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc index aedebdfa6f0b..aafdb5bdeaa2 100644 --- a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc +++ b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc @@ -36,7 +36,7 @@ class GrpcHttp1BridgeFilterTest : public testing::Test { ~GrpcHttp1BridgeFilterTest() override { filter_.onDestroy(); } - Envoy::Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Grpc::ContextImpl context_; Http1BridgeFilter filter_; NiceMock decoder_callbacks_; diff --git a/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc b/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc index e0a501f48d69..b20f517ee5e9 100644 --- a/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc +++ b/test/extensions/filters/http/grpc_web/grpc_web_filter_test.cc @@ -106,7 +106,7 @@ class GrpcWebFilterTest : public testing::TestWithParamvalue().getStringView()); } - Envoy::Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Grpc::ContextImpl grpc_context_; GrpcWebFilter filter_; NiceMock decoder_callbacks_; diff --git a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc index 046bd9596c4b..cf9e44e1b9f3 100644 --- a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc +++ b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc @@ -89,7 +89,7 @@ class LightStepDriverTest : public testing::Test { SystemTime start_time_; StreamInfo::MockStreamInfo stream_info_; - Envoy::Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Grpc::ContextImpl grpc_context_; NiceMock tls_; Stats::IsolatedStoreImpl stats_; diff --git a/test/integration/BUILD b/test/integration/BUILD index 393eadbfa2e8..60ababd92f17 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -434,6 +434,7 @@ envoy_cc_test_library( "//source/common/network:utility_lib", "//source/common/runtime:runtime_lib", "//source/common/stats:isolated_store_lib", + "//source/common/stats:symbol_table_creator_lib", "//source/common/stats:thread_local_store_lib", "//source/common/thread_local:thread_local_lib", "//source/common/upstream:upstream_includes", @@ -548,6 +549,7 @@ envoy_cc_test( deps = [ ":integration_lib", "//source/common/memory:stats_lib", + "//source/common/stats:symbol_table_creator_lib", "//source/extensions/filters/http/router:config", "//source/extensions/filters/network/http_connection_manager:config", "//test/common/stats:stat_test_utility_lib", diff --git a/test/integration/integration.cc b/test/integration/integration.cc index e0c124bd3c6b..f8fe1910abf1 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -524,9 +524,9 @@ void BaseIntegrationTest::createXdsUpstream() { auto cfg = std::make_unique( tls_context, factory_context_); - static Stats::Scope* upstream_stats_store = new Stats::TestIsolatedStoreImpl(); + upstream_stats_store_ = std::make_unique(); auto context = std::make_unique( - std::move(cfg), context_manager_, *upstream_stats_store, std::vector{}); + std::move(cfg), context_manager_, *upstream_stats_store_, std::vector{}); fake_upstreams_.emplace_back(new FakeUpstream( std::move(context), 0, FakeHttpConnection::Type::HTTP2, version_, timeSystem())); } diff --git a/test/integration/integration.h b/test/integration/integration.h index db32e8a8cfc6..c61dfb0fc006 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -327,6 +327,8 @@ class BaseIntegrationTest : Logger::Loggable { bool initialized() const { return initialized_; } + std::unique_ptr upstream_stats_store_; + // The IpVersion (IPv4, IPv6) to use. Network::Address::IpVersion version_; // IP Address to use when binding sockets on upstreams. diff --git a/test/integration/integration_admin_test.cc b/test/integration/integration_admin_test.cc index 2af1c1f069df..43c3a867bfd6 100644 --- a/test/integration/integration_admin_test.cc +++ b/test/integration/integration_admin_test.cc @@ -500,9 +500,14 @@ TEST_F(IntegrationAdminIpv4Ipv6Test, Ipv4Ipv6Listen) { // Testing the behavior of StatsMatcher, which allows/denies the instantiation of stats based on // restrictions on their names. +// +// Note: using 'Event::TestUsingSimulatedTime' appears to conflict with LDS in +// StatsMatcherIntegrationTest.IncludeExact, which manifests in a coverage test +// crash, which is really difficult to debug. See #7215. It's possible this is +// due to a bad interaction between the wait-for constructs in the integration +// test framework with sim-time. class StatsMatcherIntegrationTest : public testing::Test, - public Event::TestUsingSimulatedTime, public HttpIntegrationTest, public testing::WithParamInterface { public: diff --git a/test/integration/server.cc b/test/integration/server.cc index 6161eeedf8b4..83777e19db1e 100644 --- a/test/integration/server.cc +++ b/test/integration/server.cc @@ -8,6 +8,7 @@ #include "common/common/thread.h" #include "common/local_info/local_info_impl.h" #include "common/network/utility.h" +#include "common/stats/symbol_table_creator.h" #include "common/stats/thread_local_store.h" #include "common/thread_local/thread_local_impl.h" @@ -187,10 +188,10 @@ void IntegrationTestServerImpl::createAndRunEnvoyServer( Runtime::RandomGeneratorPtr&& random_generator, absl::optional> process_object) { { - Stats::FakeSymbolTableImpl symbol_table; + Stats::SymbolTablePtr symbol_table = Stats::SymbolTableCreator::makeSymbolTable(); Server::HotRestartNopImpl restarter; ThreadLocal::InstanceImpl tls; - Stats::AllocatorImpl stats_allocator(symbol_table); + Stats::AllocatorImpl stats_allocator(*symbol_table); Stats::ThreadLocalStoreImpl stat_store(stats_allocator); std::unique_ptr process_context; if (process_object.has_value()) { diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index ceadf07bacd7..746baf523926 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -7,6 +7,7 @@ #include "common/config/well_known_names.h" #include "common/memory/stats.h" +#include "common/stats/symbol_table_creator.h" #include "test/common/stats/stat_test_utility.h" #include "test/config/utility.h" @@ -172,13 +173,24 @@ class ClusterMemoryTestHelper : public BaseIntegrationTest { return memory_test.consumedBytes(); } }; -class ClusterMemoryTestRunner : public testing::TestWithParam {}; +class ClusterMemoryTestRunner : public testing::TestWithParam { +protected: + ClusterMemoryTestRunner() : save_use_fakes_(Stats::SymbolTableCreator::useFakeSymbolTables()) {} + ~ClusterMemoryTestRunner() override { + Stats::TestUtil::SymbolTableCreatorTestPeer::setUseFakeSymbolTables(save_use_fakes_); + } + +private: + const bool save_use_fakes_; +}; INSTANTIATE_TEST_SUITE_P(IpVersions, ClusterMemoryTestRunner, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); -TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { +TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { + Stats::TestUtil::SymbolTableCreatorTestPeer::setUseFakeSymbolTables(true); + // A unique instance of ClusterMemoryTest allows for multiple runs of Envoy with // differing configuration. This is necessary for measuring the memory consumption // between the different instances within the same test. @@ -226,5 +238,40 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { EXPECT_MEMORY_LE(m_per_cluster, 44000); } +TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { + Stats::TestUtil::SymbolTableCreatorTestPeer::setUseFakeSymbolTables(false); + + // A unique instance of ClusterMemoryTest allows for multiple runs of Envoy with + // differing configuration. This is necessary for measuring the memory consumption + // between the different instances within the same test. + const size_t m1 = ClusterMemoryTestHelper::computeMemory(1); + const size_t m1001 = ClusterMemoryTestHelper::computeMemory(1001); + const size_t m_per_cluster = (m1001 - m1) / 1000; + + // Note: if you are increasing this golden value because you are adding a + // stat, please confirm that this will be generally useful to most Envoy + // users. Otherwise you are adding to the per-cluster memory overhead, which + // will be significant for Envoy installations that are massively + // multi-tenant. + // + // History of golden values: + // + // Date PR Bytes Per Cluster Notes + // exact upper-bound + // ---------- ----- ----------------- ----- + // 2019/08/09 7882 35489 36000 Initial version + + // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI + // 'release' builds, where we control the platform and tool-chain. So you + // will need to find the correct value only after failing CI and looking + // at the logs. + // + // On a local clang8/libstdc++/linux flow, the memory usage was observed in + // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may + // vary. + EXPECT_MEMORY_EQ(m_per_cluster, 35489); // 104 bytes higher than a debug build. + EXPECT_MEMORY_LE(m_per_cluster, 36000); +} + } // namespace } // namespace Envoy diff --git a/test/mocks/router/BUILD b/test/mocks/router/BUILD index eead78a0d5fc..d5d8729cccad 100644 --- a/test/mocks/router/BUILD +++ b/test/mocks/router/BUILD @@ -27,5 +27,6 @@ envoy_cc_mock( "//include/envoy/upstream:cluster_manager_interface", "//source/common/stats:fake_symbol_table_lib", "//test/mocks:common_lib", + "//test/mocks/stats:stats_mocks", ], ) diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index aeda21400f98..7e6ce48606ea 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -24,6 +24,7 @@ #include "common/stats/fake_symbol_table_impl.h" +#include "test/mocks/stats/mocks.h" #include "test/test_common/global.h" #include "gmock/gmock.h" @@ -199,7 +200,7 @@ class TestVirtualCluster : public VirtualCluster { // Router::VirtualCluster Stats::StatName statName() const override { return stat_name_.statName(); } - Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Stats::StatNameManagedStorage stat_name_{"fake_virtual_cluster", *symbol_table_}; }; @@ -223,7 +224,7 @@ class MockVirtualHost : public VirtualHost { return stat_name_->statName(); } - mutable Test::Global symbol_table_; + mutable Stats::TestSymbolTable symbol_table_; std::string name_{"fake_vhost"}; mutable std::unique_ptr stat_name_; testing::NiceMock rate_limit_policy_; diff --git a/test/mocks/server/mocks.cc b/test/mocks/server/mocks.cc index 9d2d49f52750..42bcd95947a8 100644 --- a/test/mocks/server/mocks.cc +++ b/test/mocks/server/mocks.cc @@ -77,7 +77,7 @@ MockGuardDog::MockGuardDog() : watch_dog_(new NiceMock()) { } MockGuardDog::~MockGuardDog() = default; -MockHotRestart::MockHotRestart() : stats_allocator_(symbol_table_.get()) { +MockHotRestart::MockHotRestart() : stats_allocator_(*symbol_table_) { ON_CALL(*this, logLock()).WillByDefault(ReturnRef(log_lock_)); ON_CALL(*this, accessLogLock()).WillByDefault(ReturnRef(access_log_lock_)); ON_CALL(*this, statsAllocator()).WillByDefault(ReturnRef(stats_allocator_)); diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 4ba0920f5695..fd0ee9b27ef6 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -85,6 +85,7 @@ class MockOptions : public Options { MOCK_CONST_METHOD0(signalHandlingEnabled, bool()); MOCK_CONST_METHOD0(mutexTracingEnabled, bool()); MOCK_CONST_METHOD0(libeventBufferEnabled, bool()); + MOCK_CONST_METHOD0(fakeSymbolTableEnabled, bool()); MOCK_CONST_METHOD0(cpusetThreadsEnabled, bool()); MOCK_CONST_METHOD0(toCommandLineOptions, Server::CommandLineOptionsPtr()); @@ -218,7 +219,7 @@ class MockHotRestart : public HotRestart { MOCK_METHOD0(statsAllocator, Stats::Allocator&()); private: - Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; Thread::MutexBasicLockable log_lock_; Thread::MutexBasicLockable access_log_lock_; Stats::AllocatorImpl stats_allocator_; diff --git a/test/mocks/stats/BUILD b/test/mocks/stats/BUILD index 6539e818467b..bf9048b25366 100644 --- a/test/mocks/stats/BUILD +++ b/test/mocks/stats/BUILD @@ -22,6 +22,7 @@ envoy_cc_mock( "//source/common/stats:isolated_store_lib", "//source/common/stats:stats_lib", "//source/common/stats:store_impl_lib", + "//source/common/stats:symbol_table_creator_lib", "//test/mocks:common_lib", "//test/test_common:global_lib", ], diff --git a/test/mocks/stats/mocks.cc b/test/mocks/stats/mocks.cc index 56dfbaf84d40..ef5eeebeb488 100644 --- a/test/mocks/stats/mocks.cc +++ b/test/mocks/stats/mocks.cc @@ -63,7 +63,7 @@ MockMetricSnapshot::~MockMetricSnapshot() = default; MockSink::MockSink() = default; MockSink::~MockSink() = default; -MockStore::MockStore() : StoreImpl(*fake_symbol_table_) { +MockStore::MockStore() : StoreImpl(*global_symbol_table_) { ON_CALL(*this, counter(_)).WillByDefault(ReturnRef(counter_)); ON_CALL(*this, histogram(_)).WillByDefault(Invoke([this](const std::string& name) -> Histogram& { auto* histogram = new NiceMock(); // symbol_table_); @@ -75,8 +75,7 @@ MockStore::MockStore() : StoreImpl(*fake_symbol_table_) { } MockStore::~MockStore() = default; -MockIsolatedStatsStore::MockIsolatedStatsStore() - : IsolatedStoreImpl(Test::Global::get()) {} +MockIsolatedStatsStore::MockIsolatedStatsStore() : IsolatedStoreImpl(*global_symbol_table_) {} MockIsolatedStatsStore::~MockIsolatedStatsStore() = default; MockStatsMatcher::MockStatsMatcher() = default; diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h index d418ad430f2b..d5ad120ffcae 100644 --- a/test/mocks/stats/mocks.h +++ b/test/mocks/stats/mocks.h @@ -18,6 +18,7 @@ #include "common/stats/histogram_impl.h" #include "common/stats/isolated_store_impl.h" #include "common/stats/store_impl.h" +#include "common/stats/symbol_table_creator.h" #include "test/test_common/global.h" @@ -26,6 +27,25 @@ namespace Envoy { namespace Stats { +class TestSymbolTableHelper { +public: + TestSymbolTableHelper() : symbol_table_(SymbolTableCreator::makeSymbolTable()) {} + SymbolTable& symbolTable() { return *symbol_table_; } + const SymbolTable& constSymbolTable() const { return *symbol_table_; } + +private: + SymbolTablePtr symbol_table_; +}; + +class TestSymbolTable { +public: + SymbolTable& operator*() { return global_.get().symbolTable(); } + const SymbolTable& operator*() const { return global_.get().constSymbolTable(); } + SymbolTable* operator->() { return &global_.get().symbolTable(); } + const SymbolTable* operator->() const { return &global_.get().constSymbolTable(); } + Envoy::Test::Global global_; +}; + template class MockMetric : public BaseClass { public: MockMetric() : name_(*this), tag_pool_(*symbol_table_) {} @@ -39,7 +59,7 @@ template class MockMetric : public BaseClass { explicit MetricName(MockMetric& mock_metric) : mock_metric_(mock_metric) {} ~MetricName() { if (stat_name_storage_ != nullptr) { - stat_name_storage_->free(*mock_metric_.symbol_table_); + stat_name_storage_->free(mock_metric_.symbolTable()); } } @@ -57,8 +77,8 @@ template class MockMetric : public BaseClass { std::unique_ptr stat_name_storage_; }; - SymbolTable& symbolTable() override { return symbol_table_.get(); } - const SymbolTable& constSymbolTable() const override { return symbol_table_.get(); } + SymbolTable& symbolTable() override { return *symbol_table_; } + const SymbolTable& constSymbolTable() const override { return *symbol_table_; } // Note: cannot be mocked because it is accessed as a Property in a gmock EXPECT_CALL. This // creates a deadlock in gmock and is an unintended use of mock functions. @@ -90,7 +110,7 @@ template class MockMetric : public BaseClass { } } - Test::Global symbol_table_; // Must outlive name_. + TestSymbolTable symbol_table_; // Must outlive name_. MetricName name_; void setTags(const std::vector& tags) { @@ -251,7 +271,7 @@ class MockSink : public Sink { class SymbolTableProvider { public: - Test::Global fake_symbol_table_; + TestSymbolTable global_symbol_table_; }; class MockStore : public SymbolTableProvider, public StoreImpl { @@ -285,7 +305,7 @@ class MockStore : public SymbolTableProvider, public StoreImpl { return histogram(symbol_table_->toString(name)); } - Test::Global symbol_table_; + TestSymbolTable symbol_table_; testing::NiceMock counter_; std::vector> histograms_; }; @@ -294,8 +314,7 @@ class MockStore : public SymbolTableProvider, public StoreImpl { * With IsolatedStoreImpl it's hard to test timing stats. * MockIsolatedStatsStore mocks only deliverHistogramToSinks for better testing. */ -class MockIsolatedStatsStore : private Test::Global, - public IsolatedStoreImpl { +class MockIsolatedStatsStore : public SymbolTableProvider, public IsolatedStoreImpl { public: MockIsolatedStatsStore(); ~MockIsolatedStatsStore() override; diff --git a/test/mocks/upstream/host.h b/test/mocks/upstream/host.h index 095c67c1ae88..6fea5dec1f2e 100644 --- a/test/mocks/upstream/host.h +++ b/test/mocks/upstream/host.h @@ -108,7 +108,7 @@ class MockHostDescription : public HostDescription { testing::NiceMock cluster_; testing::NiceMock stats_store_; HostStats stats_{ALL_HOST_STATS(POOL_COUNTER(stats_store_), POOL_GAUGE(stats_store_))}; - mutable Test::Global symbol_table_; + mutable Stats::TestSymbolTable symbol_table_; mutable std::unique_ptr locality_zone_stat_name_; }; @@ -186,7 +186,7 @@ class MockHost : public Host { testing::NiceMock outlier_detector_; NiceMock stats_store_; HostStats stats_{ALL_HOST_STATS(POOL_COUNTER(stats_store_), POOL_GAUGE(stats_store_))}; - mutable Test::Global symbol_table_; + mutable Stats::TestSymbolTable symbol_table_; mutable std::unique_ptr locality_zone_stat_name_; }; diff --git a/test/server/config_validation/cluster_manager_test.cc b/test/server/config_validation/cluster_manager_test.cc index 0d3639d4860b..0653c0cc8ae7 100644 --- a/test/server/config_validation/cluster_manager_test.cc +++ b/test/server/config_validation/cluster_manager_test.cc @@ -51,7 +51,6 @@ TEST(ValidationClusterManagerTest, MockedMethods) { log_manager, singleton_manager, time_system); const envoy::config::bootstrap::v2::Bootstrap bootstrap; - Stats::FakeSymbolTableImpl symbol_table; ClusterManagerPtr cluster_manager = factory.clusterManagerFromProto(bootstrap); EXPECT_EQ(nullptr, cluster_manager->httpConnPoolForCluster("cluster", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); diff --git a/test/server/http/BUILD b/test/server/http/BUILD index f513226fd1a5..20df4a917b1b 100644 --- a/test/server/http/BUILD +++ b/test/server/http/BUILD @@ -19,6 +19,7 @@ envoy_cc_test( "//source/common/profiler:profiler_lib", "//source/common/protobuf", "//source/common/protobuf:utility_lib", + "//source/common/stats:symbol_table_creator_lib", "//source/common/stats:thread_local_store_lib", "//source/extensions/transport_sockets/tls:context_config_lib", "//source/server/http:admin_lib", diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index 481a1bbfd533..2c9e933ed881 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -14,6 +14,7 @@ #include "common/profiler/profiler.h" #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" +#include "common/stats/symbol_table_creator.h" #include "common/stats/thread_local_store.h" #include "server/http/admin.h" @@ -50,7 +51,8 @@ namespace Server { class AdminStatsTest : public testing::TestWithParam { public: - AdminStatsTest() : alloc_(symbol_table_) { + AdminStatsTest() + : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_) { store_ = std::make_unique(alloc_); store_->addSink(sink_); } @@ -63,7 +65,7 @@ class AdminStatsTest : public testing::TestWithParam main_thread_dispatcher_; NiceMock tls_; Stats::AllocatorImpl alloc_; @@ -1374,15 +1376,16 @@ class HistogramWrapper { class PrometheusStatsFormatterTest : public testing::Test { protected: - PrometheusStatsFormatterTest() : alloc_(symbol_table_) {} + PrometheusStatsFormatterTest() + : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_) {} void addCounter(const std::string& name, std::vector cluster_tags) { - Stats::StatNameManagedStorage storage(name, symbol_table_); + Stats::StatNameManagedStorage storage(name, *symbol_table_); counters_.push_back(alloc_.makeCounter(storage.statName(), name, cluster_tags)); } void addGauge(const std::string& name, std::vector cluster_tags) { - Stats::StatNameManagedStorage storage(name, symbol_table_); + Stats::StatNameManagedStorage storage(name, *symbol_table_); gauges_.push_back(alloc_.makeGauge(storage.statName(), name, cluster_tags, Stats::Gauge::ImportMode::Accumulate)); } @@ -1396,7 +1399,7 @@ class PrometheusStatsFormatterTest : public testing::Test { return MockHistogramSharedPtr(new NiceMock()); } - Stats::FakeSymbolTableImpl symbol_table_; + Stats::SymbolTablePtr symbol_table_; Stats::AllocatorImpl alloc_; std::vector counters_; std::vector gauges_; diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index 71662bb36bd6..bc4cc16f7fa7 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -99,6 +99,7 @@ TEST_F(OptionsImplTest, All) { EXPECT_TRUE(options->cpusetThreadsEnabled()); EXPECT_TRUE(options->allowUnknownStaticFields()); EXPECT_TRUE(options->rejectUnknownDynamicFields()); + EXPECT_TRUE(options->fakeSymbolTableEnabled()); options = createOptionsImpl("envoy --mode init_only"); EXPECT_EQ(Server::Mode::InitOnly, options->mode()); @@ -129,6 +130,7 @@ TEST_F(OptionsImplTest, SetAll) { bool hot_restart_disabled = options->hotRestartDisabled(); bool signal_handling_enabled = options->signalHandlingEnabled(); bool cpuset_threads_enabled = options->cpusetThreadsEnabled(); + bool fake_symbol_table_enabled = options->fakeSymbolTableEnabled(); options->setBaseId(109876); options->setConcurrency(42); @@ -155,6 +157,7 @@ TEST_F(OptionsImplTest, SetAll) { options->setCpusetThreads(!options->cpusetThreadsEnabled()); options->setAllowUnkownFields(true); options->setRejectUnknownFieldsDynamic(true); + options->setFakeSymbolTableEnabled(!options->fakeSymbolTableEnabled()); EXPECT_EQ(109876, options->baseId()); EXPECT_EQ(42U, options->concurrency()); @@ -181,6 +184,7 @@ TEST_F(OptionsImplTest, SetAll) { EXPECT_EQ(!cpuset_threads_enabled, options->cpusetThreadsEnabled()); EXPECT_TRUE(options->allowUnknownStaticFields()); EXPECT_TRUE(options->rejectUnknownDynamicFields()); + EXPECT_EQ(!fake_symbol_table_enabled, options->fakeSymbolTableEnabled()); // Validate that CommandLineOptions is constructed correctly. Server::CommandLineOptionsPtr command_line_options = options->toCommandLineOptions(); @@ -247,7 +251,8 @@ TEST_F(OptionsImplTest, OptionsAreInSyncWithProto) { // 3. ignore_rest - default TCLAP argument. // 4. use-libevent-buffers - short-term override for rollout of new buffer implementation. // 5. allow-unknown-fields - deprecated alias of allow-unknown-static-fields. - EXPECT_EQ(options->count() - 5, command_line_options->GetDescriptor()->field_count()); + // 6. use-fake-symbol-table - short-term override for rollout of real symbol-table implementation. + EXPECT_EQ(options->count() - 6, command_line_options->GetDescriptor()->field_count()); } TEST_F(OptionsImplTest, BadCliOption) { diff --git a/test/tools/router_check/router.h b/test/tools/router_check/router.h index a3395de2ad87..7c81835da7ae 100644 --- a/test/tools/router_check/router.h +++ b/test/tools/router_check/router.h @@ -54,7 +54,7 @@ struct ToolConfig { private: ToolConfig(std::unique_ptr headers, int random_value); - Test::Global symbol_table_; + Stats::TestSymbolTable symbol_table_; }; /** From c7110f8d10d928a3b18ee6a05439d7b4e49e595c Mon Sep 17 00:00:00 2001 From: "Tejasvi (Teju) Nareddy" Date: Wed, 21 Aug 2019 21:12:42 -0700 Subject: [PATCH 424/542] api config: add build rules for go protos (#7987) Some BUILD files are missing build rules to generate go protos. envoyproxy/go-control-plane depends on these protos, so they should be exposed publicly. Added build rules to generate *.pb.go files. Risk Level: Low Testing: These rules were copied to google3 and tested internally. Unfortunately, I am having a bit of trouble with bazel build directly on these targets ("Package is considered deleted due to --deleted_packages"). Please let me know if there is a better way to test this change. Signed-off-by: Teju Nareddy --- api/envoy/config/accesslog/v2/BUILD | 8 +++++++- api/envoy/config/filter/network/tcp_proxy/v2/BUILD | 12 +++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/api/envoy/config/accesslog/v2/BUILD b/api/envoy/config/accesslog/v2/BUILD index d6fd2d358929..85ac228ccf13 100644 --- a/api/envoy/config/accesslog/v2/BUILD +++ b/api/envoy/config/accesslog/v2/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") licenses(["notice"]) # Apache 2 @@ -14,3 +14,9 @@ api_proto_library_internal( name = "file", srcs = ["file.proto"], ) + +api_go_proto_library( + name = "als", + proto = ":als", + deps = ["//envoy/api/v2/core:grpc_service_go_proto"], +) diff --git a/api/envoy/config/filter/network/tcp_proxy/v2/BUILD b/api/envoy/config/filter/network/tcp_proxy/v2/BUILD index d77d910aceb2..e75ab7036b75 100644 --- a/api/envoy/config/filter/network/tcp_proxy/v2/BUILD +++ b/api/envoy/config/filter/network/tcp_proxy/v2/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") licenses(["notice"]) # Apache 2 @@ -11,3 +11,13 @@ api_proto_library_internal( "//envoy/config/filter/accesslog/v2:accesslog", ], ) + +api_go_proto_library( + name = "tcp_proxy", + proto = ":tcp_proxy", + deps = [ + "//envoy/api/v2/core:address_go_proto", + "//envoy/api/v2/core:base_go_proto", + "//envoy/config/filter/accesslog/v2:accesslog_go_proto", + ], +) From 7267542177ef49b33d6dd7b6ae6cdaa4ce2fb9ce Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Wed, 21 Aug 2019 22:30:00 -0700 Subject: [PATCH 425/542] test: don't use on macOS. (#8000) Xcode 11 requires at least macOS 10.15 (upcoming) in order to use either or C++17 . Signed-off-by: Piotr Sikora --- test/test_common/environment.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_common/environment.cc b/test/test_common/environment.cc index 86b0a0ec3bdd..d0f0577f4763 100644 --- a/test/test_common/environment.cc +++ b/test/test_common/environment.cc @@ -8,7 +8,7 @@ #if defined(_LIBCPP_VERSION) && !defined(__APPLE__) #include #elif defined __has_include -#if __has_include() +#if __has_include() && !defined(__APPLE__) #include #endif #endif @@ -44,7 +44,7 @@ std::string makeTempDir(char* name_template) { name_template, strerror(errno))); #if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION >= 9000 && !defined(__APPLE__) std::__fs::filesystem::create_directories(dirname); -#elif defined __cpp_lib_experimental_filesystem +#elif defined __cpp_lib_experimental_filesystem && !defined(__APPLE__) std::experimental::filesystem::create_directories(dirname); #endif #else @@ -102,7 +102,7 @@ void TestEnvironment::createParentPath(const std::string& path) { // We don't want to rely on mkdir etc. if we can avoid it, since it might not // exist in some environments such as ClusterFuzz. std::__fs::filesystem::create_directories(std::__fs::filesystem::path(path).parent_path()); -#elif defined __cpp_lib_experimental_filesystem +#elif defined __cpp_lib_experimental_filesystem && !defined(__APPLE__) std::experimental::filesystem::create_directories( std::experimental::filesystem::path(path).parent_path()); #else @@ -120,7 +120,7 @@ void TestEnvironment::removePath(const std::string& path) { return; } std::__fs::filesystem::remove_all(std::__fs::filesystem::path(path)); -#elif defined __cpp_lib_experimental_filesystem +#elif defined __cpp_lib_experimental_filesystem && !defined(__APPLE__) if (!std::experimental::filesystem::exists(path)) { return; } From ffeffd77a1c29664df06b3f649f9ce7c31c6ad59 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 22 Aug 2019 13:39:35 -0400 Subject: [PATCH 426/542] event: adding the capability of creating an alarm with a given scope (#7920) Precursor to #7782 Adding scope tracking functionality to the basic alarm functions. Risk Level: Medium (should be a no-op but is a large enough refactor) Testing: new unit tests Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- include/envoy/event/timer.h | 13 +- source/common/event/BUILD | 1 + source/common/event/dispatcher_impl.cc | 2 +- source/common/event/libevent_scheduler.cc | 4 +- source/common/event/libevent_scheduler.h | 2 +- source/common/event/real_time_system.cc | 4 +- source/common/event/timer_impl.cc | 18 +- source/common/event/timer_impl.h | 10 +- .../access_log_manager_impl_test.cc | 8 +- .../config/delta_subscription_test_harness.h | 2 +- test/common/config/grpc_mux_impl_test.cc | 7 +- .../config/grpc_subscription_impl_test.cc | 4 +- .../config/grpc_subscription_test_harness.h | 2 +- .../config/http_subscription_impl_test.cc | 4 +- .../config/http_subscription_test_harness.h | 4 +- test/common/event/dispatcher_impl_test.cc | 34 ++ test/common/http/async_client_impl_test.cc | 10 +- test/common/http/conn_manager_impl_test.cc | 72 +-- test/common/http/date_provider_impl_test.cc | 4 +- test/common/http/http1/conn_pool_test.cc | 4 +- test/common/http/http2/conn_pool_test.cc | 2 +- test/common/network/connection_impl_test.cc | 10 +- test/common/network/dns_impl_test.cc | 2 +- test/common/router/retry_state_impl_test.cc | 28 +- test/common/router/router_test.cc | 8 +- .../common/router/router_upstream_log_test.cc | 4 +- test/common/tcp/conn_pool_test.cc | 8 +- test/common/tcp_proxy/tcp_proxy_test.cc | 30 +- test/common/upstream/eds_test.cc | 8 +- test/common/upstream/hds_test.cc | 16 +- .../upstream/health_checker_impl_test.cc | 479 +++++++++--------- .../upstream/load_stats_reporter_test.cc | 18 +- .../upstream/logical_dns_cluster_test.cc | 10 +- .../upstream/original_dst_cluster_test.cc | 24 +- .../upstream/outlier_detection_impl_test.cc | 88 ++-- test/common/upstream/upstream_impl_test.cc | 52 +- .../grpc/grpc_access_log_impl_test.cc | 8 +- .../clusters/redis/redis_cluster_test.cc | 8 +- .../dns_cache_impl_test.cc | 24 +- .../filters/http/fault/fault_filter_test.cc | 16 +- .../http/health_check/health_check_test.cc | 6 +- .../filters/http/squash/squash_filter_test.cc | 17 +- .../client_ssl_auth/client_ssl_auth_test.cc | 10 +- .../network/common/redis/client_impl_test.cc | 24 +- .../filters/network/mongo_proxy/proxy_test.cc | 8 +- .../health_checkers/redis/redis_test.cc | 40 +- .../datadog/datadog_tracer_impl_test.cc | 4 +- .../lightstep/lightstep_tracer_impl_test.cc | 4 +- .../tracers/zipkin/zipkin_tracer_impl_test.cc | 4 +- test/mocks/common.h | 1 + test/mocks/event/mocks.cc | 7 +- test/mocks/event/mocks.h | 13 +- test/server/connection_handler_test.cc | 6 +- test/server/drain_manager_impl_test.cc | 6 +- test/test_common/BUILD | 1 + test/test_common/simulated_time_system.cc | 24 +- .../test_common/simulated_time_system_test.cc | 40 +- 57 files changed, 689 insertions(+), 578 deletions(-) diff --git a/include/envoy/event/timer.h b/include/envoy/event/timer.h index 3a6771904cd7..c2255e8dece9 100644 --- a/include/envoy/event/timer.h +++ b/include/envoy/event/timer.h @@ -8,8 +8,13 @@ #include "envoy/common/time.h" namespace Envoy { + +class ScopeTrackedObject; + namespace Event { +class Dispatcher; + /** * Callback invoked when a timer event fires. */ @@ -30,8 +35,12 @@ class Timer { /** * Enable a pending timeout. If a timeout is already pending, it will be reset to the new timeout. + * + * @param ms supplies the duration of the alarm in milliseconds. + * @param object supplies an optional scope for the duration of the alarm. */ - virtual void enableTimer(const std::chrono::milliseconds& d) PURE; + virtual void enableTimer(const std::chrono::milliseconds& ms, + const ScopeTrackedObject* object = nullptr) PURE; /** * Return whether the timer is currently armed. @@ -48,7 +57,7 @@ class Scheduler { /** * Creates a timer. */ - virtual TimerPtr createTimer(const TimerCb& cb) PURE; + virtual TimerPtr createTimer(const TimerCb& cb, Dispatcher& dispatcher) PURE; }; using SchedulerPtr = std::unique_ptr; diff --git a/source/common/event/BUILD b/source/common/event/BUILD index 78461dcd3181..76c0dc627cc2 100644 --- a/source/common/event/BUILD +++ b/source/common/event/BUILD @@ -120,6 +120,7 @@ envoy_cc_library( ":event_impl_base_lib", ":libevent_lib", "//include/envoy/event:timer_interface", + "//source/common/common:scope_tracker", ], ) diff --git a/source/common/event/dispatcher_impl.cc b/source/common/event/dispatcher_impl.cc index 7b718b8daf57..28fc0f6b8312 100644 --- a/source/common/event/dispatcher_impl.cc +++ b/source/common/event/dispatcher_impl.cc @@ -149,7 +149,7 @@ TimerPtr DispatcherImpl::createTimer(TimerCb cb) { return createTimerInternal(cb TimerPtr DispatcherImpl::createTimerInternal(TimerCb cb) { ASSERT(isThreadSafe()); - return scheduler_->createTimer(cb); + return scheduler_->createTimer(cb, *this); } void DispatcherImpl::deferredDelete(DeferredDeletablePtr&& to_delete) { diff --git a/source/common/event/libevent_scheduler.cc b/source/common/event/libevent_scheduler.cc index df22b45ba737..dcdca25c69a1 100644 --- a/source/common/event/libevent_scheduler.cc +++ b/source/common/event/libevent_scheduler.cc @@ -19,8 +19,8 @@ LibeventScheduler::LibeventScheduler() : libevent_(event_base_new()) { RELEASE_ASSERT(Libevent::Global::initialized(), ""); } -TimerPtr LibeventScheduler::createTimer(const TimerCb& cb) { - return std::make_unique(libevent_, cb); +TimerPtr LibeventScheduler::createTimer(const TimerCb& cb, Dispatcher& dispatcher) { + return std::make_unique(libevent_, cb, dispatcher); }; void LibeventScheduler::run(Dispatcher::RunType mode) { diff --git a/source/common/event/libevent_scheduler.h b/source/common/event/libevent_scheduler.h index b9157bf4059b..7694a69ea8d3 100644 --- a/source/common/event/libevent_scheduler.h +++ b/source/common/event/libevent_scheduler.h @@ -17,7 +17,7 @@ class LibeventScheduler : public Scheduler { LibeventScheduler(); // Scheduler - TimerPtr createTimer(const TimerCb& cb) override; + TimerPtr createTimer(const TimerCb& cb, Dispatcher& dispatcher) override; /** * Runs the event loop. diff --git a/source/common/event/real_time_system.cc b/source/common/event/real_time_system.cc index 9621611c2df0..c528b58b4e8c 100644 --- a/source/common/event/real_time_system.cc +++ b/source/common/event/real_time_system.cc @@ -12,7 +12,9 @@ namespace { class RealScheduler : public Scheduler { public: RealScheduler(Scheduler& base_scheduler) : base_scheduler_(base_scheduler) {} - TimerPtr createTimer(const TimerCb& cb) override { return base_scheduler_.createTimer(cb); }; + TimerPtr createTimer(const TimerCb& cb, Dispatcher& d) override { + return base_scheduler_.createTimer(cb, d); + }; private: Scheduler& base_scheduler_; diff --git a/source/common/event/timer_impl.cc b/source/common/event/timer_impl.cc index 4725e2018c39..ba9ea231c57b 100644 --- a/source/common/event/timer_impl.cc +++ b/source/common/event/timer_impl.cc @@ -17,16 +17,28 @@ void TimerUtils::millisecondsToTimeval(const std::chrono::milliseconds& d, timev tv.tv_usec = usecs.count(); } -TimerImpl::TimerImpl(Libevent::BasePtr& libevent, TimerCb cb) : cb_(cb) { +TimerImpl::TimerImpl(Libevent::BasePtr& libevent, TimerCb cb, Dispatcher& dispatcher) + : cb_(cb), dispatcher_(dispatcher) { ASSERT(cb_); evtimer_assign( &raw_event_, libevent.get(), - [](evutil_socket_t, short, void* arg) -> void { static_cast(arg)->cb_(); }, this); + [](evutil_socket_t, short, void* arg) -> void { + TimerImpl* timer = static_cast(arg); + if (timer->object_ == nullptr) { + timer->cb_(); + return; + } + ScopeTrackerScopeState scope(timer->object_, timer->dispatcher_); + timer->object_ = nullptr; + timer->cb_(); + }, + this); } void TimerImpl::disableTimer() { event_del(&raw_event_); } -void TimerImpl::enableTimer(const std::chrono::milliseconds& d) { +void TimerImpl::enableTimer(const std::chrono::milliseconds& d, const ScopeTrackedObject* object) { + object_ = object; if (d.count() == 0) { event_active(&raw_event_, EV_TIMEOUT, 0); } else { diff --git a/source/common/event/timer_impl.h b/source/common/event/timer_impl.h index 206525ec1e44..172f1b142c0f 100644 --- a/source/common/event/timer_impl.h +++ b/source/common/event/timer_impl.h @@ -4,6 +4,7 @@ #include "envoy/event/timer.h" +#include "common/common/scope_tracker.h" #include "common/event/event_impl_base.h" #include "common/event/libevent.h" @@ -23,15 +24,20 @@ class TimerUtils { */ class TimerImpl : public Timer, ImplBase { public: - TimerImpl(Libevent::BasePtr& libevent, TimerCb cb); + TimerImpl(Libevent::BasePtr& libevent, TimerCb cb, Event::Dispatcher& dispatcher); // Timer void disableTimer() override; - void enableTimer(const std::chrono::milliseconds& d) override; + void enableTimer(const std::chrono::milliseconds& d, const ScopeTrackedObject* scope) override; bool enabled() override; private: TimerCb cb_; + Dispatcher& dispatcher_; + // This has to be atomic for alarms which are handled out of thread, for + // example if the DispatcherImpl::post is called by two threads, they race to + // both set this to null. + std::atomic object_{}; }; } // namespace Event diff --git a/test/common/access_log/access_log_manager_impl_test.cc b/test/common/access_log/access_log_manager_impl_test.cc index 151ac7544b47..e7bf863df009 100644 --- a/test/common/access_log/access_log_manager_impl_test.cc +++ b/test/common/access_log/access_log_manager_impl_test.cc @@ -59,7 +59,7 @@ TEST_F(AccessLogManagerImplTest, flushToLogFilePeriodically) { EXPECT_CALL(*file_, open_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); - EXPECT_CALL(*timer, enableTimer(timeout_40ms_)); + EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); EXPECT_CALL(*file_, write_(_)) .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { EXPECT_EQ(0, data.compare("test")); @@ -83,7 +83,7 @@ TEST_F(AccessLogManagerImplTest, flushToLogFilePeriodically) { // make sure timer is re-enabled on callback call log_file->write("test2"); - EXPECT_CALL(*timer, enableTimer(timeout_40ms_)); + EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); timer->invokeCallback(); { @@ -101,7 +101,7 @@ TEST_F(AccessLogManagerImplTest, flushToLogFileOnDemand) { EXPECT_CALL(*file_, open_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); - EXPECT_CALL(*timer, enableTimer(timeout_40ms_)); + EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); // The first write to a given file will start the flush thread, which can flush // immediately (race on whether it will or not). So do a write and flush to @@ -146,7 +146,7 @@ TEST_F(AccessLogManagerImplTest, flushToLogFileOnDemand) { // make sure timer is re-enabled on callback call log_file->write("test2"); - EXPECT_CALL(*timer, enableTimer(timeout_40ms_)); + EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); timer->invokeCallback(); expected_writes++; diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index cae4cebbf298..4414aa77489c 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -157,7 +157,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { init_timeout_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout))); + EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout), _)); } void expectDisableInitFetchTimeoutTimer() override { diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 72b556277a62..c2493c0934f5 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -154,7 +154,7 @@ TEST_F(GrpcMuxImplTest, ResetStream) { .Times(3); EXPECT_CALL(random_, random()); ASSERT_TRUE(timer != nullptr); // initialized from dispatcher mock. - EXPECT_CALL(*timer, enableTimer(_)); + EXPECT_CALL(*timer, enableTimer(_, _)); grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); EXPECT_EQ(0, control_plane_connected_state_.value()); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); @@ -481,7 +481,7 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithEmptyRateLimitSetti grpc_mux_->start(); // Validate that drain_request_timer is enabled when there are no tokens. - EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100))); + EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100), _)); onReceiveMessage(99); EXPECT_EQ(1, stats_.counter("control_plane.rate_limit_enforced").value()); EXPECT_EQ( @@ -541,7 +541,8 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { EXPECT_EQ(0, stats_.counter("control_plane.rate_limit_enforced").value()); // Validate that drain_request_timer is enabled when there are no tokens. - EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(500))).Times(AtLeast(1)); + EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(500), _)) + .Times(AtLeast(1)); onReceiveMessage(160); EXPECT_EQ(12, stats_.counter("control_plane.rate_limit_enforced").value()); Stats::Gauge& pending_requests = diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index ea773b24f1f8..7a1ca435985d 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -18,7 +18,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { EXPECT_CALL(callbacks_, onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); EXPECT_CALL(random_, random()); - EXPECT_CALL(*timer_, enableTimer(_)); + EXPECT_CALL(*timer_, enableTimer(_, _)); subscription_->start({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); // Ensure this doesn't cause an issue by sending a request, since we don't @@ -40,7 +40,7 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); - EXPECT_CALL(*timer_, enableTimer(_)); + EXPECT_CALL(*timer_, enableTimer(_, _)); EXPECT_CALL(random_, random()); subscription_->grpcMux().grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 3031c2e947ef..9e68838ee646 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -142,7 +142,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { init_timeout_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout))); + EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout), _)); } void expectDisableInitFetchTimeoutTimer() override { diff --git a/test/common/config/http_subscription_impl_test.cc b/test/common/config/http_subscription_impl_test.cc index 258357452c12..afd592c0220e 100644 --- a/test/common/config/http_subscription_impl_test.cc +++ b/test/common/config/http_subscription_impl_test.cc @@ -14,7 +14,7 @@ class HttpSubscriptionImplTest : public testing::Test, public HttpSubscriptionTe TEST_F(HttpSubscriptionImplTest, OnRequestReset) { startSubscription({"cluster0", "cluster1"}); EXPECT_CALL(random_gen_, random()).WillOnce(Return(0)); - EXPECT_CALL(*timer_, enableTimer(_)); + EXPECT_CALL(*timer_, enableTimer(_, _)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); http_callbacks_->onFailure(Http::AsyncClient::FailureReason::Reset); @@ -32,7 +32,7 @@ TEST_F(HttpSubscriptionImplTest, BadJsonRecovery) { Http::MessagePtr message{new Http::ResponseMessageImpl(std::move(response_headers))}; message->body() = std::make_unique(";!@#badjso n"); EXPECT_CALL(random_gen_, random()).WillOnce(Return(0)); - EXPECT_CALL(*timer_, enableTimer(_)); + EXPECT_CALL(*timer_, enableTimer(_, _)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); http_callbacks_->onSuccess(std::move(message)); diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index 95d1123be5b5..1d41ee38019c 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -144,7 +144,7 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); } EXPECT_CALL(random_gen_, random()).WillOnce(Return(0)); - EXPECT_CALL(*timer_, enableTimer(_)); + EXPECT_CALL(*timer_, enableTimer(_, _)); http_callbacks_->onSuccess(std::move(message)); if (accept) { version_ = version; @@ -159,7 +159,7 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { init_timeout_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout))); + EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout), _)); } void expectDisableInitFetchTimeoutTimer() override { diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index c0605a70c3ca..8df3bf5c55f4 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -185,6 +185,40 @@ TEST_F(DispatcherImplTest, Timer) { } } +TEST_F(DispatcherImplTest, TimerWithScope) { + TimerPtr timer; + MockScopedTrackedObject scope; + dispatcher_->post([this, &timer, &scope]() { + { + // Expect a call to dumpState. The timer will call onFatalError during + // the alarm interval, and if the scope is tracked correctly this will + // result in a dumpState call. + EXPECT_CALL(scope, dumpState(_, _)); + Thread::LockGuard lock(mu_); + timer = dispatcher_->createTimer([this]() { + { + Thread::LockGuard lock(mu_); + static_cast(dispatcher_.get())->onFatalError(); + work_finished_ = true; + } + cv_.notifyOne(); + }); + EXPECT_FALSE(timer->enabled()); + } + cv_.notifyOne(); + }); + + Thread::LockGuard lock(mu_); + while (timer == nullptr) { + cv_.wait(mu_); + } + timer->enableTimer(std::chrono::milliseconds(50), &scope); + + while (!work_finished_) { + cv_.wait(mu_); + } +} + TEST_F(DispatcherImplTest, IsThreadSafe) { dispatcher_->post([this]() { { diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index ae5b7cdbb6fb..29985046fa7f 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -719,7 +719,7 @@ TEST_F(AsyncClientImplTest, StreamTimeout) { EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(&message_->headers()), true)); timer_ = new NiceMock(&dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(40))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(40), _)); EXPECT_CALL(stream_encoder_.stream_, resetStream(_)); TestHeaderMapImpl expected_timeout{ @@ -754,7 +754,7 @@ TEST_F(AsyncClientImplTest, StreamTimeoutHeadReply) { HttpTestUtility::addDefaultHeaders(message->headers(), "HEAD"); EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(&message->headers()), true)); timer_ = new NiceMock(&dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(40))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(40), _)); EXPECT_CALL(stream_encoder_.stream_, resetStream(_)); TestHeaderMapImpl expected_timeout{ @@ -779,7 +779,7 @@ TEST_F(AsyncClientImplTest, RequestTimeout) { EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(&message_->headers()), true)); expectSuccess(504); timer_ = new NiceMock(&dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(40))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(40), _)); EXPECT_CALL(stream_encoder_.stream_, resetStream(_)); client_.send(std::move(message_), callbacks_, AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds(40))); @@ -804,7 +804,7 @@ TEST_F(AsyncClientImplTest, DisableTimer) { EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(&message_->headers()), true)); timer_ = new NiceMock(&dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(200))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(200), _)); EXPECT_CALL(*timer_, disableTimer()); EXPECT_CALL(stream_encoder_.stream_, resetStream(_)); AsyncClient::Request* request = @@ -823,7 +823,7 @@ TEST_F(AsyncClientImplTest, DisableTimerWithStream) { EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(&message_->headers()), true)); timer_ = new NiceMock(&dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(40))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(40), _)); EXPECT_CALL(*timer_, disableTimer()); EXPECT_CALL(stream_encoder_.stream_, resetStream(_)); EXPECT_CALL(stream_callbacks_, onReset()); diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 083bbbe73753..ea6f27e236a0 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -1403,12 +1403,12 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutGlobal) { EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* idle_timer = setUpTimer(); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10), _)); conn_manager_->newStream(response_encoder_); // Expect resetIdleTimer() to be called for the response // encodeHeaders()/encodeData(). - EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); + EXPECT_CALL(*idle_timer, enableTimer(_, _)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); idle_timer->invokeCallback(); })); @@ -1441,12 +1441,12 @@ TEST_F(HttpConnectionManagerImplTest, AccessEncoderRouteBeforeHeadersArriveOnIdl EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* idle_timer = setUpTimer(); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10), _)); conn_manager_->newStream(response_encoder_); // Expect resetIdleTimer() to be called for the response // encodeHeaders()/encodeData(). - EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); + EXPECT_CALL(*idle_timer, enableTimer(_, _)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); // Simulate and idle timeout so that the filter chain gets created. idle_timer->invokeCallback(); @@ -1482,12 +1482,12 @@ TEST_F(HttpConnectionManagerImplTest, TestStreamIdleAccessLog) { NiceMock encoder; EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* idle_timer = setUpTimer(); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10), _)); conn_manager_->newStream(response_encoder_); // Expect resetIdleTimer() to be called for the response // encodeHeaders()/encodeData(). - EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); + EXPECT_CALL(*idle_timer, enableTimer(_, _)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); idle_timer->invokeCallback(); })); @@ -1534,12 +1534,12 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutRouteOverride) { EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance& data) -> void { Event::MockTimer* idle_timer = setUpTimer(); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10), _)); StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(30))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(30), _)); decoder->decodeHeaders(std::move(headers), false); data.drain(4); @@ -1560,7 +1560,7 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutRouteZeroOverride) { EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance& data) -> void { Event::MockTimer* idle_timer = setUpTimer(); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(10), _)); StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ @@ -1590,12 +1590,12 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterDownstreamHeaders Event::MockTimer* idle_timer = setUpTimer(); HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); decoder->decodeHeaders(std::move(headers), false); // Expect resetIdleTimer() to be called for the response // encodeHeaders()/encodeData(). - EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); + EXPECT_CALL(*idle_timer, enableTimer(_, _)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); idle_timer->invokeCallback(); @@ -1630,7 +1630,7 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutNormalTermination) { HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); decoder->decodeHeaders(std::move(headers), false); data.drain(4); @@ -1659,15 +1659,15 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterDownstreamHeaders Event::MockTimer* idle_timer = setUpTimer(); HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); decoder->decodeHeaders(std::move(headers), false); - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); decoder->decodeData(data, false); // Expect resetIdleTimer() to be called for the response // encodeHeaders()/encodeData(). - EXPECT_CALL(*idle_timer, enableTimer(_)).Times(2); + EXPECT_CALL(*idle_timer, enableTimer(_, _)).Times(2); EXPECT_CALL(*idle_timer, disableTimer()); idle_timer->invokeCallback(); @@ -1712,11 +1712,11 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterUpstreamHeaders) Event::MockTimer* idle_timer = setUpTimer(); HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); decoder->decodeHeaders(std::move(headers), false); HeaderMapPtr response_headers{new TestHeaderMapImpl{{":status", "200"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); filter->callbacks_->encodeHeaders(std::move(response_headers), false); EXPECT_CALL(*idle_timer, disableTimer()); @@ -1761,26 +1761,26 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterBidiData) { decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); decoder->decodeHeaders(std::move(headers), false); HeaderMapPtr response_continue_headers{new TestHeaderMapImpl{{":status", "100"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); filter->callbacks_->encode100ContinueHeaders(std::move(response_continue_headers)); HeaderMapPtr response_headers{new TestHeaderMapImpl{{":status", "200"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); filter->callbacks_->encodeHeaders(std::move(response_headers), false); - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); decoder->decodeData(data, false); HeaderMapPtr trailers{new TestHeaderMapImpl{{"foo", "bar"}}}; - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); decoder->decodeTrailers(std::move(trailers)); Buffer::OwnedImpl fake_response("world"); - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); filter->callbacks_->encodeData(fake_response, false); EXPECT_CALL(*idle_timer, disableTimer()); @@ -1839,7 +1839,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutValidlyConfigured) { EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); - EXPECT_CALL(*request_timer, enableTimer(request_timeout_)); + EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)); conn_manager_->newStream(response_encoder_); })); @@ -1855,7 +1855,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutCallbackDisarmsAndReturns408 std::string response_body; EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); - EXPECT_CALL(*request_timer, enableTimer(request_timeout_)).Times(1); + EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); EXPECT_CALL(*request_timer, disableTimer()).Times(AtLeast(1)); EXPECT_CALL(response_encoder_, encodeHeaders(_, false)) @@ -1881,7 +1881,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsNotDisarmedOnIncompleteReq EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); - EXPECT_CALL(*request_timer, enableTimer(request_timeout_)).Times(1); + EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); EXPECT_CALL(*request_timer, disableTimer()).Times(0); StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); @@ -1904,7 +1904,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsDisarmedOnCompleteRequestW EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); - EXPECT_CALL(*request_timer, enableTimer(request_timeout_)).Times(1); + EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ @@ -1926,7 +1926,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsDisarmedOnCompleteRequestW EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> void { Event::MockTimer* request_timer = setUpTimer(); - EXPECT_CALL(*request_timer, enableTimer(request_timeout_)).Times(1); + EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ @@ -1949,7 +1949,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsDisarmedOnCompleteRequestW EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> void { Event::MockTimer* request_timer = setUpTimer(); - EXPECT_CALL(*request_timer, enableTimer(request_timeout_)).Times(1); + EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ @@ -1980,7 +1980,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsDisarmedOnEncodeHeaders) { EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); - EXPECT_CALL(*request_timer, enableTimer(request_timeout_)).Times(1); + EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ @@ -2014,7 +2014,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsDisarmedOnConnectionTermin Buffer::OwnedImpl fake_input("1234"); - EXPECT_CALL(*request_timer, enableTimer(request_timeout_)).Times(1); + EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); conn_manager_->onData(fake_input, false); // kick off request EXPECT_CALL(*request_timer, disableTimer()).Times(1); @@ -2138,7 +2138,7 @@ TEST_F(HttpConnectionManagerImplTest, DrainClose) { HeaderMapPtr response_headers{new TestHeaderMapImpl{{":status", "300"}}}; Event::MockTimer* drain_timer = setUpTimer(); - EXPECT_CALL(*drain_timer, enableTimer(_)); + EXPECT_CALL(*drain_timer, enableTimer(_, _)); EXPECT_CALL(drain_close_, drainClose()).WillOnce(Return(true)); EXPECT_CALL(*codec_, shutdownNotice()); filter->callbacks_->encodeHeaders(std::move(response_headers), true); @@ -2376,7 +2376,7 @@ TEST_F(HttpConnectionManagerImplTest, IdleTimeoutNoCodec) { idle_timeout_ = (std::chrono::milliseconds(10)); Event::MockTimer* idle_timer = setUpTimer(); - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); setup(false, ""); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); @@ -2389,7 +2389,7 @@ TEST_F(HttpConnectionManagerImplTest, IdleTimeoutNoCodec) { TEST_F(HttpConnectionManagerImplTest, IdleTimeout) { idle_timeout_ = (std::chrono::milliseconds(10)); Event::MockTimer* idle_timer = setUpTimer(); - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); setup(false, ""); MockStreamDecoderFilter* filter = new NiceMock(); @@ -2420,12 +2420,12 @@ TEST_F(HttpConnectionManagerImplTest, IdleTimeout) { Buffer::OwnedImpl fake_input("1234"); conn_manager_->onData(fake_input, false); - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); HeaderMapPtr response_headers{new TestHeaderMapImpl{{":status", "200"}}}; filter->callbacks_->encodeHeaders(std::move(response_headers), true); Event::MockTimer* drain_timer = setUpTimer(); - EXPECT_CALL(*drain_timer, enableTimer(_)); + EXPECT_CALL(*drain_timer, enableTimer(_, _)); idle_timer->invokeCallback(); EXPECT_CALL(*codec_, goAway()); diff --git a/test/common/http/date_provider_impl_test.cc b/test/common/http/date_provider_impl_test.cc index 619a5f9a96e3..41594a30a6ea 100644 --- a/test/common/http/date_provider_impl_test.cc +++ b/test/common/http/date_provider_impl_test.cc @@ -19,14 +19,14 @@ TEST(DateProviderImplTest, All) { Event::MockDispatcher dispatcher; NiceMock tls; Event::MockTimer* timer = new Event::MockTimer(&dispatcher); - EXPECT_CALL(*timer, enableTimer(std::chrono::milliseconds(500))); + EXPECT_CALL(*timer, enableTimer(std::chrono::milliseconds(500), _)); TlsCachingDateProviderImpl provider(dispatcher, tls); HeaderMapImpl headers; provider.setDateHeader(headers); EXPECT_NE(nullptr, headers.Date()); - EXPECT_CALL(*timer, enableTimer(std::chrono::milliseconds(500))); + EXPECT_CALL(*timer, enableTimer(std::chrono::milliseconds(500), _)); timer->invokeCallback(); headers.removeDate(); diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc index d91c9cd862a7..73b79b35205f 100644 --- a/test/common/http/http1/conn_pool_test.cc +++ b/test/common/http/http1/conn_pool_test.cc @@ -98,13 +98,13 @@ class ConnPoolImplForTest : public ConnPoolImpl { EXPECT_CALL(mock_dispatcher_, createClientConnection_(_, _, _, _)) .WillOnce(Return(test_client.connection_)); EXPECT_CALL(*this, createCodecClient_()).WillOnce(Return(test_client.codec_client_)); - EXPECT_CALL(*test_client.connect_timer_, enableTimer(_)); + EXPECT_CALL(*test_client.connect_timer_, enableTimer(_, _)); ON_CALL(*test_client.codec_, protocol()).WillByDefault(Return(protocol)); } void expectEnableUpstreamReady() { EXPECT_FALSE(upstream_ready_enabled_); - EXPECT_CALL(*mock_upstream_ready_timer_, enableTimer(_)).Times(1).RetiresOnSaturation(); + EXPECT_CALL(*mock_upstream_ready_timer_, enableTimer(_, _)).Times(1).RetiresOnSaturation(); } void expectAndRunUpstreamReady() { diff --git a/test/common/http/http2/conn_pool_test.cc b/test/common/http/http2/conn_pool_test.cc index 6490da7337e1..5e2cac038136 100644 --- a/test/common/http/http2/conn_pool_test.cc +++ b/test/common/http/http2/conn_pool_test.cc @@ -96,7 +96,7 @@ class Http2ConnPoolImplTest : public testing::Test { .WillOnce(Invoke([this](Upstream::Host::CreateConnectionData&) -> CodecClient* { return test_clients_.back().codec_client_; })); - EXPECT_CALL(*test_client.connect_timer_, enableTimer(_)); + EXPECT_CALL(*test_client.connect_timer_, enableTimer(_, _)); } // Connects a pending connection for client with the given index, asserting diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 734734f09bfc..7e22ba77e941 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -1224,7 +1224,7 @@ TEST_P(ConnectionImplTest, DelayedCloseTimerResetWithPendingWriteBufferFlushes) Buffer::OwnedImpl data("data"); server_connection->write(data, false); - EXPECT_CALL(*mocks.timer_, enableTimer(timeout)).Times(1); + EXPECT_CALL(*mocks.timer_, enableTimer(timeout, _)).Times(1); server_connection->close(ConnectionCloseType::FlushWriteAndDelay); // The write ready event cb (ConnectionImpl::onWriteReady()) will reset the timer to its original @@ -1234,7 +1234,7 @@ TEST_P(ConnectionImplTest, DelayedCloseTimerResetWithPendingWriteBufferFlushes) // Partial flush. return IoResult{PostIoAction::KeepOpen, 1, false}; })); - EXPECT_CALL(*mocks.timer_, enableTimer(timeout)).Times(1); + EXPECT_CALL(*mocks.timer_, enableTimer(timeout, _)).Times(1); (*mocks.file_ready_cb_)(Event::FileReadyType::Write); EXPECT_CALL(*transport_socket, doWrite(BufferStringEqual("data"), _)) @@ -1243,7 +1243,7 @@ TEST_P(ConnectionImplTest, DelayedCloseTimerResetWithPendingWriteBufferFlushes) buffer.drain(buffer.length()); return IoResult{PostIoAction::KeepOpen, buffer.length(), false}; })); - EXPECT_CALL(*mocks.timer_, enableTimer(timeout)).Times(1); + EXPECT_CALL(*mocks.timer_, enableTimer(timeout, _)).Times(1); (*mocks.file_ready_cb_)(Event::FileReadyType::Write); // Force the delayed close timeout to trigger so the connection is cleaned up. @@ -1277,7 +1277,7 @@ TEST_P(ConnectionImplTest, DelayedCloseTimeoutDisableOnSocketClose) { return IoResult{PostIoAction::KeepOpen, buffer.length(), false}; })); server_connection->write(data, false); - EXPECT_CALL(*mocks.timer_, enableTimer(_)).Times(1); + EXPECT_CALL(*mocks.timer_, enableTimer(_, _)).Times(1); // Enable the delayed close timer. server_connection->close(ConnectionCloseType::FlushWriteAndDelay); EXPECT_CALL(*mocks.timer_, disableTimer()).Times(1); @@ -1318,7 +1318,7 @@ TEST_P(ConnectionImplTest, DelayedCloseTimeoutNullStats) { })); server_connection->write(data, false); - EXPECT_CALL(*mocks.timer_, enableTimer(_)).Times(1); + EXPECT_CALL(*mocks.timer_, enableTimer(_, _)).Times(1); server_connection->close(ConnectionCloseType::FlushWriteAndDelay); EXPECT_CALL(*mocks.timer_, disableTimer()).Times(1); // The following close() will call closeSocket() and reset internal data structures such as diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index eaa925b9e06a..49d01a430d0c 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -857,7 +857,7 @@ TEST(DnsImplUnitTest, PendingTimerEnable) { DnsResolverImpl resolver(dispatcher, {}); Event::FileEvent* file_event = new NiceMock(); EXPECT_CALL(dispatcher, createFileEvent_(_, _, _, _)).WillOnce(Return(file_event)); - EXPECT_CALL(*timer, enableTimer(_)); + EXPECT_CALL(*timer, enableTimer(_, _)); EXPECT_NE(nullptr, resolver.resolve("some.bad.domain.invalid", DnsLookupFamily::V4Only, [&](std::list&& results) { UNREFERENCED_PARAMETER(results); diff --git a/test/common/router/retry_state_impl_test.cc b/test/common/router/retry_state_impl_test.cc index 6120470db1cb..64de11d0215d 100644 --- a/test/common/router/retry_state_impl_test.cc +++ b/test/common/router/retry_state_impl_test.cc @@ -43,7 +43,7 @@ class RouterRetryStateImplTest : public testing::Test { void expectTimerCreateAndEnable() { retry_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*retry_timer_, enableTimer(_)); + EXPECT_CALL(*retry_timer_, enableTimer(_, _)); } NiceMock policy_; @@ -466,12 +466,12 @@ TEST_F(RouterRetryStateImplTest, MaxRetriesHeader) { EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); - EXPECT_CALL(*retry_timer_, enableTimer(_)); + EXPECT_CALL(*retry_timer_, enableTimer(_, _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); - EXPECT_CALL(*retry_timer_, enableTimer(_)); + EXPECT_CALL(*retry_timer_, enableTimer(_, _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); @@ -493,19 +493,19 @@ TEST_F(RouterRetryStateImplTest, Backoff) { EXPECT_CALL(random_, random()).WillOnce(Return(49)); retry_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(24))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(24), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(149)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(74))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(74), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(349)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(174))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(174), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); @@ -530,25 +530,25 @@ TEST_F(RouterRetryStateImplTest, CustomBackOffInterval) { EXPECT_CALL(random_, random()).WillOnce(Return(149)); retry_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(49))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(49), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(350)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(50))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(50), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(751)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(51))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(51), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(1499)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(1200))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(1200), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); @@ -565,25 +565,25 @@ TEST_F(RouterRetryStateImplTest, CustomBackOffIntervalDefaultMax) { EXPECT_CALL(random_, random()).WillOnce(Return(149)); retry_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(49))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(49), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(350)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(50))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(50), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(751)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(51))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(51), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); EXPECT_CALL(random_, random()).WillOnce(Return(1499)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_EQ(RetryStatus::Yes, state_->shouldRetryReset(connect_failure_, callback_)); EXPECT_CALL(callback_ready_, ready()); retry_timer_->invokeCallback(); diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 770ce8bef6ae..fd91559acc1e 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -106,13 +106,13 @@ class RouterTestBase : public testing::Test { void expectResponseTimerCreate() { response_timeout_ = new Event::MockTimer(&callbacks_.dispatcher_); - EXPECT_CALL(*response_timeout_, enableTimer(_)); + EXPECT_CALL(*response_timeout_, enableTimer(_, _)); EXPECT_CALL(*response_timeout_, disableTimer()); } void expectPerTryTimerCreate() { per_try_timeout_ = new Event::MockTimer(&callbacks_.dispatcher_); - EXPECT_CALL(*per_try_timeout_, enableTimer(_)); + EXPECT_CALL(*per_try_timeout_, enableTimer(_, _)); EXPECT_CALL(*per_try_timeout_, disableTimer()); } @@ -1491,7 +1491,7 @@ TEST_F(RouterTest, UpstreamPerTryTimeoutExcludesNewStream) { })); response_timeout_ = new Event::MockTimer(&callbacks_.dispatcher_); - EXPECT_CALL(*response_timeout_, enableTimer(_)); + EXPECT_CALL(*response_timeout_, enableTimer(_, _)); EXPECT_CALL(callbacks_.stream_info_, onUpstreamHostSelected(_)) .WillOnce(Invoke([&](const Upstream::HostDescriptionConstSharedPtr host) -> void { @@ -1506,7 +1506,7 @@ TEST_F(RouterTest, UpstreamPerTryTimeoutExcludesNewStream) { router_.decodeData(data, true); per_try_timeout_ = new Event::MockTimer(&callbacks_.dispatcher_); - EXPECT_CALL(*per_try_timeout_, enableTimer(_)); + EXPECT_CALL(*per_try_timeout_, enableTimer(_, _)); // The per try timeout timer should not be started yet. pool_callbacks->onPoolReady(encoder, cm_.conn_pool_.host_); diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index 581db1087f3c..554aa3c198f5 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -106,13 +106,13 @@ class RouterUpstreamLogTest : public testing::Test { void expectResponseTimerCreate() { response_timeout_ = new Event::MockTimer(&callbacks_.dispatcher_); - EXPECT_CALL(*response_timeout_, enableTimer(_)); + EXPECT_CALL(*response_timeout_, enableTimer(_, _)); EXPECT_CALL(*response_timeout_, disableTimer()); } void expectPerTryTimerCreate() { per_try_timeout_ = new Event::MockTimer(&callbacks_.dispatcher_); - EXPECT_CALL(*per_try_timeout_, enableTimer(_)); + EXPECT_CALL(*per_try_timeout_, enableTimer(_, _)); EXPECT_CALL(*per_try_timeout_, disableTimer()); } diff --git a/test/common/tcp/conn_pool_test.cc b/test/common/tcp/conn_pool_test.cc index ec42cf60e602..d1521e28226a 100644 --- a/test/common/tcp/conn_pool_test.cc +++ b/test/common/tcp/conn_pool_test.cc @@ -106,12 +106,12 @@ class ConnPoolImplForTest : public ConnPoolImpl { .WillOnce(Invoke( [&](Network::ReadFilterSharedPtr filter) -> void { test_conn.filter_ = filter; })); EXPECT_CALL(*test_conn.connection_, connect()); - EXPECT_CALL(*test_conn.connect_timer_, enableTimer(_)); + EXPECT_CALL(*test_conn.connect_timer_, enableTimer(_, _)); } void expectEnableUpstreamReady() { EXPECT_FALSE(upstream_ready_enabled_); - EXPECT_CALL(*mock_upstream_ready_timer_, enableTimer(_)).Times(1).RetiresOnSaturation(); + EXPECT_CALL(*mock_upstream_ready_timer_, enableTimer(_, _)).Times(1).RetiresOnSaturation(); } void expectAndRunUpstreamReady() { @@ -186,7 +186,7 @@ class TcpConnPoolImplDestructorTest : public testing::Test { connection_ = new NiceMock(); connect_timer_ = new NiceMock(&dispatcher_); EXPECT_CALL(dispatcher_, createClientConnection_(_, _, _, _)).WillOnce(Return(connection_)); - EXPECT_CALL(*connect_timer_, enableTimer(_)); + EXPECT_CALL(*connect_timer_, enableTimer(_, _)); callbacks_ = std::make_unique(); ConnectionPool::Cancellable* handle = conn_pool_->newConnection(*callbacks_); @@ -921,7 +921,7 @@ TEST_F(TcpConnPoolImplDestructorTest, TestPendingConnectionsAreClosed) { connection_ = new NiceMock(); connect_timer_ = new NiceMock(&dispatcher_); EXPECT_CALL(dispatcher_, createClientConnection_(_, _, _, _)).WillOnce(Return(connection_)); - EXPECT_CALL(*connect_timer_, enableTimer(_)); + EXPECT_CALL(*connect_timer_, enableTimer(_, _)); callbacks_ = std::make_unique(); ConnectionPool::Cancellable* handle = conn_pool_->newConnection(*callbacks_); diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index 903249d9a06e..88d21bfdc4b6 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -821,21 +821,21 @@ TEST_F(TcpProxyTest, IdleTimeout) { setup(1, config); Event::MockTimer* idle_timer = new Event::MockTimer(&filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); raiseEventUpstreamConnected(0); Buffer::OwnedImpl buffer("hello"); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); filter_->onData(buffer, false); buffer.add("hello2"); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); upstream_callbacks_->onUpstreamData(buffer, false); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); filter_callbacks_.connection_.raiseBytesSentCallbacks(1); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); upstream_connections_.at(0)->raiseBytesSentCallbacks(2); EXPECT_CALL(*upstream_connections_.at(0), close(Network::ConnectionCloseType::NoFlush)); @@ -851,7 +851,7 @@ TEST_F(TcpProxyTest, IdleTimerDisabledDownstreamClose) { setup(1, config); Event::MockTimer* idle_timer = new Event::MockTimer(&filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); raiseEventUpstreamConnected(0); EXPECT_CALL(*idle_timer, disableTimer()); @@ -865,7 +865,7 @@ TEST_F(TcpProxyTest, IdleTimerDisabledUpstreamClose) { setup(1, config); Event::MockTimer* idle_timer = new Event::MockTimer(&filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); raiseEventUpstreamConnected(0); EXPECT_CALL(*idle_timer, disableTimer()); @@ -879,21 +879,21 @@ TEST_F(TcpProxyTest, IdleTimeoutWithOutstandingDataFlushed) { setup(1, config); Event::MockTimer* idle_timer = new Event::MockTimer(&filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); raiseEventUpstreamConnected(0); Buffer::OwnedImpl buffer("hello"); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); filter_->onData(buffer, false); buffer.add("hello2"); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); upstream_callbacks_->onUpstreamData(buffer, false); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); filter_callbacks_.connection_.raiseBytesSentCallbacks(1); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); upstream_connections_.at(0)->raiseBytesSentCallbacks(2); // Mark the upstream connection as blocked. @@ -1043,7 +1043,7 @@ TEST_F(TcpProxyTest, UpstreamFlushTimeoutConfigured) { NiceMock* idle_timer = new NiceMock(&filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); raiseEventUpstreamConnected(0); EXPECT_CALL(*upstream_connections_.at(0), @@ -1056,7 +1056,7 @@ TEST_F(TcpProxyTest, UpstreamFlushTimeoutConfigured) { filter_.reset(); EXPECT_EQ(1U, config_->stats().upstream_flush_active_.value()); - EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*idle_timer, enableTimer(std::chrono::milliseconds(1000), _)); upstream_connections_.at(0)->raiseBytesSentCallbacks(1); // Simulate flush complete. @@ -1075,7 +1075,7 @@ TEST_F(TcpProxyTest, UpstreamFlushTimeoutExpired) { NiceMock* idle_timer = new NiceMock(&filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*idle_timer, enableTimer(_)); + EXPECT_CALL(*idle_timer, enableTimer(_, _)); raiseEventUpstreamConnected(0); EXPECT_CALL(*upstream_connections_.at(0), diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index 988df7fc7ee4..b71150712939 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -1713,9 +1713,9 @@ TEST_F(EdsAssignmentTimeoutTest, AssignmentTimeoutEnableDisable) { cluster_load_assignment_lease.mutable_policy()->mutable_endpoint_stale_after()->MergeFrom( Protobuf::util::TimeUtil::SecondsToDuration(1)); - EXPECT_CALL(*interval_timer_, enableTimer(_)).Times(2); // Timer enabled twice. - EXPECT_CALL(*interval_timer_, disableTimer()).Times(1); // Timer disabled once. - EXPECT_CALL(*interval_timer_, enabled()).Times(6); // Includes calls by test. + EXPECT_CALL(*interval_timer_, enableTimer(_, _)).Times(2); // Timer enabled twice. + EXPECT_CALL(*interval_timer_, disableTimer()).Times(1); // Timer disabled once. + EXPECT_CALL(*interval_timer_, enabled()).Times(6); // Includes calls by test. doOnConfigUpdateVerifyNoThrow(cluster_load_assignment_lease); // Check that the timer is enabled. EXPECT_EQ(interval_timer_->enabled(), true); @@ -1755,7 +1755,7 @@ TEST_F(EdsAssignmentTimeoutTest, AssignmentLeaseExpired) { add_endpoint(81); // Expect the timer to be enabled once. - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); // Expect the timer to be disabled when stale assignments are removed. EXPECT_CALL(*interval_timer_, disableTimer()); EXPECT_CALL(*interval_timer_, enabled()).Times(2); diff --git a/test/common/upstream/hds_test.cc b/test/common/upstream/hds_test.cc index c7fe3af8d73e..3a442a1924b5 100644 --- a/test/common/upstream/hds_test.cc +++ b/test/common/upstream/hds_test.cc @@ -232,7 +232,7 @@ TEST_F(HdsTest, TestMinimalOnReceiveMessage) { message->mutable_interval()->set_seconds(1); // Process message - EXPECT_CALL(*server_response_timer_, enableTimer(_)).Times(AtLeast(1)); + EXPECT_CALL(*server_response_timer_, enableTimer(_, _)).Times(AtLeast(1)); hds_delegate_->onReceiveMessage(std::move(message)); } @@ -248,7 +248,7 @@ TEST_F(HdsTest, TestMinimalSendResponse) { message->mutable_interval()->set_seconds(1); // Process message and send 2 responses - EXPECT_CALL(*server_response_timer_, enableTimer(_)).Times(AtLeast(1)); + EXPECT_CALL(*server_response_timer_, enableTimer(_, _)).Times(AtLeast(1)); EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)).Times(2); hds_delegate_->onReceiveMessage(std::move(message)); hds_delegate_->sendResponse(); @@ -265,11 +265,11 @@ TEST_F(HdsTest, TestStreamConnectionFailure) { .WillOnce(Return(&async_stream_)); EXPECT_CALL(random_, random()).WillOnce(Return(1000005)).WillRepeatedly(Return(1234567)); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(5))); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(1567))); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(2567))); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(4567))); - EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(25567))); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(5), _)); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(1567), _)); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(2567), _)); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(4567), _)); + EXPECT_CALL(*retry_timer_, enableTimer(std::chrono::milliseconds(25567), _)); EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)); // Test connection failure and retry @@ -297,7 +297,7 @@ TEST_F(HdsTest, TestSendResponseOneEndpointTimeout) { Network::MockClientConnection* connection_ = new NiceMock(); EXPECT_CALL(dispatcher_, createClientConnection_(_, _, _, _)).WillRepeatedly(Return(connection_)); - EXPECT_CALL(*server_response_timer_, enableTimer(_)).Times(2); + EXPECT_CALL(*server_response_timer_, enableTimer(_, _)).Times(2); EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)); EXPECT_CALL(test_factory_, createClusterInfo(_)).WillOnce(Return(cluster_info_)); EXPECT_CALL(*connection_, setBufferLimits(_)); diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index 7c8d7cfb5701..cb599a08b5b8 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -543,12 +543,12 @@ class HttpHealthCheckerImplTest : public testing::Test { Host::HealthFlag::FAILED_ACTIVE_HC); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); // Test that failing first disables fast success. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); respond(0, "503", false, false, false, false, health_checked_cluster); @@ -557,12 +557,12 @@ class HttpHealthCheckerImplTest : public testing::Test { EXPECT_EQ(Host::Health::Unhealthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, false, false, health_checked_cluster); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( @@ -570,13 +570,13 @@ class HttpHealthCheckerImplTest : public testing::Test { EXPECT_EQ(Host::Health::Unhealthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, false, false, health_checked_cluster); EXPECT_EQ(Host::Health::Healthy, @@ -606,13 +606,14 @@ TEST_F(HttpHealthCheckerImplTest, Success) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -627,7 +628,7 @@ TEST_F(HttpHealthCheckerImplTest, Degraded) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); @@ -635,19 +636,19 @@ TEST_F(HttpHealthCheckerImplTest, Degraded) { .WillRepeatedly(Return(45000)); // We start off as healthy, and should go degraded after receiving the degraded health response. - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); EXPECT_CALL(*event_logger_, logDegraded(_, _)); respond(0, "200", false, false, true, false, {}, true); EXPECT_EQ(Host::Health::Degraded, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); // Then, after receiving a regular health check response we should go back to healthy. - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->interval_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*event_logger_, logNoLongerDegraded(_, _)); respond(0, "200", false, false, true, false, {}, false); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -661,22 +662,22 @@ TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitter) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); for (int i = 0; i < 50000; i += 239) { EXPECT_CALL(random_, random()).WillOnce(Return(i)); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // the jitter is 1000ms here EXPECT_CALL(*test_sessions_[0]->interval_timer_, - enableTimer(std::chrono::milliseconds(5000 + i % 1000))); + enableTimer(std::chrono::milliseconds(5000 + i % 1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); } @@ -690,24 +691,24 @@ TEST_F(HttpHealthCheckerImplTest, InitialJitterNoTraffic) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); test_sessions_[0]->interval_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); for (int i = 0; i < 2; i += 1) { EXPECT_CALL(random_, random()).WillOnce(Return(i)); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // the jitter is 40% of 5000, so should be 2000 EXPECT_CALL(*test_sessions_[0]->interval_timer_, - enableTimer(std::chrono::milliseconds(5000 + i % 2000))); + enableTimer(std::chrono::milliseconds(5000 + i % 2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); } @@ -721,22 +722,22 @@ TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitterPercentNoTraffic) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); for (int i = 0; i < 50000; i += 239) { EXPECT_CALL(random_, random()).WillOnce(Return(i)); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // the jitter is 40% of 5000, so should be 2000 EXPECT_CALL(*test_sessions_[0]->interval_timer_, - enableTimer(std::chrono::milliseconds(5000 + i % 2000))); + enableTimer(std::chrono::milliseconds(5000 + i % 2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); } @@ -751,22 +752,22 @@ TEST_F(HttpHealthCheckerImplTest, SuccessIntervalJitterPercent) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); for (int i = 0; i < 50000; i += 239) { EXPECT_CALL(random_, random()).WillOnce(Return(i)); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // the jitter is 40% of 1000, so should be 400 EXPECT_CALL(*test_sessions_[0]->interval_timer_, - enableTimer(std::chrono::milliseconds(1000 + i % 400))); + enableTimer(std::chrono::milliseconds(1000 + i % 400), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); } @@ -781,13 +782,14 @@ TEST_F(HttpHealthCheckerImplTest, SuccessWithSpurious100Continue) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); std::unique_ptr continue_headers( @@ -808,13 +810,14 @@ TEST_F(HttpHealthCheckerImplTest, SuccessWithSpuriousMetadata) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); std::unique_ptr metadata_map(new Http::MetadataMap()); @@ -837,19 +840,21 @@ TEST_F(HttpHealthCheckerImplTest, SuccessWithMultipleHosts) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectSessionCreate(); expectStreamCreate(1); - EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).Times(2); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .Times(2) .WillRepeatedly(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - EXPECT_CALL(*test_sessions_[1]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[1]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[1]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true); respond(1, "200", false, false, true); @@ -870,19 +875,21 @@ TEST_F(HttpHealthCheckerImplTest, SuccessWithMultipleHostSets) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectSessionCreate(); expectStreamCreate(1); - EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).Times(2); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .Times(2) .WillRepeatedly(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - EXPECT_CALL(*test_sessions_[1]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[1]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[1]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true); respond(1, "200", false, false, true); @@ -924,7 +931,7 @@ TEST_F(HttpHealthCheckerImplTest, ZeroRetryInterval) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true)) .WillOnce(Invoke([&](const Http::HeaderMap& headers, bool) { EXPECT_EQ(headers.Host()->value().getStringView(), host); @@ -936,7 +943,7 @@ TEST_F(HttpHealthCheckerImplTest, ZeroRetryInterval) { EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).WillOnce(Return(0)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)).WillOnce(Return(0)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); absl::optional health_checked_cluster("locations-production-iad"); respond(0, "200", false, false, true, false, health_checked_cluster); @@ -957,7 +964,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheck) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true)) .WillOnce(Invoke([&](const Http::HeaderMap& headers, bool) { EXPECT_EQ(headers.Host()->value().getStringView(), host); @@ -970,7 +977,8 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheck) { EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); absl::optional health_checked_cluster("locations-production-iad"); respond(0, "200", false, false, true, false, health_checked_cluster); @@ -992,7 +1000,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithCustomHostValue) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true)) .WillOnce(Invoke([&](const Http::HeaderMap& headers, bool) { EXPECT_EQ(headers.Host()->value().getStringView(), host); @@ -1003,7 +1011,8 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithCustomHostValue) { EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); absl::optional health_checked_cluster("locations-production-iad"); respond(0, "200", false, false, true, false, health_checked_cluster); @@ -1053,7 +1062,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithAdditionalHeaders) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true)) .WillRepeatedly(Invoke([&](const Http::HeaderMap& headers, bool) { EXPECT_EQ(headers.get(header_ok)->value().getStringView(), value_ok); @@ -1079,13 +1088,14 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithAdditionalHeaders) { EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); absl::optional health_checked_cluster("locations-production-iad"); respond(0, "200", false, false, true, false, health_checked_cluster); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); } @@ -1110,7 +1120,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithoutUserAgent) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true)) .WillRepeatedly(Invoke( [&](const Http::HeaderMap& headers, bool) { EXPECT_EQ(headers.UserAgent(), nullptr); })); @@ -1119,13 +1129,14 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithoutUserAgent) { EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); absl::optional health_checked_cluster("locations-production-iad"); respond(0, "200", false, false, true, false, health_checked_cluster); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); } @@ -1144,13 +1155,14 @@ TEST_F(HttpHealthCheckerImplTest, ServiceDoesNotMatchFail) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); absl::optional health_checked_cluster("api-production-iad"); respond(0, "200", false, false, true, false, health_checked_cluster); @@ -1174,13 +1186,14 @@ TEST_F(HttpHealthCheckerImplTest, ServiceNotPresentInResponseFail) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, false); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( @@ -1201,13 +1214,14 @@ TEST_F(HttpHealthCheckerImplTest, ServiceCheckRuntimeOff) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); absl::optional health_checked_cluster("api-production-iad"); respond(0, "200", false, false, true, false, health_checked_cluster); @@ -1230,10 +1244,10 @@ TEST_F(HttpHealthCheckerImplTest, SuccessNoTraffic) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true, true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1247,7 +1261,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessStartFailedSuccessFirst) { Host::HealthFlag::FAILED_ACTIVE_HC); expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); // Test fast success immediately moves us to healthy. @@ -1255,7 +1269,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessStartFailedSuccessFirst) { EXPECT_CALL(*event_logger_, logAddHealthy(_, _, true)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).WillOnce(Return(500)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(500))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(500), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1278,7 +1292,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailRemoveHostInCallbackNoClose) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)) @@ -1287,7 +1301,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailRemoveHostInCallbackNoClose) { cluster_->prioritySet().runUpdateCallbacks(0, {}, {host}); })); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)).Times(0); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)).Times(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()).Times(0); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); respond(0, "503", false); @@ -1300,7 +1314,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailRemoveHostInCallbackClose) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)) @@ -1309,7 +1323,7 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailRemoveHostInCallbackClose) { cluster_->prioritySet().runUpdateCallbacks(0, {}, {host}); })); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)).Times(0); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)).Times(0); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()).Times(0); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); respond(0, "503", true); @@ -1321,12 +1335,12 @@ TEST_F(HttpHealthCheckerImplTest, HttpFail) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); respond(0, "503", false); @@ -1337,12 +1351,12 @@ TEST_F(HttpHealthCheckerImplTest, HttpFail) { EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::UNHEALTHY); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( @@ -1350,13 +1364,13 @@ TEST_F(HttpHealthCheckerImplTest, HttpFail) { EXPECT_EQ(Host::Health::Unhealthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1368,12 +1382,12 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailLogError) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); respond(0, "503", false); @@ -1384,13 +1398,13 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailLogError) { EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::UNHEALTHY); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // logUnhealthy is called with first_check == false EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, false)); respond(0, "503", false); @@ -1401,12 +1415,12 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailLogError) { EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::UNHEALTHY); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( @@ -1414,13 +1428,13 @@ TEST_F(HttpHealthCheckerImplTest, HttpFailLogError) { EXPECT_EQ(Host::Health::Unhealthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1435,23 +1449,23 @@ TEST_F(HttpHealthCheckerImplTest, Disconnect) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); expectClientCreate(0); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(cluster_->prioritySet().getMockHostSet(0)->hosts_[0], HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( @@ -1466,12 +1480,12 @@ TEST_F(HttpHealthCheckerImplTest, Timeout) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); @@ -1491,7 +1505,7 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutThenSuccess) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); // Do a response that is not complete but includes headers. @@ -1502,18 +1516,18 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutThenSuccess) { EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); expectClientCreate(0); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1526,24 +1540,24 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutThenRemoteClose) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); EXPECT_CALL(*test_sessions_[0]->client_connection_, close(_)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); expectClientCreate(0); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->client_connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( @@ -1562,11 +1576,11 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutAfterDisconnect) { expectSessionCreate(); expectStreamCreate(0); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)).Times(2); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)).Times(2); health_checker_->start(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)).Times(1); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)).Times(2); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)).Times(2); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); for (auto& session : test_sessions_) { session->client_connection_->close(Network::ConnectionCloseType::NoFlush); @@ -1576,7 +1590,7 @@ TEST_F(HttpHealthCheckerImplTest, TimeoutAfterDisconnect) { EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - test_sessions_[0]->timeout_timer_->enableTimer(std::chrono::seconds(10)); + test_sessions_[0]->timeout_timer_->enableTimer(std::chrono::seconds(10), nullptr); test_sessions_[0]->timeout_timer_->invokeCallback(); EXPECT_EQ(Host::Health::Unhealthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1590,7 +1604,7 @@ TEST_F(HttpHealthCheckerImplTest, DynamicAddAndRemove) { expectStreamCreate(0); cluster_->prioritySet().getMockHostSet(0)->hosts_ = { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); cluster_->prioritySet().getMockHostSet(0)->runCallbacks( {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {}); @@ -1608,17 +1622,17 @@ TEST_F(HttpHealthCheckerImplTest, ConnectionClose) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); expectClientCreate(0); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); test_sessions_[0]->interval_timer_->invokeCallback(); } @@ -1630,17 +1644,17 @@ TEST_F(HttpHealthCheckerImplTest, ProxyConnectionClose) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false, true); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); expectClientCreate(0); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); test_sessions_[0]->interval_timer_->invokeCallback(); } @@ -1650,39 +1664,39 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { makeTestHost(cluster_->info_, "tcp://128.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); // First check should respect no_traffic_interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); cluster_->info_->stats().upstream_cx_total_.inc(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Follow up successful checks should respect interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Follow up successful checks should respect interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -1692,33 +1706,33 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { // check respects "unhealthy_interval". EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "503", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "503", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "503", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -1726,21 +1740,21 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { // When transitioning to a successful state, checks should respect healthy_edge_interval. Health // state should be delayed pending healthy threshold. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -1749,22 +1763,22 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { // the default interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -1773,11 +1787,11 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { // timeout, being a network type failure, should respect unhealthy threshold before changing the // health state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a network timeout. expectClientCreate(0); // Needed after a response is sent. @@ -1785,11 +1799,11 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a network timeout. expectClientCreate(0); // Needed after a response is sent. @@ -1800,11 +1814,11 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { // reached, health state should also change. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a network timeout. expectClientCreate(0); // Needed after a response is sent. @@ -1813,11 +1827,11 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { // Remaining failing checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a network timeout. expectClientCreate(0); // Needed after a response is sent. @@ -1826,21 +1840,21 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { // When transitioning to a successful state, checks should respect healthy_edge_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -1849,18 +1863,18 @@ TEST_F(HttpHealthCheckerImplTest, HealthCheckIntervals) { // the default interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); } @@ -1873,10 +1887,10 @@ TEST_F(HttpHealthCheckerImplTest, RemoteCloseBetweenChecks) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1885,10 +1899,10 @@ TEST_F(HttpHealthCheckerImplTest, RemoteCloseBetweenChecks) { expectClientCreate(0); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); test_sessions_[0]->interval_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1903,10 +1917,10 @@ TEST_F(HttpHealthCheckerImplTest, DontReuseConnectionBetweenChecks) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1916,10 +1930,10 @@ TEST_F(HttpHealthCheckerImplTest, DontReuseConnectionBetweenChecks) { // closes the connection. expectClientCreate(0); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); test_sessions_[0]->interval_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respond(0, "200", false); EXPECT_EQ(Host::Health::Healthy, cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->health()); @@ -1933,10 +1947,10 @@ TEST_F(HttpHealthCheckerImplTest, StreamReachesWatermarkDuringCheck) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->request_encoder_.stream_.runHighWatermarkCallbacks(); @@ -1954,10 +1968,10 @@ TEST_F(HttpHealthCheckerImplTest, ConnectionReachesWatermarkDuringCheck) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(_, _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->client_connection_->runHighWatermarkCallbacks(); @@ -1982,7 +1996,7 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithAltPort) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(hosts); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); EXPECT_CALL(test_sessions_[0]->request_encoder_, encodeHeaders(_, true)) .WillOnce(Invoke([&](const Http::HeaderMap& headers, bool) { EXPECT_EQ(headers.Host()->value().getStringView(), host); @@ -1993,7 +2007,8 @@ TEST_F(HttpHealthCheckerImplTest, SuccessServiceCheckWithAltPort) { EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .WillOnce(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); absl::optional health_checked_cluster("locations-production-iad"); respond(0, "200", false, false, true, false, health_checked_cluster); @@ -2013,19 +2028,21 @@ TEST_F(HttpHealthCheckerImplTest, SuccessWithMultipleHostsAndAltPort) { cluster_->info_->stats().upstream_cx_total_.inc(); expectSessionCreate(hosts); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); expectSessionCreate(hosts); expectStreamCreate(1); - EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[1]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.max_interval", _)).Times(2); EXPECT_CALL(runtime_.snapshot_, getInteger("health_check.min_interval", _)) .Times(2) .WillRepeatedly(Return(45000)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); - EXPECT_CALL(*test_sessions_[1]->interval_timer_, enableTimer(std::chrono::milliseconds(45000))); + EXPECT_CALL(*test_sessions_[1]->interval_timer_, + enableTimer(std::chrono::milliseconds(45000), _)); EXPECT_CALL(*test_sessions_[1]->timeout_timer_, disableTimer()); respond(0, "200", false, false, true); respond(1, "200", false, false, true); @@ -2410,7 +2427,7 @@ TEST_F(TcpHealthCheckerImplTest, Success) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); health_checker_->start(); connection_->runHighWatermarkCallbacks(); @@ -2418,7 +2435,7 @@ TEST_F(TcpHealthCheckerImplTest, Success) { connection_->raiseEvent(Network::ConnectionEvent::Connected); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); Buffer::OwnedImpl response; add_uint8(response, 2); read_filter_->onData(response, false); @@ -2434,14 +2451,14 @@ TEST_F(TcpHealthCheckerImplTest, DataWithoutReusingConnection) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(1); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); health_checker_->start(); connection_->raiseEvent(Network::ConnectionEvent::Connected); // Expected execution flow when a healthcheck is successful and reuse_connection is false. EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); EXPECT_CALL(*connection_, close(Network::ConnectionCloseType::NoFlush)).Times(1); Buffer::OwnedImpl response; @@ -2463,7 +2480,7 @@ TEST_F(TcpHealthCheckerImplTest, WrongData) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(1); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); health_checker_->start(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2492,7 +2509,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutThenRemoteClose) { cluster_->prioritySet().getMockHostSet(0)->hosts_ = { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); cluster_->prioritySet().getMockHostSet(0)->runCallbacks( {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {}); @@ -2506,7 +2523,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutThenRemoteClose) { EXPECT_CALL(*connection_, close(_)); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); timeout_timer_->invokeCallback(); EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::TIMEOUT); @@ -2514,14 +2531,14 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutThenRemoteClose) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( Host::HealthFlag::FAILED_ACTIVE_HC)); @@ -2530,7 +2547,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutThenRemoteClose) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2552,7 +2569,7 @@ TEST_F(TcpHealthCheckerImplTest, Timeout) { cluster_->prioritySet().getMockHostSet(0)->hosts_ = { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); cluster_->prioritySet().getMockHostSet(0)->runCallbacks( {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {}); @@ -2567,7 +2584,7 @@ TEST_F(TcpHealthCheckerImplTest, Timeout) { EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); timeout_timer_->invokeCallback(); EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::TIMEOUT); @@ -2586,7 +2603,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { cluster_->prioritySet().getMockHostSet(0)->hosts_ = { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); cluster_->prioritySet().getMockHostSet(0)->runCallbacks( {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {}); @@ -2600,7 +2617,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { EXPECT_CALL(*connection_, close(_)); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); timeout_timer_->invokeCallback(); EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::TIMEOUT); @@ -2608,7 +2625,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2616,7 +2633,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { EXPECT_CALL(*connection_, close(_)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); timeout_timer_->invokeCallback(); EXPECT_EQ(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->getActiveHealthFailureType(), Host::ActiveHealthFailureType::TIMEOUT); @@ -2625,7 +2642,7 @@ TEST_F(TcpHealthCheckerImplTest, DoubleTimeout) { expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2646,14 +2663,14 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutWithoutReusingConnection) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(1); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); health_checker_->start(); connection_->raiseEvent(Network::ConnectionEvent::Connected); // Expected flow when a healthcheck is successful and reuse_connection is false. EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); EXPECT_CALL(*connection_, close(Network::ConnectionCloseType::NoFlush)).Times(1); Buffer::OwnedImpl response; @@ -2666,14 +2683,14 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutWithoutReusingConnection) { // The healthcheck will run again. expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); // Expected flow when a healthcheck times out. EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); // The healthcheck is not yet at the unhealthy threshold. EXPECT_FALSE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( @@ -2687,7 +2704,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutWithoutReusingConnection) { // The healthcheck will run again, it should be failing after this attempt. expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); interval_timer_->invokeCallback(); connection_->raiseEvent(Network::ConnectionEvent::Connected); @@ -2695,7 +2712,7 @@ TEST_F(TcpHealthCheckerImplTest, TimeoutWithoutReusingConnection) { // Expected flow when a healthcheck times out. EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); connection_->raiseEvent(Network::ConnectionEvent::RemoteClose); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( Host::HealthFlag::FAILED_ACTIVE_HC)); @@ -2716,17 +2733,17 @@ TEST_F(TcpHealthCheckerImplTest, NoData) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(0); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); health_checker_->start(); EXPECT_CALL(*connection_, close(_)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); connection_->raiseEvent(Network::ConnectionEvent::Connected); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(0); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); interval_timer_->invokeCallback(); } @@ -2739,7 +2756,7 @@ TEST_F(TcpHealthCheckerImplTest, PassiveFailure) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(0); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); health_checker_->start(); @@ -2755,7 +2772,7 @@ TEST_F(TcpHealthCheckerImplTest, PassiveFailure) { // A single success should not bring us back to healthy. EXPECT_CALL(*connection_, close(_)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); connection_->raiseEvent(Network::ConnectionEvent::Connected); EXPECT_TRUE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( Host::HealthFlag::FAILED_ACTIVE_HC)); @@ -2777,7 +2794,7 @@ TEST_F(TcpHealthCheckerImplTest, PassiveFailureCrossThreadRemoveHostRace) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(0); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); health_checker_->start(); // Do a passive failure. This will not reset the active HC timers. @@ -2806,7 +2823,7 @@ TEST_F(TcpHealthCheckerImplTest, PassiveFailureCrossThreadRemoveClusterRace) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)).Times(0); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); health_checker_->start(); // Do a passive failure. This will not reset the active HC timers. @@ -2834,13 +2851,13 @@ TEST_F(TcpHealthCheckerImplTest, ConnectionLocalFailure) { expectSessionCreate(); expectClientCreate(); EXPECT_CALL(*connection_, write(_, _)); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); health_checker_->start(); // Expect the LocalClose to be handled as a health check failure EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); // Raise a LocalClose that is not triggered by the health monitor itself. // e.g. a failure to setsockopt(). @@ -3092,16 +3109,16 @@ class GrpcHealthCheckerImplTestBase { // Hides timer/stream-related boilerplate of healthcheck start. void expectHealthcheckStart(size_t index) { expectStreamCreate(index); - EXPECT_CALL(*test_sessions_[index]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[index]->timeout_timer_, enableTimer(_, _)); } // Hides timer-related boilerplate of healthcheck stop. void expectHealthcheckStop(size_t index, int interval_ms = 0) { if (interval_ms > 0) { EXPECT_CALL(*test_sessions_[index]->interval_timer_, - enableTimer(std::chrono::milliseconds(interval_ms))); + enableTimer(std::chrono::milliseconds(interval_ms), _)); } else { - EXPECT_CALL(*test_sessions_[index]->interval_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[index]->interval_timer_, enableTimer(_, _)); } EXPECT_CALL(*test_sessions_[index]->timeout_timer_, disableTimer()); } @@ -3532,7 +3549,7 @@ TEST_F(GrpcHealthCheckerImplTest, DynamicAddAndRemove) { expectStreamCreate(0); cluster_->prioritySet().getMockHostSet(0)->hosts_ = { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); cluster_->prioritySet().getMockHostSet(0)->runCallbacks( {cluster_->prioritySet().getMockHostSet(0)->hosts_.back()}, {}); @@ -3548,39 +3565,39 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { makeTestHost(cluster_->info_, "tcp://128.0.0.1:80")}; expectSessionCreate(); expectStreamCreate(0); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); health_checker_->start(); // First check should respect no_traffic_interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(5000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); cluster_->info_->stats().upstream_cx_total_.inc(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Follow up successful checks should respect interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Follow up successful checks should respect interval setting. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -3590,33 +3607,33 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { // check respects "unhealthy_interval". EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent failing checks should respect unhealthy_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -3624,21 +3641,21 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { // When transitioning to a successful state, checks should respect healthy_edge_interval. Health // state should be delayed pending healthy threshold. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -3647,22 +3664,22 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { // the default interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -3671,21 +3688,21 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { // timeout, being a network type failure, should respect unhealthy threshold before changing the // health state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(3000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -3694,43 +3711,43 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { // reached, health state should also change. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Remaining failing checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(2000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); test_sessions_[0]->timeout_timer_->invokeCallback(); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // When transitioning to a successful state, checks should respect healthy_edge_interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); EXPECT_CALL(*this, onHostStatus(_, HealthTransition::ChangePending)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); @@ -3739,18 +3756,18 @@ TEST_F(GrpcHealthCheckerImplTest, HealthCheckIntervals) { // the default interval. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); - EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_)); + EXPECT_CALL(*test_sessions_[0]->timeout_timer_, enableTimer(_, _)); // Needed after a response is sent. expectStreamCreate(0); test_sessions_[0]->interval_timer_->invokeCallback(); // Subsequent checks shouldn't change the state. EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); - EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*test_sessions_[0]->interval_timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(*test_sessions_[0]->timeout_timer_, disableTimer()); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); } diff --git a/test/common/upstream/load_stats_reporter_test.cc b/test/common/upstream/load_stats_reporter_test.cc index 05dcdf9f22ee..11a2bdfc9bce 100644 --- a/test/common/upstream/load_stats_reporter_test.cc +++ b/test/common/upstream/load_stats_reporter_test.cc @@ -63,7 +63,7 @@ class LoadStatsReporterTest : public testing::Test { std::copy(cluster_names.begin(), cluster_names.end(), Protobuf::RepeatedPtrFieldBackInserter(response->mutable_clusters())); - EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000))); + EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000), _)); load_stats_reporter_->onReceiveMessage(std::move(response)); } @@ -84,7 +84,7 @@ class LoadStatsReporterTest : public testing::Test { // Validate that stream creation results in a timer based retry. TEST_F(LoadStatsReporterTest, StreamCreationFailure) { EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(*retry_timer_, enableTimer(_)); + EXPECT_CALL(*retry_timer_, enableTimer(_, _)); createLoadStatsReporter(); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage({}); @@ -98,13 +98,13 @@ TEST_F(LoadStatsReporterTest, TestPubSub) { deliverLoadStatsResponse({"foo"}); EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)); - EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000))); + EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000), _)); response_timer_cb_(); deliverLoadStatsResponse({"bar"}); EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)); - EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000))); + EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000), _)); response_timer_cb_(); } @@ -135,7 +135,7 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { Protobuf::util::TimeUtil::MicrosecondsToDuration(1)); expectSendMessage({foo_cluster_stats}); } - EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000))); + EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000), _)); response_timer_cb_(); // Some traffic on foo/bar in between previous request and next response. @@ -163,7 +163,7 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { Protobuf::util::TimeUtil::MicrosecondsToDuration(22)); expectSendMessage({bar_cluster_stats, foo_cluster_stats}); } - EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000))); + EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000), _)); response_timer_cb_(); // Some traffic on foo/bar in between previous request and next response. @@ -184,7 +184,7 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { Protobuf::util::TimeUtil::MicrosecondsToDuration(5)); expectSendMessage({bar_cluster_stats}); } - EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000))); + EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000), _)); response_timer_cb_(); // Some traffic on foo/bar in between previous request and next response. @@ -212,7 +212,7 @@ TEST_F(LoadStatsReporterTest, ExistingClusters) { Protobuf::util::TimeUtil::MicrosecondsToDuration(14)); expectSendMessage({bar_cluster_stats, foo_cluster_stats}); } - EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000))); + EXPECT_CALL(*response_timer_, enableTimer(std::chrono::milliseconds(42000), _)); response_timer_cb_(); } @@ -222,7 +222,7 @@ TEST_F(LoadStatsReporterTest, RemoteStreamClose) { expectSendMessage({}); createLoadStatsReporter(); EXPECT_CALL(*response_timer_, disableTimer()); - EXPECT_CALL(*retry_timer_, enableTimer(_)); + EXPECT_CALL(*retry_timer_, enableTimer(_, _)); load_stats_reporter_->onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage({}); diff --git a/test/common/upstream/logical_dns_cluster_test.cc b/test/common/upstream/logical_dns_cluster_test.cc index e09890e76ebe..b3d36e7c43b4 100644 --- a/test/common/upstream/logical_dns_cluster_test.cc +++ b/test/common/upstream/logical_dns_cluster_test.cc @@ -75,7 +75,7 @@ class LogicalDnsClusterTest : public testing::Test { EXPECT_CALL(membership_updated_, ready()); EXPECT_CALL(initialized_, ready()); - EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(4000), _)); dns_callback_(TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); @@ -104,7 +104,7 @@ class LogicalDnsClusterTest : public testing::Test { resolve_timer_->invokeCallback(); // Should not cause any changes. - EXPECT_CALL(*resolve_timer_, enableTimer(_)); + EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); dns_callback_(TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2", "127.0.0.3"})); EXPECT_EQ("127.0.0.1:" + std::to_string(expected_hc_port), @@ -136,7 +136,7 @@ class LogicalDnsClusterTest : public testing::Test { resolve_timer_->invokeCallback(); // Should cause a change. - EXPECT_CALL(*resolve_timer_, enableTimer(_)); + EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); dns_callback_(TestUtility::makeDnsResponse({"127.0.0.3", "127.0.0.1", "127.0.0.2"})); EXPECT_EQ("127.0.0.3:" + std::to_string(expected_hc_port), @@ -154,7 +154,7 @@ class LogicalDnsClusterTest : public testing::Test { resolve_timer_->invokeCallback(); // Empty should not cause any change. - EXPECT_CALL(*resolve_timer_, enableTimer(_)); + EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); dns_callback_({}); EXPECT_EQ(logical_host, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]); @@ -255,7 +255,7 @@ TEST_P(LogicalDnsParamTest, ImmediateResolve) { EXPECT_CALL(*dns_resolver_, resolve("foo.bar.com", std::get<1>(GetParam()), _)) .WillOnce(Invoke([&](const std::string&, Network::DnsLookupFamily, Network::DnsResolver::ResolveCb cb) -> Network::ActiveDnsQuery* { - EXPECT_CALL(*resolve_timer_, enableTimer(_)); + EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); cb(TestUtility::makeDnsResponse(std::get<2>(GetParam()))); return nullptr; })); diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index eccedcc9437c..3c6546f472f7 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -167,7 +167,7 @@ TEST_F(OriginalDstClusterTest, CleanupInterval) { EXPECT_CALL(initialized_, ready()); EXPECT_CALL(membership_updated_, ready()).Times(0); - EXPECT_CALL(*cleanup_timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*cleanup_timer_, enableTimer(std::chrono::milliseconds(1000), _)); setupFromYaml(yaml); EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); @@ -184,7 +184,7 @@ TEST_F(OriginalDstClusterTest, NoContext) { EXPECT_CALL(initialized_, ready()); EXPECT_CALL(membership_updated_, ready()).Times(0); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); setupFromYaml(yaml); EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); @@ -241,7 +241,7 @@ TEST_F(OriginalDstClusterTest, Membership) { )EOF"; EXPECT_CALL(initialized_, ready()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); setupFromYaml(yaml); EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); @@ -289,7 +289,7 @@ TEST_F(OriginalDstClusterTest, Membership) { // Make host time out, no membership changes happen on the first timeout. ASSERT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(true, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->used()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); cleanup_timer_->invokeCallback(); EXPECT_EQ( cluster_hosts, @@ -299,7 +299,7 @@ TEST_F(OriginalDstClusterTest, Membership) { ASSERT_EQ(1UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(false, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->used()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); EXPECT_CALL(membership_updated_, ready()); cleanup_timer_->invokeCallback(); EXPECT_NE(cluster_hosts, @@ -332,7 +332,7 @@ TEST_F(OriginalDstClusterTest, Membership2) { )EOF"; EXPECT_CALL(initialized_, ready()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); setupFromYaml(yaml); EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); @@ -391,7 +391,7 @@ TEST_F(OriginalDstClusterTest, Membership2) { ASSERT_EQ(2UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(true, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->used()); EXPECT_EQ(true, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[1]->used()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); cleanup_timer_->invokeCallback(); EXPECT_EQ( cluster_hosts, @@ -402,7 +402,7 @@ TEST_F(OriginalDstClusterTest, Membership2) { EXPECT_EQ(false, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]->used()); EXPECT_EQ(false, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[1]->used()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); EXPECT_CALL(membership_updated_, ready()); cleanup_timer_->invokeCallback(); EXPECT_NE(cluster_hosts, @@ -420,7 +420,7 @@ TEST_F(OriginalDstClusterTest, Connection) { )EOF"; EXPECT_CALL(initialized_, ready()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); setupFromYaml(yaml); EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); @@ -460,7 +460,7 @@ TEST_F(OriginalDstClusterTest, MultipleClusters) { )EOF"; EXPECT_CALL(initialized_, ready()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); setupFromYaml(yaml); PrioritySetImpl second; @@ -514,7 +514,7 @@ TEST_F(OriginalDstClusterTest, UseHttpHeaderEnabled) { )EOF"; EXPECT_CALL(initialized_, ready()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); setupFromYaml(yaml); EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); @@ -585,7 +585,7 @@ TEST_F(OriginalDstClusterTest, UseHttpHeaderDisabled) { )EOF"; EXPECT_CALL(initialized_, ready()); - EXPECT_CALL(*cleanup_timer_, enableTimer(_)); + EXPECT_CALL(*cleanup_timer_, enableTimer(_, _)); setupFromYaml(yaml); EXPECT_EQ(0UL, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts().size()); diff --git a/test/common/upstream/outlier_detection_impl_test.cc b/test/common/upstream/outlier_detection_impl_test.cc index 6de146f062a7..8c8c51062031 100644 --- a/test/common/upstream/outlier_detection_impl_test.cc +++ b/test/common/upstream/outlier_detection_impl_test.cc @@ -133,7 +133,7 @@ success_rate_stdev_factor: 3000 envoy::api::v2::cluster::OutlierDetection outlier_detection; TestUtility::loadFromYaml(yaml, outlier_detection); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(100))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(100), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, outlier_detection, dispatcher_, runtime_, time_system_, event_logger_)); @@ -156,7 +156,7 @@ TEST_F(OutlierDetectorImplTest, DestroyWithActive) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}, true); addHosts({"tcp://127.0.0.1:81"}, false); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -188,7 +188,7 @@ TEST_F(OutlierDetectorImplTest, DestroyWithActive) { TEST_F(OutlierDetectorImplTest, DestroyHostInUse) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -205,7 +205,7 @@ TEST_F(OutlierDetectorImplTest, DestroyHostInUse) { TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodes) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -231,7 +231,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodes) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -240,7 +240,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodes) { EXPECT_CALL(checker_, check(hosts_[0])); EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -276,7 +276,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaHttpCodes) { TEST_F(OutlierDetectorImplTest, ConnectSuccessWithOptionalHTTP_OK) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -300,7 +300,7 @@ TEST_F(OutlierDetectorImplTest, ConnectSuccessWithOptionalHTTP_OK) { TEST_F(OutlierDetectorImplTest, ExternalOriginEventsNonSplit) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -327,7 +327,7 @@ TEST_F(OutlierDetectorImplTest, ExternalOriginEventsNonSplit) { TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaNonHttpCodes) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -358,7 +358,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaNonHttpCodes) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -367,7 +367,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaNonHttpCodes) { EXPECT_CALL(checker_, check(hosts_[0])); EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -410,7 +410,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlow5xxViaNonHttpCodes) { TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailure) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); @@ -449,7 +449,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailure) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -458,7 +458,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailure) { EXPECT_CALL(checker_, check(hosts_[0])); EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -512,7 +512,7 @@ TEST_F(OutlierDetectorImplTest, TimeoutWithHttpCode) { "tcp://127.0.0.1:84", }); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -536,7 +536,7 @@ TEST_F(OutlierDetectorImplTest, TimeoutWithHttpCode) { EXPECT_CALL(checker_, check(hosts_[0])); EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); @@ -581,7 +581,7 @@ TEST_F(OutlierDetectorImplTest, TimeoutWithHttpCode) { TEST_F(OutlierDetectorImplTest, BasicFlowLocalOriginFailure) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}, true); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, event_logger_)); @@ -610,7 +610,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowLocalOriginFailure) { // Wait short time - not enough to be unejected time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -619,7 +619,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowLocalOriginFailure) { EXPECT_CALL(checker_, check(hosts_[0])); EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -665,7 +665,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowLocalOriginFailure) { TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailureAnd5xx) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); @@ -698,7 +698,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailureAnd5xx) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -707,7 +707,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailureAnd5xx) { EXPECT_CALL(checker_, check(hosts_[0])); EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[0]))); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[0]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_TRUE(hosts_[0]->outlierDetector().lastUnejectionTime()); @@ -757,7 +757,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowGatewayFailureAnd5xx) { TEST_F(OutlierDetectorImplTest, BasicFlowNonHttpCodesExternalOrigin) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -812,7 +812,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { "tcp://127.0.0.1:84", }); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -845,7 +845,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { EXPECT_CALL(*event_logger_, logEject(std::static_pointer_cast(hosts_[4]), _, envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE, true)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_stdev_factor", 1900)) .WillByDefault(Return(1900)); interval_timer_->invokeCallback(); @@ -867,7 +867,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(19999)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); @@ -877,7 +877,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { EXPECT_CALL(checker_, check(hosts_[4])); EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[4]))); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); @@ -900,7 +900,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { loadRq(hosts_[4], 25, 503); time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( @@ -916,7 +916,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { TEST_F(OutlierDetectorImplTest, ExternalOriginEventsWithSplit) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}, true); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, event_logger_)); @@ -951,7 +951,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { "tcp://127.0.0.1:84", }); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -979,7 +979,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { logEject(std::static_pointer_cast(hosts_[4]), _, envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN, true)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_stdev_factor", 1900)) .WillByDefault(Return(1900)); interval_timer_->invokeCallback(); @@ -1001,7 +1001,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { // Interval that doesn't bring the host back in. time_system_.setMonotonicTime(std::chrono::milliseconds(19999)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); @@ -1011,7 +1011,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { EXPECT_CALL(checker_, check(hosts_[4])); EXPECT_CALL(*event_logger_, logUneject(std::static_pointer_cast(hosts_[4]))); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_FALSE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); @@ -1030,7 +1030,7 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { loadRq(hosts_[4], 25, Result::LOCAL_ORIGIN_CONNECT_FAILED); time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( @@ -1044,13 +1044,13 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { // Validate that empty hosts doesn't crash success rate handling when success_rate_minimum_hosts is // zero. This is a regression test for earlier divide-by-zero behavior. TEST_F(OutlierDetectorImplTest, EmptySuccessRate) { - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); loadRq(hosts_, 200, 503); time_system_.setMonotonicTime(std::chrono::milliseconds(10000)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_minimum_hosts", 5)) .WillByDefault(Return(0)); interval_timer_->invokeCallback(); @@ -1059,7 +1059,7 @@ TEST_F(OutlierDetectorImplTest, EmptySuccessRate) { TEST_F(OutlierDetectorImplTest, RemoveWhileEjected) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -1082,14 +1082,14 @@ TEST_F(OutlierDetectorImplTest, RemoveWhileEjected) { EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); time_system_.setMonotonicTime(std::chrono::milliseconds(9999)); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); } TEST_F(OutlierDetectorImplTest, Overflow) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80", "tcp://127.0.0.1:81"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -1118,7 +1118,7 @@ TEST_F(OutlierDetectorImplTest, Overflow) { TEST_F(OutlierDetectorImplTest, NotEnforcing) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -1163,7 +1163,7 @@ TEST_F(OutlierDetectorImplTest, NotEnforcing) { TEST_F(OutlierDetectorImplTest, CrossThreadRemoveRace) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -1185,7 +1185,7 @@ TEST_F(OutlierDetectorImplTest, CrossThreadRemoveRace) { TEST_F(OutlierDetectorImplTest, CrossThreadDestroyRace) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -1208,7 +1208,7 @@ TEST_F(OutlierDetectorImplTest, CrossThreadDestroyRace) { TEST_F(OutlierDetectorImplTest, CrossThreadFailRace) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); @@ -1236,7 +1236,7 @@ TEST_F(OutlierDetectorImplTest, CrossThreadFailRace) { TEST_F(OutlierDetectorImplTest, Consecutive_5xxAlreadyEjected) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); - EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); std::shared_ptr detector(DetectorImpl::create( cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index f55f2884097a..3bb78af76687 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -221,7 +221,7 @@ TEST_F(StrictDnsClusterImplTest, ZeroHostsHealthChecker) { EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)); EXPECT_CALL(initialized, ready()); - EXPECT_CALL(*resolver.timer_, enableTimer(_)); + EXPECT_CALL(*resolver.timer_, enableTimer(_, _)); resolver.dns_callback_({}); EXPECT_EQ(0UL, cluster.prioritySet().hostSetsPerPriority()[0]->hosts().size()); EXPECT_EQ(0UL, cluster.prioritySet().hostSetsPerPriority()[0]->healthyHosts().size()); @@ -301,7 +301,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { cluster.initialize([] {}); resolver1.expectResolve(*dns_resolver_); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_THAT( @@ -312,7 +312,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -320,14 +320,14 @@ TEST_F(StrictDnsClusterImplTest, Basic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.3"})); EXPECT_THAT( @@ -335,7 +335,7 @@ TEST_F(StrictDnsClusterImplTest, Basic) { ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); // Make sure we de-dup the same address. - EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver2.dns_callback_(TestUtility::makeDnsResponse({"10.0.0.1", "10.0.0.1"})); EXPECT_THAT( @@ -390,7 +390,7 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalActiveHealthSkipped) { cluster.initialize([&]() -> void {}); EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)); - EXPECT_CALL(*resolver.timer_, enableTimer(_)).Times(2); + EXPECT_CALL(*resolver.timer_, enableTimer(_, _)).Times(2); resolver.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); // Verify that both endpoints are initially marked with FAILED_ACTIVE_HC, then @@ -443,7 +443,7 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalAfterHcFail) { cluster.initialize([&initialized]() { initialized.ready(); }); EXPECT_CALL(*health_checker, addHostCheckCompleteCb(_)); - EXPECT_CALL(*resolver.timer_, enableTimer(_)).Times(2); + EXPECT_CALL(*resolver.timer_, enableTimer(_, _)).Times(2); resolver.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); // Verify that both endpoints are initially marked with FAILED_ACTIVE_HC, then @@ -592,7 +592,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { cluster.initialize([] {}); resolver1.expectResolve(*dns_resolver_); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_THAT( @@ -611,7 +611,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -623,7 +623,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -633,7 +633,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { // Since no change for localhost1, we expect no rebuild. EXPECT_EQ(2UL, stats_.counter("cluster.name.update_no_rebuild").value()); - EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver2.dns_callback_(TestUtility::makeDnsResponse({"10.0.0.1", "10.0.0.1"})); @@ -642,14 +642,14 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); // We again received the same set as before for localhost1. No rebuild this time. EXPECT_EQ(3UL, stats_.counter("cluster.name.update_no_rebuild").value()); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.3"})); EXPECT_THAT( @@ -657,7 +657,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); // Make sure we de-dup the same address. - EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver2.dns_callback_(TestUtility::makeDnsResponse({"10.0.0.1", "10.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.3:11001", "10.0.0.1:11002"}), @@ -670,7 +670,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { cluster.prioritySet().hostSetsPerPriority()[0]->healthyHostsPerLocality().get().size()); // Make sure that we *don't* de-dup between resolve targets. - EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver3.dns_callback_(TestUtility::makeDnsResponse({"10.0.0.1"})); @@ -704,11 +704,11 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { } }); - EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver2.dns_callback_(TestUtility::makeDnsResponse({})); - EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver3.dns_callback_(TestUtility::makeDnsResponse({})); @@ -789,7 +789,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { cluster.initialize([] {}); resolver1.expectResolve(*dns_resolver_); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.1", "127.0.0.2"})); EXPECT_THAT( @@ -800,7 +800,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), @@ -808,14 +808,14 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.2", "127.0.0.1"})); EXPECT_THAT( std::list({"127.0.0.1:11001", "127.0.0.2:11001"}), ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); resolver1.timer_->invokeCallback(); - EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver1.dns_callback_(TestUtility::makeDnsResponse({"127.0.0.3"})); EXPECT_THAT( @@ -823,7 +823,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); // Make sure we de-dup the same address. - EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver2.dns_callback_(TestUtility::makeDnsResponse({"10.0.0.1", "10.0.0.1"})); EXPECT_THAT( @@ -839,7 +839,7 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasicMultiplePriorities) { EXPECT_EQ(cluster.info().get(), &host->cluster()); } - EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000), _)); EXPECT_CALL(membership_updated, ready()); resolver3.dns_callback_(TestUtility::makeDnsResponse({"192.168.1.1", "192.168.1.2"})); @@ -915,7 +915,7 @@ TEST_F(StrictDnsClusterImplTest, RecordTtlAsDnsRefreshRate) { EXPECT_CALL(membership_updated, ready()); - EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(5000))); + EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(5000), _)); resolver.dns_callback_( TestUtility::makeDnsResponse({"192.168.1.1", "192.168.1.2"}, std::chrono::seconds(5))); } @@ -944,7 +944,7 @@ TEST_F(StrictDnsClusterImplTest, DefaultTtlAsDnsRefreshRateWhenResponseEmpty) { std::move(scope), false); cluster.initialize([] {}); - EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(4000))); + EXPECT_CALL(*resolver.timer_, enableTimer(std::chrono::milliseconds(4000), _)); resolver.dns_callback_(TestUtility::makeDnsResponse({}, std::chrono::seconds(5))); } diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 571d29fe48dd..83ba234f7973 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -34,7 +34,7 @@ class GrpcAccessLoggerImplTest : public testing::Test { void initLogger(std::chrono::milliseconds buffer_flush_interval_msec, size_t buffer_size_bytes) { timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*timer_, enableTimer(buffer_flush_interval_msec)); + EXPECT_CALL(*timer_, enableTimer(buffer_flush_interval_msec, _)); logger_ = std::make_unique(Grpc::RawAsyncClientPtr{async_client_}, log_name_, buffer_flush_interval_msec, buffer_size_bytes, dispatcher_, local_info_); @@ -202,7 +202,7 @@ TEST_F(GrpcAccessLoggerImplTest, Flushing) { initLogger(FlushInterval, 100); // Nothing to do yet. - EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + EXPECT_CALL(*timer_, enableTimer(FlushInterval, _)); timer_->invokeCallback(); envoy::data::accesslog::v2::HTTPAccessLogEntry entry; @@ -227,11 +227,11 @@ TEST_F(GrpcAccessLoggerImplTest, Flushing) { - request: path: /test/path1 )EOF")); - EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + EXPECT_CALL(*timer_, enableTimer(FlushInterval, _)); timer_->invokeCallback(); // Flush on empty message does nothing. - EXPECT_CALL(*timer_, enableTimer(FlushInterval)); + EXPECT_CALL(*timer_, enableTimer(FlushInterval, _)); timer_->invokeCallback(); } diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index a3190debb640..8899a22f4e8a 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -170,12 +170,12 @@ class RedisClusterTest : public testing::Test, } void expectClusterSlotResponse(NetworkFilters::Common::Redis::RespValuePtr&& response) { - EXPECT_CALL(*resolve_timer_, enableTimer(_)); + EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); pool_callbacks_->onResponse(std::move(response)); } void expectClusterSlotFailure() { - EXPECT_CALL(*resolve_timer_, enableTimer(_)); + EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); pool_callbacks_->onFailure(); } @@ -646,7 +646,7 @@ TEST_F(RedisClusterTest, EmptyDnsResponse) { Event::MockTimer* dns_timer = new NiceMock(&dispatcher_); setupFromV2Yaml(BasicConfig); const std::list resolved_addresses{}; - EXPECT_CALL(*dns_timer, enableTimer(_)); + EXPECT_CALL(*dns_timer, enableTimer(_, _)); expectResolveDiscovery(Network::DnsLookupFamily::V4Only, "foo.bar.com", resolved_addresses); EXPECT_CALL(initialized_, ready()); @@ -657,7 +657,7 @@ TEST_F(RedisClusterTest, EmptyDnsResponse) { EXPECT_EQ(1U, cluster_->info()->stats().update_empty_.value()); // Does not recreate the timer on subsequent DNS resolve calls. - EXPECT_CALL(*dns_timer, enableTimer(_)); + EXPECT_CALL(*dns_timer, enableTimer(_, _)); expectResolveDiscovery(Network::DnsLookupFamily::V4Only, "foo.bar.com", resolved_addresses); dns_timer->invokeCallback(); diff --git a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc index 7f69ca894a68..0097b254329b 100644 --- a/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc +++ b/test/extensions/common/dynamic_forward_proxy/dns_cache_impl_test.cc @@ -98,7 +98,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, @@ -113,7 +113,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { 1 /* added */, 0 /* removed */, 1 /* num hosts */); // Address does not change. - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, @@ -130,7 +130,7 @@ TEST_F(DnsCacheImplTest, ResolveSuccess) { // Address does change. EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.2:80", "foo.com", false))); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.2"})); checkStats(3 /* attempt */, 3 /* success */, 0 /* failure */, 2 /* address changed */, @@ -155,7 +155,7 @@ TEST_F(DnsCacheImplTest, Ipv4Address) { update_callbacks_, onDnsHostAddOrUpdate("127.0.0.1", DnsHostInfoEquals("127.0.0.1:80", "127.0.0.1", true))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"127.0.0.1"})); } @@ -177,7 +177,7 @@ TEST_F(DnsCacheImplTest, Ipv4AddressWithPort) { onDnsHostAddOrUpdate("127.0.0.1:10000", DnsHostInfoEquals("127.0.0.1:10000", "127.0.0.1", true))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"127.0.0.1"})); } @@ -198,7 +198,7 @@ TEST_F(DnsCacheImplTest, Ipv6Address) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("[::1]", DnsHostInfoEquals("[::1]:80", "::1", true))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"::1"})); } @@ -219,7 +219,7 @@ TEST_F(DnsCacheImplTest, Ipv6AddressWithPort) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("[::1]:10000", DnsHostInfoEquals("[::1]:10000", "::1", true))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"::1"})); } @@ -243,7 +243,7 @@ TEST_F(DnsCacheImplTest, TTL) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"}, std::chrono::seconds(0))); checkStats(1 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, @@ -257,7 +257,7 @@ TEST_F(DnsCacheImplTest, TTL) { checkStats(2 /* attempt */, 1 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); checkStats(2 /* attempt */, 2 /* success */, 0 /* failure */, 1 /* address changed */, 1 /* added */, 0 /* removed */, 1 /* num hosts */); @@ -300,7 +300,7 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { EXPECT_CALL(update_callbacks_, onDnsHostAddOrUpdate("foo.com", DnsHostInfoEquals("10.0.0.1:80", "foo.com", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000), _)); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"}, std::chrono::seconds(0))); // Re-resolve with ~30s passed. TTL should still be OK at 60s. @@ -308,7 +308,7 @@ TEST_F(DnsCacheImplTest, TTLWithCustomParameters) { EXPECT_CALL(*resolver_, resolve("foo.com", _, _)) .WillOnce(DoAll(SaveArg<2>(&resolve_cb), Return(&resolver_->active_query_))); resolve_timer->invokeCallback(); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(30000), _)); resolve_cb(TestUtility::makeDnsResponse({"10.0.0.1"})); // Re-resolve with ~30s passed. TTL should expire. @@ -340,7 +340,7 @@ TEST_F(DnsCacheImplTest, InlineResolve) { update_callbacks_, onDnsHostAddOrUpdate("localhost", DnsHostInfoEquals("127.0.0.1:80", "localhost", false))); EXPECT_CALL(callbacks, onLoadDnsCacheComplete()); - EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000))); + EXPECT_CALL(*resolve_timer, enableTimer(std::chrono::milliseconds(60000), _)); post_cb(); } diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index e6cbf4ace6b9..038e674c73b3 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -142,7 +142,7 @@ class FaultFilterTest : public testing::Test { void expectDelayTimer(uint64_t duration_ms) { timer_ = new Event::MockTimer(&decoder_filter_callbacks_.dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(duration_ms))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(duration_ms), _)); EXPECT_CALL(*timer_, disableTimer()); } @@ -766,7 +766,7 @@ TEST_F(FaultFilterTest, TimerResetAfterStreamReset) { SCOPED_TRACE("FixedDelayWithStreamReset"); timer_ = new Event::MockTimer(&decoder_filter_callbacks_.dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(5000UL))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(5000UL), _)); EXPECT_CALL(decoder_filter_callbacks_.stream_info_, setResponseFlag(StreamInfo::ResponseFlag::DelayInjected)); @@ -1053,7 +1053,7 @@ TEST_F(FaultFilterRateLimitTest, ResponseRateLimitEnabled) { // Send a small amount of data which should be within limit. Buffer::OwnedImpl data1("hello"); - EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0))); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data1, false)); EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual("hello"), false)); @@ -1064,11 +1064,11 @@ TEST_F(FaultFilterRateLimitTest, ResponseRateLimitEnabled) { // Send 1152 bytes of data which is 1s + 2 refill cycles of data. EXPECT_CALL(encoder_filter_callbacks_, onEncoderFilterAboveWriteBufferHighWatermark()); - EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0))); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); Buffer::OwnedImpl data2(std::string(1152, 'a')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data2, false)); - EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(63))); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(63), _)); EXPECT_CALL(encoder_filter_callbacks_, onEncoderFilterBelowWriteBufferLowWatermark()); EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual(std::string(1024, 'a')), false)); @@ -1076,7 +1076,7 @@ TEST_F(FaultFilterRateLimitTest, ResponseRateLimitEnabled) { // Fire timer, also advance time. time_system_.sleep(std::chrono::milliseconds(63)); - EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(63))); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(63), _)); EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual(std::string(64, 'a')), false)); token_timer->invokeCallback(); @@ -1087,7 +1087,7 @@ TEST_F(FaultFilterRateLimitTest, ResponseRateLimitEnabled) { // Fire timer, also advance time. time_system_.sleep(std::chrono::milliseconds(63)); - EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(63))); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(63), _)); EXPECT_CALL(encoder_filter_callbacks_, injectEncodedDataToFilterChain(BufferStringEqual(std::string(64, 'a')), false)); token_timer->invokeCallback(); @@ -1102,7 +1102,7 @@ TEST_F(FaultFilterRateLimitTest, ResponseRateLimitEnabled) { time_system_.sleep(std::chrono::seconds(1)); // Now send 1024 in one shot with end_stream true which should go through and end the stream. - EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0))); + EXPECT_CALL(*token_timer, enableTimer(std::chrono::milliseconds(0), _)); Buffer::OwnedImpl data4(std::string(1024, 'c')); EXPECT_EQ(Http::FilterDataStatus::StopIterationNoBuffer, filter_->encodeData(data4, true)); EXPECT_CALL(encoder_filter_callbacks_, diff --git a/test/extensions/filters/http/health_check/health_check_test.cc b/test/extensions/filters/http/health_check/health_check_test.cc index 85f2d242c39d..34df211019b7 100644 --- a/test/extensions/filters/http/health_check/health_check_test.cc +++ b/test/extensions/filters/http/health_check/health_check_test.cc @@ -37,7 +37,7 @@ class HealthCheckFilterTest : public testing::Test { if (caching) { cache_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*cache_timer_, enableTimer(_)); + EXPECT_CALL(*cache_timer_, enableTimer(_, _)); cache_manager_.reset(new HealthCheckCacheManager(dispatcher_, std::chrono::milliseconds(1))); } @@ -359,7 +359,7 @@ TEST_F(HealthCheckFilterCachingTest, All) { filter_->decodeHeaders(request_headers_, true)); // Fire the timer, this should result in the next request going through. - EXPECT_CALL(*cache_timer_, enableTimer(_)); + EXPECT_CALL(*cache_timer_, enableTimer(_, _)); cache_timer_->invokeCallback(); prepareFilter(true); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, true)); @@ -393,7 +393,7 @@ TEST_F(HealthCheckFilterCachingTest, DegradedHeader) { filter_->decodeHeaders(request_headers_, true)); // Fire the timer, this should result in the next request going through. - EXPECT_CALL(*cache_timer_, enableTimer(_)); + EXPECT_CALL(*cache_timer_, enableTimer(_, _)); cache_timer_->invokeCallback(); prepareFilter(true); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers_, true)); diff --git a/test/extensions/filters/http/squash/squash_filter_test.cc b/test/extensions/filters/http/squash/squash_filter_test.cc index 0800a51499cb..2cf509c59b5f 100644 --- a/test/extensions/filters/http/squash/squash_filter_test.cc +++ b/test/extensions/filters/http/squash/squash_filter_test.cc @@ -196,7 +196,7 @@ class SquashFilterTest : public testing::Test { expectAsyncClientSend(); - EXPECT_CALL(*attachmentTimeout_timer_, enableTimer(config_->attachmentTimeout())); + EXPECT_CALL(*attachmentTimeout_timer_, enableTimer(config_->attachmentTimeout(), _)); Envoy::Http::TestHeaderMapImpl headers{{":method", "GET"}, {":authority", "www.solo.io"}, @@ -354,7 +354,7 @@ TEST_F(SquashFilterTest, CheckRetryPollingAttachment) { NiceMock* retry_timer; retry_timer = new NiceMock(&filter_callbacks_.dispatcher_); - EXPECT_CALL(*retry_timer, enableTimer(config_->attachmentPollPeriod())); + EXPECT_CALL(*retry_timer, enableTimer(config_->attachmentPollPeriod(), _)); completeGetStatusRequest("attaching"); // Expect the second get attachment request @@ -372,7 +372,7 @@ TEST_F(SquashFilterTest, CheckRetryPollingAttachmentOnFailure) { NiceMock* retry_timer; retry_timer = new NiceMock(&filter_callbacks_.dispatcher_); - EXPECT_CALL(*retry_timer, enableTimer(config_->attachmentPollPeriod())); + EXPECT_CALL(*retry_timer, enableTimer(config_->attachmentPollPeriod(), _)); popPendingCallback()->onFailure(Envoy::Http::AsyncClient::FailureReason::Reset); // Expect the second get attachment request @@ -391,7 +391,7 @@ TEST_F(SquashFilterTest, DestroyedInTheMiddle) { completeCreateRequest(); auto retry_timer = new NiceMock(&filter_callbacks_.dispatcher_); - EXPECT_CALL(*retry_timer, enableTimer(config_->attachmentPollPeriod())); + EXPECT_CALL(*retry_timer, enableTimer(config_->attachmentPollPeriod(), _)); completeGetStatusRequest("attaching"); EXPECT_CALL(*attachmentTimeout_timer_, disableTimer()); @@ -413,7 +413,7 @@ TEST_F(SquashFilterTest, InvalidJsonForGetAttachment) { completeCreateRequest(); auto retry_timer = new NiceMock(&filter_callbacks_.dispatcher_); - EXPECT_CALL(*retry_timer, enableTimer(config_->attachmentPollPeriod())); + EXPECT_CALL(*retry_timer, enableTimer(config_->attachmentPollPeriod(), _)); completeRequest("200", "This is not a JSON object"); } @@ -430,10 +430,9 @@ TEST_F(SquashFilterTest, TimerExpiresInline) { initFilter(); attachmentTimeout_timer_ = new NiceMock(&filter_callbacks_.dispatcher_); - // TODO: this is a really synthetic test as the callback can't actually be called under the stack - // of enableTimer. It'd be good to clean this up. - EXPECT_CALL(*attachmentTimeout_timer_, enableTimer(config_->attachmentTimeout())) - .WillOnce(Invoke([&](const std::chrono::milliseconds&) { + EXPECT_CALL(*attachmentTimeout_timer_, enableTimer(config_->attachmentTimeout(), _)) + .WillOnce(Invoke([&](const std::chrono::milliseconds&, const ScopeTrackedObject* scope) { + attachmentTimeout_timer_->scope_ = scope; attachmentTimeout_timer_->enabled_ = true; // timer expires inline attachmentTimeout_timer_->invokeCallback(); diff --git a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc index 23b7f9acf6d8..5e50b3769158 100644 --- a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc +++ b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc @@ -167,7 +167,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); // Respond. - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); Http::MessagePtr message(new Http::ResponseMessageImpl( Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}})); message->body() = std::make_unique( @@ -224,7 +224,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { interval_timer_->invokeCallback(); // Error response. - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); message = std::make_unique( Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "503"}}}); callbacks_->onSuccess(std::move(message)); @@ -234,7 +234,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { interval_timer_->invokeCallback(); // Parsing error - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); message = std::make_unique( Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "200"}}}); message->body() = std::make_unique("bad_json"); @@ -245,7 +245,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { interval_timer_->invokeCallback(); // No response failure. - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); callbacks_->onFailure(Http::AsyncClient::FailureReason::Reset); // Interval timer fires, cannot obtain async client. @@ -258,7 +258,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "503"}}})}); return nullptr; })); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); interval_timer_->invokeCallback(); EXPECT_EQ(4U, stats_store_.counter("auth.clientssl.vpn.update_failure").value()); diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index e9d28235610d..a6646c86741a 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -66,7 +66,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF connect_or_op_timer_ = new Event::MockTimer(&dispatcher_); flush_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(*host_, createConnection_(_, _)).WillOnce(Return(conn_info)); EXPECT_CALL(*upstream_connection_, addReadFilter(_)) .WillOnce(SaveArg<0>(&upstream_read_filter_)); @@ -85,7 +85,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF } void onConnected() { - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); upstream_connection_->raiseEvent(Network::ConnectionEvent::Connected); } @@ -170,7 +170,7 @@ TEST_F(RedisClientImplTest, BatchWithTimerFiring) { Common::Redis::RespValue request1; MockPoolCallbacks callbacks1; EXPECT_CALL(*encoder_, encode(Ref(request1), _)); - EXPECT_CALL(*flush_timer_, enableTimer(_)); + EXPECT_CALL(*flush_timer_, enableTimer(_, _)); PoolRequest* handle1 = client_->makeRequest(request1, callbacks1); EXPECT_NE(nullptr, handle1); @@ -212,7 +212,7 @@ TEST_F(RedisClientImplTest, BatchWithTimerCancelledByBufferFlush) { Common::Redis::RespValue request1; MockPoolCallbacks callbacks1; EXPECT_CALL(*encoder_, encode(Ref(request1), _)); - EXPECT_CALL(*flush_timer_, enableTimer(_)); + EXPECT_CALL(*flush_timer_, enableTimer(_, _)); PoolRequest* handle1 = client_->makeRequest(request1, callbacks1); EXPECT_NE(nullptr, handle1); @@ -232,7 +232,7 @@ TEST_F(RedisClientImplTest, BatchWithTimerCancelledByBufferFlush) { InSequence s; Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); @@ -282,7 +282,7 @@ TEST_F(RedisClientImplTest, Basic) { InSequence s; Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); @@ -330,7 +330,7 @@ TEST_F(RedisClientImplTest, Cancel) { Common::Redis::RespValuePtr response1(new Common::Redis::RespValue()); EXPECT_CALL(callbacks1, onResponse_(_)).Times(0); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); @@ -543,7 +543,7 @@ TEST_F(RedisClientImplTest, OpTimeout) { EXPECT_CALL(*encoder_, encode(Ref(request1), _)); EXPECT_CALL(*flush_timer_, enabled()).WillOnce(Return(false)); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); handle1 = client_->makeRequest(request1, callbacks1); EXPECT_NE(nullptr, handle1); @@ -596,7 +596,7 @@ TEST_F(RedisClientImplTest, AskRedirection) { // Simulate redirection failure. EXPECT_CALL(callbacks1, onRedirection(Ref(*response1))).WillOnce(Return(false)); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); @@ -658,7 +658,7 @@ TEST_F(RedisClientImplTest, MovedRedirection) { // Simulate redirection failure. EXPECT_CALL(callbacks1, onRedirection(Ref(*response1))).WillOnce(Return(false)); EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); @@ -719,7 +719,7 @@ TEST_F(RedisClientImplTest, AskRedirectionNotEnabled) { response1->asString() = "ASK 1111 10.1.2.3:4321"; // Simulate redirection failure. EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); @@ -781,7 +781,7 @@ TEST_F(RedisClientImplTest, MovedRedirectionNotEnabled) { // The exact values of the hash slot and IP info are not important. response1->asString() = "MOVED 1111 10.1.2.3:4321"; EXPECT_CALL(callbacks1, onResponse_(Ref(response1))); - EXPECT_CALL(*connect_or_op_timer_, enableTimer(_)); + EXPECT_CALL(*connect_or_op_timer_, enableTimer(_, _)); EXPECT_CALL(host_->outlier_detector_, putResult(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS, _)); callbacks_->onRespValue(std::move(response1)); diff --git a/test/extensions/filters/network/mongo_proxy/proxy_test.cc b/test/extensions/filters/network/mongo_proxy/proxy_test.cc index 2fb9b66221db..01d0c8082332 100644 --- a/test/extensions/filters/network/mongo_proxy/proxy_test.cc +++ b/test/extensions/filters/network/mongo_proxy/proxy_test.cc @@ -133,7 +133,7 @@ TEST_F(MongoProxyFilterTest, DelayFaults) { Event::MockTimer* delay_timer = new Event::MockTimer(&read_filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*delay_timer, enableTimer(std::chrono::milliseconds(10))); + EXPECT_CALL(*delay_timer, enableTimer(std::chrono::milliseconds(10), _)); EXPECT_CALL(*file_, write(_)).Times(AtLeast(1)); EXPECT_CALL(*filter_->decoder_, onData(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { @@ -551,7 +551,7 @@ TEST_F(MongoProxyFilterTest, ConcurrentQueryWithDrainClose) { .WillByDefault(Return(true)); EXPECT_CALL(drain_decision_, drainClose()).WillOnce(Return(true)); drain_timer = new Event::MockTimer(&read_filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*drain_timer, enableTimer(std::chrono::milliseconds(0))); + EXPECT_CALL(*drain_timer, enableTimer(std::chrono::milliseconds(0), _)); filter_->callbacks_->decodeReply(std::move(message)); })); filter_->onWrite(fake_data_, false); @@ -595,7 +595,7 @@ TEST_F(MongoProxyFilterTest, ConnectionDestroyLocal) { Event::MockTimer* delay_timer = new Event::MockTimer(&read_filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*delay_timer, enableTimer(std::chrono::milliseconds(10))); + EXPECT_CALL(*delay_timer, enableTimer(std::chrono::milliseconds(10), _)); EXPECT_CALL(*filter_->decoder_, onData(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { QueryMessagePtr message(new QueryMessageImpl(0, 0)); @@ -619,7 +619,7 @@ TEST_F(MongoProxyFilterTest, ConnectionDestroyRemote) { Event::MockTimer* delay_timer = new Event::MockTimer(&read_filter_callbacks_.connection_.dispatcher_); - EXPECT_CALL(*delay_timer, enableTimer(std::chrono::milliseconds(10))); + EXPECT_CALL(*delay_timer, enableTimer(std::chrono::milliseconds(10), _)); EXPECT_CALL(*filter_->decoder_, onData(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { QueryMessagePtr message(new QueryMessageImpl(0, 0)); diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index f8be7912d129..2d721cacf56d 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -146,13 +146,13 @@ class RedisHealthCheckerTest void expectExistsRequestCreate() { EXPECT_CALL(*client_, makeRequest(Ref(RedisHealthChecker::existsHealthCheckRequest("")), _)) .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&pool_callbacks_)), Return(&pool_request_))); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); } void expectPingRequestCreate() { EXPECT_CALL(*client_, makeRequest(Ref(RedisHealthChecker::pingHealthCheckRequest()), _)) .WillOnce(DoAll(WithArg<1>(SaveArgAddress(&pool_callbacks_)), Return(&pool_request_))); - EXPECT_CALL(*timeout_timer_, enableTimer(_)); + EXPECT_CALL(*timeout_timer_, enableTimer(_, _)); } void exerciseStubs() { @@ -204,7 +204,7 @@ TEST_F(RedisHealthCheckerTest, PingAndVariousFailures) { // Success EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); NetworkFilters::Common::Redis::RespValuePtr response( new NetworkFilters::Common::Redis::RespValue()); response->type(NetworkFilters::Common::Redis::RespType::SimpleString); @@ -217,7 +217,7 @@ TEST_F(RedisHealthCheckerTest, PingAndVariousFailures) { // Failure EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); response = std::make_unique(); pool_callbacks_->onResponse(std::move(response)); @@ -226,7 +226,7 @@ TEST_F(RedisHealthCheckerTest, PingAndVariousFailures) { // Redis failure via disconnect EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); pool_callbacks_->onFailure(); client_->raiseEvent(Network::ConnectionEvent::RemoteClose); @@ -238,7 +238,7 @@ TEST_F(RedisHealthCheckerTest, PingAndVariousFailures) { EXPECT_CALL(pool_request_, cancel()); EXPECT_CALL(*client_, close()); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); timeout_timer_->invokeCallback(); expectClientCreate(); @@ -272,7 +272,7 @@ TEST_F(RedisHealthCheckerTest, FailuresLogging) { // Success EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); NetworkFilters::Common::Redis::RespValuePtr response( new NetworkFilters::Common::Redis::RespValue()); response->type(NetworkFilters::Common::Redis::RespType::SimpleString); @@ -286,7 +286,7 @@ TEST_F(RedisHealthCheckerTest, FailuresLogging) { EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, false)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); response = std::make_unique(); pool_callbacks_->onResponse(std::move(response)); @@ -296,7 +296,7 @@ TEST_F(RedisHealthCheckerTest, FailuresLogging) { // Fail again EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, false)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); response = std::make_unique(); pool_callbacks_->onResponse(std::move(response)); @@ -332,7 +332,7 @@ TEST_F(RedisHealthCheckerTest, LogInitialFailure) { EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); pool_callbacks_->onFailure(); client_->raiseEvent(Network::ConnectionEvent::RemoteClose); @@ -343,7 +343,7 @@ TEST_F(RedisHealthCheckerTest, LogInitialFailure) { // Success EXPECT_CALL(*event_logger_, logAddHealthy(_, _, false)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); NetworkFilters::Common::Redis::RespValuePtr response( new NetworkFilters::Common::Redis::RespValue()); response->type(NetworkFilters::Common::Redis::RespType::SimpleString); @@ -380,7 +380,7 @@ TEST_F(RedisHealthCheckerTest, Exists) { // Success EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); NetworkFilters::Common::Redis::RespValuePtr response( new NetworkFilters::Common::Redis::RespValue()); response->type(NetworkFilters::Common::Redis::RespType::Integer); @@ -393,7 +393,7 @@ TEST_F(RedisHealthCheckerTest, Exists) { // Failure, exists EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); response = std::make_unique(); response->type(NetworkFilters::Common::Redis::RespType::Integer); response->asInteger() = 1; @@ -404,7 +404,7 @@ TEST_F(RedisHealthCheckerTest, Exists) { // Failure, no value EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); response = std::make_unique(); pool_callbacks_->onResponse(std::move(response)); @@ -432,7 +432,7 @@ TEST_F(RedisHealthCheckerTest, ExistsRedirected) { // Success with moved redirection EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); NetworkFilters::Common::Redis::RespValue moved_response; moved_response.type(NetworkFilters::Common::Redis::RespType::Error); moved_response.asString() = "MOVED 1111 127.0.0.1:81"; // exact values not important @@ -443,7 +443,7 @@ TEST_F(RedisHealthCheckerTest, ExistsRedirected) { // Success with ask redirection EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); NetworkFilters::Common::Redis::RespValue ask_response; ask_response.type(NetworkFilters::Common::Redis::RespType::Error); ask_response.asString() = "ASK 1111 127.0.0.1:81"; // exact values not important @@ -471,7 +471,7 @@ TEST_F(RedisHealthCheckerTest, NoConnectionReuse) { // The connection will close on success. EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); EXPECT_CALL(*client_, close()); NetworkFilters::Common::Redis::RespValuePtr response( new NetworkFilters::Common::Redis::RespValue()); @@ -486,7 +486,7 @@ TEST_F(RedisHealthCheckerTest, NoConnectionReuse) { // The connection will close on failure. EXPECT_CALL(*event_logger_, logEjectUnhealthy(_, _, _)); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); EXPECT_CALL(*client_, close()); response = std::make_unique(); pool_callbacks_->onResponse(std::move(response)); @@ -497,7 +497,7 @@ TEST_F(RedisHealthCheckerTest, NoConnectionReuse) { // Redis failure via disconnect, the connection was closed by the other end. EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); pool_callbacks_->onFailure(); client_->raiseEvent(Network::ConnectionEvent::RemoteClose); @@ -509,7 +509,7 @@ TEST_F(RedisHealthCheckerTest, NoConnectionReuse) { EXPECT_CALL(pool_request_, cancel()); EXPECT_CALL(*client_, close()); EXPECT_CALL(*timeout_timer_, disableTimer()); - EXPECT_CALL(*interval_timer_, enableTimer(_)); + EXPECT_CALL(*interval_timer_, enableTimer(_, _)); timeout_timer_->invokeCallback(); expectClientCreate(); diff --git a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc index 50e2722f7cfc..cc80f5672c0e 100644 --- a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc +++ b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc @@ -48,7 +48,7 @@ class DatadogDriverTest : public testing::Test { if (init_timer) { timer_ = new NiceMock(&tls_.dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900), _)); } driver_ = std::make_unique(datadog_config, cm_, stats_, tls_, runtime_); @@ -146,7 +146,7 @@ TEST_F(DatadogDriverTest, FlushSpansTimer) { span->finishSpan(); // Timer should be re-enabled. - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(900), _)); timer_->invokeCallback(); diff --git a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc index cf9e44e1b9f3..e830c55d8217 100644 --- a/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc +++ b/test/extensions/tracers/lightstep/lightstep_tracer_impl_test.cc @@ -60,7 +60,7 @@ class LightStepDriverTest : public testing::Test { if (init_timer) { timer_ = new NiceMock(&tls_.dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000), _)); } driver_ = std::make_unique(lightstep_config, cm_, stats_, tls_, runtime_, @@ -336,7 +336,7 @@ TEST_F(LightStepDriverTest, FlushSpansTimer) { span->finishSpan(); // Timer should be re-enabled. - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.request_timeout", 5000U)) .WillOnce(Return(5000U)); EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.lightstep.flush_interval_ms", 1000U)) diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index eceef5245fee..b8d18baca67e 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -49,7 +49,7 @@ class ZipkinDriverTest : public testing::Test { if (init_timer) { timer_ = new NiceMock(&tls_.dispatcher_); - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(5000))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(5000), _)); } driver_ = std::make_unique(zipkin_config, cm_, stats_, tls_, runtime_, local_info_, @@ -240,7 +240,7 @@ TEST_F(ZipkinDriverTest, FlushSpansTimer) { span->finishSpan(); // Timer should be re-enabled. - EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(5000))); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(5000), _)); EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.request_timeout", 5000U)) .WillOnce(Return(5000U)); EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.flush_interval_ms", 5000U)) diff --git a/test/mocks/common.h b/test/mocks/common.h index 22f9fcafa422..1cbffa2eeded 100644 --- a/test/mocks/common.h +++ b/test/mocks/common.h @@ -88,6 +88,7 @@ inline bool operator==(const StringViewSaver& saver, const char* str) { } class MockScopedTrackedObject : public ScopeTrackedObject { +public: MOCK_CONST_METHOD2(dumpState, void(std::ostream&, int)); }; diff --git a/test/mocks/event/mocks.cc b/test/mocks/event/mocks.cc index 0cd2f11d33be..af00b66b06c0 100644 --- a/test/mocks/event/mocks.cc +++ b/test/mocks/event/mocks.cc @@ -27,7 +27,11 @@ MockDispatcher::MockDispatcher() { MockDispatcher::~MockDispatcher() = default; MockTimer::MockTimer() { - ON_CALL(*this, enableTimer(_)).WillByDefault(Assign(&enabled_, true)); + ON_CALL(*this, enableTimer(_, _)) + .WillByDefault(Invoke([&](const std::chrono::milliseconds&, const ScopeTrackedObject* scope) { + enabled_ = true; + scope_ = scope; + })); ON_CALL(*this, disableTimer()).WillByDefault(Assign(&enabled_, false)); ON_CALL(*this, enabled()).WillByDefault(ReturnPointee(&enabled_)); } @@ -36,6 +40,7 @@ MockTimer::MockTimer() { // createTimer_(), so to avoid destructing it twice, the MockTimer must have been dynamically // allocated and must not be deleted by it's creator. MockTimer::MockTimer(MockDispatcher* dispatcher) : MockTimer() { + dispatcher_ = dispatcher; EXPECT_CALL(*dispatcher, createTimer_(_)) .WillOnce(DoAll(SaveArg<0>(&callback_), Return(this))) .RetiresOnSaturation(); diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 0e65635e4b08..327214a6903f 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -17,6 +17,8 @@ #include "envoy/network/transport_socket.h" #include "envoy/ssl/context.h" +#include "common/common/scope_tracker.h" + #include "test/mocks/buffer/mocks.h" #include "test/test_common/test_time.h" @@ -131,14 +133,23 @@ class MockTimer : public Timer { void invokeCallback() { EXPECT_TRUE(enabled_); enabled_ = false; + if (scope_ == nullptr) { + callback_(); + return; + } + ScopeTrackerScopeState scope(scope_, *dispatcher_); + scope_ = nullptr; callback_(); } // Timer MOCK_METHOD0(disableTimer, void()); - MOCK_METHOD1(enableTimer, void(const std::chrono::milliseconds&)); + MOCK_METHOD2(enableTimer, + void(const std::chrono::milliseconds&, const ScopeTrackedObject* scope)); MOCK_METHOD0(enabled, bool()); + MockDispatcher* dispatcher_{}; + const ScopeTrackedObject* scope_{}; bool enabled_{}; private: diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 7e4e2e03237c..59d14ffb5e2f 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -601,7 +601,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeout) { return Network::FilterStatus::StopIteration; })); Event::MockTimer* timeout = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000))); + EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000), _)); Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}, true); Stats::Gauge& downstream_pre_cx_active = @@ -649,7 +649,7 @@ TEST_F(ConnectionHandlerTest, ContinueOnListenerFilterTimeout) { return Network::FilterStatus::StopIteration; })); Event::MockTimer* timeout = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000))); + EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000), _)); Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}, true); Stats::Gauge& downstream_pre_cx_active = @@ -698,7 +698,7 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeoutResetOnSuccess) { return Network::FilterStatus::StopIteration; })); Event::MockTimer* timeout = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000))); + EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000), _)); Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}, true); diff --git a/test/server/drain_manager_impl_test.cc b/test/server/drain_manager_impl_test.cc index 58919df1ce7e..0a3076c265c2 100644 --- a/test/server/drain_manager_impl_test.cc +++ b/test/server/drain_manager_impl_test.cc @@ -33,7 +33,7 @@ TEST_F(DrainManagerImplTest, Default) { // Test parent shutdown. Event::MockTimer* shutdown_timer = new Event::MockTimer(&server_.dispatcher_); - EXPECT_CALL(*shutdown_timer, enableTimer(std::chrono::milliseconds(900000))); + EXPECT_CALL(*shutdown_timer, enableTimer(std::chrono::milliseconds(900000), _)); drain_manager.startParentShutdownSequence(); EXPECT_CALL(server_.hot_restart_, sendParentTerminateRequest()); @@ -47,14 +47,14 @@ TEST_F(DrainManagerImplTest, Default) { // Test drain sequence. Event::MockTimer* drain_timer = new Event::MockTimer(&server_.dispatcher_); - EXPECT_CALL(*drain_timer, enableTimer(_)); + EXPECT_CALL(*drain_timer, enableTimer(_, _)); ReadyWatcher drain_complete; drain_manager.startDrainSequence([&drain_complete]() -> void { drain_complete.ready(); }); // 600s which is the default drain time. for (size_t i = 0; i < 599; i++) { if (i < 598) { - EXPECT_CALL(*drain_timer, enableTimer(_)); + EXPECT_CALL(*drain_timer, enableTimer(_, _)); } else { EXPECT_CALL(drain_complete, ready()); } diff --git a/test/test_common/BUILD b/test/test_common/BUILD index c1d4c4f692bf..911ae90a26bd 100644 --- a/test/test_common/BUILD +++ b/test/test_common/BUILD @@ -231,6 +231,7 @@ envoy_cc_test( ":simulated_time_system_lib", ":utility_lib", "//source/common/event:libevent_scheduler_lib", + "//test/mocks/event:event_mocks", ], ) diff --git a/test/test_common/simulated_time_system.cc b/test/test_common/simulated_time_system.cc index c4801ae68bda..43db9e511148 100644 --- a/test/test_common/simulated_time_system.cc +++ b/test/test_common/simulated_time_system.cc @@ -50,8 +50,9 @@ class UnlockGuard { // mechanism used in RealTimeSystem timers is employed for simulated alarms. class SimulatedTimeSystemHelper::Alarm : public Timer { public: - Alarm(SimulatedTimeSystemHelper& time_system, Scheduler& base_scheduler, TimerCb cb) - : base_timer_(base_scheduler.createTimer([this, cb] { runAlarm(cb); })), + Alarm(SimulatedTimeSystemHelper& time_system, Scheduler& base_scheduler, TimerCb cb, + Dispatcher& dispatcher) + : base_timer_(base_scheduler.createTimer([this, cb] { runAlarm(cb); }, dispatcher)), time_system_(time_system), index_(time_system.nextIndex()), armed_(false), pending_(false) { } @@ -59,7 +60,8 @@ class SimulatedTimeSystemHelper::Alarm : public Timer { // Timer void disableTimer() override; - void enableTimer(const std::chrono::milliseconds& duration) override; + void enableTimer(const std::chrono::milliseconds& duration, + const ScopeTrackedObject* scope) override; bool enabled() override { Thread::LockGuard lock(time_system_.mutex_); return armed_ || base_timer_->enabled(); @@ -75,7 +77,8 @@ class SimulatedTimeSystemHelper::Alarm : public Timer { * Activates the timer so it will be run the next time the libevent loop is run, * typically via Dispatcher::run(). */ - void activateLockHeld() EXCLUSIVE_LOCKS_REQUIRED(time_system_.mutex_) { + void activateLockHeld(const ScopeTrackedObject* scope = nullptr) + EXCLUSIVE_LOCKS_REQUIRED(time_system_.mutex_) { ASSERT(armed_); armed_ = false; if (pending_) { @@ -91,7 +94,7 @@ class SimulatedTimeSystemHelper::Alarm : public Timer { // time_system_.mutex_ prior to running libevent, which may delete this. UnlockGuard unlocker(time_system_.mutex_); std::chrono::milliseconds duration = std::chrono::milliseconds::zero(); - base_timer_->enableTimer(duration); + base_timer_->enableTimer(duration, scope); } MonotonicTime time() const EXCLUSIVE_LOCKS_REQUIRED(time_system_.mutex_) { @@ -146,8 +149,9 @@ class SimulatedTimeSystemHelper::SimulatedScheduler : public Scheduler { public: SimulatedScheduler(SimulatedTimeSystemHelper& time_system, Scheduler& base_scheduler) : time_system_(time_system), base_scheduler_(base_scheduler) {} - TimerPtr createTimer(const TimerCb& cb) override { - return std::make_unique(time_system_, base_scheduler_, cb); + TimerPtr createTimer(const TimerCb& cb, Dispatcher& dispatcher) override { + return std::make_unique(time_system_, base_scheduler_, cb, + dispatcher); }; private: @@ -173,13 +177,13 @@ void SimulatedTimeSystemHelper::Alarm::Alarm::disableTimerLockHeld() { } } -void SimulatedTimeSystemHelper::Alarm::Alarm::enableTimer( - const std::chrono::milliseconds& duration) { +void SimulatedTimeSystemHelper::Alarm::Alarm::enableTimer(const std::chrono::milliseconds& duration, + const ScopeTrackedObject* scope) { Thread::LockGuard lock(time_system_.mutex_); disableTimerLockHeld(); armed_ = true; if (duration.count() == 0) { - activateLockHeld(); + activateLockHeld(scope); } else { time_system_.addAlarmLockHeld(this, duration); } diff --git a/test/test_common/simulated_time_system_test.cc b/test/test_common/simulated_time_system_test.cc index 399ee31a87b1..f584c29401d5 100644 --- a/test/test_common/simulated_time_system_test.cc +++ b/test/test_common/simulated_time_system_test.cc @@ -3,6 +3,7 @@ #include "common/event/libevent_scheduler.h" #include "common/event/timer_impl.h" +#include "test/mocks/event/mocks.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -23,10 +24,12 @@ class SimulatedTimeSystemTest : public testing::Test { void addTask(int64_t delay_ms, char marker) { std::chrono::milliseconds delay(delay_ms); - TimerPtr timer = scheduler_->createTimer([this, marker, delay]() { - output_.append(1, marker); - EXPECT_GE(time_system_.monotonicTime(), start_monotonic_time_ + delay); - }); + TimerPtr timer = scheduler_->createTimer( + [this, marker, delay]() { + output_.append(1, marker); + EXPECT_GE(time_system_.monotonicTime(), start_monotonic_time_ + delay); + }, + dispatcher_); timer->enableTimer(delay); timers_.push_back(std::move(timer)); } @@ -41,6 +44,7 @@ class SimulatedTimeSystemTest : public testing::Test { base_scheduler_.run(Dispatcher::RunType::NonBlock); } + testing::NiceMock dispatcher_; LibeventScheduler base_scheduler_; SimulatedTimeSystem time_system_; SchedulerPtr scheduler_; @@ -71,11 +75,13 @@ TEST_F(SimulatedTimeSystemTest, WaitFor) { }); Thread::CondVar condvar; Thread::MutexBasicLockable mutex; - TimerPtr timer = scheduler_->createTimer([&condvar, &mutex, &done]() { - Thread::LockGuard lock(mutex); - done = true; - condvar.notifyOne(); - }); + TimerPtr timer = scheduler_->createTimer( + [&condvar, &mutex, &done]() { + Thread::LockGuard lock(mutex); + done = true; + condvar.notifyOne(); + }, + dispatcher_); timer->enableTimer(std::chrono::seconds(60)); // Wait 50 simulated seconds of simulated time, which won't be enough to @@ -201,7 +207,7 @@ TEST_F(SimulatedTimeSystemTest, DeleteTime) { TEST_F(SimulatedTimeSystemTest, DuplicateTimer) { // Set one alarm two times to test that pending does not get duplicated.. std::chrono::milliseconds delay(0); - TimerPtr zero_timer = scheduler_->createTimer([this]() { output_.append(1, '2'); }); + TimerPtr zero_timer = scheduler_->createTimer([this]() { output_.append(1, '2'); }, dispatcher_); zero_timer->enableTimer(delay); zero_timer->enableTimer(delay); sleepMsAndLoop(1); @@ -216,11 +222,13 @@ TEST_F(SimulatedTimeSystemTest, DuplicateTimer) { }); Thread::CondVar condvar; Thread::MutexBasicLockable mutex; - TimerPtr timer = scheduler_->createTimer([&condvar, &mutex, &done]() { - Thread::LockGuard lock(mutex); - done = true; - condvar.notifyOne(); - }); + TimerPtr timer = scheduler_->createTimer( + [&condvar, &mutex, &done]() { + Thread::LockGuard lock(mutex); + done = true; + condvar.notifyOne(); + }, + dispatcher_); timer->enableTimer(std::chrono::seconds(10)); { @@ -234,7 +242,7 @@ TEST_F(SimulatedTimeSystemTest, DuplicateTimer) { } TEST_F(SimulatedTimeSystemTest, Enabled) { - TimerPtr timer = scheduler_->createTimer({}); + TimerPtr timer = scheduler_->createTimer({}, dispatcher_); timer->enableTimer(std::chrono::milliseconds(0)); EXPECT_TRUE(timer->enabled()); } From d4dc0a5b74acd42aafaaef8bb9c4ebed832674bd Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Thu, 22 Aug 2019 23:46:51 +0530 Subject: [PATCH 427/542] ext authz: add dns san support for ext authz service (#7948) Adds support for DNS SAN in ext authz peer validation Risk Level: Low Testing: Added Docs Changes: Added Release Notes: N/A Signed-off-by: Rama Chavali --- .../service/auth/v2/attribute_context.proto | 4 +- .../common/ext_authz/check_request_utils.cc | 31 ++++-- .../ext_authz/check_request_utils_test.cc | 104 ++++++++++++------ 3 files changed, 90 insertions(+), 49 deletions(-) diff --git a/api/envoy/service/auth/v2/attribute_context.proto b/api/envoy/service/auth/v2/attribute_context.proto index 822e361dd81b..fe5bee033909 100644 --- a/api/envoy/service/auth/v2/attribute_context.proto +++ b/api/envoy/service/auth/v2/attribute_context.proto @@ -50,8 +50,8 @@ message AttributeContext { // The authenticated identity of this peer. // For example, the identity associated with the workload such as a service account. // If an X.509 certificate is used to assert the identity this field should be sourced from - // `Subject` or `Subject Alternative Names`. The primary identity should be the principal. - // The principal format is issuer specific. + // `URI Subject Alternative Names`, `DNS Subject Alternate Names` or `Subject` in that order. + // The primary identity should be the principal. The principal format is issuer specific. // // Example: // * SPIFFE format is `spiffe://trust-domain/path` diff --git a/source/extensions/filters/common/ext_authz/check_request_utils.cc b/source/extensions/filters/common/ext_authz/check_request_utils.cc index b98752cc5a14..707053f46bf0 100644 --- a/source/extensions/filters/common/ext_authz/check_request_utils.cc +++ b/source/extensions/filters/common/ext_authz/check_request_utils.cc @@ -37,24 +37,33 @@ void CheckRequestUtils::setAttrContextPeer(envoy::service::auth::v2::AttributeCo Envoy::Network::Utility::addressToProtobufAddress(*connection.remoteAddress(), *addr); } - // Set the principal - // Preferably the SAN from the peer's cert or - // Subject from the peer's cert. + // Set the principal. Preferably the URI SAN, DNS SAN or Subject in that order from the peer's + // cert. Ssl::ConnectionInfo* ssl = const_cast(connection.ssl()); if (ssl != nullptr) { if (local) { - const auto uriSans = ssl->uriSanLocalCertificate(); - if (uriSans.empty()) { - peer.set_principal(ssl->subjectLocalCertificate()); + const auto uri_sans = ssl->uriSanLocalCertificate(); + if (uri_sans.empty()) { + const auto dns_sans = ssl->dnsSansLocalCertificate(); + if (dns_sans.empty()) { + peer.set_principal(ssl->subjectLocalCertificate()); + } else { + peer.set_principal(dns_sans[0]); + } } else { - peer.set_principal(uriSans[0]); + peer.set_principal(uri_sans[0]); } } else { - const auto uriSans = ssl->uriSanPeerCertificate(); - if (uriSans.empty()) { - peer.set_principal(ssl->subjectPeerCertificate()); + const auto uri_sans = ssl->uriSanPeerCertificate(); + if (uri_sans.empty()) { + const auto dns_sans = ssl->dnsSansPeerCertificate(); + if (dns_sans.empty()) { + peer.set_principal(ssl->subjectPeerCertificate()); + } else { + peer.set_principal(dns_sans[0]); + } } else { - peer.set_principal(uriSans[0]); + peer.set_principal(uri_sans[0]); } } } diff --git a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc index fa925bf6b8bf..bc5df7ec323d 100644 --- a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc +++ b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc @@ -30,7 +30,7 @@ class CheckRequestUtilsTest : public testing::Test { buffer_ = CheckRequestUtilsTest::newTestBuffer(8192); }; - void ExpectBasicHttp() { + void expectBasicHttp() { EXPECT_CALL(callbacks_, connection()).Times(2).WillRepeatedly(Return(&connection_)); EXPECT_CALL(connection_, remoteAddress()).WillOnce(ReturnRef(addr_)); EXPECT_CALL(connection_, localAddress()).WillOnce(ReturnRef(addr_)); @@ -41,6 +41,33 @@ class CheckRequestUtilsTest : public testing::Test { EXPECT_CALL(req_info_, protocol()).Times(2).WillRepeatedly(ReturnPointee(&protocol_)); } + void callHttpCheckAndValidateRequestAttributes() { + Http::TestHeaderMapImpl request_headers{{"x-envoy-downstream-service-cluster", "foo"}, + {":path", "/bar"}}; + envoy::service::auth::v2::CheckRequest request; + Protobuf::Map context_extensions; + context_extensions["key"] = "value"; + + envoy::api::v2::core::Metadata metadata_context; + auto metadata_val = MessageUtil::keyValueStruct("foo", "bar"); + (*metadata_context.mutable_filter_metadata())["meta.key"] = metadata_val; + + CheckRequestUtils::createHttpCheck(&callbacks_, request_headers, std::move(context_extensions), + std::move(metadata_context), request, false); + + EXPECT_EQ("source", request.attributes().source().principal()); + EXPECT_EQ("destination", request.attributes().destination().principal()); + EXPECT_EQ("foo", request.attributes().source().service()); + EXPECT_EQ("value", request.attributes().context_extensions().at("key")); + EXPECT_EQ("bar", request.attributes() + .metadata_context() + .filter_metadata() + .at("meta.key") + .fields() + .at("foo") + .string_value()); + } + static Buffer::InstancePtr newTestBuffer(uint64_t size) { auto buffer = std::make_unique(); while (buffer->length() < size) { @@ -84,7 +111,7 @@ TEST_F(CheckRequestUtilsTest, BasicHttp) { // A client supplied EnvoyAuthPartialBody header should be ignored. Http::TestHeaderMapImpl request_headers{{Http::Headers::get().EnvoyAuthPartialBody.get(), "1"}}; - ExpectBasicHttp(); + expectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, request_headers, Protobuf::Map(), envoy::api::v2::core::Metadata(), request_, size); @@ -101,7 +128,7 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithPartialBody) { Http::HeaderMapImpl headers_; envoy::service::auth::v2::CheckRequest request_; - ExpectBasicHttp(); + expectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, headers_, Protobuf::Map(), envoy::api::v2::core::Metadata(), request_, size); @@ -116,7 +143,7 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithFullBody) { Http::HeaderMapImpl headers_; envoy::service::auth::v2::CheckRequest request_; - ExpectBasicHttp(); + expectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, headers_, Protobuf::Map(), envoy::api::v2::core::Metadata(), request_, buffer_->length()); @@ -127,45 +154,50 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithFullBody) { Http::Headers::get().EnvoyAuthPartialBody.get())); } -// Verify that createHttpCheck extract the proper attributes from the http request into CheckRequest -// proto object. -TEST_F(CheckRequestUtilsTest, CheckAttrContextPeer) { - Http::TestHeaderMapImpl request_headers{{"x-envoy-downstream-service-cluster", "foo"}, - {":path", "/bar"}}; - envoy::service::auth::v2::CheckRequest request; - EXPECT_CALL(callbacks_, connection()).WillRepeatedly(Return(&connection_)); - EXPECT_CALL(connection_, remoteAddress()).WillRepeatedly(ReturnRef(addr_)); - EXPECT_CALL(connection_, localAddress()).WillRepeatedly(ReturnRef(addr_)); - EXPECT_CALL(Const(connection_), ssl()).WillRepeatedly(Return(&ssl_)); - EXPECT_CALL(callbacks_, streamId()).WillRepeatedly(Return(0)); - EXPECT_CALL(callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); - EXPECT_CALL(callbacks_, decodingBuffer()).Times(1); - EXPECT_CALL(req_info_, protocol()).WillRepeatedly(ReturnPointee(&protocol_)); +// Verify that createHttpCheck extract the attributes from the HTTP request into CheckRequest +// proto object and URI SAN is used as principal if present. +TEST_F(CheckRequestUtilsTest, CheckAttrContextPeerUriSans) { + expectBasicHttp(); + EXPECT_CALL(ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{"source"})); EXPECT_CALL(ssl_, uriSanLocalCertificate()) .WillOnce(Return(std::vector{"destination"})); + callHttpCheckAndValidateRequestAttributes(); +} + +// Verify that createHttpCheck extract the attributes from the HTTP request into CheckRequest +// proto object and DNS SAN is used as principal if URI SAN is absent. +TEST_F(CheckRequestUtilsTest, CheckAttrContextPeerDnsSans) { + expectBasicHttp(); + + EXPECT_CALL(ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(ssl_, dnsSansPeerCertificate()).WillOnce(Return(std::vector{"source"})); + + EXPECT_CALL(ssl_, uriSanLocalCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(ssl_, dnsSansLocalCertificate()) + .WillOnce(Return(std::vector{"destination"})); + Protobuf::Map context_extensions; context_extensions["key"] = "value"; - envoy::api::v2::core::Metadata metadata_context; - auto metadata_val = MessageUtil::keyValueStruct("foo", "bar"); - (*metadata_context.mutable_filter_metadata())["meta.key"] = metadata_val; - - CheckRequestUtils::createHttpCheck(&callbacks_, request_headers, std::move(context_extensions), - std::move(metadata_context), request, false); - - EXPECT_EQ("source", request.attributes().source().principal()); - EXPECT_EQ("destination", request.attributes().destination().principal()); - EXPECT_EQ("foo", request.attributes().source().service()); - EXPECT_EQ("value", request.attributes().context_extensions().at("key")); - EXPECT_EQ("bar", request.attributes() - .metadata_context() - .filter_metadata() - .at("meta.key") - .fields() - .at("foo") - .string_value()); + callHttpCheckAndValidateRequestAttributes(); +} + +// Verify that createHttpCheck extract the attributes from the HTTP request into CheckRequest +// proto object and Subject is used as principal if both URI SAN and DNS SAN are absent. +TEST_F(CheckRequestUtilsTest, CheckAttrContextSubject) { + expectBasicHttp(); + + EXPECT_CALL(ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(ssl_, dnsSansPeerCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(ssl_, subjectPeerCertificate()).WillOnce(Return("source")); + + EXPECT_CALL(ssl_, uriSanLocalCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(ssl_, dnsSansLocalCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(ssl_, subjectLocalCertificate()).WillOnce(Return("destination")); + + callHttpCheckAndValidateRequestAttributes(); } } // namespace From e1ecb027cb3e17ba9d041b7eaf68648128c8cfcd Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 22 Aug 2019 11:42:31 -0700 Subject: [PATCH 428/542] accesslog: don't open log file with read flag (#7998) Description: File access log shouldn't need read access for a file. Risk Level: Low Testing: local in mac, CI Docs Changes: Release Notes: Fixes #7997 Signed-off-by: Lizan Zhou --- .../access_log/access_log_manager_impl.cc | 6 +-- .../access_log_manager_impl_test.cc | 39 +++++++++++++------ test/extensions/access_loggers/file/BUILD | 1 + test/mocks/filesystem/mocks.cc | 4 +- test/mocks/filesystem/mocks.h | 3 +- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/source/common/access_log/access_log_manager_impl.cc b/source/common/access_log/access_log_manager_impl.cc index dd2615421db1..cf5a98daaf47 100644 --- a/source/common/access_log/access_log_manager_impl.cc +++ b/source/common/access_log/access_log_manager_impl.cc @@ -42,9 +42,9 @@ AccessLogFileImpl::AccessLogFileImpl(Filesystem::FilePtr&& file, Event::Dispatch } Filesystem::FlagSet AccessLogFileImpl::defaultFlags() { - static constexpr Filesystem::FlagSet default_flags{ - 1 << Filesystem::File::Operation::Read | 1 << Filesystem::File::Operation::Write | - 1 << Filesystem::File::Operation::Create | 1 << Filesystem::File::Operation::Append}; + static constexpr Filesystem::FlagSet default_flags{1 << Filesystem::File::Operation::Write | + 1 << Filesystem::File::Operation::Create | + 1 << Filesystem::File::Operation::Append}; return default_flags; } diff --git a/test/common/access_log/access_log_manager_impl_test.cc b/test/common/access_log/access_log_manager_impl_test.cc index e7bf863df009..d8d9446d856f 100644 --- a/test/common/access_log/access_log_manager_impl_test.cc +++ b/test/common/access_log/access_log_manager_impl_test.cc @@ -14,6 +14,7 @@ using testing::_; using testing::ByMove; +using testing::Invoke; using testing::NiceMock; using testing::Return; using testing::ReturnNew; @@ -49,14 +50,28 @@ class AccessLogManagerImplTest : public testing::Test { TEST_F(AccessLogManagerImplTest, BadFile) { EXPECT_CALL(dispatcher_, createTimer_(_)); - EXPECT_CALL(*file_, open_()).WillOnce(Return(ByMove(Filesystem::resultFailure(false, 0)))); + EXPECT_CALL(*file_, open_(_)).WillOnce(Return(ByMove(Filesystem::resultFailure(false, 0)))); EXPECT_THROW(access_log_manager_.createAccessLog("foo"), EnvoyException); } +TEST_F(AccessLogManagerImplTest, OpenFileWithRightFlags) { + EXPECT_CALL(dispatcher_, createTimer_(_)); + EXPECT_CALL(*file_, open_(_)) + .WillOnce(Invoke([](Filesystem::FlagSet flags) -> Api::IoCallBoolResult { + EXPECT_FALSE(flags[Filesystem::File::Operation::Read]); + EXPECT_TRUE(flags[Filesystem::File::Operation::Write]); + EXPECT_TRUE(flags[Filesystem::File::Operation::Append]); + EXPECT_TRUE(flags[Filesystem::File::Operation::Create]); + return Filesystem::resultSuccess(true); + })); + EXPECT_NE(nullptr, access_log_manager_.createAccessLog("foo")); + EXPECT_CALL(*file_, close_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); +} + TEST_F(AccessLogManagerImplTest, flushToLogFilePeriodically) { NiceMock* timer = new NiceMock(&dispatcher_); - EXPECT_CALL(*file_, open_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); + EXPECT_CALL(*file_, open_(_)).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); @@ -98,7 +113,7 @@ TEST_F(AccessLogManagerImplTest, flushToLogFilePeriodically) { TEST_F(AccessLogManagerImplTest, flushToLogFileOnDemand) { NiceMock* timer = new NiceMock(&dispatcher_); - EXPECT_CALL(*file_, open_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); + EXPECT_CALL(*file_, open_(_)).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); @@ -163,7 +178,7 @@ TEST_F(AccessLogManagerImplTest, reopenFile) { NiceMock* timer = new NiceMock(&dispatcher_); Sequence sq; - EXPECT_CALL(*file_, open_()) + EXPECT_CALL(*file_, open_(_)) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); @@ -188,7 +203,7 @@ TEST_F(AccessLogManagerImplTest, reopenFile) { EXPECT_CALL(*file_, close_()) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); - EXPECT_CALL(*file_, open_()) + EXPECT_CALL(*file_, open_(_)) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); @@ -224,7 +239,7 @@ TEST_F(AccessLogManagerImplTest, reopenThrows) { })); Sequence sq; - EXPECT_CALL(*file_, open_()) + EXPECT_CALL(*file_, open_(_)) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); @@ -232,7 +247,7 @@ TEST_F(AccessLogManagerImplTest, reopenThrows) { EXPECT_CALL(*file_, close_()) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); - EXPECT_CALL(*file_, open_()) + EXPECT_CALL(*file_, open_(_)) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultFailure(false, 0)))); @@ -262,7 +277,7 @@ TEST_F(AccessLogManagerImplTest, reopenThrows) { } TEST_F(AccessLogManagerImplTest, bigDataChunkShouldBeFlushedWithoutTimer) { - EXPECT_CALL(*file_, open_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); + EXPECT_CALL(*file_, open_(_)).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); EXPECT_CALL(*file_, write_(_)) @@ -305,7 +320,7 @@ TEST_F(AccessLogManagerImplTest, reopenAllFiles) { EXPECT_CALL(dispatcher_, createTimer_(_)).WillRepeatedly(ReturnNew>()); Sequence sq; - EXPECT_CALL(*file_, open_()) + EXPECT_CALL(*file_, open_(_)) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log = access_log_manager_.createAccessLog("foo"); @@ -315,7 +330,7 @@ TEST_F(AccessLogManagerImplTest, reopenAllFiles) { .WillOnce(Return(ByMove(std::unique_ptr>(file2)))); Sequence sq2; - EXPECT_CALL(*file2, open_()) + EXPECT_CALL(*file2, open_(_)) .InSequence(sq2) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log2 = access_log_manager_.createAccessLog("bar"); @@ -342,10 +357,10 @@ TEST_F(AccessLogManagerImplTest, reopenAllFiles) { .InSequence(sq2) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); - EXPECT_CALL(*file_, open_()) + EXPECT_CALL(*file_, open_(_)) .InSequence(sq) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); - EXPECT_CALL(*file2, open_()) + EXPECT_CALL(*file2, open_(_)) .InSequence(sq2) .WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); diff --git a/test/extensions/access_loggers/file/BUILD b/test/extensions/access_loggers/file/BUILD index 3feb9c7509a6..461c4d5bdaea 100644 --- a/test/extensions/access_loggers/file/BUILD +++ b/test/extensions/access_loggers/file/BUILD @@ -18,5 +18,6 @@ envoy_extension_cc_test( deps = [ "//source/extensions/access_loggers/file:config", "//test/mocks/server:server_mocks", + "//test/test_common:environment_lib", ], ) diff --git a/test/mocks/filesystem/mocks.cc b/test/mocks/filesystem/mocks.cc index 7cf733315030..af5d9fcddc7c 100644 --- a/test/mocks/filesystem/mocks.cc +++ b/test/mocks/filesystem/mocks.cc @@ -9,10 +9,10 @@ namespace Filesystem { MockFile::MockFile() : num_opens_(0), num_writes_(0), is_open_(false) {} MockFile::~MockFile() = default; -Api::IoCallBoolResult MockFile::open(FlagSet) { +Api::IoCallBoolResult MockFile::open(FlagSet flag) { Thread::LockGuard lock(open_mutex_); - Api::IoCallBoolResult result = open_(); + Api::IoCallBoolResult result = open_(flag); is_open_ = result.rc_; num_opens_++; open_event_.notifyOne(); diff --git a/test/mocks/filesystem/mocks.h b/test/mocks/filesystem/mocks.h index 459cdfd8ce65..c4ae006a83ad 100644 --- a/test/mocks/filesystem/mocks.h +++ b/test/mocks/filesystem/mocks.h @@ -25,7 +25,8 @@ class MockFile : public File { bool isOpen() const override { return is_open_; }; MOCK_CONST_METHOD0(path, std::string()); - MOCK_METHOD0(open_, Api::IoCallBoolResult()); + // The first parameter here must be `const FlagSet&` otherwise it doesn't compile with libstdc++ + MOCK_METHOD1(open_, Api::IoCallBoolResult(const FlagSet& flag)); MOCK_METHOD1(write_, Api::IoCallSizeResult(absl::string_view buffer)); MOCK_METHOD0(close_, Api::IoCallBoolResult()); From 43c4acd27b85e4e28beeff9858aca7a110ce5863 Mon Sep 17 00:00:00 2001 From: htuch Date: Thu, 22 Aug 2019 14:53:23 -0400 Subject: [PATCH 429/542] protobuf: towards unifying PGV, deprecated and unknown field validation. (#8002) This is part of #7980; basically, we want to leverage the recursive pass that already exists for the deprecated check. This PR does not implement the recursive behavior yet for unknown fields though, because there is a ton of churn, so this PR just has the mechanical bits. We switch plumbing of validation visitor into places such as anyConvert() and instead pass this to MessageUtil::validate. There are a bunch of future followups planned in additional PRs: * Combine the recursive pass for unknown/deprecated check in MessageUtil::validate(). * Add mitigation for #5965 by copying to a temporary before recursive expansion. * [Future] consider moving deprecated reporting into a message validation visitor handler. Risk level: Low Testing: Some new //test/common/protobuf::utility_test unit test. Signed-off-by: Harvey Tuch --- include/envoy/server/BUILD | 1 + include/envoy/server/filter_config.h | 5 ++- .../envoy/server/resource_monitor_config.h | 7 +++++ include/envoy/upstream/retry.h | 6 ++-- source/common/access_log/access_log_impl.cc | 31 +++++++++++-------- source/common/access_log/access_log_impl.h | 12 ++++--- source/common/protobuf/utility.h | 19 +++++++----- source/common/router/config_impl.cc | 6 ++-- source/common/router/config_impl.h | 1 + source/common/router/rds_impl.cc | 5 ++- source/common/router/rds_impl.h | 4 +-- .../route_config_update_receiver_impl.cc | 5 ++- source/common/router/scoped_rds.cc | 6 ++-- source/common/router/scoped_rds.h | 4 +-- source/common/router/vhds.cc | 3 +- source/common/router/vhds.h | 5 +-- source/common/runtime/runtime_impl.cc | 5 ++- source/common/runtime/runtime_impl.h | 4 +-- source/common/secret/sds_api.cc | 5 ++- source/common/secret/sds_api.h | 3 +- source/common/upstream/cds_api_impl.cc | 8 ++--- source/common/upstream/cds_api_impl.h | 2 +- source/common/upstream/cluster_factory_impl.h | 3 +- source/common/upstream/eds.cc | 6 ++-- source/common/upstream/eds.h | 4 +-- source/common/upstream/upstream_impl.cc | 2 +- .../extensions/access_loggers/file/config.cc | 3 +- .../access_loggers/grpc/http_config.cc | 3 +- .../filters/http/common/factory_base.h | 9 ++++-- .../original_src_config_factory.cc | 5 +-- .../filters/network/common/factory_base.h | 12 ++++--- .../dubbo_proxy/filters/factory_base.h | 5 +-- .../thrift_proxy/filters/factory_base.h | 5 +-- .../grpc_credentials/aws_iam/config.cc | 5 ++- .../file_based_metadata/config.cc | 7 ++--- .../health_checkers/redis/utility.h | 2 +- .../resource_monitors/common/factory_base.h | 5 +-- .../priority/previous_priorities/config.cc | 10 +++--- .../priority/previous_priorities/config.h | 6 ++-- .../stat_sinks/dog_statsd/config.cc | 3 +- .../extensions/stat_sinks/hystrix/config.cc | 3 +- .../stat_sinks/metrics_service/config.cc | 2 +- source/extensions/stat_sinks/statsd/config.cc | 3 +- .../extensions/tracers/common/factory_base.h | 6 ++-- .../transport_sockets/alts/config.cc | 2 +- .../transport_sockets/tap/config.cc | 4 +-- .../transport_sockets/tls/config.cc | 6 ++-- source/server/lds_api.cc | 8 ++--- source/server/lds_api.h | 2 +- source/server/overload_manager_impl.cc | 2 +- source/server/resource_monitor_config_impl.h | 10 ++++-- source/server/server.cc | 2 +- .../common/access_log/access_log_impl_test.cc | 2 +- test/common/http/codec_impl_fuzz_test.cc | 2 +- test/common/protobuf/utility_test.cc | 30 +++++++++--------- test/common/router/config_impl_test.cc | 2 +- test/common/router/header_formatter_test.cc | 4 +-- test/common/router/header_parser_fuzz_test.cc | 2 +- test/common/router/rds_impl_test.cc | 10 +++--- test/common/router/route_fuzz_test.cc | 2 +- test/common/router/router_ratelimit_test.cc | 2 +- test/common/router/scoped_rds_test.cc | 8 ++--- .../upstream/health_checker_impl_test.cc | 8 ++--- test/common/upstream/upstream_impl_test.cc | 6 ++-- .../clusters/redis/redis_cluster_test.cc | 2 +- .../filters/http/ext_authz/config_test.cc | 2 ++ .../filters/http/ext_authz/ext_authz_test.cc | 2 +- .../original_src_config_factory_test.cc | 3 +- .../original_src_config_factory_test.cc | 3 +- .../network/dubbo_proxy/conn_manager_test.cc | 2 +- .../network/dubbo_proxy/route_matcher_test.cc | 4 +-- .../network/ext_authz/ext_authz_test.cc | 2 +- .../network/thrift_proxy/conn_manager_test.cc | 2 +- .../filters/network/thrift_proxy/mocks.cc | 3 +- .../thrift_proxy/route_matcher_test.cc | 2 +- .../fixed_heap/config_test.cc | 3 +- .../injected_resource/config_test.cc | 3 +- .../injected_resource_monitor_test.cc | 3 +- .../previous_priorities/config_test.cc | 3 +- .../transport_sockets/alts/config_test.cc | 4 +-- .../tls/context_impl_test.cc | 2 +- test/mocks/upstream/mocks.h | 4 ++- test/test_common/utility.h | 13 ++++++-- test/tools/router_check/router.cc | 2 +- test/tools/schema_validator/validator.cc | 4 +-- 85 files changed, 250 insertions(+), 188 deletions(-) diff --git a/include/envoy/server/BUILD b/include/envoy/server/BUILD index 2dbd768aab91..591868dde496 100644 --- a/include/envoy/server/BUILD +++ b/include/envoy/server/BUILD @@ -234,6 +234,7 @@ envoy_cc_library( ":resource_monitor_interface", "//include/envoy/api:api_interface", "//include/envoy/event:dispatcher_interface", + "//include/envoy/protobuf:message_validator_interface", ], ) diff --git a/include/envoy/server/filter_config.h b/include/envoy/server/filter_config.h index 8cd1b4058353..4864d5bf9da6 100644 --- a/include/envoy/server/filter_config.h +++ b/include/envoy/server/filter_config.h @@ -269,11 +269,14 @@ class ProtocolOptionsFactory { * implementation is unable to produce a factory with the provided parameters, it should throw an * EnvoyException. * @param config supplies the protobuf configuration for the filter + * @param validation_visitor message validation visitor instance. * @return Upstream::ProtocolOptionsConfigConstSharedPtr the protocol options */ virtual Upstream::ProtocolOptionsConfigConstSharedPtr - createProtocolOptionsConfig(const Protobuf::Message& config) { + createProtocolOptionsConfig(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor& validation_visitor) { UNREFERENCED_PARAMETER(config); + UNREFERENCED_PARAMETER(validation_visitor); return nullptr; } diff --git a/include/envoy/server/resource_monitor_config.h b/include/envoy/server/resource_monitor_config.h index aef576b8e4bf..db7cc786a11e 100644 --- a/include/envoy/server/resource_monitor_config.h +++ b/include/envoy/server/resource_monitor_config.h @@ -3,6 +3,7 @@ #include "envoy/api/api.h" #include "envoy/common/pure.h" #include "envoy/event/dispatcher.h" +#include "envoy/protobuf/message_validator.h" #include "envoy/server/resource_monitor.h" #include "common/protobuf/protobuf.h" @@ -25,6 +26,12 @@ class ResourceMonitorFactoryContext { * @return reference to the Api object */ virtual Api::Api& api() PURE; + + /** + * @return ProtobufMessage::ValidationVisitor& validation visitor for filter configuration + * messages. + */ + virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; }; /** diff --git a/include/envoy/upstream/retry.h b/include/envoy/upstream/retry.h index dd518eb13ea5..82928f661f39 100644 --- a/include/envoy/upstream/retry.h +++ b/include/envoy/upstream/retry.h @@ -79,8 +79,10 @@ class RetryPriorityFactory { public: virtual ~RetryPriorityFactory() = default; - virtual RetryPrioritySharedPtr createRetryPriority(const Protobuf::Message& config, - uint32_t retry_count) PURE; + virtual RetryPrioritySharedPtr + createRetryPriority(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor& validation_visitor, + uint32_t retry_count) PURE; virtual std::string name() const PURE; diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc index f59c96d1b66e..498c3e18176d 100644 --- a/source/common/access_log/access_log_impl.cc +++ b/source/common/access_log/access_log_impl.cc @@ -53,7 +53,8 @@ bool ComparisonFilter::compareAgainstValue(uint64_t lhs) { FilterPtr FilterFactory::fromProto(const envoy::config::filter::accesslog::v2::AccessLogFilter& config, - Runtime::Loader& runtime, Runtime::RandomGenerator& random) { + Runtime::Loader& runtime, Runtime::RandomGenerator& random, + ProtobufMessage::ValidationVisitor& validation_visitor) { switch (config.filter_specifier_case()) { case envoy::config::filter::accesslog::v2::AccessLogFilter::kStatusCodeFilter: return FilterPtr{new StatusCodeFilter(config.status_code_filter(), runtime)}; @@ -66,19 +67,19 @@ FilterFactory::fromProto(const envoy::config::filter::accesslog::v2::AccessLogFi case envoy::config::filter::accesslog::v2::AccessLogFilter::kRuntimeFilter: return FilterPtr{new RuntimeFilter(config.runtime_filter(), runtime, random)}; case envoy::config::filter::accesslog::v2::AccessLogFilter::kAndFilter: - return FilterPtr{new AndFilter(config.and_filter(), runtime, random)}; + return FilterPtr{new AndFilter(config.and_filter(), runtime, random, validation_visitor)}; case envoy::config::filter::accesslog::v2::AccessLogFilter::kOrFilter: - return FilterPtr{new OrFilter(config.or_filter(), runtime, random)}; + return FilterPtr{new OrFilter(config.or_filter(), runtime, random, validation_visitor)}; case envoy::config::filter::accesslog::v2::AccessLogFilter::kHeaderFilter: return FilterPtr{new HeaderFilter(config.header_filter())}; case envoy::config::filter::accesslog::v2::AccessLogFilter::kResponseFlagFilter: - MessageUtil::validate(config); + MessageUtil::validate(config, validation_visitor); return FilterPtr{new ResponseFlagFilter(config.response_flag_filter())}; case envoy::config::filter::accesslog::v2::AccessLogFilter::kGrpcStatusFilter: - MessageUtil::validate(config); + MessageUtil::validate(config, validation_visitor); return FilterPtr{new GrpcStatusFilter(config.grpc_status_filter())}; case envoy::config::filter::accesslog::v2::AccessLogFilter::kExtensionFilter: - MessageUtil::validate(config); + MessageUtil::validate(config, validation_visitor); { auto& factory = Config::Utility::getAndCheckFactory( config.extension_filter().name()); @@ -140,19 +141,22 @@ bool RuntimeFilter::evaluate(const StreamInfo::StreamInfo&, const Http::HeaderMa OperatorFilter::OperatorFilter(const Protobuf::RepeatedPtrField< envoy::config::filter::accesslog::v2::AccessLogFilter>& configs, - Runtime::Loader& runtime, Runtime::RandomGenerator& random) { + Runtime::Loader& runtime, Runtime::RandomGenerator& random, + ProtobufMessage::ValidationVisitor& validation_visitor) { for (const auto& config : configs) { - filters_.emplace_back(FilterFactory::fromProto(config, runtime, random)); + filters_.emplace_back(FilterFactory::fromProto(config, runtime, random, validation_visitor)); } } OrFilter::OrFilter(const envoy::config::filter::accesslog::v2::OrFilter& config, - Runtime::Loader& runtime, Runtime::RandomGenerator& random) - : OperatorFilter(config.filters(), runtime, random) {} + Runtime::Loader& runtime, Runtime::RandomGenerator& random, + ProtobufMessage::ValidationVisitor& validation_visitor) + : OperatorFilter(config.filters(), runtime, random, validation_visitor) {} AndFilter::AndFilter(const envoy::config::filter::accesslog::v2::AndFilter& config, - Runtime::Loader& runtime, Runtime::RandomGenerator& random) - : OperatorFilter(config.filters(), runtime, random) {} + Runtime::Loader& runtime, Runtime::RandomGenerator& random, + ProtobufMessage::ValidationVisitor& validation_visitor) + : OperatorFilter(config.filters(), runtime, random, validation_visitor) {} bool OrFilter::evaluate(const StreamInfo::StreamInfo& info, const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers, @@ -268,7 +272,8 @@ AccessLogFactory::fromProto(const envoy::config::filter::accesslog::v2::AccessLo Server::Configuration::FactoryContext& context) { FilterPtr filter; if (config.has_filter()) { - filter = FilterFactory::fromProto(config.filter(), context.runtime(), context.random()); + filter = FilterFactory::fromProto(config.filter(), context.runtime(), context.random(), + context.messageValidationVisitor()); } auto& factory = diff --git a/source/common/access_log/access_log_impl.h b/source/common/access_log/access_log_impl.h index 899e81a2902e..7e3aa51b3329 100644 --- a/source/common/access_log/access_log_impl.h +++ b/source/common/access_log/access_log_impl.h @@ -28,7 +28,8 @@ class FilterFactory { * Read a filter definition from proto and instantiate a concrete filter class. */ static FilterPtr fromProto(const envoy::config::filter::accesslog::v2::AccessLogFilter& config, - Runtime::Loader& runtime, Runtime::RandomGenerator& random); + Runtime::Loader& runtime, Runtime::RandomGenerator& random, + ProtobufMessage::ValidationVisitor& validation_visitor); }; /** @@ -82,7 +83,8 @@ class OperatorFilter : public Filter { public: OperatorFilter(const Protobuf::RepeatedPtrField< envoy::config::filter::accesslog::v2::AccessLogFilter>& configs, - Runtime::Loader& runtime, Runtime::RandomGenerator& random); + Runtime::Loader& runtime, Runtime::RandomGenerator& random, + ProtobufMessage::ValidationVisitor& validation_visitor); protected: std::vector filters_; @@ -94,7 +96,8 @@ class OperatorFilter : public Filter { class AndFilter : public OperatorFilter { public: AndFilter(const envoy::config::filter::accesslog::v2::AndFilter& config, Runtime::Loader& runtime, - Runtime::RandomGenerator& random); + Runtime::RandomGenerator& random, + ProtobufMessage::ValidationVisitor& validation_visitor); // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::HeaderMap& request_headers, @@ -108,7 +111,8 @@ class AndFilter : public OperatorFilter { class OrFilter : public OperatorFilter { public: OrFilter(const envoy::config::filter::accesslog::v2::OrFilter& config, Runtime::Loader& runtime, - Runtime::RandomGenerator& random); + Runtime::RandomGenerator& random, + ProtobufMessage::ValidationVisitor& validation_visitor); // AccessLog::Filter bool evaluate(const StreamInfo::StreamInfo& info, const Http::HeaderMap& request_headers, diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 2f2aff2a3f6e..d0451893d479 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -235,9 +235,12 @@ class MessageUtil { * @param message message to validate. * @throw ProtoValidationException if the message does not satisfy its type constraints. */ - template static void validate(const MessageType& message) { + template + static void validate(const MessageType& message, + ProtobufMessage::ValidationVisitor& validation_visitor) { // Log warnings or throw errors if deprecated fields are in use. checkForDeprecation(message); + checkUnknownFields(message, validation_visitor); std::string err; if (!Validate(message, &err)) { @@ -249,14 +252,14 @@ class MessageUtil { static void loadFromFileAndValidate(const std::string& path, MessageType& message, ProtobufMessage::ValidationVisitor& validation_visitor) { loadFromFile(path, message, validation_visitor); - validate(message); + validate(message, validation_visitor); } template static void loadFromYamlAndValidate(const std::string& yaml, MessageType& message, ProtobufMessage::ValidationVisitor& validation_visitor) { loadFromYaml(yaml, message, validation_visitor); - validate(message); + validate(message, validation_visitor); } /** @@ -268,9 +271,11 @@ class MessageUtil { * @throw ProtoValidationException if the message does not satisfy its type constraints. */ template - static const MessageType& downcastAndValidate(const Protobuf::Message& config) { + static const MessageType& + downcastAndValidate(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor& validation_visitor) { const auto& typed_config = dynamic_cast(config); - validate(typed_config); + validate(typed_config, validation_visitor); return typed_config; } @@ -282,13 +287,11 @@ class MessageUtil { * @return MessageType the typed message inside the Any. */ template - static inline MessageType anyConvert(const ProtobufWkt::Any& message, - ProtobufMessage::ValidationVisitor& validation_visitor) { + static inline MessageType anyConvert(const ProtobufWkt::Any& message) { MessageType typed_message; if (!message.UnpackTo(&typed_message)) { throw EnvoyException("Unable to unpack " + message.DebugString()); } - checkUnknownFields(typed_message, validation_visitor); return typed_message; }; diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index a589af954269..408689e95d80 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -66,7 +66,8 @@ HedgePolicyImpl::HedgePolicyImpl() : initial_requests_(1), additional_request_chance_({}), hedge_on_per_try_timeout_(false) {} RetryPolicyImpl::RetryPolicyImpl(const envoy::api::v2::route::RetryPolicy& retry_policy, - ProtobufMessage::ValidationVisitor& validation_visitor) { + ProtobufMessage::ValidationVisitor& validation_visitor) + : validation_visitor_(&validation_visitor) { per_try_timeout_ = std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(retry_policy, per_try_timeout, 0)); num_retries_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(retry_policy, num_retries, 1); @@ -141,7 +142,8 @@ Upstream::RetryPrioritySharedPtr RetryPolicyImpl::retryPriority() const { auto& factory = Envoy::Config::Utility::getAndCheckFactory( retry_priority_config_.first); - return factory.createRetryPriority(*retry_priority_config_.second, num_retries_); + return factory.createRetryPriority(*retry_priority_config_.second, *validation_visitor_, + num_retries_); } CorsPolicyImpl::CorsPolicyImpl(const envoy::api::v2::route::CorsPolicy& config, diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 22ed8ae088a6..202ffe6bea6d 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -258,6 +258,7 @@ class RetryPolicyImpl : public RetryPolicy { std::vector retriable_status_codes_; absl::optional base_interval_; absl::optional max_interval_; + ProtobufMessage::ValidationVisitor* validation_visitor_{}; }; /** diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index d94910325cdb..b95554601aef 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -92,9 +92,8 @@ void RdsRouteConfigSubscription::onConfigUpdate( if (!validateUpdateSize(resources.size())) { return; } - auto route_config = MessageUtil::anyConvert( - resources[0], validation_visitor_); - MessageUtil::validate(route_config); + auto route_config = MessageUtil::anyConvert(resources[0]); + MessageUtil::validate(route_config, validation_visitor_); if (route_config.name() != route_config_name_) { throw EnvoyException(fmt::format("Unexpected RDS configuration (expecting {}): {}", route_config_name_, route_config.name())); diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 2e8bbc61a5b4..21acac47cfa7 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -118,9 +118,7 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { - return MessageUtil::anyConvert(resource, - validation_visitor_) - .name(); + return MessageUtil::anyConvert(resource).name(); } RdsRouteConfigSubscription( diff --git a/source/common/router/route_config_update_receiver_impl.cc b/source/common/router/route_config_update_receiver_impl.cc index ecaa5dbb20e8..81931e482c90 100644 --- a/source/common/router/route_config_update_receiver_impl.cc +++ b/source/common/router/route_config_update_receiver_impl.cc @@ -60,9 +60,8 @@ void RouteConfigUpdateReceiverImpl::updateVhosts( const Protobuf::RepeatedPtrField& added_resources) { for (const auto& resource : added_resources) { envoy::api::v2::route::VirtualHost vhost = - MessageUtil::anyConvert(resource.resource(), - validation_visitor_); - MessageUtil::validate(vhost); + MessageUtil::anyConvert(resource.resource()); + MessageUtil::validate(vhost, validation_visitor_); auto found = vhosts.find(vhost.name()); if (found != vhosts.end()) { vhosts.erase(found); diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index e30ea4296b7c..26ef93327284 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -105,8 +105,8 @@ void ScopedRdsConfigSubscription::onConfigUpdate( const std::string& version_info) { std::vector scoped_routes; for (const auto& resource_any : resources) { - scoped_routes.emplace_back(MessageUtil::anyConvert( - resource_any, validation_visitor_)); + scoped_routes.emplace_back( + MessageUtil::anyConvert(resource_any)); } std::unordered_set resource_names; @@ -117,7 +117,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( } } for (const auto& scoped_route : scoped_routes) { - MessageUtil::validate(scoped_route); + MessageUtil::validate(scoped_route, validation_visitor_); } // TODO(AndresGuedez): refactor such that it can be shared with other delta APIs (e.g., CDS). diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 4fd6c72c3ff8..3d006b6a7039 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -115,9 +115,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio ConfigSubscriptionCommonBase::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { - return MessageUtil::anyConvert(resource, - validation_visitor_) - .name(); + return MessageUtil::anyConvert(resource).name(); } const std::string name_; diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index f48449f67897..6cd05ed015f0 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -28,8 +28,7 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, scope_(factory_context.scope().createScope(stat_prefix + "vhds." + config_update_info_->routeConfigName() + ".")), stats_({ALL_VHDS_STATS(POOL_COUNTER(*scope_))}), - route_config_providers_(route_config_providers), - validation_visitor_(factory_context.messageValidationVisitor()) { + route_config_providers_(route_config_providers) { const auto& config_source = config_update_info_->routeConfiguration() .vhds() .config_source() diff --git a/source/common/router/vhds.h b/source/common/router/vhds.h index 0959bc6eca2a..a2d3cb6beb81 100644 --- a/source/common/router/vhds.h +++ b/source/common/router/vhds.h @@ -59,9 +59,7 @@ class VhdsSubscription : Envoy::Config::SubscriptionCallbacks, void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { - return MessageUtil::anyConvert(resource, - validation_visitor_) - .name(); + return MessageUtil::anyConvert(resource).name(); } RouteConfigUpdatePtr& config_update_info_; @@ -70,7 +68,6 @@ class VhdsSubscription : Envoy::Config::SubscriptionCallbacks, Stats::ScopePtr scope_; VhdsStats stats_; std::unordered_set& route_config_providers_; - ProtobufMessage::ValidationVisitor& validation_visitor_; }; using VhdsSubscriptionPtr = std::unique_ptr; diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 3e86fcba3329..b668b9aa7187 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -517,9 +517,8 @@ RtdsSubscription::RtdsSubscription( void RtdsSubscription::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string&) { validateUpdateSize(resources.size()); - auto runtime = MessageUtil::anyConvert( - resources[0], validation_visitor_); - MessageUtil::validate(runtime); + auto runtime = MessageUtil::anyConvert(resources[0]); + MessageUtil::validate(runtime, validation_visitor_); if (runtime.name() != resource_name_) { throw EnvoyException( fmt::format("Unexpected RTDS runtime (expecting {}): {}", resource_name_, runtime.name())); diff --git a/source/common/runtime/runtime_impl.h b/source/common/runtime/runtime_impl.h index 06be894c5aec..7832fe970c87 100644 --- a/source/common/runtime/runtime_impl.h +++ b/source/common/runtime/runtime_impl.h @@ -209,9 +209,7 @@ struct RtdsSubscription : Config::SubscriptionCallbacks, Logger::Loggable(resource, - validation_visitor_) - .name(); + return MessageUtil::anyConvert(resource).name(); } void start(); diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index 1aa00bb8c75b..7d695040dfa9 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -30,9 +30,8 @@ SdsApi::SdsApi(envoy::api::v2::core::ConfigSource sds_config, absl::string_view void SdsApi::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { validateUpdateSize(resources.size()); - auto secret = - MessageUtil::anyConvert(resources[0], validation_visitor_); - MessageUtil::validate(secret); + auto secret = MessageUtil::anyConvert(resources[0]); + MessageUtil::validate(secret, validation_visitor_); if (secret.name() != sds_config_name_) { throw EnvoyException( diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index d6b9235157fa..762de28c96b4 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -59,8 +59,7 @@ class SdsApi : public Config::SubscriptionCallbacks { void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { - return MessageUtil::anyConvert(resource, validation_visitor_) - .name(); + return MessageUtil::anyConvert(resource).name(); } private: diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 2d0271f56e58..3ec334cf9b52 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -35,8 +35,7 @@ void CdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField clusters; for (const auto& cluster_blob : resources) { - clusters.push_back( - MessageUtil::anyConvert(cluster_blob, validation_visitor_)); + clusters.push_back(MessageUtil::anyConvert(cluster_blob)); clusters_to_remove.erase(clusters.back().name()); } Protobuf::RepeatedPtrField to_remove_repeated; @@ -66,9 +65,8 @@ void CdsApiImpl::onConfigUpdate( for (const auto& resource : added_resources) { envoy::api::v2::Cluster cluster; try { - cluster = MessageUtil::anyConvert(resource.resource(), - validation_visitor_); - MessageUtil::validate(cluster); + cluster = MessageUtil::anyConvert(resource.resource()); + MessageUtil::validate(cluster, validation_visitor_); if (!cluster_names.insert(cluster.name()).second) { // NOTE: at this point, the first of these duplicates has already been successfully applied. throw EnvoyException(fmt::format("duplicate cluster {} found", cluster.name())); diff --git a/source/common/upstream/cds_api_impl.h b/source/common/upstream/cds_api_impl.h index 2825213f3153..b17d4bbc9989 100644 --- a/source/common/upstream/cds_api_impl.h +++ b/source/common/upstream/cds_api_impl.h @@ -43,7 +43,7 @@ class CdsApiImpl : public CdsApi, void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { - return MessageUtil::anyConvert(resource, validation_visitor_).name(); + return MessageUtil::anyConvert(resource).name(); } CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, diff --git a/source/common/upstream/cluster_factory_impl.h b/source/common/upstream/cluster_factory_impl.h index ae709eb094c2..4b3f536dd3c5 100644 --- a/source/common/upstream/cluster_factory_impl.h +++ b/source/common/upstream/cluster_factory_impl.h @@ -178,7 +178,8 @@ template class ConfigurableClusterFactoryBase : public Clust cluster.cluster_type().typed_config(), ProtobufWkt::Struct::default_instance(), socket_factory_context.messageValidationVisitor(), *config); return createClusterWithConfig(cluster, - MessageUtil::downcastAndValidate(*config), + MessageUtil::downcastAndValidate( + *config, context.messageValidationVisitor()), context, socket_factory_context, std::move(stats_scope)); } diff --git a/source/common/upstream/eds.cc b/source/common/upstream/eds.cc index 7205d5aec7d2..3d795c9a207b 100644 --- a/source/common/upstream/eds.cc +++ b/source/common/upstream/eds.cc @@ -101,9 +101,9 @@ void EdsClusterImpl::onConfigUpdate(const Protobuf::RepeatedPtrField( - resources[0], validation_visitor_); - MessageUtil::validate(cluster_load_assignment); + auto cluster_load_assignment = + MessageUtil::anyConvert(resources[0]); + MessageUtil::validate(cluster_load_assignment, validation_visitor_); if (cluster_load_assignment.cluster_name() != cluster_name_) { throw EnvoyException(fmt::format("Unexpected EDS cluster (expecting {}): {}", cluster_name_, cluster_load_assignment.cluster_name())); diff --git a/source/common/upstream/eds.h b/source/common/upstream/eds.h index c7b3ffaba024..0df5f4c84473 100644 --- a/source/common/upstream/eds.h +++ b/source/common/upstream/eds.h @@ -38,9 +38,7 @@ class EdsClusterImpl : public BaseDynamicClusterImpl, Config::SubscriptionCallba void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { - return MessageUtil::anyConvert(resource, - validation_visitor_) - .cluster_name(); + return MessageUtil::anyConvert(resource).cluster_name(); } using LocalityWeightsMap = diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index f44e01f5a5de..8443868f68f0 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -140,7 +140,7 @@ createProtocolOptionsConfig(const std::string& name, const ProtobufWkt::Any& typ Envoy::Config::Utility::translateOpaqueConfig(typed_config, config, validation_visitor, *proto_config); - return factory->createProtocolOptionsConfig(*proto_config); + return factory->createProtocolOptionsConfig(*proto_config, validation_visitor); } std::map diff --git a/source/extensions/access_loggers/file/config.cc b/source/extensions/access_loggers/file/config.cc index d04f3965a008..fcf432a861bc 100644 --- a/source/extensions/access_loggers/file/config.cc +++ b/source/extensions/access_loggers/file/config.cc @@ -24,7 +24,8 @@ FileAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, Server::Configuration::FactoryContext& context) { const auto& fal_config = - MessageUtil::downcastAndValidate(config); + MessageUtil::downcastAndValidate( + config, context.messageValidationVisitor()); AccessLog::FormatterPtr formatter; if (fal_config.access_log_format_case() == envoy::config::accesslog::v2::FileAccessLog::kFormat || diff --git a/source/extensions/access_loggers/grpc/http_config.cc b/source/extensions/access_loggers/grpc/http_config.cc index 2e3f19e87927..1047beaa579a 100644 --- a/source/extensions/access_loggers/grpc/http_config.cc +++ b/source/extensions/access_loggers/grpc/http_config.cc @@ -29,7 +29,8 @@ HttpGrpcAccessLogFactory::createAccessLogInstance(const Protobuf::Message& confi validateProtoDescriptors(); const auto& proto_config = MessageUtil::downcastAndValidate< - const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig&>(config); + const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig&>( + config, context.messageValidationVisitor()); std::shared_ptr grpc_access_logger_cache = context.singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(grpc_access_logger_cache), [&context] { diff --git a/source/extensions/filters/http/common/factory_base.h b/source/extensions/filters/http/common/factory_base.h index 8e3a9669173b..73c91d4d1be8 100644 --- a/source/extensions/filters/http/common/factory_base.h +++ b/source/extensions/filters/http/common/factory_base.h @@ -25,8 +25,9 @@ class FactoryBase : public Server::Configuration::NamedHttpFilterConfigFactory { createFilterFactoryFromProto(const Protobuf::Message& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override { - return createFilterFactoryFromProtoTyped( - MessageUtil::downcastAndValidate(proto_config), stats_prefix, context); + return createFilterFactoryFromProtoTyped(MessageUtil::downcastAndValidate( + proto_config, context.messageValidationVisitor()), + stats_prefix, context); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { @@ -41,7 +42,9 @@ class FactoryBase : public Server::Configuration::NamedHttpFilterConfigFactory { createRouteSpecificFilterConfig(const Protobuf::Message& proto_config, Server::Configuration::FactoryContext& context) override { return createRouteSpecificFilterConfigTyped( - MessageUtil::downcastAndValidate(proto_config), context); + MessageUtil::downcastAndValidate( + proto_config, context.messageValidationVisitor()), + context); } std::string name() override { return name_; } diff --git a/source/extensions/filters/listener/original_src/original_src_config_factory.cc b/source/extensions/filters/listener/original_src/original_src_config_factory.cc index aa1efc58b6e2..14a1a7384bfe 100644 --- a/source/extensions/filters/listener/original_src/original_src_config_factory.cc +++ b/source/extensions/filters/listener/original_src/original_src_config_factory.cc @@ -14,9 +14,10 @@ namespace ListenerFilters { namespace OriginalSrc { Network::ListenerFilterFactoryCb OriginalSrcConfigFactory::createFilterFactoryFromProto( - const Protobuf::Message& message, Server::Configuration::ListenerFactoryContext&) { + const Protobuf::Message& message, Server::Configuration::ListenerFactoryContext& context) { auto proto_config = MessageUtil::downcastAndValidate< - const envoy::config::filter::listener::original_src::v2alpha1::OriginalSrc&>(message); + const envoy::config::filter::listener::original_src::v2alpha1::OriginalSrc&>( + message, context.messageValidationVisitor()); Config config(proto_config); return [config](Network::ListenerFilterManager& filter_manager) -> void { filter_manager.addAcceptFilter(std::make_unique(config)); diff --git a/source/extensions/filters/network/common/factory_base.h b/source/extensions/filters/network/common/factory_base.h index 0c241878d2e8..8b3bf610b83a 100644 --- a/source/extensions/filters/network/common/factory_base.h +++ b/source/extensions/filters/network/common/factory_base.h @@ -25,8 +25,9 @@ class FactoryBase : public Server::Configuration::NamedNetworkFilterConfigFactor Network::FilterFactoryCb createFilterFactoryFromProto(const Protobuf::Message& proto_config, Server::Configuration::FactoryContext& context) override { - return createFilterFactoryFromProtoTyped( - MessageUtil::downcastAndValidate(proto_config), context); + return createFilterFactoryFromProtoTyped(MessageUtil::downcastAndValidate( + proto_config, context.messageValidationVisitor()), + context); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { @@ -38,9 +39,10 @@ class FactoryBase : public Server::Configuration::NamedNetworkFilterConfigFactor } Upstream::ProtocolOptionsConfigConstSharedPtr - createProtocolOptionsConfig(const Protobuf::Message& proto_config) override { - return createProtocolOptionsTyped( - MessageUtil::downcastAndValidate(proto_config)); + createProtocolOptionsConfig(const Protobuf::Message& proto_config, + ProtobufMessage::ValidationVisitor& validation_visitor) override { + return createProtocolOptionsTyped(MessageUtil::downcastAndValidate( + proto_config, validation_visitor)); } std::string name() override { return name_; } diff --git a/source/extensions/filters/network/dubbo_proxy/filters/factory_base.h b/source/extensions/filters/network/dubbo_proxy/filters/factory_base.h index 1319a846170b..021d99b52e2f 100644 --- a/source/extensions/filters/network/dubbo_proxy/filters/factory_base.h +++ b/source/extensions/filters/network/dubbo_proxy/filters/factory_base.h @@ -21,8 +21,9 @@ template class FactoryBase : public NamedDubboFilterConfigFa createFilterFactoryFromProto(const Protobuf::Message& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override { - return createFilterFactoryFromProtoTyped( - MessageUtil::downcastAndValidate(proto_config), stats_prefix, context); + return createFilterFactoryFromProtoTyped(MessageUtil::downcastAndValidate( + proto_config, context.messageValidationVisitor()), + stats_prefix, context); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { diff --git a/source/extensions/filters/network/thrift_proxy/filters/factory_base.h b/source/extensions/filters/network/thrift_proxy/filters/factory_base.h index bf2bb292f043..159328b73a94 100644 --- a/source/extensions/filters/network/thrift_proxy/filters/factory_base.h +++ b/source/extensions/filters/network/thrift_proxy/filters/factory_base.h @@ -16,8 +16,9 @@ template class FactoryBase : public NamedThriftFilterConfigF createFilterFactoryFromProto(const Protobuf::Message& proto_config, const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override { - return createFilterFactoryFromProtoTyped( - MessageUtil::downcastAndValidate(proto_config), stats_prefix, context); + return createFilterFactoryFromProtoTyped(MessageUtil::downcastAndValidate( + proto_config, context.messageValidationVisitor()), + stats_prefix, context); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { diff --git a/source/extensions/grpc_credentials/aws_iam/config.cc b/source/extensions/grpc_credentials/aws_iam/config.cc index 3128a9608434..d2b423f744d7 100644 --- a/source/extensions/grpc_credentials/aws_iam/config.cc +++ b/source/extensions/grpc_credentials/aws_iam/config.cc @@ -34,12 +34,15 @@ std::shared_ptr AwsIamGrpcCredentialsFactory::getChann case envoy::api::v2::core::GrpcService::GoogleGrpc::CallCredentials::kFromPlugin: { if (credential.from_plugin().name() == GrpcCredentialsNames::get().AwsIam) { AwsIamGrpcCredentialsFactory credentials_factory; + // We don't deal with validation failures here at runtime today, see + // https://github.com/envoyproxy/envoy/issues/8010. const Envoy::ProtobufTypes::MessagePtr config_message = Envoy::Config::Utility::translateToFactoryConfig( credential.from_plugin(), ProtobufMessage::getNullValidationVisitor(), credentials_factory); const auto& config = Envoy::MessageUtil::downcastAndValidate< - const envoy::config::grpc_credential::v2alpha::AwsIamConfig&>(*config_message); + const envoy::config::grpc_credential::v2alpha::AwsIamConfig&>( + *config_message, ProtobufMessage::getNullValidationVisitor()); auto credentials_provider = std::make_shared( api, HttpFilters::Common::Aws::Utility::metadataFetcher); diff --git a/source/extensions/grpc_credentials/file_based_metadata/config.cc b/source/extensions/grpc_credentials/file_based_metadata/config.cc index e40c4549f101..03eabb5e9094 100644 --- a/source/extensions/grpc_credentials/file_based_metadata/config.cc +++ b/source/extensions/grpc_credentials/file_based_metadata/config.cc @@ -28,16 +28,15 @@ FileBasedMetadataGrpcCredentialsFactory::getChannelCredentials( case envoy::api::v2::core::GrpcService::GoogleGrpc::CallCredentials::kFromPlugin: { if (credential.from_plugin().name() == GrpcCredentialsNames::get().FileBasedMetadata) { FileBasedMetadataGrpcCredentialsFactory file_based_metadata_credentials_factory; - // TODO(htuch): This should probably follow the standard metadata validation pattern, but - // needs error handling, since this happens at runtime, not config time, and we need to - // catch any validation exceptions. + // We don't deal with validation failures here at runtime today, see + // https://github.com/envoyproxy/envoy/issues/8010. const Envoy::ProtobufTypes::MessagePtr file_based_metadata_config_message = Envoy::Config::Utility::translateToFactoryConfig( credential.from_plugin(), ProtobufMessage::getNullValidationVisitor(), file_based_metadata_credentials_factory); const auto& file_based_metadata_config = Envoy::MessageUtil::downcastAndValidate< const envoy::config::grpc_credential::v2alpha::FileBasedMetadataConfig&>( - *file_based_metadata_config_message); + *file_based_metadata_config_message, ProtobufMessage::getNullValidationVisitor()); std::shared_ptr new_call_creds = grpc::MetadataCredentialsFromPlugin( std::make_unique(file_based_metadata_config, api)); if (call_creds == nullptr) { diff --git a/source/extensions/health_checkers/redis/utility.h b/source/extensions/health_checkers/redis/utility.h index df868f85a3e8..b05a1856f273 100644 --- a/source/extensions/health_checkers/redis/utility.h +++ b/source/extensions/health_checkers/redis/utility.h @@ -21,7 +21,7 @@ getRedisHealthCheckConfig(const envoy::api::v2::core::HealthCheck& health_check_ MessageUtil::jsonConvert(health_check_config.custom_health_check().config(), validation_visitor, *config); return MessageUtil::downcastAndValidate( - *config); + *config, validation_visitor); } } // namespace diff --git a/source/extensions/resource_monitors/common/factory_base.h b/source/extensions/resource_monitors/common/factory_base.h index 124367132ddc..65350bc6950c 100644 --- a/source/extensions/resource_monitors/common/factory_base.h +++ b/source/extensions/resource_monitors/common/factory_base.h @@ -15,8 +15,9 @@ class FactoryBase : public Server::Configuration::ResourceMonitorFactory { Server::ResourceMonitorPtr createResourceMonitor(const Protobuf::Message& config, Server::Configuration::ResourceMonitorFactoryContext& context) override { - return createResourceMonitorFromProtoTyped( - MessageUtil::downcastAndValidate(config), context); + return createResourceMonitorFromProtoTyped(MessageUtil::downcastAndValidate( + config, context.messageValidationVisitor()), + context); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { diff --git a/source/extensions/retry/priority/previous_priorities/config.cc b/source/extensions/retry/priority/previous_priorities/config.cc index 5eaf923967db..d528442359f7 100644 --- a/source/extensions/retry/priority/previous_priorities/config.cc +++ b/source/extensions/retry/priority/previous_priorities/config.cc @@ -9,12 +9,14 @@ namespace Extensions { namespace Retry { namespace Priority { -Upstream::RetryPrioritySharedPtr -PreviousPrioritiesRetryPriorityFactory::createRetryPriority(const Protobuf::Message& config, - uint32_t max_retries) { +Upstream::RetryPrioritySharedPtr PreviousPrioritiesRetryPriorityFactory::createRetryPriority( + const Protobuf::Message& config, ProtobufMessage::ValidationVisitor& validation_visitor, + + uint32_t max_retries) { return std::make_shared( MessageUtil::downcastAndValidate< - const envoy::config::retry::previous_priorities::PreviousPrioritiesConfig&>(config) + const envoy::config::retry::previous_priorities::PreviousPrioritiesConfig&>( + config, validation_visitor) .update_frequency(), max_retries); } diff --git a/source/extensions/retry/priority/previous_priorities/config.h b/source/extensions/retry/priority/previous_priorities/config.h index b98555235299..8fe761b3fe69 100644 --- a/source/extensions/retry/priority/previous_priorities/config.h +++ b/source/extensions/retry/priority/previous_priorities/config.h @@ -15,8 +15,10 @@ namespace Priority { class PreviousPrioritiesRetryPriorityFactory : public Upstream::RetryPriorityFactory { public: - Upstream::RetryPrioritySharedPtr createRetryPriority(const Protobuf::Message& config, - uint32_t max_retries) override; + Upstream::RetryPrioritySharedPtr + createRetryPriority(const Protobuf::Message& config, + ProtobufMessage::ValidationVisitor& validation_visitor, + uint32_t max_retries) override; std::string name() const override { return RetryPriorityValues::get().PreviousPrioritiesRetryPriority; diff --git a/source/extensions/stat_sinks/dog_statsd/config.cc b/source/extensions/stat_sinks/dog_statsd/config.cc index 9902d3534451..6b7ce0182bb5 100644 --- a/source/extensions/stat_sinks/dog_statsd/config.cc +++ b/source/extensions/stat_sinks/dog_statsd/config.cc @@ -19,7 +19,8 @@ namespace DogStatsd { Stats::SinkPtr DogStatsdSinkFactory::createStatsSink(const Protobuf::Message& config, Server::Instance& server) { const auto& sink_config = - MessageUtil::downcastAndValidate(config); + MessageUtil::downcastAndValidate( + config, server.messageValidationContext().staticValidationVisitor()); Network::Address::InstanceConstSharedPtr address = Network::Address::resolveProtoAddress(sink_config.address()); ENVOY_LOG(debug, "dog_statsd UDP ip address: {}", address->asString()); diff --git a/source/extensions/stat_sinks/hystrix/config.cc b/source/extensions/stat_sinks/hystrix/config.cc index 3034efad99ae..a9fc2d697d60 100644 --- a/source/extensions/stat_sinks/hystrix/config.cc +++ b/source/extensions/stat_sinks/hystrix/config.cc @@ -19,7 +19,8 @@ namespace Hystrix { Stats::SinkPtr HystrixSinkFactory::createStatsSink(const Protobuf::Message& config, Server::Instance& server) { const auto& hystrix_sink = - MessageUtil::downcastAndValidate(config); + MessageUtil::downcastAndValidate( + config, server.messageValidationContext().staticValidationVisitor()); return std::make_unique(server, hystrix_sink.num_buckets()); } diff --git a/source/extensions/stat_sinks/metrics_service/config.cc b/source/extensions/stat_sinks/metrics_service/config.cc index 78bad7587992..80e74ea9925f 100644 --- a/source/extensions/stat_sinks/metrics_service/config.cc +++ b/source/extensions/stat_sinks/metrics_service/config.cc @@ -23,7 +23,7 @@ Stats::SinkPtr MetricsServiceSinkFactory::createStatsSink(const Protobuf::Messag const auto& sink_config = MessageUtil::downcastAndValidate( - config); + config, server.messageValidationContext().staticValidationVisitor()); const auto& grpc_service = sink_config.grpc_service(); ENVOY_LOG(debug, "Metrics Service gRPC service configuration: {}", grpc_service.DebugString()); diff --git a/source/extensions/stat_sinks/statsd/config.cc b/source/extensions/stat_sinks/statsd/config.cc index 6fe112882ae7..f7e352e30bd4 100644 --- a/source/extensions/stat_sinks/statsd/config.cc +++ b/source/extensions/stat_sinks/statsd/config.cc @@ -20,7 +20,8 @@ Stats::SinkPtr StatsdSinkFactory::createStatsSink(const Protobuf::Message& confi Server::Instance& server) { const auto& statsd_sink = - MessageUtil::downcastAndValidate(config); + MessageUtil::downcastAndValidate( + config, server.messageValidationContext().staticValidationVisitor()); switch (statsd_sink.statsd_specifier_case()) { case envoy::config::metrics::v2::StatsdSink::kAddress: { Network::Address::InstanceConstSharedPtr address = diff --git a/source/extensions/tracers/common/factory_base.h b/source/extensions/tracers/common/factory_base.h index 6b57de7b3c35..b32975a74e9f 100644 --- a/source/extensions/tracers/common/factory_base.h +++ b/source/extensions/tracers/common/factory_base.h @@ -17,8 +17,10 @@ template class FactoryBase : public Server::Configuration::T // Server::Configuration::TracerFactory Tracing::HttpTracerPtr createHttpTracer(const Protobuf::Message& config, Server::Instance& server) override { - return createHttpTracerTyped(MessageUtil::downcastAndValidate(config), - server); + return createHttpTracerTyped( + MessageUtil::downcastAndValidate( + config, server.messageValidationContext().staticValidationVisitor()), + server); } ProtobufTypes::MessagePtr createEmptyConfigProto() override { diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index a85bb9d7fa85..712f69282078 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -79,7 +79,7 @@ Network::TransportSocketFactoryPtr createTransportSocketFactoryHelper( [] { return std::make_shared(); }); auto config = MessageUtil::downcastAndValidate( - message); + message, factory_ctxt.messageValidationVisitor()); HandshakeValidator validator = createHandshakeValidator(config); const std::string& handshaker_service = config.handshaker_service(); diff --git a/source/extensions/transport_sockets/tap/config.cc b/source/extensions/transport_sockets/tap/config.cc index c376fe7cec8a..fb04ed511b26 100644 --- a/source/extensions/transport_sockets/tap/config.cc +++ b/source/extensions/transport_sockets/tap/config.cc @@ -36,7 +36,7 @@ Network::TransportSocketFactoryPtr UpstreamTapSocketConfigFactory::createTranspo Server::Configuration::TransportSocketFactoryContext& context) { const auto& outer_config = MessageUtil::downcastAndValidate( - message); + message, context.messageValidationVisitor()); auto& inner_config_factory = Config::Utility::getAndCheckFactory< Server::Configuration::UpstreamTransportSocketConfigFactory>( outer_config.transport_socket().name()); @@ -55,7 +55,7 @@ Network::TransportSocketFactoryPtr DownstreamTapSocketConfigFactory::createTrans const std::vector& server_names) { const auto& outer_config = MessageUtil::downcastAndValidate( - message); + message, context.messageValidationVisitor()); auto& inner_config_factory = Config::Utility::getAndCheckFactory< Server::Configuration::DownstreamTransportSocketConfigFactory>( outer_config.transport_socket().name()); diff --git a/source/extensions/transport_sockets/tls/config.cc b/source/extensions/transport_sockets/tls/config.cc index 15854a3698bb..4d04b0f2c3b3 100644 --- a/source/extensions/transport_sockets/tls/config.cc +++ b/source/extensions/transport_sockets/tls/config.cc @@ -17,7 +17,8 @@ Network::TransportSocketFactoryPtr UpstreamSslSocketFactory::createTransportSock const Protobuf::Message& message, Server::Configuration::TransportSocketFactoryContext& context) { auto client_config = std::make_unique( - MessageUtil::downcastAndValidate(message), + MessageUtil::downcastAndValidate( + message, context.messageValidationVisitor()), context); return std::make_unique( std::move(client_config), context.sslContextManager(), context.statsScope()); @@ -34,7 +35,8 @@ Network::TransportSocketFactoryPtr DownstreamSslSocketFactory::createTransportSo const Protobuf::Message& message, Server::Configuration::TransportSocketFactoryContext& context, const std::vector& server_names) { auto server_config = std::make_unique( - MessageUtil::downcastAndValidate(message), + MessageUtil::downcastAndValidate( + message, context.messageValidationVisitor()), context); return std::make_unique( std::move(server_config), context.sslContextManager(), context.statsScope(), server_names); diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index 581abba24b71..957603de82f1 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -49,9 +49,8 @@ void LdsApiImpl::onConfigUpdate( for (const auto& resource : added_resources) { envoy::api::v2::Listener listener; try { - listener = MessageUtil::anyConvert(resource.resource(), - validation_visitor_); - MessageUtil::validate(listener); + listener = MessageUtil::anyConvert(resource.resource()); + MessageUtil::validate(listener, validation_visitor_); if (!listener_names.insert(listener.name()).second) { // NOTE: at this point, the first of these duplicates has already been successfully applied. throw EnvoyException(fmt::format("duplicate listener {} found", listener.name())); @@ -91,8 +90,7 @@ void LdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField(listener_blob, validation_visitor_) - .name(); + MessageUtil::anyConvert(listener_blob).name(); to_add->set_name(listener_name); to_add->set_version(version_info); to_add->mutable_resource()->MergeFrom(listener_blob); diff --git a/source/server/lds_api.h b/source/server/lds_api.h index 199bd8413243..24ca66cfe73f 100644 --- a/source/server/lds_api.h +++ b/source/server/lds_api.h @@ -39,7 +39,7 @@ class LdsApiImpl : public LdsApi, void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override { - return MessageUtil::anyConvert(resource, validation_visitor_).name(); + return MessageUtil::anyConvert(resource).name(); } std::unique_ptr subscription_; diff --git a/source/server/overload_manager_impl.cc b/source/server/overload_manager_impl.cc index 6eaad4b82c9b..7795802b6c76 100644 --- a/source/server/overload_manager_impl.cc +++ b/source/server/overload_manager_impl.cc @@ -93,7 +93,7 @@ OverloadManagerImpl::OverloadManagerImpl( : started_(false), dispatcher_(dispatcher), tls_(slot_allocator.allocateSlot()), refresh_interval_( std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(config, refresh_interval, 1000))) { - Configuration::ResourceMonitorFactoryContextImpl context(dispatcher, api); + Configuration::ResourceMonitorFactoryContextImpl context(dispatcher, api, validation_visitor); for (const auto& resource : config.resource_monitors()) { const auto& name = resource.name(); ENVOY_LOG(debug, "Adding resource monitor for {}", name); diff --git a/source/server/resource_monitor_config_impl.h b/source/server/resource_monitor_config_impl.h index bab2f6c3a64f..e565f6fd78ad 100644 --- a/source/server/resource_monitor_config_impl.h +++ b/source/server/resource_monitor_config_impl.h @@ -8,16 +8,22 @@ namespace Configuration { class ResourceMonitorFactoryContextImpl : public ResourceMonitorFactoryContext { public: - ResourceMonitorFactoryContextImpl(Event::Dispatcher& dispatcher, Api::Api& api) - : dispatcher_(dispatcher), api_(api) {} + ResourceMonitorFactoryContextImpl(Event::Dispatcher& dispatcher, Api::Api& api, + ProtobufMessage::ValidationVisitor& validation_visitor) + : dispatcher_(dispatcher), api_(api), validation_visitor_(validation_visitor) {} Event::Dispatcher& dispatcher() override { return dispatcher_; } Api::Api& api() override { return api_; } + ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { + return validation_visitor_; + } + private: Event::Dispatcher& dispatcher_; Api::Api& api_; + ProtobufMessage::ValidationVisitor& validation_visitor_; }; } // namespace Configuration diff --git a/source/server/server.cc b/source/server/server.cc index 6e8cd16af655..4994d53ccdf8 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -231,7 +231,7 @@ InstanceUtil::BootstrapVersion InstanceUtil::loadBootstrapConfig( if (config_proto.ByteSize() != 0) { bootstrap.MergeFrom(config_proto); } - MessageUtil::validate(bootstrap); + MessageUtil::validate(bootstrap, validation_visitor); return BootstrapVersion::V2; } diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index fe29b7839499..02677fc783ed 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -1143,7 +1143,7 @@ class TestHeaderFilterFactory : public ExtensionFilterFactory { auto factory_config = Config::Utility::translateToFactoryConfig( config, Envoy::ProtobufMessage::getNullValidationVisitor(), *this); const auto& header_config = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *factory_config); return std::make_unique(header_config); } diff --git a/test/common/http/codec_impl_fuzz_test.cc b/test/common/http/codec_impl_fuzz_test.cc index 28cb56d0e592..f411565fa465 100644 --- a/test/common/http/codec_impl_fuzz_test.cc +++ b/test/common/http/codec_impl_fuzz_test.cc @@ -506,7 +506,7 @@ void codecFuzz(const test::common::http::CodecImplFuzzTestCase& input, HttpVersi DEFINE_PROTO_FUZZER(const test::common::http::CodecImplFuzzTestCase& input) { try { // Validate input early. - MessageUtil::validate(input); + TestUtility::validate(input); codecFuzz(input, HttpVersion::Http1); codecFuzz(input, HttpVersion::Http2); } catch (const EnvoyException& e) { diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 9d437982a0e0..050fc0485e02 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -123,15 +123,28 @@ TEST_F(ProtobufUtilityTest, RepeatedPtrUtilDebugString) { EXPECT_EQ("[value: 10\n, value: 20\n]", RepeatedPtrUtil::debugString(repeated)); } -TEST_F(ProtobufUtilityTest, DowncastAndValidate) { +// Validated exception thrown when downcastAndValidate observes a PGV failures. +TEST_F(ProtobufUtilityTest, DowncastAndValidateFailedValidation) { envoy::config::bootstrap::v2::Bootstrap bootstrap; bootstrap.mutable_static_resources()->add_clusters(); - EXPECT_THROW(MessageUtil::validate(bootstrap), ProtoValidationException); + EXPECT_THROW(TestUtility::validate(bootstrap), ProtoValidationException); EXPECT_THROW( - MessageUtil::downcastAndValidate(bootstrap), + TestUtility::downcastAndValidate(bootstrap), ProtoValidationException); } +// Validated exception thrown when downcastAndValidate observes a unknown field. +TEST_F(ProtobufUtilityTest, DowncastAndValidateUnknownFields) { + envoy::config::bootstrap::v2::Bootstrap bootstrap; + bootstrap.GetReflection()->MutableUnknownFields(&bootstrap)->AddVarint(1, 0); + EXPECT_THROW_WITH_MESSAGE(TestUtility::validate(bootstrap), EnvoyException, + "Protobuf message (type envoy.config.bootstrap.v2.Bootstrap with " + "unknown field set {1}) has unknown fields"); + EXPECT_THROW_WITH_MESSAGE(TestUtility::validate(bootstrap), EnvoyException, + "Protobuf message (type envoy.config.bootstrap.v2.Bootstrap with " + "unknown field set {1}) has unknown fields"); +} + TEST_F(ProtobufUtilityTest, LoadBinaryProtoFromFile) { envoy::config::bootstrap::v2::Bootstrap bootstrap; bootstrap.mutable_cluster_manager() @@ -342,17 +355,6 @@ TEST_F(ProtobufUtilityTest, AnyConvertWrongType) { EnvoyException, "Unable to unpack .*"); } -TEST_F(ProtobufUtilityTest, AnyConvertWrongFields) { - const ProtobufWkt::Struct obj = MessageUtil::keyValueStruct("test_key", "test_value"); - ProtobufWkt::Any source_any; - source_any.PackFrom(obj); - source_any.set_type_url("type.google.com/google.protobuf.Timestamp"); - EXPECT_THROW_WITH_MESSAGE(TestUtility::anyConvert(source_any), - EnvoyException, - "Protobuf message (type google.protobuf.Timestamp with unknown " - "field set {1}) has unknown fields"); -} - TEST_F(ProtobufUtilityTest, JsonConvertSuccess) { envoy::config::bootstrap::v2::Bootstrap source; source.set_flags_path("foo"); diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 6b7c5743dc90..ca9073d4b712 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -89,7 +89,7 @@ Http::TestHeaderMapImpl genHeaders(const std::string& host, const std::string& p envoy::api::v2::RouteConfiguration parseRouteConfigurationFromV2Yaml(const std::string& yaml) { envoy::api::v2::RouteConfiguration route_config; TestUtility::loadFromYaml(yaml, route_config); - MessageUtil::validate(route_config); + TestUtility::validate(route_config); return route_config; } diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 3ab5ead740d7..e6c7c6aa7f45 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -505,7 +505,7 @@ TEST_F(StreamInfoHeaderFormatterTest, ValidateLimitsOnUserDefinedHeaders) { header->mutable_header()->set_key("header_name"); header->mutable_header()->set_value(long_string); header->mutable_append()->set_value(true); - EXPECT_THROW_WITH_REGEX(MessageUtil::validate(route), ProtoValidationException, + EXPECT_THROW_WITH_REGEX(TestUtility::validate(route), ProtoValidationException, "Proto constraint validation failed.*"); } { @@ -516,7 +516,7 @@ TEST_F(StreamInfoHeaderFormatterTest, ValidateLimitsOnUserDefinedHeaders) { header->mutable_header()->set_key("header_name"); header->mutable_header()->set_value("value"); } - EXPECT_THROW_WITH_REGEX(MessageUtil::validate(route), ProtoValidationException, + EXPECT_THROW_WITH_REGEX(TestUtility::validate(route), ProtoValidationException, "Proto constraint validation failed.*"); } } diff --git a/test/common/router/header_parser_fuzz_test.cc b/test/common/router/header_parser_fuzz_test.cc index 451aae702c4e..cb62744f52d1 100644 --- a/test/common/router/header_parser_fuzz_test.cc +++ b/test/common/router/header_parser_fuzz_test.cc @@ -11,7 +11,7 @@ namespace { DEFINE_PROTO_FUZZER(const test::common::router::TestCase& input) { try { - MessageUtil::validate(input); + TestUtility::validate(input); auto headers_to_add = replaceInvalidHeaders(input.headers_to_add()); Protobuf::RepeatedPtrField headers_to_remove; for (const auto& s : input.headers_to_remove()) { diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 1046b5827be0..fb74aa9694f7 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -295,7 +295,7 @@ envoy::api::v2::RouteConfiguration parseRouteConfigurationFromV2Yaml(const std:: TEST_F(RouteConfigProviderManagerImplTest, ConfigDump) { auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["routes"](); const auto& route_config_dump = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); // No routes at all, no last_updated timestamp @@ -325,7 +325,7 @@ name: foo parseRouteConfigurationFromV2Yaml(config_yaml), factory_context_); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["routes"](); const auto& route_config_dump2 = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); TestUtility::loadFromYaml(R"EOF( static_route_configs: @@ -368,7 +368,7 @@ name: foo rds_callbacks_->onConfigUpdate(response1.resources(), response1.version_info()); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["routes"](); const auto& route_config_dump3 = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); TestUtility::loadFromYaml(R"EOF( static_route_configs: @@ -497,7 +497,7 @@ TEST_F(RouteConfigProviderManagerImplTest, onConfigUpdateWrongSize) { TEST_F(RouteConfigProviderManagerImplTest, ConfigDumpAfterConfigRejected) { auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["routes"](); const auto& route_config_dump = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); // No routes at all, no last_updated timestamp @@ -549,7 +549,7 @@ version_info: '1' message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["routes"](); const auto& route_config_dump3 = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); TestUtility::loadFromYaml(R"EOF( static_route_configs: diff --git a/test/common/router/route_fuzz_test.cc b/test/common/router/route_fuzz_test.cc index 865c30f3f8aa..b2cecf913b83 100644 --- a/test/common/router/route_fuzz_test.cc +++ b/test/common/router/route_fuzz_test.cc @@ -72,7 +72,7 @@ DEFINE_PROTO_FUZZER(const test::common::router::RouteTestCase& input) { try { NiceMock stream_info; NiceMock factory_context; - MessageUtil::validate(input.config()); + TestUtility::validate(input.config()); ConfigImpl config(cleanRouteConfig(input.config()), factory_context, true); Http::TestHeaderMapImpl headers = Fuzz::fromHeaders(input.headers()); // It's a precondition of routing that {:authority, :path, x-forwarded-proto} headers exists, diff --git a/test/common/router/router_ratelimit_test.cc b/test/common/router/router_ratelimit_test.cc index f49f4e943de9..207a82a2829f 100644 --- a/test/common/router/router_ratelimit_test.cc +++ b/test/common/router/router_ratelimit_test.cc @@ -31,7 +31,7 @@ namespace { envoy::api::v2::route::RateLimit parseRateLimitFromV2Yaml(const std::string& yaml_string) { envoy::api::v2::route::RateLimit rate_limit; TestUtility::loadFromYaml(yaml_string, rate_limit); - MessageUtil::validate(rate_limit); + TestUtility::validate(rate_limit); return rate_limit; } diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 30d3a671ed0a..0bc871f232b8 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -191,7 +191,7 @@ TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); // No routes at all, no last_updated timestamp @@ -237,7 +237,7 @@ stat_prefix: foo factory_context_, "foo.", *config_provider_manager_); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump2 = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: @@ -310,7 +310,7 @@ route_configuration_name: dynamic-foo-route-config expected_config_dump); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump3 = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump3.DebugString()); @@ -342,7 +342,7 @@ route_configuration_name: dynamic-foo-route-config expected_config_dump); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump4 = - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( *message_ptr); EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump4.DebugString()); } diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index cb599a08b5b8..4753cf9da7be 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -4087,7 +4087,7 @@ TEST(HealthCheckProto, Validation) { service_name: locations path: /healthcheck )EOF"; - EXPECT_THROW_WITH_REGEX(MessageUtil::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException, + EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException, "Proto constraint validation failed.*value must be greater than.*"); } { @@ -4099,7 +4099,7 @@ TEST(HealthCheckProto, Validation) { service_name: locations path: /healthcheck )EOF"; - EXPECT_THROW_WITH_REGEX(MessageUtil::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException, + EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException, "Proto constraint validation failed.*value must be greater than.*"); } { @@ -4111,7 +4111,7 @@ TEST(HealthCheckProto, Validation) { service_name: locations path: /healthcheck )EOF"; - EXPECT_THROW_WITH_REGEX(MessageUtil::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException, + EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException, "Proto constraint validation failed.*value must be greater than.*"); } { @@ -4123,7 +4123,7 @@ TEST(HealthCheckProto, Validation) { service_name: locations path: /healthcheck )EOF"; - EXPECT_THROW_WITH_REGEX(MessageUtil::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException, + EXPECT_THROW_WITH_REGEX(TestUtility::validate(parseHealthCheckFromV2Yaml(yaml)), EnvoyException, "Proto constraint validation failed.*value must be greater than.*"); } } diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 3bb78af76687..c03093e01b33 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -2094,7 +2094,8 @@ class TestNetworkFilterConfigFactory return parent_.createEmptyProtocolOptionsProto(); } Upstream::ProtocolOptionsConfigConstSharedPtr - createProtocolOptionsConfig(const Protobuf::Message& msg) override { + createProtocolOptionsConfig(const Protobuf::Message& msg, + ProtobufMessage::ValidationVisitor&) override { return parent_.createProtocolOptionsConfig(msg); } std::string name() override { CONSTRUCT_ON_FIRST_USE(std::string, "envoy.test.filter"); } @@ -2130,7 +2131,8 @@ class TestHttpFilterConfigFactory : public Server::Configuration::NamedHttpFilte return parent_.createEmptyProtocolOptionsProto(); } Upstream::ProtocolOptionsConfigConstSharedPtr - createProtocolOptionsConfig(const Protobuf::Message& msg) override { + createProtocolOptionsConfig(const Protobuf::Message& msg, + ProtobufMessage::ValidationVisitor&) override { return parent_.createProtocolOptionsConfig(msg); } std::string name() override { CONSTRUCT_ON_FIRST_USE(std::string, "envoy.test.filter"); } diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 8899a22f4e8a..39f23ef1be78 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -101,7 +101,7 @@ class RedisClusterTest : public testing::Test, cluster_callback_ = std::make_shared>(); cluster_.reset(new RedisCluster( cluster_config, - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( config), *this, cm, runtime_, *api_, dns_resolver_, factory_context, std::move(scope), false, cluster_callback_)); diff --git a/test/extensions/filters/http/ext_authz/config_test.cc b/test/extensions/filters/http/ext_authz/config_test.cc index 52942c6a7d6e..29e4a1097de4 100644 --- a/test/extensions/filters/http/ext_authz/config_test.cc +++ b/test/extensions/filters/http/ext_authz/config_test.cc @@ -31,6 +31,7 @@ TEST(HttpExtAuthzConfigTest, CorrectProtoGrpc) { TestUtility::loadFromYaml(yaml, *proto_config); testing::StrictMock context; + EXPECT_CALL(context, messageValidationVisitor()).Times(1); EXPECT_CALL(context, localInfo()).Times(1); EXPECT_CALL(context, clusterManager()).Times(1); EXPECT_CALL(context, runtime()).Times(1); @@ -85,6 +86,7 @@ TEST(HttpExtAuthzConfigTest, CorrectProtoHttp) { ProtobufTypes::MessagePtr proto_config = factory.createEmptyConfigProto(); TestUtility::loadFromYaml(yaml, *proto_config); testing::StrictMock context; + EXPECT_CALL(context, messageValidationVisitor()).Times(1); EXPECT_CALL(context, localInfo()).Times(1); EXPECT_CALL(context, clusterManager()).Times(1); EXPECT_CALL(context, runtime()).Times(1); diff --git a/test/extensions/filters/http/ext_authz/ext_authz_test.cc b/test/extensions/filters/http/ext_authz/ext_authz_test.cc index 3586d9d9b550..9207f2e9c6c7 100644 --- a/test/extensions/filters/http/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/http/ext_authz/ext_authz_test.cc @@ -306,7 +306,7 @@ TEST_F(HttpFilterTest, BadConfig) { envoy::config::filter::http::ext_authz::v2::ExtAuthz proto_config{}; TestUtility::loadFromYaml(filter_config, proto_config); EXPECT_THROW( - MessageUtil::downcastAndValidate( + TestUtility::downcastAndValidate( proto_config), ProtoValidationException); } diff --git a/test/extensions/filters/http/original_src/original_src_config_factory_test.cc b/test/extensions/filters/http/original_src/original_src_config_factory_test.cc index e581bfbd59a4..ea164cacc971 100644 --- a/test/extensions/filters/http/original_src/original_src_config_factory_test.cc +++ b/test/extensions/filters/http/original_src/original_src_config_factory_test.cc @@ -11,6 +11,7 @@ #include "gtest/gtest.h" using testing::Invoke; +using testing::NiceMock; namespace Envoy { namespace Extensions { @@ -27,7 +28,7 @@ TEST(OriginalSrcHttpConfigFactoryTest, TestCreateFactory) { ProtobufTypes::MessagePtr proto_config = factory.createEmptyConfigProto(); TestUtility::loadFromYaml(yaml, *proto_config); - Server::Configuration::MockFactoryContext context; + NiceMock context; Http::FilterFactoryCb cb = factory.createFilterFactoryFromProto(*proto_config, "", context); diff --git a/test/extensions/filters/listener/original_src/original_src_config_factory_test.cc b/test/extensions/filters/listener/original_src/original_src_config_factory_test.cc index 08ec55111ae2..25f758e9e6b2 100644 --- a/test/extensions/filters/listener/original_src/original_src_config_factory_test.cc +++ b/test/extensions/filters/listener/original_src/original_src_config_factory_test.cc @@ -11,6 +11,7 @@ #include "gtest/gtest.h" using testing::Invoke; +using testing::NiceMock; namespace Envoy { namespace Extensions { @@ -28,7 +29,7 @@ TEST(OriginalSrcConfigFactoryTest, TestCreateFactory) { ProtobufTypes::MessagePtr proto_config = factory.createEmptyConfigProto(); TestUtility::loadFromYaml(yaml, *proto_config); - Server::Configuration::MockListenerFactoryContext context; + NiceMock context; Network::ListenerFilterFactoryCb cb = factory.createFilterFactoryFromProto(*proto_config, context); diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index 2b0429d2da77..bd9de61eeb2a 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -134,7 +134,7 @@ class ConnectionManagerTest : public testing::Test { if (!yaml.empty()) { TestUtility::loadFromYaml(yaml, proto_config_); - MessageUtil::validate(proto_config_); + TestUtility::validate(proto_config_); } proto_config_.set_stat_prefix("test"); diff --git a/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc b/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc index 623c6f657cd1..ad648d51fdc4 100644 --- a/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc @@ -24,7 +24,7 @@ envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration parseRouteConfigurationFromV2Yaml(const std::string& yaml) { envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteConfiguration route_config; TestUtility::loadFromYaml(yaml, route_config); - MessageUtil::validate(route_config); + TestUtility::validate(route_config); return route_config; } @@ -32,7 +32,7 @@ envoy::config::filter::network::dubbo_proxy::v2alpha1::DubboProxy parseDubboProxyFromV2Yaml(const std::string& yaml) { envoy::config::filter::network::dubbo_proxy::v2alpha1::DubboProxy config; TestUtility::loadFromYaml(yaml, config); - MessageUtil::validate(config); + TestUtility::validate(config); return config; } diff --git a/test/extensions/filters/network/ext_authz/ext_authz_test.cc b/test/extensions/filters/network/ext_authz/ext_authz_test.cc index 8a344ed162f5..a9d86928e3ed 100644 --- a/test/extensions/filters/network/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/network/ext_authz/ext_authz_test.cc @@ -95,7 +95,7 @@ TEST_F(ExtAuthzFilterTest, BadExtAuthzConfig) { envoy::config::filter::network::ext_authz::v2::ExtAuthz proto_config{}; TestUtility::loadFromJson(json_string, proto_config); - EXPECT_THROW(MessageUtil::downcastAndValidate< + EXPECT_THROW(TestUtility::downcastAndValidate< const envoy::config::filter::network::ext_authz::v2::ExtAuthz&>(proto_config), ProtoValidationException); } diff --git a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc index a8e76e040a2c..0e19ef9e943c 100644 --- a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc @@ -91,7 +91,7 @@ class ThriftConnectionManagerTest : public testing::Test { proto_config_.set_stat_prefix("test"); } else { TestUtility::loadFromYaml(yaml, proto_config_); - MessageUtil::validate(proto_config_); + TestUtility::validate(proto_config_); } proto_config_.set_stat_prefix("test"); diff --git a/test/extensions/filters/network/thrift_proxy/mocks.cc b/test/extensions/filters/network/thrift_proxy/mocks.cc index 3f591919a4fc..ae130a3c42c6 100644 --- a/test/extensions/filters/network/thrift_proxy/mocks.cc +++ b/test/extensions/filters/network/thrift_proxy/mocks.cc @@ -12,7 +12,8 @@ using testing::ReturnRef; namespace Envoy { // Provide a specialization for ProtobufWkt::Struct (for MockFilterConfigFactory) -template <> void MessageUtil::validate(const ProtobufWkt::Struct&) {} +template <> +void MessageUtil::validate(const ProtobufWkt::Struct&, ProtobufMessage::ValidationVisitor&) {} namespace Extensions { namespace NetworkFilters { diff --git a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc index 794cc084e012..d29c89833022 100644 --- a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc @@ -22,7 +22,7 @@ envoy::config::filter::network::thrift_proxy::v2alpha1::RouteConfiguration parseRouteConfigurationFromV2Yaml(const std::string& yaml) { envoy::config::filter::network::thrift_proxy::v2alpha1::RouteConfiguration route_config; TestUtility::loadFromYaml(yaml, route_config); - MessageUtil::validate(route_config); + TestUtility::validate(route_config); return route_config; } diff --git a/test/extensions/resource_monitors/fixed_heap/config_test.cc b/test/extensions/resource_monitors/fixed_heap/config_test.cc index 64f8d6eae058..1c91ae642dd9 100644 --- a/test/extensions/resource_monitors/fixed_heap/config_test.cc +++ b/test/extensions/resource_monitors/fixed_heap/config_test.cc @@ -25,7 +25,8 @@ TEST(FixedHeapMonitorFactoryTest, CreateMonitor) { config.set_max_heap_size_bytes(std::numeric_limits::max()); Event::MockDispatcher dispatcher; Api::ApiPtr api = Api::createApiForTest(); - Server::Configuration::ResourceMonitorFactoryContextImpl context(dispatcher, *api); + Server::Configuration::ResourceMonitorFactoryContextImpl context( + dispatcher, *api, ProtobufMessage::getStrictValidationVisitor()); auto monitor = factory->createResourceMonitor(config, context); EXPECT_NE(monitor, nullptr); } diff --git a/test/extensions/resource_monitors/injected_resource/config_test.cc b/test/extensions/resource_monitors/injected_resource/config_test.cc index 8e1ce2266ac8..00862c91bf1e 100644 --- a/test/extensions/resource_monitors/injected_resource/config_test.cc +++ b/test/extensions/resource_monitors/injected_resource/config_test.cc @@ -28,7 +28,8 @@ TEST(InjectedResourceMonitorFactoryTest, CreateMonitor) { config.set_filename(TestEnvironment::temporaryPath("injected_resource")); Api::ApiPtr api = Api::createApiForTest(); Event::DispatcherPtr dispatcher(api->allocateDispatcher()); - Server::Configuration::ResourceMonitorFactoryContextImpl context(*dispatcher, *api); + Server::Configuration::ResourceMonitorFactoryContextImpl context( + *dispatcher, *api, ProtobufMessage::getStrictValidationVisitor()); Server::ResourceMonitorPtr monitor = factory->createResourceMonitor(config, context); EXPECT_NE(monitor, nullptr); } diff --git a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_test.cc b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_test.cc index b017c90f0c74..a6ac6f290e31 100644 --- a/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_test.cc +++ b/test/extensions/resource_monitors/injected_resource/injected_resource_monitor_test.cc @@ -61,7 +61,8 @@ class InjectedResourceMonitorTest : public testing::Test { std::unique_ptr createMonitor() { envoy::config::resource_monitor::injected_resource::v2alpha::InjectedResourceConfig config; config.set_filename(resource_filename_); - Server::Configuration::ResourceMonitorFactoryContextImpl context(*dispatcher_, *api_); + Server::Configuration::ResourceMonitorFactoryContextImpl context( + *dispatcher_, *api_, ProtobufMessage::getStrictValidationVisitor()); return std::make_unique(config, context); } diff --git a/test/extensions/retry/priority/previous_priorities/config_test.cc b/test/extensions/retry/priority/previous_priorities/config_test.cc index 9b2df9346020..47f425dfee60 100644 --- a/test/extensions/retry/priority/previous_priorities/config_test.cc +++ b/test/extensions/retry/priority/previous_priorities/config_test.cc @@ -31,7 +31,8 @@ class RetryPriorityTest : public testing::Test { // by that method is compatible with the downcast in createRetryPriority. auto empty = factory->createEmptyConfigProto(); empty->MergeFrom(config); - retry_priority_ = factory->createRetryPriority(*empty, 3); + retry_priority_ = + factory->createRetryPriority(*empty, ProtobufMessage::getStrictValidationVisitor(), 3); original_priority_load_ = Upstream::HealthyAndDegradedLoad{original_healthy_priority_load, original_degraded_priority_load}; } diff --git a/test/extensions/transport_sockets/alts/config_test.cc b/test/extensions/transport_sockets/alts/config_test.cc index 34fcd34ae645..68c602ea50fd 100644 --- a/test/extensions/transport_sockets/alts/config_test.cc +++ b/test/extensions/transport_sockets/alts/config_test.cc @@ -23,7 +23,7 @@ namespace Alts { namespace { TEST(UpstreamAltsConfigTest, CreateSocketFactory) { - MockTransportSocketFactoryContext factory_context; + NiceMock factory_context; Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); UpstreamAltsTransportSocketConfigFactory factory; @@ -43,7 +43,7 @@ TEST(UpstreamAltsConfigTest, CreateSocketFactory) { } TEST(DownstreamAltsConfigTest, CreateSocketFactory) { - MockTransportSocketFactoryContext factory_context; + NiceMock factory_context; Singleton::ManagerImpl singleton_manager{Thread::threadFactoryForTest()}; EXPECT_CALL(factory_context, singletonManager()).WillRepeatedly(ReturnRef(singleton_manager)); DownstreamAltsTransportSocketConfigFactory factory; diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index ef82b26fff43..92b78d793cd8 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -1074,7 +1074,7 @@ TEST_F(ServerContextConfigImplTest, MultiSdsConfig) { tls_context.mutable_common_tls_context()->add_tls_certificate_sds_secret_configs(); tls_context.mutable_common_tls_context()->add_tls_certificate_sds_secret_configs(); EXPECT_THROW_WITH_REGEX( - MessageUtil::validate(tls_context), + TestUtility::validate(tls_context), EnvoyException, "Proto constraint validation failed"); } diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index f194bffceca0..1b7f0b07b237 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -142,7 +142,9 @@ class MockRetryPriorityFactory : public RetryPriorityFactory { public: MockRetryPriorityFactory(const MockRetryPriority& retry_priority) : retry_priority_(retry_priority) {} - RetryPrioritySharedPtr createRetryPriority(const Protobuf::Message&, uint32_t) override { + RetryPrioritySharedPtr createRetryPriority(const Protobuf::Message&, + ProtobufMessage::ValidationVisitor&, + uint32_t) override { return std::make_shared>(retry_priority_); } diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 7e33eeff7d1e..3da54288da7f 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -499,8 +499,7 @@ class TestUtility { template static inline MessageType anyConvert(const ProtobufWkt::Any& message) { - return MessageUtil::anyConvert(message, - ProtobufMessage::getStrictValidationVisitor()); + return MessageUtil::anyConvert(message); } template @@ -515,6 +514,16 @@ class TestUtility { ProtobufMessage::getStrictValidationVisitor()); } + template static void validate(const MessageType& message) { + return MessageUtil::validate(message, ProtobufMessage::getStrictValidationVisitor()); + } + + template + static const MessageType& downcastAndValidate(const Protobuf::Message& config) { + return MessageUtil::downcastAndValidate( + config, ProtobufMessage::getStrictValidationVisitor()); + } + static void jsonConvert(const Protobuf::Message& source, Protobuf::Message& dest) { // Explicit round-tripping to support conversions inside tests between arbitrary messages as a // convenience. diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index c75053792f6a..6922d26cea2f 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -165,7 +165,7 @@ bool RouterCheckTool::compareEntries(const std::string& expected_routes) { auto api = Api::createApiForTest(*stats); const std::string contents = api->fileSystem().fileReadToEnd(expected_routes); TestUtility::loadFromFile(expected_routes, validation_config, *api); - MessageUtil::validate(validation_config); + TestUtility::validate(validation_config); bool no_failures = true; for (const envoy::RouterCheckToolSchema::ValidationItem& check_config : diff --git a/test/tools/schema_validator/validator.cc b/test/tools/schema_validator/validator.cc index 80b2a4137061..fee3cde141e7 100644 --- a/test/tools/schema_validator/validator.cc +++ b/test/tools/schema_validator/validator.cc @@ -59,13 +59,13 @@ void Validator::validate(const std::string& config_path, Schema::Type schema_typ case Schema::Type::DiscoveryResponse: { envoy::api::v2::DiscoveryResponse discovery_response_config; TestUtility::loadFromFile(config_path, discovery_response_config, *api_); - MessageUtil::validate(discovery_response_config); + TestUtility::validate(discovery_response_config); break; } case Schema::Type::Route: { envoy::api::v2::RouteConfiguration route_config; TestUtility::loadFromFile(config_path, route_config, *api_); - MessageUtil::validate(route_config); + TestUtility::validate(route_config); break; } default: From b2da45af1c86efc5056b8eb8f18644954be43809 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 22 Aug 2019 15:07:01 -0400 Subject: [PATCH 430/542] http: forwarding x-forwarded-proto from trusted proxies (#7995) Trusting the x-forwarded-proto header from trusted proxies. If Envoy is operating as an edge proxy but has a trusted hop in front, the trusted proxy should be allowed to set x-forwarded-proto and its x-forwarded-proto should be preserved. Guarded by envoy.reloadable_features.trusted_forwarded_proto, default on. Risk Level: Medium (L7 header changes) but guarded Testing: new unit tests Docs Changes: n/a Release Notes: inline Fixes #4496 Signed-off-by: Alyssa Wilk --- docs/root/intro/version_history.rst | 1 + source/common/http/conn_manager_utility.cc | 24 +++++++-- source/common/runtime/runtime_features.cc | 1 + test/common/http/BUILD | 1 + test/common/http/conn_manager_utility_test.cc | 45 ++++++++++++++++ test/test_common/BUILD | 15 ++++++ test/test_common/test_runtime.h | 53 +++++++++++++++++++ 7 files changed, 136 insertions(+), 4 deletions(-) create mode 100644 test/test_common/test_runtime.h diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 2a049c36c740..ade42d943ef2 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -25,6 +25,7 @@ Version history * grpc-json: added support for :ref:`ignoring unknown query parameters`. * header to metadata: added :ref:`PROTOBUF_VALUE ` and :ref:`ValueEncode ` to support protobuf Value and Base64 encoding. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. +* http: changed Envoy to forward existing x-forwarded-proto from upstream trusted proxies. Guarded by `envoy.reloadable_features.trusted_forwarded_proto` which defaults true. * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. diff --git a/source/common/http/conn_manager_utility.cc b/source/common/http/conn_manager_utility.cc index 5f9d541693b6..9c1b8a3b1ea6 100644 --- a/source/common/http/conn_manager_utility.cc +++ b/source/common/http/conn_manager_utility.cc @@ -85,6 +85,9 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest Network::Address::InstanceConstSharedPtr final_remote_address; bool single_xff_address; const uint32_t xff_num_trusted_hops = config.xffNumTrustedHops(); + const bool trusted_forwarded_proto = + Runtime::runtimeFeatureEnabled("envoy.reloadable_features.trusted_forwarded_proto"); + if (config.useRemoteAddress()) { single_xff_address = request_headers.ForwardedFor() == nullptr; // If there are any trusted proxies in front of this Envoy instance (as indicated by @@ -107,8 +110,21 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest Utility::appendXff(request_headers, *connection.remoteAddress()); } } - request_headers.insertForwardedProto().value().setReference( - connection.ssl() ? Headers::get().SchemeValues.Https : Headers::get().SchemeValues.Http); + if (trusted_forwarded_proto) { + // If the prior hop is not a trusted proxy, overwrite any x-forwarded-proto value it set as + // untrusted. Alternately if no x-forwarded-proto header exists, add one. + if (xff_num_trusted_hops == 0 || request_headers.ForwardedProto() == nullptr) { + request_headers.insertForwardedProto().value().setReference( + connection.ssl() ? Headers::get().SchemeValues.Https + : Headers::get().SchemeValues.Http); + } + } else { + // Previously, before the trusted_forwarded_proto logic, Envoy would always overwrite the + // x-forwarded-proto header even if it was set by a trusted proxy. This code path is + // deprecated and will be removed. + request_headers.insertForwardedProto().value().setReference( + connection.ssl() ? Headers::get().SchemeValues.Https : Headers::get().SchemeValues.Http); + } } else { // If we are not using remote address, attempt to pull a valid IPv4 or IPv6 address out of XFF. // If we find one, it will be used as the downstream address for logging. It may or may not be @@ -118,8 +134,8 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest single_xff_address = ret.single_address_; } - // If we didn't already replace x-forwarded-proto because we are using the remote address, and - // remote hasn't set it (trusted proxy), we set it, since we then use this for setting scheme. + // If the x-forwarded-proto header is not set, set it here, since Envoy uses it for determining + // scheme and communicating it upstream. if (!request_headers.ForwardedProto()) { request_headers.insertForwardedProto().value().setReference( connection.ssl() ? Headers::get().SchemeValues.Https : Headers::get().SchemeValues.Http); diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index e1dc2a53bc91..db42217c8233 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -26,6 +26,7 @@ constexpr const char* runtime_features[] = { // Enabled "envoy.reloadable_features.test_feature_true", "envoy.reloadable_features.buffer_filter_populate_content_length", + "envoy.reloadable_features.trusted_forwarded_proto", }; // This is a list of configuration fields which are disallowed by default in Envoy diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 8e13f1625db3..7982b15c462f 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -234,6 +234,7 @@ envoy_cc_test( "//test/mocks/runtime:runtime_mocks", "//test/mocks/ssl:ssl_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index c228f99e3c30..025a21ab1caf 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -13,6 +13,7 @@ #include "test/mocks/runtime/mocks.h" #include "test/mocks/ssl/mocks.h" #include "test/test_common/printers.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -226,6 +227,50 @@ TEST_F(ConnectionManagerUtilityTest, SkipXffAppendPassThruUseRemoteAddress) { EXPECT_EQ("198.51.100.1", headers.ForwardedFor()->value().getStringView()); } +TEST_F(ConnectionManagerUtilityTest, ForwardedProtoLegacyBehavior) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.trusted_forwarded_proto", "false"}}); + + ON_CALL(config_, useRemoteAddress()).WillByDefault(Return(true)); + ON_CALL(config_, xffNumTrustedHops()).WillByDefault(Return(1)); + EXPECT_CALL(config_, skipXffAppend()).WillOnce(Return(true)); + connection_.remote_address_ = std::make_shared("12.12.12.12"); + ON_CALL(config_, useRemoteAddress()).WillByDefault(Return(true)); + TestHeaderMapImpl headers{{"x-forwarded-proto", "https"}}; + + callMutateRequestHeaders(headers, Protocol::Http2); + EXPECT_EQ("http", headers.ForwardedProto()->value().getStringView()); +} + +TEST_F(ConnectionManagerUtilityTest, PreserveForwardedProtoWhenInternal) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.trusted_forwarded_proto", "true"}}); + + ON_CALL(config_, useRemoteAddress()).WillByDefault(Return(true)); + ON_CALL(config_, xffNumTrustedHops()).WillByDefault(Return(1)); + EXPECT_CALL(config_, skipXffAppend()).WillOnce(Return(true)); + connection_.remote_address_ = std::make_shared("12.12.12.12"); + ON_CALL(config_, useRemoteAddress()).WillByDefault(Return(true)); + TestHeaderMapImpl headers{{"x-forwarded-proto", "https"}}; + + callMutateRequestHeaders(headers, Protocol::Http2); + EXPECT_EQ("https", headers.ForwardedProto()->value().getStringView()); +} + +TEST_F(ConnectionManagerUtilityTest, OverwriteForwardedProtoWhenExternal) { + ON_CALL(config_, useRemoteAddress()).WillByDefault(Return(true)); + ON_CALL(config_, xffNumTrustedHops()).WillByDefault(Return(0)); + connection_.remote_address_ = std::make_shared("127.0.0.1"); + TestHeaderMapImpl headers{{"x-forwarded-proto", "https"}}; + Network::Address::Ipv4Instance local_address("10.3.2.1"); + ON_CALL(config_, localAddress()).WillByDefault(ReturnRef(local_address)); + + callMutateRequestHeaders(headers, Protocol::Http2); + EXPECT_EQ("http", headers.ForwardedProto()->value().getStringView()); +} + // Verify internal request and XFF is set when we are using remote address and the address is // internal according to user configuration. TEST_F(ConnectionManagerUtilityTest, UseRemoteAddressWhenUserConfiguredRemoteAddress) { diff --git a/test/test_common/BUILD b/test/test_common/BUILD index 911ae90a26bd..e1bb1467dc19 100644 --- a/test/test_common/BUILD +++ b/test/test_common/BUILD @@ -117,6 +117,21 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "test_runtime_lib", + hdrs = ["test_runtime.h"], + deps = [ + "//source/common/runtime:runtime_lib", + "//source/common/stats:isolated_store_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/init:init_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/protobuf:protobuf_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/mocks/thread_local:thread_local_mocks", + ], +) + envoy_cc_test_library( name = "thread_factory_for_test_lib", srcs = ["thread_factory_for_test.cc"], diff --git a/test/test_common/test_runtime.h b/test/test_common/test_runtime.h new file mode 100644 index 000000000000..ba9900578fa7 --- /dev/null +++ b/test/test_common/test_runtime.h @@ -0,0 +1,53 @@ +// A simple test utility to easily allow for runtime feature overloads in unit tests. +// +// As long as this class is in scope one can do runtime feature overrides: +// +// TestScopedRuntime scoped_runtime; +// Runtime::LoaderSingleton::getExisting()->mergeValues( +// {{"envoy.reloadable_features.test_feature_true", "false"}}); +// +// As long as a TestScopedRuntime exists, Runtime::LoaderSingleton::getExisting()->mergeValues() +// can safely be called to override runtime values. + +#pragma once + +#include "common/runtime/runtime_impl.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/mocks/event/mocks.h" +#include "test/mocks/init/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/mocks/thread_local/mocks.h" + +#include "gmock/gmock.h" + +namespace Envoy { + +// TODO(alyssawilk) move existing runtime tests over to using this. +class TestScopedRuntime { +public: + TestScopedRuntime() : api_(Api::createApiForTest()) { + envoy::config::bootstrap::v2::LayeredRuntime config; + // The existence of an admin layer is required for mergeValues() to work. + config.add_layers()->mutable_admin_layer(); + + loader_ = std::make_unique( + std::make_unique(dispatcher_, tls_, config, local_info_, init_manager_, + store_, generator_, validation_visitor_, *api_)); + } + +private: + Event::MockDispatcher dispatcher_; + testing::NiceMock tls_; + Stats::IsolatedStoreImpl store_; + Runtime::MockRandomGenerator generator_; + Api::ApiPtr api_; + testing::NiceMock local_info_; + Init::MockManager init_manager_; + testing::NiceMock validation_visitor_; + std::unique_ptr loader_; +}; + +} // namespace Envoy From f12adacdcdae9509baee086f7dfc24019f9bcaf3 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 22 Aug 2019 15:40:18 -0400 Subject: [PATCH 431/542] build: adding an option to hard-fail when deprecated config is used. (#7962) Adding a build option to default all deprecated protos off, and using it on the debug build. Risk Level: Low Testing: new UT Docs Changes: inline Release Notes: n/a Fixes #7548 Signed-off-by: Alyssa Wilk --- CONTRIBUTING.md | 6 +++++- bazel/BUILD | 5 +++++ bazel/README.md | 2 ++ bazel/envoy_internal.bzl | 3 +++ ci/do_ci.sh | 1 + source/common/runtime/runtime_impl.cc | 5 +++++ test/common/protobuf/utility_test.cc | 19 ++++++++++--------- test/common/runtime/runtime_impl_test.cc | 9 +++++++++ .../redis/redis_cluster_integration_test.cc | 4 +++- .../http/cors/cors_filter_integration_test.cc | 6 +++++- .../redis_proxy_integration_test.cc | 10 +++++++--- test/test_common/utility.h | 13 +++++++++++++ 12 files changed, 68 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 345192d66ddf..23e9bd13a6e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,7 +72,11 @@ maximize the chances of your PR being merged. could convert from the earlier API to the new API. A field may be deprecated if this tool would be able to perform the conversion. For example, removing a field to describe HTTP/2 window settings is valid if a more comprehensive - HTTP/2 protocol options field is being introduced to replace it. + HTTP/2 protocol options field is being introduced to replace it. The PR author + deprecating the old configuration is responsible for updating all tests and + canonical configuration, or guarding them with the DEPRECATED_FEATURE_TEST() macro. + This will be validated by the bazel.compile_time_options target, which will hard-fail when + deprecated configuration is used. * For configuration deprecations that are not covered by the above semantic replacement policy, any deprecation will only take place after community consultation on mailing lists, Slack and GitHub, over the period of diff --git a/bazel/BUILD b/bazel/BUILD index d449250d2390..e60a4ae9c06c 100755 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -120,6 +120,11 @@ config_setting( values = {"define": "object_dump_on_signal_trace=disabled"}, ) +config_setting( + name = "disable_deprecated_features", + values = {"define": "deprecated_features=disabled"}, +) + config_setting( name = "disable_hot_restart", values = {"define": "hot_restart=disabled"}, diff --git a/bazel/README.md b/bazel/README.md index 2631fbec01f3..24340fdd0fce 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -407,6 +407,8 @@ The following optional features can be disabled on the Bazel build command-line: * Backtracing on signals with `--define signal_trace=disabled` * Active stream state dump on signals with `--define signal_trace=disabled` or `--define disable_object_dump_on_signal_trace=disabled` * tcmalloc with `--define tcmalloc=disabled` +* deprecated features with `--define deprecated_features=disabled` + ## Enabling optional features diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 325ea94f5596..2562390311de 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -56,6 +56,9 @@ def envoy_copts(repository, test = False): }) + select({ repository + "//bazel:disable_object_dump_on_signal_trace": [], "//conditions:default": ["-DENVOY_OBJECT_TRACE_ON_DUMP"], + }) + select({ + repository + "//bazel:disable_deprecated_features": ["-DENVOY_DISABLE_DEPRECATED_FEATURES"], + "//conditions:default": [], }) + select({ repository + "//bazel:enable_log_debug_assert_in_release": ["-DENVOY_LOG_DEBUG_ASSERT_IN_RELEASE"], "//conditions:default": [], diff --git a/ci/do_ci.sh b/ci/do_ci.sh index d526c87d1721..db8bc0bdbd24 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -197,6 +197,7 @@ elif [[ "$CI_TARGET" == "bazel.compile_time_options" ]]; then --define log_debug_assert_in_release=enabled \ --define quiche=enabled \ --define path_normalization_by_default=true \ + --define deprecated_features=disabled \ " setup_clang_libcxx_toolchain # This doesn't go into CI but is available for developer convenience. diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index b668b9aa7187..b1f4494b27a7 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -182,9 +182,14 @@ bool SnapshotImpl::deprecatedFeatureEnabled(const std::string& key) const { // If either disallowed by default or configured off, the feature is not enabled. return false; } + // The feature is allowed. It is assumed this check is called when the feature // is about to be used, so increment the feature use stat. stats_.deprecated_feature_use_.inc(); +#ifdef ENVOY_DISABLE_DEPRECATED_FEATURES + return false; +#endif + return true; } diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 050fc0485e02..4371e4ecc9e8 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -511,7 +511,7 @@ TEST_F(DeprecatedFieldsTest, NoErrorWhenDeprecatedFieldsUnused) { EXPECT_EQ(0, runtime_deprecated_feature_use_.value()); } -TEST_F(DeprecatedFieldsTest, IndividualFieldDeprecated) { +TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(IndividualFieldDeprecated)) { envoy::test::deprecation_test::Base base; base.set_is_deprecated("foo"); // Non-fatal checks for a deprecated field should log rather than throw an exception. @@ -522,7 +522,7 @@ TEST_F(DeprecatedFieldsTest, IndividualFieldDeprecated) { } // Use of a deprecated and disallowed field should result in an exception. -TEST_F(DeprecatedFieldsTest, IndividualFieldDisallowed) { +TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(IndividualFieldDisallowed)) { envoy::test::deprecation_test::Base base; base.set_is_deprecated_fatal("foo"); EXPECT_THROW_WITH_REGEX( @@ -530,7 +530,8 @@ TEST_F(DeprecatedFieldsTest, IndividualFieldDisallowed) { "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'"); } -TEST_F(DeprecatedFieldsTest, IndividualFieldDisallowedWithRuntimeOverride) { +TEST_F(DeprecatedFieldsTest, + DEPRECATED_FEATURE_TEST(IndividualFieldDisallowedWithRuntimeOverride)) { envoy::test::deprecation_test::Base base; base.set_is_deprecated_fatal("foo"); @@ -552,7 +553,7 @@ TEST_F(DeprecatedFieldsTest, IndividualFieldDisallowedWithRuntimeOverride) { EXPECT_EQ(1, runtime_deprecated_feature_use_.value()); } -TEST_F(DeprecatedFieldsTest, DisallowViaRuntime) { +TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(DisallowViaRuntime)) { envoy::test::deprecation_test::Base base; base.set_is_deprecated("foo"); @@ -574,7 +575,7 @@ TEST_F(DeprecatedFieldsTest, DisallowViaRuntime) { // Note that given how Envoy config parsing works, the first time we hit a // 'fatal' error and throw, we won't log future warnings. That said, this tests // the case of the warning occurring before the fatal error. -TEST_F(DeprecatedFieldsTest, MixOfFatalAndWarnings) { +TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(MixOfFatalAndWarnings)) { envoy::test::deprecation_test::Base base; base.set_is_deprecated("foo"); base.set_is_deprecated_fatal("foo"); @@ -587,7 +588,7 @@ TEST_F(DeprecatedFieldsTest, MixOfFatalAndWarnings) { } // Present (unused) deprecated messages should be detected as deprecated. -TEST_F(DeprecatedFieldsTest, MessageDeprecated) { +TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(MessageDeprecated)) { envoy::test::deprecation_test::Base base; base.mutable_deprecated_message(); EXPECT_LOG_CONTAINS( @@ -596,7 +597,7 @@ TEST_F(DeprecatedFieldsTest, MessageDeprecated) { EXPECT_EQ(1, runtime_deprecated_feature_use_.value()); } -TEST_F(DeprecatedFieldsTest, InnerMessageDeprecated) { +TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(InnerMessageDeprecated)) { envoy::test::deprecation_test::Base base; base.mutable_not_deprecated_message()->set_inner_not_deprecated("foo"); // Checks for a non-deprecated field shouldn't trigger warnings @@ -612,7 +613,7 @@ TEST_F(DeprecatedFieldsTest, InnerMessageDeprecated) { } // Check that repeated sub-messages get validated. -TEST_F(DeprecatedFieldsTest, SubMessageDeprecated) { +TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(SubMessageDeprecated)) { envoy::test::deprecation_test::Base base; base.add_repeated_message(); base.add_repeated_message()->set_inner_deprecated("foo"); @@ -626,7 +627,7 @@ TEST_F(DeprecatedFieldsTest, SubMessageDeprecated) { } // Check that deprecated repeated messages trigger -TEST_F(DeprecatedFieldsTest, RepeatedMessageDeprecated) { +TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(RepeatedMessageDeprecated)) { envoy::test::deprecation_test::Base base; base.add_deprecated_repeated_message(); diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 82fedad731af..55d9bf662744 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -179,6 +179,15 @@ TEST_F(DiskLoaderImplTest, All) { // test_feature_false is not in runtime_features.cc and so is false by default. EXPECT_EQ(false, snapshot->runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); + // Deprecation +#ifdef ENVOY_DISABLE_DEPRECATED_FEATURES + EXPECT_EQ(false, snapshot->deprecatedFeatureEnabled("random_string_should_be_enabled")); +#else + EXPECT_EQ(true, snapshot->deprecatedFeatureEnabled("random_string_should_be_enabled")); +#endif + EXPECT_EQ(false, snapshot->deprecatedFeatureEnabled( + "envoy.deprecated_features.deprecated.proto:is_deprecated_fatal")); + // Feature defaults via helper function. EXPECT_EQ(false, runtimeFeatureEnabled("envoy.reloadable_features.test_feature_false")); EXPECT_EQ(true, runtimeFeatureEnabled("envoy.reloadable_features.test_feature_true")); diff --git a/test/extensions/clusters/redis/redis_cluster_integration_test.cc b/test/extensions/clusters/redis/redis_cluster_integration_test.cc index 91aecc412a3d..000928ef2441 100644 --- a/test/extensions/clusters/redis/redis_cluster_integration_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_integration_test.cc @@ -36,7 +36,9 @@ const std::string& testConfig() { name: envoy.redis_proxy config: stat_prefix: redis_stats - cluster: cluster_0 + prefix_routes: + catch_all_route: + cluster: cluster_0 settings: op_timeout: 5s clusters: diff --git a/test/extensions/filters/http/cors/cors_filter_integration_test.cc b/test/extensions/filters/http/cors/cors_filter_integration_test.cc index bca2bcff0c63..6c78c1892313 100644 --- a/test/extensions/filters/http/cors/cors_filter_integration_test.cc +++ b/test/extensions/filters/http/cors/cors_filter_integration_test.cc @@ -34,7 +34,11 @@ class CorsFilterIntegrationTest : public testing::TestWithParamadd_routes(); route->mutable_match()->set_prefix("/no-cors"); route->mutable_route()->set_cluster("cluster_0"); - route->mutable_route()->mutable_cors()->mutable_enabled()->set_value(false); + route->mutable_route() + ->mutable_cors() + ->mutable_filter_enabled() + ->mutable_default_value() + ->set_numerator(0); } { diff --git a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc index c9f8c2ce10f7..c8c2ecc5e74c 100644 --- a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc +++ b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc @@ -56,7 +56,9 @@ const std::string CONFIG = R"EOF( name: envoy.redis_proxy config: stat_prefix: redis_stats - cluster: cluster_0 + prefix_routes: + catch_all_route: + cluster: cluster_0 settings: op_timeout: 5s )EOF"; @@ -149,7 +151,8 @@ const std::string CONFIG_WITH_ROUTES_BASE = R"EOF( const std::string CONFIG_WITH_ROUTES = CONFIG_WITH_ROUTES_BASE + R"EOF( prefix_routes: - catch_all_cluster: cluster_0 + catch_all_route: + cluster: cluster_0 routes: - prefix: "foo:" cluster: cluster_1 @@ -250,7 +253,8 @@ const std::string CONFIG_WITH_ROUTES_AND_AUTH_PASSWORDS = R"EOF( settings: op_timeout: 5s prefix_routes: - catch_all_cluster: cluster_0 + catch_all_route: + cluster: cluster_0 routes: - prefix: "foo:" cluster: cluster_1 diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 3da54288da7f..4ff0ac413097 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -105,6 +105,19 @@ namespace Envoy { } \ } while (false) +// A convenience macro for testing Envoy deprecated features. This will disable the test when +// tests are built with --define deprecated_features=disabled to avoid the hard-failure mode for +// deprecated features. Sample usage is: +// +// TEST_F(FixtureName, DEPRECATED_FEATURE_TEST(TestName)) { +// ... +// } +#ifndef ENVOY_DISABLE_DEPRECATED_FEATURES +#define DEPRECATED_FEATURE_TEST(X) X +#else +#define DEPRECATED_FEATURE_TEST(X) DISABLED_##X +#endif + // Random number generator which logs its seed to stderr. To repeat a test run with a non-zero seed // one can run the test with --test_arg=--gtest_random_seed=[seed] class TestRandomGenerator { From 797d58f4f44e1f373735a5cb05c65fa588c917a9 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 22 Aug 2019 22:59:35 +0200 Subject: [PATCH 432/542] envoy_cc_library: add export of foo_with_external_headers (#8005) Add a parallel native.cc_library to envoy_cc_library for external projects that consume Envoy's libraries. This allows the consuming project to disambiguate overlapping include paths when repository overlaying is used, as it can now include envoy headers via external/envoy/... Risk Level: Low Testing: N/A Signed-off-by: Otto van der Schaaf --- bazel/envoy_library.bzl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index b0f54ea64eb4..352dbfa5efd4 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -70,6 +70,17 @@ def envoy_cc_library( strip_include_prefix = strip_include_prefix, ) + # Intended for usage by external consumers. This allows them to disambiguate + # include paths via `external/envoy...` + native.cc_library( + name = name + "_with_external_headers", + hdrs = hdrs, + copts = envoy_copts(repository) + copts, + visibility = visibility, + deps = [":" + name], + strip_include_prefix = strip_include_prefix, + ) + # Used to specify a library that only builds on POSIX def envoy_cc_posix_library(name, srcs = [], hdrs = [], **kargs): envoy_cc_library( From 57d48a3f0e8d191bda6e5f6f6570783c11d06b13 Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 22 Aug 2019 19:13:42 -0400 Subject: [PATCH 433/542] ci: add fuzz test targets to ci (#7949) Builds fuzz targets with asan+libfuzzer and runs them against their corpora. Our native bazel builds work, this PR integrates the asan+libfuzzer builds in to CI. The fuzz target binaries will be in your envoy docker build directory. Invoke with the following for all fuzz targets, or a specified one. ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.fuzz' ./ci/run_envoy_docker.sh './ci/do_ci.sh bazel.fuzz //test/common/common:utility_fuzz_test' Risk level: low Signed-off-by: Asra Ali asraa@google.com Signed-off-by: Asra Ali --- .azure-pipelines/linux.yml | 2 ++ .bazelrc | 4 +++- bazel/envoy_test.bzl | 4 ++-- ci/README.md | 2 ++ ci/do_ci.sh | 7 +++++++ 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index 842e1c992e74..f3337072613e 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -10,6 +10,8 @@ jobs: CI_TARGET: 'bazel.gcc' compile_time_options: CI_TARGET: 'bazel.compile_time_options' + fuzz: + CI_TARGET: 'bazel.fuzz' dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. timeoutInMinutes: 360 pool: diff --git a/.bazelrc b/.bazelrc index a74e3e350c02..01d32bdd3fd3 100644 --- a/.bazelrc +++ b/.bazelrc @@ -145,4 +145,6 @@ build:remote-ci --remote_executor=grpcs://remotebuildexecution.googleapis.com build:asan-fuzzer --config=asan build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -build:asan-fuzzer --copt=-fsanitize-coverage=trace-pc-guard \ No newline at end of file +build:asan-fuzzer --copt=-fsanitize-coverage=trace-pc-guard +# Remove UBSAN halt_on_error to avoid crashing on protobuf errors. +build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1 \ No newline at end of file diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index 91d65b8803e5..4e7e91e0070b 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -115,7 +115,7 @@ def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs): tags = ["manual"] + tags, ) - native.cc_binary( + native.cc_test( name = name + "_with_libfuzzer", copts = envoy_copts("@envoy", test = True), linkopts = ["-fsanitize=fuzzer"] + _envoy_test_linkopts(), @@ -123,7 +123,7 @@ def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs): testonly = 1, data = [corpus_name], deps = [":" + test_lib_name], - tags = ["manual"] + tags, + tags = ["manual", "fuzzer"] + tags, ) # Envoy C++ test targets should be specified with this function. diff --git a/ci/README.md b/ci/README.md index 1f3c4301fb30..04864d581498 100644 --- a/ci/README.md +++ b/ci/README.md @@ -98,6 +98,8 @@ The `./ci/run_envoy_docker.sh './ci/do_ci.sh '` targets are: * `bazel.coverity` — build Envoy static binary and run Coverity Scan static analysis. * `bazel.tsan` — build and run tests under `-c dbg --config=clang-tsan` with clang. * `bazel.tsan ` — build and run a specified test or test dir under `-c dbg --config=clang-tsan` with clang. +* `bazel.fuzz` — build and run fuzz tests under `-c dbg --config=asan-fuzzer` with clang. +* `bazel.fuzz ` — build and run a specified fuzz test or test dir under `-c dbg --config=asan-fuzzer` with clang. If specifying a single fuzz test, must use the full target name with "_with_libfuzzer" for ``. * `bazel.compile_time_options` — build Envoy and run tests with various compile-time options toggled to their non-default state, to ensure they still build. * `bazel.compile_time_options ` — build Envoy and run a specified test or test dir with various compile-time options toggled to their non-default state, to ensure they still build. * `bazel.clang_tidy` — build and run clang-tidy over all source files. diff --git a/ci/do_ci.sh b/ci/do_ci.sh index db8bc0bdbd24..f94d96fc8cdd 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -253,6 +253,13 @@ elif [[ "$CI_TARGET" == "bazel.coverity" ]]; then "${ENVOY_BUILD_DIR}"/envoy-coverity-output.tgz \ "${ENVOY_DELIVERY_DIR}"/envoy-coverity-output.tgz exit 0 +elif [[ "$CI_TARGET" == "bazel.fuzz" ]]; then + setup_clang_toolchain + FUZZ_TEST_TARGETS="$(bazel query "attr('tags','fuzzer',${TEST_TARGETS})")" + echo "bazel ASAN libFuzzer build with fuzz tests ${FUZZ_TEST_TARGETS}" + echo "Building envoy fuzzers and executing 100 fuzz iterations..." + bazel_with_collection test ${BAZEL_BUILD_OPTIONS} --config=asan-fuzzer ${FUZZ_TEST_TARGETS} --test_arg="-runs=10" + exit 0 elif [[ "$CI_TARGET" == "fix_format" ]]; then echo "fix_format..." ./tools/check_format.py fix From 9a3a234c3cd1f7b6707746d84eb74414b1990c73 Mon Sep 17 00:00:00 2001 From: Ismo Puustinen Date: Fri, 23 Aug 2019 02:16:54 +0300 Subject: [PATCH 434/542] tls: support BoringSSL private key async functionality (#6326) This PR adds BoringSSL private key API abstraction, as discussed in #6248. All comments and discussion is welcomed to get the API sufficient for most private key API tasks. The PR contains the proposed API and the way how it can be used from ssl_socket.h. Also there is some code showing how the PrivateKeyMethodProvider is coming from TLS certificate config. Two example private key method providers are included in the tests. Description: tls: support BoringSSL private key async functionality Risk Level: medium Testing: two basic private key provider implementation Docs Changes: TLS arch doc, cert.proto doc Signed-off-by: Ismo Puustinen --- CODEOWNERS | 2 + api/envoy/api/v2/auth/cert.proto | 27 + docs/root/extending/extending.rst | 1 + .../root/intro/arch_overview/security/ssl.rst | 4 + include/envoy/ssl/BUILD | 3 + include/envoy/ssl/context_manager.h | 7 + include/envoy/ssl/private_key/BUILD | 35 ++ include/envoy/ssl/private_key/private_key.h | 85 +++ .../ssl/private_key/private_key_callbacks.h | 25 + .../ssl/private_key/private_key_config.h | 22 + include/envoy/ssl/tls_certificate_config.h | 6 + source/common/ssl/BUILD | 2 + .../common/ssl/tls_certificate_config_impl.cc | 19 +- .../common/ssl/tls_certificate_config_impl.h | 9 +- source/extensions/transport_sockets/tls/BUILD | 4 + .../tls/context_config_impl.cc | 8 +- .../transport_sockets/tls/context_impl.cc | 95 ++- .../transport_sockets/tls/context_impl.h | 7 + .../tls/context_manager_impl.h | 7 + .../transport_sockets/tls/private_key/BUILD | 26 + .../private_key/private_key_manager_impl.cc | 30 + .../private_key/private_key_manager_impl.h | 23 + .../transport_sockets/tls/ssl_socket.cc | 58 +- .../transport_sockets/tls/ssl_socket.h | 14 +- source/server/ssl_context_manager.cc | 2 + test/common/secret/sds_api_test.cc | 4 +- .../common/secret/secret_manager_impl_test.cc | 6 +- test/extensions/transport_sockets/tls/BUILD | 25 + .../tls/context_impl_test.cc | 119 ++++ .../transport_sockets/tls/ssl_socket_test.cc | 560 +++++++++++++++++- .../tls/test_private_key_method_provider.cc | 377 ++++++++++++ .../tls/test_private_key_method_provider.h | 96 +++ test/mocks/event/mocks.h | 1 + test/mocks/ssl/mocks.cc | 6 + test/mocks/ssl/mocks.h | 24 + tools/check_format.py | 1 - tools/spelling_dictionary.txt | 2 + 37 files changed, 1679 insertions(+), 63 deletions(-) create mode 100644 include/envoy/ssl/private_key/BUILD create mode 100644 include/envoy/ssl/private_key/private_key.h create mode 100644 include/envoy/ssl/private_key/private_key_callbacks.h create mode 100644 include/envoy/ssl/private_key/private_key_config.h create mode 100644 source/extensions/transport_sockets/tls/private_key/BUILD create mode 100644 source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.cc create mode 100644 source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.h create mode 100644 test/extensions/transport_sockets/tls/test_private_key_method_provider.cc create mode 100644 test/extensions/transport_sockets/tls/test_private_key_method_provider.h diff --git a/CODEOWNERS b/CODEOWNERS index 72efd16d0562..7c3617ef6292 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -26,6 +26,8 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/http/header_to_metadata @rgs1 @zuercher # alts transport socket extension /*/extensions/transport_sockets/alts @htuch @yangminzhu +# tls transport socket extension +/*/extensions/transport_sockets/tls @PiotrSikora @lizan # sni_cluster extension /*/extensions/filters/network/sni_cluster @rshriram @lizan # tracers.datadog extension diff --git a/api/envoy/api/v2/auth/cert.proto b/api/envoy/api/v2/auth/cert.proto index 526caf292829..30db22c6d7a6 100644 --- a/api/envoy/api/v2/auth/cert.proto +++ b/api/envoy/api/v2/auth/cert.proto @@ -10,6 +10,8 @@ option go_package = "auth"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/core/config_source.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; @@ -102,6 +104,22 @@ message TlsParameters { repeated string ecdh_curves = 4; } +// BoringSSL private key method configuration. The private key methods are used for external +// (potentially asynchronous) signing and decryption operations. Some use cases for private key +// methods would be TPM support and TLS acceleration. +message PrivateKeyProvider { + // Private key method provider name. The name must match a + // supported private key method provider type. + string provider_name = 1 [(validate.rules).string.min_bytes = 1]; + + // Private key method provider specific configuration. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} + message TlsCertificate { // The TLS certificate chain. core.DataSource certificate_chain = 1; @@ -109,6 +127,15 @@ message TlsCertificate { // The TLS private key. core.DataSource private_key = 2; + // BoringSSL private key method provider. This is an alternative to :ref:`private_key + // ` field. This can't be + // marked as ``oneof`` due to API compatibility reasons. Setting both :ref:`private_key + // ` and + // :ref:`private_key_provider + // ` fields will result in an + // error. + PrivateKeyProvider private_key_provider = 6; + // The password to decrypt the TLS private key. If this field is not set, it is assumed that the // TLS private key is not password encrypted. core.DataSource password = 3; diff --git a/docs/root/extending/extending.rst b/docs/root/extending/extending.rst index 1551d0992428..51198fefacc4 100644 --- a/docs/root/extending/extending.rst +++ b/docs/root/extending/extending.rst @@ -19,6 +19,7 @@ types including: * :ref:`Stat sinks ` * :ref:`Tracers ` * Transport sockets +* BoringSSL private key methods As of this writing there is no high level extension developer documentation. The :repo:`existing extensions ` are a good way to learn what is possible. diff --git a/docs/root/intro/arch_overview/security/ssl.rst b/docs/root/intro/arch_overview/security/ssl.rst index e73d14dd3ef3..a44a8f531872 100644 --- a/docs/root/intro/arch_overview/security/ssl.rst +++ b/docs/root/intro/arch_overview/security/ssl.rst @@ -23,6 +23,10 @@ requirements (TLS1.2, SNI, etc.). Envoy supports the following TLS features: tickets (see `RFC 5077 `_). Resumption can be performed across hot restarts and between parallel Envoy instances (typically useful in a front proxy configuration). +* **BoringSSL private key methods**: TLS private key operations (signing and decrypting) can be + performed asynchronously from an extension. This allows extending Envoy to support various key + management schemes (such as TPM) and TLS acceleration. This mechanism uses + `BoringSSL private key method interface `_. Underlying implementation ------------------------- diff --git a/include/envoy/ssl/BUILD b/include/envoy/ssl/BUILD index 73373af9842c..8ea81a6e9090 100644 --- a/include/envoy/ssl/BUILD +++ b/include/envoy/ssl/BUILD @@ -48,6 +48,9 @@ envoy_cc_library( envoy_cc_library( name = "tls_certificate_config_interface", hdrs = ["tls_certificate_config.h"], + deps = [ + "//include/envoy/ssl/private_key:private_key_interface", + ], ) envoy_cc_library( diff --git a/include/envoy/ssl/context_manager.h b/include/envoy/ssl/context_manager.h index 7358b7745b45..bb0c104e5202 100644 --- a/include/envoy/ssl/context_manager.h +++ b/include/envoy/ssl/context_manager.h @@ -5,6 +5,7 @@ #include "envoy/common/time.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" +#include "envoy/ssl/private_key/private_key.h" #include "envoy/stats/scope.h" namespace Envoy { @@ -39,6 +40,12 @@ class ContextManager { * Iterate through all currently allocated contexts. */ virtual void iterateContexts(std::function callback) PURE; + + /** + * Access the private key operations manager, which is part of SSL + * context manager. + */ + virtual PrivateKeyMethodManager& privateKeyMethodManager() PURE; }; using ContextManagerPtr = std::unique_ptr; diff --git a/include/envoy/ssl/private_key/BUILD b/include/envoy/ssl/private_key/BUILD new file mode 100644 index 000000000000..4bb651d1f8d3 --- /dev/null +++ b/include/envoy/ssl/private_key/BUILD @@ -0,0 +1,35 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "private_key_interface", + hdrs = ["private_key.h"], + external_deps = ["ssl"], + deps = [ + ":private_key_callbacks_interface", + "//include/envoy/event:dispatcher_interface", + "@envoy_api//envoy/api/v2/auth:cert_cc", + ], +) + +envoy_cc_library( + name = "private_key_config_interface", + hdrs = ["private_key_config.h"], + deps = [ + ":private_key_interface", + "//include/envoy/registry", + ], +) + +envoy_cc_library( + name = "private_key_callbacks_interface", + hdrs = ["private_key_callbacks.h"], + external_deps = ["ssl"], +) diff --git a/include/envoy/ssl/private_key/private_key.h b/include/envoy/ssl/private_key/private_key.h new file mode 100644 index 000000000000..e972d608cd02 --- /dev/null +++ b/include/envoy/ssl/private_key/private_key.h @@ -0,0 +1,85 @@ +#pragma once + +#include +#include + +#include "envoy/api/v2/auth/cert.pb.h" +#include "envoy/common/pure.h" +#include "envoy/event/dispatcher.h" +#include "envoy/ssl/private_key/private_key_callbacks.h" + +#include "openssl/ssl.h" + +namespace Envoy { +namespace Server { +namespace Configuration { +// Prevent a dependency loop with the forward declaration. +class TransportSocketFactoryContext; +} // namespace Configuration +} // namespace Server + +namespace Ssl { + +using BoringSslPrivateKeyMethodSharedPtr = std::shared_ptr; + +class PrivateKeyMethodProvider { +public: + virtual ~PrivateKeyMethodProvider() = default; + + /** + * Register an SSL connection to private key operations by the provider. + * @param ssl a SSL connection object. + * @param cb a callbacks object, whose "complete" method will be invoked + * when the asynchronous processing is complete. + * @param dispatcher supplies the owning thread's dispatcher. + */ + virtual void registerPrivateKeyMethod(SSL* ssl, PrivateKeyConnectionCallbacks& cb, + Event::Dispatcher& dispatcher) PURE; + + /** + * Unregister an SSL connection from private key operations by the provider. + * @param ssl a SSL connection object. + * @throw EnvoyException if registration fails. + */ + virtual void unregisterPrivateKeyMethod(SSL* ssl) PURE; + + /** + * Check whether the private key method satisfies FIPS requirements. + * @return true if FIPS key requirements are satisfied, false if not. + */ + virtual bool checkFips() PURE; + + /** + * Get the private key methods from the provider. + * @return the private key methods associated with this provider and + * configuration. + */ + virtual BoringSslPrivateKeyMethodSharedPtr getBoringSslPrivateKeyMethod() PURE; +}; + +using PrivateKeyMethodProviderSharedPtr = std::shared_ptr; + +/** + * A manager for finding correct user-provided functions for handling BoringSSL private key + * operations. + */ +class PrivateKeyMethodManager { +public: + virtual ~PrivateKeyMethodManager() = default; + + /** + * Finds and returns a private key operations provider for BoringSSL. + * + * @param config a protobuf message object containing a PrivateKeyProvider message. + * @param factory_context context that provides components for creating and + * initializing connections using asynchronous private key operations. + * @return PrivateKeyMethodProvider the private key operations provider, or nullptr if + * no provider can be used with the context configuration. + */ + virtual PrivateKeyMethodProviderSharedPtr createPrivateKeyMethodProvider( + const envoy::api::v2::auth::PrivateKeyProvider& config, + Envoy::Server::Configuration::TransportSocketFactoryContext& factory_context) PURE; +}; + +} // namespace Ssl +} // namespace Envoy diff --git a/include/envoy/ssl/private_key/private_key_callbacks.h b/include/envoy/ssl/private_key/private_key_callbacks.h new file mode 100644 index 000000000000..1f370fda947b --- /dev/null +++ b/include/envoy/ssl/private_key/private_key_callbacks.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#include "envoy/common/pure.h" + +namespace Envoy { +namespace Ssl { + +class PrivateKeyConnectionCallbacks { +public: + virtual ~PrivateKeyConnectionCallbacks() = default; + + /** + * Callback function which is called when the asynchronous private key + * operation has been completed (with either success or failure). The + * provider will communicate the success status when SSL_do_handshake() + * is called the next time. + */ + virtual void onPrivateKeyMethodComplete() PURE; +}; + +} // namespace Ssl +} // namespace Envoy diff --git a/include/envoy/ssl/private_key/private_key_config.h b/include/envoy/ssl/private_key/private_key_config.h new file mode 100644 index 000000000000..8a5da737cac4 --- /dev/null +++ b/include/envoy/ssl/private_key/private_key_config.h @@ -0,0 +1,22 @@ +#pragma once + +#include "envoy/api/v2/auth/cert.pb.h" +#include "envoy/registry/registry.h" +#include "envoy/ssl/private_key/private_key.h" + +namespace Envoy { +namespace Ssl { + +// Base class which the private key operation provider implementations can register. + +class PrivateKeyMethodProviderInstanceFactory { +public: + virtual ~PrivateKeyMethodProviderInstanceFactory() = default; + virtual PrivateKeyMethodProviderSharedPtr createPrivateKeyMethodProviderInstance( + const envoy::api::v2::auth::PrivateKeyProvider& config, + Server::Configuration::TransportSocketFactoryContext& factory_context) PURE; + virtual std::string name() const PURE; +}; + +} // namespace Ssl +} // namespace Envoy diff --git a/include/envoy/ssl/tls_certificate_config.h b/include/envoy/ssl/tls_certificate_config.h index f934e5654a7a..882d40fe133d 100644 --- a/include/envoy/ssl/tls_certificate_config.h +++ b/include/envoy/ssl/tls_certificate_config.h @@ -4,6 +4,7 @@ #include #include "envoy/common/pure.h" +#include "envoy/ssl/private_key/private_key.h" namespace Envoy { namespace Ssl { @@ -34,6 +35,11 @@ class TlsCertificateConfig { */ virtual const std::string& privateKeyPath() const PURE; + /** + * @return private key method provider. + */ + virtual Envoy::Ssl::PrivateKeyMethodProviderSharedPtr privateKeyMethod() const PURE; + /** * @return a string of password. */ diff --git a/source/common/ssl/BUILD b/source/common/ssl/BUILD index 37dcef2ba8ea..ebbe62647302 100644 --- a/source/common/ssl/BUILD +++ b/source/common/ssl/BUILD @@ -13,7 +13,9 @@ envoy_cc_library( srcs = ["tls_certificate_config_impl.cc"], hdrs = ["tls_certificate_config_impl.h"], deps = [ + "//include/envoy/server:transport_socket_config_interface", "//include/envoy/ssl:tls_certificate_config_interface", + "//include/envoy/ssl/private_key:private_key_interface", "//source/common/common:empty_string", "//source/common/config:datasource_lib", "@envoy_api//envoy/api/v2/auth:cert_cc", diff --git a/source/common/ssl/tls_certificate_config_impl.cc b/source/common/ssl/tls_certificate_config_impl.cc index 25f993c38da6..5b28c3568e09 100644 --- a/source/common/ssl/tls_certificate_config_impl.cc +++ b/source/common/ssl/tls_certificate_config_impl.cc @@ -1,6 +1,7 @@ #include "common/ssl/tls_certificate_config_impl.h" #include "envoy/common/exception.h" +#include "envoy/server/transport_socket_config.h" #include "common/common/empty_string.h" #include "common/common/fmt.h" @@ -12,7 +13,8 @@ namespace Ssl { static const std::string INLINE_STRING = ""; TlsCertificateConfigImpl::TlsCertificateConfigImpl( - const envoy::api::v2::auth::TlsCertificate& config, Api::Api& api) + const envoy::api::v2::auth::TlsCertificate& config, + Server::Configuration::TransportSocketFactoryContext* factory_context, Api::Api& api) : certificate_chain_(Config::DataSource::read(config.certificate_chain(), true, api)), certificate_chain_path_( Config::DataSource::getPath(config.certificate_chain()) @@ -22,9 +24,18 @@ TlsCertificateConfigImpl::TlsCertificateConfigImpl( .value_or(private_key_.empty() ? EMPTY_STRING : INLINE_STRING)), password_(Config::DataSource::read(config.password(), true, api)), password_path_(Config::DataSource::getPath(config.password()) - .value_or(password_.empty() ? EMPTY_STRING : INLINE_STRING)) { - - if (certificate_chain_.empty() || private_key_.empty()) { + .value_or(password_.empty() ? EMPTY_STRING : INLINE_STRING)), + private_key_method_( + factory_context != nullptr && config.has_private_key_provider() + ? factory_context->sslContextManager() + .privateKeyMethodManager() + .createPrivateKeyMethodProvider(config.private_key_provider(), *factory_context) + : nullptr) { + if (config.has_private_key_provider() && config.has_private_key()) { + throw EnvoyException(fmt::format( + "Certificate configuration can't have both private_key and private_key_provider")); + } + if (certificate_chain_.empty() || (private_key_.empty() && private_key_method_ == nullptr)) { throw EnvoyException(fmt::format("Failed to load incomplete certificate from {}, {}", certificate_chain_path_, private_key_path_)); } diff --git a/source/common/ssl/tls_certificate_config_impl.h b/source/common/ssl/tls_certificate_config_impl.h index ed664521b187..1db9046e925a 100644 --- a/source/common/ssl/tls_certificate_config_impl.h +++ b/source/common/ssl/tls_certificate_config_impl.h @@ -4,6 +4,7 @@ #include "envoy/api/api.h" #include "envoy/api/v2/auth/cert.pb.h" +#include "envoy/ssl/private_key/private_key.h" #include "envoy/ssl/tls_certificate_config.h" namespace Envoy { @@ -11,7 +12,9 @@ namespace Ssl { class TlsCertificateConfigImpl : public TlsCertificateConfig { public: - TlsCertificateConfigImpl(const envoy::api::v2::auth::TlsCertificate& config, Api::Api& api); + TlsCertificateConfigImpl(const envoy::api::v2::auth::TlsCertificate& config, + Server::Configuration::TransportSocketFactoryContext* factory_context, + Api::Api& api); const std::string& certificateChain() const override { return certificate_chain_; } const std::string& certificateChainPath() const override { return certificate_chain_path_; } @@ -19,6 +22,9 @@ class TlsCertificateConfigImpl : public TlsCertificateConfig { const std::string& privateKeyPath() const override { return private_key_path_; } const std::string& password() const override { return password_; } const std::string& passwordPath() const override { return password_path_; } + Envoy::Ssl::PrivateKeyMethodProviderSharedPtr privateKeyMethod() const override { + return private_key_method_; + } private: const std::string certificate_chain_; @@ -27,6 +33,7 @@ class TlsCertificateConfigImpl : public TlsCertificateConfig { const std::string private_key_path_; const std::string password_; const std::string password_path_; + Envoy::Ssl::PrivateKeyMethodProviderSharedPtr private_key_method_{}; }; } // namespace Ssl diff --git a/source/extensions/transport_sockets/tls/BUILD b/source/extensions/transport_sockets/tls/BUILD index 2d9526cfcd7b..cb8bb9ec02ea 100644 --- a/source/extensions/transport_sockets/tls/BUILD +++ b/source/extensions/transport_sockets/tls/BUILD @@ -38,6 +38,8 @@ envoy_cc_library( ":utility_lib", "//include/envoy/network:connection_interface", "//include/envoy/network:transport_socket_interface", + "//include/envoy/ssl/private_key:private_key_callbacks_interface", + "//include/envoy/ssl/private_key:private_key_interface", "//include/envoy/stats:stats_macros", "//source/common/common:assert_lib", "//source/common/common:empty_string", @@ -90,6 +92,7 @@ envoy_cc_library( "//include/envoy/ssl:context_config_interface", "//include/envoy/ssl:context_interface", "//include/envoy/ssl:context_manager_interface", + "//include/envoy/ssl/private_key:private_key_interface", "//include/envoy/stats:stats_interface", "//include/envoy/stats:stats_macros", "//source/common/common:assert_lib", @@ -98,6 +101,7 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/network:address_lib", "//source/common/protobuf:utility_lib", + "//source/extensions/transport_sockets/tls/private_key:private_key_manager_lib", "@envoy_api//envoy/admin/v2alpha:certs_cc", ], ) diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc index 424e08ae2d59..5978e136c880 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.cc +++ b/source/extensions/transport_sockets/tls/context_config_impl.cc @@ -25,7 +25,8 @@ std::vector getTlsCertificateConf if (!config.tls_certificates().empty()) { std::vector providers; for (const auto& tls_certificate : config.tls_certificates()) { - if (!tls_certificate.has_certificate_chain() && !tls_certificate.has_private_key()) { + if (!tls_certificate.has_private_key_provider() && !tls_certificate.has_certificate_chain() && + !tls_certificate.has_private_key()) { continue; } providers.push_back( @@ -143,7 +144,7 @@ ContextConfigImpl::ContextConfigImpl( if (!tls_certificate_providers_.empty()) { for (auto& provider : tls_certificate_providers_) { if (provider->secret() != nullptr) { - tls_certificate_configs_.emplace_back(*provider->secret(), api_); + tls_certificate_configs_.emplace_back(*provider->secret(), &factory_context, api_); } } } @@ -174,7 +175,8 @@ void ContextConfigImpl::setSecretUpdateCallback(std::function callback) // This breaks multiple certificate support, but today SDS is only single cert. // TODO(htuch): Fix this when SDS goes multi-cert. tls_certificate_configs_.clear(); - tls_certificate_configs_.emplace_back(*tls_certificate_providers_[0]->secret(), api_); + tls_certificate_configs_.emplace_back(*tls_certificate_providers_[0]->secret(), nullptr, + api_); callback(); }); } diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index 03d247c489cc..f4795c0db2b1 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -305,40 +305,62 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c #endif } - // Load private key. - bio.reset(BIO_new_mem_buf(const_cast(tls_certificate.privateKey().data()), - tls_certificate.privateKey().size())); - RELEASE_ASSERT(bio != nullptr, ""); - bssl::UniquePtr pkey(PEM_read_bio_PrivateKey( - bio.get(), nullptr, nullptr, - !tls_certificate.password().empty() ? const_cast(tls_certificate.password().c_str()) - : nullptr)); - if (pkey == nullptr || !SSL_CTX_use_PrivateKey(ctx.ssl_ctx_.get(), pkey.get())) { - throw EnvoyException( - fmt::format("Failed to load private key from {}", tls_certificate.privateKeyPath())); - } - + Envoy::Ssl::PrivateKeyMethodProviderSharedPtr private_key_method_provider = + tls_certificate.privateKeyMethod(); + // We either have a private key or a BoringSSL private key method provider. + if (private_key_method_provider) { + ctx.private_key_method_provider_ = private_key_method_provider; + // The provider has a reference to the private key method for the context lifetime. + Ssl::BoringSslPrivateKeyMethodSharedPtr private_key_method = + private_key_method_provider->getBoringSslPrivateKeyMethod(); + if (private_key_method == nullptr) { + throw EnvoyException( + fmt::format("Failed to get BoringSSL private key method from provider")); + } #ifdef BORINGSSL_FIPS - // Verify that private keys are passing FIPS pairwise consistency tests. - switch (pkey_id) { - case EVP_PKEY_EC: { - const EC_KEY* ecdsa_private_key = EVP_PKEY_get0_EC_KEY(pkey.get()); - if (!EC_KEY_check_fips(ecdsa_private_key)) { - throw EnvoyException(fmt::format("Failed to load private key from {}, ECDSA key failed " - "pairwise consistency test required in FIPS mode", - tls_certificate.privateKeyPath())); + if (!ctx.private_key_method_provider_->checkFips()) { + throw EnvoyException( + fmt::format("Private key method doesn't support FIPS mode with current parameters")); } - } break; - case EVP_PKEY_RSA: { - RSA* rsa_private_key = EVP_PKEY_get0_RSA(pkey.get()); - if (!RSA_check_fips(rsa_private_key)) { - throw EnvoyException(fmt::format("Failed to load private key from {}, RSA key failed " - "pairwise consistency test required in FIPS mode", - tls_certificate.privateKeyPath())); +#endif + SSL_CTX_set_private_key_method(ctx.ssl_ctx_.get(), private_key_method.get()); + } else { + // Load private key. + bio.reset(BIO_new_mem_buf(const_cast(tls_certificate.privateKey().data()), + tls_certificate.privateKey().size())); + RELEASE_ASSERT(bio != nullptr, ""); + bssl::UniquePtr pkey( + PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, + !tls_certificate.password().empty() + ? const_cast(tls_certificate.password().c_str()) + : nullptr)); + if (pkey == nullptr || !SSL_CTX_use_PrivateKey(ctx.ssl_ctx_.get(), pkey.get())) { + throw EnvoyException( + fmt::format("Failed to load private key from {}", tls_certificate.privateKeyPath())); + } + +#ifdef BORINGSSL_FIPS + // Verify that private keys are passing FIPS pairwise consistency tests. + switch (pkey_id) { + case EVP_PKEY_EC: { + const EC_KEY* ecdsa_private_key = EVP_PKEY_get0_EC_KEY(pkey.get()); + if (!EC_KEY_check_fips(ecdsa_private_key)) { + throw EnvoyException(fmt::format("Failed to load private key from {}, ECDSA key failed " + "pairwise consistency test required in FIPS mode", + tls_certificate.privateKeyPath())); + } + } break; + case EVP_PKEY_RSA: { + RSA* rsa_private_key = EVP_PKEY_get0_RSA(pkey.get()); + if (!RSA_check_fips(rsa_private_key)) { + throw EnvoyException(fmt::format("Failed to load private key from {}, RSA key failed " + "pairwise consistency test required in FIPS mode", + tls_certificate.privateKeyPath())); + } + } break; } - } break; - } #endif + } } // use the server's cipher list preferences @@ -486,6 +508,19 @@ void ContextImpl::logHandshake(SSL* ssl) const { } } +std::vector ContextImpl::getPrivateKeyMethodProviders() { + std::vector providers; + + for (auto& tls_context : tls_contexts_) { + Envoy::Ssl::PrivateKeyMethodProviderSharedPtr provider = + tls_context.getPrivateKeyMethodProvider(); + if (provider) { + providers.push_back(provider); + } + } + return providers; +} + bool ContextImpl::verifySubjectAltName(X509* cert, const std::vector& subject_alt_names) { bssl::UniquePtr san_names( diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index c4cf67d6cfa8..ccb984a63efa 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -8,6 +8,7 @@ #include "envoy/network/transport_socket.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" +#include "envoy/ssl/private_key/private_key.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" @@ -79,6 +80,8 @@ class ContextImpl : public virtual Envoy::Ssl::Context { Envoy::Ssl::CertificateDetailsPtr getCaCertInformation() const override; std::vector getCertChainInformation() const override; + std::vector getPrivateKeyMethodProviders(); + protected: ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& config, TimeSource& time_source); @@ -135,11 +138,15 @@ class ContextImpl : public virtual Envoy::Ssl::Context { bssl::UniquePtr cert_chain_; std::string cert_chain_file_path_; bool is_ecdsa_{}; + Ssl::PrivateKeyMethodProviderSharedPtr private_key_method_provider_{}; std::string getCertChainFileName() const { return cert_chain_file_path_; }; void addClientValidationContext(const Envoy::Ssl::CertificateValidationContextConfig& config, bool require_client_cert); bool isCipherEnabled(uint16_t cipher_id, uint16_t client_version); + Envoy::Ssl::PrivateKeyMethodProviderSharedPtr getPrivateKeyMethodProvider() { + return private_key_method_provider_; + } }; // This is always non-empty, with the first context used for all new SSL diff --git a/source/extensions/transport_sockets/tls/context_manager_impl.h b/source/extensions/transport_sockets/tls/context_manager_impl.h index 88ef9e67d546..d08e12e97410 100644 --- a/source/extensions/transport_sockets/tls/context_manager_impl.h +++ b/source/extensions/transport_sockets/tls/context_manager_impl.h @@ -5,8 +5,11 @@ #include "envoy/common/time.h" #include "envoy/ssl/context_manager.h" +#include "envoy/ssl/private_key/private_key.h" #include "envoy/stats/scope.h" +#include "extensions/transport_sockets/tls/private_key/private_key_manager_impl.h" + namespace Envoy { namespace Extensions { namespace TransportSockets { @@ -33,11 +36,15 @@ class ContextManagerImpl final : public Envoy::Ssl::ContextManager { const std::vector& server_names) override; size_t daysUntilFirstCertExpires() const override; void iterateContexts(std::function callback) override; + Ssl::PrivateKeyMethodManager& privateKeyMethodManager() override { + return private_key_method_manager_; + }; private: void removeEmptyContexts(); TimeSource& time_source_; std::list> contexts_; + PrivateKeyMethodManagerImpl private_key_method_manager_{}; }; } // namespace Tls diff --git a/source/extensions/transport_sockets/tls/private_key/BUILD b/source/extensions/transport_sockets/tls/private_key/BUILD new file mode 100644 index 000000000000..2c181249b5d8 --- /dev/null +++ b/source/extensions/transport_sockets/tls/private_key/BUILD @@ -0,0 +1,26 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "private_key_manager_lib", + srcs = [ + "private_key_manager_impl.cc", + ], + hdrs = [ + "private_key_manager_impl.h", + ], + deps = [ + "//include/envoy/event:dispatcher_interface", + "//include/envoy/registry", + "//include/envoy/ssl/private_key:private_key_config_interface", + "//include/envoy/ssl/private_key:private_key_interface", + "@envoy_api//envoy/api/v2/auth:cert_cc", + ], +) diff --git a/source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.cc b/source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.cc new file mode 100644 index 000000000000..817b9d362616 --- /dev/null +++ b/source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.cc @@ -0,0 +1,30 @@ +#include "extensions/transport_sockets/tls/private_key/private_key_manager_impl.h" + +#include "envoy/registry/registry.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { + +Envoy::Ssl::PrivateKeyMethodProviderSharedPtr +PrivateKeyMethodManagerImpl::createPrivateKeyMethodProvider( + const envoy::api::v2::auth::PrivateKeyProvider& config, + Server::Configuration::TransportSocketFactoryContext& factory_context) { + + Ssl::PrivateKeyMethodProviderInstanceFactory* factory = + Registry::FactoryRegistry::getFactory( + config.provider_name()); + + // Create a new provider instance with the configuration. + if (factory) { + return factory->createPrivateKeyMethodProviderInstance(config, factory_context); + } + + return nullptr; +} + +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.h b/source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.h new file mode 100644 index 000000000000..1ae42d1916ec --- /dev/null +++ b/source/extensions/transport_sockets/tls/private_key/private_key_manager_impl.h @@ -0,0 +1,23 @@ +#pragma once + +#include "envoy/api/v2/auth/cert.pb.h" +#include "envoy/ssl/private_key/private_key.h" +#include "envoy/ssl/private_key/private_key_config.h" + +namespace Envoy { +namespace Extensions { +namespace TransportSockets { +namespace Tls { + +class PrivateKeyMethodManagerImpl : public virtual Ssl::PrivateKeyMethodManager { +public: + // Ssl::PrivateKeyMethodManager + Ssl::PrivateKeyMethodProviderSharedPtr createPrivateKeyMethodProvider( + const envoy::api::v2::auth::PrivateKeyProvider& config, + Server::Configuration::TransportSocketFactoryContext& factory_context) override; +}; + +} // namespace Tls +} // namespace TransportSockets +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/transport_sockets/tls/ssl_socket.cc b/source/extensions/transport_sockets/tls/ssl_socket.cc index d6b63be7d091..c079bb989fdc 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.cc +++ b/source/extensions/transport_sockets/tls/ssl_socket.cc @@ -46,7 +46,7 @@ SslSocket::SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, const Network::TransportSocketOptionsSharedPtr& transport_socket_options) : transport_socket_options_(transport_socket_options), ctx_(std::dynamic_pointer_cast(ctx)), - ssl_(ctx_->newSsl(transport_socket_options_.get())) { + ssl_(ctx_->newSsl(transport_socket_options_.get())), state_(SocketState::PreHandshake) { if (state == InitialState::Client) { SSL_set_connect_state(ssl_.get()); } else { @@ -59,6 +59,12 @@ void SslSocket::setTransportSocketCallbacks(Network::TransportSocketCallbacks& c ASSERT(!callbacks_); callbacks_ = &callbacks; + // Associate this SSL connection with all the certificates (with their potentially different + // private key methods). + for (auto const& provider : ctx_->getPrivateKeyMethodProviders()) { + provider->registerPrivateKeyMethod(ssl_.get(), *this, callbacks_->connection().dispatcher()); + } + BIO* bio = BIO_new_socket(callbacks_->ioHandle().fd(), 0); SSL_set_bio(ssl_.get(), bio, bio); } @@ -88,9 +94,9 @@ SslSocket::ReadResult SslSocket::sslReadIntoSlice(Buffer::RawSlice& slice) { } Network::IoResult SslSocket::doRead(Buffer::Instance& read_buffer) { - if (!handshake_complete_) { + if (state_ != SocketState::HandshakeComplete && state_ != SocketState::ShutdownSent) { PostIoAction action = doHandshake(); - if (action == PostIoAction::Close || !handshake_complete_) { + if (action == PostIoAction::Close || state_ != SocketState::HandshakeComplete) { // end_stream is false because either a hard error occurred (action == Close) or // the handshake isn't complete, so a half-close cannot occur yet. return {action, 0, false}; @@ -149,12 +155,24 @@ Network::IoResult SslSocket::doRead(Buffer::Instance& read_buffer) { return {action, bytes_read, end_stream}; } +void SslSocket::onPrivateKeyMethodComplete() { + ASSERT(isThreadSafe()); + ASSERT(state_ == SocketState::HandshakeInProgress); + + // Resume handshake. + PostIoAction action = doHandshake(); + if (action == PostIoAction::Close) { + ENVOY_CONN_LOG(debug, "async handshake completion error", callbacks_->connection()); + callbacks_->connection().close(Network::ConnectionCloseType::FlushWrite); + } +} + PostIoAction SslSocket::doHandshake() { - ASSERT(!handshake_complete_); + ASSERT(state_ != SocketState::HandshakeComplete && state_ != SocketState::ShutdownSent); int rc = SSL_do_handshake(ssl_.get()); if (rc == 1) { ENVOY_CONN_LOG(debug, "handshake complete", callbacks_->connection()); - handshake_complete_ = true; + state_ = SocketState::HandshakeComplete; ctx_->logHandshake(ssl_.get()); callbacks_->raiseEvent(Network::ConnectionEvent::Connected); @@ -164,12 +182,18 @@ PostIoAction SslSocket::doHandshake() { : PostIoAction::Close; } else { int err = SSL_get_error(ssl_.get(), rc); - ENVOY_CONN_LOG(debug, "handshake error: {}", callbacks_->connection(), err); switch (err) { case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: + ENVOY_CONN_LOG(debug, "handshake expecting {}", callbacks_->connection(), + err == SSL_ERROR_WANT_READ ? "read" : "write"); + return PostIoAction::KeepOpen; + case SSL_ERROR_WANT_PRIVATE_KEY_OPERATION: + ENVOY_CONN_LOG(debug, "handshake continued asynchronously", callbacks_->connection()); + state_ = SocketState::HandshakeInProgress; return PostIoAction::KeepOpen; default: + ENVOY_CONN_LOG(debug, "handshake error: {}", callbacks_->connection(), err); drainErrorQueue(); return PostIoAction::Close; } @@ -204,10 +228,10 @@ void SslSocket::drainErrorQueue() { } Network::IoResult SslSocket::doWrite(Buffer::Instance& write_buffer, bool end_stream) { - ASSERT(!shutdown_sent_ || write_buffer.length() == 0); - if (!handshake_complete_) { + ASSERT(state_ != SocketState::ShutdownSent || write_buffer.length() == 0); + if (state_ != SocketState::HandshakeComplete && state_ != SocketState::ShutdownSent) { PostIoAction action = doHandshake(); - if (action == PostIoAction::Close || !handshake_complete_) { + if (action == PostIoAction::Close || state_ != SocketState::HandshakeComplete) { return {action, 0, false}; } } @@ -260,15 +284,16 @@ Network::IoResult SslSocket::doWrite(Buffer::Instance& write_buffer, bool end_st return {PostIoAction::KeepOpen, total_bytes_written, false}; } -void SslSocket::onConnected() { ASSERT(!handshake_complete_); } +void SslSocket::onConnected() { ASSERT(state_ == SocketState::PreHandshake); } void SslSocket::shutdownSsl() { - ASSERT(handshake_complete_); - if (!shutdown_sent_ && callbacks_->connection().state() != Network::Connection::State::Closed) { + ASSERT(state_ != SocketState::PreHandshake); + if (state_ != SocketState::ShutdownSent && + callbacks_->connection().state() != Network::Connection::State::Closed) { int rc = SSL_shutdown(ssl_.get()); ENVOY_CONN_LOG(debug, "SSL shutdown: rc={}", callbacks_->connection(), rc); drainErrorQueue(); - shutdown_sent_ = true; + state_ = SocketState::ShutdownSent; } } @@ -381,10 +406,15 @@ std::vector SslSocket::dnsSansPeerCertificate() const { } void SslSocket::closeSocket(Network::ConnectionEvent) { + // Unregister the SSL connection object from private key method providers. + for (auto const& provider : ctx_->getPrivateKeyMethodProviders()) { + provider->unregisterPrivateKeyMethod(ssl_.get()); + } + // Attempt to send a shutdown before closing the socket. It's possible this won't go out if // there is no room on the socket. We can extend the state machine to handle this at some point // if needed. - if (handshake_complete_) { + if (state_ == SocketState::HandshakeInProgress || state_ == SocketState::HandshakeComplete) { shutdownSsl(); } } diff --git a/source/extensions/transport_sockets/tls/ssl_socket.h b/source/extensions/transport_sockets/tls/ssl_socket.h index 549ffbf83696..8e0eb6601058 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.h +++ b/source/extensions/transport_sockets/tls/ssl_socket.h @@ -6,6 +6,7 @@ #include "envoy/network/connection.h" #include "envoy/network/transport_socket.h" #include "envoy/secret/secret_callbacks.h" +#include "envoy/ssl/private_key/private_key_callbacks.h" #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" @@ -38,9 +39,11 @@ struct SslSocketFactoryStats { }; enum class InitialState { Client, Server }; +enum class SocketState { PreHandshake, HandshakeInProgress, HandshakeComplete, ShutdownSent }; class SslSocket : public Network::TransportSocket, public Envoy::Ssl::ConnectionInfo, + public Envoy::Ssl::PrivateKeyConnectionCallbacks, protected Logger::Loggable { public: SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, @@ -70,13 +73,16 @@ class SslSocket : public Network::TransportSocket, void setTransportSocketCallbacks(Network::TransportSocketCallbacks& callbacks) override; std::string protocol() const override; absl::string_view failureReason() const override; - bool canFlushClose() override { return handshake_complete_; } + bool canFlushClose() override { return state_ == SocketState::HandshakeComplete; } void closeSocket(Network::ConnectionEvent close_type) override; Network::IoResult doRead(Buffer::Instance& read_buffer) override; Network::IoResult doWrite(Buffer::Instance& write_buffer, bool end_stream) override; void onConnected() override; const Ssl::ConnectionInfo* ssl() const override { return this; } + // Ssl::PrivateKeyConnectionCallbacks + void onPrivateKeyMethodComplete() override; + SSL* rawSslForTest() const { return ssl_.get(); } private: @@ -89,18 +95,20 @@ class SslSocket : public Network::TransportSocket, Network::PostIoAction doHandshake(); void drainErrorQueue(); void shutdownSsl(); + bool isThreadSafe() const { + return callbacks_ != nullptr && callbacks_->connection().dispatcher().isThreadSafe(); + } const Network::TransportSocketOptionsSharedPtr transport_socket_options_; Network::TransportSocketCallbacks* callbacks_{}; ContextImplSharedPtr ctx_; bssl::UniquePtr ssl_; - bool handshake_complete_{}; - bool shutdown_sent_{}; uint64_t bytes_to_retry_{}; std::string failure_reason_; mutable std::string cached_sha_256_peer_certificate_digest_; mutable std::string cached_url_encoded_pem_encoded_peer_certificate_; mutable std::string cached_url_encoded_pem_encoded_peer_cert_chain_; + SocketState state_; }; class ClientSslSocketFactory : public Network::TransportSocketFactory, diff --git a/source/server/ssl_context_manager.cc b/source/server/ssl_context_manager.cc index 3e422643ed43..4573cdf6de2f 100644 --- a/source/server/ssl_context_manager.cc +++ b/source/server/ssl_context_manager.cc @@ -29,6 +29,8 @@ class SslContextManagerNoTlsStub final : public Envoy::Ssl::ContextManager { void iterateContexts(std::function /* callback */) override{}; + Ssl::PrivateKeyMethodManager& privateKeyMethodManager() override { throwException(); } + private: [[noreturn]] void throwException() { throw EnvoyException("SSL is not supported in this configuration"); diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index 780211864399..ea65ba4f5f10 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -87,7 +87,7 @@ TEST_F(SdsApiTest, DynamicTlsCertificateUpdateSuccess) { EXPECT_CALL(secret_callback, onAddOrUpdateSecret()); subscription_factory_.callbacks_->onConfigUpdate(secret_resources, ""); - Ssl::TlsCertificateConfigImpl tls_config(*sds_api.secret(), *api_); + Ssl::TlsCertificateConfigImpl tls_config(*sds_api.secret(), nullptr, *api_); const std::string cert_pem = "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem"; EXPECT_EQ(TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(cert_pem)), @@ -182,7 +182,7 @@ TEST_F(SdsApiTest, DeltaUpdateSuccess) { initialize(); subscription_factory_.callbacks_->onConfigUpdate(secret_resources, {}, ""); - Ssl::TlsCertificateConfigImpl tls_config(*sds_api.secret(), *api_); + Ssl::TlsCertificateConfigImpl tls_config(*sds_api.secret(), nullptr, *api_); const std::string cert_pem = "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem"; EXPECT_EQ(TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(cert_pem)), diff --git a/test/common/secret/secret_manager_impl_test.cc b/test/common/secret/secret_manager_impl_test.cc index b3a6105fb11f..69e3051ce880 100644 --- a/test/common/secret/secret_manager_impl_test.cc +++ b/test/common/secret/secret_manager_impl_test.cc @@ -66,7 +66,7 @@ name: "abc.com" ASSERT_NE(secret_manager->findStaticTlsCertificateProvider("abc.com"), nullptr); Ssl::TlsCertificateConfigImpl tls_config( - *secret_manager->findStaticTlsCertificateProvider("abc.com")->secret(), *api_); + *secret_manager->findStaticTlsCertificateProvider("abc.com")->secret(), nullptr, *api_); const std::string cert_pem = "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem"; EXPECT_EQ(TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(cert_pem)), @@ -206,7 +206,7 @@ name: "abc.com" init_target_handle->initialize(init_watcher); secret_context.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(secret_resources, ""); - Ssl::TlsCertificateConfigImpl tls_config(*secret_provider->secret(), *api_); + Ssl::TlsCertificateConfigImpl tls_config(*secret_provider->secret(), nullptr, *api_); const std::string cert_pem = "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem"; EXPECT_EQ(TestEnvironment::readFileToStringForTest(TestEnvironment::substitute(cert_pem)), @@ -261,7 +261,7 @@ name: "abc.com" init_target_handle->initialize(init_watcher); secret_context.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(secret_resources, "keycert-v1"); - Ssl::TlsCertificateConfigImpl tls_config(*secret_provider->secret(), *api_); + Ssl::TlsCertificateConfigImpl tls_config(*secret_provider->secret(), nullptr, *api_); EXPECT_EQ("DUMMY_INLINE_BYTES_FOR_CERT_CHAIN", tls_config.certificateChain()); EXPECT_EQ("DUMMY_INLINE_BYTES_FOR_PRIVATE_KEY", tls_config.privateKey()); EXPECT_EQ("DUMMY_PASSWORD", tls_config.password()); diff --git a/test/extensions/transport_sockets/tls/BUILD b/test/extensions/transport_sockets/tls/BUILD index 32ec88bb3d58..94f93e62a6dd 100644 --- a/test/extensions/transport_sockets/tls/BUILD +++ b/test/extensions/transport_sockets/tls/BUILD @@ -25,6 +25,7 @@ envoy_cc_test( external_deps = ["ssl"], shard_count = 4, deps = [ + ":test_private_key_method_provider_test_lib", "//include/envoy/network:transport_socket_interface", "//source/common/buffer:buffer_lib", "//source/common/common:empty_string", @@ -41,14 +42,17 @@ envoy_cc_test( "//source/extensions/transport_sockets/tls:context_lib", "//source/extensions/transport_sockets/tls:ssl_socket_lib", "//source/extensions/transport_sockets/tls:utility_lib", + "//source/extensions/transport_sockets/tls/private_key:private_key_manager_lib", "//test/extensions/transport_sockets/tls/test_data:cert_infos", "//test/mocks/buffer:buffer_mocks", "//test/mocks/network:network_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/server:server_mocks", + "//test/mocks/ssl:ssl_mocks", "//test/mocks/stats:stats_mocks", "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", + "//test/test_common:registry_lib", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", ], @@ -75,6 +79,7 @@ envoy_cc_test( "//test/mocks/runtime:runtime_mocks", "//test/mocks/secret:secret_mocks", "//test/mocks/server:server_mocks", + "//test/mocks/ssl:ssl_mocks", "//test/test_common:environment_lib", "//test/test_common:simulated_time_system_lib", ], @@ -109,3 +114,23 @@ envoy_cc_test_library( "//test/test_common:environment_lib", ], ) + +envoy_cc_test_library( + name = "test_private_key_method_provider_test_lib", + srcs = [ + "test_private_key_method_provider.cc", + ], + hdrs = [ + "test_private_key_method_provider.h", + ], + external_deps = ["ssl"], + deps = [ + "//include/envoy/api:api_interface", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/server:transport_socket_config_interface", + "//include/envoy/ssl/private_key:private_key_config_interface", + "//include/envoy/ssl/private_key:private_key_interface", + "//source/common/config:utility_lib", + "//source/common/protobuf:utility_lib", + ], +) diff --git a/test/extensions/transport_sockets/tls/context_impl_test.cc b/test/extensions/transport_sockets/tls/context_impl_test.cc index 92b78d793cd8..4327856f54fc 100644 --- a/test/extensions/transport_sockets/tls/context_impl_test.cc +++ b/test/extensions/transport_sockets/tls/context_impl_test.cc @@ -17,6 +17,7 @@ #include "test/extensions/transport_sockets/tls/test_data/san_dns3_cert_info.h" #include "test/mocks/secret/mocks.h" #include "test/mocks/server/mocks.h" +#include "test/mocks/ssl/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/simulated_time_system.h" #include "test/test_common/utility.h" @@ -1185,6 +1186,124 @@ TEST_F(ServerContextConfigImplTest, InvalidIgnoreCertsNoCA) { EXPECT_NO_THROW(ServerContextConfigImpl server_context_config(tls_context, factory_context_)); } +TEST_F(ServerContextConfigImplTest, PrivateKeyMethodLoadFailureNoProvider) { + envoy::api::v2::auth::DownstreamTlsContext tls_context; + NiceMock context_manager; + NiceMock private_key_method_manager; + EXPECT_CALL(factory_context_, sslContextManager()).WillOnce(ReturnRef(context_manager)); + EXPECT_CALL(context_manager, privateKeyMethodManager()) + .WillOnce(ReturnRef(private_key_method_manager)); + const std::string tls_context_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem" + private_key_provider: + provider_name: mock_provider + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + test_value: 100 + )EOF"; + TestUtility::loadFromYaml(TestEnvironment::substitute(tls_context_yaml), tls_context); + EXPECT_THROW_WITH_REGEX( + ServerContextConfigImpl server_context_config(tls_context, factory_context_), EnvoyException, + "Failed to load incomplete certificate from "); +} + +TEST_F(ServerContextConfigImplTest, PrivateKeyMethodLoadFailureNoMethod) { + envoy::api::v2::auth::DownstreamTlsContext tls_context; + tls_context.mutable_common_tls_context()->add_tls_certificates(); + Stats::IsolatedStoreImpl store; + NiceMock context_manager; + NiceMock private_key_method_manager; + auto private_key_method_provider_ptr = + std::make_shared>(); + Event::SimulatedTimeSystem time_system; + ContextManagerImpl manager(time_system); + EXPECT_CALL(factory_context_, sslContextManager()).WillOnce(ReturnRef(context_manager)); + EXPECT_CALL(context_manager, privateKeyMethodManager()) + .WillOnce(ReturnRef(private_key_method_manager)); + EXPECT_CALL(private_key_method_manager, createPrivateKeyMethodProvider(_, _)) + .WillOnce(Return(private_key_method_provider_ptr)); + const std::string tls_context_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem" + private_key_provider: + provider_name: mock_provider + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + test_value: 100 + )EOF"; + TestUtility::loadFromYaml(TestEnvironment::substitute(tls_context_yaml), tls_context); + ServerContextConfigImpl server_context_config(tls_context, factory_context_); + EXPECT_THROW_WITH_MESSAGE( + Envoy::Ssl::ServerContextSharedPtr server_ctx( + manager.createSslServerContext(store, server_context_config, std::vector{})), + EnvoyException, "Failed to get BoringSSL private key method from provider"); +} + +TEST_F(ServerContextConfigImplTest, PrivateKeyMethodLoadSuccess) { + envoy::api::v2::auth::DownstreamTlsContext tls_context; + NiceMock context_manager; + NiceMock private_key_method_manager; + auto private_key_method_provider_ptr = + std::make_shared>(); + EXPECT_CALL(factory_context_, sslContextManager()).WillOnce(ReturnRef(context_manager)); + EXPECT_CALL(context_manager, privateKeyMethodManager()) + .WillOnce(ReturnRef(private_key_method_manager)); + EXPECT_CALL(private_key_method_manager, createPrivateKeyMethodProvider(_, _)) + .WillOnce(Return(private_key_method_provider_ptr)); + const std::string tls_context_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem" + private_key_provider: + provider_name: mock_provider + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + test_value: 100 + )EOF"; + TestUtility::loadFromYaml(TestEnvironment::substitute(tls_context_yaml), tls_context); + ServerContextConfigImpl server_context_config(tls_context, factory_context_); +} + +TEST_F(ServerContextConfigImplTest, PrivateKeyMethodLoadFailureBothKeyAndMethod) { + envoy::api::v2::auth::DownstreamTlsContext tls_context; + NiceMock context_manager; + NiceMock private_key_method_manager; + auto private_key_method_provider_ptr = + std::make_shared>(); + EXPECT_CALL(factory_context_, sslContextManager()).WillOnce(ReturnRef(context_manager)); + EXPECT_CALL(context_manager, privateKeyMethodManager()) + .WillOnce(ReturnRef(private_key_method_manager)); + EXPECT_CALL(private_key_method_manager, createPrivateKeyMethodProvider(_, _)) + .WillOnce(Return(private_key_method_provider_ptr)); + const std::string tls_context_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem" + private_key: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_key.pem" + private_key_provider: + provider_name: mock_provider + typed_config: + "@type": type.googleapis.com/google.protobuf.Struct + value: + test_value: 100 + )EOF"; + TestUtility::loadFromYaml(TestEnvironment::substitute(tls_context_yaml), tls_context); + EXPECT_THROW_WITH_MESSAGE( + ServerContextConfigImpl server_context_config(tls_context, factory_context_), EnvoyException, + "Certificate configuration can't have both private_key and private_key_provider"); +} + } // namespace Tls } // namespace TransportSockets } // namespace Extensions diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 07daedca6ca0..8b6dbb4b0263 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -16,6 +16,7 @@ #include "extensions/filters/listener/tls_inspector/tls_inspector.h" #include "extensions/transport_sockets/tls/context_config_impl.h" #include "extensions/transport_sockets/tls/context_impl.h" +#include "extensions/transport_sockets/tls/private_key/private_key_manager_impl.h" #include "extensions/transport_sockets/tls/ssl_socket.h" #include "test/extensions/transport_sockets/tls/ssl_certs_test.h" @@ -25,13 +26,16 @@ #include "test/extensions/transport_sockets/tls/test_data/san_dns_cert_info.h" #include "test/extensions/transport_sockets/tls/test_data/san_uri_cert_info.h" #include "test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_cert_info.h" +#include "test/extensions/transport_sockets/tls/test_private_key_method_provider.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/secret/mocks.h" #include "test/mocks/server/mocks.h" +#include "test/mocks/ssl/mocks.h" #include "test/mocks/stats/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/network_utility.h" +#include "test/test_common/registry.h" #include "test/test_common/utility.h" #include "absl/strings/str_replace.h" @@ -95,7 +99,9 @@ class TestUtilOptions : public TestUtilOptionsBase { TestUtilOptions(const std::string& client_ctx_yaml, const std::string& server_ctx_yaml, bool expect_success, Network::Address::IpVersion version) : TestUtilOptionsBase(expect_success, version), client_ctx_yaml_(client_ctx_yaml), - server_ctx_yaml_(server_ctx_yaml), expect_no_cert_(false), expect_no_cert_chain_(false) { + server_ctx_yaml_(server_ctx_yaml), expect_no_cert_(false), expect_no_cert_chain_(false), + expect_private_key_method_(false), + expected_server_close_event_(Network::ConnectionEvent::RemoteClose) { if (expect_success) { setExpectedServerStats("ssl.handshake"); } else { @@ -204,12 +210,28 @@ class TestUtilOptions : public TestUtilOptionsBase { return expected_expiration_peer_cert_; } + TestUtilOptions& setPrivateKeyMethodExpected(bool expected_method) { + expect_private_key_method_ = expected_method; + return *this; + } + + bool expectedPrivateKeyMethod() const { return expect_private_key_method_; } + + TestUtilOptions& setExpectedServerCloseEvent(Network::ConnectionEvent expected_event) { + expected_server_close_event_ = expected_event; + return *this; + } + + Network::ConnectionEvent expectedServerCloseEvent() const { return expected_server_close_event_; } + private: const std::string client_ctx_yaml_; const std::string server_ctx_yaml_; bool expect_no_cert_; bool expect_no_cert_chain_; + bool expect_private_key_method_; + Network::ConnectionEvent expected_server_close_event_; std::string expected_digest_; std::vector expected_local_uri_; std::string expected_serial_number_; @@ -231,6 +253,21 @@ void testUtil(const TestUtilOptions& options) { server_factory_context; ON_CALL(server_factory_context, api()).WillByDefault(ReturnRef(*server_api)); + // For private key method testing. + NiceMock context_manager; + Extensions::PrivateKeyMethodProvider::TestPrivateKeyMethodFactory test_factory; + Registry::InjectFactory + test_private_key_method_factory(test_factory); + PrivateKeyMethodManagerImpl private_key_method_manager; + if (options.expectedPrivateKeyMethod()) { + EXPECT_CALL(server_factory_context, sslContextManager()) + .WillOnce(ReturnRef(context_manager)) + .WillRepeatedly(ReturnRef(context_manager)); + EXPECT_CALL(context_manager, privateKeyMethodManager()) + .WillOnce(ReturnRef(private_key_method_manager)) + .WillRepeatedly(ReturnRef(private_key_method_manager)); + } + envoy::api::v2::auth::DownstreamTlsContext server_tls_context; TestUtility::loadFromYaml(TestEnvironment::substitute(options.serverCtxYaml()), server_tls_context); @@ -376,7 +413,7 @@ void testUtil(const TestUtilOptions& options) { } else { EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::RemoteClose)) .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { close_second_time(); })); - EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::RemoteClose)) + EXPECT_CALL(server_connection_callbacks, onEvent(options.expectedServerCloseEvent())) .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { close_second_time(); })); } @@ -4145,6 +4182,525 @@ TEST_P(SslReadBufferLimitTest, SmallReadsIntoSameSlice) { dispatcher_->run(Event::Dispatcher::RunType::Block); } +// Test asynchronous signing (ECDHE) using a private key provider. +TEST_P(SslSocketTest, RsaPrivateKeyProviderAsyncSignSuccess) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + sync_mode: false + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string successful_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - ECDHE-RSA-AES128-GCM-SHA256 +)EOF"; + + TestUtilOptions successful_test_options(successful_client_ctx_yaml, server_ctx_yaml, true, + GetParam()); + testUtil(successful_test_options.setPrivateKeyMethodExpected(true)); +} + +// Test asynchronous decryption (RSA). +TEST_P(SslSocketTest, RsaPrivateKeyProviderAsyncDecryptSuccess) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: decrypt + sync_mode: false + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string successful_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - TLS_RSA_WITH_AES_128_GCM_SHA256 +)EOF"; + + TestUtilOptions successful_test_options(successful_client_ctx_yaml, server_ctx_yaml, true, + GetParam()); + testUtil(successful_test_options.setPrivateKeyMethodExpected(true)); +} + +// Test synchronous signing (ECDHE). +TEST_P(SslSocketTest, RsaPrivateKeyProviderSyncSignSuccess) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + sync_mode: true + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string successful_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - ECDHE-RSA-AES128-GCM-SHA256 +)EOF"; + + TestUtilOptions successful_test_options(successful_client_ctx_yaml, server_ctx_yaml, true, + GetParam()); + testUtil(successful_test_options.setPrivateKeyMethodExpected(true)); +} + +// Test synchronous decryption (RSA). +TEST_P(SslSocketTest, RsaPrivateKeyProviderSyncDecryptSuccess) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: decrypt + sync_mode: true + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string successful_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - TLS_RSA_WITH_AES_128_GCM_SHA256 +)EOF"; + + TestUtilOptions successful_test_options(successful_client_ctx_yaml, server_ctx_yaml, true, + GetParam()); + testUtil(successful_test_options.setPrivateKeyMethodExpected(true)); +} + +// Test asynchronous signing (ECDHE) failure (invalid signature). +TEST_P(SslSocketTest, RsaPrivateKeyProviderAsyncSignFailure) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + sync_mode: false + crypto_error: true + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string failing_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - ECDHE-RSA-AES128-GCM-SHA256 +)EOF"; + + TestUtilOptions failing_test_options(failing_client_ctx_yaml, server_ctx_yaml, false, GetParam()); + testUtil(failing_test_options.setPrivateKeyMethodExpected(true).setExpectedServerStats( + "ssl.connection_error")); +} + +// Test synchronous signing (ECDHE) failure (invalid signature). +TEST_P(SslSocketTest, RsaPrivateKeyProviderSyncSignFailure) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + sync_mode: true + crypto_error: true + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string failing_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - ECDHE-RSA-AES128-GCM-SHA256 +)EOF"; + + TestUtilOptions failing_test_options(failing_client_ctx_yaml, server_ctx_yaml, false, GetParam()); + testUtil(failing_test_options.setPrivateKeyMethodExpected(true).setExpectedServerStats( + "ssl.connection_error")); +} + +// Test the sign operation return with an error. +TEST_P(SslSocketTest, RsaPrivateKeyProviderSignFailure) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + method_error: true + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string failing_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - ECDHE-RSA-AES128-GCM-SHA256 +)EOF"; + + TestUtilOptions failing_test_options(failing_client_ctx_yaml, server_ctx_yaml, false, GetParam()); + testUtil(failing_test_options.setPrivateKeyMethodExpected(true).setExpectedServerStats( + "ssl.connection_error")); +} + +// Test the decrypt operation return with an error. +TEST_P(SslSocketTest, RsaPrivateKeyProviderDecryptFailure) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: decrypt + method_error: true + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string failing_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - TLS_RSA_WITH_AES_128_GCM_SHA256 +)EOF"; + + TestUtilOptions failing_test_options(failing_client_ctx_yaml, server_ctx_yaml, false, GetParam()); + testUtil(failing_test_options.setPrivateKeyMethodExpected(true).setExpectedServerStats( + "ssl.connection_error")); +} + +// Test the sign operation return with an error in complete. +TEST_P(SslSocketTest, RsaPrivateKeyProviderAsyncSignCompleteFailure) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + async_method_error: true + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string failing_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - ECDHE-RSA-AES128-GCM-SHA256 +)EOF"; + + TestUtilOptions failing_test_options(failing_client_ctx_yaml, server_ctx_yaml, false, GetParam()); + testUtil(failing_test_options.setPrivateKeyMethodExpected(true) + .setExpectedServerCloseEvent(Network::ConnectionEvent::LocalClose) + .setExpectedServerStats("ssl.connection_error")); +} + +// Test the decrypt operation return with an error in complete. +TEST_P(SslSocketTest, RsaPrivateKeyProviderAsyncDecryptCompleteFailure) { + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + certificate_chain: + filename: "{{ test_tmpdir }}/unittestcert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: decrypt + async_method_error: true + mode: rsa + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.pem" + crl: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/ca_cert.crl" +)EOF"; + const std::string failing_client_ctx_yaml = R"EOF( + common_tls_context: + tls_params: + cipher_suites: + - TLS_RSA_WITH_AES_128_GCM_SHA256 +)EOF"; + + TestUtilOptions failing_test_options(failing_client_ctx_yaml, server_ctx_yaml, false, GetParam()); + testUtil(failing_test_options.setPrivateKeyMethodExpected(true) + .setExpectedServerCloseEvent(Network::ConnectionEvent::LocalClose) + .setExpectedServerStats("ssl.connection_error")); +} + +// Test having one cert with private key method and another with just +// private key. +TEST_P(SslSocketTest, RsaPrivateKeyProviderMultiCertSuccess) { + const std::string client_ctx_yaml = absl::StrCat(R"EOF( + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_2 + tls_maximum_protocol_version: TLSv1_2 + cipher_suites: + - ECDHE-ECDSA-AES128-GCM-SHA256 + - ECDHE-RSA-AES128-GCM-SHA256 + validation_context: + verify_certificate_hash: )EOF", + TEST_SELFSIGNED_ECDSA_P256_CERT_HASH); + + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + sync_mode: false + mode: rsa + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_cert.pem" + private_key: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_key.pem" +)EOF"; + + TestUtilOptions test_options(client_ctx_yaml, server_ctx_yaml, true, GetParam()); + testUtil(test_options.setPrivateKeyMethodExpected(true)); +} + +// Test having two certs with private key methods. This will +// synchronously fail because the second certificate is a ECDSA one and +// the RSA method can't handle it. +TEST_P(SslSocketTest, RsaPrivateKeyProviderMultiCertFail) { + const std::string client_ctx_yaml = absl::StrCat(R"EOF( + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_2 + tls_maximum_protocol_version: TLSv1_2 + cipher_suites: + - ECDHE-ECDSA-AES128-GCM-SHA256 + - ECDHE-RSA-AES128-GCM-SHA256 + validation_context: + verify_certificate_hash: )EOF", + TEST_SELFSIGNED_ECDSA_P256_CERT_HASH); + + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + sync_mode: false + mode: rsa + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_cert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_key.pem" + expected_operation: sign + sync_mode: false + mode: rsa +)EOF"; + + TestUtilOptions failing_test_options(client_ctx_yaml, server_ctx_yaml, false, GetParam()); + EXPECT_THROW_WITH_MESSAGE(testUtil(failing_test_options.setPrivateKeyMethodExpected(true)), + EnvoyException, "Private key is not RSA.") +} + +// Test ECDSA private key method provider mode. +TEST_P(SslSocketTest, EcdsaPrivateKeyProviderSuccess) { + const std::string client_ctx_yaml = absl::StrCat(R"EOF( + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_2 + tls_maximum_protocol_version: TLSv1_2 + cipher_suites: + - ECDHE-ECDSA-AES128-GCM-SHA256 + validation_context: + verify_certificate_hash: )EOF", + TEST_SELFSIGNED_ECDSA_P256_CERT_HASH); + + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_cert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_key.pem" + expected_operation: sign + mode: ecdsa +)EOF"; + + TestUtilOptions test_options(client_ctx_yaml, server_ctx_yaml, true, GetParam()); + testUtil(test_options.setPrivateKeyMethodExpected(true)); +} + +// Test having two certs with different private key method modes. It's expected that the ECDSA +// provider mode is being used. RSA provider mode is set to fail with "async_method_error", but +// that's not happening. +TEST_P(SslSocketTest, RsaAndEcdsaPrivateKeyProviderMultiCertSuccess) { + const std::string client_ctx_yaml = absl::StrCat(R"EOF( + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_2 + tls_maximum_protocol_version: TLSv1_2 + cipher_suites: + - ECDHE-ECDSA-AES128-GCM-SHA256 + - ECDHE-RSA-AES128-GCM-SHA256 + validation_context: + verify_certificate_hash: )EOF", + TEST_SELFSIGNED_ECDSA_P256_CERT_HASH); + + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + sync_mode: false + async_method_error: true + mode: rsa + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_cert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_key.pem" + expected_operation: sign + mode: ecdsa +)EOF"; + TestUtilOptions test_options(client_ctx_yaml, server_ctx_yaml, true, GetParam()); + testUtil(test_options.setPrivateKeyMethodExpected(true)); +} + +// Test having two certs with different private key method modes. ECDSA provider is set to fail. +TEST_P(SslSocketTest, RsaAndEcdsaPrivateKeyProviderMultiCertFail) { + const std::string client_ctx_yaml = absl::StrCat(R"EOF( + common_tls_context: + tls_params: + tls_minimum_protocol_version: TLSv1_2 + tls_maximum_protocol_version: TLSv1_2 + cipher_suites: + - ECDHE-ECDSA-AES128-GCM-SHA256 + - ECDHE-RSA-AES128-GCM-SHA256 + validation_context: + verify_certificate_hash: )EOF", + TEST_SELFSIGNED_ECDSA_P256_CERT_HASH); + + const std::string server_ctx_yaml = R"EOF( + common_tls_context: + tls_certificates: + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_cert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_tmpdir }}/unittestkey.pem" + expected_operation: sign + sync_mode: false + mode: rsa + - certificate_chain: + filename: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_cert.pem" + private_key_provider: + provider_name: test + config: + private_key_file: "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/selfsigned_ecdsa_p256_key.pem" + expected_operation: sign + async_method_error: true + mode: ecdsa +)EOF"; + TestUtilOptions failing_test_options(client_ctx_yaml, server_ctx_yaml, false, GetParam()); + testUtil(failing_test_options.setPrivateKeyMethodExpected(true) + .setExpectedServerCloseEvent(Network::ConnectionEvent::LocalClose) + .setExpectedServerStats("ssl.connection_error")); +} + } // namespace Tls } // namespace TransportSockets } // namespace Extensions diff --git a/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc new file mode 100644 index 000000000000..cf78fbf3a304 --- /dev/null +++ b/test/extensions/transport_sockets/tls/test_private_key_method_provider.cc @@ -0,0 +1,377 @@ +#include "test/extensions/transport_sockets/tls/test_private_key_method_provider.h" + +#include + +#include "envoy/api/api.h" + +#include "openssl/ssl.h" + +namespace Envoy { +namespace Extensions { +namespace PrivateKeyMethodProvider { + +void TestPrivateKeyConnection::delayed_op() { + const std::chrono::milliseconds timeout_0ms{0}; + + timer_ = dispatcher_.createTimer([this]() -> void { + finished_ = true; + this->cb_.onPrivateKeyMethodComplete(); + }); + timer_->enableTimer(timeout_0ms); +} + +static int calculateDigest(const EVP_MD* md, const uint8_t* in, size_t in_len, unsigned char* hash, + unsigned int* hash_len) { + bssl::ScopedEVP_MD_CTX ctx; + + // Calculate the message digest for signing. + if (!EVP_DigestInit_ex(ctx.get(), md, nullptr) || !EVP_DigestUpdate(ctx.get(), in, in_len) || + !EVP_DigestFinal_ex(ctx.get(), hash, hash_len)) { + return 0; + } + return 1; +} + +static ssl_private_key_result_t ecdsaPrivateKeySign(SSL* ssl, uint8_t* out, size_t* out_len, + size_t max_out, uint16_t signature_algorithm, + const uint8_t* in, size_t in_len) { + unsigned char hash[EVP_MAX_MD_SIZE]; + unsigned int hash_len; + TestPrivateKeyConnection* ops = static_cast( + SSL_get_ex_data(ssl, TestPrivateKeyMethodProvider::ecdsaConnectionIndex())); + unsigned int out_len_unsigned; + + if (!ops) { + return ssl_private_key_failure; + } + + if (ops->test_options_.method_error_) { + // Have an artificial test failure. + return ssl_private_key_failure; + } + + if (!ops->test_options_.sign_expected_) { + return ssl_private_key_failure; + } + + const EVP_MD* md = SSL_get_signature_algorithm_digest(signature_algorithm); + if (!md) { + return ssl_private_key_failure; + } + + if (!calculateDigest(md, in, in_len, hash, &hash_len)) { + return ssl_private_key_failure; + } + + bssl::UniquePtr ec_key(EVP_PKEY_get1_EC_KEY(ops->getPrivateKey())); + if (!ec_key) { + return ssl_private_key_failure; + } + + // Borrow "out" because it has been already initialized to the max_out size. + if (!ECDSA_sign(0, hash, hash_len, out, &out_len_unsigned, ec_key.get())) { + return ssl_private_key_failure; + } + + if (ops->test_options_.sync_mode_) { + // Return immediately with the results. + if (out_len_unsigned > max_out) { + return ssl_private_key_failure; + } + *out_len = out_len_unsigned; + return ssl_private_key_success; + } + + ops->output_.assign(out, out + out_len_unsigned); + // Tell SSL socket that the operation is ready to be called again. + ops->delayed_op(); + + return ssl_private_key_retry; +} + +static ssl_private_key_result_t ecdsaPrivateKeyDecrypt(SSL*, uint8_t*, size_t*, size_t, + const uint8_t*, size_t) { + return ssl_private_key_failure; +} + +static ssl_private_key_result_t rsaPrivateKeySign(SSL* ssl, uint8_t* out, size_t* out_len, + size_t max_out, uint16_t signature_algorithm, + const uint8_t* in, size_t in_len) { + TestPrivateKeyConnection* ops = static_cast( + SSL_get_ex_data(ssl, TestPrivateKeyMethodProvider::rsaConnectionIndex())); + unsigned char hash[EVP_MAX_MD_SIZE]; + unsigned int hash_len; + + if (!ops) { + return ssl_private_key_failure; + } + + if (ops->test_options_.method_error_) { + return ssl_private_key_failure; + } + + if (!ops->test_options_.sign_expected_) { + return ssl_private_key_failure; + } + + const EVP_MD* md = SSL_get_signature_algorithm_digest(signature_algorithm); + if (!md) { + return ssl_private_key_failure; + } + + if (!calculateDigest(md, in, in_len, hash, &hash_len)) { + return ssl_private_key_failure; + } + + RSA* rsa = EVP_PKEY_get0_RSA(ops->getPrivateKey()); + if (rsa == nullptr) { + return ssl_private_key_failure; + } + + if (ops->test_options_.crypto_error_) { + // Flip the bits in the first byte of the digest so that the handshake will fail. + hash[0] ^= hash[0]; + } + + // Perform RSA signing. + if (SSL_is_signature_algorithm_rsa_pss(signature_algorithm)) { + RSA_sign_pss_mgf1(rsa, out_len, out, max_out, hash, hash_len, md, nullptr, -1); + } else { + unsigned int out_len_unsigned; + if (!RSA_sign(EVP_MD_type(md), hash, hash_len, out, &out_len_unsigned, rsa)) { + return ssl_private_key_failure; + } + if (out_len_unsigned > max_out) { + return ssl_private_key_failure; + } + *out_len = out_len_unsigned; + } + + if (ops->test_options_.sync_mode_) { + return ssl_private_key_success; + } + + ops->output_.assign(out, out + *out_len); + ops->delayed_op(); + + return ssl_private_key_retry; +} + +static ssl_private_key_result_t rsaPrivateKeyDecrypt(SSL* ssl, uint8_t* out, size_t* out_len, + size_t max_out, const uint8_t* in, + size_t in_len) { + TestPrivateKeyConnection* ops = static_cast( + SSL_get_ex_data(ssl, TestPrivateKeyMethodProvider::rsaConnectionIndex())); + + if (!ops) { + return ssl_private_key_failure; + } + + if (ops->test_options_.method_error_) { + return ssl_private_key_failure; + } + + if (!ops->test_options_.decrypt_expected_) { + return ssl_private_key_failure; + } + + RSA* rsa = EVP_PKEY_get0_RSA(ops->getPrivateKey()); + if (rsa == nullptr) { + return ssl_private_key_failure; + } + + if (!RSA_decrypt(rsa, out_len, out, max_out, in, in_len, RSA_NO_PADDING)) { + return ssl_private_key_failure; + } + + if (ops->test_options_.sync_mode_) { + return ssl_private_key_success; + } + + ops->output_.assign(out, out + *out_len); + ops->delayed_op(); + + return ssl_private_key_retry; +} + +static ssl_private_key_result_t privateKeyComplete(SSL* ssl, uint8_t* out, size_t* out_len, + size_t max_out, int id) { + TestPrivateKeyConnection* ops = static_cast(SSL_get_ex_data(ssl, id)); + + if (!ops->finished_) { + // The operation didn't finish yet, retry. + return ssl_private_key_retry; + } + + if (ops->test_options_.async_method_error_) { + return ssl_private_key_failure; + } + + if (ops->output_.size() > max_out) { + return ssl_private_key_failure; + } + + std::copy(ops->output_.begin(), ops->output_.end(), out); + *out_len = ops->output_.size(); + + return ssl_private_key_success; +} + +static ssl_private_key_result_t rsaPrivateKeyComplete(SSL* ssl, uint8_t* out, size_t* out_len, + size_t max_out) { + return privateKeyComplete(ssl, out, out_len, max_out, + TestPrivateKeyMethodProvider::rsaConnectionIndex()); +} + +static ssl_private_key_result_t ecdsaPrivateKeyComplete(SSL* ssl, uint8_t* out, size_t* out_len, + size_t max_out) { + return privateKeyComplete(ssl, out, out_len, max_out, + TestPrivateKeyMethodProvider::ecdsaConnectionIndex()); +} + +Ssl::BoringSslPrivateKeyMethodSharedPtr +TestPrivateKeyMethodProvider::getBoringSslPrivateKeyMethod() { + return method_; +} + +bool TestPrivateKeyMethodProvider::checkFips() { + if (mode_ == "rsa") { + RSA* rsa_private_key = EVP_PKEY_get0_RSA(pkey_.get()); + if (rsa_private_key == nullptr || !RSA_check_fips(rsa_private_key)) { + return false; + } + } else { // if (mode_ == "ecdsa") + const EC_KEY* ecdsa_private_key = EVP_PKEY_get0_EC_KEY(pkey_.get()); + if (ecdsa_private_key == nullptr || !EC_KEY_check_fips(ecdsa_private_key)) { + return false; + } + } + return true; +} + +TestPrivateKeyConnection::TestPrivateKeyConnection( + Ssl::PrivateKeyConnectionCallbacks& cb, Event::Dispatcher& dispatcher, + bssl::UniquePtr pkey, TestPrivateKeyConnectionTestOptions& test_options) + : test_options_(test_options), cb_(cb), dispatcher_(dispatcher), pkey_(std::move(pkey)) {} + +void TestPrivateKeyMethodProvider::registerPrivateKeyMethod(SSL* ssl, + Ssl::PrivateKeyConnectionCallbacks& cb, + Event::Dispatcher& dispatcher) { + TestPrivateKeyConnection* ops; + // In multi-cert case, when the same provider is used in different modes with the same SSL object, + // we need to keep both rsa and ecdsa connection objects in store because the test options for the + // two certificates may be different. We need to be able to deduct in the signing, decryption, and + // completion functions which options to use, so we associate the connection objects to the same + // SSL object using different user data indexes. + // + // Another way to do this would be to store both test options in one connection object. + int index = mode_ == "rsa" ? TestPrivateKeyMethodProvider::rsaConnectionIndex() + : TestPrivateKeyMethodProvider::ecdsaConnectionIndex(); + + // Check if there is another certificate of the same mode associated with the context. This would + // be an error. + ops = static_cast(SSL_get_ex_data(ssl, index)); + if (ops != nullptr) { + throw EnvoyException( + "Can't distinguish between two registered providers for the same SSL object."); + } + + ops = new TestPrivateKeyConnection(cb, dispatcher, bssl::UpRef(pkey_), test_options_); + SSL_set_ex_data(ssl, index, ops); +} + +void TestPrivateKeyMethodProvider::unregisterPrivateKeyMethod(SSL* ssl) { + int index = mode_ == "rsa" ? TestPrivateKeyMethodProvider::rsaConnectionIndex() + : TestPrivateKeyMethodProvider::ecdsaConnectionIndex(); + TestPrivateKeyConnection* ops = + static_cast(SSL_get_ex_data(ssl, index)); + SSL_set_ex_data(ssl, index, nullptr); + delete ops; +} + +static int createIndex() { + int index = SSL_get_ex_new_index(0, nullptr, nullptr, nullptr, nullptr); + RELEASE_ASSERT(index >= 0, "Failed to get SSL user data index."); + return index; +} + +int TestPrivateKeyMethodProvider::rsaConnectionIndex() { + CONSTRUCT_ON_FIRST_USE(int, createIndex()); +} + +int TestPrivateKeyMethodProvider::ecdsaConnectionIndex() { + CONSTRUCT_ON_FIRST_USE(int, createIndex()); +} + +TestPrivateKeyMethodProvider::TestPrivateKeyMethodProvider( + const ProtobufWkt::Struct& config, + Server::Configuration::TransportSocketFactoryContext& factory_context) { + std::string private_key_path; + + for (auto& value_it : config.fields()) { + auto& value = value_it.second; + if (value_it.first == "private_key_file" && + value.kind_case() == ProtobufWkt::Value::kStringValue) { + private_key_path = value.string_value(); + } + if (value_it.first == "sync_mode" && value.kind_case() == ProtobufWkt::Value::kBoolValue) { + test_options_.sync_mode_ = value.bool_value(); + } + if (value_it.first == "crypto_error" && value.kind_case() == ProtobufWkt::Value::kBoolValue) { + test_options_.crypto_error_ = value.bool_value(); + } + if (value_it.first == "method_error" && value.kind_case() == ProtobufWkt::Value::kBoolValue) { + test_options_.method_error_ = value.bool_value(); + } + if (value_it.first == "async_method_error" && + value.kind_case() == ProtobufWkt::Value::kBoolValue) { + test_options_.async_method_error_ = value.bool_value(); + } + if (value_it.first == "expected_operation" && + value.kind_case() == ProtobufWkt::Value::kStringValue) { + if (value.string_value() == "decrypt") { + test_options_.decrypt_expected_ = true; + } else if (value.string_value() == "sign") { + test_options_.sign_expected_ = true; + } + } + if (value_it.first == "mode" && value.kind_case() == ProtobufWkt::Value::kStringValue) { + mode_ = value.string_value(); + } + } + + std::string private_key = factory_context.api().fileSystem().fileReadToEnd(private_key_path); + bssl::UniquePtr bio( + BIO_new_mem_buf(const_cast(private_key.data()), private_key.size())); + bssl::UniquePtr pkey(PEM_read_bio_PrivateKey(bio.get(), nullptr, nullptr, nullptr)); + if (pkey == nullptr) { + throw EnvoyException("Failed to read private key from disk."); + } + + method_ = std::make_shared(); + + // Have two modes, "rsa" and "ecdsa", for testing multi-cert use cases. + if (mode_ == "rsa") { + if (EVP_PKEY_id(pkey.get()) != EVP_PKEY_RSA) { + throw EnvoyException("Private key is not RSA."); + } + method_->sign = rsaPrivateKeySign; + method_->decrypt = rsaPrivateKeyDecrypt; + method_->complete = rsaPrivateKeyComplete; + } else if (mode_ == "ecdsa") { + if (EVP_PKEY_id(pkey.get()) != EVP_PKEY_EC) { + throw EnvoyException("Private key is not ECDSA."); + } + method_->sign = ecdsaPrivateKeySign; + method_->decrypt = ecdsaPrivateKeyDecrypt; + method_->complete = ecdsaPrivateKeyComplete; + } else { + throw EnvoyException("Unknown test provider mode, supported modes are \"rsa\" and \"ecdsa\"."); + } + + pkey_ = std::move(pkey); +} + +} // namespace PrivateKeyMethodProvider +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/transport_sockets/tls/test_private_key_method_provider.h b/test/extensions/transport_sockets/tls/test_private_key_method_provider.h new file mode 100644 index 000000000000..6aadf9301077 --- /dev/null +++ b/test/extensions/transport_sockets/tls/test_private_key_method_provider.h @@ -0,0 +1,96 @@ +#pragma once + +#include "envoy/event/dispatcher.h" +#include "envoy/server/transport_socket_config.h" +#include "envoy/ssl/private_key/private_key.h" +#include "envoy/ssl/private_key/private_key_config.h" + +#include "common/config/utility.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Extensions { +namespace PrivateKeyMethodProvider { + +struct TestPrivateKeyConnectionTestOptions { + // Return private key method value directly without asynchronous operation. + bool sync_mode_{}; + + // The "decrypt" private key method is expected to he called. + bool decrypt_expected_{}; + + // The "sign" private key method is expected to he called. + bool sign_expected_{}; + + // Add a cryptographic error (invalid signature, incorrect decryption). + bool crypto_error_{}; + + // Return an error from the private key method. + bool method_error_{}; + + // Return an error from the private key method completion function. + bool async_method_error_{}; +}; + +// An example private key method provider for testing the decrypt() and sign() +// functionality. +class TestPrivateKeyConnection { +public: + TestPrivateKeyConnection(Ssl::PrivateKeyConnectionCallbacks& cb, Event::Dispatcher& dispatcher, + bssl::UniquePtr pkey, + TestPrivateKeyConnectionTestOptions& test_options); + EVP_PKEY* getPrivateKey() { return pkey_.get(); } + void delayed_op(); + // Store the output data temporarily. + std::vector output_; + // The complete callback can return other value than "retry" only after + // onPrivateKeyMethodComplete() function has been called. This is controlled by "finished" + // variable. + bool finished_{}; + TestPrivateKeyConnectionTestOptions& test_options_; + +private: + Ssl::PrivateKeyConnectionCallbacks& cb_; + Event::Dispatcher& dispatcher_; + bssl::UniquePtr pkey_; + // A zero-length timer controls the callback. + Event::TimerPtr timer_; +}; + +class TestPrivateKeyMethodProvider : public virtual Ssl::PrivateKeyMethodProvider { +public: + TestPrivateKeyMethodProvider( + const ProtobufWkt::Struct& config, + Server::Configuration::TransportSocketFactoryContext& factory_context); + // Ssl::PrivateKeyMethodProvider + void registerPrivateKeyMethod(SSL* ssl, Ssl::PrivateKeyConnectionCallbacks& cb, + Event::Dispatcher& dispatcher) override; + void unregisterPrivateKeyMethod(SSL* ssl) override; + bool checkFips() override; + Ssl::BoringSslPrivateKeyMethodSharedPtr getBoringSslPrivateKeyMethod() override; + + static int rsaConnectionIndex(); + static int ecdsaConnectionIndex(); + +private: + Ssl::BoringSslPrivateKeyMethodSharedPtr method_{}; + bssl::UniquePtr pkey_; + TestPrivateKeyConnectionTestOptions test_options_; + std::string mode_; +}; + +class TestPrivateKeyMethodFactory : public Ssl::PrivateKeyMethodProviderInstanceFactory { +public: + // Ssl::PrivateKeyMethodProviderInstanceFactory + Ssl::PrivateKeyMethodProviderSharedPtr createPrivateKeyMethodProviderInstance( + const envoy::api::v2::auth::PrivateKeyProvider& config, + Server::Configuration::TransportSocketFactoryContext& factory_context) override { + return std::make_shared(config.config(), factory_context); + } + + std::string name() const override { return std::string("test"); }; +}; + +} // namespace PrivateKeyMethodProvider +} // namespace Extensions +} // namespace Envoy diff --git a/test/mocks/event/mocks.h b/test/mocks/event/mocks.h index 327214a6903f..81e133dd50d1 100644 --- a/test/mocks/event/mocks.h +++ b/test/mocks/event/mocks.h @@ -118,6 +118,7 @@ class MockDispatcher : public Dispatcher { MOCK_METHOD1(setTrackedObject, const ScopeTrackedObject*(const ScopeTrackedObject* object)); MOCK_CONST_METHOD0(isThreadSafe, bool()); Buffer::WatermarkFactory& getWatermarkFactory() override { return buffer_factory_; } + MOCK_METHOD0(getCurrentThreadId, Thread::ThreadId()); GlobalTimeSystem time_system_; std::list to_delete_; diff --git a/test/mocks/ssl/mocks.cc b/test/mocks/ssl/mocks.cc index 72702de823ed..50ed3f3ae6c0 100644 --- a/test/mocks/ssl/mocks.cc +++ b/test/mocks/ssl/mocks.cc @@ -18,5 +18,11 @@ MockClientContextConfig::~MockClientContextConfig() = default; MockServerContextConfig::MockServerContextConfig() = default; MockServerContextConfig::~MockServerContextConfig() = default; +MockPrivateKeyMethodManager::MockPrivateKeyMethodManager() = default; +MockPrivateKeyMethodManager::~MockPrivateKeyMethodManager() = default; + +MockPrivateKeyMethodProvider::MockPrivateKeyMethodProvider() = default; +MockPrivateKeyMethodProvider::~MockPrivateKeyMethodProvider() = default; + } // namespace Ssl } // namespace Envoy diff --git a/test/mocks/ssl/mocks.h b/test/mocks/ssl/mocks.h index 20621bd29ca4..957cbf05c87c 100644 --- a/test/mocks/ssl/mocks.h +++ b/test/mocks/ssl/mocks.h @@ -29,6 +29,7 @@ class MockContextManager : public ContextManager { const std::vector& server_names)); MOCK_CONST_METHOD0(daysUntilFirstCertExpires, size_t()); MOCK_METHOD1(iterateContexts, void(std::function callback)); + MOCK_METHOD0(privateKeyMethodManager, Ssl::PrivateKeyMethodManager&()); }; class MockConnectionInfo : public ConnectionInfo { @@ -108,5 +109,28 @@ class MockServerContextConfig : public ServerContextConfig { MOCK_CONST_METHOD0(sessionTicketKeys, const std::vector&()); }; +class MockPrivateKeyMethodManager : public PrivateKeyMethodManager { +public: + MockPrivateKeyMethodManager(); + ~MockPrivateKeyMethodManager() override; + + MOCK_METHOD2(createPrivateKeyMethodProvider, + PrivateKeyMethodProviderSharedPtr( + const envoy::api::v2::auth::PrivateKeyProvider& config, + Envoy::Server::Configuration::TransportSocketFactoryContext& factory_context)); +}; + +class MockPrivateKeyMethodProvider : public PrivateKeyMethodProvider { +public: + MockPrivateKeyMethodProvider(); + ~MockPrivateKeyMethodProvider() override; + + MOCK_METHOD3(registerPrivateKeyMethod, + void(SSL* ssl, PrivateKeyConnectionCallbacks& cb, Event::Dispatcher& dispatcher)); + MOCK_METHOD1(unregisterPrivateKeyMethod, void(SSL* ssl)); + MOCK_METHOD0(checkFips, bool()); + MOCK_METHOD0(getBoringSslPrivateKeyMethod, BoringSslPrivateKeyMethodSharedPtr()); +}; + } // namespace Ssl } // namespace Envoy diff --git a/tools/check_format.py b/tools/check_format.py index 67292cea223b..40953acc8077 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -141,7 +141,6 @@ "extensions/common/tap", "extensions/transport_sockets/raw_buffer", "extensions/transport_sockets/tap", - "extensions/transport_sockets/tls", "extensions/tracers/zipkin", "extensions/tracers/dynamic_ot", "extensions/tracers/opencensus", diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 98cfe52c85a8..bb101fa95946 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -200,6 +200,7 @@ POSTs PREBIND PRNG PROT +PSS QPS QUIC RAII @@ -270,6 +271,7 @@ TLSv TLV TMPDIR TODO +TPM TPROXY TSAN TSI From b44a00bfb033903a552ea2f3a55e9086df26d0c5 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Fri, 23 Aug 2019 08:17:37 +0900 Subject: [PATCH 435/542] use SymbolTableCreator rather than fakes in a few stray places. (#8006) stats: use SymbolTableCreator rather than fakes in a few stray places. (#8006) Signed-off-by: Joshua Marantz --- test/common/stats/BUILD | 6 +-- test/common/stats/allocator_impl_test.cc | 12 +++-- test/common/stats/metric_impl_test.cc | 10 ++-- test/common/stats/stat_merger_test.cc | 6 +-- .../stats/thread_local_store_speed_test.cc | 10 ++-- test/common/stats/thread_local_store_test.cc | 52 ++++++++++--------- 6 files changed, 52 insertions(+), 44 deletions(-) diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index 78157bdee706..ab725c0a2608 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -15,7 +15,7 @@ envoy_cc_test( srcs = ["allocator_impl_test.cc"], deps = [ "//source/common/stats:allocator_lib", - "//source/common/stats:fake_symbol_table_lib", + "//source/common/stats:symbol_table_creator_lib", "//test/test_common:logging_lib", ], ) @@ -33,7 +33,7 @@ envoy_cc_test( srcs = ["metric_impl_test.cc"], deps = [ "//source/common/stats:allocator_lib", - "//source/common/stats:fake_symbol_table_lib", + "//source/common/stats:symbol_table_creator_lib", "//source/common/stats:utility_lib", "//test/test_common:logging_lib", ], @@ -43,9 +43,9 @@ envoy_cc_test( name = "stat_merger_test", srcs = ["stat_merger_test.cc"], deps = [ - "//source/common/stats:fake_symbol_table_lib", "//source/common/stats:isolated_store_lib", "//source/common/stats:stat_merger_lib", + "//source/common/stats:symbol_table_creator_lib", "//source/common/stats:thread_local_store_lib", "//test/test_common:utility_lib", ], diff --git a/test/common/stats/allocator_impl_test.cc b/test/common/stats/allocator_impl_test.cc index 37339b138478..f1b646a46b29 100644 --- a/test/common/stats/allocator_impl_test.cc +++ b/test/common/stats/allocator_impl_test.cc @@ -1,7 +1,7 @@ #include #include "common/stats/allocator_impl.h" -#include "common/stats/fake_symbol_table_impl.h" +#include "common/stats/symbol_table_creator.h" #include "test/test_common/logging.h" @@ -13,21 +13,23 @@ namespace { class AllocatorImplTest : public testing::Test { protected: - AllocatorImplTest() : alloc_(symbol_table_), pool_(symbol_table_) {} + AllocatorImplTest() + : symbol_table_(SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_), + pool_(*symbol_table_) {} ~AllocatorImplTest() override { clearStorage(); } StatNameStorage makeStatStorage(absl::string_view name) { - return StatNameStorage(name, symbol_table_); + return StatNameStorage(name, *symbol_table_); } StatName makeStat(absl::string_view name) { return pool_.add(name); } void clearStorage() { pool_.clear(); - EXPECT_EQ(0, symbol_table_.numSymbols()); + EXPECT_EQ(0, symbol_table_->numSymbols()); } - FakeSymbolTableImpl symbol_table_; + SymbolTablePtr symbol_table_; AllocatorImpl alloc_; StatNamePool pool_; }; diff --git a/test/common/stats/metric_impl_test.cc b/test/common/stats/metric_impl_test.cc index 16e90a220199..b178842fa865 100644 --- a/test/common/stats/metric_impl_test.cc +++ b/test/common/stats/metric_impl_test.cc @@ -1,7 +1,7 @@ #include #include "common/stats/allocator_impl.h" -#include "common/stats/fake_symbol_table_impl.h" +#include "common/stats/symbol_table_creator.h" #include "common/stats/utility.h" #include "test/test_common/logging.h" @@ -14,17 +14,19 @@ namespace { class MetricImplTest : public testing::Test { protected: - MetricImplTest() : alloc_(symbol_table_), pool_(symbol_table_) {} + MetricImplTest() + : symbol_table_(SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_), + pool_(*symbol_table_) {} ~MetricImplTest() override { clearStorage(); } StatName makeStat(absl::string_view name) { return pool_.add(name); } void clearStorage() { pool_.clear(); - EXPECT_EQ(0, symbol_table_.numSymbols()); + EXPECT_EQ(0, symbol_table_->numSymbols()); } - FakeSymbolTableImpl symbol_table_; + SymbolTablePtr symbol_table_; AllocatorImpl alloc_; StatNamePool pool_; }; diff --git a/test/common/stats/stat_merger_test.cc b/test/common/stats/stat_merger_test.cc index f56e8d98c718..4168bef2218c 100644 --- a/test/common/stats/stat_merger_test.cc +++ b/test/common/stats/stat_merger_test.cc @@ -1,8 +1,8 @@ #include -#include "common/stats/fake_symbol_table_impl.h" #include "common/stats/isolated_store_impl.h" #include "common/stats/stat_merger.h" +#include "common/stats/symbol_table_creator.h" #include "common/stats/thread_local_store.h" #include "test/test_common/utility.h" @@ -182,8 +182,8 @@ TEST_F(StatMergerTest, gaugeMergeImportMode) { class StatMergerThreadLocalTest : public testing::Test { protected: - FakeSymbolTableImpl symbol_table_; - AllocatorImpl alloc_{symbol_table_}; + SymbolTablePtr symbol_table_{SymbolTableCreator::makeSymbolTable()}; + AllocatorImpl alloc_{*symbol_table_}; ThreadLocalStoreImpl store_{alloc_}; }; diff --git a/test/common/stats/thread_local_store_speed_test.cc b/test/common/stats/thread_local_store_speed_test.cc index 877c6bc7bb09..6ea30cb61e84 100644 --- a/test/common/stats/thread_local_store_speed_test.cc +++ b/test/common/stats/thread_local_store_speed_test.cc @@ -22,18 +22,18 @@ namespace Envoy { class ThreadLocalStorePerf { public: ThreadLocalStorePerf() - : heap_alloc_(symbol_table_), store_(heap_alloc_), - api_(Api::createApiForTest(store_, time_system_)) { + : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), heap_alloc_(*symbol_table_), + store_(heap_alloc_), api_(Api::createApiForTest(store_, time_system_)) { store_.setTagProducer(std::make_unique(stats_config_)); Stats::TestUtil::forEachSampleStat(1000, [this](absl::string_view name) { - stat_names_.push_back(std::make_unique(name, symbol_table_)); + stat_names_.push_back(std::make_unique(name, *symbol_table_)); }); } ~ThreadLocalStorePerf() { for (auto& stat_name_storage : stat_names_) { - stat_name_storage->free(symbol_table_); + stat_name_storage->free(*symbol_table_); } store_.shutdownThreading(); if (tls_) { @@ -54,7 +54,7 @@ class ThreadLocalStorePerf { } private: - Stats::FakeSymbolTableImpl symbol_table_; + Stats::SymbolTablePtr symbol_table_; Event::SimulatedTimeSystem time_system_; Stats::AllocatorImpl heap_alloc_; Stats::ThreadLocalStoreImpl store_; diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index be0b87058ced..fb4b2d594ee4 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -39,7 +39,8 @@ const uint64_t MaxStatNameLength = 127; class StatsThreadLocalStoreTest : public testing::Test { public: StatsThreadLocalStoreTest() - : alloc_(symbol_table_), store_(std::make_unique(alloc_)) { + : symbol_table_(SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_), + store_(std::make_unique(alloc_)) { store_->addSink(sink_); } @@ -48,7 +49,7 @@ class StatsThreadLocalStoreTest : public testing::Test { store_->addSink(sink_); } - Stats::FakeSymbolTableImpl symbol_table_; + SymbolTablePtr symbol_table_; NiceMock main_thread_dispatcher_; NiceMock tls_; AllocatorImpl alloc_; @@ -78,7 +79,7 @@ class HistogramTest : public testing::Test { public: using NameHistogramMap = std::map; - HistogramTest() : alloc_(symbol_table_) {} + HistogramTest() : symbol_table_(SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_) {} void SetUp() override { store_ = std::make_unique(alloc_); @@ -166,7 +167,7 @@ class HistogramTest : public testing::Test { } } - FakeSymbolTableImpl symbol_table_; + SymbolTablePtr symbol_table_; NiceMock main_thread_dispatcher_; NiceMock tls_; AllocatorImpl alloc_; @@ -182,7 +183,7 @@ TEST_F(StatsThreadLocalStoreTest, NoTls) { Counter& c1 = store_->counter("c1"); EXPECT_EQ(&c1, &store_->counter("c1")); - StatNameManagedStorage c1_name("c1", symbol_table_); + StatNameManagedStorage c1_name("c1", *symbol_table_); c1.add(100); auto found_counter = store_->findCounter(c1_name.statName()); ASSERT_TRUE(found_counter.has_value()); @@ -193,7 +194,7 @@ TEST_F(StatsThreadLocalStoreTest, NoTls) { Gauge& g1 = store_->gauge("g1", Gauge::ImportMode::Accumulate); EXPECT_EQ(&g1, &store_->gauge("g1", Gauge::ImportMode::Accumulate)); - StatNameManagedStorage g1_name("g1", symbol_table_); + StatNameManagedStorage g1_name("g1", *symbol_table_); g1.set(100); auto found_gauge = store_->findGauge(g1_name.statName()); ASSERT_TRUE(found_gauge.has_value()); @@ -204,7 +205,7 @@ TEST_F(StatsThreadLocalStoreTest, NoTls) { Histogram& h1 = store_->histogram("h1"); EXPECT_EQ(&h1, &store_->histogram("h1")); - StatNameManagedStorage h1_name("h1", symbol_table_); + StatNameManagedStorage h1_name("h1", *symbol_table_); auto found_histogram = store_->findHistogram(h1_name.statName()); ASSERT_TRUE(found_histogram.has_value()); EXPECT_EQ(&h1, &found_histogram->get()); @@ -230,7 +231,7 @@ TEST_F(StatsThreadLocalStoreTest, Tls) { Counter& c1 = store_->counter("c1"); EXPECT_EQ(&c1, &store_->counter("c1")); - StatNameManagedStorage c1_name("c1", symbol_table_); + StatNameManagedStorage c1_name("c1", *symbol_table_); c1.add(100); auto found_counter = store_->findCounter(c1_name.statName()); ASSERT_TRUE(found_counter.has_value()); @@ -241,7 +242,7 @@ TEST_F(StatsThreadLocalStoreTest, Tls) { Gauge& g1 = store_->gauge("g1", Gauge::ImportMode::Accumulate); EXPECT_EQ(&g1, &store_->gauge("g1", Gauge::ImportMode::Accumulate)); - StatNameManagedStorage g1_name("g1", symbol_table_); + StatNameManagedStorage g1_name("g1", *symbol_table_); g1.set(100); auto found_gauge = store_->findGauge(g1_name.statName()); ASSERT_TRUE(found_gauge.has_value()); @@ -252,7 +253,7 @@ TEST_F(StatsThreadLocalStoreTest, Tls) { Histogram& h1 = store_->histogram("h1"); EXPECT_EQ(&h1, &store_->histogram("h1")); - StatNameManagedStorage h1_name("h1", symbol_table_); + StatNameManagedStorage h1_name("h1", *symbol_table_); auto found_histogram = store_->findHistogram(h1_name.statName()); ASSERT_TRUE(found_histogram.has_value()); EXPECT_EQ(&h1, &found_histogram->get()); @@ -284,11 +285,11 @@ TEST_F(StatsThreadLocalStoreTest, BasicScope) { Counter& c2 = scope1->counter("c2"); EXPECT_EQ("c1", c1.name()); EXPECT_EQ("scope1.c2", c2.name()); - StatNameManagedStorage c1_name("c1", symbol_table_); + StatNameManagedStorage c1_name("c1", *symbol_table_); auto found_counter = store_->findCounter(c1_name.statName()); ASSERT_TRUE(found_counter.has_value()); EXPECT_EQ(&c1, &found_counter->get()); - StatNameManagedStorage c2_name("scope1.c2", symbol_table_); + StatNameManagedStorage c2_name("scope1.c2", *symbol_table_); auto found_counter2 = store_->findCounter(c2_name.statName()); ASSERT_TRUE(found_counter2.has_value()); EXPECT_EQ(&c2, &found_counter2->get()); @@ -297,11 +298,11 @@ TEST_F(StatsThreadLocalStoreTest, BasicScope) { Gauge& g2 = scope1->gauge("g2", Gauge::ImportMode::Accumulate); EXPECT_EQ("g1", g1.name()); EXPECT_EQ("scope1.g2", g2.name()); - StatNameManagedStorage g1_name("g1", symbol_table_); + StatNameManagedStorage g1_name("g1", *symbol_table_); auto found_gauge = store_->findGauge(g1_name.statName()); ASSERT_TRUE(found_gauge.has_value()); EXPECT_EQ(&g1, &found_gauge->get()); - StatNameManagedStorage g2_name("scope1.g2", symbol_table_); + StatNameManagedStorage g2_name("scope1.g2", *symbol_table_); auto found_gauge2 = store_->findGauge(g2_name.statName()); ASSERT_TRUE(found_gauge2.has_value()); EXPECT_EQ(&g2, &found_gauge2->get()); @@ -314,11 +315,11 @@ TEST_F(StatsThreadLocalStoreTest, BasicScope) { h1.recordValue(100); EXPECT_CALL(sink_, onHistogramComplete(Ref(h2), 200)); h2.recordValue(200); - StatNameManagedStorage h1_name("h1", symbol_table_); + StatNameManagedStorage h1_name("h1", *symbol_table_); auto found_histogram = store_->findHistogram(h1_name.statName()); ASSERT_TRUE(found_histogram.has_value()); EXPECT_EQ(&h1, &found_histogram->get()); - StatNameManagedStorage h2_name("scope1.h2", symbol_table_); + StatNameManagedStorage h2_name("scope1.h2", *symbol_table_); auto found_histogram2 = store_->findHistogram(h2_name.statName()); ASSERT_TRUE(found_histogram2.has_value()); EXPECT_EQ(&h2, &found_histogram2->get()); @@ -380,7 +381,7 @@ TEST_F(StatsThreadLocalStoreTest, NestedScopes) { ScopePtr scope1 = store_->createScope("scope1."); Counter& c1 = scope1->counter("foo.bar"); EXPECT_EQ("scope1.foo.bar", c1.name()); - StatNameManagedStorage c1_name("scope1.foo.bar", symbol_table_); + StatNameManagedStorage c1_name("scope1.foo.bar", *symbol_table_); auto found_counter = store_->findCounter(c1_name.statName()); ASSERT_TRUE(found_counter.has_value()); EXPECT_EQ(&c1, &found_counter->get()); @@ -389,7 +390,7 @@ TEST_F(StatsThreadLocalStoreTest, NestedScopes) { Counter& c2 = scope2->counter("bar"); EXPECT_EQ(&c1, &c2); EXPECT_EQ("scope1.foo.bar", c2.name()); - StatNameManagedStorage c2_name("scope1.foo.bar", symbol_table_); + StatNameManagedStorage c2_name("scope1.foo.bar", *symbol_table_); auto found_counter2 = store_->findCounter(c2_name.statName()); ASSERT_TRUE(found_counter2.has_value()); @@ -455,12 +456,14 @@ TEST_F(StatsThreadLocalStoreTest, OverlappingScopes) { class LookupWithStatNameTest : public testing::Test { public: - LookupWithStatNameTest() : alloc_(symbol_table_), store_(alloc_), pool_(symbol_table_) {} + LookupWithStatNameTest() + : symbol_table_(SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_), + store_(alloc_), pool_(*symbol_table_) {} ~LookupWithStatNameTest() override { store_.shutdownThreading(); } StatName makeStatName(absl::string_view name) { return pool_.add(name); } - Stats::FakeSymbolTableImpl symbol_table_; + SymbolTablePtr symbol_table_; AllocatorImpl alloc_; ThreadLocalStoreImpl store_; StatNamePool pool_; @@ -664,7 +667,8 @@ TEST_F(StatsMatcherTLSTest, TestExclusionRegex) { class RememberStatsMatcherTest : public testing::TestWithParam { public: RememberStatsMatcherTest() - : heap_alloc_(symbol_table_), store_(heap_alloc_), scope_(store_.createScope("scope.")) { + : symbol_table_(SymbolTableCreator::makeSymbolTable()), heap_alloc_(*symbol_table_), + store_(heap_alloc_), scope_(store_.createScope("scope.")) { if (GetParam()) { store_.initializeThreading(main_thread_dispatcher_, tls_); } @@ -755,7 +759,7 @@ class RememberStatsMatcherTest : public testing::TestWithParam { }; } - Stats::FakeSymbolTableImpl symbol_table_; + Stats::SymbolTablePtr symbol_table_; NiceMock main_thread_dispatcher_; NiceMock tls_; AllocatorImpl heap_alloc_; @@ -968,11 +972,11 @@ TEST_F(StatsThreadLocalStoreTest, MergeDuringShutDown) { } TEST(ThreadLocalStoreThreadTest, ConstructDestruct) { - Stats::FakeSymbolTableImpl symbol_table; + SymbolTablePtr symbol_table(SymbolTableCreator::makeSymbolTable()); Api::ApiPtr api = Api::createApiForTest(); Event::DispatcherPtr dispatcher = api->allocateDispatcher(); NiceMock tls; - AllocatorImpl alloc(symbol_table); + AllocatorImpl alloc(*symbol_table); ThreadLocalStoreImpl store(alloc); store.initializeThreading(*dispatcher, tls); From 69f805c2fc582b647bc53d55f3f13ad00bfae793 Mon Sep 17 00:00:00 2001 From: Xin Date: Thu, 22 Aug 2019 19:19:05 -0400 Subject: [PATCH 436/542] [router] Add SRDS configUpdate impl (#7451) This PR contains changes on the xRDS side for SRDS impl, cribbed from http://go/gh/stevenzzzz/envoy/pull/8/files#diff-2071ab0887162eac1fd177e89d83175a * Add onConfigUpdate impl for SRDS subscription * Remove scoped_config_manager as it's not used now. * Move ScopedConfigInfo to scoped_config_impl.h/cc * Add a hash to scopeKey and scopeKeyFragment, so we can look up scopekey by hash value in constant time when SRDS has many scopes. * Add a initManager parameter to RDS createRdsRouteConfigProvider API interface, when creating RouteConfigProvider after listener/server warmed up, we need to specify a different initManager than the one from factoryContext to avoid an assertion failure. see related:#7617 This PR only latches a SRDS provider into the connection manager, the "conn manager using SRDS to make route decision" plus integration tests will be covered in a following PR. Risk Level: LOW [not fully implemented]. Testing: unit tests Signed-off-by: Xin Zhuang --- .../router/route_config_provider_manager.h | 5 +- source/common/config/config_provider_impl.h | 44 +- source/common/router/BUILD | 20 +- source/common/router/rds_impl.cc | 13 +- source/common/router/rds_impl.h | 15 +- source/common/router/scoped_config_impl.cc | 57 ++- source/common/router/scoped_config_impl.h | 65 ++- source/common/router/scoped_config_manager.cc | 22 - source/common/router/scoped_config_manager.h | 49 -- source/common/router/scoped_rds.cc | 280 ++++++++--- source/common/router/scoped_rds.h | 96 +++- .../network/http_connection_manager/config.cc | 21 +- .../config/config_provider_impl_test.cc | 3 + test/common/router/BUILD | 5 + test/common/router/rds_impl_test.cc | 8 +- test/common/router/scoped_config_impl_test.cc | 198 +++++++- test/common/router/scoped_rds_test.cc | 437 ++++++++++++++++-- .../scoped_rds_integration_test.cc | 50 +- test/mocks/router/mocks.cc | 16 +- test/mocks/router/mocks.h | 37 +- 20 files changed, 1137 insertions(+), 304 deletions(-) delete mode 100644 source/common/router/scoped_config_manager.cc delete mode 100644 source/common/router/scoped_config_manager.h diff --git a/include/envoy/router/route_config_provider_manager.h b/include/envoy/router/route_config_provider_manager.h index ffd9922bd1a7..9912aa460356 100644 --- a/include/envoy/router/route_config_provider_manager.h +++ b/include/envoy/router/route_config_provider_manager.h @@ -33,10 +33,13 @@ class RouteConfigProviderManager { * @param rds supplies the proto configuration of an RDS-configured RouteConfigProvider. * @param factory_context is the context to use for the route config provider. * @param stat_prefix supplies the stat_prefix to use for the provider stats. + * @param init_manager the Init::Manager used to coordinate initialization of a the underlying RDS + * subscription. */ virtual RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix) PURE; + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) PURE; /** * Get a RouteConfigSharedPtr for a statically defined route. Ownership is as described for diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index a1a7b02d71b7..4a7499c5abdb 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -1,7 +1,5 @@ #pragma once -#include - #include "envoy/config/config_provider.h" #include "envoy/config/config_provider_manager.h" #include "envoy/init/manager.h" @@ -146,9 +144,7 @@ class MutableConfigProviderCommonBase; * shared ownership of the underlying subscription. * */ -class ConfigSubscriptionCommonBase - : protected Logger::Loggable, - public std::enable_shared_from_this { +class ConfigSubscriptionCommonBase : protected Logger::Loggable { public: // Callback for updating a Config implementation held in each worker thread, the callback is // called in applyConfigUpdate() with the current version Config, and is expected to return the @@ -224,21 +220,14 @@ class ConfigSubscriptionCommonBase */ void applyConfigUpdate( const ConfigUpdateCb& update_fn, const Event::PostCb& complete_cb = []() {}) { - // It is safe to call shared_from_this here as this is in main thread, and destruction of a - // ConfigSubscriptionCommonBase owner (i.e., a provider) happens in main thread as well. - auto shared_this = shared_from_this(); tls_->runOnAllThreads( [this, update_fn]() { - tls_->getTyped().config_ = update_fn(this->getConfig()); + // NOTE: there is a known race condition between *this* subscription being teared down in + // main thread and the posted callback being executed before the destruction. See more + // details in https://github.com/envoyproxy/envoy/issues/7902 + tls_->getTyped().config_ = update_fn(getConfig()); }, - // During the update propagation, a subscription may get teared down in main thread due to - // all owners/providers destructed in a xDS update (e.g. LDS demolishes a - // RouteConfigProvider and its subscription). - // If such a race condition happens, holding a reference to the "*this" subscription - // instance in this cb will ensure the shared "*this" gets posted back to main thread, after - // all the workers finish calling the update_fn, at which point it's safe to destruct - // "*this" instance. - [shared_this, complete_cb]() { complete_cb(); }); + complete_cb); } void setLastUpdated() { last_updated_ = time_source_.systemTime(); } @@ -287,8 +276,8 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { /** * Must be called by the derived class' constructor. - * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with - * the underlying subscription, shared across all providers and workers. + * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated + * with the underlying subscription, shared across all providers and workers. */ void initialize(const ConfigProvider::ConfigConstSharedPtr& initial_config) { tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { @@ -302,7 +291,8 @@ class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { * @param config_proto supplies the newly received config proto. * @param config_name supplies the name associated with the config. * @param version_info supplies the version associated with the config. - * @return bool false when the config proto has no delta from the previous config, true otherwise. + * @return bool false when the config proto has no delta from the previous config, true + * otherwise. */ bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info); @@ -369,8 +359,8 @@ class MutableConfigProviderCommonBase : public ConfigProvider { }; /** - * Provides generic functionality required by all config provider managers, such as managing shared - * lifetime of subscriptions and dynamic config providers, along with determining which + * Provides generic functionality required by all config provider managers, such as managing + * shared lifetime of subscriptions and dynamic config providers, along with determining which * subscriptions should be associated with newly instantiated providers. * * The implementation of this class is not thread safe. Note that ImmutableConfigProviderBase @@ -379,9 +369,9 @@ class MutableConfigProviderCommonBase : public ConfigProvider { * * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care - * must be taken with regards to destruction of these objects, since it must also happen on the main - * thread _prior_ to destruction of the ConfigProviderManagerImplBase object from which they were - * created. + * must be taken with regards to destruction of these objects, since it must also happen on the + * main thread _prior_ to destruction of the ConfigProviderManagerImplBase object from which they + * were created. * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. @@ -415,8 +405,8 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl const ConfigProviderSet& immutableConfigProviders(ConfigProviderInstanceType type) const; /** - * Returns the subscription associated with the config_source_proto; if none exists, a new one is - * allocated according to the subscription_factory_fn. + * Returns the subscription associated with the config_source_proto; if none exists, a new one + * is allocated according to the subscription_factory_fn. * @param config_source_proto supplies the proto specifying the config subscription parameters. * @param init_manager supplies the init manager. * @param subscription_factory_fn supplies a function to be called when a new subscription needs diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 31d354477d5e..21970908db10 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -132,6 +132,7 @@ envoy_cc_library( "//include/envoy/singleton:instance_interface", "//include/envoy/thread_local:thread_local_interface", "//source/common/common:assert_lib", + "//source/common/common:callback_impl_lib", "//source/common/common:minimal_logger_lib", "//source/common/config:rds_json_lib", "//source/common/config:subscription_factory_lib", @@ -146,22 +147,16 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "scoped_config_manager_lib", - srcs = ["scoped_config_manager.cc"], - hdrs = ["scoped_config_manager.h"], - deps = [ - "@envoy_api//envoy/api/v2:srds_cc", - ], -) - envoy_cc_library( name = "scoped_config_lib", srcs = ["scoped_config_impl.cc"], hdrs = ["scoped_config_impl.h"], + external_deps = [ + "abseil_str_format", + ], deps = [ ":config_lib", - ":scoped_config_manager_lib", + "//include/envoy/router:rds_interface", "//include/envoy/router:scopes_interface", "//include/envoy/thread_local:thread_local_interface", "@envoy_api//envoy/api/v2:srds_cc", @@ -174,13 +169,18 @@ envoy_cc_library( srcs = ["scoped_rds.cc"], hdrs = ["scoped_rds.h"], deps = [ + ":rds_lib", ":scoped_config_lib", "//include/envoy/config:config_provider_interface", "//include/envoy/config:subscription_interface", + "//include/envoy/router:route_config_provider_manager_interface", "//include/envoy/stats:stats_interface", "//source/common/common:assert_lib", + "//source/common/common:cleanup_lib", "//source/common/common:minimal_logger_lib", "//source/common/config:config_provider_lib", + "//source/common/init:manager_lib", + "//source/common/init:watcher_lib", "@envoy_api//envoy/admin/v2alpha:config_dump_cc", "@envoy_api//envoy/api/v2:srds_cc", ], diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index b95554601aef..64a60062a30c 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -30,8 +30,8 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( return route_config_provider_manager.createStaticRouteConfigProvider(config.route_config(), factory_context); case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds: - return route_config_provider_manager.createRdsRouteConfigProvider(config.rds(), factory_context, - stat_prefix); + return route_config_provider_manager.createRdsRouteConfigProvider( + config.rds(), factory_context, stat_prefix, factory_context.initManager()); default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -124,6 +124,7 @@ void RdsRouteConfigSubscription::onConfigUpdate( } vhds_subscription_.release(); } + update_callback_manager_.runCallbacks(); } init_target_.ready(); @@ -218,8 +219,8 @@ RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& ad Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix) { - + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) { // RdsRouteConfigSubscriptions are unique based on their serialized RDS config. const uint64_t manager_identifier = MessageUtil::hash(rds); @@ -232,9 +233,7 @@ Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteCon // of simplicity. subscription.reset(new RdsRouteConfigSubscription(rds, manager_identifier, factory_context, stat_prefix, *this)); - - factory_context.initManager().add(subscription->init_target_); - + init_manager.add(subscription->init_target_); route_config_subscriptions_.insert({manager_identifier, subscription}); } else { // Because the RouteConfigProviderManager's weak_ptrs only get cleaned up diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 21acac47cfa7..418512bb6e50 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -22,6 +22,7 @@ #include "envoy/stats/scope.h" #include "envoy/thread_local/thread_local.h" +#include "common/common/callback_impl.h" #include "common/common/logger.h" #include "common/init/target_impl.h" #include "common/protobuf/utility.h" @@ -31,6 +32,9 @@ namespace Envoy { namespace Router { +// For friend class declaration in RdsRouteConfigSubscription. +class ScopedRdsConfigSubscription; + /** * Route configuration provider utilities. */ @@ -121,6 +125,10 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, return MessageUtil::anyConvert(resource).name(); } + Common::CallbackHandle* addUpdateCallback(std::function callback) { + return update_callback_manager_.add(callback); + } + RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, @@ -142,8 +150,11 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, VhdsSubscriptionPtr vhds_subscription_; RouteConfigUpdatePtr config_update_info_; ProtobufMessage::ValidationVisitor& validation_visitor_; + Common::CallbackManager<> update_callback_manager_; friend class RouteConfigProviderManagerImpl; + // Access to addUpdateCallback + friend class ScopedRdsConfigSubscription; }; using RdsRouteConfigSubscriptionSharedPtr = std::shared_ptr; @@ -195,8 +206,8 @@ class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, // RouteConfigProviderManager RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix) override; + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Init::Manager& init_manager) override; RouteConfigProviderPtr createStaticRouteConfigProvider(const envoy::api::v2::RouteConfiguration& route_config, diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index f5c8007f9e74..c6f00f58ea1e 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -10,13 +10,7 @@ bool ScopeKey::operator==(const ScopeKey& other) const { // An empty key equals to nothing, "NULL" != "NULL". return false; } - return std::equal(fragments_.begin(), fragments_.end(), other.fragments_.begin(), - other.fragments_.end(), - [](const std::unique_ptr& left, - const std::unique_ptr& right) -> bool { - // Both should be non-NULL now. - return *left == *right; - }); + return this->hash() == other.hash(); } HeaderValueExtractorImpl::HeaderValueExtractorImpl( @@ -76,6 +70,22 @@ HeaderValueExtractorImpl::computeFragment(const Http::HeaderMap& headers) const return nullptr; } +ScopedRouteInfo::ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, + ConfigConstSharedPtr&& route_config) + : config_proto_(std::move(config_proto)), route_config_(std::move(route_config)) { + // TODO(stevenzzzz): Maybe worth a KeyBuilder abstraction when there are more than one type of + // Fragment. + for (const auto& fragment : config_proto_.key().fragments()) { + switch (fragment.type_case()) { + case envoy::api::v2::ScopedRouteConfiguration::Key::Fragment::kStringKey: + scope_key_.addFragment(std::make_unique(fragment.string_key())); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + } +} + ScopeKeyBuilderImpl::ScopeKeyBuilderImpl(ScopedRoutes::ScopeKeyBuilder&& config) : ScopeKeyBuilderBase(std::move(config)) { for (const auto& fragment_builder : config_.fragments()) { @@ -104,12 +114,37 @@ ScopeKeyBuilderImpl::computeScopeKey(const Http::HeaderMap& headers) const { return std::make_unique(std::move(key)); } -void ScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} +void ScopedConfigImpl::addOrUpdateRoutingScope( + const ScopedRouteInfoConstSharedPtr& scoped_route_info) { + const auto iter = scoped_route_info_by_name_.find(scoped_route_info->scopeName()); + if (iter != scoped_route_info_by_name_.end()) { + ASSERT(scoped_route_info_by_key_.contains(iter->second->scopeKey().hash())); + scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); + } + scoped_route_info_by_name_[scoped_route_info->scopeName()] = scoped_route_info; + scoped_route_info_by_key_[scoped_route_info->scopeKey().hash()] = scoped_route_info; +} -void ScopedConfigImpl::removeRoutingScope(const std::string&) {} +void ScopedConfigImpl::removeRoutingScope(const std::string& scope_name) { + const auto iter = scoped_route_info_by_name_.find(scope_name); + if (iter != scoped_route_info_by_name_.end()) { + ASSERT(scoped_route_info_by_key_.contains(iter->second->scopeKey().hash())); + scoped_route_info_by_key_.erase(iter->second->scopeKey().hash()); + scoped_route_info_by_name_.erase(iter); + } +} -Router::ConfigConstSharedPtr ScopedConfigImpl::getRouteConfig(const Http::HeaderMap&) const { - return std::make_shared(); +Router::ConfigConstSharedPtr +ScopedConfigImpl::getRouteConfig(const Http::HeaderMap& headers) const { + std::unique_ptr scope_key = scope_key_builder_.computeScopeKey(headers); + if (scope_key == nullptr) { + return nullptr; + } + auto iter = scoped_route_info_by_key_.find(scope_key->hash()); + if (iter != scoped_route_info_by_key_.end()) { + return iter->second->routeConfig(); + } + return nullptr; } } // namespace Router diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 184929f94664..91e67e841b14 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -4,13 +4,17 @@ #include "envoy/api/v2/srds.pb.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/router/rds.h" #include "envoy/router/router.h" #include "envoy/router/scopes.h" #include "envoy/thread_local/thread_local.h" +#include "common/common/hash.h" #include "common/protobuf/utility.h" #include "common/router/config_impl.h" -#include "common/router/scoped_config_manager.h" + +#include "absl/numeric/int128.h" +#include "absl/strings/str_format.h" namespace Envoy { namespace Router { @@ -26,15 +30,14 @@ class ScopeKeyFragmentBase { bool operator==(const ScopeKeyFragmentBase& other) const { if (typeid(*this) == typeid(other)) { - return equals(other); + return hash() == other.hash(); } return false; } virtual ~ScopeKeyFragmentBase() = default; -private: - // Returns true if the two fragments equal else false. - virtual bool equals(const ScopeKeyFragmentBase&) const PURE; + // Hash of the fragment. + virtual uint64_t hash() const PURE; }; /** @@ -52,28 +55,39 @@ class ScopeKey { // Caller should guarantee the fragment is not nullptr. void addFragment(std::unique_ptr&& fragment) { ASSERT(fragment != nullptr, "null fragment not allowed in ScopeKey."); + updateHash(*fragment); fragments_.emplace_back(std::move(fragment)); } + uint64_t hash() const { return hash_; } bool operator!=(const ScopeKey& other) const; - bool operator==(const ScopeKey& other) const; private: + // Update the key's hash with the new fragment hash. + void updateHash(const ScopeKeyFragmentBase& fragment) { + std::stringbuf buffer; + buffer.sputn(reinterpret_cast(&hash_), sizeof(hash_)); + const auto& fragment_hash = fragment.hash(); + buffer.sputn(reinterpret_cast(&fragment_hash), sizeof(fragment_hash)); + hash_ = HashUtil::xxHash64(buffer.str()); + } + + uint64_t hash_{0}; std::vector> fragments_; }; // String fragment. class StringKeyFragment : public ScopeKeyFragmentBase { public: - explicit StringKeyFragment(absl::string_view value) : value_(value) {} + explicit StringKeyFragment(absl::string_view value) + : value_(value), hash_(HashUtil::xxHash64(value_)) {} -private: - bool equals(const ScopeKeyFragmentBase& other) const override { - return value_ == static_cast(other).value_; - } + uint64_t hash() const override { return hash_; } +private: const std::string value_; + const uint64_t hash_; }; /** @@ -132,9 +146,27 @@ class ScopeKeyBuilderImpl : public ScopeKeyBuilderBase { std::vector> fragment_builders_; }; +// ScopedRouteConfiguration and corresponding RouteConfigProvider. +class ScopedRouteInfo { +public: + ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration&& config_proto, + ConfigConstSharedPtr&& route_config); + + const ConfigConstSharedPtr& routeConfig() const { return route_config_; } + const ScopeKey& scopeKey() const { return scope_key_; } + const envoy::api::v2::ScopedRouteConfiguration& configProto() const { return config_proto_; } + const std::string& scopeName() const { return config_proto_.name(); } + +private: + envoy::api::v2::ScopedRouteConfiguration config_proto_; + ScopeKey scope_key_; + ConfigConstSharedPtr route_config_; +}; +using ScopedRouteInfoConstSharedPtr = std::shared_ptr; +// Ordered map for consistent config dumping. +using ScopedRouteMap = std::map; + /** - * TODO(AndresGuedez): implement scoped routing logic. - * * Each Envoy worker is assigned an instance of this type. When config updates are received, * addOrUpdateRoutingScope() and removeRoutingScope() are called to update the set of scoped routes. * @@ -153,8 +185,11 @@ class ScopedConfigImpl : public ScopedConfig { Router::ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap& headers) const override; private: - const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder - scope_key_builder_; + ScopeKeyBuilderImpl scope_key_builder_; + // From scope name to cached ScopedRouteInfo. + absl::flat_hash_map scoped_route_info_by_name_; + // Hash by ScopeKey hash to lookup in constant time. + absl::flat_hash_map scoped_route_info_by_key_; }; /** diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc deleted file mode 100644 index 2a5b75f3b29c..000000000000 --- a/source/common/router/scoped_config_manager.cc +++ /dev/null @@ -1,22 +0,0 @@ -#include "common/router/scoped_config_manager.h" - -#include "envoy/common/exception.h" - -#include "common/common/fmt.h" - -namespace Envoy { -namespace Router { - -ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( - const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { - auto scoped_route_info = std::make_shared(config_proto); - scoped_route_map_[config_proto.name()] = scoped_route_info; - return scoped_route_info; -} - -bool ScopedConfigManager::removeRoutingScope(const std::string& name) { - return scoped_route_map_.erase(name) == 0; -} - -} // namespace Router -} // namespace Envoy diff --git a/source/common/router/scoped_config_manager.h b/source/common/router/scoped_config_manager.h deleted file mode 100644 index 5f8dd6fda878..000000000000 --- a/source/common/router/scoped_config_manager.h +++ /dev/null @@ -1,49 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/api/v2/srds.pb.h" - -namespace Envoy { -namespace Router { - -// The internal representation of the configuration distributed via the ScopedRouteConfiguration -// proto. -class ScopedRouteInfo { -public: - ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration& config_proto) - : config_proto_(config_proto) {} - - // TODO(AndresGuedez): Add the necessary APIs required for the scoped routing logic. - - const envoy::api::v2::ScopedRouteConfiguration config_proto_; -}; -using ScopedRouteInfoConstSharedPtr = std::shared_ptr; - -// A manager for routing configuration scopes. -// An instance of the manager is owned by each ScopedRdsConfigSubscription. When config updates are -// received (on the main thread), the manager is called to track changes to the set of scoped route -// configurations and build s as needed. -class ScopedConfigManager { -public: - // Ordered map for consistent config dumping. - using ScopedRouteMap = std::map; - - // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the - // set of scopes matched against the scope keys built for each HTTP request. - ScopedRouteInfoConstSharedPtr - addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, - const std::string& version_info); - - // Removes a routing scope from the set of scopes matched against each HTTP request. - bool removeRoutingScope(const std::string& scope_name); - - const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } - -private: - ScopedRouteMap scoped_route_map_; -}; - -} // namespace Router -} // namespace Envoy diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 26ef93327284..989e284668c4 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -6,7 +6,12 @@ #include "envoy/api/v2/srds.pb.validate.h" #include "common/common/assert.h" +#include "common/common/cleanup.h" #include "common/common/logger.h" +#include "common/common/utility.h" +#include "common/config/resources.h" +#include "common/init/manager_impl.h" +#include "common/init/watcher_impl.h" // Types are deeply nested under Envoy::Config::ConfigProvider; use 'using-directives' across all // ConfigProvider related types for consistency. @@ -17,9 +22,7 @@ using Envoy::Config::ConfigProviderPtr; namespace Envoy { namespace Router { - namespace ScopedRoutesConfigProviderUtil { - ConfigProviderPtr create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, @@ -79,13 +82,17 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + envoy::api::v2::core::ConfigSource rds_config_source, + RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, factory_context), - name_(name), scope_key_builder_(scope_key_builder), + factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), - validation_visitor_(factory_context.messageValidationVisitor()) { + rds_config_source_(std::move(rds_config_source)), + validation_visitor_(factory_context.messageValidationVisitor()), stat_prefix_(stat_prefix), + route_config_provider_manager_(route_config_provider_manager) { subscription_ = factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( scoped_rds.scoped_rds_config_source(), @@ -100,72 +107,220 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( }); } +ScopedRdsConfigSubscription::RdsRouteConfigProviderHelper::RdsRouteConfigProviderHelper( + ScopedRdsConfigSubscription& parent, std::string scope_name, + envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + Init::Manager& init_manager) + : parent_(parent), scope_name_(scope_name), + route_provider_(static_cast( + parent_.route_config_provider_manager_ + .createRdsRouteConfigProvider(rds, parent_.factory_context_, parent_.stat_prefix_, + init_manager) + .release())), + rds_update_callback_handle_(route_provider_->subscription().addUpdateCallback([this]() { + // Subscribe to RDS update. + parent_.onRdsConfigUpdate(scope_name_, route_provider_->subscription()); + })) {} + +bool ScopedRdsConfigSubscription::addOrUpdateScopes( + const Protobuf::RepeatedPtrField& resources, + Init::Manager& init_manager, const std::string& version_info, + std::vector& exception_msgs) { + bool any_applied = false; + envoy::config::filter::network::http_connection_manager::v2::Rds rds; + rds.mutable_config_source()->MergeFrom(rds_config_source_); + absl::flat_hash_set unique_resource_names; + for (const auto& resource : resources) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_config; + try { + scoped_route_config = + MessageUtil::anyConvert(resource.resource()); + MessageUtil::validate(scoped_route_config, validation_visitor_); + const std::string scope_name = scoped_route_config.name(); + if (!unique_resource_names.insert(scope_name).second) { + throw EnvoyException( + fmt::format("duplicate scoped route configuration '{}' found", scope_name)); + } + // TODO(stevenzzz): Creating a new RdsRouteConfigProvider likely expensive, migrate RDS to + // config-provider-framework to make it light weight. + rds.set_route_config_name(scoped_route_config.route_configuration_name()); + auto rds_config_provider_helper = + std::make_unique(*this, scope_name, rds, init_manager); + auto scoped_route_info = std::make_shared( + std::move(scoped_route_config), rds_config_provider_helper->routeConfig()); + // Detect if there is key conflict between two scopes, in which case Envoy won't be able to + // tell which RouteConfiguration to use. Reject the second scope in the delta form API. + auto iter = scope_name_by_hash_.find(scoped_route_info->scopeKey().hash()); + if (iter != scope_name_by_hash_.end()) { + if (iter->second != scoped_route_info->scopeName()) { + throw EnvoyException( + fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", + iter->second, scoped_route_info->scopeName())); + } + } + // NOTE: delete previous route provider if any. + route_provider_by_scope_.insert({scope_name, std::move(rds_config_provider_helper)}); + scope_name_by_hash_[scoped_route_info->scopeKey().hash()] = scoped_route_info->scopeName(); + scoped_route_map_[scoped_route_info->scopeName()] = scoped_route_info; + applyConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + return config; + }); + any_applied = true; + ENVOY_LOG(debug, "srds: add/update scoped_route '{}', version: {}", + scoped_route_info->scopeName(), version_info); + } catch (const EnvoyException& e) { + exception_msgs.emplace_back(fmt::format("{}", e.what())); + } + } + return any_applied; +} + +bool ScopedRdsConfigSubscription::removeScopes( + const Protobuf::RepeatedPtrField& scope_names, const std::string& version_info) { + bool any_applied = false; + for (const auto& scope_name : scope_names) { + auto iter = scoped_route_map_.find(scope_name); + if (iter != scoped_route_map_.end()) { + route_provider_by_scope_.erase(scope_name); + scope_name_by_hash_.erase(iter->second->scopeKey().hash()); + scoped_route_map_.erase(iter); + applyConfigUpdate([scope_name](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->removeRoutingScope(scope_name); + return config; + }); + any_applied = true; + ENVOY_LOG(debug, "srds: remove scoped route '{}', version: {}", scope_name, version_info); + } + } + return any_applied; +} + void ScopedRdsConfigSubscription::onConfigUpdate( - const Protobuf::RepeatedPtrField& resources, + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, const std::string& version_info) { - std::vector scoped_routes; - for (const auto& resource_any : resources) { - scoped_routes.emplace_back( - MessageUtil::anyConvert(resource_any)); + // If new route config sources come after the factory_context_.initManager()'s initialize() been + // called, that initManager can't accept new targets. Instead we use a local override which will + // start new subscriptions but not wait on them to be ready. + // NOTE: For now we use a local init-manager, in the future when Envoy supports on-demand xDS, we + // will probably make this init-manager as a member of the subscription. + std::unique_ptr noop_init_manager; + // NOTE: This should be defined after noop_init_manager as it depends on the + // noop_init_manager. + std::unique_ptr resume_rds; + if (factory_context_.initManager().state() == Init::Manager::State::Initialized) { + noop_init_manager = + std::make_unique(fmt::format("SRDS {}:{}", name_, version_info)); + // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. + // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either + // by Server init or LDS init. + factory_context_.clusterManager().adsMux().pause( + Envoy::Config::TypeUrl::get().RouteConfiguration); + resume_rds = std::make_unique([this, &noop_init_manager, version_info] { + // For new RDS subscriptions created after listener warming up, we don't wait for them to warm + // up. + Init::WatcherImpl noop_watcher( + // Note: we just throw it away. + fmt::format("SRDS ConfigUpdate watcher {}:{}", name_, version_info), + []() { /*Do nothing.*/ }); + noop_init_manager->initialize(noop_watcher); + // New RDS subscriptions should have been created, now lift the floodgate. + // Note in the case of partial acceptance, accepted RDS subscriptions should be started + // despite of any error. + factory_context_.clusterManager().adsMux().resume( + Envoy::Config::TypeUrl::get().RouteConfiguration); + }); } + std::vector exception_msgs; + bool any_applied = addOrUpdateScopes( + added_resources, + (noop_init_manager == nullptr ? factory_context_.initManager() : *noop_init_manager), + version_info, exception_msgs); + any_applied = removeScopes(removed_resources, version_info) || any_applied; + ConfigSubscriptionCommonBase::onConfigUpdate(); + if (any_applied) { + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + } + stats_.config_reload_.inc(); + if (!exception_msgs.empty()) { + throw EnvoyException(fmt::format("Error adding/updating scoped route(s): {}", + StringUtil::join(exception_msgs, ", "))); + } +} - std::unordered_set resource_names; - for (const auto& scoped_route : scoped_routes) { - if (!resource_names.insert(scoped_route.name()).second) { +void ScopedRdsConfigSubscription::onRdsConfigUpdate(const std::string& scope_name, + RdsRouteConfigSubscription& rds_subscription) { + auto iter = scoped_route_map_.find(scope_name); + ASSERT(iter != scoped_route_map_.end(), + fmt::format("trying to update route config for non-existing scope {}", scope_name)); + auto new_scoped_route_info = std::make_shared( + envoy::api::v2::ScopedRouteConfiguration(iter->second->configProto()), + std::make_shared(rds_subscription.routeConfigUpdate()->routeConfiguration(), + factory_context_, false)); + applyConfigUpdate([new_scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) + -> ConfigProvider::ConfigConstSharedPtr { + auto* thread_local_scoped_config = + const_cast(static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(new_scoped_route_info); + return config; + }); +} + +// TODO(stevenzzzz): see issue #7508, consider generalizing this function as it overlaps with +// CdsApiImpl::onConfigUpdate. +void ScopedRdsConfigSubscription::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + absl::flat_hash_map scoped_routes; + absl::flat_hash_map scope_name_by_key_hash; + for (const auto& resource_any : resources) { + // Throws (thus rejects all) on any error. + auto scoped_route = + MessageUtil::anyConvert(resource_any); + MessageUtil::validate(scoped_route, validation_visitor_); + const std::string scope_name = scoped_route.name(); + auto scope_config_inserted = scoped_routes.try_emplace(scope_name, std::move(scoped_route)); + if (!scope_config_inserted.second) { throw EnvoyException( - fmt::format("duplicate scoped route configuration {} found", scoped_route.name())); + fmt::format("duplicate scoped route configuration '{}' found", scope_name)); + } + const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config = + scope_config_inserted.first->second; + const uint64_t key_fingerprint = MessageUtil::hash(scoped_route_config.key()); + if (!scope_name_by_key_hash.try_emplace(key_fingerprint, scope_name).second) { + throw EnvoyException( + fmt::format("scope key conflict found, first scope is '{}', second scope is '{}'", + scope_name_by_key_hash[key_fingerprint], scope_name)); } } - for (const auto& scoped_route : scoped_routes) { - MessageUtil::validate(scoped_route, validation_visitor_); - } - - // TODO(AndresGuedez): refactor such that it can be shared with other delta APIs (e.g., CDS). - std::vector exception_msgs; - // We need to keep track of which scoped routes we might need to remove. - ScopedConfigManager::ScopedRouteMap scoped_routes_to_remove = - scoped_config_manager_.scopedRouteMap(); - for (auto& scoped_route : scoped_routes) { - const std::string& scoped_route_name = scoped_route.name(); - scoped_routes_to_remove.erase(scoped_route_name); - ScopedRouteInfoConstSharedPtr scoped_route_info = - scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); - ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); - applyConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) - -> ConfigProvider::ConfigConstSharedPtr { - auto* thread_local_scoped_config = - const_cast(static_cast(config.get())); - thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); - return config; - }); + ScopedRouteMap scoped_routes_to_remove = scoped_route_map_; + Protobuf::RepeatedPtrField to_add_repeated; + Protobuf::RepeatedPtrField to_remove_repeated; + for (auto& iter : scoped_routes) { + const std::string& scope_name = iter.first; + scoped_routes_to_remove.erase(scope_name); + auto* to_add = to_add_repeated.Add(); + to_add->set_name(scope_name); + to_add->set_version(version_info); + to_add->mutable_resource()->PackFrom(iter.second); } for (const auto& scoped_route : scoped_routes_to_remove) { - const std::string scoped_route_name = scoped_route.first; - ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); - scoped_config_manager_.removeRoutingScope(scoped_route_name); - applyConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) - -> ConfigProvider::ConfigConstSharedPtr { - // In place update. - auto* thread_local_scoped_config = - const_cast(static_cast(config.get())); - thread_local_scoped_config->removeRoutingScope(scoped_route_name); - return config; - }); + *to_remove_repeated.Add() = scoped_route.first; } - - DeltaConfigSubscriptionInstance::onConfigUpdate(); - setLastConfigInfo(absl::optional({absl::nullopt, version_info})); - stats_.config_reload_.inc(); -} + onConfigUpdate(to_add_repeated, to_remove_repeated, version_info); +} // namespace Router ScopedRdsConfigProvider::ScopedRdsConfigProvider( - ScopedRdsConfigSubscriptionSharedPtr&& subscription, - envoy::api::v2::core::ConfigSource rds_config_source) - : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta), - subscription_(static_cast( - MutableConfigProviderCommonBase::subscription_.get())), - rds_config_source_(std::move(rds_config_source)) {} + ScopedRdsConfigSubscriptionSharedPtr&& subscription) + : MutableConfigProviderCommonBase(std::move(subscription), ConfigProvider::ApiType::Delta) {} ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { auto config_dump = std::make_unique(); @@ -179,10 +334,9 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const const ScopedRdsConfigSubscription* typed_subscription = static_cast(subscription.get()); dynamic_config->set_name(typed_subscription->name()); - const ScopedConfigManager::ScopedRouteMap& scoped_route_map = - typed_subscription->scopedRouteMap(); + const ScopedRouteMap& scoped_route_map = typed_subscription->scopedRouteMap(); for (const auto& it : scoped_route_map) { - dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it.second->config_proto_); + dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it.second->configProto()); } TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), *dynamic_config->mutable_last_updated()); @@ -223,11 +377,13 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( return std::make_shared( scoped_rds_config_source, manager_identifier, typed_optarg.scoped_routes_name_, typed_optarg.scope_key_builder_, factory_context, stat_prefix, + typed_optarg.rds_config_source_, + static_cast(config_provider_manager) + .route_config_provider_manager(), static_cast(config_provider_manager)); }); - return std::make_unique(std::move(subscription), - typed_optarg.rds_config_source_); + return std::make_unique(std::move(subscription)); } ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 3d006b6a7039..878cca0a3280 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -3,10 +3,15 @@ #include #include "envoy/api/v2/srds.pb.h" +#include "envoy/common/callback.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/config/subscription.h" +#include "envoy/router/route_config_provider_manager.h" #include "envoy/stats/scope.h" #include "common/config/config_provider_impl.h" +#include "common/init/manager_impl.h" +#include "common/router/rds_impl.h" #include "common/router/scoped_config_impl.h" namespace Envoy { @@ -89,43 +94,90 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + envoy::api::v2::core::ConfigSource rds_config_source, + RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); ~ScopedRdsConfigSubscription() override = default; const std::string& name() const { return name_; } - const ScopedConfigManager::ScopedRouteMap& scopedRouteMap() const { - return scoped_config_manager_.scopedRouteMap(); - } + const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } private: + // A helper class that takes care the life cycle management of a RDS route provider and the + // update callback handle. + struct RdsRouteConfigProviderHelper { + RdsRouteConfigProviderHelper( + ScopedRdsConfigSubscription& parent, std::string scope_name, + envoy::config::filter::network::http_connection_manager::v2::Rds& rds, + Init::Manager& init_manager); + ~RdsRouteConfigProviderHelper() { rds_update_callback_handle_->remove(); } + ConfigConstSharedPtr routeConfig() { return route_provider_->config(); } + + ScopedRdsConfigSubscription& parent_; + std::string scope_name_; + std::unique_ptr route_provider_; + // This handle_ is owned by the route config provider's RDS subscription, when the helper + // destructs, the handle is deleted as well. + Common::CallbackHandle* rds_update_callback_handle_; + }; + + // Adds or updates scopes, create a new RDS provider for each resource, if an exception is thrown + // during updating, the exception message is collected via the exception messages vector. + // Returns true if any scope updated, false otherwise. + bool addOrUpdateScopes(const Protobuf::RepeatedPtrField& resources, + Init::Manager& init_manager, const std::string& version_info, + std::vector& exception_msgs); + // Removes given scopes from the managed set of scopes. + // Returns true if any scope updated, false otherwise. + bool removeScopes(const Protobuf::RepeatedPtrField& scope_names, + const std::string& version_info); + // Envoy::Config::DeltaConfigSubscriptionInstance void start() override { subscription_->start({}); } // Envoy::Config::SubscriptionCallbacks + + // NOTE: state-of-the-world form onConfigUpdate(resources, version_info) will throw an + // EnvoyException on any error and essentially reject an update. While the Delta form + // onConfigUpdate(added_resources, removed_resources, version_info) by design will partially + // accept correct RouteConfiguration from management server. void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override; - void onConfigUpdate(const Protobuf::RepeatedPtrField&, - const Protobuf::RepeatedPtrField&, const std::string&) override { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& version_info) override; void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, const EnvoyException*) override { - ConfigSubscriptionCommonBase::onConfigUpdateFailed(); + DeltaConfigSubscriptionInstance::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource).name(); } - + // Propagate RDS updates to ScopeConfigImpl in workers. + void onRdsConfigUpdate(const std::string& scope_name, + RdsRouteConfigSubscription& rds_subscription); + + // ScopedRouteInfo by scope name. + ScopedRouteMap scoped_route_map_; + // RdsRouteConfigProvider by scope name. + absl::flat_hash_map> + route_provider_by_scope_; + // A map of (hash, scope-name), used to detect the key conflict between scopes. + absl::flat_hash_map scope_name_by_hash_; + // For creating RDS subscriptions. + Server::Configuration::FactoryContext& factory_context_; const std::string name_; std::unique_ptr subscription_; const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder scope_key_builder_; Stats::ScopePtr scope_; ScopedRdsStats stats_; - ScopedConfigManager scoped_config_manager_; + const envoy::api::v2::core::ConfigSource rds_config_source_; ProtobufMessage::ValidationVisitor& validation_visitor_; + const std::string stat_prefix_; + RouteConfigProviderManager& route_config_provider_manager_; }; using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; @@ -134,22 +186,21 @@ using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr(subscription_.get()); + } }; // A ConfigProviderManager for scoped routing configuration that creates static/inline and dynamic // (xds) config providers. class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderManagerImplBase { public: - ScopedRoutesConfigProviderManager(Server::Admin& admin) - : Envoy::Config::ConfigProviderManagerImplBase(admin, "route_scopes") {} + ScopedRoutesConfigProviderManager( + Server::Admin& admin, Router::RouteConfigProviderManager& route_config_provider_manager) + : Envoy::Config::ConfigProviderManagerImplBase(admin, "route_scopes"), + route_config_provider_manager_(route_config_provider_manager) {} ~ScopedRoutesConfigProviderManager() override = default; @@ -174,6 +225,13 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa std::vector>&& config_protos, Server::Configuration::FactoryContext& factory_context, const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; + + RouteConfigProviderManager& route_config_provider_manager() { + return route_config_provider_manager_; + } + +private: + RouteConfigProviderManager& route_config_provider_manager_; }; // The optional argument passed to the ConfigProviderManager::create*() functions. diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index cc03d2392364..82fb5274bb96 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -89,10 +89,12 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( return std::make_shared(context.admin()); }); - std::shared_ptr scoped_routes_config_provider_manager = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), [&context] { - return std::make_shared(context.admin()); + std::shared_ptr scoped_routes_config_provider_manager = + context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), + [&context, route_config_provider_manager] { + return std::make_shared( + context.admin(), *route_config_provider_manager); }); std::shared_ptr filter_config(new HttpConnectionManagerConfig( @@ -100,10 +102,11 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( *scoped_routes_config_provider_manager)); // This lambda captures the shared_ptrs created above, thus preserving the - // reference count. Moreover, keep in mind the capture list determines - // destruction order. - return [route_config_provider_manager, scoped_routes_config_provider_manager, filter_config, - &context, date_provider](Network::FilterManager& filter_manager) -> void { + // reference count. + // Keep in mind the lambda capture list **doesn't** determine the destruction order, but it's fine + // as these captured objects are also global singletons. + return [scoped_routes_config_provider_manager, route_config_provider_manager, date_provider, + filter_config, &context](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl( *filter_config, context.drainDecision(), context.random(), context.httpContext(), context.runtime(), context.localInfo(), context.clusterManager(), @@ -183,8 +186,6 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( break; case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kScopedRoutes: - ENVOY_LOG(warn, "Scoped routing has been enabled but it is not yet fully implemented! HTTP " - "request routing DOES NOT work (yet) with this configuration."); scoped_routes_config_provider_ = Router::ScopedRoutesConfigProviderUtil::create( config, context_, stats_prefix_, scoped_routes_config_provider_manager_); break; diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 5e7e65eeec70..6bf5e0f9ce04 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -629,6 +629,7 @@ class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { const std::string&, const Envoy::Config::ConfigProviderManager::OptionalArg&) override { DeltaDummyConfigSubscriptionSharedPtr subscription = + getSubscription( config_source_proto, factory_context.initManager(), [&factory_context](const uint64_t manager_identifier, @@ -709,6 +710,8 @@ TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { subscription.onConfigUpdate(untyped_dummy_configs, "2"); // NOTE: the config implementation is append only and _does not_ track updates/removals to the // config proto set, so the expectation is to double the size of the set. + EXPECT_EQ(provider1->config().get(), + provider2->config().get()); EXPECT_EQ(provider1->config()->numProtos(), 4); EXPECT_EQ(provider1->configProtoInfoVector().value().version_, "2"); diff --git a/test/common/router/BUILD b/test/common/router/BUILD index eb07c6558972..2ac71059b8bc 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -85,6 +85,7 @@ envoy_cc_test( ], deps = [ "//source/common/router:scoped_config_lib", + "//test/mocks/router:router_mocks", "//test/test_common:utility_lib", ], ) @@ -96,12 +97,16 @@ envoy_cc_test( "abseil_strings", ], deps = [ + "//include/envoy/config:subscription_interface", + "//include/envoy/init:manager_interface", "//source/common/config:utility_lib", "//source/common/http:message_lib", "//source/common/json:json_loader_lib", "//source/common/router:scoped_rds_lib", "//source/server/http:admin_lib", + "//test/mocks/config:config_mocks", "//test/mocks/init:init_mocks", + "//test/mocks/router:router_mocks", "//test/mocks/server:server_mocks", "//test/test_common:simulated_time_system_lib", "//test/test_common:utility_lib", diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index fb74aa9694f7..8bd32f455302 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -266,8 +266,8 @@ class RouteConfigProviderManagerImplTest : public RdsTestBase { // Get a RouteConfigProvider. This one should create an entry in the RouteConfigProviderManager. rds_.set_route_config_name("foo_route_config"); rds_.mutable_config_source()->set_path("foo_path"); - provider_ = route_config_provider_manager_->createRdsRouteConfigProvider(rds_, factory_context_, - "foo_prefix."); + provider_ = route_config_provider_manager_->createRdsRouteConfigProvider( + rds_, factory_context_, "foo_prefix.", factory_context_.initManager()); rds_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; } @@ -419,7 +419,7 @@ name: foo_route_config "1"); RouteConfigProviderPtr provider2 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, factory_context_, "foo_prefix"); + rds_, factory_context_, "foo_prefix", factory_context_.initManager()); // provider2 should have route config immediately after create EXPECT_TRUE(provider2->configInfo().has_value()); @@ -433,7 +433,7 @@ name: foo_route_config rds2.set_route_config_name("foo_route_config"); rds2.mutable_config_source()->set_path("bar_path"); RouteConfigProviderPtr provider3 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds2, factory_context_, "foo_prefix"); + rds2, factory_context_, "foo_prefix", factory_context_.initManager()); EXPECT_NE(provider3, provider_); factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(route_configs, "provider3"); diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index 9c9a06a25934..cf9bfd83055c 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -2,6 +2,7 @@ #include "common/router/scoped_config_impl.h" +#include "test/mocks/router/mocks.h" #include "test/test_common/utility.h" #include "gtest/gtest.h" @@ -11,10 +12,12 @@ namespace Router { namespace { using ::Envoy::Http::TestHeaderMapImpl; +using ::testing::NiceMock; +using ::testing::Return; class FooFragment : public ScopeKeyFragmentBase { -private: - bool equals(const ScopeKeyFragmentBase&) const override { return true; }; +public: + uint64_t hash() const override { return 1; } }; TEST(ScopeKeyFragmentBaseTest, EqualSign) { @@ -24,14 +27,33 @@ TEST(ScopeKeyFragmentBaseTest, EqualSign) { EXPECT_NE(foo, bar); } +TEST(ScopeKeyFragmentBaseTest, HashStable) { + FooFragment foo1; + FooFragment foo2; + + // Two FooFragments equal because their hash equals. + EXPECT_EQ(foo1, foo2); + EXPECT_EQ(foo1.hash(), foo2.hash()); + + // Hash value doesn't change. + StringKeyFragment a("abcdefg"); + auto hash_value = a.hash(); + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(hash_value, a.hash()); + EXPECT_EQ(StringKeyFragment("abcdefg").hash(), hash_value); + } +} + TEST(StringKeyFragmentTest, Empty) { StringKeyFragment a(""); StringKeyFragment b(""); EXPECT_EQ(a, b); + EXPECT_EQ(a.hash(), b.hash()); StringKeyFragment non_empty("ABC"); EXPECT_NE(a, non_empty); + EXPECT_NE(a.hash(), non_empty.hash()); } TEST(StringKeyFragmentTest, Normal) { @@ -217,7 +239,9 @@ ScopeKey makeKey(const std::vector& parts) { TEST(ScopeKeyDeathTest, AddNullFragment) { ScopeKey key; +#if !defined(NDEBUG) EXPECT_DEBUG_DEATH(key.addFragment(nullptr), "null fragment not allowed in ScopeKey."); +#endif } TEST(ScopeKeyTest, Unmatches) { @@ -231,6 +255,10 @@ TEST(ScopeKeyTest, Unmatches) { EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); + // Order matters. + EXPECT_EQ(makeKey({"a", "b", "c"}), makeKey({"a", "b", "c"})); + EXPECT_NE(makeKey({"a", "c", "b"}), makeKey({"a", "b", "c"})); + // Two keys of different length won't match. EXPECT_NE(makeKey({"a", "b"}), makeKey({"a", "b", "c"})); @@ -243,7 +271,7 @@ TEST(ScopeKeyTest, Matches) { EXPECT_EQ(makeKey({"", ""}), makeKey({"", ""})); EXPECT_EQ(makeKey({"a", "", ""}), makeKey({"a", "", ""})); - // Non empty fragments comparison. + // Non empty fragments comparison. EXPECT_EQ(makeKey({"A", "b"}), makeKey({"A", "b"})); } @@ -317,6 +345,170 @@ TEST(ScopeKeyBuilderImplTest, Parse) { EXPECT_EQ(key, nullptr); } +class ScopedRouteInfoTest : public testing::Test { +public: + void SetUp() override { + std::string yaml_plain = R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: foo + - string_key: bar +)EOF"; + TestUtility::loadFromYaml(yaml_plain, scoped_route_config_); + + route_config_ = std::make_shared>(); + route_config_->name_ = "foo_route"; + } + + envoy::api::v2::RouteConfiguration route_configuration_; + envoy::api::v2::ScopedRouteConfiguration scoped_route_config_; + std::shared_ptr route_config_; + std::unique_ptr info_; +}; + +TEST_F(ScopedRouteInfoTest, Creation) { + envoy::api::v2::ScopedRouteConfiguration config_copy = scoped_route_config_; + info_ = std::make_unique(std::move(scoped_route_config_), route_config_); + EXPECT_EQ(info_->routeConfig().get(), route_config_.get()); + EXPECT_TRUE(TestUtility::protoEqual(info_->configProto(), config_copy)); + EXPECT_EQ(info_->scopeName(), "foo_scope"); + EXPECT_EQ(info_->scopeKey(), makeKey({"foo", "bar"})); +} + +class ScopedConfigImplTest : public testing::Test { +public: + void SetUp() override { + std::string yaml_plain = R"EOF( + fragments: + - header_value_extractor: + name: 'foo_header' + element_separator: ',' + element: + key: 'bar' + separator: '=' + - header_value_extractor: + name: 'bar_header' + element_separator: ';' + index: 2 +)EOF"; + TestUtility::loadFromYaml(yaml_plain, key_builder_config_); + + scope_info_a_ = makeScopedRouteInfo(R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: foo + - string_key: bar +)EOF"); + scope_info_a_v2_ = makeScopedRouteInfo(R"EOF( + name: foo_scope + route_configuration_name: foo_route + key: + fragments: + - string_key: xyz + - string_key: xyz +)EOF"); + scope_info_b_ = makeScopedRouteInfo(R"EOF( + name: bar_scope + route_configuration_name: bar_route + key: + fragments: + - string_key: bar + - string_key: baz +)EOF"); + } + std::shared_ptr makeScopedRouteInfo(const std::string& route_config_yaml) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_config; + TestUtility::loadFromYaml(route_config_yaml, scoped_route_config); + + std::shared_ptr route_config = std::make_shared>(); + route_config->name_ = scoped_route_config.route_configuration_name(); + return std::make_shared(std::move(scoped_route_config), + std::move(route_config)); + } + + std::shared_ptr scope_info_a_; + std::shared_ptr scope_info_a_v2_; + std::shared_ptr scope_info_b_; + ScopedRoutes::ScopeKeyBuilder key_builder_config_; + std::unique_ptr scoped_config_impl_; +}; + +// Test a ScopedConfigImpl returns the correct route Config. +TEST_F(ScopedConfigImplTest, PickRoute) { + scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_b_); + + // Key (foo, bar) maps to scope_info_a_. + ConfigConstSharedPtr route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",,key=value,bar=foo,"}, + {"bar_header", ";val1;bar;val3"}, + }); + EXPECT_EQ(route_config, scope_info_a_->routeConfig()); + + // Key (bar, baz) maps to scope_info_b_. + route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",,key=value,bar=bar,"}, + {"bar_header", ";val1;baz;val3"}, + }); + EXPECT_EQ(route_config, scope_info_b_->routeConfig()); + + // No such key (bar, NOT_BAZ). + route_config = scoped_config_impl_->getRouteConfig(TestHeaderMapImpl{ + {"foo_header", ",key=value,bar=bar,"}, + {"bar_header", ";val1;NOT_BAZ;val3"}, + }); + EXPECT_EQ(route_config, nullptr); +} + +// Test a ScopedConfigImpl returns the correct route Config before and after scope config update. +TEST_F(ScopedConfigImplTest, Update) { + scoped_config_impl_ = std::make_unique(std::move(key_builder_config_)); + + TestHeaderMapImpl headers{ + {"foo_header", ",,key=value,bar=foo,"}, + {"bar_header", ";val1;bar;val3"}, + }; + // Empty ScopeConfig. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // Add scope_key (bar, baz). + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_b_); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + EXPECT_EQ(scoped_config_impl_->getRouteConfig( + TestHeaderMapImpl{{"foo_header", ",,key=v,bar=bar,"}, {"bar_header", ";val1;baz"}}), + scope_info_b_->routeConfig()); + + // Add scope_key (foo, bar). + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_); + // Found scope_info_a_. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), scope_info_a_->routeConfig()); + + // Update scope foo_scope. + scoped_config_impl_->addOrUpdateRoutingScope(scope_info_a_v2_); + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // foo_scope now is keyed by (xyz, xyz). + EXPECT_EQ(scoped_config_impl_->getRouteConfig( + TestHeaderMapImpl{{"foo_header", ",bar=xyz,foo=bar"}, {"bar_header", ";;xyz"}}), + scope_info_a_v2_->routeConfig()); + + // Remove scope "foo_scope". + scoped_config_impl_->removeRoutingScope("foo_scope"); + // scope_info_a_ is gone. + EXPECT_EQ(scoped_config_impl_->getRouteConfig(headers), nullptr); + + // Now delete some non-existent scopes. + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("foo_scope1")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("base_scope")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("bluh_scope")); + EXPECT_NO_THROW(scoped_config_impl_->removeRoutingScope("xyz_scope")); +} + } // namespace } // namespace Router } // namespace Envoy diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 0bc871f232b8..00936537bc7f 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -2,25 +2,40 @@ #include "envoy/admin/v2alpha/config_dump.pb.h" #include "envoy/admin/v2alpha/config_dump.pb.validate.h" +#include "envoy/config/subscription.h" +#include "envoy/init/manager.h" #include "envoy/stats/scope.h" #include "common/router/scoped_rds.h" +#include "test/mocks/config/mocks.h" +#include "test/mocks/router/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::AnyNumber; +using testing::ByMove; +using testing::DoAll; +using testing::Eq; using testing::InSequence; +using testing::Invoke; +using testing::IsNull; +using testing::NiceMock; using testing::Return; +using testing::ReturnRefOfCopy; namespace Envoy { namespace Router { namespace { +using ::Envoy::Http::TestHeaderMapImpl; + envoy::api::v2::ScopedRouteConfiguration parseScopedRouteConfigurationFromYaml(const std::string& yaml) { envoy::api::v2::ScopedRouteConfiguration scoped_route_config; @@ -44,21 +59,39 @@ parseHttpConnectionManagerFromYaml(const std::string& config_yaml) { class ScopedRoutesTestBase : public testing::Test { protected: ScopedRoutesTestBase() { + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("routes", _)); + route_config_provider_manager_ = + std::make_unique(factory_context_.admin_); + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); - config_provider_manager_ = - std::make_unique(factory_context_.admin_); + config_provider_manager_ = std::make_unique( + factory_context_.admin_, *route_config_provider_manager_); } ~ScopedRoutesTestBase() override { factory_context_.thread_local_.shutdownThread(); } + // The delta style API helper. + Protobuf::RepeatedPtrField + anyToResource(Protobuf::RepeatedPtrField& resources, + const std::string& version) { + Protobuf::RepeatedPtrField added_resources; + for (const auto& resource_any : resources) { + auto config = TestUtility::anyConvert(resource_any); + auto* to_add = added_resources.Add(); + to_add->set_name(config.name()); + to_add->set_version(version); + to_add->mutable_resource()->PackFrom(config); + } + return added_resources; + } + Event::SimulatedTimeSystem& timeSystem() { return time_system_; } NiceMock factory_context_; - Upstream::ClusterManager::ClusterInfoMap cluster_map_; - Upstream::MockClusterMockPrioritySet cluster_; + std::unique_ptr route_config_provider_manager_; std::unique_ptr config_provider_manager_; + Event::SimulatedTimeSystem time_system_; - envoy::api::v2::core::ConfigSource rds_config_source_; }; class ScopedRdsTest : public ScopedRoutesTestBase { @@ -66,11 +99,56 @@ class ScopedRdsTest : public ScopedRoutesTestBase { void setup() { InSequence s; + // Since factory_context_.cluster_manager_.subscription_factory_.callbacks_ is taken by the SRDS + // subscription. We need to return a different MockSubscription here for each RDS subscription. + // To build the map from RDS route_config_name to the RDS subscription, we need to get the + // route_config_name by mocking start() on the Config::Subscription. + EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, + subscriptionFromConfigSource(_, _, _, _)) + .Times(AnyNumber()); + EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, + subscriptionFromConfigSource( + _, + Eq(Grpc::Common::typeUrl( + envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name())), + _, _)) + .Times(AnyNumber()) + .WillRepeatedly(Invoke([this](const envoy::api::v2::core::ConfigSource&, absl::string_view, + Stats::Scope&, + Envoy::Config::SubscriptionCallbacks& callbacks) { + auto ret = std::make_unique>(); + rds_subscription_by_config_subscription_[ret.get()] = &callbacks; + EXPECT_CALL(*ret, start(_)) + .WillOnce(Invoke( + [this, config_sub_addr = ret.get()](const std::set& resource_names) { + EXPECT_EQ(resource_names.size(), 1); + auto iter = rds_subscription_by_config_subscription_.find(config_sub_addr); + EXPECT_NE(iter, rds_subscription_by_config_subscription_.end()); + rds_subscription_by_name_[*resource_names.begin()] = iter->second; + })); + return ret; + })); + + ON_CALL(factory_context_.init_manager_, add(_)) + .WillByDefault(Invoke([this](const Init::Target& target) { + target_handles_.push_back(target.createHandle("test")); + })); + ON_CALL(factory_context_.init_manager_, initialize(_)) + .WillByDefault(Invoke([this](const Init::Watcher& watcher) { + for (auto& handle_ : target_handles_) { + handle_->initialize(watcher); + } + })); + const std::string config_yaml = R"EOF( name: foo_scoped_routes scope_key_builder: fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: Addr + element: + key: x-foo-key + separator: ; )EOF"; envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes scoped_routes_config; TestUtility::loadFromYaml(config_yaml, scoped_routes_config); @@ -79,11 +157,44 @@ name: foo_scoped_routes ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), scoped_routes_config.rds_config_source(), scoped_routes_config.scope_key_builder())); - subscription_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; + srds_subscription_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; + } + + // Helper function which pushes an update to given RDS subscription, the start(_) of the + // subscription must have been called. + void pushRdsConfig(const std::string& route_config_name, const std::string& version) { + const std::string route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: test + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: bluh }} +)EOF"; + Protobuf::RepeatedPtrField resources; + resources.Add()->PackFrom(TestUtility::parseYaml( + fmt::format(route_config_tmpl, route_config_name))); + rds_subscription_by_name_[route_config_name]->onConfigUpdate(resources, version); + } + + ScopedRdsConfigProvider* getScopedRdsProvider() const { + return dynamic_cast(provider_.get()); + } + // Helper function which returns the ScopedRouteMap of the subscription. + const ScopedRouteMap& getScopedRouteMap() const { + return getScopedRdsProvider()->subscription().scopedRouteMap(); } - Envoy::Config::SubscriptionCallbacks* subscription_callbacks_{}; + Envoy::Config::SubscriptionCallbacks* srds_subscription_{}; Envoy::Config::ConfigProviderPtr provider_; + std::list target_handles_; + Init::ExpectableWatcherImpl init_watcher_; + + // RDS mocks. + absl::flat_hash_map + rds_subscription_by_config_subscription_; + absl::flat_hash_map rds_subscription_by_name_; }; TEST_F(ScopedRdsTest, ValidateFail) { @@ -99,7 +210,11 @@ route_configuration_name: foo_routes )EOF"; Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources, "1"), ProtoValidationException); + + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed.*"); // 'route_configuration_name' validation: value must be > 1 byte. const std::string config_yaml2 = R"EOF( @@ -111,7 +226,10 @@ name: foo_scope )EOF"; Protobuf::RepeatedPtrField resources2; parseScopedRouteConfigurationFromYaml(*resources2.Add(), config_yaml2); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources2, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources2, "1"), ProtoValidationException); + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(anyToResource(resources2, "1"), {}, "1"), EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed.*"); // 'key' validation: must define at least 1 fragment. const std::string config_yaml3 = R"EOF( @@ -121,11 +239,15 @@ route_configuration_name: foo_routes )EOF"; Protobuf::RepeatedPtrField resources3; parseScopedRouteConfigurationFromYaml(*resources3.Add(), config_yaml3); - EXPECT_THROW(subscription_callbacks_->onConfigUpdate(resources3, "1"), ProtoValidationException); + EXPECT_THROW(srds_subscription_->onConfigUpdate(resources3, "1"), ProtoValidationException); + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(anyToResource(resources3, "1"), {}, "1"), EnvoyException, + "Error adding/updating scoped route\\(s\\): Proto constraint validation failed .*value is " + "required.*"); } -// Tests that multiple uniquely named resources are allowed in config updates. -TEST_F(ScopedRdsTest, MultipleResources) { +// Tests that multiple uniquely named non-conflict resources are allowed in config updates. +TEST_F(ScopedRdsTest, MultipleResourcesStow) { setup(); const std::string config_yaml = R"EOF( @@ -140,20 +262,210 @@ route_configuration_name: foo_routes const std::string config_yaml2 = R"EOF( name: foo_scope2 route_configuration_name: foo_routes +key: + fragments: + - string_key: x-bar-key +)EOF"; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(resources, "1")); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times(2); // SRDS and RDS "foo_routes" + EXPECT_EQ( + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + + // Verify the config is a ScopedConfigImpl instance, both scopes point to "" as RDS hasn't kicked + // in yet(NullConfigImpl returned). + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + ""); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + ""); + // RDS updates foo_routes. + pushRdsConfig("foo_routes", "111"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + "foo_routes"); + + // Delete foo_scope2. + resources.RemoveLast(); + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(resources, "3")); + EXPECT_EQ(getScopedRouteMap().size(), 1); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); + EXPECT_EQ( + 2UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // now scope key "x-bar-key" points to nowhere. + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + IsNull()); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); +} + +// Tests that multiple uniquely named non-conflict resources are allowed in config updates. +TEST_F(ScopedRdsTest, MultipleResourcesDelta) { + setup(); + init_watcher_.expectReady().Times(2); // SRDS and RDS "foo_routes" + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes key: fragments: - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + const std::string config_yaml2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-bar-key )EOF"; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); - EXPECT_NO_THROW(subscription_callbacks_->onConfigUpdate(resources, "1")); + + // Delta API. + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(anyToResource(resources, "2"), {}, "1")); + factory_context_.init_manager_.initialize(init_watcher_); EXPECT_EQ( 1UL, factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + EXPECT_EQ(getScopedRouteMap().size(), 2); + + // Verify the config is a ScopedConfigImpl instance, both scopes point to "" as RDS hasn't kicked + // in yet(NullConfigImpl returned). + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + ""); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + ""); + // RDS updates foo_routes. + pushRdsConfig("foo_routes", "111"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}) + ->name(), + "foo_routes"); + + // Delete foo_scope2. + resources.RemoveLast(); + Protobuf::RepeatedPtrField deletes; + *deletes.Add() = "foo_scope2"; + EXPECT_NO_THROW(srds_subscription_->onConfigUpdate(anyToResource(resources, "4"), deletes, "2")); + EXPECT_EQ(getScopedRouteMap().size(), 1); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); + EXPECT_EQ( + 2UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // now scope key "x-bar-key" points to nowhere. + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-bar-key"}}), + IsNull()); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); +} + +// Tests that conflict resources are detected. +TEST_F(ScopedRdsTest, MultipleResourcesWithKeyConflict) { + setup(); + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + const std::string config_yaml2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(resources, "1"), EnvoyException, + ".*scope key conflict found, first scope is 'foo_scope', second scope is 'foo_scope2'"); + EXPECT_EQ( + // Fully rejected. + 0UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // Scope key "x-foo-key" points to nowhere. + EXPECT_NE(getScopedRdsProvider(), nullptr); + EXPECT_NE(getScopedRdsProvider()->config(), nullptr); + EXPECT_THAT(getScopedRdsProvider()->config()->getRouteConfig( + TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}), + IsNull()); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times( + 1); // Just SRDS, RDS "foo_routes" will initialized by the noop init-manager. + EXPECT_EQ(factory_context_.scope_.counter("foo.rds.foo_routes.config_reload").value(), 0UL); + + // Delta API. + EXPECT_CALL(factory_context_.init_manager_, state()) + .WillOnce(Return(Init::Manager::State::Initialized)); + EXPECT_THROW_WITH_REGEX( + srds_subscription_->onConfigUpdate(anyToResource(resources, "2"), {}, "2"), EnvoyException, + ".*scope key conflict found, first scope is 'foo_scope', second scope is 'foo_scope2'"); + EXPECT_EQ( + // Partially reject. + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // foo_scope update is applied. + EXPECT_EQ(getScopedRouteMap().size(), 1UL); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); + // Scope key "x-foo-key" points to foo_routes due to partial rejection. + pushRdsConfig("foo_routes", "111"); // Push some real route configuration. + EXPECT_EQ(1UL, factory_context_.scope_.counter("foo.rds.foo_routes.config_reload").value()); + EXPECT_EQ(getScopedRdsProvider() + ->config() + ->getRouteConfig(TestHeaderMapImpl{{"Addr", "x-foo-key;x-foo-key"}}) + ->name(), + "foo_routes"); } // Tests that only one resource is provided during a config update. -TEST_F(ScopedRdsTest, InvalidDuplicateResource) { +TEST_F(ScopedRdsTest, InvalidDuplicateResourceSotw) { setup(); + factory_context_.init_manager_.initialize(init_watcher_); + init_watcher_.expectReady().Times(0); const std::string config_yaml = R"EOF( name: foo_scope @@ -165,8 +477,42 @@ route_configuration_name: foo_routes Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); - EXPECT_THROW_WITH_MESSAGE(subscription_callbacks_->onConfigUpdate(resources, "1"), EnvoyException, - "duplicate scoped route configuration foo_scope found"); + EXPECT_THROW_WITH_MESSAGE(srds_subscription_->onConfigUpdate(resources, "1"), EnvoyException, + "duplicate scoped route configuration 'foo_scope' found"); +} + +// Tests that only one resource is provided during a config update. +TEST_F(ScopedRdsTest, InvalidDuplicateResourceDelta) { + setup(); + factory_context_.init_manager_.initialize(init_watcher_); + // After the above initialize, the default init_manager should return "Initialized". + EXPECT_CALL(factory_context_.init_manager_, state()) + .WillOnce(Return(Init::Manager::State::Initialized)); + init_watcher_.expectReady().Times( + 1); // SRDS onConfigUpdate breaks, but first foo_routes will + // kick start if it's initialized post-Server/LDS initialization. + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + EXPECT_THROW_WITH_MESSAGE( + srds_subscription_->onConfigUpdate(anyToResource(resources, "1"), {}, "1"), EnvoyException, + "Error adding/updating scoped route(s): duplicate scoped route configuration 'foo_scope' " + "found"); + EXPECT_EQ( + // Partially reject. + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); + // foo_scope update is applied. + EXPECT_EQ(getScopedRouteMap().size(), 1UL); + EXPECT_EQ(getScopedRouteMap().count("foo_scope"), 1); } // Tests a config update failure. @@ -177,31 +523,37 @@ TEST_F(ScopedRdsTest, ConfigUpdateFailure) { timeSystem().setSystemTime(time); const EnvoyException ex(fmt::format("config failure")); // Verify the failure updates the lastUpdated() timestamp. - subscription_callbacks_->onConfigUpdateFailed( - Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &ex); + srds_subscription_->onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, + &ex); EXPECT_EQ(std::chrono::time_point_cast(provider_->lastUpdated()) .time_since_epoch(), time); } -using ScopedRoutesConfigProviderManagerTest = ScopedRoutesTestBase; +// Tests that the /config_dump handler returns the corresponding scoped routing +// config. +TEST_F(ScopedRdsTest, ConfigDump) { + setup(); + factory_context_.init_manager_.initialize(init_watcher_); + EXPECT_CALL(factory_context_.init_manager_, state()) + .Times(2) // There are two SRDS pushes. + .WillRepeatedly(Return(Init::Manager::State::Initialized)); + init_watcher_.expectReady().Times(1); // SRDS only, no RDS push. -// Tests that the /config_dump handler returns the corresponding scoped routing config. -TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump = TestUtility::downcastAndValidate( *message_ptr); - // No routes at all, no last_updated timestamp + // No routes at all(no SRDS push yet), no last_updated timestamp envoy::admin::v2alpha::ScopedRoutesConfigDump expected_config_dump; TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: dynamic_scoped_route_configs: )EOF", expected_config_dump); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump)); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); @@ -215,7 +567,9 @@ stat_prefix: foo name: $0 scope_key_builder: fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: Addr + index: 0 $1 )EOF"; const std::string inline_scoped_route_configs_yaml = R"EOF( @@ -257,17 +611,9 @@ stat_prefix: foo dynamic_scoped_route_configs: )EOF", expected_config_dump); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump2.DebugString()); - - const std::string scoped_rds_config_yaml = R"EOF( - scoped_rds: - scoped_rds_config_source: -)EOF"; - Envoy::Config::ConfigProviderPtr dynamic_provider = ScopedRoutesConfigProviderUtil::create( - parseHttpConnectionManagerFromYaml(absl::Substitute( - hcm_base_config_yaml, "foo-dynamic-scoped-routes", scoped_rds_config_yaml)), - factory_context_, "foo.", *config_provider_manager_); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump2)); + // Now SRDS kicks off. Protobuf::RepeatedPtrField resources; resources.Add()->PackFrom(parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo @@ -277,8 +623,7 @@ route_configuration_name: dynamic-foo-route-config )EOF")); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891567)); - factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(resources, - "1"); + srds_subscription_->onConfigUpdate(resources, "1"); TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: @@ -296,7 +641,7 @@ route_configuration_name: dynamic-foo-route-config seconds: 1234567891 nanos: 234000000 dynamic_scoped_route_configs: - - name: foo-dynamic-scoped-routes + - name: foo_scoped_routes scoped_route_configs: - name: dynamic-foo route_configuration_name: dynamic-foo-route-config @@ -312,11 +657,10 @@ route_configuration_name: dynamic-foo-route-config const auto& scoped_routes_config_dump3 = TestUtility::downcastAndValidate( *message_ptr); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump3.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump3)); resources.Clear(); - factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(resources, - "2"); + srds_subscription_->onConfigUpdate(resources, "2"); TestUtility::loadFromYaml(R"EOF( inline_scoped_route_configs: - name: foo-scoped-routes @@ -333,7 +677,7 @@ route_configuration_name: dynamic-foo-route-config seconds: 1234567891 nanos: 234000000 dynamic_scoped_route_configs: - - name: foo-dynamic-scoped-routes + - name: foo_scoped_routes last_updated: seconds: 1234567891 nanos: 567000000 @@ -344,14 +688,13 @@ route_configuration_name: dynamic-foo-route-config const auto& scoped_routes_config_dump4 = TestUtility::downcastAndValidate( *message_ptr); - EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump4.DebugString()); + EXPECT_TRUE(TestUtility::protoEqual(expected_config_dump, scoped_routes_config_dump4)); } -using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderManagerTest; - // Tests that SRDS only allows creation of delta static config providers. -TEST_F(ScopedRoutesConfigProviderManagerDeathTest, DeltaStaticConfigProviderOnly) { - // Use match all regex due to lack of distinctive matchable output for coverage test. +TEST_F(ScopedRdsTest, DeltaStaticConfigProviderOnly) { + // Use match all regex due to lack of distinctive matchable output for + // coverage test. EXPECT_DEATH(config_provider_manager_->createStaticConfigProvider( parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 32e916da6d9a..e14e87d08fa7 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -48,7 +48,11 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, http_connection_manager) { const std::string& scope_key_builder_config_yaml = R"EOF( fragments: - - header_value_extractor: { name: X-Google-VIP } + - header_value_extractor: + name: Addr + element: + key: x-foo-key + separator: ; )EOF"; envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder scope_key_builder; @@ -124,6 +128,15 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, createStream(&scoped_rds_upstream_info_, getScopedRdsFakeUpstream()); } + void sendRdsResponse(const std::string& route_config, const std::string& version) { + envoy::api::v2::DiscoveryResponse response; + response.set_version_info(version); + response.set_type_url(Config::TypeUrl::get().RouteConfiguration); + response.add_resources()->PackFrom( + TestUtility::parseYaml(route_config)); + rds_upstream_info_.stream_->sendGrpcMessage(response); + } + void sendScopedRdsResponse(const std::vector& resource_protos, const std::string& version) { ASSERT(scoped_rds_upstream_info_.stream_ != nullptr); @@ -159,19 +172,31 @@ route_configuration_name: foo_route1 )EOF"; const std::string scope_route2 = R"EOF( name: foo_scope2 -route_configuration_name: foo_route2 +route_configuration_name: foo_route1 key: fragments: - - string_key: x-foo-key + - string_key: x-bar-key +)EOF"; + + const std::string route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: integration + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: {} }} )EOF"; - on_server_init_function_ = [this, &scope_route1, &scope_route2]() { + on_server_init_function_ = [&]() { createScopedRdsStream(); sendScopedRdsResponse({scope_route1, scope_route2}, "1"); + createRdsStream(); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_1"), "1"); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_2"), "2"); }; initialize(); - - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); // The version gauge should be set to xxHash64("1"). test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", @@ -179,20 +204,21 @@ route_configuration_name: foo_route2 const std::string scope_route3 = R"EOF( name: foo_scope3 -route_configuration_name: foo_route3 +route_configuration_name: foo_route1 key: fragments: - string_key: x-baz-key )EOF"; sendScopedRdsResponse({scope_route3}, "2"); - - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 2); test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", 6927017134761466251UL); - - // TODO(AndresGuedez): test actual scoped routing logic; only the config handling is implemented - // at this point. + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_attempt", 3); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_3"), "3"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 3); + // RDS updates won't affect SRDS. + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", + 6927017134761466251UL); } // Test that a bad config update updates the corresponding stats. diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index 827e9c84f875..291bd3248e9d 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -116,11 +116,25 @@ MockRoute::MockRoute() { } MockRoute::~MockRoute() = default; +MockRouteConfigProvider::MockRouteConfigProvider() { + ON_CALL(*this, config()).WillByDefault(Return(route_config_)); +} +MockRouteConfigProvider::~MockRouteConfigProvider() = default; + MockRouteConfigProviderManager::MockRouteConfigProviderManager() = default; MockRouteConfigProviderManager::~MockRouteConfigProviderManager() = default; -MockScopedConfig::MockScopedConfig() = default; +MockScopedConfig::MockScopedConfig() { + ON_CALL(*this, getRouteConfig(_)).WillByDefault(Return(route_config_)); +} MockScopedConfig::~MockScopedConfig() = default; +MockScopedRouteConfigProvider::MockScopedRouteConfigProvider() + : config_(std::make_shared()) { + ON_CALL(*this, getConfig()).WillByDefault(Return(config_)); + ON_CALL(*this, apiType()).WillByDefault(Return(ApiType::Delta)); +} +MockScopedRouteConfigProvider::~MockScopedRouteConfigProvider() = default; + } // namespace Router } // namespace Envoy diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 7e6ce48606ea..75249eeaad47 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -8,6 +8,8 @@ #include #include +#include "envoy/common/time.h" +#include "envoy/config/config_provider.h" #include "envoy/config/typed_metadata.h" #include "envoy/event/dispatcher.h" #include "envoy/json/json_object.h" @@ -31,6 +33,7 @@ namespace Envoy { namespace Router { +using ::testing::NiceMock; class MockDirectResponseEntry : public DirectResponseEntry { public: @@ -380,16 +383,29 @@ class MockConfig : public Config { std::string name_{"fake_config"}; }; +class MockRouteConfigProvider : public RouteConfigProvider { +public: + MockRouteConfigProvider(); + ~MockRouteConfigProvider() override; + + MOCK_METHOD0(config, ConfigConstSharedPtr()); + MOCK_CONST_METHOD0(configInfo, absl::optional()); + MOCK_CONST_METHOD0(lastUpdated, SystemTime()); + MOCK_METHOD0(onConfigUpdate, void()); + + std::shared_ptr> route_config_{new NiceMock()}; +}; + class MockRouteConfigProviderManager : public RouteConfigProviderManager { public: MockRouteConfigProviderManager(); ~MockRouteConfigProviderManager() override; - MOCK_METHOD3(createRdsRouteConfigProvider, + MOCK_METHOD4(createRdsRouteConfigProvider, RouteConfigProviderPtr( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix)); + const std::string& stat_prefix, Init::Manager& init_manager)); MOCK_METHOD2(createStaticRouteConfigProvider, RouteConfigProviderPtr(const envoy::api::v2::RouteConfiguration& route_config, Server::Configuration::FactoryContext& factory_context)); @@ -401,6 +417,23 @@ class MockScopedConfig : public ScopedConfig { ~MockScopedConfig() override; MOCK_CONST_METHOD1(getRouteConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); + + std::shared_ptr route_config_{new NiceMock()}; +}; + +class MockScopedRouteConfigProvider : public Envoy::Config::ConfigProvider { +public: + MockScopedRouteConfigProvider(); + ~MockScopedRouteConfigProvider() override; + + // Config::ConfigProvider + MOCK_CONST_METHOD0(lastUpdated, SystemTime()); + MOCK_CONST_METHOD0(getConfigProto, Protobuf::Message*()); + MOCK_CONST_METHOD0(getConfigProtos, Envoy::Config::ConfigProvider::ConfigProtoVector()); + MOCK_CONST_METHOD0(getConfig, ConfigConstSharedPtr()); + MOCK_CONST_METHOD0(apiType, ApiType()); + + std::shared_ptr config_; }; } // namespace Router From 225ad90b558aeb9228021663a60cd45cbfb034df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Guti=C3=A9rrez=20Segal=C3=A9s?= Date: Thu, 22 Aug 2019 23:06:12 -0400 Subject: [PATCH 437/542] Fix version history (#8021) Follow-up for #7995. Signed-off-by: Raul Gutierrez Segales --- docs/root/intro/version_history.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ade42d943ef2..60b1d5cbf208 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -25,7 +25,7 @@ Version history * grpc-json: added support for :ref:`ignoring unknown query parameters`. * header to metadata: added :ref:`PROTOBUF_VALUE ` and :ref:`ValueEncode ` to support protobuf Value and Base64 encoding. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. -* http: changed Envoy to forward existing x-forwarded-proto from upstream trusted proxies. Guarded by `envoy.reloadable_features.trusted_forwarded_proto` which defaults true. +* http: changed Envoy to forward existing x-forwarded-proto from downstream trusted proxies. Guarded by `envoy.reloadable_features.trusted_forwarded_proto` which defaults true. * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. From 7f970608bbbc08b6c61510bca5034bf00ab8e134 Mon Sep 17 00:00:00 2001 From: htuch Date: Thu, 22 Aug 2019 23:52:58 -0400 Subject: [PATCH 438/542] tools: sync tool for envoyproxy/assignable team. (#8015) Bulk update of team to match envoyproxy organization. While at it, cleaned up some venv stuff in shell_utils.sh. Risk level: Low Testing: Synced 157 members from envoyproxy to envoyproxy/assignable. Signed-off-by: Harvey Tuch --- .../deprecate_features/deprecate_features.sh | 9 +--- tools/deprecate_version/deprecate_version.sh | 9 +--- tools/github/requirements.txt | 1 + tools/github/sync_assignable.py | 53 +++++++++++++++++++ tools/github/sync_assignable.sh | 7 +++ tools/shell_utils.sh | 12 +++++ 6 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 tools/github/requirements.txt create mode 100644 tools/github/sync_assignable.py create mode 100755 tools/github/sync_assignable.sh diff --git a/tools/deprecate_features/deprecate_features.sh b/tools/deprecate_features/deprecate_features.sh index 43878548be09..661b348e0f0d 100644 --- a/tools/deprecate_features/deprecate_features.sh +++ b/tools/deprecate_features/deprecate_features.sh @@ -4,11 +4,4 @@ set -e -SCRIPT_DIR=$(realpath "$(dirname "$0")") -BUILD_DIR=build_tools -VENV_DIR="$BUILD_DIR"/deprecate_features - -source_venv "$VENV_DIR" -pip install -r "${SCRIPT_DIR}"/requirements.txt - -python "${SCRIPT_DIR}/deprecate_features.py" $* +python_venv deprecate_features diff --git a/tools/deprecate_version/deprecate_version.sh b/tools/deprecate_version/deprecate_version.sh index fe77384bcb6c..5421f66565b5 100755 --- a/tools/deprecate_version/deprecate_version.sh +++ b/tools/deprecate_version/deprecate_version.sh @@ -4,11 +4,4 @@ set -e -SCRIPT_DIR=$(realpath "$(dirname "$0")") -BUILD_DIR=build_tools -VENV_DIR="$BUILD_DIR"/deprecate_version - -source_venv "$VENV_DIR" -pip install -r "${SCRIPT_DIR}"/requirements.txt - -python "${SCRIPT_DIR}/deprecate_version.py" $* +python_venv deprecate_version diff --git a/tools/github/requirements.txt b/tools/github/requirements.txt new file mode 100644 index 000000000000..e1b66335b79b --- /dev/null +++ b/tools/github/requirements.txt @@ -0,0 +1 @@ +PyGithub==1.43.8 diff --git a/tools/github/sync_assignable.py b/tools/github/sync_assignable.py new file mode 100644 index 000000000000..3a437fa8d35f --- /dev/null +++ b/tools/github/sync_assignable.py @@ -0,0 +1,53 @@ +# Sync envoyproxy organization users to envoyproxy/assignable team. +# +# This can be used for bulk cleanups if envoyproxy/assignable is not consistent +# with organization membership. In general, prefer to add new members by editing +# the envoyproxy/assignable in the GitHub UI, which will also cause an +# organization invite to be sent; this reduces the need to manually manage +# access tokens. +# +# Note: the access token supplied must have admin:org (write:org, read:org) +# permissions (and ideally be scoped no more widely than this). See Settings -> +# Developer settings -> Personal access tokens for access token generation. +# Ideally, these should be cleaned up after use. + +import os +import sys + +import github + + +def GetConfirmation(): + """Obtain stdin confirmation to add users in GH.""" + return input('Add users to envoyproxy/assignable ? [yN] ').strip().lower() in ('y', 'yes') + + +def SyncAssignable(access_token): + organization = github.Github(access_token).get_organization('envoyproxy') + team = organization.get_team_by_slug('assignable') + organization_members = set(organization.get_members()) + assignable_members = set(team.get_members()) + missing = organization_members.difference(assignable_members) + + if not missing: + print('envoyproxy/assignable is consistent with organization membership.') + return 0 + + print('The following organization members are missing from envoyproxy/assignable:') + for m in missing: + print(m.login) + + if not GetConfirmation(): + return 1 + + for m in missing: + team.add_membership(m, 'member') + + +if __name__ == '__main__': + access_token = os.getenv('GH_ACCESS_TOKEN') + if not access_token: + print('Missing GH_ACCESS_TOKEN') + sys.exit(1) + + sys.exit(SyncAssignable(access_token)) diff --git a/tools/github/sync_assignable.sh b/tools/github/sync_assignable.sh new file mode 100755 index 000000000000..ac11d9ccc3c8 --- /dev/null +++ b/tools/github/sync_assignable.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +. tools/shell_utils.sh + +set -e + +python_venv sync_assignable diff --git a/tools/shell_utils.sh b/tools/shell_utils.sh index 4d0471e89aea..bb5a14c25e9e 100644 --- a/tools/shell_utils.sh +++ b/tools/shell_utils.sh @@ -9,3 +9,15 @@ source_venv() { echo "Found existing virtualenv" fi } + +python_venv() { + SCRIPT_DIR=$(realpath "$(dirname "$0")") + + BUILD_DIR=build_tools + VENV_DIR="$BUILD_DIR/$1" + + source_venv "$VENV_DIR" + pip install -r "${SCRIPT_DIR}"/requirements.txt + + python3 "${SCRIPT_DIR}/$1.py" $* +} From faad477782ec1c19614bbe8802685f433b50ca78 Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Thu, 22 Aug 2019 21:07:42 -0700 Subject: [PATCH 439/542] redis: fix onHostHealthUpdate got called before the cluster is resolved. (#8018) Signed-off-by: Henry Yang --- source/extensions/clusters/redis/redis_cluster_lb.cc | 5 +++++ test/extensions/clusters/redis/redis_cluster_lb_test.cc | 1 + 2 files changed, 6 insertions(+) diff --git a/source/extensions/clusters/redis/redis_cluster_lb.cc b/source/extensions/clusters/redis/redis_cluster_lb.cc index fefd17fff2fc..b4f3b1d06a2c 100644 --- a/source/extensions/clusters/redis/redis_cluster_lb.cc +++ b/source/extensions/clusters/redis/redis_cluster_lb.cc @@ -75,6 +75,11 @@ void RedisClusterLoadBalancerFactory::onHostHealthUpdate() { current_shard_vector = shard_vector_; } + // This can get called by cluster initialization before the Redis Cluster topology is resolved. + if (!current_shard_vector) { + return; + } + auto shard_vector = std::make_shared>(); for (auto const& shard : *current_shard_vector) { diff --git a/test/extensions/clusters/redis/redis_cluster_lb_test.cc b/test/extensions/clusters/redis/redis_cluster_lb_test.cc index dedcf8e51ec8..5e0ae78de9b2 100644 --- a/test/extensions/clusters/redis/redis_cluster_lb_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_lb_test.cc @@ -44,6 +44,7 @@ class RedisClusterLoadBalancerTest : public testing::Test { factory_ = std::make_shared(random_); lb_ = std::make_unique(factory_); lb_->initialize(); + factory_->onHostHealthUpdate(); } void validateAssignment(Upstream::HostVector& hosts, From e67923fc0b60d0cdf3935fcb30af61d131301fad Mon Sep 17 00:00:00 2001 From: htuch Date: Fri, 23 Aug 2019 03:01:23 -0400 Subject: [PATCH 440/542] api/build: migrate UDPA proto tree to external cncf/udpa repository. (#8017) This is a one-time movement of all UDPA content from envoyproxy/envoy to cncf/udpa. The permanent home of UDPA will be https://github.com/cncf/udpa. Risk level: Low Testing: Added UDPA service entry to build_test. Signed-off-by: Harvey Tuch --- api/bazel/api_build_system.bzl | 3 +- api/bazel/repositories.bzl | 4 +++ api/bazel/repository_locations.bzl | 8 +++++ api/test/build/BUILD | 1 + api/test/build/build_test.cc | 1 + api/udpa/data/orca/v1/BUILD | 16 --------- api/udpa/data/orca/v1/orca_load_report.proto | 36 ------------------- api/udpa/service/orca/v1/BUILD | 20 ----------- api/udpa/service/orca/v1/orca.proto | 38 -------------------- 9 files changed, 16 insertions(+), 111 deletions(-) delete mode 100644 api/udpa/data/orca/v1/BUILD delete mode 100644 api/udpa/data/orca/v1/orca_load_report.proto delete mode 100644 api/udpa/service/orca/v1/BUILD delete mode 100644 api/udpa/service/orca/v1/orca.proto diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index d9a3e2d943e6..14cc6f89359d 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -5,6 +5,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_test") _PY_SUFFIX = "_py" _CC_SUFFIX = "_cc" +_CC_EXPORT_SUFFIX = "_export_cc" _GO_PROTO_SUFFIX = "_go_proto" _GO_GRPC_SUFFIX = "_go_grpc" _GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/data-plane-api/api/" @@ -169,7 +170,7 @@ def api_cc_test(name, srcs, proto_deps): native.cc_test( name = name, srcs = srcs, - deps = [_LibrarySuffix(d, _CC_SUFFIX) for d in proto_deps], + deps = [_LibrarySuffix(d, _CC_EXPORT_SUFFIX) for d in proto_deps], ) def api_go_test(name, size, importpath, srcs = [], deps = []): diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index 3185d0f74072..d76337dc1200 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -15,6 +15,10 @@ def api_dependencies(): name = "com_google_googleapis", locations = REPOSITORY_LOCATIONS, ) + envoy_http_archive( + name = "com_github_cncf_udpa", + locations = REPOSITORY_LOCATIONS, + ) envoy_http_archive( name = "com_github_gogo_protobuf", locations = REPOSITORY_LOCATIONS, diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 0d0600a984bd..7b0bfeb07539 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -18,6 +18,9 @@ PROMETHEUS_SHA = "783bdaf8ee0464b35ec0c8704871e1e72afa0005c3f3587f65d9d6694bf391 KAFKA_SOURCE_SHA = "ae7a1696c0a0302b43c5b21e515c37e6ecd365941f68a510a7e442eebddf39a1" # 2.2.0-rc2 +UDPA_GIT_SHA = "4cbdcb9931ca743a915a7c5fda51b2ee793ed157" # Aug 22, 2019 +UDPA_SHA256 = "6291d0c0e3a4d5f08057ea7a00ed0b0ec3dd4e5a3b1cf20f803774680b5a806f" + REPOSITORY_LOCATIONS = dict( bazel_skylib = dict( sha256 = BAZEL_SKYLIB_SHA256, @@ -34,6 +37,11 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "googleapis-" + GOOGLEAPIS_GIT_SHA, urls = ["https://github.com/googleapis/googleapis/archive/" + GOOGLEAPIS_GIT_SHA + ".tar.gz"], ), + com_github_cncf_udpa = dict( + sha256 = UDPA_SHA256, + strip_prefix = "udpa-" + UDPA_GIT_SHA, + urls = ["https://github.com/cncf/udpa/archive/" + UDPA_GIT_SHA + ".tar.gz"], + ), com_github_gogo_protobuf = dict( sha256 = GOGOPROTO_SHA256, strip_prefix = "protobuf-" + GOGOPROTO_RELEASE, diff --git a/api/test/build/BUILD b/api/test/build/BUILD index ab1e8640a97d..a271aa5daf82 100644 --- a/api/test/build/BUILD +++ b/api/test/build/BUILD @@ -16,6 +16,7 @@ api_cc_test( "//envoy/service/discovery/v2:rtds", "//envoy/service/metrics/v2:metrics_service", "//envoy/service/ratelimit/v2:rls", + "@com_github_cncf_udpa//udpa/service/orca/v1:orca", ], ) diff --git a/api/test/build/build_test.cc b/api/test/build/build_test.cc index f0a8e7bf432c..540b2a51506d 100644 --- a/api/test/build/build_test.cc +++ b/api/test/build/build_test.cc @@ -23,6 +23,7 @@ int main(int argc, char* argv[]) { "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs", "envoy.service.metrics.v2.MetricsService.StreamMetrics", "envoy.service.ratelimit.v2.RateLimitService.ShouldRateLimit", + "udpa.service.orca.v1.OpenRcaService.StreamCoreMetrics", }; for (const auto& method : methods) { diff --git a/api/udpa/data/orca/v1/BUILD b/api/udpa/data/orca/v1/BUILD deleted file mode 100644 index 096ca28bac3b..000000000000 --- a/api/udpa/data/orca/v1/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library") - -licenses(["notice"]) # Apache 2 - -api_proto_library( - name = "orca_load_report", - srcs = ["orca_load_report.proto"], - visibility = [ - "//visibility:public", - ], -) - -api_go_proto_library( - name = "orca_load_report", - proto = ":orca_load_report", -) diff --git a/api/udpa/data/orca/v1/orca_load_report.proto b/api/udpa/data/orca/v1/orca_load_report.proto deleted file mode 100644 index 5c2eda6cd267..000000000000 --- a/api/udpa/data/orca/v1/orca_load_report.proto +++ /dev/null @@ -1,36 +0,0 @@ -syntax = "proto3"; - -package udpa.data.orca.v1; - -option java_outer_classname = "OrcaLoadReportProto"; -option java_multiple_files = true; -option java_package = "io.envoyproxy.udpa.data.orca.v1"; -option go_package = "v1"; - -import "validate/validate.proto"; - -// See section `ORCA load report format` of the design document in -// :ref:`https://github.com/envoyproxy/envoy/issues/6614`. - -message OrcaLoadReport { - // CPU utilization expressed as a fraction of available CPU resources. This - // should be derived from the latest sample or measurement. - double cpu_utilization = 1 [(validate.rules).double.gte = 0, (validate.rules).double.lte = 1]; - - // Memory utilization expressed as a fraction of available memory - // resources. This should be derived from the latest sample or measurement. - double mem_utilization = 2 [(validate.rules).double.gte = 0, (validate.rules).double.lte = 1]; - - // Total RPS being served by an endpoint. This should cover all services that an endpoint is - // responsible for. - uint64 rps = 3; - - // Application specific requests costs. Each value is an absolute cost (e.g. 3487 bytes of - // storage) associated with the request. - map request_cost = 4; - - // Resource utilization values. Each value is expressed as a fraction of total resources - // available, derived from the latest sample or measurement. - map utilization = 5 - [(validate.rules).map.values.double.gte = 0, (validate.rules).map.values.double.lte = 1]; -} diff --git a/api/udpa/service/orca/v1/BUILD b/api/udpa/service/orca/v1/BUILD deleted file mode 100644 index 72543e809221..000000000000 --- a/api/udpa/service/orca/v1/BUILD +++ /dev/null @@ -1,20 +0,0 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") - -licenses(["notice"]) # Apache 2 - -api_proto_library_internal( - name = "orca", - srcs = ["orca.proto"], - has_services = 1, - deps = [ - "//udpa/data/orca/v1:orca_load_report", - ], -) - -api_go_grpc_library( - name = "orca", - proto = ":orca", - deps = [ - "//udpa/data/orca/v1:orca_load_report_go_proto", - ], -) diff --git a/api/udpa/service/orca/v1/orca.proto b/api/udpa/service/orca/v1/orca.proto deleted file mode 100644 index 87871d209a4c..000000000000 --- a/api/udpa/service/orca/v1/orca.proto +++ /dev/null @@ -1,38 +0,0 @@ -syntax = "proto3"; - -package udpa.service.orca.v1; - -option java_outer_classname = "OrcaProto"; -option java_multiple_files = true; -option java_package = "io.envoyproxy.udpa.service.orca.v1"; -option go_package = "v1"; - -import "udpa/data/orca/v1/orca_load_report.proto"; - -import "google/protobuf/duration.proto"; - -import "validate/validate.proto"; - -// See section `Out-of-band (OOB) reporting` of the design document in -// :ref:`https://github.com/envoyproxy/envoy/issues/6614`. - -// Out-of-band (OOB) load reporting service for the additional load reporting -// agent that does not sit in the request path. Reports are periodically sampled -// with sufficient frequency to provide temporal association with requests. -// OOB reporting compensates the limitation of in-band reporting in revealing -// costs for backends that do not provide a steady stream of telemetry such as -// long running stream operations and zero QPS services. This is a server -// streaming service, client needs to terminate current RPC and initiate -// a new call to change backend reporting frequency. -service OpenRcaService { - rpc StreamCoreMetrics(OrcaLoadReportRequest) returns (stream udpa.data.orca.v1.OrcaLoadReport); -} - -message OrcaLoadReportRequest { - // Interval for generating Open RCA core metric responses. - google.protobuf.Duration report_interval = 1; - // Request costs to collect. If this is empty, all known requests costs tracked by - // the load reporting agent will be returned. This provides an opportunity for - // the client to selectively obtain a subset of tracked costs. - repeated string request_cost_names = 2; -} From e958cf9e58742d0caf4d07b59810f688c0212fe3 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Fri, 23 Aug 2019 12:18:38 -0400 Subject: [PATCH 441/542] http: tracking active session under L7 timers (#7782) Signed-off-by: Alyssa Wilk --- source/common/http/conn_manager_impl.cc | 2 +- source/common/router/router.cc | 2 ++ source/extensions/filters/http/fault/fault_filter.cc | 12 ++++++------ source/extensions/filters/http/fault/fault_filter.h | 4 +++- .../extensions/filters/http/squash/squash_filter.cc | 6 ++++-- test/common/http/conn_manager_impl_test.cc | 1 + test/common/router/router_test.cc | 1 + .../filters/http/fault/fault_filter_test.cc | 3 +++ .../filters/http/squash/squash_filter_test.cc | 4 ++++ 9 files changed, 25 insertions(+), 10 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 8ee08cf2b988..e5d7fe942155 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -471,7 +471,7 @@ ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connect std::chrono::milliseconds request_timeout_ms_ = connection_manager_.config_.requestTimeout(); request_timer_ = connection_manager.read_callbacks_->connection().dispatcher().createTimer( [this]() -> void { onRequestTimeout(); }); - request_timer_->enableTimer(request_timeout_ms_); + request_timer_->enableTimer(request_timeout_ms_, this); } stream_info_.setRequestedServerName( diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 5ad636479afc..8296c009525d 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -1539,6 +1539,8 @@ void Filter::UpstreamRequest::onPoolFailure(Http::ConnectionPool::PoolFailureRea void Filter::UpstreamRequest::onPoolReady(Http::StreamEncoder& request_encoder, Upstream::HostDescriptionConstSharedPtr host) { + // This may be called under an existing ScopeTrackerScopeState but it will unwind correctly. + ScopeTrackerScopeState scope(&parent_.callbacks_->scope(), parent_.callbacks_->dispatcher()); ENVOY_STREAM_LOG(debug, "pool ready", *parent_.callbacks_); host->outlierDetector().putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS); diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 7d914f553c2c..05dccd35c891 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -148,7 +148,7 @@ Http::FilterHeadersStatus FaultFilter::decodeHeaders(Http::HeaderMap& headers, b delay_timer_ = decoder_callbacks_->dispatcher().createTimer([this]() -> void { postDelayInjection(); }); ENVOY_LOG(debug, "fault: delaying request {}ms", duration.value().count()); - delay_timer_->enableTimer(duration.value()); + delay_timer_->enableTimer(duration.value(), &decoder_callbacks_->scope()); recordDelaysInjectedStats(); decoder_callbacks_->streamInfo().setResponseFlag(StreamInfo::ResponseFlag::DelayInjected); return Http::FilterHeadersStatus::StopIteration; @@ -192,7 +192,7 @@ void FaultFilter::maybeSetupResponseRateLimit(const Http::HeaderMap& request_hea encoder_callbacks_->injectEncodedDataToFilterChain(data, end_stream); }, [this] { encoder_callbacks_->continueEncoding(); }, config_->timeSource(), - decoder_callbacks_->dispatcher()); + decoder_callbacks_->dispatcher(), decoder_callbacks_->scope()); } bool FaultFilter::faultOverflow() { @@ -423,10 +423,10 @@ StreamRateLimiter::StreamRateLimiter(uint64_t max_kbps, uint64_t max_buffered_da std::function resume_data_cb, std::function write_data_cb, std::function continue_cb, TimeSource& time_source, - Event::Dispatcher& dispatcher) + Event::Dispatcher& dispatcher, const ScopeTrackedObject& scope) : // bytes_per_time_slice is KiB converted to bytes divided by the number of ticks per second. bytes_per_time_slice_((max_kbps * 1024) / SecondDivisor), write_data_cb_(write_data_cb), - continue_cb_(continue_cb), + continue_cb_(continue_cb), scope_(scope), // The token bucket is configured with a max token count of the number of ticks per second, // and refills at the same rate, so that we have a per second limit which refills gradually in // ~63ms intervals. @@ -472,7 +472,7 @@ void StreamRateLimiter::onTokenTimer() { const std::chrono::milliseconds ms = token_bucket_.nextTokenAvailable(); if (ms.count() > 0) { ENVOY_LOG(trace, "limiter: scheduling wakeup for {}ms", ms.count()); - token_timer_->enableTimer(ms); + token_timer_->enableTimer(ms, &scope_); } } @@ -498,7 +498,7 @@ void StreamRateLimiter::writeData(Buffer::Instance& incoming_buffer, bool end_st // The filter API does not currently support that and it will not be a trivial change to add. // Instead we cheat here by scheduling the token timer to run immediately after the stack is // unwound, at which point we can directly called encode/decodeData. - token_timer_->enableTimer(std::chrono::milliseconds(0)); + token_timer_->enableTimer(std::chrono::milliseconds(0), &scope_); } } diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index 824fecd64ea1..c4ae48fdcc58 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -144,12 +144,13 @@ class StreamRateLimiter : Logger::Loggable { * trailers that have been paused during body flush. * @param time_source the time source to run the token bucket with. * @param dispatcher the stream's dispatcher to use for creating timers. + * @param scope the stream's scope */ StreamRateLimiter(uint64_t max_kbps, uint64_t max_buffered_data, std::function pause_data_cb, std::function resume_data_cb, std::function write_data_cb, std::function continue_cb, TimeSource& time_source, - Event::Dispatcher& dispatcher); + Event::Dispatcher& dispatcher, const ScopeTrackedObject& scope); /** * Called by the stream to write data. All data writes happen asynchronously, the stream should @@ -180,6 +181,7 @@ class StreamRateLimiter : Logger::Loggable { const uint64_t bytes_per_time_slice_; const std::function write_data_cb_; const std::function continue_cb_; + const ScopeTrackedObject& scope_; TokenBucketImpl token_bucket_; Event::TimerPtr token_timer_; bool saw_data_{}; diff --git a/source/extensions/filters/http/squash/squash_filter.cc b/source/extensions/filters/http/squash/squash_filter.cc index 5a3b7d7b12cc..a4c58205c4e8 100644 --- a/source/extensions/filters/http/squash/squash_filter.cc +++ b/source/extensions/filters/http/squash/squash_filter.cc @@ -163,7 +163,8 @@ Http::FilterHeadersStatus SquashFilter::decodeHeaders(Http::HeaderMap& headers, attachment_timeout_timer_ = decoder_callbacks_->dispatcher().createTimer([this]() -> void { doneSquashing(); }); - attachment_timeout_timer_->enableTimer(config_->attachmentTimeout()); + attachment_timeout_timer_->enableTimer(config_->attachmentTimeout(), + &decoder_callbacks_->scope()); // Check if the timer expired inline. if (!is_squashing_) { return Http::FilterHeadersStatus::Continue; @@ -261,7 +262,8 @@ void SquashFilter::scheduleRetry() { attachment_poll_period_timer_ = decoder_callbacks_->dispatcher().createTimer([this]() -> void { pollForAttachment(); }); } - attachment_poll_period_timer_->enableTimer(config_->attachmentPollPeriod()); + attachment_poll_period_timer_->enableTimer(config_->attachmentPollPeriod(), + &decoder_callbacks_->scope()); } void SquashFilter::pollForAttachment() { diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index ea6f27e236a0..251c62394907 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -1865,6 +1865,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutCallbackDisarmsAndReturns408 EXPECT_CALL(response_encoder_, encodeData(_, true)).WillOnce(AddBufferToString(&response_body)); conn_manager_->newStream(response_encoder_); + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, setTrackedObject(_)).Times(2); request_timer->invokeCallback(); })); diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index fd91559acc1e..342c82c1150c 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -205,6 +205,7 @@ class RouterTestBase : public testing::Test { [&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder_ = &decoder; + EXPECT_CALL(callbacks_.dispatcher_, setTrackedObject(_)).Times(testing::AtLeast(2)); callbacks.onPoolReady(original_encoder_, cm_.conn_pool_.host_); return nullptr; })); diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index 038e674c73b3..d4f8ad2eb90e 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -25,6 +25,7 @@ #include "gtest/gtest.h" using testing::_; +using testing::AnyNumber; using testing::DoAll; using testing::Invoke; using testing::Matcher; @@ -136,6 +137,7 @@ class FaultFilterTest : public testing::Test { filter_ = std::make_unique(config_); filter_->setDecoderFilterCallbacks(decoder_filter_callbacks_); filter_->setEncoderFilterCallbacks(encoder_filter_callbacks_); + EXPECT_CALL(decoder_filter_callbacks_.dispatcher_, setTrackedObject(_)).Times(AnyNumber()); } void SetUpTest(const std::string json) { SetUpTest(convertJsonStrToProtoConfig(json)); } @@ -465,6 +467,7 @@ TEST_F(FaultFilterTest, DelayForDownstreamCluster) { EXPECT_CALL(decoder_filter_callbacks_, continueDecoding()); EXPECT_EQ(Http::FilterDataStatus::StopIterationAndWatermark, filter_->decodeData(data_, false)); + EXPECT_CALL(decoder_filter_callbacks_.dispatcher_, setTrackedObject(_)).Times(2); timer_->invokeCallback(); EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_headers_)); diff --git a/test/extensions/filters/http/squash/squash_filter_test.cc b/test/extensions/filters/http/squash/squash_filter_test.cc index 2cf509c59b5f..491f9930afa3 100644 --- a/test/extensions/filters/http/squash/squash_filter_test.cc +++ b/test/extensions/filters/http/squash/squash_filter_test.cc @@ -330,6 +330,7 @@ TEST_F(SquashFilterTest, Timeout) { EXPECT_CALL(request_, cancel()); EXPECT_CALL(filter_callbacks_, continueDecoding()); + EXPECT_CALL(filter_callbacks_.dispatcher_, setTrackedObject(_)).Times(2); attachmentTimeout_timer_->invokeCallback(); EXPECT_EQ(Envoy::Http::FilterDataStatus::Continue, filter_->decodeData(buffer, false)); @@ -359,6 +360,7 @@ TEST_F(SquashFilterTest, CheckRetryPollingAttachment) { // Expect the second get attachment request expectAsyncClientSend(); + EXPECT_CALL(filter_callbacks_.dispatcher_, setTrackedObject(_)).Times(2); retry_timer->invokeCallback(); EXPECT_CALL(filter_callbacks_, continueDecoding()); completeGetStatusRequest("attached"); @@ -378,6 +380,7 @@ TEST_F(SquashFilterTest, CheckRetryPollingAttachmentOnFailure) { // Expect the second get attachment request expectAsyncClientSend(); + EXPECT_CALL(filter_callbacks_.dispatcher_, setTrackedObject(_)).Times(2); retry_timer->invokeCallback(); EXPECT_CALL(filter_callbacks_, continueDecoding()); @@ -435,6 +438,7 @@ TEST_F(SquashFilterTest, TimerExpiresInline) { attachmentTimeout_timer_->scope_ = scope; attachmentTimeout_timer_->enabled_ = true; // timer expires inline + EXPECT_CALL(filter_callbacks_.dispatcher_, setTrackedObject(_)).Times(2); attachmentTimeout_timer_->invokeCallback(); })); From 73c2b64997b0e0662b5b4aeb23cdc77a048d54a5 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Fri, 23 Aug 2019 21:49:34 +0530 Subject: [PATCH 442/542] upstream: remove thread local cluster after triggering call backs (#8004) Signed-off-by: Rama Chavali --- .../common/upstream/cluster_manager_impl.cc | 2 +- .../upstream/cluster_manager_impl_test.cc | 2 +- test/integration/BUILD | 1 + test/integration/ads_integration.cc | 34 ++++++++++++++++ test/integration/ads_integration.h | 4 ++ test/integration/ads_integration_test.cc | 40 +++++++++++++++++++ 6 files changed, 81 insertions(+), 2 deletions(-) diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 7c4ac14f1bee..78bfade0a9d3 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -553,10 +553,10 @@ bool ClusterManagerImpl::removeCluster(const std::string& cluster_name) { ASSERT(cluster_manager.thread_local_clusters_.count(cluster_name) == 1); ENVOY_LOG(debug, "removing TLS cluster {}", cluster_name); - cluster_manager.thread_local_clusters_.erase(cluster_name); for (auto& cb : cluster_manager.update_callbacks_) { cb->onClusterRemoval(cluster_name); } + cluster_manager.thread_local_clusters_.erase(cluster_name); }); } diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index a7ada1599f80..29908486a15f 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1330,9 +1330,9 @@ TEST_F(ClusterManagerImplTest, DynamicAddRemove) { // tcp connections. Http::ConnectionPool::Instance::DrainedCb drained_cb; Tcp::ConnectionPool::Instance::DrainedCb drained_cb2; + EXPECT_CALL(*callbacks, onClusterRemoval(_)).Times(1); EXPECT_CALL(*cp, addDrainedCallback(_)).WillOnce(SaveArg<0>(&drained_cb)); EXPECT_CALL(*cp2, addDrainedCallback(_)).WillOnce(SaveArg<0>(&drained_cb2)); - EXPECT_CALL(*callbacks, onClusterRemoval(_)).Times(1); EXPECT_TRUE(cluster_manager_->removeCluster("fake_cluster")); EXPECT_EQ(nullptr, cluster_manager_->get("fake_cluster")); EXPECT_EQ(0UL, cluster_manager_->clusters().size()); diff --git a/test/integration/BUILD b/test/integration/BUILD index 60ababd92f17..96e499dce644 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -33,6 +33,7 @@ envoy_cc_test_library( "//source/common/config:protobuf_link_hacks", "//source/common/config:resources_lib", "//source/common/protobuf:utility_lib", + "//source/extensions/filters/network/redis_proxy:config", "//test/common/grpc:grpc_client_integration_lib", "//test/test_common:network_utility_lib", "//test/test_common:utility_lib", diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index cf62f423aaad..48a8af72bd5e 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -46,6 +46,17 @@ envoy::api::v2::Cluster AdsIntegrationTest::buildCluster(const std::string& name name)); } +envoy::api::v2::Cluster AdsIntegrationTest::buildRedisCluster(const std::string& name) { + return TestUtility::parseYaml(fmt::format(R"EOF( + name: {} + connect_timeout: 5s + type: EDS + eds_cluster_config: {{ eds_config: {{ ads: {{}} }} }} + lb_policy: MAGLEV + )EOF", + name)); +} + envoy::api::v2::ClusterLoadAssignment AdsIntegrationTest::buildClusterLoadAssignment(const std::string& name) { return TestUtility::parseYaml( @@ -87,6 +98,29 @@ envoy::api::v2::Listener AdsIntegrationTest::buildListener(const std::string& na name, Network::Test::getLoopbackAddressString(ipVersion()), stat_prefix, route_config)); } +envoy::api::v2::Listener AdsIntegrationTest::buildRedisListener(const std::string& name, + const std::string& cluster) { + return TestUtility::parseYaml(fmt::format( + R"EOF( + name: {} + address: + socket_address: + address: {} + port_value: 0 + filter_chains: + filters: + - name: envoy.redis_proxy + config: + settings: + op_timeout: 1s + stat_prefix: {} + prefix_routes: + catch_all_route: + cluster: {} + )EOF", + name, Network::Test::getLoopbackAddressString(ipVersion()), name, cluster)); +} + envoy::api::v2::RouteConfiguration AdsIntegrationTest::buildRouteConfig(const std::string& name, const std::string& cluster) { return TestUtility::parseYaml(fmt::format(R"EOF( diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index 024bcf04d607..628550fb9cea 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -49,11 +49,15 @@ class AdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public H envoy::api::v2::Cluster buildCluster(const std::string& name); + envoy::api::v2::Cluster buildRedisCluster(const std::string& name); + envoy::api::v2::ClusterLoadAssignment buildClusterLoadAssignment(const std::string& name); envoy::api::v2::Listener buildListener(const std::string& name, const std::string& route_config, const std::string& stat_prefix = "ads_test"); + envoy::api::v2::Listener buildRedisListener(const std::string& name, const std::string& cluster); + envoy::api::v2::RouteConfiguration buildRouteConfig(const std::string& name, const std::string& cluster); diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 9479194d73be..92357a7a24bb 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -200,6 +200,46 @@ TEST_P(AdsIntegrationTest, DuplicateInitialClusters) { test_server_->waitForCounterGe("cluster_manager.cds.update_rejected", 1); } +// Validates that removing a redis cluster does not crash Envoy. +// Regression test for issue https://github.com/envoyproxy/envoy/issues/7990. +TEST_P(AdsIntegrationTest, RedisClusterRemoval) { + initialize(); + + // Send initial configuration with a redis cluster and a redis proxy listener. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, true)); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildRedisCluster("redis_cluster")}, + {buildRedisCluster("redis_cluster")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", + {"redis_cluster"}, {}, {})); + sendDiscoveryResponse( + Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("redis_cluster")}, + {buildClusterLoadAssignment("redis_cluster")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); + sendDiscoveryResponse( + Config::TypeUrl::get().Listener, {buildRedisListener("listener_0", "redis_cluster")}, + {buildRedisListener("listener_0", "redis_cluster")}, {}, "1"); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", + {"redis_cluster"}, {}, {})); + + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); + + // Validate that redis listener is successfully created. + test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); + + // Now send a CDS update, removing redis cluster added above. + sendDiscoveryResponse( + Config::TypeUrl::get().Cluster, {buildCluster("cluster_2")}, {buildCluster("cluster_2")}, + {"redis_cluster"}, "2"); + + // Validate that the cluster is removed successfully. + test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1); +} + // Validate that the request with duplicate clusters in the subsequent requests (warming clusters) // is rejected. TEST_P(AdsIntegrationTest, DuplicateWarmingClusters) { From 07e3e287189dc1ce4c54b4e0252e29ae9cf6c19c Mon Sep 17 00:00:00 2001 From: Kateryna Nezdolii Date: Fri, 23 Aug 2019 18:27:44 +0200 Subject: [PATCH 443/542] upstream: Introducing close_connections_on_host_set_change property (#7675) Signed-off-by: Kateryna Nezdolii --- api/envoy/api/v2/cds.proto | 4 + docs/root/intro/version_history.rst | 1 + .../common/upstream/cluster_manager_impl.cc | 25 ++- source/common/upstream/cluster_manager_impl.h | 3 +- .../upstream/cluster_manager_impl_test.cc | 191 +++++++++++++++++- 5 files changed, 214 insertions(+), 10 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 647a5015063f..e7df4a940bc6 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -583,6 +583,10 @@ message Cluster { // If panic mode is triggered, new hosts are still eligible for traffic; they simply do not // contribute to the calculation when deciding whether panic mode is enabled or not. bool ignore_new_hosts_until_first_hc = 5; + + // If set to `true`, the cluster manager will drain all existing + // connections to upstream hosts whenever hosts are added or removed from the cluster. + bool close_connections_on_host_set_change = 6; } // Common configuration for all load balancer implementations. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 60b1d5cbf208..609554b4d374 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -42,6 +42,7 @@ Version history certificate validation context. * upstream: added network filter chains to upstream connections, see :ref:`filters`. * upstream: use p2c to select hosts for least-requests load balancers if all host weights are the same, even in cases where weights are not equal to 1. +* upstream: added :ref:`an option ` that allows draining HTTP, TCP connection pools on cluster membership change. * zookeeper: parse responses and emit latency stats. 1.11.1 (August 13, 2019) diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 78bfade0a9d3..1dd78f2dabfa 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -314,12 +314,21 @@ void ClusterManagerImpl::onClusterInit(Cluster& cluster) { // Now setup for cross-thread updates. cluster.prioritySet().addMemberUpdateCb( [&cluster, this](const HostVector&, const HostVector& hosts_removed) -> void { - // TODO(snowp): Should this be subject to merge windows? - - // Whenever hosts are removed from the cluster, we make each TLS cluster drain it's - // connection pools for the removed hosts. - if (!hosts_removed.empty()) { - postThreadLocalHostRemoval(cluster, hosts_removed); + if (cluster.info()->lbConfig().close_connections_on_host_set_change()) { + for (const auto& host_set : cluster.prioritySet().hostSetsPerPriority()) { + // This will drain all tcp and http connection pools. + postThreadLocalDrainConnections(cluster, host_set->hosts()); + } + } else { + // TODO(snowp): Should this be subject to merge windows? + + // Whenever hosts are removed from the cluster, we make each TLS cluster drain it's + // connection pools for the removed hosts. If `close_connections_on_host_set_change` is + // enabled, this case will be covered by first `if` statement, where all + // connection pools are drained. + if (!hosts_removed.empty()) { + postThreadLocalDrainConnections(cluster, hosts_removed); + } } }); @@ -712,8 +721,8 @@ Tcp::ConnectionPool::Instance* ClusterManagerImpl::tcpConnPoolForCluster( return entry->second->tcpConnPool(priority, context, transport_socket_options); } -void ClusterManagerImpl::postThreadLocalHostRemoval(const Cluster& cluster, - const HostVector& hosts_removed) { +void ClusterManagerImpl::postThreadLocalDrainConnections(const Cluster& cluster, + const HostVector& hosts_removed) { tls_->runOnAllThreads([this, name = cluster.info()->name(), hosts_removed]() { ThreadLocalClusterManagerImpl::removeHosts(name, hosts_removed, *tls_); }); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index ae0ecb2dbd40..00b892eaba67 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -232,7 +232,8 @@ class ClusterManagerImpl : public ClusterManager, Logger::LoggablesetInitializedCb([&]() -> void { initialized.ready(); }); + + std::unique_ptr callbacks(new NiceMock()); + ClusterUpdateCallbacksHandlePtr cb = + cluster_manager_->addThreadLocalClusterUpdateCallbacks(*callbacks); + + EXPECT_FALSE(cluster_manager_->get("cluster_1")->info()->addedViaApi()); + + // Verify that we get no hosts when the HostSet is empty. + EXPECT_EQ(nullptr, cluster_manager_->httpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); + EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, + nullptr, nullptr)); + EXPECT_EQ(nullptr, + cluster_manager_->tcpConnForCluster("cluster_1", nullptr, nullptr).connection_); + + Cluster& cluster = cluster_manager_->activeClusters().begin()->second; + + // Set up the HostSet. + HostSharedPtr host1 = makeTestHost(cluster.info(), "tcp://127.0.0.1:80"); + HostSharedPtr host2 = makeTestHost(cluster.info(), "tcp://127.0.0.1:81"); + + HostVector hosts{host1, host2}; + auto hosts_ptr = std::make_shared(hosts); + + // Sending non-mergeable updates. + cluster.prioritySet().updateHosts( + 0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {}, + 100); + + EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); + EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); + EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); + + EXPECT_CALL(factory_, allocateConnPool_(_, _)) + .Times(3) + .WillRepeatedly(ReturnNew()); + + EXPECT_CALL(factory_, allocateTcpConnPool_(_)) + .Times(3) + .WillRepeatedly(ReturnNew()); + + // This should provide us a CP for each of the above hosts. + Http::ConnectionPool::MockInstance* cp1 = + dynamic_cast(cluster_manager_->httpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); + // Create persistent connection for host2. + Http::ConnectionPool::MockInstance* cp2 = + dynamic_cast(cluster_manager_->httpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, Http::Protocol::Http2, nullptr)); + + Tcp::ConnectionPool::MockInstance* tcp1 = + dynamic_cast(cluster_manager_->tcpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + + Tcp::ConnectionPool::MockInstance* tcp2 = + dynamic_cast(cluster_manager_->tcpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + + EXPECT_NE(cp1, cp2); + EXPECT_NE(tcp1, tcp2); + + EXPECT_CALL(*cp2, addDrainedCallback(_)) + .WillOnce(Invoke([](Http::ConnectionPool::Instance::DrainedCb cb) { cb(); })); + + EXPECT_CALL(*cp1, addDrainedCallback(_)) + .WillOnce(Invoke([](Http::ConnectionPool::Instance::DrainedCb cb) { cb(); })); + + EXPECT_CALL(*tcp1, addDrainedCallback(_)) + .WillOnce(Invoke([](Tcp::ConnectionPool::Instance::DrainedCb cb) { cb(); })); + + EXPECT_CALL(*tcp2, addDrainedCallback(_)) + .WillOnce(Invoke([](Tcp::ConnectionPool::Instance::DrainedCb cb) { cb(); })); + + HostVector hosts_removed; + hosts_removed.push_back(host2); + + // This update should drain all connection pools (host1, host2). + cluster.prioritySet().updateHosts( + 0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, {}, + hosts_removed, 100); + + // Recreate connection pool for host1. + cp1 = dynamic_cast(cluster_manager_->httpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); + + tcp1 = dynamic_cast(cluster_manager_->tcpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + + HostSharedPtr host3 = makeTestHost(cluster.info(), "tcp://127.0.0.1:82"); + + HostVector hosts_added; + hosts_added.push_back(host3); + + EXPECT_CALL(*cp1, addDrainedCallback(_)) + .WillOnce(Invoke([](Http::ConnectionPool::Instance::DrainedCb cb) { cb(); })); + + EXPECT_CALL(*tcp1, addDrainedCallback(_)) + .WillOnce(Invoke([](Tcp::ConnectionPool::Instance::DrainedCb cb) { cb(); })); + + // Adding host3 should drain connection pool for host1. + cluster.prioritySet().updateHosts( + 0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, + hosts_added, {}, 100); +} + +TEST_F(ClusterManagerImplTest, ConnPoolsNotDrainedOnHostSetChange) { + const std::string yaml = R"EOF( + static_resources: + clusters: + - name: cluster_1 + connect_timeout: 0.250s + lb_policy: ROUND_ROBIN + type: STATIC + )EOF"; + + ReadyWatcher initialized; + EXPECT_CALL(initialized, ready()); + create(parseBootstrapFromV2Yaml(yaml)); + + // Set up for an initialize callback. + cluster_manager_->setInitializedCb([&]() -> void { initialized.ready(); }); + + std::unique_ptr callbacks(new NiceMock()); + ClusterUpdateCallbacksHandlePtr cb = + cluster_manager_->addThreadLocalClusterUpdateCallbacks(*callbacks); + + Cluster& cluster = cluster_manager_->activeClusters().begin()->second; + + // Set up the HostSet. + HostSharedPtr host1 = makeTestHost(cluster.info(), "tcp://127.0.0.1:80"); + + HostVector hosts{host1}; + auto hosts_ptr = std::make_shared(hosts); + + // Sending non-mergeable updates. + cluster.prioritySet().updateHosts( + 0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {}, + 100); + + EXPECT_CALL(factory_, allocateConnPool_(_, _)) + .Times(1) + .WillRepeatedly(ReturnNew()); + + EXPECT_CALL(factory_, allocateTcpConnPool_(_)) + .Times(1) + .WillRepeatedly(ReturnNew()); + + // This should provide us a CP for each of the above hosts. + Http::ConnectionPool::MockInstance* cp1 = + dynamic_cast(cluster_manager_->httpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); + + Tcp::ConnectionPool::MockInstance* tcp1 = + dynamic_cast(cluster_manager_->tcpConnPoolForCluster( + "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + + HostSharedPtr host2 = makeTestHost(cluster.info(), "tcp://127.0.0.1:82"); + HostVector hosts_added; + hosts_added.push_back(host2); + + // No connection pools should be drained. + EXPECT_CALL(*cp1, drainConnections()).Times(0); + EXPECT_CALL(*tcp1, drainConnections()).Times(0); + + // No connection pools should be drained. + cluster.prioritySet().updateHosts( + 0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, + hosts_added, {}, 100); +} + } // namespace } // namespace Upstream } // namespace Envoy From dbb11fd6743e8dbc69c2cd3ff63251e898ccafc0 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 23 Aug 2019 11:02:14 -0700 Subject: [PATCH 444/542] upstream: delete stale TODO (#8028) This was fixed in https://github.com/envoyproxy/envoy/pull/7877 Signed-off-by: Matt Klein --- source/common/upstream/upstream_impl.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 8443868f68f0..c50aa84f85d4 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -1311,12 +1311,6 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const HostVector& new_hosts, // At this point we've accounted for all the new hosts as well the hosts that previously // existed in this priority. - - // TODO(mattklein123): This stat is used by both the RR and LR load balancer to decide at - // runtime whether to use either the weighted or unweighted mode. If we extend weights to - // static clusters or DNS SRV clusters we need to make sure this gets set. Better, we should - // avoid pivoting on this entirely and probably just force a host set refresh if any weights - // change. info_->stats().max_host_weight_.set(max_host_weight); // Whatever remains in current_priority_hosts should be removed. From 640b5a436d2ce8e637d28225d5b4f0aae307dede Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Fri, 23 Aug 2019 12:18:04 -0700 Subject: [PATCH 445/542] Enhance comment about MonotonicTime (#8011) Depending on the execution environment in which envoy is being run, it is possible that some of the assumption on the clock are maybe not holding as previously commented. With some sandboxing technologies the clock does not reference the machine boot time but the sandbox boot time. This invalidates the assumtpion that the first update in the cluster_manager will most likely fall out of the windows and ends up showing a non intuitive behavior difficult to catch. This PR simply adds a comment that will allow the reader to consider this option while reading to the code. Signed-off-by: Flavio Crisciani --- api/envoy/api/v2/cds.proto | 3 ++- source/common/upstream/cluster_manager_impl.h | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index e7df4a940bc6..dec942853616 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -553,7 +553,8 @@ message Cluster { // the first update happens. This is useful for big clusters, with potentially noisy deploys // that might trigger excessive CPU usage due to a constant stream of healthcheck state changes // or metadata updates. The first set of updates to be seen apply immediately (e.g.: a new - // cluster). + // cluster). Please always keep in mind that the use of sandbox technologies may change this + // behavior. // // If this is not set, we default to a merge window of 1000ms. To disable it, set the merge // window to 0. diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 00b892eaba67..6b8f205ed9b2 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -420,9 +420,13 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable Date: Fri, 23 Aug 2019 15:41:03 -0400 Subject: [PATCH 446/542] build: some missing dep fixups for Google import. (#8026) Signed-off-by: Harvey Tuch --- source/common/upstream/BUILD | 2 ++ source/extensions/tracers/opencensus/BUILD | 1 + 2 files changed, 3 insertions(+) diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index e11007731ec6..cf2e800ef622 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -23,6 +23,7 @@ envoy_cc_library( "//source/common/config:utility_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/api/v2:cds_cc", + "@envoy_api//envoy/api/v2/cluster:outlier_detection_cc", ], ) @@ -267,6 +268,7 @@ envoy_cc_library( "//source/common/http:codes_lib", "//source/common/protobuf", "@envoy_api//envoy/api/v2:cds_cc", + "@envoy_api//envoy/api/v2/cluster:outlier_detection_cc", "@envoy_api//envoy/data/cluster/v2alpha:outlier_detection_event_cc", ], ) diff --git a/source/extensions/tracers/opencensus/BUILD b/source/extensions/tracers/opencensus/BUILD index 492648fdf088..c318395cad66 100644 --- a/source/extensions/tracers/opencensus/BUILD +++ b/source/extensions/tracers/opencensus/BUILD @@ -39,5 +39,6 @@ envoy_cc_library( deps = [ "//source/common/config:utility_lib", "//source/common/tracing:http_tracer_lib", + "@envoy_api//envoy/config/trace/v2:trace_cc", ], ) From eff020170c6267e6c8dc235473f7fc85c5b1e07d Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 23 Aug 2019 12:47:30 -0700 Subject: [PATCH 447/542] introduce safe regex matcher based on re2 engine (#7878) The libstdc++ std::regex implementation is not safe in all cases for user provided input. This change deprecates the used of std::regex in all user facing paths and introduces a new safe regex matcher with an explicitly configurable engine, right now limited to Google's re2 regex engine. This is not a drop in replacement for std::regex as all language features are not supported. As such we will go through a deprecation period for the old regex engine. Fixes https://github.com/envoyproxy/envoy/issues/7728 Signed-off-by: Matt Klein --- api/docs/BUILD | 1 + api/envoy/api/v2/route/BUILD | 4 + api/envoy/api/v2/route/route.proto | 88 ++++++++++++-- api/envoy/type/matcher/BUILD | 17 +++ api/envoy/type/matcher/regex.proto | 37 ++++++ api/envoy/type/matcher/string.proto | 11 +- bazel/repository_locations.bzl | 6 +- docs/build.sh | 1 + docs/root/api-v2/types/types.rst | 1 + docs/root/intro/deprecated.rst | 21 +++- docs/root/intro/version_history.rst | 12 +- include/envoy/common/BUILD | 13 +++ include/envoy/common/matchers.h | 28 +++++ include/envoy/common/regex.h | 21 ++++ include/envoy/router/BUILD | 1 + include/envoy/router/router.h | 10 +- .../common/access_log/access_log_formatter.cc | 1 + source/common/access_log/access_log_impl.cc | 7 +- source/common/access_log/access_log_impl.h | 2 +- source/common/common/BUILD | 15 +++ source/common/common/matchers.cc | 19 ++- source/common/common/matchers.h | 19 ++- source/common/common/regex.cc | 78 +++++++++++++ source/common/common/regex.h | 44 +++++++ source/common/common/utility.cc | 11 +- source/common/common/utility.h | 17 --- source/common/http/BUILD | 1 + source/common/http/header_utility.cc | 15 ++- source/common/http/header_utility.h | 35 ++++-- source/common/router/config_impl.cc | 109 +++++++++++------- source/common/router/config_impl.h | 23 ++-- source/common/router/config_utility.cc | 48 ++++++-- source/common/router/config_utility.h | 15 +-- source/common/router/router_ratelimit.cc | 7 +- source/common/router/router_ratelimit.h | 2 +- source/common/stats/BUILD | 1 + source/common/stats/stats_matcher_impl.cc | 6 +- source/common/stats/stats_matcher_impl.h | 2 +- source/common/stats/tag_extractor_impl.cc | 5 +- source/extensions/common/tap/tap_matcher.cc | 7 +- source/extensions/common/tap/tap_matcher.h | 2 +- .../common/ext_authz/ext_authz_http_impl.cc | 32 ++--- .../common/ext_authz/ext_authz_http_impl.h | 6 +- .../extensions/filters/common/rbac/matchers.h | 8 +- .../filters/http/cors/cors_filter.cc | 35 +----- .../filters/http/cors/cors_filter.h | 5 +- .../filters/http/csrf/csrf_filter.cc | 9 +- .../filters/http/csrf/csrf_filter.h | 31 ++--- .../filters/http/fault/fault_filter.cc | 7 +- .../filters/http/fault/fault_filter.h | 4 +- .../filters/http/health_check/config.cc | 8 +- .../filters/http/health_check/health_check.h | 2 +- .../filters/http/jwt_authn/matcher.cc | 38 +++--- .../filters/http/squash/squash_filter.h | 2 + .../dubbo_proxy/router/route_matcher.cc | 7 +- .../dubbo_proxy/router/route_matcher.h | 4 +- .../thrift_proxy/router/router_impl.cc | 8 +- .../network/thrift_proxy/router/router_impl.h | 2 +- .../router/router_ratelimit_impl.cc | 7 +- .../router/router_ratelimit_impl.h | 2 +- test/common/common/BUILD | 9 ++ test/common/common/matchers_test.cc | 9 ++ test/common/common/regex_test.cc | 61 ++++++++++ test/common/common/utility_test.cc | 16 --- test/common/http/header_utility_test.cc | 95 ++++++++++----- test/common/router/config_impl_test.cc | 74 ++++++++++-- .../http/cors/cors_filter_integration_test.cc | 23 ++-- .../filters/http/cors/cors_filter_test.cc | 40 +++++-- .../http/health_check/health_check_test.cc | 4 +- .../filters/http/jwt_authn/matcher_test.cc | 22 ++++ .../network/thrift_proxy/integration_test.cc | 4 +- test/integration/http_integration.cc | 2 +- test/mocks/router/mocks.h | 17 ++- test/test_common/utility.cc | 1 + tools/check_format.py | 17 +++ tools/check_format_test_helper.py | 2 + tools/protodoc/protodoc.py | 5 + tools/testdata/check_format/regex.cc | 9 ++ 78 files changed, 1004 insertions(+), 386 deletions(-) create mode 100644 api/envoy/type/matcher/regex.proto create mode 100644 include/envoy/common/matchers.h create mode 100644 include/envoy/common/regex.h create mode 100644 source/common/common/regex.cc create mode 100644 source/common/common/regex.h create mode 100644 test/common/common/regex_test.cc create mode 100644 tools/testdata/check_format/regex.cc diff --git a/api/docs/BUILD b/api/docs/BUILD index da20fe5e7171..11ef3876b218 100644 --- a/api/docs/BUILD +++ b/api/docs/BUILD @@ -102,6 +102,7 @@ proto_library( "//envoy/type:range", "//envoy/type/matcher:metadata", "//envoy/type/matcher:number", + "//envoy/type/matcher:regex", "//envoy/type/matcher:string", ], ) diff --git a/api/envoy/api/v2/route/BUILD b/api/envoy/api/v2/route/BUILD index 2fec56ae389b..968ab1c67be2 100644 --- a/api/envoy/api/v2/route/BUILD +++ b/api/envoy/api/v2/route/BUILD @@ -10,6 +10,8 @@ api_proto_library_internal( "//envoy/api/v2/core:base", "//envoy/type:percent", "//envoy/type:range", + "//envoy/type/matcher:regex", + "//envoy/type/matcher:string", ], ) @@ -20,5 +22,7 @@ api_go_proto_library( "//envoy/api/v2/core:base_go_proto", "//envoy/type:percent_go_proto", "//envoy/type:range_go_proto", + "//envoy/type/matcher:regex_go_proto", + "//envoy/type/matcher:string_go_proto", ], ) diff --git a/api/envoy/api/v2/route/route.proto b/api/envoy/api/v2/route/route.proto index 93d021ddb216..e0fbaf0fd685 100644 --- a/api/envoy/api/v2/route/route.proto +++ b/api/envoy/api/v2/route/route.proto @@ -9,6 +9,8 @@ option go_package = "route"; option java_generic_services = true; import "envoy/api/v2/core/base.proto"; +import "envoy/type/matcher/regex.proto"; +import "envoy/type/matcher/string.proto"; import "envoy/type/percent.proto"; import "envoy/type/range.proto"; @@ -349,7 +351,25 @@ message RouteMatch { // * The regex */b[io]t* matches the path */bot* // * The regex */b[io]t* does not match the path */bite* // * The regex */b[io]t* does not match the path */bit/bot* - string regex = 3 [(validate.rules).string.max_bytes = 1024]; + // + // .. attention:: + // This field has been deprecated in favor of `safe_regex` as it is not safe for use with + // untrusted input in all cases. + string regex = 3 [(validate.rules).string.max_bytes = 1024, deprecated = true]; + + // If specified, the route is a regular expression rule meaning that the + // regex must match the *:path* header once the query string is removed. The entire path + // (without the query string) must match the regex. The rule will not match if only a + // subsequence of the *:path* header matches the regex. + // + // [#next-major-version: In the v3 API we should redo how path specification works such + // that we utilize StringMatcher, and additionally have consistent options around whether we + // strip query strings, do a case sensitive match, etc. In the interim it will be too disruptive + // to deprecate the existing options. We should even consider whether we want to do away with + // path_specifier entirely and just rely on a set of header matchers which can already match + // on :path, etc. The issue with that is it is unclear how to generically deal with query string + // stripping. This needs more thought.] + type.matcher.RegexMatcher safe_regex = 10 [(validate.rules).message.required = true]; } // Indicates that prefix/path matching should be case insensitive. The default @@ -404,12 +424,24 @@ message CorsPolicy { // Specifies the origins that will be allowed to do CORS requests. // // An origin is allowed if either allow_origin or allow_origin_regex match. - repeated string allow_origin = 1; + // + // .. attention:: + // This field has been deprecated in favor of `allow_origin_string_match`. + repeated string allow_origin = 1 [deprecated = true]; // Specifies regex patterns that match allowed origins. // // An origin is allowed if either allow_origin or allow_origin_regex match. - repeated string allow_origin_regex = 8 [(validate.rules).repeated .items.string.max_bytes = 1024]; + // + // .. attention:: + // This field has been deprecated in favor of `allow_origin_string_match` as it is not safe for + // use with untrusted input in all cases. + repeated string allow_origin_regex = 8 + [(validate.rules).repeated .items.string.max_bytes = 1024, deprecated = true]; + + // Specifies string patterns that match allowed origins. An origin is allowed if any of the + // string matchers match. + repeated type.matcher.StringMatcher allow_origin_string_match = 11; // Specifies the content for the *access-control-allow-methods* header. string allow_methods = 2; @@ -1077,18 +1109,28 @@ message VirtualCluster { // * The regex */rides/\d+* matches the path */rides/0* // * The regex */rides/\d+* matches the path */rides/123* // * The regex */rides/\d+* does not match the path */rides/123/456* - string pattern = 1 [(validate.rules).string = {min_bytes: 1, max_bytes: 1024}]; + // + // .. attention:: + // This field has been deprecated in favor of `headers` as it is not safe for use with + // untrusted input in all cases. + string pattern = 1 [(validate.rules).string.max_bytes = 1024, deprecated = true]; + + // Specifies a list of header matchers to use for matching requests. Each specified header must + // match. The pseudo-headers `:path` and `:method` can be used to match the request path and + // method, respectively. + repeated HeaderMatcher headers = 4; - // Specifies the name of the virtual cluster. The virtual cluster name as well + // Specifies the name of the virtual cluster. The virtual cluster name as well // as the virtual host name are used when emitting statistics. The statistics are emitted by the // router filter and are documented :ref:`here `. string name = 2 [(validate.rules).string.min_bytes = 1]; // Optionally specifies the HTTP method to match on. For example GET, PUT, // etc. - // [#comment:TODO(htuch): add (validate.rules).enum.defined_only = true once - // https://github.com/lyft/protoc-gen-validate/issues/42 is resolved.] - core.RequestMethod method = 3; + // + // .. attention:: + // This field has been deprecated in favor of `headers`. + core.RequestMethod method = 3 [deprecated = true]; } // Global rate limiting :ref:`architecture overview `. @@ -1248,6 +1290,7 @@ message RateLimit { // ` header will match, regardless of the header's // value. // +// [#next-major-version: HeaderMatcher should be refactored to use StringMatcher.] message HeaderMatcher { // Specifies the name of the header in the request. string name = 1 [(validate.rules).string.min_bytes = 1]; @@ -1273,7 +1316,16 @@ message HeaderMatcher { // * The regex *\d{3}* matches the value *123* // * The regex *\d{3}* does not match the value *1234* // * The regex *\d{3}* does not match the value *123.456* - string regex_match = 5 [(validate.rules).string.max_bytes = 1024]; + // + // .. attention:: + // This field has been deprecated in favor of `safe_regex_match` as it is not safe for use + // with untrusted input in all cases. + string regex_match = 5 [(validate.rules).string.max_bytes = 1024, deprecated = true]; + + // If specified, this regex string is a regular expression rule which implies the entire request + // header value must match the regex. The rule will not match if only a subsequence of the + // request header value matches the regex. + type.matcher.RegexMatcher safe_regex_match = 11; // If specified, header match will be performed based on range. // The rule will match if the request header value is within this range. @@ -1328,11 +1380,25 @@ message QueryParameterMatcher { // Specifies the value of the key. If the value is absent, a request // that contains the key in its query string will match, whether the // key appears with a value (e.g., "?debug=true") or not (e.g., "?debug") - string value = 3; + // + // ..attention:: + // This field is deprecated. Use an `exact` match inside the `string_match` field. + string value = 3 [deprecated = true]; // Specifies whether the query parameter value is a regular expression. // Defaults to false. The entire query parameter value (i.e., the part to // the right of the equals sign in "key=value") must match the regex. // E.g., the regex "\d+$" will match "123" but not "a123" or "123a". - google.protobuf.BoolValue regex = 4; + // + // ..attention:: + // This field is deprecated. Use a `safe_regex` match inside the `string_match` field. + google.protobuf.BoolValue regex = 4 [deprecated = true]; + + oneof query_parameter_match_specifier { + // Specifies whether a query parameter value should match against a string. + type.matcher.StringMatcher string_match = 5 [(validate.rules).message.required = true]; + + // Specifies whether a query parameter should be present. + bool present_match = 6; + } } diff --git a/api/envoy/type/matcher/BUILD b/api/envoy/type/matcher/BUILD index ec4aa09b6c63..5fe594db4ca2 100644 --- a/api/envoy/type/matcher/BUILD +++ b/api/envoy/type/matcher/BUILD @@ -40,11 +40,17 @@ api_proto_library_internal( name = "string", srcs = ["string.proto"], visibility = ["//visibility:public"], + deps = [ + ":regex", + ], ) api_go_proto_library( name = "string", proto = ":string", + deps = [ + ":regex_go_proto", + ], ) api_proto_library_internal( @@ -65,3 +71,14 @@ api_go_proto_library( ":string_go_proto", ], ) + +api_proto_library_internal( + name = "regex", + srcs = ["regex.proto"], + visibility = ["//visibility:public"], +) + +api_go_proto_library( + name = "regex", + proto = ":regex", +) diff --git a/api/envoy/type/matcher/regex.proto b/api/envoy/type/matcher/regex.proto new file mode 100644 index 000000000000..b3b7194441eb --- /dev/null +++ b/api/envoy/type/matcher/regex.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +package envoy.type.matcher; + +option java_outer_classname = "StringProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.type.matcher"; +option go_package = "matcher"; + +import "google/protobuf/wrappers.proto"; +import "validate/validate.proto"; + +// [#protodoc-title: RegexMatcher] + +// A regex matcher designed for safety when used with untrusted input. +message RegexMatcher { + // Google's `RE2 `_ regex engine. The regex string must adhere to + // the documented `syntax `_. The engine is designed + // to complete execution in linear time as well as limit the amount of memory used. + message GoogleRE2 { + // This field controls the RE2 "program size" which is a rough estimate of how complex a + // compiled regex is to evaluate. A regex that has a program size greater than the configured + // value will fail to compile. In this case, the configured max program size can be increased + // or the regex can be simplified. If not specified, the default is 100. + google.protobuf.UInt32Value max_program_size = 1; + } + + oneof engine_type { + option (validate.required) = true; + + // Google's RE2 regex engine. + GoogleRE2 google_re2 = 1 [(validate.rules).message.required = true]; + } + + // The regex match string. The string must be supported by the configured engine. + string regex = 2 [(validate.rules).string.min_bytes = 1]; +} diff --git a/api/envoy/type/matcher/string.proto b/api/envoy/type/matcher/string.proto index 55f2171af53e..35628b7ccb10 100644 --- a/api/envoy/type/matcher/string.proto +++ b/api/envoy/type/matcher/string.proto @@ -7,6 +7,8 @@ option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type.matcher"; option go_package = "matcher"; +import "envoy/type/matcher/regex.proto"; + import "validate/validate.proto"; // [#protodoc-title: StringMatcher] @@ -48,7 +50,14 @@ message StringMatcher { // * The regex *\d{3}* matches the value *123* // * The regex *\d{3}* does not match the value *1234* // * The regex *\d{3}* does not match the value *123.456* - string regex = 4 [(validate.rules).string.max_bytes = 1024]; + // + // .. attention:: + // This field has been deprecated in favor of `safe_regex` as it is not safe for use with + // untrusted input in all cases. + string regex = 4 [(validate.rules).string.max_bytes = 1024, deprecated = true]; + + // The input string must match the regular expression specified here. + RegexMatcher safe_regex = 5 [(validate.rules).message.required = true]; } } diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index ebb3e8569064..7c2c5ad23e62 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -253,8 +253,8 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/google/cel-cpp/archive/d9d02b20ab85da2444dbdd03410bac6822141364.tar.gz"], ), com_googlesource_code_re2 = dict( - sha256 = "f31db9cd224d018a7e4fe88ef84aaa874b0b3ed91d4d98ee5a1531101d3fdc64", - strip_prefix = "re2-87e2ad45e7b18738e1551474f7ee5886ff572059", - urls = ["https://github.com/google/re2/archive/87e2ad45e7b18738e1551474f7ee5886ff572059.tar.gz"], + sha256 = "38bc0426ee15b5ed67957017fd18201965df0721327be13f60496f2b356e3e01", + strip_prefix = "re2-2019-08-01", + urls = ["https://github.com/google/re2/archive/2019-08-01.tar.gz"], ), ) diff --git a/docs/build.sh b/docs/build.sh index b147712dd537..7b89527bb24c 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -156,6 +156,7 @@ PROTO_RST=" /envoy/type/matcher/metadata/envoy/type/matcher/metadata.proto.rst /envoy/type/matcher/value/envoy/type/matcher/value.proto.rst /envoy/type/matcher/number/envoy/type/matcher/number.proto.rst + /envoy/type/matcher/regex/envoy/type/matcher/regex.proto.rst /envoy/type/matcher/string/envoy/type/matcher/string.proto.rst " diff --git a/docs/root/api-v2/types/types.rst b/docs/root/api-v2/types/types.rst index 4a6ba6194c62..a5a84c33a13d 100644 --- a/docs/root/api-v2/types/types.rst +++ b/docs/root/api-v2/types/types.rst @@ -10,5 +10,6 @@ Types ../type/range.proto ../type/matcher/metadata.proto ../type/matcher/number.proto + ../type/matcher/regex.proto ../type/matcher/string.proto ../type/matcher/value.proto diff --git a/docs/root/intro/deprecated.rst b/docs/root/intro/deprecated.rst index 6ca0b034b759..fb7832739383 100644 --- a/docs/root/intro/deprecated.rst +++ b/docs/root/intro/deprecated.rst @@ -12,8 +12,25 @@ Deprecated items below are listed in chronological order. Version 1.12.0 (pending) ======================== -* The ORIGINAL_DST_LB :ref:`load balancing policy ` is deprecated, use CLUSTER_PROVIDED policy instead when configuring an :ref:`original destination cluster `. -* The :option:`--allow-unknown-fields` command-line option, use :option:`--allow-unknown-static-fields` instead. +* The ORIGINAL_DST_LB :ref:`load balancing policy ` is + deprecated, use CLUSTER_PROVIDED policy instead when configuring an :ref:`original destination + cluster `. +* The `regex` field in :ref:`StringMatcher ` has been + deprecated in favor of the `safe_regex` field. +* The `regex` field in :ref:`RouteMatch ` has been + deprecated in favor of the `safe_regex` field. +* The `allow_origin` and `allow_origin_regex` fields in :ref:`CorsPolicy + ` have been deprecated in favor of the + `allow_origin_string_match` field. +* The `pattern` and `method` fields in :ref:`VirtualCluster ` + have been deprecated in favor of the `headers` field. +* The `regex_match` field in :ref:`HeaderMatcher ` has been + deprecated in favor of the `safe_regex_match` field. +* The `value` and `regex` fields in :ref:`QueryParameterMatcher + ` has been deprecated in favor of the `string_match` + and `present_match` fields. +* The :option:`--allow-unknown-fields` command-line option, + use :option:`--allow-unknown-static-fields` instead. Version 1.11.0 (July 11, 2019) ============================== diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 609554b4d374..caf541e796d5 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -7,8 +7,8 @@ Version history * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * api: added ::ref:`set_node_on_first_message_only ` option to omit the node identifier from the subsequent discovery requests on the same stream. -* config: enforcing that terminal filters (e.g. HttpConnectionManager for L4, router for L7) be the last in their respective filter chains. * buffer filter: the buffer filter populates content-length header if not present, behavior can be disabled using the runtime feature `envoy.reloadable_features.buffer_filter_populate_content_length`. +* config: enforcing that terminal filters (e.g. HttpConnectionManager for L4, router for L7) be the last in their respective filter chains. * config: added access log :ref:`extension filter`. * config: added support for :option:`--reject-unknown-dynamic-fields`, providing independent control over whether unknown fields are rejected in static and dynamic configuration. By default, unknown @@ -29,11 +29,15 @@ Version history * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. -* performance: stats symbol table implementation (disabled by default; to test it, add "--use-fake-symbol-table 0" to the command-line arguments when starting Envoy). -* redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. -* rbac: added support for DNS SAN as :ref:`principal_name `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). +* performance: stats symbol table implementation (disabled by default; to test it, add "--use-fake-symbol-table 0" to the command-line arguments when starting Envoy). +* rbac: added support for DNS SAN as :ref:`principal_name `. +* redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. +* regex: introduce new :ref:`RegexMatcher ` type that + provides a safe regex implementation for untrusted user input. This type is now used in all + configuration that processes user provided input. See :ref:`deprecated configuration details + ` for more information. * rbac: added conditions to the policy, see :ref:`condition `. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * router check tool: add coverage reporting & enforcement. diff --git a/include/envoy/common/BUILD b/include/envoy/common/BUILD index 4e04008c25d0..105f8b4374b7 100644 --- a/include/envoy/common/BUILD +++ b/include/envoy/common/BUILD @@ -29,6 +29,19 @@ envoy_cc_library( hdrs = ["time.h"], ) +envoy_cc_library( + name = "matchers_interface", + hdrs = ["matchers.h"], +) + +envoy_cc_library( + name = "regex_interface", + hdrs = ["regex.h"], + deps = [ + ":matchers_interface", + ], +) + envoy_cc_library( name = "token_bucket_interface", hdrs = ["token_bucket.h"], diff --git a/include/envoy/common/matchers.h b/include/envoy/common/matchers.h new file mode 100644 index 000000000000..4a79d00b97d6 --- /dev/null +++ b/include/envoy/common/matchers.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "envoy/common/pure.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Matchers { + +/** + * Generic string matching interface. + */ +class StringMatcher { +public: + virtual ~StringMatcher() = default; + + /** + * Return whether a passed string value matches. + */ + virtual bool match(const absl::string_view value) const PURE; +}; + +using StringMatcherPtr = std::unique_ptr; + +} // namespace Matchers +} // namespace Envoy diff --git a/include/envoy/common/regex.h b/include/envoy/common/regex.h new file mode 100644 index 000000000000..f4cdc1699ef7 --- /dev/null +++ b/include/envoy/common/regex.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "envoy/common/matchers.h" + +namespace Envoy { +namespace Regex { + +/** + * A compiled regex expression matcher which uses an abstract regex engine. + * + * NOTE: Currently this is the same as StringMatcher, however has been split out as in the future + * we are likely to add other methods such as returning captures, etc. + */ +class CompiledMatcher : public Matchers::StringMatcher {}; + +using CompiledMatcherPtr = std::unique_ptr; + +} // namespace Regex +} // namespace Envoy diff --git a/include/envoy/router/BUILD b/include/envoy/router/BUILD index 3ebe1a1b70e3..d62054044670 100644 --- a/include/envoy/router/BUILD +++ b/include/envoy/router/BUILD @@ -48,6 +48,7 @@ envoy_cc_library( external_deps = ["abseil_optional"], deps = [ "//include/envoy/access_log:access_log_interface", + "//include/envoy/common:matchers_interface", "//include/envoy/config:typed_metadata_interface", "//include/envoy/http:codec_interface", "//include/envoy/http:codes_interface", diff --git a/include/envoy/router/router.h b/include/envoy/router/router.h index a14864b4b287..5b793ec8a841 100644 --- a/include/envoy/router/router.h +++ b/include/envoy/router/router.h @@ -10,6 +10,7 @@ #include "envoy/access_log/access_log.h" #include "envoy/api/v2/core/base.pb.h" +#include "envoy/common/matchers.h" #include "envoy/config/typed_metadata.h" #include "envoy/http/codec.h" #include "envoy/http/codes.h" @@ -101,14 +102,9 @@ class CorsPolicy { virtual ~CorsPolicy() = default; /** - * @return std::list& access-control-allow-origin values. + * @return std::vector& access-control-allow-origin matchers. */ - virtual const std::list& allowOrigins() const PURE; - - /* - * @return std::list& regexes that match allowed origins. - */ - virtual const std::list& allowOriginRegexes() const PURE; + virtual const std::vector& allowOrigins() const PURE; /** * @return std::string access-control-allow-methods value. diff --git a/source/common/access_log/access_log_formatter.cc b/source/common/access_log/access_log_formatter.cc index 899ae27ff8d1..295c544aa76e 100644 --- a/source/common/access_log/access_log_formatter.cc +++ b/source/common/access_log/access_log_formatter.cc @@ -1,6 +1,7 @@ #include "common/access_log/access_log_formatter.h" #include +#include #include #include diff --git a/source/common/access_log/access_log_impl.cc b/source/common/access_log/access_log_impl.cc index 498c3e18176d..837842f621ae 100644 --- a/source/common/access_log/access_log_impl.cc +++ b/source/common/access_log/access_log_impl.cc @@ -193,13 +193,12 @@ bool NotHealthCheckFilter::evaluate(const StreamInfo::StreamInfo& info, const Ht return !info.healthCheck(); } -HeaderFilter::HeaderFilter(const envoy::config::filter::accesslog::v2::HeaderFilter& config) { - header_data_.push_back(Http::HeaderUtility::HeaderData(config.header())); -} +HeaderFilter::HeaderFilter(const envoy::config::filter::accesslog::v2::HeaderFilter& config) + : header_data_(std::make_unique(config.header())) {} bool HeaderFilter::evaluate(const StreamInfo::StreamInfo&, const Http::HeaderMap& request_headers, const Http::HeaderMap&, const Http::HeaderMap&) { - return Http::HeaderUtility::matchHeaders(request_headers, header_data_); + return Http::HeaderUtility::matchHeaders(request_headers, *header_data_); } ResponseFlagFilter::ResponseFlagFilter( diff --git a/source/common/access_log/access_log_impl.h b/source/common/access_log/access_log_impl.h index 7e3aa51b3329..345ea5a4938c 100644 --- a/source/common/access_log/access_log_impl.h +++ b/source/common/access_log/access_log_impl.h @@ -178,7 +178,7 @@ class HeaderFilter : public Filter { const Http::HeaderMap& response_trailers) override; private: - std::vector header_data_; + const Http::HeaderUtility::HeaderDataPtr header_data_; }; /** diff --git a/source/common/common/BUILD b/source/common/common/BUILD index 6ace52dbb343..fbe3714e6374 100644 --- a/source/common/common/BUILD +++ b/source/common/common/BUILD @@ -165,6 +165,8 @@ envoy_cc_library( external_deps = ["abseil_optional"], deps = [ ":utility_lib", + "//include/envoy/common:matchers_interface", + "//source/common/common:regex_lib", "//source/common/config:metadata_lib", "//source/common/protobuf", "@envoy_api//envoy/type/matcher:metadata_cc", @@ -174,6 +176,19 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "regex_lib", + srcs = ["regex.cc"], + hdrs = ["regex.h"], + deps = [ + ":assert_lib", + "//include/envoy/common:regex_interface", + "//source/common/protobuf:utility_lib", + "@com_googlesource_code_re2//:re2", + "@envoy_api//envoy/type/matcher:regex_cc", + ], +) + envoy_cc_library( name = "non_copyable", hdrs = ["non_copyable.h"], diff --git a/source/common/common/matchers.cc b/source/common/common/matchers.cc index 2fb5f4b918b5..4c648ebe8eb6 100644 --- a/source/common/common/matchers.cc +++ b/source/common/common/matchers.cc @@ -2,6 +2,7 @@ #include "envoy/api/v2/core/base.pb.h" +#include "common/common/regex.h" #include "common/config/metadata.h" #include "absl/strings/match.h" @@ -16,7 +17,7 @@ ValueMatcherConstSharedPtr ValueMatcher::create(const envoy::type::matcher::Valu case envoy::type::matcher::ValueMatcher::kDoubleMatch: return std::make_shared(v.double_match()); case envoy::type::matcher::ValueMatcher::kStringMatch: - return std::make_shared(v.string_match()); + return std::make_shared(v.string_match()); case envoy::type::matcher::ValueMatcher::kBoolMatch: return std::make_shared(v.bool_match()); case envoy::type::matcher::ValueMatcher::kPresentMatch: @@ -56,7 +57,16 @@ bool DoubleMatcher::match(const ProtobufWkt::Value& value) const { }; } -bool StringMatcher::match(const ProtobufWkt::Value& value) const { +StringMatcherImpl::StringMatcherImpl(const envoy::type::matcher::StringMatcher& matcher) + : matcher_(matcher) { + if (matcher.match_pattern_case() == envoy::type::matcher::StringMatcher::kRegex) { + regex_ = Regex::Utility::parseStdRegexAsCompiledMatcher(matcher_.regex()); + } else if (matcher.match_pattern_case() == envoy::type::matcher::StringMatcher::kSafeRegex) { + regex_ = Regex::Utility::parseRegex(matcher_.safe_regex()); + } +} + +bool StringMatcherImpl::match(const ProtobufWkt::Value& value) const { if (value.kind_case() != ProtobufWkt::Value::kStringValue) { return false; } @@ -64,7 +74,7 @@ bool StringMatcher::match(const ProtobufWkt::Value& value) const { return match(value.string_value()); } -bool StringMatcher::match(const absl::string_view value) const { +bool StringMatcherImpl::match(const absl::string_view value) const { switch (matcher_.match_pattern_case()) { case envoy::type::matcher::StringMatcher::kExact: return matcher_.exact() == value; @@ -73,7 +83,8 @@ bool StringMatcher::match(const absl::string_view value) const { case envoy::type::matcher::StringMatcher::kSuffix: return absl::EndsWith(value, matcher_.suffix()); case envoy::type::matcher::StringMatcher::kRegex: - return std::regex_match(value.begin(), value.end(), regex_); + case envoy::type::matcher::StringMatcher::kSafeRegex: + return regex_->match(value); default: NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/source/common/common/matchers.h b/source/common/common/matchers.h index 8305a3ee4b3f..bb89941f3d0e 100644 --- a/source/common/common/matchers.h +++ b/source/common/common/matchers.h @@ -3,6 +3,8 @@ #include #include "envoy/api/v2/core/base.pb.h" +#include "envoy/common/matchers.h" +#include "envoy/common/regex.h" #include "envoy/type/matcher/metadata.pb.h" #include "envoy/type/matcher/number.pb.h" #include "envoy/type/matcher/string.pb.h" @@ -70,21 +72,16 @@ class DoubleMatcher : public ValueMatcher { const envoy::type::matcher::DoubleMatcher matcher_; }; -class StringMatcher : public ValueMatcher { +class StringMatcherImpl : public ValueMatcher, public StringMatcher { public: - StringMatcher(const envoy::type::matcher::StringMatcher& matcher) : matcher_(matcher) { - if (matcher.match_pattern_case() == envoy::type::matcher::StringMatcher::kRegex) { - regex_ = RegexUtil::parseRegex(matcher_.regex()); - } - } - - bool match(const absl::string_view value) const; + explicit StringMatcherImpl(const envoy::type::matcher::StringMatcher& matcher); + bool match(const absl::string_view value) const override; bool match(const ProtobufWkt::Value& value) const override; private: const envoy::type::matcher::StringMatcher matcher_; - std::regex regex_; + Regex::CompiledMatcherPtr regex_; }; class LowerCaseStringMatcher : public ValueMatcher { @@ -100,9 +97,11 @@ class LowerCaseStringMatcher : public ValueMatcher { envoy::type::matcher::StringMatcher toLowerCase(const envoy::type::matcher::StringMatcher& matcher); - const StringMatcher matcher_; + const StringMatcherImpl matcher_; }; +using LowerCaseStringMatcherPtr = std::unique_ptr; + class ListMatcher : public ValueMatcher { public: ListMatcher(const envoy::type::matcher::ListMatcher& matcher); diff --git a/source/common/common/regex.cc b/source/common/common/regex.cc new file mode 100644 index 000000000000..b78beef7ce3a --- /dev/null +++ b/source/common/common/regex.cc @@ -0,0 +1,78 @@ +#include "common/common/regex.h" + +#include "envoy/common/exception.h" + +#include "common/common/assert.h" +#include "common/common/fmt.h" +#include "common/protobuf/utility.h" + +#include "re2/re2.h" + +namespace Envoy { +namespace Regex { +namespace { + +class CompiledStdMatcher : public CompiledMatcher { +public: + CompiledStdMatcher(std::regex&& regex) : regex_(std::move(regex)) {} + + // CompiledMatcher + bool match(absl::string_view value) const override { + return std::regex_match(value.begin(), value.end(), regex_); + } + +private: + const std::regex regex_; +}; + +class CompiledGoogleReMatcher : public CompiledMatcher { +public: + CompiledGoogleReMatcher(const envoy::type::matcher::RegexMatcher& config) + : regex_(config.regex(), re2::RE2::Quiet) { + if (!regex_.ok()) { + throw EnvoyException(regex_.error()); + } + + const uint32_t max_program_size = + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config.google_re2(), max_program_size, 100); + if (static_cast(regex_.ProgramSize()) > max_program_size) { + throw EnvoyException(fmt::format("regex '{}' RE2 program size of {} > max program size of " + "{}. Increase configured max program size if necessary.", + config.regex(), regex_.ProgramSize(), max_program_size)); + } + } + + // CompiledMatcher + bool match(absl::string_view value) const override { + return re2::RE2::FullMatch(re2::StringPiece(value.data(), value.size()), regex_); + } + +private: + const re2::RE2 regex_; +}; + +} // namespace + +CompiledMatcherPtr Utility::parseRegex(const envoy::type::matcher::RegexMatcher& matcher) { + // Google Re is the only currently supported engine. + ASSERT(matcher.has_google_re2()); + return std::make_unique(matcher); +} + +CompiledMatcherPtr Utility::parseStdRegexAsCompiledMatcher(const std::string& regex, + std::regex::flag_type flags) { + return std::make_unique(parseStdRegex(regex, flags)); +} + +std::regex Utility::parseStdRegex(const std::string& regex, std::regex::flag_type flags) { + // TODO(zuercher): In the future, PGV (https://github.com/lyft/protoc-gen-validate) annotations + // may allow us to remove this in favor of direct validation of regular expressions. + try { + return std::regex(regex, flags); + } catch (const std::regex_error& e) { + throw EnvoyException(fmt::format("Invalid regex '{}': {}", regex, e.what())); + } +} + +} // namespace Regex +} // namespace Envoy diff --git a/source/common/common/regex.h b/source/common/common/regex.h new file mode 100644 index 000000000000..a88c4739addb --- /dev/null +++ b/source/common/common/regex.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +#include "envoy/common/regex.h" +#include "envoy/type/matcher/regex.pb.h" + +namespace Envoy { +namespace Regex { + +/** + * Utilities for constructing regular expressions. + */ +class Utility { +public: + /** + * Constructs a std::regex, converting any std::regex_error exception into an EnvoyException. + * @param regex std::string containing the regular expression to parse. + * @param flags std::regex::flag_type containing parser flags. Defaults to std::regex::optimize. + * @return std::regex constructed from regex and flags. + * @throw EnvoyException if the regex string is invalid. + */ + static std::regex parseStdRegex(const std::string& regex, + std::regex::flag_type flags = std::regex::optimize); + + /** + * Construct an std::regex compiled regex matcher. + * + * TODO(mattklein123): In general this is only currently used in deprecated code paths and can be + * removed once all of those code paths are removed. + */ + static CompiledMatcherPtr + parseStdRegexAsCompiledMatcher(const std::string& regex, + std::regex::flag_type flags = std::regex::optimize); + + /** + * Construct a compiled regex matcher from a match config. + */ + static CompiledMatcherPtr parseRegex(const envoy::type::matcher::RegexMatcher& matcher); +}; + +} // namespace Regex +} // namespace Envoy diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index ed6483795814..3809da40e6d1 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "envoy/common/exception.h" @@ -499,16 +500,6 @@ uint32_t Primes::findPrimeLargerThan(uint32_t x) { return x; } -std::regex RegexUtil::parseRegex(const std::string& regex, std::regex::flag_type flags) { - // TODO(zuercher): In the future, PGV (https://github.com/lyft/protoc-gen-validate) annotations - // may allow us to remove this in favor of direct validation of regular expressions. - try { - return std::regex(regex, flags); - } catch (const std::regex_error& e) { - throw EnvoyException(fmt::format("Invalid regex '{}': {}", regex, e.what())); - } -} - // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm void WelfordStandardDeviation::update(double newValue) { ++count_; diff --git a/source/common/common/utility.h b/source/common/common/utility.h index 6ff000af46dc..48c09be97548 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -388,22 +387,6 @@ class Primes { static uint32_t findPrimeLargerThan(uint32_t x); }; -/** - * Utilities for constructing regular expressions. - */ -class RegexUtil { -public: - /* - * Constructs a std::regex, converting any std::regex_error exception into an EnvoyException. - * @param regex std::string containing the regular expression to parse. - * @param flags std::regex::flag_type containing parser flags. Defaults to std::regex::optimize. - * @return std::regex constructed from regex and flags. - * @throw EnvoyException if the regex string is invalid. - */ - static std::regex parseRegex(const std::string& regex, - std::regex::flag_type flags = std::regex::optimize); -}; - /** * Utilities for working with weighted clusters. */ diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 1dcf5045fa26..a41fdf2bb925 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -177,6 +177,7 @@ envoy_cc_library( "//source/common/common:empty_string", "//source/common/common:enum_to_int", "//source/common/common:linked_object", + "//source/common/common:regex_lib", "//source/common/common:scope_tracker", "//source/common/common:utility_lib", "//source/common/http/http1:codec_lib", diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc index b6af8ab75f00..2c2f79ca7265 100644 --- a/source/common/http/header_utility.cc +++ b/source/common/http/header_utility.cc @@ -1,5 +1,6 @@ #include "common/http/header_utility.h" +#include "common/common/regex.h" #include "common/common/utility.h" #include "common/config/rds_json.h" #include "common/http/header_map_impl.h" @@ -32,7 +33,11 @@ HeaderUtility::HeaderData::HeaderData(const envoy::api::v2::route::HeaderMatcher break; case envoy::api::v2::route::HeaderMatcher::kRegexMatch: header_match_type_ = HeaderMatchType::Regex; - regex_pattern_ = RegexUtil::parseRegex(config.regex_match()); + regex_ = Regex::Utility::parseStdRegexAsCompiledMatcher(config.regex_match()); + break; + case envoy::api::v2::route::HeaderMatcher::kSafeRegexMatch: + header_match_type_ = HeaderMatchType::Regex; + regex_ = Regex::Utility::parseRegex(config.safe_regex_match()); break; case envoy::api::v2::route::HeaderMatcher::kRangeMatch: header_match_type_ = HeaderMatchType::Range; @@ -82,11 +87,11 @@ void HeaderUtility::getAllOfHeader(const Http::HeaderMap& headers, absl::string_ } bool HeaderUtility::matchHeaders(const Http::HeaderMap& request_headers, - const std::vector& config_headers) { + const std::vector& config_headers) { // No headers to match is considered a match. if (!config_headers.empty()) { - for (const HeaderData& cfg_header_data : config_headers) { - if (!matchHeaders(request_headers, cfg_header_data)) { + for (const HeaderDataPtr& cfg_header_data : config_headers) { + if (!matchHeaders(request_headers, *cfg_header_data)) { return false; } } @@ -110,7 +115,7 @@ bool HeaderUtility::matchHeaders(const Http::HeaderMap& request_headers, match = header_data.value_.empty() || header_view == header_data.value_; break; case HeaderMatchType::Regex: - match = std::regex_match(header_view.begin(), header_view.end(), header_data.regex_pattern_); + match = header_data.regex_->match(header_view); break; case HeaderMatchType::Range: { int64_t header_value = 0; diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h index 7d3e13853059..a4f8c75639bd 100644 --- a/source/common/http/header_utility.h +++ b/source/common/http/header_utility.h @@ -1,13 +1,15 @@ #pragma once -#include #include #include "envoy/api/v2/route/route.pb.h" +#include "envoy/common/regex.h" #include "envoy/http/header_map.h" #include "envoy/json/json_object.h" #include "envoy/type/range.pb.h" +#include "common/protobuf/protobuf.h" + namespace Envoy { namespace Http { @@ -18,7 +20,8 @@ class HeaderUtility { public: enum class HeaderMatchType { Value, Regex, Range, Present, Prefix, Suffix }; - /* Get all instances of the header key specified, and return the values in the vector provided. + /** + * Get all instances of the header key specified, and return the values in the vector provided. * * This should not be used for inline headers, as it turns a constant time lookup into O(n). * @@ -26,7 +29,7 @@ class HeaderUtility { * @param key the header key to return values for * @param out the vector to return values in */ - static void getAllOfHeader(const Http::HeaderMap& headers, absl::string_view key, + static void getAllOfHeader(const HeaderMap& headers, absl::string_view key, std::vector& out); // A HeaderData specifies one of exact value or regex or range element @@ -36,14 +39,28 @@ class HeaderUtility { HeaderData(const envoy::api::v2::route::HeaderMatcher& config); HeaderData(const Json::Object& config); - const Http::LowerCaseString name_; + const LowerCaseString name_; HeaderMatchType header_match_type_; std::string value_; - std::regex regex_pattern_; + Regex::CompiledMatcherPtr regex_; envoy::type::Int64Range range_; const bool invert_match_; }; + using HeaderDataPtr = std::unique_ptr; + + /** + * Build a vector of HeaderData given input config. + */ + static std::vector buildHeaderDataVector( + const Protobuf::RepeatedPtrField& header_matchers) { + std::vector ret; + for (const auto& header_match : header_matchers) { + ret.emplace_back(std::make_unique(header_match)); + } + return ret; + } + /** * See if the headers specified in the config are present in a request. * @param request_headers supplies the headers from the request. @@ -51,10 +68,10 @@ class HeaderUtility { * @return bool true if all the headers (and values) in the config_headers are found in the * request_headers. If no config_headers are specified, returns true. */ - static bool matchHeaders(const Http::HeaderMap& request_headers, - const std::vector& config_headers); + static bool matchHeaders(const HeaderMap& request_headers, + const std::vector& config_headers); - static bool matchHeaders(const Http::HeaderMap& request_headers, const HeaderData& config_header); + static bool matchHeaders(const HeaderMap& request_headers, const HeaderData& config_header); /** * Validates that a header value is valid, according to RFC 7230, section 3.2. @@ -68,7 +85,7 @@ class HeaderUtility { * @param headers target where headers will be added * @param headers_to_add supplies the headers to be added */ - static void addHeaders(Http::HeaderMap& headers, const Http::HeaderMap& headers_to_add); + static void addHeaders(HeaderMap& headers, const HeaderMap& headers_to_add); }; } // namespace Http } // namespace Envoy diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 408689e95d80..7b9898ab7ecc 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -20,6 +19,7 @@ #include "common/common/fmt.h" #include "common/common/hash.h" #include "common/common/logger.h" +#include "common/common/regex.h" #include "common/common/utility.h" #include "common/config/metadata.h" #include "common/config/rds_json.h" @@ -153,10 +153,17 @@ CorsPolicyImpl::CorsPolicyImpl(const envoy::api::v2::route::CorsPolicy& config, max_age_(config.max_age()), legacy_enabled_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enabled, true)) { for (const auto& origin : config.allow_origin()) { - allow_origin_.push_back(origin); + envoy::type::matcher::StringMatcher matcher_config; + matcher_config.set_exact(origin); + allow_origins_.push_back(std::make_unique(matcher_config)); } for (const auto& regex : config.allow_origin_regex()) { - allow_origin_regex_.push_back(RegexUtil::parseRegex(regex)); + envoy::type::matcher::StringMatcher matcher_config; + matcher_config.set_regex(regex); + allow_origins_.push_back(std::make_unique(matcher_config)); + } + for (const auto& string_match : config.allow_origin_string_match()) { + allow_origins_.push_back(std::make_unique(string_match)); } if (config.has_allow_credentials()) { allow_credentials_ = PROTOBUF_GET_WRAPPED_REQUIRED(config, allow_credentials); @@ -392,6 +399,7 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, factory_context.messageValidationVisitor())), rate_limit_policy_(route.route().rate_limits()), shadow_policy_(route.route()), priority_(ConfigUtility::parsePriority(route.route().priority())), + config_headers_(Http::HeaderUtility::buildHeaderDataVector(route.match().headers())), total_cluster_weight_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(route.route().weighted_clusters(), total_weight, 100UL)), request_headers_parser_(HeaderParser::configure(route.request_headers_to_add(), @@ -440,12 +448,9 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, } } - for (const auto& header_map : route.match().headers()) { - config_headers_.push_back(header_map); - } - for (const auto& query_parameter : route.match().query_parameters()) { - config_query_parameters_.push_back(query_parameter); + config_query_parameters_.push_back( + std::make_unique(query_parameter)); } if (!route.route().hash_policy().empty()) { @@ -557,7 +562,7 @@ RouteEntryImplBase::loadRuntimeData(const envoy::api::v2::route::RouteMatch& rou } void RouteEntryImplBase::finalizePathHeader(Http::HeaderMap& headers, - const std::string& matched_path, + absl::string_view matched_path, bool insert_envoy_original_path) const { const auto& rewrite = getPathRewrite(); if (rewrite.empty()) { @@ -898,32 +903,36 @@ RouteConstSharedPtr PathRouteEntryImpl::matches(const Http::HeaderMap& headers, RegexRouteEntryImpl::RegexRouteEntryImpl(const VirtualHostImpl& vhost, const envoy::api::v2::route::Route& route, Server::Configuration::FactoryContext& factory_context) - : RouteEntryImplBase(vhost, route, factory_context), - regex_(RegexUtil::parseRegex(route.match().regex())), regex_str_(route.match().regex()) {} + : RouteEntryImplBase(vhost, route, factory_context) { + if (route.match().path_specifier_case() == envoy::api::v2::route::RouteMatch::kRegex) { + regex_ = Regex::Utility::parseStdRegexAsCompiledMatcher(route.match().regex()); + regex_str_ = route.match().regex(); + } else { + ASSERT(route.match().path_specifier_case() == envoy::api::v2::route::RouteMatch::kSafeRegex); + regex_ = Regex::Utility::parseRegex(route.match().safe_regex()); + regex_str_ = route.match().safe_regex().regex(); + } +} -void RegexRouteEntryImpl::rewritePathHeader(Http::HeaderMap& headers, - bool insert_envoy_original_path) const { +absl::string_view RegexRouteEntryImpl::pathOnly(const Http::HeaderMap& headers) const { const Http::HeaderString& path = headers.Path()->value(); const absl::string_view query_string = Http::Utility::findQueryStringStart(path); const size_t path_string_length = path.size() - query_string.length(); + return path.getStringView().substr(0, path_string_length); +} + +void RegexRouteEntryImpl::rewritePathHeader(Http::HeaderMap& headers, + bool insert_envoy_original_path) const { // TODO(yuval-k): This ASSERT can happen if the path was changed by a filter without clearing the // route cache. We should consider if ASSERT-ing is the desired behavior in this case. - - const absl::string_view path_view = path.getStringView(); - ASSERT(std::regex_match(path_view.begin(), path_view.begin() + path_string_length, regex_)); - const std::string matched_path(path_view.begin(), path_view.begin() + path_string_length); - - finalizePathHeader(headers, matched_path, insert_envoy_original_path); + ASSERT(regex_->match(pathOnly(headers))); + finalizePathHeader(headers, pathOnly(headers), insert_envoy_original_path); } RouteConstSharedPtr RegexRouteEntryImpl::matches(const Http::HeaderMap& headers, uint64_t random_value) const { if (RouteEntryImplBase::matchRoute(headers, random_value)) { - const Http::HeaderString& path = headers.Path()->value(); - const absl::string_view query_string = Http::Utility::findQueryStringStart(path); - if (std::regex_match(path.getStringView().begin(), - path.getStringView().begin() + (path.size() - query_string.length()), - regex_)) { + if (regex_->match(pathOnly(headers))) { return clusterEntry(headers, random_value); } } @@ -969,19 +978,22 @@ VirtualHostImpl::VirtualHostImpl(const envoy::api::v2::route::VirtualHost& virtu } for (const auto& route : virtual_host.routes()) { - const bool has_prefix = - route.match().path_specifier_case() == envoy::api::v2::route::RouteMatch::kPrefix; - const bool has_path = - route.match().path_specifier_case() == envoy::api::v2::route::RouteMatch::kPath; - const bool has_regex = - route.match().path_specifier_case() == envoy::api::v2::route::RouteMatch::kRegex; - if (has_prefix) { + switch (route.match().path_specifier_case()) { + case envoy::api::v2::route::RouteMatch::kPrefix: { routes_.emplace_back(new PrefixRouteEntryImpl(*this, route, factory_context)); - } else if (has_path) { + break; + } + case envoy::api::v2::route::RouteMatch::kPath: { routes_.emplace_back(new PathRouteEntryImpl(*this, route, factory_context)); - } else { - ASSERT(has_regex); + break; + } + case envoy::api::v2::route::RouteMatch::kRegex: + case envoy::api::v2::route::RouteMatch::kSafeRegex: { routes_.emplace_back(new RegexRouteEntryImpl(*this, route, factory_context)); + break; + } + case envoy::api::v2::route::RouteMatch::PATH_SPECIFIER_NOT_SET: + NOT_REACHED_GCOVR_EXCL_LINE; } if (validate_clusters) { @@ -1006,10 +1018,27 @@ VirtualHostImpl::VirtualHostImpl(const envoy::api::v2::route::VirtualHost& virtu VirtualHostImpl::VirtualClusterEntry::VirtualClusterEntry( const envoy::api::v2::route::VirtualCluster& virtual_cluster, Stats::StatNamePool& pool) - : pattern_(RegexUtil::parseRegex(virtual_cluster.pattern())), - stat_name_(pool.add(virtual_cluster.name())) { + : stat_name_(pool.add(virtual_cluster.name())) { + if (virtual_cluster.pattern().empty() == virtual_cluster.headers().empty()) { + throw EnvoyException("virtual clusters must define either 'pattern' or 'headers'"); + } + + if (!virtual_cluster.pattern().empty()) { + envoy::api::v2::route::HeaderMatcher matcher_config; + matcher_config.set_name(Http::Headers::get().Path.get()); + matcher_config.set_regex_match(virtual_cluster.pattern()); + headers_.push_back(std::make_unique(matcher_config)); + } else { + ASSERT(!virtual_cluster.headers().empty()); + headers_ = Http::HeaderUtility::buildHeaderDataVector(virtual_cluster.headers()); + } + if (virtual_cluster.method() != envoy::api::v2::core::RequestMethod::METHOD_UNSPECIFIED) { - method_ = envoy::api::v2::core::RequestMethod_Name(virtual_cluster.method()); + envoy::api::v2::route::HeaderMatcher matcher_config; + matcher_config.set_name(Http::Headers::get().Method.get()); + matcher_config.set_exact_match( + envoy::api::v2::core::RequestMethod_Name(virtual_cluster.method())); + headers_.push_back(std::make_unique(matcher_config)); } } @@ -1154,11 +1183,7 @@ const std::shared_ptr VirtualHostImpl::SSL_REDIRECT_ROUT const VirtualCluster* VirtualHostImpl::virtualClusterFromEntries(const Http::HeaderMap& headers) const { for (const VirtualClusterEntry& entry : virtual_clusters_) { - bool method_matches = - !entry.method_ || headers.Method()->value().getStringView() == entry.method_.value(); - - absl::string_view path_view = headers.Path()->value().getStringView(); - if (method_matches && std::regex_match(path_view.begin(), path_view.end(), entry.pattern_)) { + if (Http::HeaderUtility::matchHeaders(headers, entry.headers_)) { return &entry; } } diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 202ffe6bea6d..39a51ba67899 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -104,8 +104,9 @@ class CorsPolicyImpl : public CorsPolicy { CorsPolicyImpl(const envoy::api::v2::route::CorsPolicy& config, Runtime::Loader& loader); // Router::CorsPolicy - const std::list& allowOrigins() const override { return allow_origin_; }; - const std::list& allowOriginRegexes() const override { return allow_origin_regex_; } + const std::vector& allowOrigins() const override { + return allow_origins_; + }; const std::string& allowMethods() const override { return allow_methods_; }; const std::string& allowHeaders() const override { return allow_headers_; }; const std::string& exposeHeaders() const override { return expose_headers_; }; @@ -131,8 +132,7 @@ class CorsPolicyImpl : public CorsPolicy { private: const envoy::api::v2::route::CorsPolicy config_; Runtime::Loader& loader_; - std::list allow_origin_; - std::list allow_origin_regex_; + std::vector allow_origins_; const std::string allow_methods_; const std::string allow_headers_; const std::string expose_headers_; @@ -182,9 +182,8 @@ class VirtualHostImpl : public VirtualHost { // Router::VirtualCluster Stats::StatName statName() const override { return stat_name_; } - const std::regex pattern_; - absl::optional method_; const Stats::StatName stat_name_; + std::vector headers_; }; class CatchAllVirtualCluster : public VirtualCluster { @@ -480,7 +479,7 @@ class RouteEntryImplBase : public RouteEntry, return (isRedirect()) ? prefix_rewrite_redirect_ : prefix_rewrite_; } - void finalizePathHeader(Http::HeaderMap& headers, const std::string& matched_path, + void finalizePathHeader(Http::HeaderMap& headers, absl::string_view matched_path, bool insert_envoy_original_path) const; private: @@ -670,8 +669,8 @@ class RouteEntryImplBase : public RouteEntry, const RateLimitPolicyImpl rate_limit_policy_; const ShadowPolicyImpl shadow_policy_; const Upstream::ResourcePriority priority_; - std::vector config_headers_; - std::vector config_query_parameters_; + std::vector config_headers_; + std::vector config_query_parameters_; std::vector weighted_clusters_; UpgradeMap upgrade_map_; @@ -760,8 +759,10 @@ class RegexRouteEntryImpl : public RouteEntryImplBase { void rewritePathHeader(Http::HeaderMap& headers, bool insert_envoy_original_path) const override; private: - const std::regex regex_; - const std::string regex_str_; + absl::string_view pathOnly(const Http::HeaderMap& headers) const; + + Regex::CompiledMatcherPtr regex_; + std::string regex_str_; }; /** diff --git a/source/common/router/config_utility.cc b/source/common/router/config_utility.cc index c48dd794d5f4..09dc85db59cc 100644 --- a/source/common/router/config_utility.cc +++ b/source/common/router/config_utility.cc @@ -1,25 +1,59 @@ #include "common/router/config_utility.h" -#include #include #include #include "common/common/assert.h" +#include "common/common/regex.h" namespace Envoy { namespace Router { +namespace { + +absl::optional +maybeCreateStringMatcher(const envoy::api::v2::route::QueryParameterMatcher& config) { + switch (config.query_parameter_match_specifier_case()) { + case envoy::api::v2::route::QueryParameterMatcher::kStringMatch: { + return Matchers::StringMatcherImpl(config.string_match()); + } + case envoy::api::v2::route::QueryParameterMatcher::kPresentMatch: { + return absl::nullopt; + } + case envoy::api::v2::route::QueryParameterMatcher::QUERY_PARAMETER_MATCH_SPECIFIER_NOT_SET: { + if (config.value().empty()) { + // Present match. + return absl::nullopt; + } + + envoy::type::matcher::StringMatcher matcher_config; + if (PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, regex, false)) { + matcher_config.set_regex(config.value()); + } else { + matcher_config.set_exact(config.value()); + } + return Matchers::StringMatcherImpl(matcher_config); + } + } + + NOT_REACHED_GCOVR_EXCL_LINE; // Needed for gcc +} + +} // namespace + +ConfigUtility::QueryParameterMatcher::QueryParameterMatcher( + const envoy::api::v2::route::QueryParameterMatcher& config) + : name_(config.name()), matcher_(maybeCreateStringMatcher(config)) {} bool ConfigUtility::QueryParameterMatcher::matches( const Http::Utility::QueryParams& request_query_params) const { auto query_param = request_query_params.find(name_); if (query_param == request_query_params.end()) { return false; - } else if (is_regex_) { - return std::regex_match(query_param->second, regex_pattern_); - } else if (value_.length() == 0) { + } else if (!matcher_.has_value()) { + // Present match. return true; } else { - return (value_ == query_param->second); + return matcher_.value().match(query_param->second); } } @@ -37,9 +71,9 @@ ConfigUtility::parsePriority(const envoy::api::v2::core::RoutingPriority& priori bool ConfigUtility::matchQueryParams( const Http::Utility::QueryParams& query_params, - const std::vector& config_query_params) { + const std::vector& config_query_params) { for (const auto& config_query_param : config_query_params) { - if (!config_query_param.matches(query_params)) { + if (!config_query_param->matches(query_params)) { return false; } } diff --git a/source/common/router/config_utility.h b/source/common/router/config_utility.h index 4a753964059f..030e20ca4cd1 100644 --- a/source/common/router/config_utility.h +++ b/source/common/router/config_utility.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -11,6 +10,7 @@ #include "envoy/upstream/resource_manager.h" #include "common/common/empty_string.h" +#include "common/common/matchers.h" #include "common/common/utility.h" #include "common/config/rds_json.h" #include "common/http/headers.h" @@ -32,10 +32,7 @@ class ConfigUtility { // equivalent of the QueryParameterMatcher proto in the RDS v2 API. class QueryParameterMatcher { public: - QueryParameterMatcher(const envoy::api::v2::route::QueryParameterMatcher& config) - : name_(config.name()), value_(config.value()), - is_regex_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, regex, false)), - regex_pattern_(is_regex_ ? RegexUtil::parseRegex(value_) : std::regex()) {} + QueryParameterMatcher(const envoy::api::v2::route::QueryParameterMatcher& config); /** * Check if the query parameters for a request contain a match for this @@ -47,11 +44,11 @@ class ConfigUtility { private: const std::string name_; - const std::string value_; - const bool is_regex_; - const std::regex regex_pattern_; + const absl::optional matcher_; }; + using QueryParameterMatcherPtr = std::unique_ptr; + /** * @return the resource priority parsed from proto. */ @@ -66,7 +63,7 @@ class ConfigUtility { * query_params */ static bool matchQueryParams(const Http::Utility::QueryParams& query_params, - const std::vector& config_query_params); + const std::vector& config_query_params); /** * Returns the redirect HTTP Status Code enum parsed from proto. diff --git a/source/common/router/router_ratelimit.cc b/source/common/router/router_ratelimit.cc index f8827836367f..28519851ea83 100644 --- a/source/common/router/router_ratelimit.cc +++ b/source/common/router/router_ratelimit.cc @@ -67,11 +67,8 @@ bool GenericKeyAction::populateDescriptor(const Router::RouteEntry&, HeaderValueMatchAction::HeaderValueMatchAction( const envoy::api::v2::route::RateLimit::Action::HeaderValueMatch& action) : descriptor_value_(action.descriptor_value()), - expect_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(action, expect_match, true)) { - for (const auto& header_matcher : action.headers()) { - action_headers_.push_back(header_matcher); - } -} + expect_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(action, expect_match, true)), + action_headers_(Http::HeaderUtility::buildHeaderDataVector(action.headers())) {} bool HeaderValueMatchAction::populateDescriptor(const Router::RouteEntry&, RateLimit::Descriptor& descriptor, diff --git a/source/common/router/router_ratelimit.h b/source/common/router/router_ratelimit.h index e15f493ddbfa..7f3a368ffbca 100644 --- a/source/common/router/router_ratelimit.h +++ b/source/common/router/router_ratelimit.h @@ -97,7 +97,7 @@ class HeaderValueMatchAction : public RateLimitAction { private: const std::string descriptor_value_; const bool expect_match_; - std::vector action_headers_; + const std::vector action_headers_; }; /* diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index 7bc419766cdc..ed5efa052b47 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -180,6 +180,7 @@ envoy_cc_library( deps = [ "//include/envoy/stats:stats_interface", "//source/common/common:perf_annotation_lib", + "//source/common/common:regex_lib", ], ) diff --git a/source/common/stats/stats_matcher_impl.cc b/source/common/stats/stats_matcher_impl.cc index e4f14872ae4d..bd73c46adf5c 100644 --- a/source/common/stats/stats_matcher_impl.cc +++ b/source/common/stats/stats_matcher_impl.cc @@ -19,14 +19,14 @@ StatsMatcherImpl::StatsMatcherImpl(const envoy::config::metrics::v2::StatsConfig case envoy::config::metrics::v2::StatsMatcher::kInclusionList: // If we have an inclusion list, we are being default-exclusive. for (const auto& stats_matcher : config.stats_matcher().inclusion_list().patterns()) { - matchers_.push_back(Matchers::StringMatcher(stats_matcher)); + matchers_.push_back(Matchers::StringMatcherImpl(stats_matcher)); } is_inclusive_ = false; break; case envoy::config::metrics::v2::StatsMatcher::kExclusionList: // If we have an exclusion list, we are being default-inclusive. for (const auto& stats_matcher : config.stats_matcher().exclusion_list().patterns()) { - matchers_.push_back(Matchers::StringMatcher(stats_matcher)); + matchers_.push_back(Matchers::StringMatcherImpl(stats_matcher)); } FALLTHRU; default: @@ -48,7 +48,7 @@ bool StatsMatcherImpl::rejects(const std::string& name) const { // This is an XNOR, which can be evaluated by checking for equality. return (is_inclusive_ == std::any_of(matchers_.begin(), matchers_.end(), - [&name](auto matcher) { return matcher.match(name); })); + [&name](auto& matcher) { return matcher.match(name); })); } } // namespace Stats diff --git a/source/common/stats/stats_matcher_impl.h b/source/common/stats/stats_matcher_impl.h index 3712a466d3f6..7fe65e7fc49e 100644 --- a/source/common/stats/stats_matcher_impl.h +++ b/source/common/stats/stats_matcher_impl.h @@ -33,7 +33,7 @@ class StatsMatcherImpl : public StatsMatcher { // StatsMatcherImpl::rejects() for much more detail. bool is_inclusive_{true}; - std::vector matchers_; + std::vector matchers_; }; } // namespace Stats diff --git a/source/common/stats/tag_extractor_impl.cc b/source/common/stats/tag_extractor_impl.cc index 7bb02e6f53e1..a56398895c25 100644 --- a/source/common/stats/tag_extractor_impl.cc +++ b/source/common/stats/tag_extractor_impl.cc @@ -5,8 +5,9 @@ #include "envoy/common/exception.h" +#include "common/common/fmt.h" #include "common/common/perf_annotation.h" -#include "common/common/utility.h" +#include "common/common/regex.h" #include "absl/strings/ascii.h" #include "absl/strings/match.h" @@ -25,7 +26,7 @@ bool regexStartsWithDot(absl::string_view regex) { TagExtractorImpl::TagExtractorImpl(const std::string& name, const std::string& regex, const std::string& substr) : name_(name), prefix_(std::string(extractRegexPrefix(regex))), substr_(substr), - regex_(RegexUtil::parseRegex(regex)) {} + regex_(Regex::Utility::parseStdRegex(regex)) {} std::string TagExtractorImpl::extractRegexPrefix(absl::string_view regex) { std::string prefix; diff --git a/source/extensions/common/tap/tap_matcher.cc b/source/extensions/common/tap/tap_matcher.cc index 1b0d53bfa614..66511e2f8856 100644 --- a/source/extensions/common/tap/tap_matcher.cc +++ b/source/extensions/common/tap/tap_matcher.cc @@ -111,11 +111,8 @@ void NotMatcher::updateLocalStatus(MatchStatusVector& statuses, HttpHeaderMatcherBase::HttpHeaderMatcherBase( const envoy::service::tap::v2alpha::HttpHeadersMatch& config, const std::vector& matchers) - : SimpleMatcher(matchers) { - for (const auto& header_match : config.headers()) { - headers_to_match_.emplace_back(header_match); - } -} + : SimpleMatcher(matchers), + headers_to_match_(Http::HeaderUtility::buildHeaderDataVector(config.headers())) {} void HttpHeaderMatcherBase::matchHeaders(const Http::HeaderMap& headers, MatchStatusVector& statuses) const { diff --git a/source/extensions/common/tap/tap_matcher.h b/source/extensions/common/tap/tap_matcher.h index b80ff24256db..5f8e89c38bea 100644 --- a/source/extensions/common/tap/tap_matcher.h +++ b/source/extensions/common/tap/tap_matcher.h @@ -239,7 +239,7 @@ class HttpHeaderMatcherBase : public SimpleMatcher { protected: void matchHeaders(const Http::HeaderMap& headers, MatchStatusVector& statuses) const; - std::vector headers_to_match_; + const std::vector headers_to_match_; }; /** diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc index aff72f7a533f..b40a927fdbfb 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc @@ -50,18 +50,28 @@ struct SuccessResponse { const MatcherSharedPtr& matchers_; ResponsePtr response_; }; + +std::vector +createLowerCaseMatchers(const envoy::type::matcher::ListStringMatcher& list) { + std::vector matchers; + for (const auto& matcher : list.patterns()) { + matchers.push_back(std::make_unique(matcher)); + } + return matchers; +} + } // namespace // Matchers -HeaderKeyMatcher::HeaderKeyMatcher(std::vector&& list) +HeaderKeyMatcher::HeaderKeyMatcher(std::vector&& list) : matchers_(std::move(list)) {} bool HeaderKeyMatcher::matches(absl::string_view key) const { return std::any_of(matchers_.begin(), matchers_.end(), - [&key](auto matcher) { return matcher.match(key); }); + [&key](auto& matcher) { return matcher->match(key); }); } -NotHeaderKeyMatcher::NotHeaderKeyMatcher(std::vector&& list) +NotHeaderKeyMatcher::NotHeaderKeyMatcher(std::vector&& list) : matcher_(std::move(list)) {} bool NotHeaderKeyMatcher::matches(absl::string_view key) const { return !matcher_.matches(key); } @@ -86,12 +96,11 @@ ClientConfig::toRequestMatchers(const envoy::type::matcher::ListStringMatcher& l {Http::Headers::get().Authorization, Http::Headers::get().Method, Http::Headers::get().Path, Http::Headers::get().Host}}; - std::vector matchers{list.patterns().begin(), - list.patterns().end()}; + std::vector matchers(createLowerCaseMatchers(list)); for (const auto& key : keys) { envoy::type::matcher::StringMatcher matcher; matcher.set_exact(key.get()); - matchers.push_back(matcher); + matchers.push_back(std::make_unique(matcher)); } return std::make_shared(std::move(matchers)); @@ -99,15 +108,14 @@ ClientConfig::toRequestMatchers(const envoy::type::matcher::ListStringMatcher& l MatcherSharedPtr ClientConfig::toClientMatchers(const envoy::type::matcher::ListStringMatcher& list) { - std::vector matchers{list.patterns().begin(), - list.patterns().end()}; + std::vector matchers(createLowerCaseMatchers(list)); // If list is empty, all authorization response headers, except Host, should be added to // the client response. if (matchers.empty()) { envoy::type::matcher::StringMatcher matcher; matcher.set_exact(Http::Headers::get().Host.get()); - matchers.push_back(matcher); + matchers.push_back(std::make_unique(matcher)); return std::make_shared(std::move(matchers)); } @@ -121,7 +129,7 @@ ClientConfig::toClientMatchers(const envoy::type::matcher::ListStringMatcher& li for (const auto& key : keys) { envoy::type::matcher::StringMatcher matcher; matcher.set_exact(key.get()); - matchers.push_back(matcher); + matchers.push_back(std::make_unique(matcher)); } return std::make_shared(std::move(matchers)); @@ -129,9 +137,7 @@ ClientConfig::toClientMatchers(const envoy::type::matcher::ListStringMatcher& li MatcherSharedPtr ClientConfig::toUpstreamMatchers(const envoy::type::matcher::ListStringMatcher& list) { - std::vector matchers{list.patterns().begin(), - list.patterns().end()}; - return std::make_unique(std::move(matchers)); + return std::make_unique(createLowerCaseMatchers(list)); } Http::LowerCaseStrPairVector ClientConfig::toHeadersAdd( diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h index a26d5e4532f4..6d4c57128a84 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.h @@ -34,17 +34,17 @@ class Matcher { class HeaderKeyMatcher : public Matcher { public: - HeaderKeyMatcher(std::vector&& list); + HeaderKeyMatcher(std::vector&& list); bool matches(absl::string_view key) const override; private: - const std::vector matchers_; + const std::vector matchers_; }; class NotHeaderKeyMatcher : public Matcher { public: - NotHeaderKeyMatcher(std::vector&& list); + NotHeaderKeyMatcher(std::vector&& list); bool matches(absl::string_view key) const override; diff --git a/source/extensions/filters/common/rbac/matchers.h b/source/extensions/filters/common/rbac/matchers.h index 98f369d49e20..8b2ef2bf1076 100644 --- a/source/extensions/filters/common/rbac/matchers.h +++ b/source/extensions/filters/common/rbac/matchers.h @@ -166,14 +166,14 @@ class AuthenticatedMatcher : public Matcher { public: AuthenticatedMatcher(const envoy::config::rbac::v2::Principal_Authenticated& auth) : matcher_(auth.has_principal_name() - ? absl::make_optional(auth.principal_name()) + ? absl::make_optional(auth.principal_name()) : absl::nullopt) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, const StreamInfo::StreamInfo&) const override; private: - const absl::optional matcher_; + const absl::optional matcher_; }; /** @@ -217,10 +217,10 @@ class MetadataMatcher : public Matcher { * Perform a match against the request server from the client's connection * request. This is typically TLS SNI. */ -class RequestedServerNameMatcher : public Matcher, Envoy::Matchers::StringMatcher { +class RequestedServerNameMatcher : public Matcher, Envoy::Matchers::StringMatcherImpl { public: RequestedServerNameMatcher(const envoy::type::matcher::StringMatcher& requested_server_name) - : Envoy::Matchers::StringMatcher(requested_server_name) {} + : Envoy::Matchers::StringMatcherImpl(requested_server_name) {} bool matches(const Network::Connection& connection, const Envoy::Http::HeaderMap& headers, const StreamInfo::StreamInfo&) const override; diff --git a/source/extensions/filters/http/cors/cors_filter.cc b/source/extensions/filters/http/cors/cors_filter.cc index 997beb5385fc..2bd26c8b8438 100644 --- a/source/extensions/filters/http/cors/cors_filter.cc +++ b/source/extensions/filters/http/cors/cors_filter.cc @@ -115,35 +115,19 @@ void CorsFilter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& c } bool CorsFilter::isOriginAllowed(const Http::HeaderString& origin) { - return isOriginAllowedString(origin) || isOriginAllowedRegex(origin); -} - -bool CorsFilter::isOriginAllowedString(const Http::HeaderString& origin) { - if (allowOrigins() == nullptr) { - return false; - } - for (const auto& o : *allowOrigins()) { - if (o == "*" || origin == o.c_str()) { - return true; - } - } - return false; -} - -bool CorsFilter::isOriginAllowedRegex(const Http::HeaderString& origin) { - if (allowOriginRegexes() == nullptr) { + const auto allow_origins = allowOrigins(); + if (allow_origins == nullptr) { return false; } - for (const auto& regex : *allowOriginRegexes()) { - const absl::string_view origin_view = origin.getStringView(); - if (std::regex_match(origin_view.begin(), origin_view.end(), regex)) { + for (const auto& allow_origin : *allow_origins) { + if (allow_origin->match("*") || allow_origin->match(origin.getStringView())) { return true; } } return false; } -const std::list* CorsFilter::allowOrigins() { +const std::vector* CorsFilter::allowOrigins() { for (const auto policy : policies_) { if (policy && !policy->allowOrigins().empty()) { return &policy->allowOrigins(); @@ -152,15 +136,6 @@ const std::list* CorsFilter::allowOrigins() { return nullptr; } -const std::list* CorsFilter::allowOriginRegexes() { - for (const auto policy : policies_) { - if (policy && !policy->allowOriginRegexes().empty()) { - return &policy->allowOriginRegexes(); - } - } - return nullptr; -} - const std::string& CorsFilter::allowMethods() { for (const auto policy : policies_) { if (policy && !policy->allowMethods().empty()) { diff --git a/source/extensions/filters/http/cors/cors_filter.h b/source/extensions/filters/http/cors/cors_filter.h index b70b382a2177..51b13efeeb28 100644 --- a/source/extensions/filters/http/cors/cors_filter.h +++ b/source/extensions/filters/http/cors/cors_filter.h @@ -82,8 +82,7 @@ class CorsFilter : public Http::StreamFilter { private: friend class CorsFilterTest; - const std::list* allowOrigins(); - const std::list* allowOriginRegexes(); + const std::vector* allowOrigins(); const std::string& allowMethods(); const std::string& allowHeaders(); const std::string& exposeHeaders(); @@ -92,8 +91,6 @@ class CorsFilter : public Http::StreamFilter { bool shadowEnabled(); bool enabled(); bool isOriginAllowed(const Http::HeaderString& origin); - bool isOriginAllowedString(const Http::HeaderString& origin); - bool isOriginAllowedRegex(const Http::HeaderString& origin); Http::StreamDecoderFilterCallbacks* decoder_callbacks_{}; Http::StreamEncoderFilterCallbacks* encoder_callbacks_{}; diff --git a/source/extensions/filters/http/csrf/csrf_filter.cc b/source/extensions/filters/http/csrf/csrf_filter.cc index 85edec966ce3..ee8db9074c1a 100644 --- a/source/extensions/filters/http/csrf/csrf_filter.cc +++ b/source/extensions/filters/http/csrf/csrf_filter.cc @@ -59,10 +59,9 @@ static CsrfStats generateStats(const std::string& prefix, Stats::Scope& scope) { return CsrfStats{ALL_CSRF_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))}; } -static const CsrfPolicy -generatePolicy(const envoy::config::filter::http::csrf::v2::CsrfPolicy& policy, - Runtime::Loader& runtime) { - return CsrfPolicy(policy, runtime); +static CsrfPolicyPtr generatePolicy(const envoy::config::filter::http::csrf::v2::CsrfPolicy& policy, + Runtime::Loader& runtime) { + return std::make_unique(policy, runtime); } } // namespace @@ -128,7 +127,7 @@ bool CsrfFilter::isValid(const absl::string_view source_origin, Http::HeaderMap& } for (const auto& additional_origin : policy_->additional_origins()) { - if (additional_origin.match(source_origin)) { + if (additional_origin->match(source_origin)) { return true; } } diff --git a/source/extensions/filters/http/csrf/csrf_filter.h b/source/extensions/filters/http/csrf/csrf_filter.h index 57def213db6e..0a36058f73e7 100644 --- a/source/extensions/filters/http/csrf/csrf_filter.h +++ b/source/extensions/filters/http/csrf/csrf_filter.h @@ -17,12 +17,10 @@ namespace Csrf { /** * All CSRF filter stats. @see stats_macros.h */ -// clang-format off -#define ALL_CSRF_STATS(COUNTER) \ - COUNTER(missing_source_origin)\ - COUNTER(request_invalid) \ - COUNTER(request_valid) \ -// clang-format on +#define ALL_CSRF_STATS(COUNTER) \ + COUNTER(missing_source_origin) \ + COUNTER(request_invalid) \ + COUNTER(request_valid) /** * Struct definition for CSRF stats. @see stats_macros.h @@ -37,9 +35,11 @@ struct CsrfStats { class CsrfPolicy : public Router::RouteSpecificFilterConfig { public: CsrfPolicy(const envoy::config::filter::http::csrf::v2::CsrfPolicy& policy, - Runtime::Loader& runtime) : policy_(policy), runtime_(runtime) { + Runtime::Loader& runtime) + : policy_(policy), runtime_(runtime) { for (const auto& additional_origin : policy.additional_origins()) { - additional_origins_.emplace_back(Matchers::StringMatcher(additional_origin)); + additional_origins_.emplace_back( + std::make_unique(additional_origin)); } } @@ -58,14 +58,16 @@ class CsrfPolicy : public Router::RouteSpecificFilterConfig { shadow_enabled.default_value()); } - const std::vector& additional_origins() const { return additional_origins_; }; + const std::vector& additional_origins() const { + return additional_origins_; + }; private: const envoy::config::filter::http::csrf::v2::CsrfPolicy policy_; - std::vector additional_origins_; + std::vector additional_origins_; Runtime::Loader& runtime_; - }; +using CsrfPolicyPtr = std::unique_ptr; /** * Configuration for the CSRF filter. @@ -73,15 +75,14 @@ class CsrfPolicy : public Router::RouteSpecificFilterConfig { class CsrfFilterConfig { public: CsrfFilterConfig(const envoy::config::filter::http::csrf::v2::CsrfPolicy& policy, - const std::string& stats_prefix, Stats::Scope& scope, - Runtime::Loader& runtime); + const std::string& stats_prefix, Stats::Scope& scope, Runtime::Loader& runtime); CsrfStats& stats() { return stats_; } - const CsrfPolicy* policy() { return &policy_; } + const CsrfPolicy* policy() { return policy_.get(); } private: CsrfStats stats_; - const CsrfPolicy policy_; + const CsrfPolicyPtr policy_; }; using CsrfFilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 05dccd35c891..0649366a7e5c 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -33,7 +33,8 @@ struct RcDetailsValues { using RcDetails = ConstSingleton; FaultSettings::FaultSettings(const envoy::config::filter::http::fault::v2::HTTPFault& fault) - : delay_percent_runtime_(PROTOBUF_GET_STRING_OR_DEFAULT(fault, delay_percent_runtime, + : fault_filter_headers_(Http::HeaderUtility::buildHeaderDataVector(fault.headers())), + delay_percent_runtime_(PROTOBUF_GET_STRING_OR_DEFAULT(fault, delay_percent_runtime, RuntimeKeys::get().DelayPercentKey)), abort_percent_runtime_(PROTOBUF_GET_STRING_OR_DEFAULT(fault, abort_percent_runtime, RuntimeKeys::get().AbortPercentKey)), @@ -57,10 +58,6 @@ FaultSettings::FaultSettings(const envoy::config::filter::http::fault::v2::HTTPF std::make_unique(fault.delay()); } - for (const Http::HeaderUtility::HeaderData& header_map : fault.headers()) { - fault_filter_headers_.push_back(header_map); - } - upstream_cluster_ = fault.upstream_cluster(); for (const auto& node : fault.downstream_nodes()) { diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index c4ae48fdcc58..be6ef435420b 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -48,7 +48,7 @@ class FaultSettings : public Router::RouteSpecificFilterConfig { public: FaultSettings(const envoy::config::filter::http::fault::v2::HTTPFault& fault); - const std::vector& filterHeaders() const { + const std::vector& filterHeaders() const { return fault_filter_headers_; } envoy::type::FractionalPercent abortPercentage() const { return abort_percentage_; } @@ -88,7 +88,7 @@ class FaultSettings : public Router::RouteSpecificFilterConfig { uint64_t http_status_{}; // HTTP or gRPC return codes Filters::Common::Fault::FaultDelayConfigPtr request_delay_config_; std::string upstream_cluster_; // restrict faults to specific upstream cluster - std::vector fault_filter_headers_; + const std::vector fault_filter_headers_; absl::flat_hash_set downstream_nodes_{}; // Inject failures for specific downstream absl::optional max_active_faults_; Filters::Common::Fault::FaultRateLimitConfigPtr response_rate_limit_; diff --git a/source/extensions/filters/http/health_check/config.cc b/source/extensions/filters/http/health_check/config.cc index 5db283e07272..6db2fa33efc5 100644 --- a/source/extensions/filters/http/health_check/config.cc +++ b/source/extensions/filters/http/health_check/config.cc @@ -21,12 +21,8 @@ Http::FilterFactoryCb HealthCheckFilterConfig::createFilterFactoryFromProtoTyped const bool pass_through_mode = proto_config.pass_through_mode().value(); const int64_t cache_time_ms = PROTOBUF_GET_MS_OR_DEFAULT(proto_config, cache_time, 0); - auto header_match_data = std::make_shared>(); - - for (const envoy::api::v2::route::HeaderMatcher& matcher : proto_config.headers()) { - Http::HeaderUtility::HeaderData single_header_match(matcher); - header_match_data->push_back(std::move(single_header_match)); - } + auto header_match_data = std::make_shared>(); + *header_match_data = Http::HeaderUtility::buildHeaderDataVector(proto_config.headers()); if (!pass_through_mode && cache_time_ms) { throw EnvoyException("cache_time_ms must not be set when path_through_mode is disabled"); diff --git a/source/extensions/filters/http/health_check/health_check.h b/source/extensions/filters/http/health_check/health_check.h index 5515e52e0435..c000e4136618 100644 --- a/source/extensions/filters/http/health_check/health_check.h +++ b/source/extensions/filters/http/health_check/health_check.h @@ -53,7 +53,7 @@ using ClusterMinHealthyPercentages = std::map; using ClusterMinHealthyPercentagesConstSharedPtr = std::shared_ptr; -using HeaderDataVectorSharedPtr = std::shared_ptr>; +using HeaderDataVectorSharedPtr = std::shared_ptr>; /** * Health check responder filter. diff --git a/source/extensions/filters/http/jwt_authn/matcher.cc b/source/extensions/filters/http/jwt_authn/matcher.cc index 53c61c2c5867..75753cdb42e1 100644 --- a/source/extensions/filters/http/jwt_authn/matcher.cc +++ b/source/extensions/filters/http/jwt_authn/matcher.cc @@ -1,6 +1,7 @@ #include "extensions/filters/http/jwt_authn/matcher.h" #include "common/common/logger.h" +#include "common/common/regex.h" #include "common/router/config_impl.h" #include "absl/strings/match.h" @@ -21,14 +22,11 @@ namespace { class BaseMatcherImpl : public Matcher, public Logger::Loggable { public: BaseMatcherImpl(const RequirementRule& rule) - : case_sensitive_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(rule.match(), case_sensitive, true)) { - - for (const auto& header_map : rule.match().headers()) { - config_headers_.push_back(header_map); - } - + : case_sensitive_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(rule.match(), case_sensitive, true)), + config_headers_(Http::HeaderUtility::buildHeaderDataVector(rule.match().headers())) { for (const auto& query_parameter : rule.match().query_parameters()) { - config_query_parameters_.push_back(query_parameter); + config_query_parameters_.push_back( + std::make_unique(query_parameter)); } } @@ -50,8 +48,8 @@ class BaseMatcherImpl : public Matcher, public Logger::Loggable const bool case_sensitive_; private: - std::vector config_headers_; - std::vector config_query_parameters_; + std::vector config_headers_; + std::vector config_query_parameters_; }; /** @@ -108,12 +106,20 @@ class PathMatcherImpl : public BaseMatcherImpl { /** * Perform a match against any path with a regex rule. + * TODO(mattklein123): This code needs dedup with RegexRouteEntryImpl. */ class RegexMatcherImpl : public BaseMatcherImpl { public: - RegexMatcherImpl(const RequirementRule& rule) - : BaseMatcherImpl(rule), regex_(RegexUtil::parseRegex(rule.match().regex())), - regex_str_(rule.match().regex()) {} + RegexMatcherImpl(const RequirementRule& rule) : BaseMatcherImpl(rule) { + if (rule.match().path_specifier_case() == envoy::api::v2::route::RouteMatch::kRegex) { + regex_ = Regex::Utility::parseStdRegexAsCompiledMatcher(rule.match().regex()); + regex_str_ = rule.match().regex(); + } else { + ASSERT(rule.match().path_specifier_case() == envoy::api::v2::route::RouteMatch::kSafeRegex); + regex_ = Regex::Utility::parseRegex(rule.match().safe_regex()); + regex_str_ = rule.match().safe_regex().regex(); + } + } bool matches(const Http::HeaderMap& headers) const override { if (BaseMatcherImpl::matchRoute(headers)) { @@ -121,7 +127,7 @@ class RegexMatcherImpl : public BaseMatcherImpl { const absl::string_view query_string = Http::Utility::findQueryStringStart(path); absl::string_view path_view = path.getStringView(); path_view.remove_suffix(query_string.length()); - if (std::regex_match(path_view.begin(), path_view.end(), regex_)) { + if (regex_->match(path_view)) { ENVOY_LOG(debug, "Regex requirement '{}' matched.", regex_str_); return true; } @@ -130,10 +136,9 @@ class RegexMatcherImpl : public BaseMatcherImpl { } private: - // regex object - const std::regex regex_; + Regex::CompiledMatcherPtr regex_; // raw regex string, for logging. - const std::string regex_str_; + std::string regex_str_; }; } // namespace @@ -145,6 +150,7 @@ MatcherConstPtr Matcher::create(const RequirementRule& rule) { case RouteMatch::PathSpecifierCase::kPath: return std::make_unique(rule); case RouteMatch::PathSpecifierCase::kRegex: + case RouteMatch::PathSpecifierCase::kSafeRegex: return std::make_unique(rule); // path specifier is required. case RouteMatch::PathSpecifierCase::PATH_SPECIFIER_NOT_SET: diff --git a/source/extensions/filters/http/squash/squash_filter.h b/source/extensions/filters/http/squash/squash_filter.h index 0e35c3dd5245..08f63aa4b291 100644 --- a/source/extensions/filters/http/squash/squash_filter.h +++ b/source/extensions/filters/http/squash/squash_filter.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "envoy/config/filter/http/squash/v2/squash.pb.h" #include "envoy/http/async_client.h" #include "envoy/http/filter.h" diff --git a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc index 81eac5f0f396..35850cfee78b 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc +++ b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.cc @@ -15,11 +15,8 @@ namespace Router { RouteEntryImplBase::RouteEntryImplBase( const envoy::config::filter::network::dubbo_proxy::v2alpha1::Route& route) - : cluster_name_(route.route().cluster()) { - for (const auto& header_map : route.match().headers()) { - config_headers_.emplace_back(header_map); - } - + : cluster_name_(route.route().cluster()), + config_headers_(Http::HeaderUtility::buildHeaderDataVector(route.match().headers())) { if (route.route().cluster_specifier_case() == envoy::config::filter::network::dubbo_proxy::v2alpha1::RouteAction::kWeightedClusters) { total_cluster_weight_ = 0UL; diff --git a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h index caa8b9ca3112..9ce491f0aec5 100644 --- a/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h +++ b/source/extensions/filters/network/dubbo_proxy/router/route_matcher.h @@ -77,7 +77,7 @@ class RouteEntryImplBase : public RouteEntry, uint64_t total_cluster_weight_; const std::string cluster_name_; - std::vector config_headers_; + const std::vector config_headers_; std::vector weighted_clusters_; // TODO(gengleilei) Implement it. @@ -123,7 +123,7 @@ class MethodRouteEntryImpl : public RouteEntryImplBase { uint64_t random_value) const override; private: - const Matchers::StringMatcher method_name_; + const Matchers::StringMatcherImpl method_name_; std::shared_ptr parameter_route_; }; diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc index 1fc44b5f78ae..81a9b015ee65 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc @@ -22,11 +22,9 @@ namespace Router { RouteEntryImplBase::RouteEntryImplBase( const envoy::config::filter::network::thrift_proxy::v2alpha1::Route& route) - : cluster_name_(route.route().cluster()), rate_limit_policy_(route.route().rate_limits()) { - for (const auto& header_map : route.match().headers()) { - config_headers_.push_back(header_map); - } - + : cluster_name_(route.route().cluster()), + config_headers_(Http::HeaderUtility::buildHeaderDataVector(route.match().headers())), + rate_limit_policy_(route.route().rate_limits()) { if (route.route().has_metadata_match()) { const auto filter_it = route.route().metadata_match().filter_metadata().find( Envoy::Config::MetadataFilters::get().ENVOY_LB); diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.h b/source/extensions/filters/network/thrift_proxy/router/router_impl.h index de8bc9e0d470..e6db1bef68f0 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.h +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.h @@ -83,7 +83,7 @@ class RouteEntryImplBase : public RouteEntry, using WeightedClusterEntrySharedPtr = std::shared_ptr; const std::string cluster_name_; - std::vector config_headers_; + const std::vector config_headers_; std::vector weighted_clusters_; uint64_t total_cluster_weight_; Envoy::Router::MetadataMatchCriteriaConstPtr metadata_match_criteria_; diff --git a/source/extensions/filters/network/thrift_proxy/router/router_ratelimit_impl.cc b/source/extensions/filters/network/thrift_proxy/router/router_ratelimit_impl.cc index c1f3f99f7cb2..a9b2b1196f16 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_ratelimit_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/router/router_ratelimit_impl.cc @@ -70,11 +70,8 @@ bool GenericKeyAction::populateDescriptor(const RouteEntry&, RateLimit::Descript HeaderValueMatchAction::HeaderValueMatchAction( const envoy::api::v2::route::RateLimit::Action::HeaderValueMatch& action) : descriptor_value_(action.descriptor_value()), - expect_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(action, expect_match, true)) { - for (const auto& header_matcher : action.headers()) { - action_headers_.push_back(header_matcher); - } -} + expect_match_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(action, expect_match, true)), + action_headers_(Http::HeaderUtility::buildHeaderDataVector(action.headers())) {} bool HeaderValueMatchAction::populateDescriptor(const RouteEntry&, RateLimit::Descriptor& descriptor, diff --git a/source/extensions/filters/network/thrift_proxy/router/router_ratelimit_impl.h b/source/extensions/filters/network/thrift_proxy/router/router_ratelimit_impl.h index 4423e0633881..976456ec2069 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_ratelimit_impl.h +++ b/source/extensions/filters/network/thrift_proxy/router/router_ratelimit_impl.h @@ -104,7 +104,7 @@ class HeaderValueMatchAction : public RateLimitAction { private: const std::string descriptor_value_; const bool expect_match_; - std::vector action_headers_; + const std::vector action_headers_; }; /* diff --git a/test/common/common/BUILD b/test/common/common/BUILD index 4714f1ff5c72..dd9b768e0ad5 100644 --- a/test/common/common/BUILD +++ b/test/common/common/BUILD @@ -143,6 +143,15 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "regex_test", + srcs = ["regex_test.cc"], + deps = [ + "//source/common/common:regex_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "perf_annotation_test", srcs = ["perf_annotation_test.cc"], diff --git a/test/common/common/matchers_test.cc b/test/common/common/matchers_test.cc index d5b509a89082..3c0a64f4ca05 100644 --- a/test/common/common/matchers_test.cc +++ b/test/common/common/matchers_test.cc @@ -257,6 +257,15 @@ TEST(MetadataTest, MatchDoubleListValue) { metadataValue.Clear(); } +TEST(StringMatcher, SafeRegexValue) { + envoy::type::matcher::StringMatcher matcher; + matcher.mutable_safe_regex()->mutable_google_re2(); + matcher.mutable_safe_regex()->set_regex("foo.*"); + EXPECT_TRUE(Matchers::StringMatcherImpl(matcher).match("foo")); + EXPECT_TRUE(Matchers::StringMatcherImpl(matcher).match("foobar")); + EXPECT_FALSE(Matchers::StringMatcherImpl(matcher).match("bar")); +} + TEST(LowerCaseStringMatcher, MatchExactValue) { envoy::type::matcher::StringMatcher matcher; matcher.set_exact("Foo"); diff --git a/test/common/common/regex_test.cc b/test/common/common/regex_test.cc new file mode 100644 index 000000000000..b12454d7cdb0 --- /dev/null +++ b/test/common/common/regex_test.cc @@ -0,0 +1,61 @@ +#include "envoy/common/exception.h" + +#include "common/common/regex.h" + +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Regex { +namespace { + +TEST(Utility, ParseStdRegex) { + EXPECT_THROW_WITH_REGEX(Utility::parseStdRegex("(+invalid)"), EnvoyException, + "Invalid regex '\\(\\+invalid\\)': .+"); + + { + std::regex regex = Utility::parseStdRegex("x*"); + EXPECT_NE(0, regex.flags() & std::regex::optimize); + } + + { + std::regex regex = Utility::parseStdRegex("x*", std::regex::icase); + EXPECT_NE(0, regex.flags() & std::regex::icase); + EXPECT_EQ(0, regex.flags() & std::regex::optimize); + } +} + +TEST(Utility, ParseRegex) { + { + envoy::type::matcher::RegexMatcher matcher; + matcher.mutable_google_re2(); + matcher.set_regex("(+invalid)"); + EXPECT_THROW_WITH_MESSAGE(Utility::parseRegex(matcher), EnvoyException, + "no argument for repetition operator: +"); + } + + // Regression test for https://github.com/envoyproxy/envoy/issues/7728 + { + envoy::type::matcher::RegexMatcher matcher; + matcher.mutable_google_re2(); + matcher.set_regex("/asdf/.*"); + const auto compiled_matcher = Utility::parseRegex(matcher); + const std::string long_string = "/asdf/" + std::string(50 * 1024, 'a'); + EXPECT_TRUE(compiled_matcher->match(long_string)); + } + + // Verify max program size. + { + envoy::type::matcher::RegexMatcher matcher; + matcher.mutable_google_re2()->mutable_max_program_size()->set_value(1); + matcher.set_regex("/asdf/.*"); + EXPECT_THROW_WITH_MESSAGE(Utility::parseRegex(matcher), EnvoyException, + "regex '/asdf/.*' RE2 program size of 24 > max program size of 1. " + "Increase configured max program size if necessary."); + } +} + +} // namespace +} // namespace Regex +} // namespace Envoy diff --git a/test/common/common/utility_test.cc b/test/common/common/utility_test.cc index 3ad89010ac49..750973e79d7f 100644 --- a/test/common/common/utility_test.cc +++ b/test/common/common/utility_test.cc @@ -401,22 +401,6 @@ TEST(Primes, findPrimeLargerThan) { EXPECT_EQ(10007, Primes::findPrimeLargerThan(9991)); } -TEST(RegexUtil, parseRegex) { - EXPECT_THROW_WITH_REGEX(RegexUtil::parseRegex("(+invalid)"), EnvoyException, - "Invalid regex '\\(\\+invalid\\)': .+"); - - { - std::regex regex = RegexUtil::parseRegex("x*"); - EXPECT_NE(0, regex.flags() & std::regex::optimize); - } - - { - std::regex regex = RegexUtil::parseRegex("x*", std::regex::icase); - EXPECT_NE(0, regex.flags() & std::regex::icase); - EXPECT_EQ(0, regex.flags() & std::regex::optimize); - } -} - class WeightedClusterEntry { public: WeightedClusterEntry(const std::string name, const uint64_t weight) diff --git a/test/common/http/header_utility_test.cc b/test/common/http/header_utility_test.cc index 97f6b95b34b6..c61443feec2f 100644 --- a/test/common/http/header_utility_test.cc +++ b/test/common/http/header_utility_test.cc @@ -175,8 +175,9 @@ name: match-header regex_match: (a|b) )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_FALSE(HeaderUtility::matchHeaders(headers, header_data)); headers.addCopy("match-header", "a"); @@ -202,9 +203,11 @@ name: match-header-A name: match-header-B )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yamlA))); - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yamlB))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yamlA))); + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yamlB))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers_1, header_data)); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers_2, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers_1, header_data)); @@ -220,8 +223,9 @@ TEST(MatchHeadersTest, HeaderPresence) { name: match-header )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -235,8 +239,9 @@ name: match-header exact_match: match-value )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -252,8 +257,9 @@ exact_match: match-value invert_match: true )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -266,8 +272,26 @@ name: match-header regex_match: \d{3} )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); + EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); + EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); +} + +TEST(MatchHeadersTest, HeaderSafeRegexMatch) { + TestHeaderMapImpl matching_headers{{"match-header", "123"}}; + TestHeaderMapImpl unmatching_headers{{"match-header", "1234"}, {"match-header", "123.456"}}; + const std::string yaml = R"EOF( +name: match-header +safe_regex_match: + google_re2: {} + regex: \d{3} + )EOF"; + + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -282,8 +306,9 @@ regex_match: \d{3} invert_match: true )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -301,8 +326,9 @@ name: match-header end: 0 )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -322,8 +348,9 @@ name: match-header invert_match: true )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -338,8 +365,9 @@ name: match-header present_match: true )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -355,8 +383,9 @@ present_match: true invert_match: true )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -370,8 +399,9 @@ name: match-header prefix_match: value )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -386,8 +416,9 @@ prefix_match: value invert_match: true )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -401,8 +432,9 @@ name: match-header suffix_match: value )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } @@ -417,8 +449,9 @@ suffix_match: value invert_match: true )EOF"; - std::vector header_data; - header_data.push_back(HeaderUtility::HeaderData(parseHeaderMatcherFromYaml(yaml))); + std::vector header_data; + header_data.push_back( + std::make_unique(parseHeaderMatcherFromYaml(yaml))); EXPECT_TRUE(HeaderUtility::matchHeaders(matching_headers, header_data)); EXPECT_FALSE(HeaderUtility::matchHeaders(unmatching_headers, header_data)); } diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index ca9073d4b712..ebed75c45251 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -175,7 +175,9 @@ TEST_F(RouteMatcherTest, TestRoutes) { route: cluster: clock - match: - regex: "/baa+" + safe_regex: + google_re2: {} + regex: "/baa+" route: cluster: sheep - match: @@ -295,8 +297,13 @@ TEST_F(RouteMatcherTest, TestRoutes) { - pattern: "^/users/\\d+$" method: PUT name: update_user - - pattern: "^/users/\\d+/location$" - method: POST + - headers: + - name: ":path" + safe_regex_match: + google_re2: {} + regex: "^/users/\\d+/location$" + - name: ":method" + exact_match: POST name: ulu )EOF"; @@ -643,6 +650,26 @@ TEST_F(RouteMatcherTest, TestRoutesWithInvalidRegex) { EnvoyException, "Invalid regex '\\^/\\(\\+invalid\\)':"); } +// Virtual cluster that contains neither pattern nor regex. This must be checked while pattern is +// deprecated. +TEST_F(RouteMatcherTest, TestRoutesWithInvalidVirtualCluster) { + const std::string invalid_virtual_cluster = R"EOF( +virtual_hosts: + - name: regex + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: "regex" } + virtual_clusters: + - name: "invalid" + )EOF"; + + EXPECT_THROW_WITH_REGEX(TestConfigImpl(parseRouteConfigurationFromV2Yaml(invalid_virtual_cluster), + factory_context_, true), + EnvoyException, + "virtual clusters must define either 'pattern' or 'headers'"); +} + // Validates behavior of request_headers_to_add at router, vhost, and route levels. TEST_F(RouteMatcherTest, TestAddRemoveRequestHeaders) { const std::string yaml = R"EOF( @@ -1325,6 +1352,21 @@ TEST_F(RouteMatcherTest, QueryParamMatchedRouting) { - name: debug route: cluster: local_service_with_valueless_query_parameter + - match: + prefix: "/" + query_parameters: + - name: debug2 + present_match: true + route: + cluster: local_service_with_present_match_query_parameter + - match: + prefix: "/" + query_parameters: + - name: debug3 + string_match: + exact: foo + route: + cluster: local_service_with_string_match_query_parameter - match: prefix: "/" route: @@ -1364,6 +1406,18 @@ TEST_F(RouteMatcherTest, QueryParamMatchedRouting) { config.route(headers, 0)->routeEntry()->clusterName()); } + { + Http::TestHeaderMapImpl headers = genHeaders("example.com", "/?debug2", "GET"); + EXPECT_EQ("local_service_with_present_match_query_parameter", + config.route(headers, 0)->routeEntry()->clusterName()); + } + + { + Http::TestHeaderMapImpl headers = genHeaders("example.com", "/?debug3=foo", "GET"); + EXPECT_EQ("local_service_with_string_match_query_parameter", + config.route(headers, 0)->routeEntry()->clusterName()); + } + { Http::TestHeaderMapImpl headers = genHeaders("example.com", "/?debug=2", "GET"); EXPECT_EQ("local_service_with_valueless_query_parameter", @@ -4217,6 +4271,12 @@ TEST_F(RoutePropertyTest, TestVHostCorsConfig) { domains: ["*"] cors: allow_origin: ["test-origin"] + allow_origin_regex: + - .*\.envoyproxy\.io + allow_origin_string_match: + - safe_regex: + google_re2: {} + regex: .*\.envoyproxy\.io allow_methods: "test-methods" allow_headers: "test-headers" expose_headers: "test-expose-headers" @@ -4258,7 +4318,7 @@ TEST_F(RoutePropertyTest, TestVHostCorsConfig) { EXPECT_EQ(cors_policy->enabled(), false); EXPECT_EQ(cors_policy->shadowEnabled(), true); - EXPECT_THAT(cors_policy->allowOrigins(), ElementsAreArray({"test-origin"})); + EXPECT_EQ(3, cors_policy->allowOrigins().size()); EXPECT_EQ(cors_policy->allowMethods(), "test-methods"); EXPECT_EQ(cors_policy->allowHeaders(), "test-headers"); EXPECT_EQ(cors_policy->exposeHeaders(), "test-expose-headers"); @@ -4311,7 +4371,7 @@ TEST_F(RoutePropertyTest, TestRouteCorsConfig) { EXPECT_EQ(cors_policy->enabled(), false); EXPECT_EQ(cors_policy->shadowEnabled(), true); - EXPECT_THAT(cors_policy->allowOrigins(), ElementsAreArray({"test-origin"})); + EXPECT_EQ(1, cors_policy->allowOrigins().size()); EXPECT_EQ(cors_policy->allowMethods(), "test-methods"); EXPECT_EQ(cors_policy->allowHeaders(), "test-headers"); EXPECT_EQ(cors_policy->exposeHeaders(), "test-expose-headers"); @@ -4350,7 +4410,7 @@ TEST_F(RoutePropertyTest, TestVHostCorsLegacyConfig) { EXPECT_EQ(cors_policy->enabled(), true); EXPECT_EQ(cors_policy->shadowEnabled(), false); - EXPECT_THAT(cors_policy->allowOrigins(), ElementsAreArray({"test-origin"})); + EXPECT_EQ(1, cors_policy->allowOrigins().size()); EXPECT_EQ(cors_policy->allowMethods(), "test-methods"); EXPECT_EQ(cors_policy->allowHeaders(), "test-headers"); EXPECT_EQ(cors_policy->exposeHeaders(), "test-expose-headers"); @@ -4386,7 +4446,7 @@ TEST_F(RoutePropertyTest, TestRouteCorsLegacyConfig) { EXPECT_EQ(cors_policy->enabled(), true); EXPECT_EQ(cors_policy->shadowEnabled(), false); - EXPECT_THAT(cors_policy->allowOrigins(), ElementsAreArray({"test-origin"})); + EXPECT_EQ(1, cors_policy->allowOrigins().size()); EXPECT_EQ(cors_policy->allowMethods(), "test-methods"); EXPECT_EQ(cors_policy->allowHeaders(), "test-headers"); EXPECT_EQ(cors_policy->exposeHeaders(), "test-expose-headers"); diff --git a/test/extensions/filters/http/cors/cors_filter_integration_test.cc b/test/extensions/filters/http/cors/cors_filter_integration_test.cc index 6c78c1892313..b313d6162497 100644 --- a/test/extensions/filters/http/cors/cors_filter_integration_test.cc +++ b/test/extensions/filters/http/cors/cors_filter_integration_test.cc @@ -54,6 +54,8 @@ class CorsFilterIntegrationTest : public testing::TestWithParamadd_routes(); route->mutable_match()->set_prefix("/cors-credentials-allowed"); route->mutable_route()->set_cluster("cluster_0"); @@ -67,7 +69,10 @@ class CorsFilterIntegrationTest : public testing::TestWithParammutable_match()->set_prefix("/cors-allow-origin-regex"); route->mutable_route()->set_cluster("cluster_0"); auto* cors = route->mutable_route()->mutable_cors(); - cors->add_allow_origin_regex(".*\\.envoyproxy\\.io"); + auto* safe_regex = + cors->mutable_allow_origin_string_match()->Add()->mutable_safe_regex(); + safe_regex->mutable_google_re2(); + safe_regex->set_regex(".*\\.envoyproxy\\.io"); } { @@ -115,7 +120,7 @@ INSTANTIATE_TEST_SUITE_P(IpVersions, CorsFilterIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); -TEST_P(CorsFilterIntegrationTest, TestVHostConfigSuccess) { +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestVHostConfigSuccess)) { testPreflight( Http::TestHeaderMapImpl{ {":method", "OPTIONS"}, @@ -135,7 +140,7 @@ TEST_P(CorsFilterIntegrationTest, TestVHostConfigSuccess) { }); } -TEST_P(CorsFilterIntegrationTest, TestRouteConfigSuccess) { +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestRouteConfigSuccess)) { testPreflight( Http::TestHeaderMapImpl{ {":method", "OPTIONS"}, @@ -156,7 +161,7 @@ TEST_P(CorsFilterIntegrationTest, TestRouteConfigSuccess) { }); } -TEST_P(CorsFilterIntegrationTest, TestRouteConfigBadOrigin) { +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestRouteConfigBadOrigin)) { testNormalRequest( Http::TestHeaderMapImpl{ {":method", "OPTIONS"}, @@ -173,7 +178,7 @@ TEST_P(CorsFilterIntegrationTest, TestRouteConfigBadOrigin) { }); } -TEST_P(CorsFilterIntegrationTest, TestCorsDisabled) { +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestCorsDisabled)) { testNormalRequest( Http::TestHeaderMapImpl{ {":method", "OPTIONS"}, @@ -190,7 +195,7 @@ TEST_P(CorsFilterIntegrationTest, TestCorsDisabled) { }); } -TEST_P(CorsFilterIntegrationTest, TestEncodeHeaders) { +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestEncodeHeaders)) { testNormalRequest( Http::TestHeaderMapImpl{ {":method", "GET"}, @@ -207,7 +212,7 @@ TEST_P(CorsFilterIntegrationTest, TestEncodeHeaders) { }); } -TEST_P(CorsFilterIntegrationTest, TestEncodeHeadersCredentialsAllowed) { +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestEncodeHeadersCredentialsAllowed)) { testNormalRequest( Http::TestHeaderMapImpl{ {":method", "GET"}, @@ -225,7 +230,7 @@ TEST_P(CorsFilterIntegrationTest, TestEncodeHeadersCredentialsAllowed) { }); } -TEST_P(CorsFilterIntegrationTest, TestAllowedOriginRegex) { +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestAllowedOriginRegex)) { testNormalRequest( Http::TestHeaderMapImpl{ {":method", "GET"}, @@ -243,7 +248,7 @@ TEST_P(CorsFilterIntegrationTest, TestAllowedOriginRegex) { }); } -TEST_P(CorsFilterIntegrationTest, TestExposeHeaders) { +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestExposeHeaders)) { testNormalRequest( Http::TestHeaderMapImpl{ {":method", "GET"}, diff --git a/test/extensions/filters/http/cors/cors_filter_test.cc b/test/extensions/filters/http/cors/cors_filter_test.cc index d56cb8ac6a4a..863bef99a8ef 100644 --- a/test/extensions/filters/http/cors/cors_filter_test.cc +++ b/test/extensions/filters/http/cors/cors_filter_test.cc @@ -1,3 +1,4 @@ +#include "common/common/matchers.h" #include "common/http/header_map_impl.h" #include "extensions/filters/http/cors/cors_filter.h" @@ -23,6 +24,21 @@ namespace Envoy { namespace Extensions { namespace HttpFilters { namespace Cors { +namespace { + +Matchers::StringMatcherPtr makeExactStringMatcher(const std::string& exact_match) { + envoy::type::matcher::StringMatcher config; + config.set_exact(exact_match); + return std::make_unique(config); +} + +Matchers::StringMatcherPtr makeStdRegexStringMatcher(const std::string& regex) { + envoy::type::matcher::StringMatcher config; + config.set_regex(regex); + return std::make_unique(config); +} + +} // namespace class CorsFilterTest : public testing::Test { public: @@ -30,7 +46,7 @@ class CorsFilterTest : public testing::Test { cors_policy_ = std::make_unique(); cors_policy_->enabled_ = true; cors_policy_->shadow_enabled_ = false; - cors_policy_->allow_origin_.emplace_back("*"); + cors_policy_->allow_origins_.emplace_back(makeExactStringMatcher("*")); cors_policy_->allow_methods_ = "GET"; cors_policy_->allow_headers_ = "content-type"; cors_policy_->expose_headers_ = "content-type"; @@ -299,8 +315,8 @@ TEST_F(CorsFilterTest, OptionsRequestNotMatchingOrigin) { Http::TestHeaderMapImpl request_headers{ {":method", "OPTIONS"}, {"origin", "test-host"}, {"access-control-request-method", "GET"}}; - cors_policy_->allow_origin_.clear(); - cors_policy_->allow_origin_.emplace_back("localhost"); + cors_policy_->allow_origins_.clear(); + cors_policy_->allow_origins_.emplace_back(makeExactStringMatcher("localhost")); EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); @@ -319,7 +335,7 @@ TEST_F(CorsFilterTest, OptionsRequestEmptyOriginList) { Http::TestHeaderMapImpl request_headers{ {":method", "OPTIONS"}, {"origin", "test-host"}, {"access-control-request-method", "GET"}}; - cors_policy_->allow_origin_.clear(); + cors_policy_->allow_origins_.clear(); EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); @@ -339,8 +355,8 @@ TEST_F(CorsFilterTest, ValidOptionsRequestWithAllowCredentialsTrue) { {":method", "OPTIONS"}, {"origin", "localhost"}, {"access-control-request-method", "GET"}}; cors_policy_->allow_credentials_ = true; - cors_policy_->allow_origin_.clear(); - cors_policy_->allow_origin_.emplace_back("localhost"); + cors_policy_->allow_origins_.clear(); + cors_policy_->allow_origins_.emplace_back(makeExactStringMatcher("localhost")); Http::TestHeaderMapImpl response_headers{ {":status", "200"}, @@ -485,8 +501,8 @@ TEST_F(CorsFilterTest, EncodeWithAllowCredentialsFalse) { TEST_F(CorsFilterTest, EncodeWithNonMatchingOrigin) { Http::TestHeaderMapImpl request_headers{{"origin", "test-host"}}; - cors_policy_->allow_origin_.clear(); - cors_policy_->allow_origin_.emplace_back("localhost"); + cors_policy_->allow_origins_.clear(); + cors_policy_->allow_origins_.emplace_back(makeExactStringMatcher("localhost")); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); EXPECT_EQ(Http::FilterDataStatus::Continue, filter_.decodeData(data_, false)); @@ -647,8 +663,8 @@ TEST_F(CorsFilterTest, OptionsRequestMatchingOriginByRegex) { {"access-control-max-age", "0"}, }; - cors_policy_->allow_origin_.clear(); - cors_policy_->allow_origin_regex_.emplace_back(std::regex(".*")); + cors_policy_->allow_origins_.clear(); + cors_policy_->allow_origins_.emplace_back(makeStdRegexStringMatcher(".*")); EXPECT_CALL(decoder_callbacks_, encodeHeaders_(HeaderMapEqualRef(&response_headers), true)); @@ -670,8 +686,8 @@ TEST_F(CorsFilterTest, OptionsRequestNotMatchingOriginByRegex) { {"origin", "www.envoyproxy.com"}, {"access-control-request-method", "GET"}}; - cors_policy_->allow_origin_.clear(); - cors_policy_->allow_origin_regex_.emplace_back(std::regex(".*.envoyproxy.io")); + cors_policy_->allow_origins_.clear(); + cors_policy_->allow_origins_.emplace_back(makeStdRegexStringMatcher(".*.envoyproxy.io")); EXPECT_CALL(decoder_callbacks_, encodeHeaders_(_, false)).Times(0); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.decodeHeaders(request_headers, false)); diff --git a/test/extensions/filters/http/health_check/health_check_test.cc b/test/extensions/filters/http/health_check/health_check_test.cc index 34df211019b7..a6cfb78104ea 100644 --- a/test/extensions/filters/http/health_check/health_check_test.cc +++ b/test/extensions/filters/http/health_check/health_check_test.cc @@ -47,11 +47,11 @@ class HealthCheckFilterTest : public testing::Test { void prepareFilter( bool pass_through, ClusterMinHealthyPercentagesConstSharedPtr cluster_min_healthy_percentages = nullptr) { - header_data_ = std::make_shared>(); + header_data_ = std::make_shared>(); envoy::api::v2::route::HeaderMatcher matcher; matcher.set_name(":path"); matcher.set_exact_match("/healthcheck"); - header_data_->emplace_back(matcher); + header_data_->emplace_back(std::make_unique(matcher)); filter_ = std::make_unique(context_, pass_through, cache_manager_, header_data_, cluster_min_healthy_percentages); filter_->setDecoderFilterCallbacks(callbacks_); diff --git a/test/extensions/filters/http/jwt_authn/matcher_test.cc b/test/extensions/filters/http/jwt_authn/matcher_test.cc index 0ca3e6035089..360ac000f462 100644 --- a/test/extensions/filters/http/jwt_authn/matcher_test.cc +++ b/test/extensions/filters/http/jwt_authn/matcher_test.cc @@ -55,6 +55,28 @@ TEST_F(MatcherTest, TestMatchRegex) { EXPECT_FALSE(matcher->matches(headers)); } +TEST_F(MatcherTest, TestMatchSafeRegex) { + const char config[] = R"( +match: + safe_regex: + google_re2: {} + regex: "/[^c][au]t")"; + + RequirementRule rule; + TestUtility::loadFromYaml(config, rule); + MatcherConstPtr matcher = Matcher::create(rule); + auto headers = TestHeaderMapImpl{{":path", "/but"}}; + EXPECT_TRUE(matcher->matches(headers)); + headers = TestHeaderMapImpl{{":path", "/mat?ok=bye"}}; + EXPECT_TRUE(matcher->matches(headers)); + headers = TestHeaderMapImpl{{":path", "/maut"}}; + EXPECT_FALSE(matcher->matches(headers)); + headers = TestHeaderMapImpl{{":path", "/cut"}}; + EXPECT_FALSE(matcher->matches(headers)); + headers = TestHeaderMapImpl{{":path", "/mut/"}}; + EXPECT_FALSE(matcher->matches(headers)); +} + TEST_F(MatcherTest, TestMatchPath) { const char config[] = R"(match: path: "/match" diff --git a/test/extensions/filters/network/thrift_proxy/integration_test.cc b/test/extensions/filters/network/thrift_proxy/integration_test.cc index 0350244d29fb..ec6db8fd0856 100644 --- a/test/extensions/filters/network/thrift_proxy/integration_test.cc +++ b/test/extensions/filters/network/thrift_proxy/integration_test.cc @@ -39,7 +39,9 @@ class ThriftConnManagerIntegrationTest - name: "x-header-1" exact_match: "x-value-1" - name: "x-header-2" - regex_match: "0.[5-9]" + safe_regex_match: + google_re2: {} + regex: "0.[5-9]" - name: "x-header-3" range_match: start: 100 diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index fbd7308fd4a6..62d2eb45ba41 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -755,7 +755,7 @@ void HttpIntegrationTest::testEnvoyProxying100Continue(bool continue_before_upst auto* virtual_host = route_config->mutable_virtual_hosts(0); { auto* cors = virtual_host->mutable_cors(); - cors->add_allow_origin("*"); + cors->mutable_allow_origin_string_match()->Add()->set_exact("*"); cors->set_allow_headers("content-type,x-grpc-web"); cors->set_allow_methods("GET,POST"); } diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 75249eeaad47..06e5875c032a 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -12,7 +12,6 @@ #include "envoy/config/config_provider.h" #include "envoy/config/typed_metadata.h" #include "envoy/event/dispatcher.h" -#include "envoy/json/json_object.h" #include "envoy/local_info/local_info.h" #include "envoy/router/rds.h" #include "envoy/router/route_config_provider_manager.h" @@ -54,8 +53,9 @@ class MockDirectResponseEntry : public DirectResponseEntry { class TestCorsPolicy : public CorsPolicy { public: // Router::CorsPolicy - const std::list& allowOrigins() const override { return allow_origin_; }; - const std::list& allowOriginRegexes() const override { return allow_origin_regex_; }; + const std::vector& allowOrigins() const override { + return allow_origins_; + }; const std::string& allowMethods() const override { return allow_methods_; }; const std::string& allowHeaders() const override { return allow_headers_; }; const std::string& exposeHeaders() const override { return expose_headers_; }; @@ -64,13 +64,12 @@ class TestCorsPolicy : public CorsPolicy { bool enabled() const override { return enabled_; }; bool shadowEnabled() const override { return shadow_enabled_; }; - std::list allow_origin_{}; - std::list allow_origin_regex_{}; - std::string allow_methods_{}; - std::string allow_headers_{}; - std::string expose_headers_{}; + std::vector allow_origins_; + std::string allow_methods_; + std::string allow_headers_; + std::string expose_headers_; std::string max_age_{}; - absl::optional allow_credentials_{}; + absl::optional allow_credentials_; bool enabled_{}; bool shadow_enabled_{}; }; diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 88f627143894..8cf850b1a300 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/tools/check_format.py b/tools/check_format.py index 40953acc8077..2adcc22e8c35 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -63,6 +63,16 @@ # Files in these paths can use Protobuf::util::JsonStringToMessage JSON_STRING_TO_MESSAGE_WHITELIST = ("./source/common/protobuf/utility.cc") +# Files in these paths can use std::regex +STD_REGEX_WHITELIST = ("./source/common/common/utility.cc", "./source/common/common/regex.h", + "./source/common/common/regex.cc", + "./source/common/stats/tag_extractor_impl.h", + "./source/common/stats/tag_extractor_impl.cc", + "./source/common/access_log/access_log_formatter.cc", + "./source/extensions/filters/http/squash/squash_filter.h", + "./source/extensions/filters/http/squash/squash_filter.cc", + "./source/server/http/admin.h", "./source/server/http/admin.cc") + CLANG_FORMAT_PATH = os.getenv("CLANG_FORMAT", "clang-format-8") BUILDIFIER_PATH = os.getenv("BUILDIFIER_BIN", "$GOPATH/bin/buildifier") ENVOY_BUILD_FIXER_PATH = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), @@ -333,6 +343,10 @@ def whitelistedForStatFromString(file_path): return file_path in STAT_FROM_STRING_WHITELIST +def whitelistedForStdRegex(file_path): + return file_path.startswith("./test") or file_path in STD_REGEX_WHITELIST + + def findSubstringAndReturnError(pattern, file_path, error_message): with open(file_path) as f: text = f.read() @@ -581,6 +595,9 @@ def checkSourceLine(line, file_path, reportError): ('.counter(' in line or '.gauge(' in line or '.histogram(' in line): reportError("Don't lookup stats by name at runtime; use StatName saved during construction") + if not whitelistedForStdRegex(file_path) and "std::regex" in line: + reportError("Don't use std::regex in code that handles untrusted input. Use RegexMatcher") + def checkBuildLine(line, file_path, reportError): if "@bazel_tools" in line and not (isSkylarkFile(file_path) or file_path.startswith("./bazel/")): diff --git a/tools/check_format_test_helper.py b/tools/check_format_test_helper.py index 7309a4bcaff1..c41abbc60890 100755 --- a/tools/check_format_test_helper.py +++ b/tools/check_format_test_helper.py @@ -217,6 +217,8 @@ def checkFileExpectingOK(filename): errors += checkUnfixableError( "histogram_from_string.cc", "Don't lookup stats by name at runtime; use StatName saved during construction") + errors += checkUnfixableError( + "regex.cc", "Don't use std::regex in code that handles untrusted input. Use RegexMatcher") errors += fixFileExpectingFailure( "api/missing_package.proto", diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 8835dce7e40a..bf6a869f0797 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -44,6 +44,10 @@ # field. NOT_IMPLEMENTED_HIDE_ANNOTATION = 'not-implemented-hide' +# Comment that allows for easy searching for things that need cleaning up in the next major +# API version. +NEXT_MAJOR_VERSION_ANNOTATION = 'next-major-version' + # Comment. Just used for adding text that will not go into the docs at all. COMMENT_ANNOTATION = 'comment' @@ -58,6 +62,7 @@ NOT_IMPLEMENTED_WARN_ANNOTATION, NOT_IMPLEMENTED_HIDE_ANNOTATION, V2_API_DIFF_ANNOTATION, + NEXT_MAJOR_VERSION_ANNOTATION, COMMENT_ANNOTATION, PROTO_STATUS_ANNOTATION, ]) diff --git a/tools/testdata/check_format/regex.cc b/tools/testdata/check_format/regex.cc new file mode 100644 index 000000000000..53241fae9cba --- /dev/null +++ b/tools/testdata/check_format/regex.cc @@ -0,0 +1,9 @@ +#include + +namespace Envoy { + +struct BadRegex { + std::regex bad_; +} + +} // namespace Envoy From 25e3e86ccf50ec7469a854974c7212f4ffc8c7e1 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Fri, 23 Aug 2019 14:03:30 -0700 Subject: [PATCH 448/542] docs: reorganize configuration tree (#8027) This is similar to what I did for the arch overview a while ago as this section is also getting out of control. Signed-off-by: Matt Klein --- docs/root/configuration/advanced/advanced.rst | 7 ++++++ .../well_known_dynamic_metadata.rst | 0 docs/root/configuration/configuration.rst | 25 ++++++------------- docs/root/configuration/http/http.rst | 8 ++++++ .../http_conn_man/header_sanitizing.rst | 0 .../{ => http}/http_conn_man/headers.rst | 0 .../http_conn_man/http_conn_man.rst | 0 .../{ => http}/http_conn_man/overview.rst | 0 .../{ => http}/http_conn_man/rds.rst | 0 .../http_conn_man/route_matching.rst | 0 .../{ => http}/http_conn_man/runtime.rst | 0 .../{ => http}/http_conn_man/stats.rst | 0 .../http_conn_man/traffic_splitting.rst | 0 .../{ => http}/http_filters/buffer_filter.rst | 0 .../{ => http}/http_filters/cors_filter.rst | 0 .../{ => http}/http_filters/csrf_filter.rst | 0 .../dynamic_forward_proxy_filter.rst | 0 .../http_filters/dynamodb_filter.rst | 0 .../http_filters/ext_authz_filter.rst | 0 .../{ => http}/http_filters/fault_filter.rst | 0 .../http_filters/grpc_http1_bridge_filter.rst | 0 .../grpc_http1_reverse_bridge_filter.rst | 0 .../grpc_json_transcoder_filter.rst | 0 .../http_filters/grpc_web_filter.rst | 0 .../{ => http}/http_filters/gzip_filter.rst | 0 .../header_to_metadata_filter.rst | 0 .../http_filters/health_check_filter.rst | 0 .../{ => http}/http_filters/http_filters.rst | 0 .../http_filters/ip_tagging_filter.rst | 0 .../http_filters/jwt_authn_filter.rst | 0 .../{ => http}/http_filters/lua_filter.rst | 0 .../http_filters/original_src_filter.rst | 0 .../http_filters/rate_limit_filter.rst | 0 .../{ => http}/http_filters/rbac_filter.rst | 0 .../{ => http}/http_filters/router_filter.rst | 0 .../{ => http}/http_filters/squash_filter.rst | 0 .../{ => http}/http_filters/tap_filter.rst | 0 .../listener_filters/http_inspector.rst | 0 .../listener_filters/listener_filters.rst | 0 .../listener_filters/original_dst_filter.rst | 0 .../listener_filters/original_src_filter.rst | 0 .../listener_filters/proxy_protocol.rst | 0 .../listener_filters/tls_inspector.rst | 0 .../configuration/listeners/listeners.rst | 2 ++ .../client_ssl_auth_filter.rst | 0 .../network_filters/dubbo_proxy_filter.rst | 0 .../network_filters/echo_filter.rst | 0 .../network_filters/ext_authz_filter.rst | 0 .../network_filters/mongo_proxy_filter.rst | 0 .../network_filters/mysql_proxy_filter.rst | 0 .../network_filters/network_filters.rst | 0 .../network_filters/rate_limit_filter.rst | 0 .../network_filters/rbac_filter.rst | 0 .../network_filters/redis_proxy_filter.rst | 0 .../network_filters/sni_cluster_filter.rst | 0 .../network_filters/tcp_proxy_filter.rst | 0 .../network_filters/thrift_proxy_filter.rst | 0 .../zookeeper_proxy_filter.rst | 0 .../{ => observability}/access_log.rst | 0 .../observability/observability.rst | 8 ++++++ .../{ => observability}/statistics.rst | 0 .../configuration/operations/operations.rst | 9 +++++++ .../overload_manager/overload_manager.rst | 0 .../{ => operations}/runtime.rst | 0 .../{ => operations}/tools/router_check.rst | 0 .../other_features/other_features.rst | 7 ++++++ .../{ => other_features}/rate_limit.rst | 0 .../dubbo_filters/dubbo_filters.rst | 0 .../dubbo_filters/router_filter.rst | 0 .../other_protocols/other_protocols.rst | 8 ++++++ .../thrift_filters/rate_limit_filter.rst | 0 .../thrift_filters/router_filter.rst | 0 .../thrift_filters/thrift_filters.rst | 0 .../configuration/overview/v2_overview.rst | 24 ++++++++++++++++++ .../configuration/{ => security}/secret.rst | 0 docs/root/configuration/security/security.rst | 7 ++++++ .../{ => upstream}/cluster_manager/cds.rst | 0 .../cluster_circuit_breakers.rst | 0 .../cluster_manager/cluster_hc.rst | 0 .../cluster_manager/cluster_manager.rst | 0 .../cluster_manager/cluster_runtime.rst | 0 .../cluster_manager/cluster_stats.rst | 0 .../cluster_manager/overview.rst | 0 .../health_checkers/health_checkers.rst | 0 .../{ => upstream}/health_checkers/redis.rst | 0 docs/root/configuration/upstream/upstream.rst | 8 ++++++ .../configuration/xds_subscription_stats.rst | 23 ----------------- 87 files changed, 96 insertions(+), 40 deletions(-) create mode 100644 docs/root/configuration/advanced/advanced.rst rename docs/root/configuration/{ => advanced}/well_known_dynamic_metadata.rst (100%) create mode 100644 docs/root/configuration/http/http.rst rename docs/root/configuration/{ => http}/http_conn_man/header_sanitizing.rst (100%) rename docs/root/configuration/{ => http}/http_conn_man/headers.rst (100%) rename docs/root/configuration/{ => http}/http_conn_man/http_conn_man.rst (100%) rename docs/root/configuration/{ => http}/http_conn_man/overview.rst (100%) rename docs/root/configuration/{ => http}/http_conn_man/rds.rst (100%) rename docs/root/configuration/{ => http}/http_conn_man/route_matching.rst (100%) rename docs/root/configuration/{ => http}/http_conn_man/runtime.rst (100%) rename docs/root/configuration/{ => http}/http_conn_man/stats.rst (100%) rename docs/root/configuration/{ => http}/http_conn_man/traffic_splitting.rst (100%) rename docs/root/configuration/{ => http}/http_filters/buffer_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/cors_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/csrf_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/dynamic_forward_proxy_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/dynamodb_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/ext_authz_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/fault_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/grpc_http1_bridge_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/grpc_http1_reverse_bridge_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/grpc_json_transcoder_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/grpc_web_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/gzip_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/header_to_metadata_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/health_check_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/http_filters.rst (100%) rename docs/root/configuration/{ => http}/http_filters/ip_tagging_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/jwt_authn_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/lua_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/original_src_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/rate_limit_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/rbac_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/router_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/squash_filter.rst (100%) rename docs/root/configuration/{ => http}/http_filters/tap_filter.rst (100%) rename docs/root/configuration/{ => listeners}/listener_filters/http_inspector.rst (100%) rename docs/root/configuration/{ => listeners}/listener_filters/listener_filters.rst (100%) rename docs/root/configuration/{ => listeners}/listener_filters/original_dst_filter.rst (100%) rename docs/root/configuration/{ => listeners}/listener_filters/original_src_filter.rst (100%) rename docs/root/configuration/{ => listeners}/listener_filters/proxy_protocol.rst (100%) rename docs/root/configuration/{ => listeners}/listener_filters/tls_inspector.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/client_ssl_auth_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/dubbo_proxy_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/echo_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/ext_authz_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/mongo_proxy_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/mysql_proxy_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/network_filters.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/rate_limit_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/rbac_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/redis_proxy_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/sni_cluster_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/tcp_proxy_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/thrift_proxy_filter.rst (100%) rename docs/root/configuration/{ => listeners}/network_filters/zookeeper_proxy_filter.rst (100%) rename docs/root/configuration/{ => observability}/access_log.rst (100%) create mode 100644 docs/root/configuration/observability/observability.rst rename docs/root/configuration/{ => observability}/statistics.rst (100%) create mode 100644 docs/root/configuration/operations/operations.rst rename docs/root/configuration/{ => operations}/overload_manager/overload_manager.rst (100%) rename docs/root/configuration/{ => operations}/runtime.rst (100%) rename docs/root/configuration/{ => operations}/tools/router_check.rst (100%) create mode 100644 docs/root/configuration/other_features/other_features.rst rename docs/root/configuration/{ => other_features}/rate_limit.rst (100%) rename docs/root/configuration/{ => other_protocols}/dubbo_filters/dubbo_filters.rst (100%) rename docs/root/configuration/{ => other_protocols}/dubbo_filters/router_filter.rst (100%) create mode 100644 docs/root/configuration/other_protocols/other_protocols.rst rename docs/root/configuration/{ => other_protocols}/thrift_filters/rate_limit_filter.rst (100%) rename docs/root/configuration/{ => other_protocols}/thrift_filters/router_filter.rst (100%) rename docs/root/configuration/{ => other_protocols}/thrift_filters/thrift_filters.rst (100%) rename docs/root/configuration/{ => security}/secret.rst (100%) create mode 100644 docs/root/configuration/security/security.rst rename docs/root/configuration/{ => upstream}/cluster_manager/cds.rst (100%) rename docs/root/configuration/{ => upstream}/cluster_manager/cluster_circuit_breakers.rst (100%) rename docs/root/configuration/{ => upstream}/cluster_manager/cluster_hc.rst (100%) rename docs/root/configuration/{ => upstream}/cluster_manager/cluster_manager.rst (100%) rename docs/root/configuration/{ => upstream}/cluster_manager/cluster_runtime.rst (100%) rename docs/root/configuration/{ => upstream}/cluster_manager/cluster_stats.rst (100%) rename docs/root/configuration/{ => upstream}/cluster_manager/overview.rst (100%) rename docs/root/configuration/{ => upstream}/health_checkers/health_checkers.rst (100%) rename docs/root/configuration/{ => upstream}/health_checkers/redis.rst (100%) create mode 100644 docs/root/configuration/upstream/upstream.rst delete mode 100644 docs/root/configuration/xds_subscription_stats.rst diff --git a/docs/root/configuration/advanced/advanced.rst b/docs/root/configuration/advanced/advanced.rst new file mode 100644 index 000000000000..2753c5bb8995 --- /dev/null +++ b/docs/root/configuration/advanced/advanced.rst @@ -0,0 +1,7 @@ +Advanced +======== + +.. toctree:: + :maxdepth: 2 + + well_known_dynamic_metadata diff --git a/docs/root/configuration/well_known_dynamic_metadata.rst b/docs/root/configuration/advanced/well_known_dynamic_metadata.rst similarity index 100% rename from docs/root/configuration/well_known_dynamic_metadata.rst rename to docs/root/configuration/advanced/well_known_dynamic_metadata.rst diff --git a/docs/root/configuration/configuration.rst b/docs/root/configuration/configuration.rst index ffbce401b32d..ec5da63ffcd1 100644 --- a/docs/root/configuration/configuration.rst +++ b/docs/root/configuration/configuration.rst @@ -8,21 +8,12 @@ Configuration reference overview/v2_overview listeners/listeners - listener_filters/listener_filters - network_filters/network_filters - http_conn_man/http_conn_man - http_filters/http_filters - thrift_filters/thrift_filters - dubbo_filters/dubbo_filters - cluster_manager/cluster_manager - health_checkers/health_checkers - access_log - rate_limit - runtime - statistics - xds_subscription_stats - tools/router_check - overload_manager/overload_manager - secret - well_known_dynamic_metadata + http/http + upstream/upstream + observability/observability + security/security + operations/operations + other_features/other_features + other_protocols/other_protocols + advanced/advanced best_practices/best_practices diff --git a/docs/root/configuration/http/http.rst b/docs/root/configuration/http/http.rst new file mode 100644 index 000000000000..f9c8124ff2b6 --- /dev/null +++ b/docs/root/configuration/http/http.rst @@ -0,0 +1,8 @@ +HTTP +==== + +.. toctree:: + :maxdepth: 2 + + http_conn_man/http_conn_man + http_filters/http_filters diff --git a/docs/root/configuration/http_conn_man/header_sanitizing.rst b/docs/root/configuration/http/http_conn_man/header_sanitizing.rst similarity index 100% rename from docs/root/configuration/http_conn_man/header_sanitizing.rst rename to docs/root/configuration/http/http_conn_man/header_sanitizing.rst diff --git a/docs/root/configuration/http_conn_man/headers.rst b/docs/root/configuration/http/http_conn_man/headers.rst similarity index 100% rename from docs/root/configuration/http_conn_man/headers.rst rename to docs/root/configuration/http/http_conn_man/headers.rst diff --git a/docs/root/configuration/http_conn_man/http_conn_man.rst b/docs/root/configuration/http/http_conn_man/http_conn_man.rst similarity index 100% rename from docs/root/configuration/http_conn_man/http_conn_man.rst rename to docs/root/configuration/http/http_conn_man/http_conn_man.rst diff --git a/docs/root/configuration/http_conn_man/overview.rst b/docs/root/configuration/http/http_conn_man/overview.rst similarity index 100% rename from docs/root/configuration/http_conn_man/overview.rst rename to docs/root/configuration/http/http_conn_man/overview.rst diff --git a/docs/root/configuration/http_conn_man/rds.rst b/docs/root/configuration/http/http_conn_man/rds.rst similarity index 100% rename from docs/root/configuration/http_conn_man/rds.rst rename to docs/root/configuration/http/http_conn_man/rds.rst diff --git a/docs/root/configuration/http_conn_man/route_matching.rst b/docs/root/configuration/http/http_conn_man/route_matching.rst similarity index 100% rename from docs/root/configuration/http_conn_man/route_matching.rst rename to docs/root/configuration/http/http_conn_man/route_matching.rst diff --git a/docs/root/configuration/http_conn_man/runtime.rst b/docs/root/configuration/http/http_conn_man/runtime.rst similarity index 100% rename from docs/root/configuration/http_conn_man/runtime.rst rename to docs/root/configuration/http/http_conn_man/runtime.rst diff --git a/docs/root/configuration/http_conn_man/stats.rst b/docs/root/configuration/http/http_conn_man/stats.rst similarity index 100% rename from docs/root/configuration/http_conn_man/stats.rst rename to docs/root/configuration/http/http_conn_man/stats.rst diff --git a/docs/root/configuration/http_conn_man/traffic_splitting.rst b/docs/root/configuration/http/http_conn_man/traffic_splitting.rst similarity index 100% rename from docs/root/configuration/http_conn_man/traffic_splitting.rst rename to docs/root/configuration/http/http_conn_man/traffic_splitting.rst diff --git a/docs/root/configuration/http_filters/buffer_filter.rst b/docs/root/configuration/http/http_filters/buffer_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/buffer_filter.rst rename to docs/root/configuration/http/http_filters/buffer_filter.rst diff --git a/docs/root/configuration/http_filters/cors_filter.rst b/docs/root/configuration/http/http_filters/cors_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/cors_filter.rst rename to docs/root/configuration/http/http_filters/cors_filter.rst diff --git a/docs/root/configuration/http_filters/csrf_filter.rst b/docs/root/configuration/http/http_filters/csrf_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/csrf_filter.rst rename to docs/root/configuration/http/http_filters/csrf_filter.rst diff --git a/docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst b/docs/root/configuration/http/http_filters/dynamic_forward_proxy_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/dynamic_forward_proxy_filter.rst rename to docs/root/configuration/http/http_filters/dynamic_forward_proxy_filter.rst diff --git a/docs/root/configuration/http_filters/dynamodb_filter.rst b/docs/root/configuration/http/http_filters/dynamodb_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/dynamodb_filter.rst rename to docs/root/configuration/http/http_filters/dynamodb_filter.rst diff --git a/docs/root/configuration/http_filters/ext_authz_filter.rst b/docs/root/configuration/http/http_filters/ext_authz_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/ext_authz_filter.rst rename to docs/root/configuration/http/http_filters/ext_authz_filter.rst diff --git a/docs/root/configuration/http_filters/fault_filter.rst b/docs/root/configuration/http/http_filters/fault_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/fault_filter.rst rename to docs/root/configuration/http/http_filters/fault_filter.rst diff --git a/docs/root/configuration/http_filters/grpc_http1_bridge_filter.rst b/docs/root/configuration/http/http_filters/grpc_http1_bridge_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/grpc_http1_bridge_filter.rst rename to docs/root/configuration/http/http_filters/grpc_http1_bridge_filter.rst diff --git a/docs/root/configuration/http_filters/grpc_http1_reverse_bridge_filter.rst b/docs/root/configuration/http/http_filters/grpc_http1_reverse_bridge_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/grpc_http1_reverse_bridge_filter.rst rename to docs/root/configuration/http/http_filters/grpc_http1_reverse_bridge_filter.rst diff --git a/docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst b/docs/root/configuration/http/http_filters/grpc_json_transcoder_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/grpc_json_transcoder_filter.rst rename to docs/root/configuration/http/http_filters/grpc_json_transcoder_filter.rst diff --git a/docs/root/configuration/http_filters/grpc_web_filter.rst b/docs/root/configuration/http/http_filters/grpc_web_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/grpc_web_filter.rst rename to docs/root/configuration/http/http_filters/grpc_web_filter.rst diff --git a/docs/root/configuration/http_filters/gzip_filter.rst b/docs/root/configuration/http/http_filters/gzip_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/gzip_filter.rst rename to docs/root/configuration/http/http_filters/gzip_filter.rst diff --git a/docs/root/configuration/http_filters/header_to_metadata_filter.rst b/docs/root/configuration/http/http_filters/header_to_metadata_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/header_to_metadata_filter.rst rename to docs/root/configuration/http/http_filters/header_to_metadata_filter.rst diff --git a/docs/root/configuration/http_filters/health_check_filter.rst b/docs/root/configuration/http/http_filters/health_check_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/health_check_filter.rst rename to docs/root/configuration/http/http_filters/health_check_filter.rst diff --git a/docs/root/configuration/http_filters/http_filters.rst b/docs/root/configuration/http/http_filters/http_filters.rst similarity index 100% rename from docs/root/configuration/http_filters/http_filters.rst rename to docs/root/configuration/http/http_filters/http_filters.rst diff --git a/docs/root/configuration/http_filters/ip_tagging_filter.rst b/docs/root/configuration/http/http_filters/ip_tagging_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/ip_tagging_filter.rst rename to docs/root/configuration/http/http_filters/ip_tagging_filter.rst diff --git a/docs/root/configuration/http_filters/jwt_authn_filter.rst b/docs/root/configuration/http/http_filters/jwt_authn_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/jwt_authn_filter.rst rename to docs/root/configuration/http/http_filters/jwt_authn_filter.rst diff --git a/docs/root/configuration/http_filters/lua_filter.rst b/docs/root/configuration/http/http_filters/lua_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/lua_filter.rst rename to docs/root/configuration/http/http_filters/lua_filter.rst diff --git a/docs/root/configuration/http_filters/original_src_filter.rst b/docs/root/configuration/http/http_filters/original_src_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/original_src_filter.rst rename to docs/root/configuration/http/http_filters/original_src_filter.rst diff --git a/docs/root/configuration/http_filters/rate_limit_filter.rst b/docs/root/configuration/http/http_filters/rate_limit_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/rate_limit_filter.rst rename to docs/root/configuration/http/http_filters/rate_limit_filter.rst diff --git a/docs/root/configuration/http_filters/rbac_filter.rst b/docs/root/configuration/http/http_filters/rbac_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/rbac_filter.rst rename to docs/root/configuration/http/http_filters/rbac_filter.rst diff --git a/docs/root/configuration/http_filters/router_filter.rst b/docs/root/configuration/http/http_filters/router_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/router_filter.rst rename to docs/root/configuration/http/http_filters/router_filter.rst diff --git a/docs/root/configuration/http_filters/squash_filter.rst b/docs/root/configuration/http/http_filters/squash_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/squash_filter.rst rename to docs/root/configuration/http/http_filters/squash_filter.rst diff --git a/docs/root/configuration/http_filters/tap_filter.rst b/docs/root/configuration/http/http_filters/tap_filter.rst similarity index 100% rename from docs/root/configuration/http_filters/tap_filter.rst rename to docs/root/configuration/http/http_filters/tap_filter.rst diff --git a/docs/root/configuration/listener_filters/http_inspector.rst b/docs/root/configuration/listeners/listener_filters/http_inspector.rst similarity index 100% rename from docs/root/configuration/listener_filters/http_inspector.rst rename to docs/root/configuration/listeners/listener_filters/http_inspector.rst diff --git a/docs/root/configuration/listener_filters/listener_filters.rst b/docs/root/configuration/listeners/listener_filters/listener_filters.rst similarity index 100% rename from docs/root/configuration/listener_filters/listener_filters.rst rename to docs/root/configuration/listeners/listener_filters/listener_filters.rst diff --git a/docs/root/configuration/listener_filters/original_dst_filter.rst b/docs/root/configuration/listeners/listener_filters/original_dst_filter.rst similarity index 100% rename from docs/root/configuration/listener_filters/original_dst_filter.rst rename to docs/root/configuration/listeners/listener_filters/original_dst_filter.rst diff --git a/docs/root/configuration/listener_filters/original_src_filter.rst b/docs/root/configuration/listeners/listener_filters/original_src_filter.rst similarity index 100% rename from docs/root/configuration/listener_filters/original_src_filter.rst rename to docs/root/configuration/listeners/listener_filters/original_src_filter.rst diff --git a/docs/root/configuration/listener_filters/proxy_protocol.rst b/docs/root/configuration/listeners/listener_filters/proxy_protocol.rst similarity index 100% rename from docs/root/configuration/listener_filters/proxy_protocol.rst rename to docs/root/configuration/listeners/listener_filters/proxy_protocol.rst diff --git a/docs/root/configuration/listener_filters/tls_inspector.rst b/docs/root/configuration/listeners/listener_filters/tls_inspector.rst similarity index 100% rename from docs/root/configuration/listener_filters/tls_inspector.rst rename to docs/root/configuration/listeners/listener_filters/tls_inspector.rst diff --git a/docs/root/configuration/listeners/listeners.rst b/docs/root/configuration/listeners/listeners.rst index 1efab35c090b..73605a853658 100644 --- a/docs/root/configuration/listeners/listeners.rst +++ b/docs/root/configuration/listeners/listeners.rst @@ -8,4 +8,6 @@ Listeners overview stats + listener_filters/listener_filters + network_filters/network_filters lds diff --git a/docs/root/configuration/network_filters/client_ssl_auth_filter.rst b/docs/root/configuration/listeners/network_filters/client_ssl_auth_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/client_ssl_auth_filter.rst rename to docs/root/configuration/listeners/network_filters/client_ssl_auth_filter.rst diff --git a/docs/root/configuration/network_filters/dubbo_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/dubbo_proxy_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/dubbo_proxy_filter.rst rename to docs/root/configuration/listeners/network_filters/dubbo_proxy_filter.rst diff --git a/docs/root/configuration/network_filters/echo_filter.rst b/docs/root/configuration/listeners/network_filters/echo_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/echo_filter.rst rename to docs/root/configuration/listeners/network_filters/echo_filter.rst diff --git a/docs/root/configuration/network_filters/ext_authz_filter.rst b/docs/root/configuration/listeners/network_filters/ext_authz_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/ext_authz_filter.rst rename to docs/root/configuration/listeners/network_filters/ext_authz_filter.rst diff --git a/docs/root/configuration/network_filters/mongo_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/mongo_proxy_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/mongo_proxy_filter.rst rename to docs/root/configuration/listeners/network_filters/mongo_proxy_filter.rst diff --git a/docs/root/configuration/network_filters/mysql_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/mysql_proxy_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/mysql_proxy_filter.rst rename to docs/root/configuration/listeners/network_filters/mysql_proxy_filter.rst diff --git a/docs/root/configuration/network_filters/network_filters.rst b/docs/root/configuration/listeners/network_filters/network_filters.rst similarity index 100% rename from docs/root/configuration/network_filters/network_filters.rst rename to docs/root/configuration/listeners/network_filters/network_filters.rst diff --git a/docs/root/configuration/network_filters/rate_limit_filter.rst b/docs/root/configuration/listeners/network_filters/rate_limit_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/rate_limit_filter.rst rename to docs/root/configuration/listeners/network_filters/rate_limit_filter.rst diff --git a/docs/root/configuration/network_filters/rbac_filter.rst b/docs/root/configuration/listeners/network_filters/rbac_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/rbac_filter.rst rename to docs/root/configuration/listeners/network_filters/rbac_filter.rst diff --git a/docs/root/configuration/network_filters/redis_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/redis_proxy_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/redis_proxy_filter.rst rename to docs/root/configuration/listeners/network_filters/redis_proxy_filter.rst diff --git a/docs/root/configuration/network_filters/sni_cluster_filter.rst b/docs/root/configuration/listeners/network_filters/sni_cluster_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/sni_cluster_filter.rst rename to docs/root/configuration/listeners/network_filters/sni_cluster_filter.rst diff --git a/docs/root/configuration/network_filters/tcp_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/tcp_proxy_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/tcp_proxy_filter.rst rename to docs/root/configuration/listeners/network_filters/tcp_proxy_filter.rst diff --git a/docs/root/configuration/network_filters/thrift_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/thrift_proxy_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/thrift_proxy_filter.rst rename to docs/root/configuration/listeners/network_filters/thrift_proxy_filter.rst diff --git a/docs/root/configuration/network_filters/zookeeper_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst similarity index 100% rename from docs/root/configuration/network_filters/zookeeper_proxy_filter.rst rename to docs/root/configuration/listeners/network_filters/zookeeper_proxy_filter.rst diff --git a/docs/root/configuration/access_log.rst b/docs/root/configuration/observability/access_log.rst similarity index 100% rename from docs/root/configuration/access_log.rst rename to docs/root/configuration/observability/access_log.rst diff --git a/docs/root/configuration/observability/observability.rst b/docs/root/configuration/observability/observability.rst new file mode 100644 index 000000000000..6c2be157e839 --- /dev/null +++ b/docs/root/configuration/observability/observability.rst @@ -0,0 +1,8 @@ +Observability +============= + +.. toctree:: + :maxdepth: 2 + + statistics + access_log diff --git a/docs/root/configuration/statistics.rst b/docs/root/configuration/observability/statistics.rst similarity index 100% rename from docs/root/configuration/statistics.rst rename to docs/root/configuration/observability/statistics.rst diff --git a/docs/root/configuration/operations/operations.rst b/docs/root/configuration/operations/operations.rst new file mode 100644 index 000000000000..faa9c0f37419 --- /dev/null +++ b/docs/root/configuration/operations/operations.rst @@ -0,0 +1,9 @@ +Operations +========== + +.. toctree:: + :maxdepth: 2 + + runtime + overload_manager/overload_manager + tools/router_check diff --git a/docs/root/configuration/overload_manager/overload_manager.rst b/docs/root/configuration/operations/overload_manager/overload_manager.rst similarity index 100% rename from docs/root/configuration/overload_manager/overload_manager.rst rename to docs/root/configuration/operations/overload_manager/overload_manager.rst diff --git a/docs/root/configuration/runtime.rst b/docs/root/configuration/operations/runtime.rst similarity index 100% rename from docs/root/configuration/runtime.rst rename to docs/root/configuration/operations/runtime.rst diff --git a/docs/root/configuration/tools/router_check.rst b/docs/root/configuration/operations/tools/router_check.rst similarity index 100% rename from docs/root/configuration/tools/router_check.rst rename to docs/root/configuration/operations/tools/router_check.rst diff --git a/docs/root/configuration/other_features/other_features.rst b/docs/root/configuration/other_features/other_features.rst new file mode 100644 index 000000000000..84d8f49483ce --- /dev/null +++ b/docs/root/configuration/other_features/other_features.rst @@ -0,0 +1,7 @@ +Other features +============== + +.. toctree:: + :maxdepth: 2 + + rate_limit diff --git a/docs/root/configuration/rate_limit.rst b/docs/root/configuration/other_features/rate_limit.rst similarity index 100% rename from docs/root/configuration/rate_limit.rst rename to docs/root/configuration/other_features/rate_limit.rst diff --git a/docs/root/configuration/dubbo_filters/dubbo_filters.rst b/docs/root/configuration/other_protocols/dubbo_filters/dubbo_filters.rst similarity index 100% rename from docs/root/configuration/dubbo_filters/dubbo_filters.rst rename to docs/root/configuration/other_protocols/dubbo_filters/dubbo_filters.rst diff --git a/docs/root/configuration/dubbo_filters/router_filter.rst b/docs/root/configuration/other_protocols/dubbo_filters/router_filter.rst similarity index 100% rename from docs/root/configuration/dubbo_filters/router_filter.rst rename to docs/root/configuration/other_protocols/dubbo_filters/router_filter.rst diff --git a/docs/root/configuration/other_protocols/other_protocols.rst b/docs/root/configuration/other_protocols/other_protocols.rst new file mode 100644 index 000000000000..8ad84b892de1 --- /dev/null +++ b/docs/root/configuration/other_protocols/other_protocols.rst @@ -0,0 +1,8 @@ +Other protocols +=============== + +.. toctree:: + :maxdepth: 2 + + thrift_filters/thrift_filters + dubbo_filters/dubbo_filters diff --git a/docs/root/configuration/thrift_filters/rate_limit_filter.rst b/docs/root/configuration/other_protocols/thrift_filters/rate_limit_filter.rst similarity index 100% rename from docs/root/configuration/thrift_filters/rate_limit_filter.rst rename to docs/root/configuration/other_protocols/thrift_filters/rate_limit_filter.rst diff --git a/docs/root/configuration/thrift_filters/router_filter.rst b/docs/root/configuration/other_protocols/thrift_filters/router_filter.rst similarity index 100% rename from docs/root/configuration/thrift_filters/router_filter.rst rename to docs/root/configuration/other_protocols/thrift_filters/router_filter.rst diff --git a/docs/root/configuration/thrift_filters/thrift_filters.rst b/docs/root/configuration/other_protocols/thrift_filters/thrift_filters.rst similarity index 100% rename from docs/root/configuration/thrift_filters/thrift_filters.rst rename to docs/root/configuration/other_protocols/thrift_filters/thrift_filters.rst diff --git a/docs/root/configuration/overview/v2_overview.rst b/docs/root/configuration/overview/v2_overview.rst index 7802030f434c..3dcefb0067d5 100644 --- a/docs/root/configuration/overview/v2_overview.rst +++ b/docs/root/configuration/overview/v2_overview.rst @@ -599,6 +599,30 @@ Management Server has a statistics tree rooted at *control_plane.* with the foll rate_limit_enforced, Counter, Total number of times rate limit was enforced for management server requests pending_requests, Gauge, Total number of pending requests when the rate limit was enforced +.. _subscription_statistics: + +xDS subscription statistics +--------------------------- + +Envoy discovers its various dynamic resources via discovery +services referred to as *xDS*. Resources are requested via :ref:`subscriptions `, +by specifying a filesystem path to watch, initiating gRPC streams or polling a REST-JSON URL. + +The following statistics are generated for all subscriptions. + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + config_reload, Counter, Total API fetches that resulted in a config reload due to a different config + init_fetch_timeout, Counter, Total :ref:`initial fetch timeouts ` + update_attempt, Counter, Total API fetches attempted + update_success, Counter, Total API fetches completed successfully + update_failure, Counter, Total API fetches that failed because of network errors + update_rejected, Counter, Total API fetches that failed because of schema/validation errors + version, Gauge, Hash of the contents from the last successful API fetch + control_plane.connected_state, Gauge, A boolean (1 for connected and 0 for disconnected) that indicates the current connection state with management server + .. _config_overview_v2_status: Status diff --git a/docs/root/configuration/secret.rst b/docs/root/configuration/security/secret.rst similarity index 100% rename from docs/root/configuration/secret.rst rename to docs/root/configuration/security/secret.rst diff --git a/docs/root/configuration/security/security.rst b/docs/root/configuration/security/security.rst new file mode 100644 index 000000000000..223a9ee5e348 --- /dev/null +++ b/docs/root/configuration/security/security.rst @@ -0,0 +1,7 @@ +Security +======== + +.. toctree:: + :maxdepth: 2 + + secret diff --git a/docs/root/configuration/cluster_manager/cds.rst b/docs/root/configuration/upstream/cluster_manager/cds.rst similarity index 100% rename from docs/root/configuration/cluster_manager/cds.rst rename to docs/root/configuration/upstream/cluster_manager/cds.rst diff --git a/docs/root/configuration/cluster_manager/cluster_circuit_breakers.rst b/docs/root/configuration/upstream/cluster_manager/cluster_circuit_breakers.rst similarity index 100% rename from docs/root/configuration/cluster_manager/cluster_circuit_breakers.rst rename to docs/root/configuration/upstream/cluster_manager/cluster_circuit_breakers.rst diff --git a/docs/root/configuration/cluster_manager/cluster_hc.rst b/docs/root/configuration/upstream/cluster_manager/cluster_hc.rst similarity index 100% rename from docs/root/configuration/cluster_manager/cluster_hc.rst rename to docs/root/configuration/upstream/cluster_manager/cluster_hc.rst diff --git a/docs/root/configuration/cluster_manager/cluster_manager.rst b/docs/root/configuration/upstream/cluster_manager/cluster_manager.rst similarity index 100% rename from docs/root/configuration/cluster_manager/cluster_manager.rst rename to docs/root/configuration/upstream/cluster_manager/cluster_manager.rst diff --git a/docs/root/configuration/cluster_manager/cluster_runtime.rst b/docs/root/configuration/upstream/cluster_manager/cluster_runtime.rst similarity index 100% rename from docs/root/configuration/cluster_manager/cluster_runtime.rst rename to docs/root/configuration/upstream/cluster_manager/cluster_runtime.rst diff --git a/docs/root/configuration/cluster_manager/cluster_stats.rst b/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst similarity index 100% rename from docs/root/configuration/cluster_manager/cluster_stats.rst rename to docs/root/configuration/upstream/cluster_manager/cluster_stats.rst diff --git a/docs/root/configuration/cluster_manager/overview.rst b/docs/root/configuration/upstream/cluster_manager/overview.rst similarity index 100% rename from docs/root/configuration/cluster_manager/overview.rst rename to docs/root/configuration/upstream/cluster_manager/overview.rst diff --git a/docs/root/configuration/health_checkers/health_checkers.rst b/docs/root/configuration/upstream/health_checkers/health_checkers.rst similarity index 100% rename from docs/root/configuration/health_checkers/health_checkers.rst rename to docs/root/configuration/upstream/health_checkers/health_checkers.rst diff --git a/docs/root/configuration/health_checkers/redis.rst b/docs/root/configuration/upstream/health_checkers/redis.rst similarity index 100% rename from docs/root/configuration/health_checkers/redis.rst rename to docs/root/configuration/upstream/health_checkers/redis.rst diff --git a/docs/root/configuration/upstream/upstream.rst b/docs/root/configuration/upstream/upstream.rst new file mode 100644 index 000000000000..3e84e6352d39 --- /dev/null +++ b/docs/root/configuration/upstream/upstream.rst @@ -0,0 +1,8 @@ +Upstream clusters +================= + +.. toctree:: + :maxdepth: 2 + + cluster_manager/cluster_manager + health_checkers/health_checkers diff --git a/docs/root/configuration/xds_subscription_stats.rst b/docs/root/configuration/xds_subscription_stats.rst deleted file mode 100644 index c15bbcc22a1a..000000000000 --- a/docs/root/configuration/xds_subscription_stats.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. _subscription_statistics: - -xDS subscription statistics -=========================== - -Envoy discovers its various dynamic resources via discovery -services referred to as *xDS*. Resources are requested via :ref:`subscriptions `, -by specifying a filesystem path to watch, initiating gRPC streams or polling a REST-JSON URL. - -The following statistics are generated for all subscriptions. - -.. csv-table:: - :header: Name, Type, Description - :widths: 1, 1, 2 - - config_reload, Counter, Total API fetches that resulted in a config reload due to a different config - init_fetch_timeout, Counter, Total :ref:`initial fetch timeouts ` - update_attempt, Counter, Total API fetches attempted - update_success, Counter, Total API fetches completed successfully - update_failure, Counter, Total API fetches that failed because of network errors - update_rejected, Counter, Total API fetches that failed because of schema/validation errors - version, Gauge, Hash of the contents from the last successful API fetch - control_plane.connected_state, Gauge, A boolean (1 for connected and 0 for disconnected) that indicates the current connection state with management server From 903351fb41e1d6219f16be39abf71c082de52153 Mon Sep 17 00:00:00 2001 From: htuch Date: Fri, 23 Aug 2019 19:28:04 -0400 Subject: [PATCH 449/542] build: missing regex include. (#8032) Signed-off-by: Harvey Tuch --- test/test_runner.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_runner.h b/test/test_runner.h index 26e373ddd324..8f70da98a945 100644 --- a/test/test_runner.h +++ b/test/test_runner.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "common/common/logger.h" #include "common/common/logger_delegates.h" #include "common/common/thread.h" From 6ff0bce8ff417a252cde4d04dfb9cba2bab463d8 Mon Sep 17 00:00:00 2001 From: asraa Date: Sat, 24 Aug 2019 20:07:41 -0400 Subject: [PATCH 450/542] [headermap] speedup for appending data (#8029) For debug builds, performance testing and fuzzers reveal that when appending to a header, we scan both the existing value and the data to append for invalid characters. This PR moves the validation check to just the data that is appended, to avoid hangups on re-scanning long header values multiple times. Testing: Added corpus entry that reveals time spent in validHeaderString Signed-off-by: Asra Ali --- source/common/http/header_map_impl.cc | 3 +- .../http/header_map_impl_corpus/appendheader | 5377 +++++++++++++++++ 2 files changed, 5378 insertions(+), 2 deletions(-) create mode 100644 test/common/http/header_map_impl_corpus/appendheader diff --git a/source/common/http/header_map_impl.cc b/source/common/http/header_map_impl.cc index ff018b77f9c6..7472b51e75aa 100644 --- a/source/common/http/header_map_impl.cc +++ b/source/common/http/header_map_impl.cc @@ -145,10 +145,9 @@ void HeaderString::append(const char* data, uint32_t size) { } } } - + ASSERT(validHeaderString(absl::string_view(data, size))); memcpy(buffer_.dynamic_ + string_length_, data, size); string_length_ += size; - ASSERT(valid()); } void HeaderString::clear() { diff --git a/test/common/http/header_map_impl_corpus/appendheader b/test/common/http/header_map_impl_corpus/appendheader new file mode 100644 index 000000000000..bb772dcb6ef8 --- /dev/null +++ b/test/common/http/header_map_impl_corpus/appendheader @@ -0,0 +1,5377 @@ +actions { + set_reference_key { + key: ":method" + value: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + set_reference_key { + key: ":method" + value: "baz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + set_reference_key { + key: ":method" + value: "baz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + set_reference_key { + key: ":method" + value: "baz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} +actions { + get_and_mutate { + key: ":method" + append: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + } +} + From 4d78ff50a63d9d9b648bfd4f1e070a9dee86dbc0 Mon Sep 17 00:00:00 2001 From: l8huang Date: Sun, 25 Aug 2019 14:36:35 -0700 Subject: [PATCH 451/542] eds: avoid send too many ClusterLoadAssignment requests (#7976) During initializing secondary clusters, for each initialized cluster, a ClusterLoadAssignment request is sent to istio pilot with the cluster's name appended into request's resource_names list. With a huge number of clusters(e.g 10k clusters), this behavior slows down Envoy's initialization and consumes ton of memory. This change pauses ADS mux for ClusterLoadAssignment to avoid that. Risk Level: Medium Testing: tiny change, no test case added Fixes #7955 Signed-off-by: lhuang8 --- source/common/config/grpc_mux_impl.h | 2 +- source/common/upstream/cds_api_impl.cc | 3 + .../common/upstream/cluster_manager_impl.cc | 36 +++++++---- source/common/upstream/cluster_manager_impl.h | 7 ++- .../upstream/cluster_manager_impl_test.cc | 59 ++++++++++++++++++- 5 files changed, 92 insertions(+), 15 deletions(-) diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index d1aa49459053..ae245c48d7fd 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -125,7 +125,7 @@ class NullGrpcMuxImpl : public GrpcMux { } void pause(const std::string&) override {} void resume(const std::string&) override {} - bool paused(const std::string&) const override { NOT_REACHED_GCOVR_EXCL_LINE; } + bool paused(const std::string&) const override { return false; } }; } // namespace Config diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 3ec334cf9b52..41f7ec4faca6 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -59,6 +59,9 @@ void CdsApiImpl::onConfigUpdate( cm_.adsMux().pause(Config::TypeUrl::get().ClusterLoadAssignment); Cleanup eds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + ENVOY_LOG(info, "cds: add {} cluster(s), remove {} cluster(s)", added_resources.size(), + removed_resources.size()); + std::vector exception_msgs; std::unordered_set cluster_names; bool any_applied = false; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 1dd78f2dabfa..cad4bd02d679 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -103,6 +103,19 @@ void ClusterManagerInitHelper::removeCluster(Cluster& cluster) { maybeFinishInitialize(); } +void ClusterManagerInitHelper::initializeSecondaryClusters() { + started_secondary_initialize_ = true; + // Cluster::initialize() method can modify the list of secondary_init_clusters_ to remove + // the item currently being initialized, so we eschew range-based-for and do this complicated + // dance to increment the iterator before calling initialize. + for (auto iter = secondary_init_clusters_.begin(); iter != secondary_init_clusters_.end();) { + Cluster* cluster = *iter; + ++iter; + ENVOY_LOG(debug, "initializing secondary cluster {}", cluster->info()->name()); + cluster->initialize([cluster, this] { onClusterInit(*cluster); }); + } +} + void ClusterManagerInitHelper::maybeFinishInitialize() { // Do not do anything if we are still doing the initial static load or if we are waiting for // CDS initialize. @@ -121,15 +134,16 @@ void ClusterManagerInitHelper::maybeFinishInitialize() { if (!secondary_init_clusters_.empty()) { if (!started_secondary_initialize_) { ENVOY_LOG(info, "cm init: initializing secondary clusters"); - started_secondary_initialize_ = true; - // Cluster::initialize() method can modify the list of secondary_init_clusters_ to remove - // the item currently being initialized, so we eschew range-based-for and do this complicated - // dance to increment the iterator before calling initialize. - for (auto iter = secondary_init_clusters_.begin(); iter != secondary_init_clusters_.end();) { - Cluster* cluster = *iter; - ++iter; - ENVOY_LOG(debug, "initializing secondary cluster {}", cluster->info()->name()); - cluster->initialize([cluster, this] { onClusterInit(*cluster); }); + // If the first CDS response doesn't have any primary cluster, ClusterLoadAssignment + // should be already paused by CdsApiImpl::onConfigUpdate(). Need to check that to + // avoid double pause ClusterLoadAssignment. + if (cm_.adsMux().paused(Config::TypeUrl::get().ClusterLoadAssignment)) { + initializeSecondaryClusters(); + } else { + cm_.adsMux().pause(Config::TypeUrl::get().ClusterLoadAssignment); + Cleanup eds_resume( + [this] { cm_.adsMux().resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + initializeSecondaryClusters(); } } @@ -188,7 +202,7 @@ ClusterManagerImpl::ClusterManagerImpl( : factory_(factory), runtime_(runtime), stats_(stats), tls_(tls.allocateSlot()), random_(random), bind_config_(bootstrap.cluster_manager().upstream_bind_config()), local_info_(local_info), cm_stats_(generateStats(stats)), - init_helper_([this](Cluster& cluster) { onClusterInit(cluster); }), + init_helper_(*this, [this](Cluster& cluster) { onClusterInit(cluster); }), config_tracker_entry_( admin.getConfigTracker().add("clusters", [this] { return dumpClusterConfigs(); })), time_source_(main_thread_dispatcher.timeSource()), dispatcher_(main_thread_dispatcher), @@ -496,7 +510,7 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::api::v2::Cluster& clust loadCluster(cluster, version_info, true, use_active_map ? active_clusters_ : warming_clusters_); if (use_active_map) { - ENVOY_LOG(info, "add/update cluster {} during init", cluster_name); + ENVOY_LOG(debug, "add/update cluster {} during init", cluster_name); auto& cluster_entry = active_clusters_.at(cluster_name); createOrUpdateThreadLocalCluster(*cluster_entry); init_helper_.addCluster(*cluster_entry->cluster_); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 6b8f205ed9b2..e45e9752372f 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -100,8 +100,9 @@ class ClusterManagerInitHelper : Logger::Loggable { * @param per_cluster_init_callback supplies the callback to call when a cluster has itself * initialized. The cluster manager can use this for post-init processing. */ - ClusterManagerInitHelper(const std::function& per_cluster_init_callback) - : per_cluster_init_callback_(per_cluster_init_callback) {} + ClusterManagerInitHelper(ClusterManager& cm, + const std::function& per_cluster_init_callback) + : cm_(cm), per_cluster_init_callback_(per_cluster_init_callback) {} enum class State { // Initial state. During this state all static clusters are loaded. Any phase 1 clusters @@ -128,9 +129,11 @@ class ClusterManagerInitHelper : Logger::Loggable { State state() const { return state_; } private: + void initializeSecondaryClusters(); void maybeFinishInitialize(); void onClusterInit(Cluster& cluster); + ClusterManager& cm_; std::function per_cluster_init_callback_; CdsApi* cds_{}; std::function initialized_callback_; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index d8a85b0e13a2..f9632b12da76 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -42,6 +42,8 @@ #include "gtest/gtest.h" using testing::_; +using testing::ByRef; +using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::Mock; @@ -2752,7 +2754,8 @@ class ClusterManagerInitHelperTest : public testing::Test { public: MOCK_METHOD1(onClusterInit, void(Cluster& cluster)); - ClusterManagerInitHelper init_helper_{[this](Cluster& cluster) { onClusterInit(cluster); }}; + NiceMock cm_; + ClusterManagerInitHelper init_helper_{cm_, [this](Cluster& cluster) { onClusterInit(cluster); }}; }; TEST_F(ClusterManagerInitHelperTest, ImmediateInitialize) { @@ -2824,6 +2827,60 @@ TEST_F(ClusterManagerInitHelperTest, UpdateAlreadyInitialized) { cluster2.initialize_callback_(); } +// If secondary clusters initialization triggered outside of CdsApiImpl::onConfigUpdate()'s +// callback flows, sending ClusterLoadAssignment should not be paused before calling +// ClusterManagerInitHelper::maybeFinishInitialize(). This case tests that +// ClusterLoadAssignment request is paused and resumed properly. +TEST_F(ClusterManagerInitHelperTest, InitSecondaryWithoutEdsPaused) { + InSequence s; + + ReadyWatcher cm_initialized; + init_helper_.setInitializedCb([&]() -> void { cm_initialized.ready(); }); + + NiceMock cluster1; + ON_CALL(cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); + init_helper_.addCluster(cluster1); + + const auto& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + EXPECT_CALL(cm_.ads_mux_, paused(Eq(ByRef(type_url)))).WillRepeatedly(Return(false)); + EXPECT_CALL(cm_.ads_mux_, pause(Eq(ByRef(type_url)))); + EXPECT_CALL(cluster1, initialize(_)); + EXPECT_CALL(cm_.ads_mux_, resume(Eq(ByRef(type_url)))); + + init_helper_.onStaticLoadComplete(); + + EXPECT_CALL(*this, onClusterInit(Ref(cluster1))); + EXPECT_CALL(cm_initialized, ready()); + cluster1.initialize_callback_(); +} + +// If secondary clusters initialization triggered inside of CdsApiImpl::onConfigUpdate()'s +// callback flows, that's, the CDS response didn't have any primary cluster, sending +// ClusterLoadAssignment should be already paused by CdsApiImpl::onConfigUpdate(). +// This case tests that ClusterLoadAssignment request isn't paused again. +TEST_F(ClusterManagerInitHelperTest, InitSecondaryWithEdsPaused) { + InSequence s; + + ReadyWatcher cm_initialized; + init_helper_.setInitializedCb([&]() -> void { cm_initialized.ready(); }); + + NiceMock cluster1; + ON_CALL(cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); + init_helper_.addCluster(cluster1); + + const auto& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + EXPECT_CALL(cm_.ads_mux_, paused(Eq(ByRef(type_url)))).WillRepeatedly(Return(true)); + EXPECT_CALL(cm_.ads_mux_, pause(Eq(ByRef(type_url)))).Times(0); + EXPECT_CALL(cluster1, initialize(_)); + EXPECT_CALL(cm_.ads_mux_, resume(Eq(ByRef(type_url)))).Times(0); + + init_helper_.onStaticLoadComplete(); + + EXPECT_CALL(*this, onClusterInit(Ref(cluster1))); + EXPECT_CALL(cm_initialized, ready()); + cluster1.initialize_callback_(); +} + TEST_F(ClusterManagerInitHelperTest, AddSecondaryAfterSecondaryInit) { InSequence s; From b0aca3045192bc88e7f0b5f94f2fb310285d1bfc Mon Sep 17 00:00:00 2001 From: Yikun Jiang Date: Mon, 26 Aug 2019 10:42:27 -0500 Subject: [PATCH 452/542] Set the bazel verison to 0.28.1 explicitly (#8037) In https://github.com/theopenlab/openlab-zuul-jobs/pull/622 , the OpenLab add the ability to set the bazel to specific version explicitly. This patch add the bazel role for the envoy job. Signed-off-by: Yikun Jiang --- .zuul/playbooks/envoy-build/run.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.zuul/playbooks/envoy-build/run.yaml b/.zuul/playbooks/envoy-build/run.yaml index 1ca8f832e08a..0087029e4a4f 100644 --- a/.zuul/playbooks/envoy-build/run.yaml +++ b/.zuul/playbooks/envoy-build/run.yaml @@ -3,6 +3,8 @@ roles: - role: config-gcc gcc_version: 7 + - role: config-bazel + bazel_version: 0.28.1 tasks: - name: Build envoy shell: From fc32b645a60a1cf70f889702cd8f24576be6db67 Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Mon, 26 Aug 2019 09:10:53 -0700 Subject: [PATCH 453/542] Read_policy is not set correctly. (#8034) Add more integration test and additional checks in the unit tests. Signed-off-by: Henry Yang --- .../network/common/redis/client_impl.cc | 6 +-- .../network/redis_proxy/conn_pool_impl.cc | 4 +- .../redis/redis_cluster_integration_test.cc | 49 +++++++++++++++++-- .../redis_proxy/conn_pool_impl_test.cc | 18 +++++-- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index a2610417a873..6f941d223ca0 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -31,14 +31,14 @@ ConfigImpl::ConfigImpl( break; case envoy::config::filter::network::redis_proxy::v2:: RedisProxy_ConnPoolSettings_ReadPolicy_REPLICA: - read_policy_ = ReadPolicy::PreferMaster; + read_policy_ = ReadPolicy::Replica; break; case envoy::config::filter::network::redis_proxy::v2:: RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_REPLICA: - read_policy_ = ReadPolicy::PreferMaster; + read_policy_ = ReadPolicy::PreferReplica; break; case envoy::config::filter::network::redis_proxy::v2::RedisProxy_ConnPoolSettings_ReadPolicy_ANY: - read_policy_ = ReadPolicy::PreferMaster; + read_policy_ = ReadPolicy::Any; break; default: NOT_REACHED_GCOVR_EXCL_LINE; diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index 2e1414e0252c..a097924ea734 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -225,8 +225,8 @@ InstanceImpl::ThreadLocalPool::makeRequest(const std::string& key, } const bool use_crc16 = is_redis_cluster_; - Clusters::Redis::RedisLoadBalancerContextImpl lb_context(key, parent_.config_.enableHashtagging(), - use_crc16, request); + Clusters::Redis::RedisLoadBalancerContextImpl lb_context( + key, parent_.config_.enableHashtagging(), use_crc16, request, parent_.config_.readPolicy()); Upstream::HostConstSharedPtr host = cluster_->loadBalancer().chooseHost(&lb_context); if (!host) { return nullptr; diff --git a/test/extensions/clusters/redis/redis_cluster_integration_test.cc b/test/extensions/clusters/redis/redis_cluster_integration_test.cc index 000928ef2441..bb1bfbd53d98 100644 --- a/test/extensions/clusters/redis/redis_cluster_integration_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_integration_test.cc @@ -15,8 +15,7 @@ namespace { // This is a basic redis_proxy configuration with a single host // in the cluster. The load balancing policy must be set // to random for proper test operation. - -const std::string& testConfig() { +const std::string& listenerConfig() { CONSTRUCT_ON_FIRST_USE(std::string, R"EOF( admin: access_log_path: /dev/null @@ -40,7 +39,11 @@ const std::string& testConfig() { catch_all_route: cluster: cluster_0 settings: - op_timeout: 5s + op_timeout: 5s)EOF"); +} + +const std::string& clusterConfig() { + CONSTRUCT_ON_FIRST_USE(std::string, R"EOF( clusters: - name: cluster_0 lb_policy: CLUSTER_PROVIDED @@ -58,6 +61,16 @@ const std::string& testConfig() { )EOF"); } +const std::string& testConfig() { + CONSTRUCT_ON_FIRST_USE(std::string, listenerConfig() + clusterConfig()); +} + +const std::string& testConfigWithReadPolicy() { + CONSTRUCT_ON_FIRST_USE(std::string, listenerConfig() + R"EOF( + read_policy: REPLICA +)EOF" + clusterConfig()); +} + // This is the basic redis_proxy configuration with an upstream // authentication password specified. @@ -278,6 +291,13 @@ class RedisClusterWithAuthIntegrationTest : public RedisClusterIntegrationTest { : RedisClusterIntegrationTest(config, num_upstreams) {} }; +class RedisClusterWithReadPolicyIntegrationTest : public RedisClusterIntegrationTest { +public: + RedisClusterWithReadPolicyIntegrationTest(const std::string& config = testConfigWithReadPolicy(), + int num_upstreams = 3) + : RedisClusterIntegrationTest(config, num_upstreams) {} +}; + INSTANTIATE_TEST_SUITE_P(IpVersions, RedisClusterIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), TestUtility::ipTestParamsToString); @@ -333,6 +353,29 @@ TEST_P(RedisClusterIntegrationTest, TwoSlot) { simpleRequestAndResponse(1, makeBulkStringArray({"get", "foo"}), "$3\r\nbar\r\n"); } +// This test sends simple "set foo" and "get foo" command from a fake +// downstream client through the proxy to a fake upstream +// Redis cluster with a single slot with master and replica. +// The envoy proxy is set with read_policy to read from replica, the expected result +// is that the set command will be sent to the master and the get command will be sent +// to the replica + +TEST_P(RedisClusterWithReadPolicyIntegrationTest, SingleSlotMasterReplicaReadReplica) { + random_index_ = 0; + + on_server_init_function_ = [this]() { + std::string cluster_slot_response = singleSlotMasterReplica( + fake_upstreams_[0]->localAddress()->ip(), fake_upstreams_[1]->localAddress()->ip()); + expectCallClusterSlot(random_index_, cluster_slot_response); + }; + + initialize(); + + // foo hashes to slot 12182 which has master node in upstream 0 and replica in upstream 1 + simpleRequestAndResponse(0, makeBulkStringArray({"set", "foo", "bar"}), ":1\r\n"); + simpleRequestAndResponse(1, makeBulkStringArray({"get", "foo"}), "$3\r\nbar\r\n"); +} + // This test sends a simple "get foo" command from a fake // downstream client through the proxy to a fake upstream // Redis cluster with a single slot with master and replica. diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 22161d0e00fb..9dd249ec6002 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -161,7 +161,8 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client void testReadPolicy( envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings::ReadPolicy - read_policy) { + read_policy, + NetworkFilters::Common::Redis::Client::ReadPolicy expected_read_policy) { InSequence s; read_policy_ = read_policy; @@ -178,6 +179,9 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client EXPECT_EQ(context->computeHashKey().value(), MurmurHash::murmurHash2_64("hash_key")); EXPECT_EQ(context->metadataMatchCriteria(), nullptr); EXPECT_EQ(context->downstreamConnection(), nullptr); + auto redis_context = + dynamic_cast(context); + EXPECT_EQ(redis_context->readPolicy(), expected_read_policy); return cm_.thread_local_cluster_.lb_.host_; })); EXPECT_CALL(*this, create_(_)).WillOnce(Return(client)); @@ -279,13 +283,17 @@ TEST_F(RedisConnPoolImplTest, BasicWithAuthPassword) { TEST_F(RedisConnPoolImplTest, BasicWithReadPolicy) { testReadPolicy(envoy::config::filter::network::redis_proxy::v2:: - RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_MASTER); + RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_MASTER, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferMaster); testReadPolicy(envoy::config::filter::network::redis_proxy::v2:: - RedisProxy_ConnPoolSettings_ReadPolicy_REPLICA); + RedisProxy_ConnPoolSettings_ReadPolicy_REPLICA, + NetworkFilters::Common::Redis::Client::ReadPolicy::Replica); testReadPolicy(envoy::config::filter::network::redis_proxy::v2:: - RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_REPLICA); + RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_REPLICA, + NetworkFilters::Common::Redis::Client::ReadPolicy::PreferReplica); testReadPolicy( - envoy::config::filter::network::redis_proxy::v2::RedisProxy_ConnPoolSettings_ReadPolicy_ANY); + envoy::config::filter::network::redis_proxy::v2::RedisProxy_ConnPoolSettings_ReadPolicy_ANY, + NetworkFilters::Common::Redis::Client::ReadPolicy::Any); }; TEST_F(RedisConnPoolImplTest, Hashtagging) { From 5c2b34bdefb8ef6d3bd8426d289128847c0768b2 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Mon, 26 Aug 2019 11:03:45 -0700 Subject: [PATCH 454/542] admin: fix /server_info hot restart version (#8022) Signed-off-by: Matt Klein --- api/envoy/admin/v2alpha/server_info.proto | 6 ++++-- source/server/http/admin.cc | 1 + test/server/http/admin_test.cc | 2 ++ test/server/options_impl_test.cc | 5 +++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/api/envoy/admin/v2alpha/server_info.proto b/api/envoy/admin/v2alpha/server_info.proto index 7389af5b0860..78cc6fa7020a 100644 --- a/api/envoy/admin/v2alpha/server_info.proto +++ b/api/envoy/admin/v2alpha/server_info.proto @@ -36,6 +36,9 @@ message ServerInfo { // Uptime since the start of the first epoch. google.protobuf.Duration uptime_all_epochs = 4; + // Hot restart version. + string hot_restart_version = 5; + // Command line options the server is currently running with. CommandLineOptions command_line_options = 6; } @@ -82,8 +85,7 @@ message CommandLineOptions { // See :option:`--log-path` for details. string log_path = 11; - // See :option:`--hot-restart-version` for details. - bool hot_restart_version = 12; + reserved 12; // See :option:`--service-cluster` for details. string service_cluster = 13; diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 5faacc1eb76d..71fd23d1118c 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -719,6 +719,7 @@ Http::Code AdminImpl::handlerServerInfo(absl::string_view, Http::HeaderMap& head time_t current_time = time(nullptr); envoy::admin::v2alpha::ServerInfo server_info; server_info.set_version(VersionInfo::version()); + server_info.set_hot_restart_version(server_.hotRestart().version()); server_info.set_state( Utility::serverState(server_.initManager().state(), server_.healthCheckFailed())); diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index 2c9e933ed881..24dbed7b348e 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -1239,6 +1239,7 @@ TEST_P(AdminInstanceTest, GetRequest) { })); NiceMock initManager; ON_CALL(server_, initManager()).WillByDefault(ReturnRef(initManager)); + ON_CALL(server_.hot_restart_, version()).WillByDefault(Return("foo_version")); { Http::HeaderMapImpl response_headers; @@ -1254,6 +1255,7 @@ TEST_P(AdminInstanceTest, GetRequest) { // values such as timestamps + Envoy version are tricky to test for. TestUtility::loadFromJson(body, server_info_proto); EXPECT_EQ(server_info_proto.state(), envoy::admin::v2alpha::ServerInfo::LIVE); + EXPECT_EQ(server_info_proto.hot_restart_version(), "foo_version"); EXPECT_EQ(server_info_proto.command_line_options().restart_epoch(), 2); EXPECT_EQ(server_info_proto.command_line_options().service_cluster(), "cluster"); } diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index bc4cc16f7fa7..4372585c4b48 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -245,14 +245,15 @@ TEST_F(OptionsImplTest, OptionsAreInSyncWithProto) { Server::CommandLineOptionsPtr command_line_options = options->toCommandLineOptions(); // Failure of this condition indicates that the server_info proto is not in sync with the options. // If an option is added/removed, please update server_info proto as well to keep it in sync. - // Currently the following 5 options are not defined in proto, hence the count differs by 5. + // Currently the following 7 options are not defined in proto, hence the count differs by 7. // 1. version - default TCLAP argument. // 2. help - default TCLAP argument. // 3. ignore_rest - default TCLAP argument. // 4. use-libevent-buffers - short-term override for rollout of new buffer implementation. // 5. allow-unknown-fields - deprecated alias of allow-unknown-static-fields. // 6. use-fake-symbol-table - short-term override for rollout of real symbol-table implementation. - EXPECT_EQ(options->count() - 6, command_line_options->GetDescriptor()->field_count()); + // 7. hot restart version - print the hot restart version and exit. + EXPECT_EQ(options->count() - 7, command_line_options->GetDescriptor()->field_count()); } TEST_F(OptionsImplTest, BadCliOption) { From fdd0e016301c07f2254edbbbabceecbd57ddb6a2 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 26 Aug 2019 14:12:01 -0400 Subject: [PATCH 455/542] test: adding debug hints for integration test config failures (#8038) Risk Level: n/a (test only) Testing: manual Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- test/integration/integration.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/integration.cc b/test/integration/integration.cc index f8fe1910abf1..595c78f1e24b 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -442,7 +442,8 @@ void BaseIntegrationTest::createGeneratedApiTestServer(const std::string& bootst if (!allow_lds_rejection) { RELEASE_ASSERT(test_server_->counter(rejected) == nullptr || test_server_->counter(rejected)->value() == 0, - "Lds update failed"); + "Lds update failed. For details, run test with -l trace and look for " + "\"Error adding/updating listener(s)\" in the logs."); } time_system_.sleep(std::chrono::milliseconds(10)); } From d39bd813a156989369597d69efd4e50489cfeebb Mon Sep 17 00:00:00 2001 From: danzh Date: Mon, 26 Aug 2019 18:10:46 -0400 Subject: [PATCH 456/542] udp_listener: refactor ActiveUdpListener creation (#7884) Signed-off-by: Dan Zhang --- api/envoy/api/v2/BUILD | 2 + api/envoy/api/v2/lds.proto | 11 +- api/envoy/api/v2/listener/BUILD | 17 +++ .../api/v2/listener/udp_listener_config.proto | 31 ++++++ docs/build.sh | 1 + docs/root/api-v2/listeners/listeners.rst | 1 + include/envoy/network/connection_handler.h | 50 +++++++++ include/envoy/network/listener.h | 7 ++ include/envoy/server/BUILD | 6 + .../envoy/server/active_udp_listener_config.h | 29 +++++ source/server/BUILD | 21 ++++ .../server/active_raw_udp_listener_config.cc | 26 +++++ .../server/active_raw_udp_listener_config.h | 31 ++++++ source/server/connection_handler_impl.cc | 91 +++++++--------- source/server/connection_handler_impl.h | 103 +++++++++--------- source/server/http/admin.h | 3 + source/server/listener_manager_impl.cc | 12 ++ source/server/listener_manager_impl.h | 4 + source/server/well_known_names.h | 21 ++++ .../proxy_protocol/proxy_protocol_test.cc | 2 + test/integration/BUILD | 1 + test/integration/fake_upstream.h | 4 + test/mocks/network/mocks.h | 1 + test/server/BUILD | 2 + test/server/connection_handler_test.cc | 11 ++ 25 files changed, 386 insertions(+), 102 deletions(-) create mode 100644 api/envoy/api/v2/listener/udp_listener_config.proto create mode 100644 include/envoy/server/active_udp_listener_config.h create mode 100644 source/server/active_raw_udp_listener_config.cc create mode 100644 source/server/active_raw_udp_listener_config.h create mode 100644 source/server/well_known_names.h diff --git a/api/envoy/api/v2/BUILD b/api/envoy/api/v2/BUILD index 48ddb2e13025..b86ca2a788bf 100644 --- a/api/envoy/api/v2/BUILD +++ b/api/envoy/api/v2/BUILD @@ -109,6 +109,7 @@ api_proto_library_internal( "//envoy/api/v2/core:address", "//envoy/api/v2/core:base", "//envoy/api/v2/listener", + "//envoy/api/v2/listener:udp_listener_config", ], ) @@ -120,6 +121,7 @@ api_go_grpc_library( "//envoy/api/v2/core:address_go_proto", "//envoy/api/v2/core:base_go_proto", "//envoy/api/v2/listener:listener_go_proto", + "//envoy/api/v2/listener:udp_listener_config_go_proto", ], ) diff --git a/api/envoy/api/v2/lds.proto b/api/envoy/api/v2/lds.proto index 195401341c96..643ac146213b 100644 --- a/api/envoy/api/v2/lds.proto +++ b/api/envoy/api/v2/lds.proto @@ -12,6 +12,7 @@ import "envoy/api/v2/core/address.proto"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/discovery.proto"; import "envoy/api/v2/listener/listener.proto"; +import "envoy/api/v2/listener/udp_listener_config.proto"; import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; @@ -44,7 +45,7 @@ service ListenerDiscoveryService { } } -// [#comment:next free field: 18] +// [#comment:next free field: 19] message Listener { // The unique name by which this listener is known. If no name is provided, // Envoy will allocate an internal UUID for the listener. If the listener is to be dynamically @@ -194,4 +195,12 @@ message Listener { // Specifies the intended direction of the traffic relative to the local Envoy. core.TrafficDirection traffic_direction = 16; + + // If the protocol in the listener socket address in :ref:`protocol + // ` is :ref:'UDP + // `, this field specifies the actual udp listener to create, + // i.e. :ref:`udp_listener_name + // ` = "raw_udp_listener" for + // creating a packet-oriented UDP listener. If not present, treat it as "raw_udp_listener". + listener.UdpListenerConfig udp_listener_config = 18; } diff --git a/api/envoy/api/v2/listener/BUILD b/api/envoy/api/v2/listener/BUILD index 9eb0c0ec982f..e539c4b8c090 100644 --- a/api/envoy/api/v2/listener/BUILD +++ b/api/envoy/api/v2/listener/BUILD @@ -22,3 +22,20 @@ api_go_proto_library( "//envoy/api/v2/core:base_go_proto", ], ) + +api_proto_library_internal( + name = "udp_listener_config", + srcs = ["udp_listener_config.proto"], + visibility = ["//envoy/api/v2:friends"], + deps = [ + "//envoy/api/v2/core:base", + ], +) + +api_go_proto_library( + name = "udp_listener_config", + proto = ":udp_listener_config", + deps = [ + "//envoy/api/v2/core:base_go_proto", + ], +) diff --git a/api/envoy/api/v2/listener/udp_listener_config.proto b/api/envoy/api/v2/listener/udp_listener_config.proto new file mode 100644 index 000000000000..88a2a35d3cfc --- /dev/null +++ b/api/envoy/api/v2/listener/udp_listener_config.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package envoy.api.v2.listener; + +option java_outer_classname = "ListenerProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v2.listener"; +option go_package = "listener"; +option csharp_namespace = "Envoy.Api.V2.ListenerNS"; +option ruby_package = "Envoy::Api::V2::ListenerNS"; + +import "google/protobuf/struct.proto"; +import "google/protobuf/any.proto"; + +// [#protodoc-title: Udp Listener Config] +// Listener :ref:`configuration overview ` + +message UdpListenerConfig { + // Used to look up UDP listener factory, matches "raw_udp_listener" or + // "quic_listener" to create a specific udp listener. + // If not specified, treat as "raw_udp_listener". + string udp_listener_name = 1; + + // Used to create a specific listener factory. To some factory, e.g. + // "raw_udp_listener", config is not needed. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} diff --git a/docs/build.sh b/docs/build.sh index 7b89527bb24c..1b8cac8b06fa 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -82,6 +82,7 @@ PROTO_RST=" /envoy/api/v2/srds/envoy/api/v2/srds.proto.rst /envoy/api/v2/lds/envoy/api/v2/lds.proto.rst /envoy/api/v2/listener/listener/envoy/api/v2/listener/listener.proto.rst + /envoy/api/v2/listener/udp_listener_config/envoy/api/v2/listener/udp_listener_config.proto.rst /envoy/api/v2/ratelimit/ratelimit/envoy/api/v2/ratelimit/ratelimit.proto.rst /envoy/config/accesslog/v2/als/envoy/config/accesslog/v2/als.proto.rst /envoy/config/accesslog/v2/file/envoy/config/accesslog/v2/file.proto.rst diff --git a/docs/root/api-v2/listeners/listeners.rst b/docs/root/api-v2/listeners/listeners.rst index d933ccd32d66..6ed0279da7de 100644 --- a/docs/root/api-v2/listeners/listeners.rst +++ b/docs/root/api-v2/listeners/listeners.rst @@ -7,3 +7,4 @@ Listeners ../api/v2/lds.proto ../api/v2/listener/listener.proto + ../api/v2/listener/udp_listener_config.proto diff --git a/include/envoy/network/connection_handler.h b/include/envoy/network/connection_handler.h index 9a36aed1cc49..6c24a814db5f 100644 --- a/include/envoy/network/connection_handler.h +++ b/include/envoy/network/connection_handler.h @@ -68,9 +68,59 @@ class ConnectionHandler { * after they have been temporarily disabled. */ virtual void enableListeners() PURE; + + /** + * Used by ConnectionHandler to manage listeners. + */ + class ActiveListener { + public: + virtual ~ActiveListener() = default; + + /** + * @return the tag value as configured. + */ + virtual uint64_t listenerTag() PURE; + /** + * @return the actual Listener object. + */ + virtual Listener* listener() PURE; + /** + * Destroy the actual Listener it wraps. + */ + virtual void destroy() PURE; + }; + + using ActiveListenerPtr = std::unique_ptr; }; using ConnectionHandlerPtr = std::unique_ptr; +/** + * A registered factory interface to create different kinds of + * ActiveUdpListener. + */ +class ActiveUdpListenerFactory { +public: + virtual ~ActiveUdpListenerFactory() = default; + + /** + * Creates an ActiveUdpListener object and a corresponding UdpListener + * according to given config. + * @param parent is the owner of the created ActiveListener objects. + * @param dispatcher is used to create actual UDP listener. + * @param logger might not need to be passed in. + * TODO(danzh): investigate if possible to use statically defined logger in ActiveUdpListener + * implementation instead. + * @param config provides information needed to create ActiveUdpListener and + * UdpListener objects. + * @return the ActiveUdpListener created. + */ + virtual ConnectionHandler::ActiveListenerPtr + createActiveUdpListener(ConnectionHandler& parent, Event::Dispatcher& disptacher, + spdlog::logger& logger, Network::ListenerConfig& config) const PURE; +}; + +using ActiveUdpListenerFactoryPtr = std::unique_ptr; + } // namespace Network } // namespace Envoy diff --git a/include/envoy/network/listener.h b/include/envoy/network/listener.h index 53b06b01be65..451a76508581 100644 --- a/include/envoy/network/listener.h +++ b/include/envoy/network/listener.h @@ -14,6 +14,7 @@ namespace Envoy { namespace Network { class UdpListenerFilterManager; +class ActiveUdpListenerFactory; /** * A configuration for an individual listener. @@ -90,6 +91,12 @@ class ListenerConfig { * @return const std::string& the listener's name. */ virtual const std::string& name() const PURE; + + /** + * @return factory pointer if listening on UDP socket, otherwise return + * nullptr. + */ + virtual const ActiveUdpListenerFactory* udpListenerFactory() PURE; }; /** diff --git a/include/envoy/server/BUILD b/include/envoy/server/BUILD index 591868dde496..db070132fbcf 100644 --- a/include/envoy/server/BUILD +++ b/include/envoy/server/BUILD @@ -256,3 +256,9 @@ envoy_cc_library( "@envoy_api//envoy/config/trace/v2:trace_cc", ], ) + +envoy_cc_library( + name = "active_udp_listener_config_interface", + hdrs = ["active_udp_listener_config.h"], + deps = ["//include/envoy/network:connection_handler_interface"], +) diff --git a/include/envoy/server/active_udp_listener_config.h b/include/envoy/server/active_udp_listener_config.h new file mode 100644 index 000000000000..810d25add389 --- /dev/null +++ b/include/envoy/server/active_udp_listener_config.h @@ -0,0 +1,29 @@ +#pragma once + +#include "envoy/network/connection_handler.h" + +namespace Envoy { +namespace Server { + +/** + * Interface to create udp listener according to + * envoy::api::v2::listener::UdpListenerConfig.udp_listener_name. + */ +class ActiveUdpListenerConfigFactory { +public: + virtual ~ActiveUdpListenerConfigFactory() = default; + + /** + * Create an ActiveUdpListenerFactory object according to given message. + */ + virtual Network::ActiveUdpListenerFactoryPtr + createActiveUdpListenerFactory(const Protobuf::Message& message) PURE; + + /** + * Used to identify which udp listener to create: quic or raw udp. + */ + virtual std::string name() PURE; +}; + +} // namespace Server +} // namespace Envoy diff --git a/source/server/BUILD b/source/server/BUILD index bd9c72b56e63..5750846d1dca 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -64,6 +64,7 @@ envoy_cc_library( "//include/envoy/network:filter_interface", "//include/envoy/network:listen_socket_interface", "//include/envoy/network:listener_interface", + "//include/envoy/server:active_udp_listener_config_interface", "//include/envoy/server:listener_manager_interface", "//include/envoy/stats:timespan", "//source/common/common:linked_object", @@ -260,6 +261,8 @@ envoy_cc_library( ":filter_chain_manager_lib", ":lds_api_lib", ":transport_socket_config_lib", + ":well_known_names_lib", + "//include/envoy/server:active_udp_listener_config_interface", "//include/envoy/server:filter_config_interface", "//include/envoy/server:listener_manager_interface", "//include/envoy/server:transport_socket_config_interface", @@ -441,3 +444,21 @@ envoy_cc_library( "//include/envoy/server:transport_socket_config_interface", ], ) + +envoy_cc_library( + name = "well_known_names_lib", + hdrs = ["well_known_names.h"], + deps = ["//source/common/singleton:const_singleton"], +) + +envoy_cc_library( + name = "active_raw_udp_listener_config", + srcs = ["active_raw_udp_listener_config.cc"], + hdrs = ["active_raw_udp_listener_config.h"], + deps = [ + ":connection_handler_lib", + ":well_known_names_lib", + "//include/envoy/registry", + "//include/envoy/server:active_udp_listener_config_interface", + ], +) diff --git a/source/server/active_raw_udp_listener_config.cc b/source/server/active_raw_udp_listener_config.cc new file mode 100644 index 000000000000..f60074dd4ea0 --- /dev/null +++ b/source/server/active_raw_udp_listener_config.cc @@ -0,0 +1,26 @@ +#include "server/active_raw_udp_listener_config.h" + +#include "server/connection_handler_impl.h" +#include "server/well_known_names.h" + +namespace Envoy { +namespace Server { + +Network::ConnectionHandler::ActiveListenerPtr ActiveRawUdpListenerFactory::createActiveUdpListener( + Network::ConnectionHandler& /*parent*/, Event::Dispatcher& dispatcher, + spdlog::logger& /*logger*/, Network::ListenerConfig& config) const { + return std::make_unique(dispatcher, config); +} + +Network::ActiveUdpListenerFactoryPtr +ActiveRawUdpListenerConfigFactory::createActiveUdpListenerFactory( + const Protobuf::Message& /*message*/) { + return std::make_unique(); +} + +std::string ActiveRawUdpListenerConfigFactory::name() { return UdpListenerNames::get().RawUdp; } + +REGISTER_FACTORY(ActiveRawUdpListenerConfigFactory, Server::ActiveUdpListenerConfigFactory); + +} // namespace Server +} // namespace Envoy diff --git a/source/server/active_raw_udp_listener_config.h b/source/server/active_raw_udp_listener_config.h new file mode 100644 index 000000000000..157ff28f6b41 --- /dev/null +++ b/source/server/active_raw_udp_listener_config.h @@ -0,0 +1,31 @@ +#pragma once + +#include "envoy/network/connection_handler.h" +#include "envoy/registry/registry.h" +#include "envoy/server/active_udp_listener_config.h" + +namespace Envoy { +namespace Server { + +class ActiveRawUdpListenerFactory : public Network::ActiveUdpListenerFactory { +public: + Network::ConnectionHandler::ActiveListenerPtr + createActiveUdpListener(Network::ConnectionHandler& parent, Event::Dispatcher& disptacher, + spdlog::logger& logger, Network::ListenerConfig& config) const override; +}; + +// This class uses a protobuf config to create a UDP listener factory which +// creates a Server::ConnectionHandlerImpl::ActiveUdpListener. +// This is the default UDP listener if not specified in config. +class ActiveRawUdpListenerConfigFactory : public ActiveUdpListenerConfigFactory { +public: + Network::ActiveUdpListenerFactoryPtr + createActiveUdpListenerFactory(const Protobuf::Message&) override; + + std::string name() override; +}; + +DECLARE_FACTORY(ActiveRawUdpListenerConfigFactory); + +} // namespace Server +} // namespace Envoy diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index b7e08a31441b..31fb7e64f2d0 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -18,28 +18,27 @@ ConnectionHandlerImpl::ConnectionHandlerImpl(spdlog::logger& logger, Event::Disp : logger_(logger), dispatcher_(dispatcher), disable_listeners_(false) {} void ConnectionHandlerImpl::addListener(Network::ListenerConfig& config) { - ActiveListenerBasePtr listener; + Network::ConnectionHandler::ActiveListenerPtr listener; Network::Address::SocketType socket_type = config.socket().socketType(); if (socket_type == Network::Address::SocketType::Stream) { - ActiveTcpListenerPtr tcp(new ActiveTcpListener(*this, config)); - listener = std::move(tcp); + listener = std::make_unique(*this, config); } else { ASSERT(socket_type == Network::Address::SocketType::Datagram, "Only datagram/stream listener supported"); - ActiveUdpListenerPtr udp(new ActiveUdpListener(*this, config)); - listener = std::move(udp); + listener = + config.udpListenerFactory()->createActiveUdpListener(*this, dispatcher_, logger_, config); } if (disable_listeners_) { - listener->listener_->disable(); + listener->listener()->disable(); } listeners_.emplace_back(config.socket().localAddress(), std::move(listener)); } void ConnectionHandlerImpl::removeListeners(uint64_t listener_tag) { for (auto listener = listeners_.begin(); listener != listeners_.end();) { - if (listener->second->listener_tag_ == listener_tag) { + if (listener->second->listenerTag() == listener_tag) { listener = listeners_.erase(listener); } else { ++listener; @@ -49,29 +48,29 @@ void ConnectionHandlerImpl::removeListeners(uint64_t listener_tag) { void ConnectionHandlerImpl::stopListeners(uint64_t listener_tag) { for (auto& listener : listeners_) { - if (listener.second->listener_tag_ == listener_tag) { - listener.second->listener_.reset(); + if (listener.second->listenerTag() == listener_tag) { + listener.second->destroy(); } } } void ConnectionHandlerImpl::stopListeners() { for (auto& listener : listeners_) { - listener.second->listener_.reset(); + listener.second->destroy(); } } void ConnectionHandlerImpl::disableListeners() { disable_listeners_ = true; for (auto& listener : listeners_) { - listener.second->listener_->disable(); + listener.second->listener()->disable(); } } void ConnectionHandlerImpl::enableListeners() { disable_listeners_ = false; for (auto& listener : listeners_) { - listener.second->listener_->enable(); + listener.second->listener()->enable(); } } @@ -84,11 +83,9 @@ void ConnectionHandlerImpl::ActiveTcpListener::removeConnection(ActiveConnection parent_.num_connections_--; } -ConnectionHandlerImpl::ActiveListenerBase::ActiveListenerBase(ConnectionHandlerImpl& parent, - Network::ListenerPtr&& listener, - Network::ListenerConfig& config) - : parent_(parent), listener_(std::move(listener)), - stats_(generateStats(config.listenerScope())), +ConnectionHandlerImpl::ActiveListenerImplBase::ActiveListenerImplBase( + Network::ListenerPtr&& listener, Network::ListenerConfig& config) + : listener_(std::move(listener)), stats_(generateStats(config.listenerScope())), listener_filters_timeout_(config.listenerFiltersTimeout()), continue_on_listener_filters_timeout_(config.continueOnListenerFiltersTimeout()), listener_tag_(config.listenerTag()), config_(config) {} @@ -104,7 +101,7 @@ ConnectionHandlerImpl::ActiveTcpListener::ActiveTcpListener(ConnectionHandlerImp ConnectionHandlerImpl::ActiveTcpListener::ActiveTcpListener(ConnectionHandlerImpl& parent, Network::ListenerPtr&& listener, Network::ListenerConfig& config) - : ConnectionHandlerImpl::ActiveListenerBase(parent, std::move(listener), config) {} + : ConnectionHandlerImpl::ActiveListenerImplBase(std::move(listener), config), parent_(parent) {} ConnectionHandlerImpl::ActiveTcpListener::~ActiveTcpListener() { // Purge sockets that have not progressed to connections. This should only happen when @@ -123,22 +120,22 @@ ConnectionHandlerImpl::ActiveTcpListener::~ActiveTcpListener() { Network::Listener* ConnectionHandlerImpl::findListenerByAddress(const Network::Address::Instance& address) { - ActiveListenerBase* listener = findActiveListenerByAddress(address); - return listener ? listener->listener_.get() : nullptr; + Network::ConnectionHandler::ActiveListener* listener = findActiveListenerByAddress(address); + return listener ? listener->listener() : nullptr; } -ConnectionHandlerImpl::ActiveListenerBase* +Network::ConnectionHandler::ActiveListener* ConnectionHandlerImpl::findActiveListenerByAddress(const Network::Address::Instance& address) { // This is a linear operation, may need to add a map to improve performance. // However, linear performance might be adequate since the number of listeners is small. // We do not return stopped listeners. - auto listener_it = std::find_if( - listeners_.begin(), listeners_.end(), - [&address]( - const std::pair& p) { - return p.second->listener_ != nullptr && p.first->type() == Network::Address::Type::Ip && - *(p.first) == address; - }); + auto listener_it = + std::find_if(listeners_.begin(), listeners_.end(), + [&address](const std::pair& p) { + return p.second->listener() != nullptr && + p.first->type() == Network::Address::Type::Ip && *(p.first) == address; + }); // If there is exact address match, return the corresponding listener. if (listener_it != listeners_.end()) { @@ -150,9 +147,9 @@ ConnectionHandlerImpl::findActiveListenerByAddress(const Network::Address::Insta // TODO(wattli): consolidate with previous search for more efficiency. listener_it = std::find_if( listeners_.begin(), listeners_.end(), - [&address]( - const std::pair& p) { - return p.second->listener_ != nullptr && p.first->type() == Network::Address::Type::Ip && + [&address](const std::pair& p) { + return p.second->listener() != nullptr && p.first->type() == Network::Address::Type::Ip && p.first->ip()->port() == address.ip()->port() && p.first->ip()->isAnyAddress(); }); return (listener_it != listeners_.end()) ? listener_it->second.get() : nullptr; @@ -210,7 +207,7 @@ void ConnectionHandlerImpl::ActiveSocket::continueFilterChain(bool success) { void ConnectionHandlerImpl::ActiveSocket::newConnection() { // Check if the socket may need to be redirected to another listener. - ActiveListenerBase* new_listener = nullptr; + ConnectionHandler::ActiveListener* new_listener = nullptr; if (hand_off_restored_destination_connections_ && socket_->localAddressRestored()) { // Find a listener associated with the original destination address. @@ -318,15 +315,12 @@ ListenerStats ConnectionHandlerImpl::generateStats(Stats::Scope& scope) { return {ALL_LISTENER_STATS(POOL_COUNTER(scope), POOL_GAUGE(scope), POOL_HISTOGRAM(scope))}; } -ConnectionHandlerImpl::ActiveUdpListener::ActiveUdpListener(ConnectionHandlerImpl& parent, - Network::ListenerConfig& config) - : ActiveUdpListener(parent, parent.dispatcher_.createUdpListener(config.socket(), *this), - config) {} +ActiveUdpListener::ActiveUdpListener(Event::Dispatcher& dispatcher, Network::ListenerConfig& config) + : ActiveUdpListener(dispatcher.createUdpListener(config.socket(), *this), config) {} -ConnectionHandlerImpl::ActiveUdpListener::ActiveUdpListener(ConnectionHandlerImpl& parent, - Network::ListenerPtr&& listener, - Network::ListenerConfig& config) - : ConnectionHandlerImpl::ActiveListenerBase(parent, std::move(listener), config), +ActiveUdpListener::ActiveUdpListener(Network::ListenerPtr&& listener, + Network::ListenerConfig& config) + : ConnectionHandlerImpl::ActiveListenerImplBase(std::move(listener), config), udp_listener_(dynamic_cast(listener_.get())), read_filter_(nullptr) { // TODO(sumukhs): Try to avoid dynamic_cast by coming up with a better interface design ASSERT(udp_listener_ != nullptr, ""); @@ -342,32 +336,27 @@ ConnectionHandlerImpl::ActiveUdpListener::ActiveUdpListener(ConnectionHandlerImp } } -void ConnectionHandlerImpl::ActiveUdpListener::onData(Network::UdpRecvData& data) { - read_filter_->onData(data); -} +void ActiveUdpListener::onData(Network::UdpRecvData& data) { read_filter_->onData(data); } -void ConnectionHandlerImpl::ActiveUdpListener::onWriteReady(const Network::Socket&) { +void ActiveUdpListener::onWriteReady(const Network::Socket&) { // TODO(sumukhs): This is not used now. When write filters are implemented, this is a // trigger to invoke the on write ready API on the filters which is when they can write // data } -void ConnectionHandlerImpl::ActiveUdpListener::onReceiveError( - const Network::UdpListenerCallbacks::ErrorCode&, Api::IoError::IoErrorCode) { +void ActiveUdpListener::onReceiveError(const Network::UdpListenerCallbacks::ErrorCode&, + Api::IoError::IoErrorCode) { // TODO(sumukhs): Determine what to do on receive error. // Would the filters need to know on error? Can't foresee a scenario where they // would take an action } -void ConnectionHandlerImpl::ActiveUdpListener::addReadFilter( - Network::UdpListenerReadFilterPtr&& filter) { +void ActiveUdpListener::addReadFilter(Network::UdpListenerReadFilterPtr&& filter) { ASSERT(read_filter_ == nullptr, "Cannot add a 2nd UDP read filter"); read_filter_ = std::move(filter); } -Network::UdpListener& ConnectionHandlerImpl::ActiveUdpListener::udpListener() { - return *udp_listener_; -} +Network::UdpListener& ActiveUdpListener::udpListener() { return *udp_listener_; } } // namespace Server } // namespace Envoy diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index 7ca0088abe61..4c1354e29870 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -12,6 +12,7 @@ #include "envoy/network/filter.h" #include "envoy/network/listen_socket.h" #include "envoy/network/listener.h" +#include "envoy/server/active_udp_listener_config.h" #include "envoy/server/listener_manager.h" #include "envoy/stats/scope.h" #include "envoy/stats/timespan.h" @@ -59,33 +60,21 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { Network::Listener* findListenerByAddress(const Network::Address::Instance& address) override; -private: - struct ActiveListenerBase; - using ActiveListenerBasePtr = std::unique_ptr; - - struct ActiveTcpListener; - using ActiveTcpListenerPtr = std::unique_ptr; - - struct ActiveUdpListener; - using ActiveUdpListenerPtr = std::unique_ptr; - - ActiveListenerBase* findActiveListenerByAddress(const Network::Address::Instance& address); - - struct ActiveConnection; - using ActiveConnectionPtr = std::unique_ptr; - struct ActiveSocket; - using ActiveSocketPtr = std::unique_ptr; + Network::ConnectionHandler::ActiveListener* + findActiveListenerByAddress(const Network::Address::Instance& address); /** * Wrapper for an active listener owned by this handler. */ - struct ActiveListenerBase { - ActiveListenerBase(ConnectionHandlerImpl& parent, Network::ListenerPtr&& listener, - Network::ListenerConfig& config); + class ActiveListenerImplBase : public Network::ConnectionHandler::ActiveListener { + public: + ActiveListenerImplBase(Network::ListenerPtr&& listener, Network::ListenerConfig& config); - virtual ~ActiveListenerBase() = default; + // Network::ConnectionHandler::ActiveListener. + uint64_t listenerTag() override { return listener_tag_; } + Network::Listener* listener() override { return listener_.get(); } + void destroy() override { listener_.reset(); } - ConnectionHandlerImpl& parent_; Network::ListenerPtr listener_; ListenerStats stats_; const std::chrono::milliseconds listener_filters_timeout_; @@ -94,38 +83,19 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { Network::ListenerConfig& config_; }; - /** - * Wrapper for an active udp listener owned by this handler. - */ - struct ActiveUdpListener : public Network::UdpListenerCallbacks, - public ActiveListenerBase, - public Network::UdpListenerFilterManager, - public Network::UdpReadFilterCallbacks { - ActiveUdpListener(ConnectionHandlerImpl& parent, Network::ListenerConfig& config); - - ActiveUdpListener(ConnectionHandlerImpl& parent, Network::ListenerPtr&& listener, - Network::ListenerConfig& config); - - // Network::UdpListenerCallbacks - void onData(Network::UdpRecvData& data) override; - void onWriteReady(const Network::Socket& socket) override; - void onReceiveError(const Network::UdpListenerCallbacks::ErrorCode& error_code, - Api::IoError::IoErrorCode err) override; - - // Network::UdpListenerFilterManager - void addReadFilter(Network::UdpListenerReadFilterPtr&& filter) override; - - // Network::UdpReadFilterCallbacks - Network::UdpListener& udpListener() override; - - Network::UdpListener* udp_listener_; - Network::UdpListenerReadFilterPtr read_filter_; - }; +private: + class ActiveTcpListener; + using ActiveTcpListenerPtr = std::unique_ptr; + struct ActiveConnection; + using ActiveConnectionPtr = std::unique_ptr; + struct ActiveSocket; + using ActiveSocketPtr = std::unique_ptr; /** * Wrapper for an active tcp listener owned by this handler. */ - struct ActiveTcpListener : public Network::ListenerCallbacks, public ActiveListenerBase { + class ActiveTcpListener : public Network::ListenerCallbacks, public ActiveListenerImplBase { + public: ActiveTcpListener(ConnectionHandlerImpl& parent, Network::ListenerConfig& config); ActiveTcpListener(ConnectionHandlerImpl& parent, Network::ListenerPtr&& listener, @@ -149,6 +119,7 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { */ void newConnection(Network::ConnectionSocketPtr&& socket); + ConnectionHandlerImpl& parent_; std::list sockets_; std::list connections_; }; @@ -225,10 +196,42 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { spdlog::logger& logger_; Event::Dispatcher& dispatcher_; - std::list> listeners_; + std::list> + listeners_; std::atomic num_connections_{}; bool disable_listeners_; }; +/** + * Wrapper for an active udp listener owned by this handler. + * TODO(danzh): rename to ActiveRawUdpListener. + */ +class ActiveUdpListener : public Network::UdpListenerCallbacks, + public ConnectionHandlerImpl::ActiveListenerImplBase, + public Network::UdpListenerFilterManager, + public Network::UdpReadFilterCallbacks { +public: + ActiveUdpListener(Event::Dispatcher& dispatcher, Network::ListenerConfig& config); + + ActiveUdpListener(Network::ListenerPtr&& listener, Network::ListenerConfig& config); + + // Network::UdpListenerCallbacks + void onData(Network::UdpRecvData& data) override; + void onWriteReady(const Network::Socket& socket) override; + void onReceiveError(const Network::UdpListenerCallbacks::ErrorCode& error_code, + Api::IoError::IoErrorCode err) override; + + // Network::UdpListenerFilterManager + void addReadFilter(Network::UdpListenerReadFilterPtr&& filter) override; + + // Network::UdpReadFilterCallbacks + Network::UdpListener& udpListener() override; + +private: + Network::UdpListener* udp_listener_; + Network::UdpListenerReadFilterPtr read_filter_; +}; + } // namespace Server } // namespace Envoy diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 09bf1ecedf4c..ac01902068a7 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -320,6 +320,9 @@ class AdminImpl : public Admin, Stats::Scope& listenerScope() override { return *scope_; } uint64_t listenerTag() const override { return 0; } const std::string& name() const override { return name_; } + const Network::ActiveUdpListenerFactory* udpListenerFactory() override { + NOT_REACHED_GCOVR_EXCL_LINE; + } AdminImpl& parent_; const std::string name_; diff --git a/source/server/listener_manager_impl.cc b/source/server/listener_manager_impl.cc index 0f2ec74ee62b..1b305d3157fd 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/server/listener_manager_impl.cc @@ -4,6 +4,7 @@ #include "envoy/admin/v2alpha/config_dump.pb.h" #include "envoy/registry/registry.h" +#include "envoy/server/active_udp_listener_config.h" #include "envoy/server/transport_socket_config.h" #include "envoy/stats/scope.h" @@ -23,6 +24,7 @@ #include "server/drain_manager_impl.h" #include "server/filter_chain_manager_impl.h" #include "server/transport_socket_config_impl.h" +#include "server/well_known_names.h" #include "extensions/filters/listener/well_known_names.h" #include "extensions/transport_sockets/well_known_names.h" @@ -226,6 +228,16 @@ ListenerImpl::ListenerImpl(const envoy::api::v2::Listener& config, const std::st addListenSocketOptions(Network::SocketOptionFactory::buildIpPacketInfoOptions()); // Needed to return receive buffer overflown indicator. addListenSocketOptions(Network::SocketOptionFactory::buildRxQueueOverFlowOptions()); + std::string listener_name = + config.has_udp_listener_config() ? config.udp_listener_config().udp_listener_name() : ""; + if (listener_name.empty()) { + listener_name = UdpListenerNames::get().RawUdp; + } + udp_listener_factory_ = + Config::Utility::getAndCheckFactory(listener_name) + .createActiveUdpListenerFactory(config.has_udp_listener_config() + ? config.udp_listener_config() + : envoy::api::v2::listener::UdpListenerConfig()); } if (!config.listener_filters().empty()) { diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 1197ac82a9be..b634a1bf6537 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -279,6 +279,9 @@ class ListenerImpl : public Network::ListenerConfig, Stats::Scope& listenerScope() override { return *listener_scope_; } uint64_t listenerTag() const override { return listener_tag_; } const std::string& name() const override { return name_; } + const Network::ActiveUdpListenerFactory* udpListenerFactory() override { + return udp_listener_factory_.get(); + } // Server::Configuration::ListenerFactoryContext AccessLog::AccessLogManager& accessLogManager() override { @@ -379,6 +382,7 @@ class ListenerImpl : public Network::ListenerConfig, Network::Socket::OptionsSharedPtr listen_socket_options_; const std::chrono::milliseconds listener_filters_timeout_; const bool continue_on_listener_filters_timeout_; + Network::ActiveUdpListenerFactoryPtr udp_listener_factory_; // to access ListenerManagerImpl::factory_. friend class ListenerFilterChainFactoryBuilder; }; diff --git a/source/server/well_known_names.h b/source/server/well_known_names.h new file mode 100644 index 000000000000..23d202d996ae --- /dev/null +++ b/source/server/well_known_names.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "common/singleton/const_singleton.h" + +namespace Envoy { +namespace Server { + +/** + * Well-known active UDP listener names. + */ +class UdpListenerNameValues { +public: + const std::string RawUdp = "raw_udp_listener"; +}; + +using UdpListenerNames = ConstSingleton; + +} // namespace Server +} // namespace Envoy diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index 2f37b861df43..cdc225696c3d 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -77,6 +77,7 @@ class ProxyProtocolTest : public testing::TestWithParam, Stats::Scope& listenerScope() override { return parent_.stats_store_; } uint64_t listenerTag() const override { return 0; } const std::string& name() const override { return name_; } + Network::ActiveUdpListenerFactory* udpListenerFactory() override { + return udp_listener_factory_.get(); + } FakeUpstream& parent_; + Network::ActiveUdpListenerFactoryPtr udp_listener_factory_; std::string name_; }; diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index e63d3f22957b..33931209fffe 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -311,6 +311,7 @@ class MockListenerConfig : public ListenerConfig { MOCK_METHOD0(listenerScope, Stats::Scope&()); MOCK_CONST_METHOD0(listenerTag, uint64_t()); MOCK_CONST_METHOD0(name, const std::string&()); + MOCK_METHOD0(udpListenerFactory, const Network::ActiveUdpListenerFactory*()); testing::NiceMock filter_chain_factory_; testing::NiceMock socket_; diff --git a/test/server/BUILD b/test/server/BUILD index 6f2893539e87..7ffe0fcfd36b 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -53,6 +53,7 @@ envoy_cc_test( "//source/common/common:utility_lib", "//source/common/network:address_lib", "//source/common/stats:stats_lib", + "//source/server:active_raw_udp_listener_config", "//source/server:connection_handler_lib", "//test/mocks/network:network_mocks", "//test/mocks/server:server_mocks", @@ -180,6 +181,7 @@ envoy_cc_test( "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:config", "//source/extensions/transport_sockets/tls:ssl_socket_lib", + "//source/server:active_raw_udp_listener_config", "//source/server:listener_manager_lib", "//test/mocks/network:network_mocks", "//test/mocks/server:server_mocks", diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 59d14ffb5e2f..092ff2780649 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -1,3 +1,4 @@ +#include "envoy/server/active_udp_listener_config.h" #include "envoy/stats/scope.h" #include "common/common/utility.h" @@ -44,6 +45,12 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable(listener_name) + .createActiveUdpListenerFactory(dummy); EXPECT_CALL(socket_, socketType()).WillOnce(Return(socket_type)); } @@ -66,6 +73,9 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable udp_listener_factory_; }; using TestListenerPtr = std::unique_ptr; From 8bdebbfb936c533a822df9e96fd37be2cbdbab94 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 26 Aug 2019 16:07:39 -0700 Subject: [PATCH 457/542] accesslog: implement TCP gRPC access logger (#7941) Description: Initial implementation for TCP gRPC access logger. Risk Level: Low (extension only) Testing: integration test Docs Changes: Added Release Notes: Added Signed-off-by: Lizan Zhou --- api/envoy/config/accesslog/v2/als.proto | 1 - .../filter/accesslog/v2/accesslog.proto | 3 + api/envoy/data/accesslog/v2/accesslog.proto | 13 +- api/envoy/service/accesslog/v2/als.proto | 2 - docs/root/intro/version_history.rst | 1 + source/extensions/access_loggers/grpc/BUILD | 40 +++- .../access_loggers/grpc/config_utils.cc | 25 +++ .../access_loggers/grpc/config_utils.h | 18 ++ .../grpc/grpc_access_log_impl.cc | 18 +- .../grpc/grpc_access_log_impl.h | 23 ++- .../grpc/grpc_access_log_proto_descriptors.cc | 4 +- .../grpc/grpc_access_log_proto_descriptors.h | 4 +- .../grpc/grpc_access_log_utils.cc | 3 +- .../access_loggers/grpc/http_config.cc | 22 +- .../access_loggers/grpc/http_config.h | 2 - .../grpc/http_grpc_access_log_impl.cc | 4 +- .../access_loggers/grpc/tcp_config.cc | 51 +++++ .../access_loggers/grpc/tcp_config.h | 29 +++ .../grpc/tcp_grpc_access_log_impl.cc | 48 +++++ .../grpc/tcp_grpc_access_log_impl.h | 60 ++++++ .../access_loggers/well_known_names.h | 2 + source/extensions/extensions_build_config.bzl | 1 + test/extensions/access_loggers/grpc/BUILD | 17 ++ .../grpc/grpc_access_log_impl_test.cc | 15 +- .../grpc/http_grpc_access_log_impl_test.cc | 21 +- .../tcp_grpc_access_log_integration_test.cc | 189 ++++++++++++++++++ 26 files changed, 564 insertions(+), 52 deletions(-) create mode 100644 source/extensions/access_loggers/grpc/config_utils.cc create mode 100644 source/extensions/access_loggers/grpc/config_utils.h create mode 100644 source/extensions/access_loggers/grpc/tcp_config.cc create mode 100644 source/extensions/access_loggers/grpc/tcp_config.h create mode 100644 source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc create mode 100644 source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h create mode 100644 test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc diff --git a/api/envoy/config/accesslog/v2/als.proto b/api/envoy/config/accesslog/v2/als.proto index a7291e4e9780..9d83ebfcfb91 100644 --- a/api/envoy/config/accesslog/v2/als.proto +++ b/api/envoy/config/accesslog/v2/als.proto @@ -38,7 +38,6 @@ message HttpGrpcAccessLogConfig { // Configuration for the built-in *envoy.tcp_grpc_access_log* type. This configuration will // populate *StreamAccessLogsMessage.tcp_logs*. -// [#not-implemented-hide:] message TcpGrpcAccessLogConfig { CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message.required = true]; } diff --git a/api/envoy/config/filter/accesslog/v2/accesslog.proto b/api/envoy/config/filter/accesslog/v2/accesslog.proto index 66d901c2d413..76fc4baf80c7 100644 --- a/api/envoy/config/filter/accesslog/v2/accesslog.proto +++ b/api/envoy/config/filter/accesslog/v2/accesslog.proto @@ -24,6 +24,7 @@ message AccessLog { // // #. "envoy.file_access_log" // #. "envoy.http_grpc_access_log" + // #. "envoy.tcp_grpc_access_log" string name = 1; // Filter which is used to determine if the access log needs to be written. @@ -36,6 +37,8 @@ message AccessLog { // ` // #. "envoy.http_grpc_access_log": :ref:`HttpGrpcAccessLogConfig // ` + // #. "envoy.tcp_grpc_access_log": :ref:`TcpGrpcAccessLogConfig + // ` oneof config_type { google.protobuf.Struct config = 3; diff --git a/api/envoy/data/accesslog/v2/accesslog.proto b/api/envoy/data/accesslog/v2/accesslog.proto index 1396c729f88e..7309d4a362a6 100644 --- a/api/envoy/data/accesslog/v2/accesslog.proto +++ b/api/envoy/data/accesslog/v2/accesslog.proto @@ -28,10 +28,12 @@ option (gogoproto.stable_marshaler_all) = true; // Fields describing *upstream* interaction will explicitly include ``upstream`` // in their name. -// [#not-implemented-hide:] message TCPAccessLogEntry { // Common properties shared by all Envoy access logs. AccessLogCommon common_properties = 1; + + // Properties of the TCP connection. + ConnectionProperties connection_properties = 2; } message HTTPAccessLogEntry { @@ -54,6 +56,15 @@ message HTTPAccessLogEntry { HTTPResponseProperties response = 4; } +// Defines fields for a connection +message ConnectionProperties { + // Number of bytes received from downstream. + uint64 received_bytes = 1; + + // Number of bytes sent to downstream. + uint64 sent_bytes = 2; +} + // Defines fields that are shared by all Envoy access logs. message AccessLogCommon { // [#not-implemented-hide:] diff --git a/api/envoy/service/accesslog/v2/als.proto b/api/envoy/service/accesslog/v2/als.proto index 1ee6ccd0094c..52788e0659c6 100644 --- a/api/envoy/service/accesslog/v2/als.proto +++ b/api/envoy/service/accesslog/v2/als.proto @@ -53,7 +53,6 @@ message StreamAccessLogsMessage { [(validate.rules).repeated .min_items = 1]; } - // [#not-implemented-hide:] // Wrapper for batches of TCP access log entries. message TCPAccessLogEntries { repeated envoy.data.accesslog.v2.TCPAccessLogEntry log_entry = 1 @@ -67,7 +66,6 @@ message StreamAccessLogsMessage { HTTPAccessLogEntries http_logs = 2; - // [#not-implemented-hide:] TCPAccessLogEntries tcp_logs = 3; } } diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index caf541e796d5..63fb953c060a 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -4,6 +4,7 @@ Version history 1.12.0 (pending) ================ * access log: added :ref:`buffering ` and :ref:`periodical flushing ` support to gRPC access logger. Defaults to 16KB buffer and flushing every 1 second. +* access log: gRPC Access Log Service (ALS) support added for :ref:`TCP access logs `. * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * api: added ::ref:`set_node_on_first_message_only ` option to omit the node identifier from the subsequent discovery requests on the same stream. diff --git a/source/extensions/access_loggers/grpc/BUILD b/source/extensions/access_loggers/grpc/BUILD index e0e705952699..e7ad7b299588 100644 --- a/source/extensions/access_loggers/grpc/BUILD +++ b/source/extensions/access_loggers/grpc/BUILD @@ -11,6 +11,18 @@ load( envoy_package() +envoy_cc_library( + name = "config_utils", + srcs = ["config_utils.cc"], + hdrs = ["config_utils.h"], + deps = [ + ":grpc_access_log_lib", + "//include/envoy/registry", + "//include/envoy/server:filter_config_interface", + "//include/envoy/singleton:instance_interface", + ], +) + envoy_cc_library( name = "grpc_access_log_lib", srcs = ["grpc_access_log_impl.cc"], @@ -18,7 +30,6 @@ envoy_cc_library( deps = [ "//include/envoy/grpc:async_client_interface", "//include/envoy/grpc:async_client_manager_interface", - "//include/envoy/singleton:instance_interface", "//include/envoy/thread_local:thread_local_interface", "//include/envoy/upstream:cluster_manager_interface", "//include/envoy/upstream:upstream_interface", @@ -54,6 +65,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "tcp_grpc_access_log_lib", + srcs = ["tcp_grpc_access_log_impl.cc"], + hdrs = ["tcp_grpc_access_log_impl.h"], + deps = [ + ":grpc_access_log_lib", + ":grpc_access_log_utils", + ], +) + envoy_cc_library( name = "grpc_access_log_proto_descriptors_lib", srcs = ["grpc_access_log_proto_descriptors.cc"], @@ -70,7 +91,7 @@ envoy_cc_library( srcs = ["http_config.cc"], hdrs = ["http_config.h"], deps = [ - "//include/envoy/registry", + ":config_utils", "//include/envoy/server:access_log_config_interface", "//source/common/common:assert_lib", "//source/common/protobuf", @@ -79,3 +100,18 @@ envoy_cc_library( "//source/extensions/access_loggers/grpc:http_grpc_access_log_lib", ], ) + +envoy_cc_library( + name = "tcp_config", + srcs = ["tcp_config.cc"], + hdrs = ["tcp_config.h"], + deps = [ + ":config_utils", + "//include/envoy/server:access_log_config_interface", + "//source/common/common:assert_lib", + "//source/common/protobuf", + "//source/extensions/access_loggers:well_known_names", + "//source/extensions/access_loggers/grpc:grpc_access_log_proto_descriptors_lib", + "//source/extensions/access_loggers/grpc:tcp_grpc_access_log_lib", + ], +) diff --git a/source/extensions/access_loggers/grpc/config_utils.cc b/source/extensions/access_loggers/grpc/config_utils.cc new file mode 100644 index 000000000000..5d2a648a0f5d --- /dev/null +++ b/source/extensions/access_loggers/grpc/config_utils.cc @@ -0,0 +1,25 @@ +#include "extensions/access_loggers/grpc/config_utils.h" + +#include "envoy/singleton/manager.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace GrpcCommon { + +// Singleton registration via macro defined in envoy/singleton/manager.h +SINGLETON_MANAGER_REGISTRATION(grpc_access_logger_cache); + +std::shared_ptr +getGrpcAccessLoggerCacheSingleton(Server::Configuration::FactoryContext& context) { + return context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(grpc_access_logger_cache), [&context] { + return std::make_shared( + context.clusterManager().grpcAsyncClientManager(), context.scope(), + context.threadLocal(), context.localInfo()); + }); +} +} // namespace GrpcCommon +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/access_loggers/grpc/config_utils.h b/source/extensions/access_loggers/grpc/config_utils.h new file mode 100644 index 000000000000..f95b53f6a790 --- /dev/null +++ b/source/extensions/access_loggers/grpc/config_utils.h @@ -0,0 +1,18 @@ +#pragma once + +#include "envoy/server/filter_config.h" + +#include "extensions/access_loggers/grpc/grpc_access_log_impl.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace GrpcCommon { + +GrpcAccessLoggerCacheSharedPtr +getGrpcAccessLoggerCacheSingleton(Server::Configuration::FactoryContext& context); + +} // namespace GrpcCommon +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc index 38020326b7da..962e3a68084d 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.cc @@ -38,14 +38,22 @@ GrpcAccessLoggerImpl::GrpcAccessLoggerImpl(Grpc::RawAsyncClientPtr&& client, std void GrpcAccessLoggerImpl::log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) { approximate_message_size_bytes_ += entry.ByteSizeLong(); - message_.mutable_http_logs()->add_log_entry()->Swap(&entry); + message_.mutable_http_logs()->mutable_log_entry()->Add(std::move(entry)); + if (approximate_message_size_bytes_ >= buffer_size_bytes_) { + flush(); + } +} + +void GrpcAccessLoggerImpl::log(envoy::data::accesslog::v2::TCPAccessLogEntry&& entry) { + approximate_message_size_bytes_ += entry.ByteSizeLong(); + message_.mutable_tcp_logs()->mutable_log_entry()->Add(std::move(entry)); if (approximate_message_size_bytes_ >= buffer_size_bytes_) { flush(); } } void GrpcAccessLoggerImpl::flush() { - if (!message_.has_http_logs()) { + if (!message_.has_http_logs() && !message_.has_tcp_logs()) { // Nothing to flush. return; } @@ -88,11 +96,11 @@ GrpcAccessLoggerCacheImpl::GrpcAccessLoggerCacheImpl(Grpc::AsyncClientManager& a } GrpcAccessLoggerSharedPtr GrpcAccessLoggerCacheImpl::getOrCreateLogger( - const envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) { + const envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config, + GrpcAccessLoggerType logger_type) { // TODO(euroelessar): Consider cleaning up loggers. auto& cache = tls_slot_->getTyped(); - // TODO(lizan): Include logger type in the hash - const std::size_t cache_key = MessageUtil::hash(config); + const auto cache_key = std::make_pair(MessageUtil::hash(config), logger_type); const auto it = cache.access_loggers_.find(cache_key); if (it != cache.access_loggers_.end()) { return it->second; diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h index 71745adc54d1..8c254e47bca0 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_impl.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_impl.h @@ -36,10 +36,18 @@ class GrpcAccessLogger { * @param entry supplies the access log to send. */ virtual void log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) PURE; + + /** + * Log tcp access entry. + * @param entry supplies the access log to send. + */ + virtual void log(envoy::data::accesslog::v2::TCPAccessLogEntry&& entry) PURE; }; using GrpcAccessLoggerSharedPtr = std::shared_ptr; +enum class GrpcAccessLoggerType { TCP, HTTP }; + /** * Interface for an access logger cache. The cache deals with threading and de-duplicates loggers * for the same configuration. @@ -54,7 +62,8 @@ class GrpcAccessLoggerCache { * @return GrpcAccessLoggerSharedPtr ready for logging requests. */ virtual GrpcAccessLoggerSharedPtr - getOrCreateLogger(const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) PURE; + getOrCreateLogger(const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config, + GrpcAccessLoggerType logger_type) PURE; }; using GrpcAccessLoggerCacheSharedPtr = std::shared_ptr; @@ -66,7 +75,9 @@ class GrpcAccessLoggerImpl : public GrpcAccessLogger { uint64_t buffer_size_bytes, Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info); + // Extensions::AccessLoggers::GrpcCommon::GrpcAccessLogger void log(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) override; + void log(envoy::data::accesslog::v2::TCPAccessLogEntry&& entry) override; private: struct LocalStream @@ -106,8 +117,9 @@ class GrpcAccessLoggerCacheImpl : public Singleton::Instance, public GrpcAccessL ThreadLocal::SlotAllocator& tls, const LocalInfo::LocalInfo& local_info); - GrpcAccessLoggerSharedPtr getOrCreateLogger( - const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) override; + GrpcAccessLoggerSharedPtr + getOrCreateLogger(const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config, + GrpcAccessLoggerType logger_type) override; private: /** @@ -117,8 +129,9 @@ class GrpcAccessLoggerCacheImpl : public Singleton::Instance, public GrpcAccessL ThreadLocalCache(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} Event::Dispatcher& dispatcher_; - // Access loggers indexed by the hash of logger's configuration. - absl::flat_hash_map access_loggers_; + // Access loggers indexed by the hash of logger's configuration and logger type. + absl::flat_hash_map, GrpcAccessLoggerSharedPtr> + access_loggers_; }; Grpc::AsyncClientManager& async_client_manager_; diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.cc b/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.cc index 3b038045ee89..6936800be0ff 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.cc @@ -9,7 +9,7 @@ namespace Envoy { namespace Extensions { namespace AccessLoggers { -namespace HttpGrpc { +namespace GrpcCommon { void validateProtoDescriptors() { const auto method = "envoy.service.accesslog.v2.AccessLogService.StreamAccessLogs"; @@ -17,7 +17,7 @@ void validateProtoDescriptors() { RELEASE_ASSERT(Protobuf::DescriptorPool::generated_pool()->FindMethodByName(method) != nullptr, ""); }; -} // namespace HttpGrpc +} // namespace GrpcCommon } // namespace AccessLoggers } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h b/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h index 62b70a387a7b..988723cfa2da 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h +++ b/source/extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h @@ -3,12 +3,12 @@ namespace Envoy { namespace Extensions { namespace AccessLoggers { -namespace HttpGrpc { +namespace GrpcCommon { // This function validates that the method descriptors for gRPC services and type descriptors that // are referenced in Any messages are available in the descriptor pool. void validateProtoDescriptors(); -} // namespace HttpGrpc +} // namespace GrpcCommon } // namespace AccessLoggers } // namespace Extensions } // namespace Envoy diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc index 6bab3fd1e274..66cf3c0a7489 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc @@ -116,6 +116,7 @@ void Utility::responseFlagsToAccessLogResponseFlags( void Utility::extractCommonAccessLogProperties( envoy::data::accesslog::v2::AccessLogCommon& common_access_log, const StreamInfo::StreamInfo& stream_info) { + // TODO(mattklein123): Populate sample_rate field. if (stream_info.downstreamRemoteAddress() != nullptr) { Network::Utility::addressToProtobufAddress( *stream_info.downstreamRemoteAddress(), @@ -229,4 +230,4 @@ void Utility::extractCommonAccessLogProperties( } // namespace GrpcCommon } // namespace AccessLoggers } // namespace Extensions -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/source/extensions/access_loggers/grpc/http_config.cc b/source/extensions/access_loggers/grpc/http_config.cc index 1047beaa579a..e8f6992600cd 100644 --- a/source/extensions/access_loggers/grpc/http_config.cc +++ b/source/extensions/access_loggers/grpc/http_config.cc @@ -10,6 +10,7 @@ #include "common/grpc/async_client_impl.h" #include "common/protobuf/protobuf.h" +#include "extensions/access_loggers/grpc/config_utils.h" #include "extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h" #include "extensions/access_loggers/grpc/http_grpc_access_log_impl.h" #include "extensions/access_loggers/well_known_names.h" @@ -19,32 +20,23 @@ namespace Extensions { namespace AccessLoggers { namespace HttpGrpc { -// Singleton registration via macro defined in envoy/singleton/manager.h -SINGLETON_MANAGER_REGISTRATION(grpc_access_logger_cache); - AccessLog::InstanceSharedPtr HttpGrpcAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, Server::Configuration::FactoryContext& context) { - validateProtoDescriptors(); + GrpcCommon::validateProtoDescriptors(); const auto& proto_config = MessageUtil::downcastAndValidate< const envoy::config::accesslog::v2::HttpGrpcAccessLogConfig&>( config, context.messageValidationVisitor()); - std::shared_ptr grpc_access_logger_cache = - context.singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(grpc_access_logger_cache), [&context] { - return std::make_shared( - context.clusterManager().grpcAsyncClientManager(), context.scope(), - context.threadLocal(), context.localInfo()); - }); - - return std::make_shared(std::move(filter), proto_config, context.threadLocal(), - grpc_access_logger_cache); + + return std::make_shared( + std::move(filter), proto_config, context.threadLocal(), + GrpcCommon::getGrpcAccessLoggerCacheSingleton(context)); } ProtobufTypes::MessagePtr HttpGrpcAccessLogFactory::createEmptyConfigProto() { - return ProtobufTypes::MessagePtr{new envoy::config::accesslog::v2::HttpGrpcAccessLogConfig()}; + return std::make_unique(); } std::string HttpGrpcAccessLogFactory::name() const { return AccessLogNames::get().HttpGrpc; } diff --git a/source/extensions/access_loggers/grpc/http_config.h b/source/extensions/access_loggers/grpc/http_config.h index 9e046ac39218..c88a3a5ac62d 100644 --- a/source/extensions/access_loggers/grpc/http_config.h +++ b/source/extensions/access_loggers/grpc/http_config.h @@ -23,8 +23,6 @@ class HttpGrpcAccessLogFactory : public Server::Configuration::AccessLogInstance std::string name() const override; }; -// TODO(mattklein123): Add TCP access log. - } // namespace HttpGrpc } // namespace AccessLoggers } // namespace Extensions diff --git a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc index 07b91a0a894a..e5bad26a3efa 100644 --- a/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc +++ b/source/extensions/access_loggers/grpc/http_grpc_access_log_impl.cc @@ -34,8 +34,8 @@ HttpGrpcAccessLog::HttpGrpcAccessLog(AccessLog::FilterPtr&& filter, } tls_slot_->set([this](Event::Dispatcher&) { - return std::make_shared( - access_logger_cache_->getOrCreateLogger(config_.common_config())); + return std::make_shared(access_logger_cache_->getOrCreateLogger( + config_.common_config(), GrpcCommon::GrpcAccessLoggerType::HTTP)); }); } diff --git a/source/extensions/access_loggers/grpc/tcp_config.cc b/source/extensions/access_loggers/grpc/tcp_config.cc new file mode 100644 index 000000000000..b8c053ae0c44 --- /dev/null +++ b/source/extensions/access_loggers/grpc/tcp_config.cc @@ -0,0 +1,51 @@ +#include "extensions/access_loggers/grpc/tcp_config.h" + +#include "envoy/config/accesslog/v2/als.pb.validate.h" +#include "envoy/config/filter/accesslog/v2/accesslog.pb.validate.h" +#include "envoy/registry/registry.h" +#include "envoy/server/filter_config.h" + +#include "common/common/assert.h" +#include "common/common/macros.h" +#include "common/grpc/async_client_impl.h" +#include "common/protobuf/protobuf.h" + +#include "extensions/access_loggers/grpc/config_utils.h" +#include "extensions/access_loggers/grpc/grpc_access_log_proto_descriptors.h" +#include "extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h" +#include "extensions/access_loggers/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace TcpGrpc { + +AccessLog::InstanceSharedPtr +TcpGrpcAccessLogFactory::createAccessLogInstance(const Protobuf::Message& config, + AccessLog::FilterPtr&& filter, + Server::Configuration::FactoryContext& context) { + GrpcCommon::validateProtoDescriptors(); + + const auto& proto_config = + MessageUtil::downcastAndValidate( + config, context.messageValidationVisitor()); + + return std::make_shared(std::move(filter), proto_config, context.threadLocal(), + GrpcCommon::getGrpcAccessLoggerCacheSingleton(context)); +} + +ProtobufTypes::MessagePtr TcpGrpcAccessLogFactory::createEmptyConfigProto() { + return std::make_unique(); +} + +std::string TcpGrpcAccessLogFactory::name() const { return AccessLogNames::get().TcpGrpc; } + +/** + * Static registration for the TCP gRPC access log. @see RegisterFactory. + */ +REGISTER_FACTORY(TcpGrpcAccessLogFactory, Server::Configuration::AccessLogInstanceFactory); + +} // namespace TcpGrpc +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/grpc/tcp_config.h b/source/extensions/access_loggers/grpc/tcp_config.h new file mode 100644 index 000000000000..39bc986146b9 --- /dev/null +++ b/source/extensions/access_loggers/grpc/tcp_config.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include "envoy/server/access_log_config.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace TcpGrpc { + +/** + * Config registration for the TCP gRPC access log. @see AccessLogInstanceFactory. + */ +class TcpGrpcAccessLogFactory : public Server::Configuration::AccessLogInstanceFactory { +public: + AccessLog::InstanceSharedPtr + createAccessLogInstance(const Protobuf::Message& config, AccessLog::FilterPtr&& filter, + Server::Configuration::FactoryContext& context) override; + + ProtobufTypes::MessagePtr createEmptyConfigProto() override; + + std::string name() const override; +}; + +} // namespace TcpGrpc +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc new file mode 100644 index 000000000000..c6c8a1bb5a58 --- /dev/null +++ b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.cc @@ -0,0 +1,48 @@ +#include "extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h" + +#include "common/common/assert.h" +#include "common/network/utility.h" +#include "common/stream_info/utility.h" + +#include "extensions/access_loggers/grpc/grpc_access_log_utils.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace TcpGrpc { + +TcpGrpcAccessLog::ThreadLocalLogger::ThreadLocalLogger(GrpcCommon::GrpcAccessLoggerSharedPtr logger) + : logger_(std::move(logger)) {} + +TcpGrpcAccessLog::TcpGrpcAccessLog(AccessLog::FilterPtr&& filter, + envoy::config::accesslog::v2::TcpGrpcAccessLogConfig config, + ThreadLocal::SlotAllocator& tls, + GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache) + : Common::ImplBase(std::move(filter)), config_(std::move(config)), + tls_slot_(tls.allocateSlot()), access_logger_cache_(std::move(access_logger_cache)) { + tls_slot_->set([this](Event::Dispatcher&) { + return std::make_shared(access_logger_cache_->getOrCreateLogger( + config_.common_config(), GrpcCommon::GrpcAccessLoggerType::TCP)); + }); +} + +void TcpGrpcAccessLog::emitLog(const Http::HeaderMap&, const Http::HeaderMap&, + const Http::HeaderMap&, const StreamInfo::StreamInfo& stream_info) { + // Common log properties. + envoy::data::accesslog::v2::TCPAccessLogEntry log_entry; + GrpcCommon::Utility::extractCommonAccessLogProperties(*log_entry.mutable_common_properties(), + stream_info); + + envoy::data::accesslog::v2::ConnectionProperties& connection_properties = + *log_entry.mutable_connection_properties(); + connection_properties.set_received_bytes(stream_info.bytesReceived()); + connection_properties.set_sent_bytes(stream_info.bytesSent()); + + // request_properties->set_request_body_bytes(stream_info.bytesReceived()); + tls_slot_->getTyped().logger_->log(std::move(log_entry)); +} + +} // namespace TcpGrpc +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h new file mode 100644 index 000000000000..115c8a467719 --- /dev/null +++ b/source/extensions/access_loggers/grpc/tcp_grpc_access_log_impl.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include "envoy/config/accesslog/v2/als.pb.h" +#include "envoy/config/filter/accesslog/v2/accesslog.pb.h" +#include "envoy/grpc/async_client.h" +#include "envoy/grpc/async_client_manager.h" +#include "envoy/local_info/local_info.h" +#include "envoy/service/accesslog/v2/als.pb.h" +#include "envoy/singleton/instance.h" +#include "envoy/thread_local/thread_local.h" + +#include "common/grpc/typed_async_client.h" + +#include "extensions/access_loggers/common/access_log_base.h" +#include "extensions/access_loggers/grpc/grpc_access_log_impl.h" + +namespace Envoy { +namespace Extensions { +namespace AccessLoggers { +namespace TcpGrpc { + +// TODO(mattklein123): Stats + +/** + * Access log Instance that streams TCP logs over gRPC. + */ +class TcpGrpcAccessLog : public Common::ImplBase { +public: + TcpGrpcAccessLog(AccessLog::FilterPtr&& filter, + envoy::config::accesslog::v2::TcpGrpcAccessLogConfig config, + ThreadLocal::SlotAllocator& tls, + GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache); + +private: + /** + * Per-thread cached logger. + */ + struct ThreadLocalLogger : public ThreadLocal::ThreadLocalObject { + ThreadLocalLogger(GrpcCommon::GrpcAccessLoggerSharedPtr logger); + + const GrpcCommon::GrpcAccessLoggerSharedPtr logger_; + }; + + // Common::ImplBase + void emitLog(const Http::HeaderMap& request_headers, const Http::HeaderMap& response_headers, + const Http::HeaderMap& response_trailers, + const StreamInfo::StreamInfo& stream_info) override; + + const envoy::config::accesslog::v2::TcpGrpcAccessLogConfig config_; + const ThreadLocal::SlotPtr tls_slot_; + const GrpcCommon::GrpcAccessLoggerCacheSharedPtr access_logger_cache_; +}; + +} // namespace TcpGrpc +} // namespace AccessLoggers +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/access_loggers/well_known_names.h b/source/extensions/access_loggers/well_known_names.h index 7c5e7576c1df..dc55f536bfad 100644 --- a/source/extensions/access_loggers/well_known_names.h +++ b/source/extensions/access_loggers/well_known_names.h @@ -18,6 +18,8 @@ class AccessLogNameValues { const std::string File = "envoy.file_access_log"; // HTTP gRPC access log const std::string HttpGrpc = "envoy.http_grpc_access_log"; + // TCP gRPC access log + const std::string TcpGrpc = "envoy.tcp_grpc_access_log"; }; using AccessLogNames = ConstSingleton; diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index f8b1ccc5158f..5366f7ba55fb 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -6,6 +6,7 @@ EXTENSIONS = { "envoy.access_loggers.file": "//source/extensions/access_loggers/file:config", "envoy.access_loggers.http_grpc": "//source/extensions/access_loggers/grpc:http_config", + "envoy.access_loggers.tcp_grpc": "//source/extensions/access_loggers/grpc:tcp_config", # # Clusters diff --git a/test/extensions/access_loggers/grpc/BUILD b/test/extensions/access_loggers/grpc/BUILD index 9e2c7b46240f..652fc010d578 100644 --- a/test/extensions/access_loggers/grpc/BUILD +++ b/test/extensions/access_loggers/grpc/BUILD @@ -77,3 +77,20 @@ envoy_extension_cc_test( "//test/test_common:utility_lib", ], ) + +envoy_extension_cc_test( + name = "tcp_grpc_access_log_integration_test", + srcs = ["tcp_grpc_access_log_integration_test.cc"], + extension_name = "envoy.access_loggers.http_grpc", + deps = [ + "//source/common/buffer:zero_copy_input_stream_lib", + "//source/common/grpc:codec_lib", + "//source/common/grpc:common_lib", + "//source/extensions/access_loggers/grpc:http_config", + "//source/extensions/access_loggers/grpc:tcp_config", + "//source/extensions/filters/network/tcp_proxy:config", + "//test/common/grpc:grpc_client_integration_lib", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 83ba234f7973..07875b008ddf 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -271,21 +271,26 @@ TEST_F(GrpcAccessLoggerCacheImplTest, Deduplication) { config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("cluster-1"); expectClientCreation(); - GrpcAccessLoggerSharedPtr logger1 = logger_cache_->getOrCreateLogger(config); - EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config)); + GrpcAccessLoggerSharedPtr logger1 = + logger_cache_->getOrCreateLogger(config, GrpcAccessLoggerType::HTTP); + EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config, GrpcAccessLoggerType::HTTP)); + + // Do not deduplicate different types of logger + expectClientCreation(); + EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config, GrpcAccessLoggerType::TCP)); // Changing log name leads to another logger. config.set_log_name("log-2"); expectClientCreation(); - EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config)); + EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config, GrpcAccessLoggerType::HTTP)); config.set_log_name("log-1"); - EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config)); + EXPECT_EQ(logger1, logger_cache_->getOrCreateLogger(config, GrpcAccessLoggerType::HTTP)); // Changing cluster name leads to another logger. config.mutable_grpc_service()->mutable_envoy_grpc()->set_cluster_name("cluster-2"); expectClientCreation(); - EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config)); + EXPECT_NE(logger1, logger_cache_->getOrCreateLogger(config, GrpcAccessLoggerType::HTTP)); } } // namespace diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 22e1f7cd59c0..893d23d61243 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -14,6 +14,7 @@ using namespace std::chrono_literals; using testing::_; +using testing::An; using testing::InSequence; using testing::Invoke; using testing::NiceMock; @@ -25,18 +26,22 @@ namespace AccessLoggers { namespace HttpGrpc { namespace { +using envoy::data::accesslog::v2::HTTPAccessLogEntry; + class MockGrpcAccessLogger : public GrpcCommon::GrpcAccessLogger { public: // GrpcAccessLogger - MOCK_METHOD1(log, void(envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry)); + MOCK_METHOD1(log, void(HTTPAccessLogEntry&& entry)); + MOCK_METHOD1(log, void(envoy::data::accesslog::v2::TCPAccessLogEntry&& entry)); }; class MockGrpcAccessLoggerCache : public GrpcCommon::GrpcAccessLoggerCache { public: // GrpcAccessLoggerCache - MOCK_METHOD1(getOrCreateLogger, + MOCK_METHOD2(getOrCreateLogger, GrpcCommon::GrpcAccessLoggerSharedPtr( - const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config)); + const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config, + GrpcCommon::GrpcAccessLoggerType logger_type)); }; class HttpGrpcAccessLogTest : public testing::Test { @@ -44,9 +49,11 @@ class HttpGrpcAccessLogTest : public testing::Test { void init() { ON_CALL(*filter_, evaluate(_, _, _, _)).WillByDefault(Return(true)); config_.mutable_common_config()->set_log_name("hello_log"); - EXPECT_CALL(*logger_cache_, getOrCreateLogger(_)) - .WillOnce([this](const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config) { + EXPECT_CALL(*logger_cache_, getOrCreateLogger(_, _)) + .WillOnce([this](const ::envoy::config::accesslog::v2::CommonGrpcAccessLogConfig& config, + GrpcCommon::GrpcAccessLoggerType logger_type) { EXPECT_EQ(config.DebugString(), config_.common_config().DebugString()); + EXPECT_EQ(GrpcCommon::GrpcAccessLoggerType::HTTP, logger_type); return logger_; }); access_log_ = std::make_unique(AccessLog::FilterPtr{filter_}, config_, tls_, @@ -58,9 +65,9 @@ class HttpGrpcAccessLogTest : public testing::Test { init(); } - envoy::data::accesslog::v2::HTTPAccessLogEntry expected_log_entry; + HTTPAccessLogEntry expected_log_entry; TestUtility::loadFromYaml(expected_log_entry_yaml, expected_log_entry); - EXPECT_CALL(*logger_, log(_)) + EXPECT_CALL(*logger_, log(An())) .WillOnce( Invoke([expected_log_entry](envoy::data::accesslog::v2::HTTPAccessLogEntry&& entry) { EXPECT_EQ(entry.DebugString(), expected_log_entry.DebugString()); diff --git a/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc b/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc new file mode 100644 index 000000000000..8ad6bbe7bb68 --- /dev/null +++ b/test/extensions/access_loggers/grpc/tcp_grpc_access_log_integration_test.cc @@ -0,0 +1,189 @@ +#include "envoy/config/accesslog/v2/als.pb.h" +#include "envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.pb.validate.h" +#include "envoy/service/accesslog/v2/als.pb.h" + +#include "common/buffer/zero_copy_input_stream_impl.h" +#include "common/common/version.h" +#include "common/grpc/codec.h" +#include "common/grpc/common.h" + +#include "test/common/grpc/grpc_client_integration.h" +#include "test/integration/http_integration.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +using testing::AssertionResult; + +namespace Envoy { +namespace { + +void clearPort(envoy::api::v2::core::Address& address) { + address.mutable_socket_address()->clear_port_specifier(); +} + +class TcpGrpcAccessLogIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, + public BaseIntegrationTest { +public: + TcpGrpcAccessLogIntegrationTest() + : BaseIntegrationTest(ipVersion(), ConfigHelper::TCP_PROXY_CONFIG) { + enable_half_close_ = true; + } + + ~TcpGrpcAccessLogIntegrationTest() override { + test_server_.reset(); + fake_upstreams_.clear(); + } + + void createUpstreams() override { + BaseIntegrationTest::createUpstreams(); + fake_upstreams_.emplace_back( + new FakeUpstream(0, FakeHttpConnection::Type::HTTP2, version_, timeSystem())); + } + + void initialize() override { + config_helper_.renameListener("tcp_proxy"); + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + auto* accesslog_cluster = bootstrap.mutable_static_resources()->add_clusters(); + accesslog_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + accesslog_cluster->set_name("accesslog"); + accesslog_cluster->mutable_http2_protocol_options(); + }); + + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* filter_chain = listener->mutable_filter_chains(0); + auto* config_blob = filter_chain->mutable_filters(0)->mutable_config(); + + envoy::config::filter::network::tcp_proxy::v2::TcpProxy tcp_proxy_config; + TestUtility::jsonConvert(*config_blob, tcp_proxy_config); + + auto* access_log = tcp_proxy_config.add_access_log(); + access_log->set_name("envoy.tcp_grpc_access_log"); + envoy::config::accesslog::v2::TcpGrpcAccessLogConfig access_log_config; + auto* common_config = access_log_config.mutable_common_config(); + common_config->set_log_name("foo"); + setGrpcService(*common_config->mutable_grpc_service(), "accesslog", + fake_upstreams_.back()->localAddress()); + TestUtility::jsonConvert(access_log_config, *access_log->mutable_config()); + + TestUtility::jsonConvert(tcp_proxy_config, *config_blob); + }); + BaseIntegrationTest::initialize(); + } + + ABSL_MUST_USE_RESULT + AssertionResult waitForAccessLogConnection() { + return fake_upstreams_[1]->waitForHttpConnection(*dispatcher_, fake_access_log_connection_); + } + + ABSL_MUST_USE_RESULT + AssertionResult waitForAccessLogStream() { + return fake_access_log_connection_->waitForNewStream(*dispatcher_, access_log_request_); + } + + ABSL_MUST_USE_RESULT + AssertionResult waitForAccessLogRequest(const std::string& expected_request_msg_yaml) { + envoy::service::accesslog::v2::StreamAccessLogsMessage request_msg; + VERIFY_ASSERTION(access_log_request_->waitForGrpcMessage(*dispatcher_, request_msg)); + EXPECT_EQ("POST", access_log_request_->headers().Method()->value().getStringView()); + EXPECT_EQ("/envoy.service.accesslog.v2.AccessLogService/StreamAccessLogs", + access_log_request_->headers().Path()->value().getStringView()); + EXPECT_EQ("application/grpc", + access_log_request_->headers().ContentType()->value().getStringView()); + + envoy::service::accesslog::v2::StreamAccessLogsMessage expected_request_msg; + TestUtility::loadFromYaml(expected_request_msg_yaml, expected_request_msg); + + // Clear fields which are not deterministic. + auto* log_entry = request_msg.mutable_tcp_logs()->mutable_log_entry(0); + clearPort(*log_entry->mutable_common_properties()->mutable_downstream_remote_address()); + clearPort(*log_entry->mutable_common_properties()->mutable_downstream_local_address()); + clearPort(*log_entry->mutable_common_properties()->mutable_upstream_remote_address()); + clearPort(*log_entry->mutable_common_properties()->mutable_upstream_local_address()); + log_entry->mutable_common_properties()->clear_start_time(); + log_entry->mutable_common_properties()->clear_time_to_last_rx_byte(); + log_entry->mutable_common_properties()->clear_time_to_first_downstream_tx_byte(); + log_entry->mutable_common_properties()->clear_time_to_last_downstream_tx_byte(); + EXPECT_EQ(request_msg.DebugString(), expected_request_msg.DebugString()); + + return AssertionSuccess(); + } + + void cleanup() { + if (fake_access_log_connection_ != nullptr) { + AssertionResult result = fake_access_log_connection_->close(); + RELEASE_ASSERT(result, result.message()); + result = fake_access_log_connection_->waitForDisconnect(); + RELEASE_ASSERT(result, result.message()); + fake_access_log_connection_ = nullptr; + } + } + + FakeHttpConnectionPtr fake_access_log_connection_; + FakeStreamPtr access_log_request_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersionsCientType, TcpGrpcAccessLogIntegrationTest, + GRPC_CLIENT_INTEGRATION_PARAMS); + +// Test a basic full access logging flow. +TEST_P(TcpGrpcAccessLogIntegrationTest, BasicAccessLogFlow) { + initialize(); + + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("tcp_proxy")); + FakeRawConnectionPtr fake_upstream_connection; + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection)); + + ASSERT_TRUE(fake_upstream_connection->write("hello")); + tcp_client->waitForData("hello"); + tcp_client->write("bar", false); + + ASSERT_TRUE(fake_upstream_connection->write("", true)); + tcp_client->waitForHalfClose(); + tcp_client->write("", true); + ASSERT_TRUE(fake_upstream_connection->waitForHalfClose()); + ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); + + ASSERT_TRUE(waitForAccessLogConnection()); + ASSERT_TRUE(waitForAccessLogStream()); + ASSERT_TRUE(waitForAccessLogRequest( + fmt::format(R"EOF( +identifier: + node: + id: node_name + cluster: cluster_name + locality: + zone: zone_name + build_version: {} + log_name: foo +tcp_logs: + log_entry: + common_properties: + downstream_remote_address: + socket_address: + address: {} + downstream_local_address: + socket_address: + address: {} + upstream_remote_address: + socket_address: + address: {} + upstream_local_address: + socket_address: + address: {} + upstream_cluster: cluster_0 + connection_properties: + received_bytes: 3 + sent_bytes: 5 +)EOF", + VersionInfo::version(), Network::Test::getLoopbackAddressString(ipVersion()), + Network::Test::getLoopbackAddressString(ipVersion()), + Network::Test::getLoopbackAddressString(ipVersion()), + Network::Test::getLoopbackAddressString(ipVersion())))); + + cleanup(); +} + +} // namespace +} // namespace Envoy From 816d6f14fd65329a8511351e74b385fafa3160a8 Mon Sep 17 00:00:00 2001 From: easy Date: Tue, 27 Aug 2019 13:15:52 +1000 Subject: [PATCH 458/542] tracing: add OpenCensus agent exporter support to OpenCensus driver. (#8023) Signed-off-by: Emil Mikulic --- api/bazel/repository_locations.bzl | 8 ++++---- api/envoy/config/trace/v2/trace.proto | 8 ++++++++ bazel/repositories.bzl | 4 ++++ bazel/repository_locations.bzl | 8 ++++---- source/extensions/tracers/opencensus/BUILD | 1 + .../tracers/opencensus/opencensus_tracer_impl.cc | 6 ++++++ test/extensions/tracers/opencensus/config_test.cc | 2 ++ 7 files changed, 29 insertions(+), 8 deletions(-) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 7b0bfeb07539..2febf71148b7 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -4,8 +4,8 @@ BAZEL_SKYLIB_SHA256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc70 GOGOPROTO_RELEASE = "1.2.1" GOGOPROTO_SHA256 = "99e423905ba8921e86817607a5294ffeedb66fdd4a85efce5eb2848f715fdb3a" -OPENCENSUS_PROTO_RELEASE = "0.2.1" -OPENCENSUS_PROTO_SHA256 = "bfcefa6093fc2ecdf5c9effea86e6982d0d6f9a5178b17fcf73a62e0f3fb43d0" +OPENCENSUS_PROTO_GIT_SHA = "5cec5ea58c3efa81fa808f2bd38ce182da9ee731" # Jul 25, 2019 +OPENCENSUS_PROTO_SHA256 = "faeb93f293ff715b0cb530d273901c0e2e99277b9ed1c0a0326bca9ec5774ad2" PGV_GIT_SHA = "2feaabb13a5d697b80fcb938c0ce37b24c9381ee" # Jul 26, 2018 PGV_SHA256 = "ddefe3dcbb25d68a2e5dfea67d19c060959c2aecc782802bd4c1a5811d44dd45" @@ -54,8 +54,8 @@ REPOSITORY_LOCATIONS = dict( ), opencensus_proto = dict( sha256 = OPENCENSUS_PROTO_SHA256, - strip_prefix = "opencensus-proto-" + OPENCENSUS_PROTO_RELEASE + "/src", - urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/v" + OPENCENSUS_PROTO_RELEASE + ".tar.gz"], + strip_prefix = "opencensus-proto-" + OPENCENSUS_PROTO_GIT_SHA + "/src", + urls = ["https://github.com/census-instrumentation/opencensus-proto/archive/" + OPENCENSUS_PROTO_GIT_SHA + ".tar.gz"], ), kafka_source = dict( sha256 = KAFKA_SOURCE_SHA, diff --git a/api/envoy/config/trace/v2/trace.proto b/api/envoy/config/trace/v2/trace.proto index e4e4dec64e95..9da6ef4313d8 100644 --- a/api/envoy/config/trace/v2/trace.proto +++ b/api/envoy/config/trace/v2/trace.proto @@ -136,6 +136,14 @@ message OpenCensusConfig { // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v2/spans" string zipkin_url = 6; + // Enables the OpenCensus Agent exporter if set to true. The address must also + // be set. + bool ocagent_exporter_enabled = 11; + + // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC + // format: https://github.com/grpc/grpc/blob/master/doc/naming.md + string ocagent_address = 12; + reserved 7; // Formerly zipkin_service_name. enum TraceContext { diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 319d374286bd..15dbd7cb0e8c 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -549,6 +549,10 @@ def _io_opencensus_cpp(): name = "opencensus_trace_trace_context", actual = "@io_opencensus_cpp//opencensus/trace:trace_context", ) + native.bind( + name = "opencensus_exporter_ocagent", + actual = "@io_opencensus_cpp//opencensus/exporters/trace/ocagent:ocagent_exporter", + ) native.bind( name = "opencensus_exporter_stdout", actual = "@io_opencensus_cpp//opencensus/exporters/trace/stdout:stdout_exporter", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 7c2c5ad23e62..2abff095453a 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -232,10 +232,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://files.pythonhosted.org/packages/b3/b2/238e2590826bfdd113244a40d9d3eb26918bd798fc187e2360a8367068db/six-1.10.0.tar.gz"], ), io_opencensus_cpp = dict( - sha256 = "8d6016e47c2e19e7acbadb6f905b8c422748c64299d71101ac8f28151677e195", - strip_prefix = "opencensus-cpp-cad0d03ff3474cf14389fc249e16847ab7b6895f", - # 2019-07-31 - urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/cad0d03ff3474cf14389fc249e16847ab7b6895f.tar.gz"], + sha256 = "b5fcc36a994a4ecb6e53c901e33579ed1ac238cff9975807db760918a15f43ff", + strip_prefix = "opencensus-cpp-8058a1b8efe6a63bd18673abc51223917d12d45d", + # 2019-08-22 + urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/8058a1b8efe6a63bd18673abc51223917d12d45d.tar.gz"], ), com_github_curl = dict( sha256 = "4376ac72b95572fb6c4fbffefb97c7ea0dd083e1974c0e44cd7e49396f454839", diff --git a/source/extensions/tracers/opencensus/BUILD b/source/extensions/tracers/opencensus/BUILD index c318395cad66..0e7c9085fcab 100644 --- a/source/extensions/tracers/opencensus/BUILD +++ b/source/extensions/tracers/opencensus/BUILD @@ -32,6 +32,7 @@ envoy_cc_library( "opencensus_trace_cloud_trace_context", "opencensus_trace_grpc_trace_bin", "opencensus_trace_trace_context", + "opencensus_exporter_ocagent", "opencensus_exporter_stdout", "opencensus_exporter_stackdriver", "opencensus_exporter_zipkin", diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index 9be9d3484868..a5562d20c81f 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -8,6 +8,7 @@ #include "absl/strings/str_cat.h" #include "google/devtools/cloudtrace/v2/tracing.grpc.pb.h" +#include "opencensus/exporters/trace/ocagent/ocagent_exporter.h" #include "opencensus/exporters/trace/stackdriver/stackdriver_exporter.h" #include "opencensus/exporters/trace/stdout/stdout_exporter.h" #include "opencensus/exporters/trace/zipkin/zipkin_exporter.h" @@ -257,6 +258,11 @@ Driver::Driver(const envoy::config::trace::v2::OpenCensusConfig& oc_config, opts.service_name = local_info_.clusterName(); ::opencensus::exporters::trace::ZipkinExporter::Register(opts); } + if (oc_config.ocagent_exporter_enabled()) { + ::opencensus::exporters::trace::OcAgentOptions opts; + opts.address = oc_config.ocagent_address(); + ::opencensus::exporters::trace::OcAgentExporter::Register(std::move(opts)); + } } void Driver::applyTraceConfig(const opencensus::proto::trace::v1::TraceConfig& config) { diff --git a/test/extensions/tracers/opencensus/config_test.cc b/test/extensions/tracers/opencensus/config_test.cc index 686f3e5b5813..95cd8767f86b 100644 --- a/test/extensions/tracers/opencensus/config_test.cc +++ b/test/extensions/tracers/opencensus/config_test.cc @@ -50,6 +50,8 @@ TEST(OpenCensusTracerConfigTest, OpenCensusHttpTracerWithTypedConfig) { stackdriver_project_id: test_project_id zipkin_exporter_enabled: true zipkin_url: http://127.0.0.1:9411/api/v2/spans + ocagent_exporter_enabled: true + ocagent_address: 127.0.0.1:55678 incoming_trace_context: b3 incoming_trace_context: trace_context incoming_trace_context: grpc_trace_bin From e1cd4cca46179562cf80de7f81774f4dc6354085 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Tue, 27 Aug 2019 17:07:55 +0200 Subject: [PATCH 459/542] Exporting platform_impl_lib headers (#8045) This allows consuming projects using repository overlaying to disambiguate overlapping include paths when it comes to platform_impl.h by going through envoy/external/... Addendum to #8005 Risk Level: Low Testing: N/A Signed-off-by: Otto van der Schaaf --- source/exe/BUILD | 29 ++++++++++++++++++++++------ source/exe/platform_impl.h | 19 ++++++++++++++++++ source/exe/posix/platform_impl.cc | 12 ++++++++++++ source/exe/posix/platform_impl.h | 18 ----------------- source/exe/win32/platform_impl.cc | 26 +++++++++++++++++++++++++ source/exe/win32/platform_impl.h | 32 ------------------------------- 6 files changed, 80 insertions(+), 56 deletions(-) create mode 100644 source/exe/platform_impl.h create mode 100644 source/exe/posix/platform_impl.cc delete mode 100644 source/exe/posix/platform_impl.h create mode 100644 source/exe/win32/platform_impl.cc delete mode 100644 source/exe/win32/platform_impl.h diff --git a/source/exe/BUILD b/source/exe/BUILD index d48f99ce33b8..7ed716a317a6 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -57,7 +57,8 @@ envoy_cc_library( ], deps = [ ":envoy_main_common_lib", - ] + envoy_cc_platform_dep("platform_impl_lib"), + ":platform_impl_lib", + ], ) envoy_cc_library( @@ -66,6 +67,7 @@ envoy_cc_library( hdrs = ["main_common.h"], deps = [ ":envoy_common_lib", + ":platform_impl_lib", ":process_wide_lib", "//source/common/api:os_sys_calls_lib", "//source/common/common:compiler_requirements_lib", @@ -80,7 +82,7 @@ envoy_cc_library( "//source/common/signal:sigaction_lib", ":terminate_handler_lib", ], - }) + envoy_cc_platform_dep("platform_impl_lib"), + }), ) envoy_cc_library( @@ -96,11 +98,26 @@ envoy_cc_library( ] + envoy_google_grpc_external_deps(), ) +envoy_cc_library( + name = "platform_impl_lib", + deps = [":platform_header_lib"] + + envoy_cc_platform_dep("platform_impl_lib"), +) + +envoy_cc_library( + name = "platform_header_lib", + hdrs = ["platform_impl.h"], + deps = [ + "//include/envoy/filesystem:filesystem_interface", + "//include/envoy/thread:thread_interface", + ], +) + envoy_cc_posix_library( name = "platform_impl_lib", - hdrs = ["posix/platform_impl.h"], - strip_include_prefix = "posix", + srcs = ["posix/platform_impl.cc"], deps = [ + ":platform_header_lib", "//source/common/common:thread_lib", "//source/common/filesystem:filesystem_lib", ], @@ -108,9 +125,9 @@ envoy_cc_posix_library( envoy_cc_win32_library( name = "platform_impl_lib", - hdrs = ["win32/platform_impl.h"], - strip_include_prefix = "win32", + srcs = ["win32/platform_impl.cc"], deps = [ + ":platform_header_lib", "//source/common/common:assert_lib", "//source/common/common:thread_lib", "//source/common/filesystem:filesystem_lib", diff --git a/source/exe/platform_impl.h b/source/exe/platform_impl.h new file mode 100644 index 000000000000..c52152c8fe6d --- /dev/null +++ b/source/exe/platform_impl.h @@ -0,0 +1,19 @@ +#pragma once + +#include "envoy/filesystem/filesystem.h" +#include "envoy/thread/thread.h" + +namespace Envoy { + +class PlatformImpl { +public: + PlatformImpl(); + Thread::ThreadFactory& threadFactory() { return *thread_factory_; } + Filesystem::Instance& fileSystem() { return *file_system_; } + +private: + std::unique_ptr thread_factory_; + std::unique_ptr file_system_; +}; + +} // namespace Envoy diff --git a/source/exe/posix/platform_impl.cc b/source/exe/posix/platform_impl.cc new file mode 100644 index 000000000000..f664071815ee --- /dev/null +++ b/source/exe/posix/platform_impl.cc @@ -0,0 +1,12 @@ +#include "common/common/thread_impl.h" +#include "common/filesystem/filesystem_impl.h" + +#include "exe/platform_impl.h" + +namespace Envoy { + +PlatformImpl::PlatformImpl() + : thread_factory_(std::make_unique()), + file_system_(std::make_unique()) {} + +} // namespace Envoy diff --git a/source/exe/posix/platform_impl.h b/source/exe/posix/platform_impl.h deleted file mode 100644 index 45fbd7340779..000000000000 --- a/source/exe/posix/platform_impl.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include "common/common/thread_impl.h" -#include "common/filesystem/filesystem_impl.h" - -namespace Envoy { - -class PlatformImpl { -public: - Thread::ThreadFactory& threadFactory() { return thread_factory_; } - Filesystem::Instance& fileSystem() { return file_system_; } - -private: - Thread::ThreadFactoryImplPosix thread_factory_; - Filesystem::InstanceImplPosix file_system_; -}; - -} // namespace Envoy diff --git a/source/exe/win32/platform_impl.cc b/source/exe/win32/platform_impl.cc new file mode 100644 index 000000000000..860a209a8e05 --- /dev/null +++ b/source/exe/win32/platform_impl.cc @@ -0,0 +1,26 @@ +#include "common/common/assert.h" +#include "common/common/thread_impl.h" +#include "common/filesystem/filesystem_impl.h" + +#include "exe/platform_impl.h" + +// clang-format off +#include +// clang-format on + +namespace Envoy { + +PlatformImpl::PlatformImpl() + : thread_factory_(std::make_unique()), + file_system_(std::make_unique()) { + const WORD wVersionRequested = MAKEWORD(2, 2); + WSADATA wsaData; + const int rc = ::WSAStartup(wVersionRequested, &wsaData); + RELEASE_ASSERT(rc == 0, "WSAStartup failed with error"); +} + +~PlatformImpl() { ::WSACleanup(); } + +}; // namespace Envoy + +} // namespace Envoy diff --git a/source/exe/win32/platform_impl.h b/source/exe/win32/platform_impl.h deleted file mode 100644 index ffb239dd7ebb..000000000000 --- a/source/exe/win32/platform_impl.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include "common/common/assert.h" -#include "common/common/thread_impl.h" -#include "common/filesystem/filesystem_impl.h" - -// clang-format off -#include -// clang-format on - -namespace Envoy { - -class PlatformImpl { -public: - PlatformImpl() { - const WORD wVersionRequested = MAKEWORD(2, 2); - WSADATA wsaData; - const int rc = ::WSAStartup(wVersionRequested, &wsaData); - RELEASE_ASSERT(rc == 0, "WSAStartup failed with error"); - } - - ~PlatformImpl() { ::WSACleanup(); } - - Thread::ThreadFactory& threadFactory() { return thread_factory_; } - Filesystem::Instance& fileSystem() { return file_system_; } - -private: - Thread::ThreadFactoryImplWin32 thread_factory_; - Filesystem::InstanceImplWin32 file_system_; -}; - -} // namespace Envoy From 854e80072f9f602c49b9633e0af3b4689cd3cbbf Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Tue, 27 Aug 2019 10:32:29 -0700 Subject: [PATCH 460/542] access_log: minimal log file error handling (#7938) Rather than ASSERT for a reasonably common error condition (e.g. disk full) record a stat that indicates log file writing failed. Also fixes a test race condition. Risk Level: low Testing: added stats checks Docs Changes: documented new stat Release Notes: updated Signed-off-by: Stephan Zuercher --- .../observability/statistics.rst | 7 +- docs/root/intro/version_history.rst | 1 + .../access_log/access_log_manager_impl.cc | 8 +- .../access_log/access_log_manager_impl.h | 7 +- .../access_log_manager_impl_test.cc | 96 +++++++++++++++++-- test/exe/main_common_test.cc | 8 +- 6 files changed, 107 insertions(+), 20 deletions(-) diff --git a/docs/root/configuration/observability/statistics.rst b/docs/root/configuration/observability/statistics.rst index 0042051e0acb..376263f42bb4 100644 --- a/docs/root/configuration/observability/statistics.rst +++ b/docs/root/configuration/observability/statistics.rst @@ -16,7 +16,7 @@ Server related statistics are rooted at *server.* with following statistics: uptime, Gauge, Current server uptime in seconds concurrency, Gauge, Number of worker threads - memory_allocated, Gauge, Current amount of allocated memory in bytes. Total of both new and old Envoy processes on hot restart. + memory_allocated, Gauge, Current amount of allocated memory in bytes. Total of both new and old Envoy processes on hot restart. memory_heap_size, Gauge, Current reserved heap size in bytes. New Envoy process heap size on hot restart. live, Gauge, "1 if the server is not currently draining, 0 otherwise" state, Gauge, Current :ref:`State ` of the Server. @@ -30,6 +30,8 @@ Server related statistics are rooted at *server.* with following statistics: static_unknown_fields, Counter, Number of messages in static configuration with unknown fields dynamic_unknown_fields, Counter, Number of messages in dynamic configuration with unknown fields +.. _filesystem_stats: + File system ----------- @@ -40,7 +42,8 @@ Statistics related to file system are emitted in the *filesystem.* namespace. :widths: 1, 1, 2 write_buffered, Counter, Total number of times file data is moved to Envoy's internal flush buffer - write_completed, Counter, Total number of times a file was written + write_completed, Counter, Total number of times a file was successfully written + write_failed, Counter, Total number of times an error occurred during a file write operation flushed_by_timer, Counter, Total number of times internal flush buffers are written to a file due to flush timeout reopen_failed, Counter, Total number of times a file was failed to be opened write_total_buffered, Gauge, Current total size of internal flush buffer in bytes diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 63fb953c060a..5152e89fdc65 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -5,6 +5,7 @@ Version history ================ * access log: added :ref:`buffering ` and :ref:`periodical flushing ` support to gRPC access logger. Defaults to 16KB buffer and flushing every 1 second. * access log: gRPC Access Log Service (ALS) support added for :ref:`TCP access logs `. +* access log: reintroduce :ref:`filesystem ` stats and added the `write_failed` counter to track failed log writes * admin: added ability to configure listener :ref:`socket options `. * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * api: added ::ref:`set_node_on_first_message_only ` option to omit the node identifier from the subsequent discovery requests on the same stream. diff --git a/source/common/access_log/access_log_manager_impl.cc b/source/common/access_log/access_log_manager_impl.cc index cf5a98daaf47..fd576b9b00e1 100644 --- a/source/common/access_log/access_log_manager_impl.cc +++ b/source/common/access_log/access_log_manager_impl.cc @@ -100,8 +100,12 @@ void AccessLogFileImpl::doWrite(Buffer::Instance& buffer) { for (const Buffer::RawSlice& slice : slices) { absl::string_view data(static_cast(slice.mem_), slice.len_); const Api::IoCallSizeResult result = file_->write(data); - ASSERT(result.rc_ == static_cast(slice.len_)); - stats_.write_completed_.inc(); + if (result.ok() && result.rc_ == static_cast(slice.len_)) { + stats_.write_completed_.inc(); + } else { + // Probably disk full. + stats_.write_failed_.inc(); + } } } diff --git a/source/common/access_log/access_log_manager_impl.h b/source/common/access_log/access_log_manager_impl.h index 03efbf2d030b..64c9c2594b49 100644 --- a/source/common/access_log/access_log_manager_impl.h +++ b/source/common/access_log/access_log_manager_impl.h @@ -20,6 +20,7 @@ namespace Envoy { COUNTER(reopen_failed) \ COUNTER(write_buffered) \ COUNTER(write_completed) \ + COUNTER(write_failed) \ GAUGE(write_total_buffered, Accumulate) struct AccessLogFileStats { @@ -34,9 +35,9 @@ class AccessLogManagerImpl : public AccessLogManager { Event::Dispatcher& dispatcher, Thread::BasicLockable& lock, Stats::Store& stats_store) : file_flush_interval_msec_(file_flush_interval_msec), api_(api), dispatcher_(dispatcher), - lock_(lock), file_stats_{ACCESS_LOG_FILE_STATS( - POOL_COUNTER_PREFIX(stats_store, "access_log_file."), - POOL_GAUGE_PREFIX(stats_store, "access_log_file."))} {} + lock_(lock), file_stats_{ + ACCESS_LOG_FILE_STATS(POOL_COUNTER_PREFIX(stats_store, "filesystem."), + POOL_GAUGE_PREFIX(stats_store, "filesystem."))} {} // AccessLog::AccessLogManager void reopen() override; diff --git a/test/common/access_log/access_log_manager_impl_test.cc b/test/common/access_log/access_log_manager_impl_test.cc index d8d9446d856f..3b28ee6526a2 100644 --- a/test/common/access_log/access_log_manager_impl_test.cc +++ b/test/common/access_log/access_log_manager_impl_test.cc @@ -8,6 +8,8 @@ #include "test/mocks/api/mocks.h" #include "test/mocks/event/mocks.h" #include "test/mocks/filesystem/mocks.h" +#include "test/test_common/test_time.h" +#include "test/test_common/utility.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -37,6 +39,14 @@ class AccessLogManagerImplTest : public testing::Test { EXPECT_CALL(api_, threadFactory()).WillRepeatedly(ReturnRef(thread_factory_)); } + void waitForCounterEq(const std::string& name, uint64_t value) { + TestUtility::waitForCounterEq(store_, name, value, time_system_); + } + + void waitForGaugeEq(const std::string& name, uint64_t value) { + TestUtility::waitForGaugeEq(store_, name, value, time_system_); + } + NiceMock api_; NiceMock file_system_; NiceMock* file_; @@ -46,6 +56,7 @@ class AccessLogManagerImplTest : public testing::Test { NiceMock dispatcher_; Thread::MutexBasicLockable lock_; AccessLogManagerImpl access_log_manager_; + Event::TestRealTimeSystem time_system_; }; TEST_F(AccessLogManagerImplTest, BadFile) { @@ -74,9 +85,18 @@ TEST_F(AccessLogManagerImplTest, flushToLogFilePeriodically) { EXPECT_CALL(*file_, open_(_)).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); + EXPECT_EQ(0UL, store_.counter("filesystem.write_failed").value()); + EXPECT_EQ(0UL, store_.counter("filesystem.write_completed").value()); + EXPECT_EQ(0UL, store_.counter("filesystem.flushed_by_timer").value()); + EXPECT_EQ(0UL, store_.counter("filesystem.write_buffered").value()); + EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); EXPECT_CALL(*file_, write_(_)) - .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { + .WillOnce(Invoke([&](absl::string_view data) -> Api::IoCallSizeResult { + EXPECT_EQ( + 4UL, + store_.gauge("filesystem.write_total_buffered", Stats::Gauge::ImportMode::Accumulate) + .value()); EXPECT_EQ(0, data.compare("test")); return Filesystem::resultSuccess(static_cast(data.length())); })); @@ -90,14 +110,25 @@ TEST_F(AccessLogManagerImplTest, flushToLogFilePeriodically) { } } + waitForCounterEq("filesystem.write_completed", 1); + EXPECT_EQ(1UL, store_.counter("filesystem.write_buffered").value()); + EXPECT_EQ(0UL, store_.counter("filesystem.flushed_by_timer").value()); + waitForGaugeEq("filesystem.write_total_buffered", 0); + EXPECT_CALL(*file_, write_(_)) - .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { + .WillOnce(Invoke([&](absl::string_view data) -> Api::IoCallSizeResult { + EXPECT_EQ( + 5UL, + store_.gauge("filesystem.write_total_buffered", Stats::Gauge::ImportMode::Accumulate) + .value()); EXPECT_EQ(0, data.compare("test2")); return Filesystem::resultSuccess(static_cast(data.length())); })); - // make sure timer is re-enabled on callback call log_file->write("test2"); + EXPECT_EQ(2UL, store_.counter("filesystem.write_buffered").value()); + + // make sure timer is re-enabled on callback call EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); timer->invokeCallback(); @@ -107,6 +138,13 @@ TEST_F(AccessLogManagerImplTest, flushToLogFilePeriodically) { file_->write_event_.wait(file_->write_mutex_); } } + + waitForCounterEq("filesystem.write_completed", 2); + EXPECT_EQ(0UL, store_.counter("filesystem.write_failed").value()); + EXPECT_EQ(1UL, store_.counter("filesystem.flushed_by_timer").value()); + EXPECT_EQ(2UL, store_.counter("filesystem.write_buffered").value()); + waitForGaugeEq("filesystem.write_total_buffered", 0); + EXPECT_CALL(*file_, close_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); } @@ -116,21 +154,24 @@ TEST_F(AccessLogManagerImplTest, flushToLogFileOnDemand) { EXPECT_CALL(*file_, open_(_)).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); + EXPECT_EQ(0UL, store_.counter("filesystem.flushed_by_timer").value()); + EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); - // The first write to a given file will start the flush thread, which can flush - // immediately (race on whether it will or not). So do a write and flush to - // get that state out of the way, then test that small writes don't trigger a flush. + // The first write to a given file will start the flush thread. Because AccessManagerImpl::write + // holds the write_lock_ when the thread is started, the thread will flush on its first loop, once + // it obtains the write_lock_. Perform a write to get all that out of the way. EXPECT_CALL(*file_, write_(_)) .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { return Filesystem::resultSuccess(static_cast(data.length())); })); log_file->write("prime-it"); - log_file->flush(); uint32_t expected_writes = 1; { Thread::LockGuard lock(file_->write_mutex_); - EXPECT_EQ(expected_writes, file_->num_writes_); + while (file_->num_writes_ != expected_writes) { + file_->write_event_.wait(file_->write_mutex_); + } } EXPECT_CALL(*file_, write_(_)) @@ -150,9 +191,14 @@ TEST_F(AccessLogManagerImplTest, flushToLogFileOnDemand) { expected_writes++; { Thread::LockGuard lock(file_->write_mutex_); - EXPECT_EQ(expected_writes, file_->num_writes_); + while (file_->num_writes_ != expected_writes) { + file_->write_event_.wait(file_->write_mutex_); + } } + waitForCounterEq("filesystem.write_completed", 2); + EXPECT_EQ(0UL, store_.counter("filesystem.flushed_by_timer").value()); + EXPECT_CALL(*file_, write_(_)) .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { EXPECT_EQ(0, data.compare("test2")); @@ -174,6 +220,36 @@ TEST_F(AccessLogManagerImplTest, flushToLogFileOnDemand) { EXPECT_CALL(*file_, close_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); } +TEST_F(AccessLogManagerImplTest, flushCountsIOErrors) { + NiceMock* timer = new NiceMock(&dispatcher_); + + EXPECT_CALL(*file_, open_(_)).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); + AccessLogFileSharedPtr log_file = access_log_manager_.createAccessLog("foo"); + + EXPECT_EQ(0UL, store_.counter("filesystem.write_failed").value()); + + EXPECT_CALL(*timer, enableTimer(timeout_40ms_, _)); + EXPECT_CALL(*file_, write_(_)) + .WillOnce(Invoke([](absl::string_view data) -> Api::IoCallSizeResult { + EXPECT_EQ(0, data.compare("test")); + return Filesystem::resultFailure(2UL, ENOSPC); + })); + + log_file->write("test"); + + { + Thread::LockGuard lock(file_->write_mutex_); + while (file_->num_writes_ != 1) { + file_->write_event_.wait(file_->write_mutex_); + } + } + + waitForCounterEq("filesystem.write_failed", 1); + EXPECT_EQ(0UL, store_.counter("filesystem.write_completed").value()); + + EXPECT_CALL(*file_, close_()).WillOnce(Return(ByMove(Filesystem::resultSuccess(true)))); +} + TEST_F(AccessLogManagerImplTest, reopenFile) { NiceMock* timer = new NiceMock(&dispatcher_); @@ -274,6 +350,8 @@ TEST_F(AccessLogManagerImplTest, reopenThrows) { // write call should not cause any exceptions log_file->write("random data"); timer->invokeCallback(); + + waitForCounterEq("filesystem.reopen_failed", 1); } TEST_F(AccessLogManagerImplTest, bigDataChunkShouldBeFlushedWithoutTimer) { diff --git a/test/exe/main_common_test.cc b/test/exe/main_common_test.cc index 4dadcb31875f..7af5aee32c6a 100644 --- a/test/exe/main_common_test.cc +++ b/test/exe/main_common_test.cc @@ -240,7 +240,7 @@ class AdminRequestTest : public MainCommonTest { TEST_P(AdminRequestTest, AdminRequestGetStatsAndQuit) { startEnvoy(); started_.WaitForNotification(); - EXPECT_THAT(adminRequest("/stats", "GET"), HasSubstr("access_log_file.reopen_failed")); + EXPECT_THAT(adminRequest("/stats", "GET"), HasSubstr("filesystem.reopen_failed")); adminRequest("/quitquitquit", "POST"); EXPECT_TRUE(waitForEnvoyToExit()); } @@ -253,7 +253,7 @@ TEST_P(AdminRequestTest, AdminRequestGetStatsAndKill) { // TODO(htuch): Remove when https://github.com/libevent/libevent/issues/779 is // fixed, started_ will then become our real synchronization point. waitForEnvoyRun(); - EXPECT_THAT(adminRequest("/stats", "GET"), HasSubstr("access_log_file.reopen_failed")); + EXPECT_THAT(adminRequest("/stats", "GET"), HasSubstr("filesystem.reopen_failed")); kill(getpid(), SIGTERM); EXPECT_TRUE(waitForEnvoyToExit()); } @@ -266,7 +266,7 @@ TEST_P(AdminRequestTest, AdminRequestGetStatsAndCtrlC) { // TODO(htuch): Remove when https://github.com/libevent/libevent/issues/779 is // fixed, started_ will then become our real synchronization point. waitForEnvoyRun(); - EXPECT_THAT(adminRequest("/stats", "GET"), HasSubstr("access_log_file.reopen_failed")); + EXPECT_THAT(adminRequest("/stats", "GET"), HasSubstr("filesystem.reopen_failed")); kill(getpid(), SIGINT); EXPECT_TRUE(waitForEnvoyToExit()); } @@ -335,7 +335,7 @@ TEST_P(AdminRequestTest, AdminRequestBeforeRun) { EXPECT_TRUE(admin_handler_was_called); // This just checks that some stat output was reported. We could pick any stat. - EXPECT_THAT(out, HasSubstr("access_log_file.reopen_failed")); + EXPECT_THAT(out, HasSubstr("filesystem.reopen_failed")); } // Class to track whether an object has been destroyed, which it does by bumping an atomic. From 1fc6c6e7cb15697e7cc8b99b7a39969f265130bd Mon Sep 17 00:00:00 2001 From: Caleb Gilmour Date: Wed, 28 Aug 2019 05:34:37 +1200 Subject: [PATCH 461/542] tracing: add grpc-status and grpc-message to spans (#7996) Signed-off-by: Caleb Gilmour --- .../arch_overview/observability/tracing.rst | 9 +- docs/root/intro/version_history.rst | 1 + source/common/http/conn_manager_impl.cc | 5 +- source/common/tracing/http_tracer_impl.cc | 26 +++ source/common/tracing/http_tracer_impl.h | 3 + test/common/tracing/http_tracer_impl_test.cc | 148 ++++++++++++++++-- 6 files changed, 176 insertions(+), 16 deletions(-) diff --git a/docs/root/intro/arch_overview/observability/tracing.rst b/docs/root/intro/arch_overview/observability/tracing.rst index bd8016424d9b..24072465e26b 100644 --- a/docs/root/intro/arch_overview/observability/tracing.rst +++ b/docs/root/intro/arch_overview/observability/tracing.rst @@ -93,9 +93,12 @@ associated with it. Each span generated by Envoy contains the following data: * Originating host set via :option:`--service-node`. * Downstream cluster set via the :ref:`config_http_conn_man_headers_downstream-service-cluster` header. -* HTTP URL. -* HTTP method. -* HTTP response code. +* HTTP request URL, method, protocol and user-agent. +* Additional HTTP request headers set via :ref:`request_headers_for_tags + ` +* HTTP response status code. +* GRPC response status and message (if available). +* An error tag when HTTP status is 5xx or GRPC status is not "OK" * Tracing system-specific metadata. The span also includes a name (or operation) which by default is defined as the host of the invoked diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 5152e89fdc65..147c65f5ce60 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -46,6 +46,7 @@ Version history * router check tool: add comprehensive coverage reporting. * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. +* tracing: added tags for gRPC response status and meesage. * upstream: added network filter chains to upstream connections, see :ref:`filters`. * upstream: use p2c to select hosts for least-requests load balancers if all host weights are the same, even in cases where weights are not equal to 1. * upstream: added :ref:`an option ` that allows draining HTTP, TCP connection pools on cluster membership change. diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index e5d7fe942155..580ba1e6c365 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -502,8 +502,9 @@ ConnectionManagerImpl::ActiveStream::~ActiveStream() { } if (active_span_) { - Tracing::HttpTracerUtility::finalizeSpan(*active_span_, request_headers_.get(), stream_info_, - *this); + Tracing::HttpTracerUtility::finalizeSpan(*active_span_, request_headers_.get(), + response_headers_.get(), response_trailers_.get(), + stream_info_, *this); } if (state_.successful_upgrade_) { connection_manager_.stats_.named_.downstream_cx_upgrades_active_.dec(); diff --git a/source/common/tracing/http_tracer_impl.cc b/source/common/tracing/http_tracer_impl.cc index 46ff74ac58a1..004e41c84a86 100644 --- a/source/common/tracing/http_tracer_impl.cc +++ b/source/common/tracing/http_tracer_impl.cc @@ -7,6 +7,7 @@ #include "common/common/fmt.h" #include "common/common/macros.h" #include "common/common/utility.h" +#include "common/grpc/common.h" #include "common/http/codes.h" #include "common/http/header_map_impl.h" #include "common/http/headers.h" @@ -81,6 +82,22 @@ Decision HttpTracerUtility::isTracing(const StreamInfo::StreamInfo& stream_info, NOT_REACHED_GCOVR_EXCL_LINE; } +static void addGrpcTags(Span& span, const Http::HeaderMap& headers) { + const Http::HeaderEntry* grpc_status_header = headers.GrpcStatus(); + if (grpc_status_header) { + span.setTag(Tracing::Tags::get().GrpcStatusCode, grpc_status_header->value().getStringView()); + } + const Http::HeaderEntry* grpc_message_header = headers.GrpcMessage(); + if (grpc_message_header) { + span.setTag(Tracing::Tags::get().GrpcMessage, grpc_message_header->value().getStringView()); + } + absl::optional grpc_status_code = Grpc::Common::getGrpcStatus(headers); + // Set error tag when status is not OK. + if (grpc_status_code && grpc_status_code.value() != Grpc::Status::GrpcStatus::Ok) { + span.setTag(Tracing::Tags::get().Error, Tracing::Tags::get().True); + } +} + static void annotateVerbose(Span& span, const StreamInfo::StreamInfo& stream_info) { const auto start_time = stream_info.startTime(); if (stream_info.lastDownstreamRxByteReceived()) { @@ -121,6 +138,8 @@ static void annotateVerbose(Span& span, const StreamInfo::StreamInfo& stream_inf } void HttpTracerUtility::finalizeSpan(Span& span, const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, const StreamInfo::StreamInfo& stream_info, const Config& tracing_config) { // Pre response data. @@ -163,6 +182,13 @@ void HttpTracerUtility::finalizeSpan(Span& span, const Http::HeaderMap* request_ span.setTag(Tracing::Tags::get().ResponseFlags, StreamInfo::ResponseFlagUtils::toShortString(stream_info)); + // GRPC data. + if (response_trailers && response_trailers->GrpcStatus() != nullptr) { + addGrpcTags(span, *response_trailers); + } else if (response_headers && response_headers->GrpcStatus() != nullptr) { + addGrpcTags(span, *response_headers); + } + if (tracing_config.verbose()) { annotateVerbose(span, stream_info); } diff --git a/source/common/tracing/http_tracer_impl.h b/source/common/tracing/http_tracer_impl.h index 50a782146043..ec421736eab4 100644 --- a/source/common/tracing/http_tracer_impl.h +++ b/source/common/tracing/http_tracer_impl.h @@ -41,6 +41,7 @@ class TracingTagValues { // Non-standard tag names. const std::string DownstreamCluster = "downstream_cluster"; const std::string GrpcStatusCode = "grpc.status_code"; + const std::string GrpcMessage = "grpc.message"; const std::string GuidXClientTraceId = "guid:x-client-trace-id"; const std::string GuidXRequestId = "guid:x-request-id"; const std::string HttpProtocol = "http.protocol"; @@ -101,6 +102,8 @@ class HttpTracerUtility { * 2) Finish active span. */ static void finalizeSpan(Span& span, const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, const StreamInfo::StreamInfo& stream_info, const Config& tracing_config); static const std::string IngressOperation; diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index 4aeee65fc6ac..4e1b991507f3 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -122,12 +122,14 @@ TEST(HttpConnManFinalizerImpl, OriginalAndLongPath) { {"x-envoy-original-path", path}, {":method", "GET"}, {"x-forwarded-proto", "http"}}; + Http::TestHeaderMapImpl response_headers; + Http::TestHeaderMapImpl response_trailers; NiceMock stream_info; absl::optional protocol = Http::Protocol::Http2; EXPECT_CALL(stream_info, bytesReceived()).WillOnce(Return(10)); EXPECT_CALL(stream_info, bytesSent()).WillOnce(Return(11)); - EXPECT_CALL(stream_info, protocol()).WillOnce(ReturnPointee(&protocol)); + EXPECT_CALL(stream_info, protocol()).WillRepeatedly(ReturnPointee(&protocol)); absl::optional response_code; EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(ReturnPointee(&response_code)); @@ -137,7 +139,8 @@ TEST(HttpConnManFinalizerImpl, OriginalAndLongPath) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/2"))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, stream_info, config); + HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, + stream_info, config); } TEST(HttpConnManFinalizerImpl, NoGeneratedId) { @@ -148,12 +151,14 @@ TEST(HttpConnManFinalizerImpl, NoGeneratedId) { Http::TestHeaderMapImpl request_headers{ {"x-envoy-original-path", path}, {":method", "GET"}, {"x-forwarded-proto", "http"}}; + Http::TestHeaderMapImpl response_headers; + Http::TestHeaderMapImpl response_trailers; NiceMock stream_info; absl::optional protocol = Http::Protocol::Http2; EXPECT_CALL(stream_info, bytesReceived()).WillOnce(Return(10)); EXPECT_CALL(stream_info, bytesSent()).WillOnce(Return(11)); - EXPECT_CALL(stream_info, protocol()).WillOnce(ReturnPointee(&protocol)); + EXPECT_CALL(stream_info, protocol()).WillRepeatedly(ReturnPointee(&protocol)); absl::optional response_code; EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(ReturnPointee(&response_code)); @@ -163,7 +168,8 @@ TEST(HttpConnManFinalizerImpl, NoGeneratedId) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/2"))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, stream_info, config); + HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, + stream_info, config); } TEST(HttpConnManFinalizerImpl, NullRequestHeaders) { @@ -184,7 +190,7 @@ TEST(HttpConnManFinalizerImpl, NullRequestHeaders) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), _)).Times(0); NiceMock config; - HttpTracerUtility::finalizeSpan(span, nullptr, stream_info, config); + HttpTracerUtility::finalizeSpan(span, nullptr, nullptr, nullptr, stream_info, config); } TEST(HttpConnManFinalizerImpl, StreamInfoLogs) { @@ -222,7 +228,7 @@ TEST(HttpConnManFinalizerImpl, StreamInfoLogs) { NiceMock config; EXPECT_CALL(config, verbose).WillOnce(Return(true)); - HttpTracerUtility::finalizeSpan(span, nullptr, stream_info, config); + HttpTracerUtility::finalizeSpan(span, nullptr, nullptr, nullptr, stream_info, config); } TEST(HttpConnManFinalizerImpl, UpstreamClusterTagSet) { @@ -244,7 +250,7 @@ TEST(HttpConnManFinalizerImpl, UpstreamClusterTagSet) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().RequestSize), Eq("10"))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, nullptr, stream_info, config); + HttpTracerUtility::finalizeSpan(span, nullptr, nullptr, nullptr, stream_info, config); } TEST(HttpConnManFinalizerImpl, SpanOptionalHeaders) { @@ -254,11 +260,13 @@ TEST(HttpConnManFinalizerImpl, SpanOptionalHeaders) { {":path", "/test"}, {":method", "GET"}, {"x-forwarded-proto", "https"}}; + Http::TestHeaderMapImpl response_headers; + Http::TestHeaderMapImpl response_trailers; NiceMock stream_info; absl::optional protocol = Http::Protocol::Http10; EXPECT_CALL(stream_info, bytesReceived()).WillOnce(Return(10)); - EXPECT_CALL(stream_info, protocol()).WillOnce(ReturnPointee(&protocol)); + EXPECT_CALL(stream_info, protocol()).WillRepeatedly(ReturnPointee(&protocol)); const std::string service_node = "i-453"; // Check that span is populated correctly. @@ -282,7 +290,8 @@ TEST(HttpConnManFinalizerImpl, SpanOptionalHeaders) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), _)).Times(0); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, stream_info, config); + HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, + stream_info, config); } TEST(HttpConnManFinalizerImpl, SpanPopulatedFailureResponse) { @@ -291,6 +300,8 @@ TEST(HttpConnManFinalizerImpl, SpanPopulatedFailureResponse) { {":path", "/test"}, {":method", "GET"}, {"x-forwarded-proto", "http"}}; + Http::TestHeaderMapImpl response_headers; + Http::TestHeaderMapImpl response_trailers; NiceMock stream_info; request_headers.insertHost().value(std::string("api")); @@ -299,7 +310,7 @@ TEST(HttpConnManFinalizerImpl, SpanPopulatedFailureResponse) { request_headers.insertClientTraceId().value(std::string("client_trace_id")); absl::optional protocol = Http::Protocol::Http10; - EXPECT_CALL(stream_info, protocol()).WillOnce(ReturnPointee(&protocol)); + EXPECT_CALL(stream_info, protocol()).WillRepeatedly(ReturnPointee(&protocol)); EXPECT_CALL(stream_info, bytesReceived()).WillOnce(Return(10)); const std::string service_node = "i-453"; @@ -339,7 +350,118 @@ TEST(HttpConnManFinalizerImpl, SpanPopulatedFailureResponse) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("UT"))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), _)).Times(0); - HttpTracerUtility::finalizeSpan(span, &request_headers, stream_info, config); + HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, + stream_info, config); +} + +TEST(HttpConnManFinalizerImpl, GrpcOkStatus) { + const std::string path_prefix = "http://"; + NiceMock span; + + Http::TestHeaderMapImpl request_headers{{":method", "POST"}, + {":scheme", "http"}, + {":path", "/pb.Foo/Bar"}, + {":authority", "example.com:80"}, + {"content-type", "application/grpc"}, + {"te", "trailers"}}; + + Http::TestHeaderMapImpl response_headers{{":status", "200"}, + {"content-type", "application/grpc"}}; + Http::TestHeaderMapImpl response_trailers{{"grpc-status", "0"}, {"grpc-message", ""}}; + NiceMock stream_info; + + absl::optional protocol = Http::Protocol::Http2; + absl::optional response_code(200); + EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(ReturnPointee(&response_code)); + EXPECT_CALL(stream_info, bytesReceived()).WillOnce(Return(10)); + EXPECT_CALL(stream_info, bytesSent()).WillOnce(Return(11)); + EXPECT_CALL(stream_info, protocol()).WillRepeatedly(ReturnPointee(&protocol)); + + EXPECT_CALL(span, setTag(_, _)).Times(testing::AnyNumber()); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpMethod), Eq("POST"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/2"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("200"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcStatusCode), Eq("0"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcMessage), Eq(""))); + + NiceMock config; + HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, + stream_info, config); +} + +TEST(HttpConnManFinalizerImpl, GrpcErrorTag) { + const std::string path_prefix = "http://"; + NiceMock span; + + Http::TestHeaderMapImpl request_headers{{":method", "POST"}, + {":scheme", "http"}, + {":path", "/pb.Foo/Bar"}, + {":authority", "example.com:80"}, + {"content-type", "application/grpc"}, + {"te", "trailers"}}; + + Http::TestHeaderMapImpl response_headers{{":status", "200"}, + {"content-type", "application/grpc"}}; + Http::TestHeaderMapImpl response_trailers{{"grpc-status", "7"}, + {"grpc-message", "permission denied"}}; + NiceMock stream_info; + + absl::optional protocol = Http::Protocol::Http2; + absl::optional response_code(200); + EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(ReturnPointee(&response_code)); + EXPECT_CALL(stream_info, bytesReceived()).WillOnce(Return(10)); + EXPECT_CALL(stream_info, bytesSent()).WillOnce(Return(11)); + EXPECT_CALL(stream_info, protocol()).WillRepeatedly(ReturnPointee(&protocol)); + + EXPECT_CALL(span, setTag(_, _)).Times(testing::AnyNumber()); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().Error), Eq(Tracing::Tags::get().True))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpMethod), Eq("POST"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/2"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("200"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcStatusCode), Eq("7"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcMessage), Eq("permission denied"))); + + NiceMock config; + HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, + stream_info, config); +} + +TEST(HttpConnManFinalizerImpl, GrpcTrailersOnly) { + const std::string path_prefix = "http://"; + NiceMock span; + + Http::TestHeaderMapImpl request_headers{{":method", "POST"}, + {":scheme", "http"}, + {":path", "/pb.Foo/Bar"}, + {":authority", "example.com:80"}, + {"content-type", "application/grpc"}, + {"te", "trailers"}}; + + Http::TestHeaderMapImpl response_headers{{":status", "200"}, + {"content-type", "application/grpc"}, + {"grpc-status", "7"}, + {"grpc-message", "permission denied"}}; + Http::TestHeaderMapImpl response_trailers; + NiceMock stream_info; + + absl::optional protocol = Http::Protocol::Http2; + absl::optional response_code(200); + EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(ReturnPointee(&response_code)); + EXPECT_CALL(stream_info, bytesReceived()).WillOnce(Return(10)); + EXPECT_CALL(stream_info, bytesSent()).WillOnce(Return(11)); + EXPECT_CALL(stream_info, protocol()).WillRepeatedly(ReturnPointee(&protocol)); + + EXPECT_CALL(span, setTag(_, _)).Times(testing::AnyNumber()); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().Error), Eq(Tracing::Tags::get().True))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpMethod), Eq("POST"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/2"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("200"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcStatusCode), Eq("7"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcMessage), Eq("permission denied"))); + + NiceMock config; + HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, + stream_info, config); } TEST(HttpTracerUtilityTest, operationTypeToString) { @@ -352,6 +474,8 @@ TEST(HttpNullTracerTest, BasicFunctionality) { MockConfig config; StreamInfo::MockStreamInfo stream_info; Http::TestHeaderMapImpl request_headers; + Http::TestHeaderMapImpl response_headers; + Http::TestHeaderMapImpl response_trailers; SpanPtr span_ptr = null_tracer.startSpan(config, request_headers, stream_info, {Reason::Sampling, true}); @@ -374,6 +498,8 @@ class HttpTracerImplTest : public testing::Test { Http::TestHeaderMapImpl request_headers_{ {":path", "/"}, {":method", "GET"}, {"x-request-id", "foo"}, {":authority", "test"}}; + Http::TestHeaderMapImpl response_headers; + Http::TestHeaderMapImpl response_trailers; StreamInfo::MockStreamInfo stream_info_; NiceMock local_info_; MockConfig config_; From 0a3fc6a091a768ba868ff4dd94cf526af4ffc0ff Mon Sep 17 00:00:00 2001 From: asraa Date: Tue, 27 Aug 2019 13:38:40 -0400 Subject: [PATCH 462/542] fuzz: add bounds to statsh flush interval (#8043) Add PGV bounds to the stats flush interval (greater than 1ms and less than 5000ms) to prevent Envoy from hanging from too small of a flush time. Risk Level: Low Testing: Corpus Entry added Fixes OSS-Fuzz issue https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=16300 Signed-off-by: Asra Ali --- api/envoy/config/bootstrap/v2/bootstrap.proto | 9 ++++++++- ...-testcase-server_fuzz_test-5734693923717120 | 18 ++++++++++++++++++ test/server/server_fuzz_test.cc | 2 ++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5734693923717120 diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index 477aeac5a141..9e3d9fe1cdb1 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -99,7 +99,14 @@ message Bootstrap { // performance reasons Envoy latches counters and only flushes counters and // gauges at a periodic interval. If not specified the default is 5000ms (5 // seconds). - google.protobuf.Duration stats_flush_interval = 7 [(gogoproto.stdduration) = true]; + // Duration must be at least 1ms and at most 5 min. + google.protobuf.Duration stats_flush_interval = 7 [ + (validate.rules).duration = { + lt: {seconds: 300}, + gte: {nanos: 1000000} + }, + (gogoproto.stdduration) = true + ]; // Optional watchdog configuration. Watchdog watchdog = 8; diff --git a/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5734693923717120 b/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5734693923717120 new file mode 100644 index 000000000000..187e397539d2 --- /dev/null +++ b/test/server/server_corpus/clusterfuzz-testcase-server_fuzz_test-5734693923717120 @@ -0,0 +1,18 @@ +static_resources { + clusters { + name: "@" + connect_timeout { + nanos: 250000000 + } + common_lb_config { + zone_aware_lb_config { + min_cluster_size { + value: 38 + } + } + } + } +} +stats_flush_interval { + nanos: 32256 +} diff --git a/test/server/server_fuzz_test.cc b/test/server/server_fuzz_test.cc index 2ee39d2f8d88..b47f1aed9622 100644 --- a/test/server/server_fuzz_test.cc +++ b/test/server/server_fuzz_test.cc @@ -1,5 +1,7 @@ #include +#include "envoy/config/bootstrap/v2/bootstrap.pb.validate.h" + #include "common/network/address_impl.h" #include "common/thread_local/thread_local_impl.h" From 44634d812f3c6bf7c9b9e3a827e650ca59b6e25c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 27 Aug 2019 17:17:31 -0400 Subject: [PATCH 463/542] Improve tools/stack_decode.py (#8041) Adjust tools/stack_decode.py to more obviously be Python 2 (not 3), and to work on stack traces that don't include the symbol names. Risk Level: Low Testing: Manually tested on a stack trace that one of our users sent us Signed-off-by: Luke Shumaker --- tools/stack_decode.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tools/stack_decode.py b/tools/stack_decode.py index 3e3c7bd87e7a..cc22bfd82a39 100755 --- a/tools/stack_decode.py +++ b/tools/stack_decode.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Call addr2line as needed to resolve addresses in a stack trace. The addresses # will be replaced if they can be resolved into file and line numbers. The @@ -11,7 +11,6 @@ # In each case this script will add file and line information to any backtrace log # lines found and echo back all non-Backtrace lines untouched. -from __future__ import print_function import collections import re import subprocess @@ -24,10 +23,14 @@ # any nonmatching lines unmodified. End when EOF received. def decode_stacktrace_log(object_file, input_source): traces = {} - # Match something like [backtrace] - # bazel-out/local-dbg/bin/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:84] + # Match something like: + # [backtrace] [bazel-out/local-dbg/bin/source/server/_virtual_includes/backtrace_lib/server/backtrace.h:84] backtrace_marker = "\[backtrace\] [^\s]+" - stackaddr_re = re.compile("%s #\d+: .* \[(0x[0-9a-fA-F]+)\]$" % backtrace_marker) + # Match something like: + # ${backtrace_marker} #10: SYMBOL [0xADDR] + # or: + # ${backtrace_marker} #10: [0xADDR] + stackaddr_re = re.compile("%s #\d+:(?: .*)? \[(0x[0-9a-fA-F]+)\]$" % backtrace_marker) try: while True: @@ -53,7 +56,7 @@ def decode_stacktrace_log(object_file, input_source): # # Returns list of result lines def run_addr2line(obj_file, addr_to_resolve): - return subprocess.check_output(["addr2line", "-Cpie", obj_file, addr_to_resolve]) + return subprocess.check_output(["addr2line", "-Cpie", obj_file, addr_to_resolve]).decode('utf-8') # Because of how bazel compiles, addr2line reports file names that begin with @@ -69,7 +72,10 @@ def trim_proc_cwd(file_and_line_number): decode_stacktrace_log(sys.argv[2], sys.stdin) sys.exit(0) elif len(sys.argv) > 1: - rununder = subprocess.Popen(sys.argv[1:], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + rununder = subprocess.Popen(sys.argv[1:], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) decode_stacktrace_log(sys.argv[1], rununder.stdout) rununder.wait() sys.exit(rununder.returncode) # Pass back test pass/fail result From 0006efc5a04dbd0a2381f25b09c77ba536330588 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 27 Aug 2019 16:35:55 -0700 Subject: [PATCH 464/542] build: tell googletest to use absl stacktrace (#8047) Description: https://github.com/google/googletest/blob/d7003576dd133856432e2e07340f45926242cc3a/BUILD.bazel#L42 Risk Level: Low (test only) Testing: CI Docs Changes: Release Notes: Signed-off-by: Lizan Zhou --- .bazelrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index 01d32bdd3fd3..6b73d8942498 100644 --- a/.bazelrc +++ b/.bazelrc @@ -20,6 +20,9 @@ build --action_env=BAZEL_LINKOPTS=-lm:-static-libgcc build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11 build --javabase=@bazel_tools//tools/jdk:remote_jdk11 +# We already have absl in the build, define absl=1 to tell googletest to use absl for backtrace. +build --define absl=1 + # Pass PATH, CC and CXX variables from the environment. build --action_env=CC build --action_env=CXX @@ -147,4 +150,4 @@ build:asan-fuzzer --define=FUZZING_ENGINE=libfuzzer build:asan-fuzzer --copt=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION build:asan-fuzzer --copt=-fsanitize-coverage=trace-pc-guard # Remove UBSAN halt_on_error to avoid crashing on protobuf errors. -build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1 \ No newline at end of file +build:asan-fuzzer --test_env=UBSAN_OPTIONS=print_stacktrace=1 From 6dd0ee161ab97e494125b1c937f7135ed0d0594f Mon Sep 17 00:00:00 2001 From: scheler Date: Tue, 27 Aug 2019 16:40:04 -0700 Subject: [PATCH 465/542] Update references to local scripts to enable using build container for filter repos (#7907) Description: This change enables using run_envoy_docker.sh to build envoy-filter-example Risk Level: Low Testing: Manually tested building envoy-filter-example using: envoy/ci/run_envoy_docker.sh './ci/do_ci.sh build' Docs Changes: N/A Release Notes: N/A Signed-off-by: Santosh Kumar Cheler --- ci/envoy_build_sha.sh | 2 +- ci/run_envoy_docker.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/envoy_build_sha.sh b/ci/envoy_build_sha.sh index 709617afe9b6..bdc6fefe409d 100644 --- a/ci/envoy_build_sha.sh +++ b/ci/envoy_build_sha.sh @@ -1,2 +1,2 @@ -ENVOY_BUILD_SHA=$(grep envoyproxy/envoy-build .circleci/config.yml | sed -e 's#.*envoyproxy/envoy-build:\(.*\)#\1#' | uniq) +ENVOY_BUILD_SHA=$(grep envoyproxy/envoy-build $(dirname $0)/../.circleci/config.yml | sed -e 's#.*envoyproxy/envoy-build:\(.*\)#\1#' | uniq) [[ $(wc -l <<< "${ENVOY_BUILD_SHA}" | awk '{$1=$1};1') == 1 ]] || (echo ".circleci/config.yml hashes are inconsistent!" && exit 1) diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index a93802b26173..ede21c5de859 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -2,7 +2,7 @@ set -e -. ci/envoy_build_sha.sh +. $(dirname $0)/envoy_build_sha.sh # We run as root and later drop permissions. This is required to setup the USER # in useradd below, which is need for correct Python execution in the Docker From d99e7f6a8ddebe6cfbe0afd0cf9056137d9ee6ae Mon Sep 17 00:00:00 2001 From: Dmitry Rozhkov Date: Wed, 28 Aug 2019 02:41:20 +0300 Subject: [PATCH 466/542] bazel: patch gRPC to fix Envoy builds with glibc v2.30 (#7971) Description: the latest glibc (v2.30) declares its own `gettid()` function (see [0]) and this creates a naming conflict in gRPC which has a function with the same name. Apply to gRPC [a patch](https://github.com/grpc/grpc/pull/18950) which renames `gettid()` to `sys_gettid()`. [0] https://sourceware.org/git/?p=glibc.git;a=commit;h=1d0fc213824eaa2a8f8c4385daaa698ee8fb7c92 Risk Level: low Testing: unit tests Docs Changes: n/a Release Notes: n/a Signed-off-by: Dmitry Rozhkov --- bazel/grpc-rename-gettid.patch | 78 ++++++++++++++++++++++++++++++++++ bazel/repositories.bzl | 2 + 2 files changed, 80 insertions(+) create mode 100644 bazel/grpc-rename-gettid.patch diff --git a/bazel/grpc-rename-gettid.patch b/bazel/grpc-rename-gettid.patch new file mode 100644 index 000000000000..90bd9115893f --- /dev/null +++ b/bazel/grpc-rename-gettid.patch @@ -0,0 +1,78 @@ +From d1d017390b799c59d6fdf7b8afa6136d218bdd61 Mon Sep 17 00:00:00 2001 +From: Benjamin Peterson +Date: Fri, 3 May 2019 08:11:00 -0700 +Subject: [PATCH] Rename gettid() functions. + +glibc 2.30 will declare its own gettid; see https://sourceware.org/git/?p=glibc.git;a=commit;h=1d0fc213824eaa2a8f8c4385daaa698ee8fb7c92. Rename the grpc versions to avoid naming conflicts. +--- + src/core/lib/gpr/log_linux.cc | 4 ++-- + src/core/lib/gpr/log_posix.cc | 4 ++-- + src/core/lib/iomgr/ev_epollex_linux.cc | 4 ++-- + 3 files changed, 6 insertions(+), 6 deletions(-) + +diff --git a/src/core/lib/gpr/log_linux.cc b/src/core/lib/gpr/log_linux.cc +index 561276f0c20..8b597b4cf2f 100644 +--- a/src/core/lib/gpr/log_linux.cc ++++ b/src/core/lib/gpr/log_linux.cc +@@ -40,7 +40,7 @@ + #include + #include + +-static long gettid(void) { return syscall(__NR_gettid); } ++static long sys_gettid(void) { return syscall(__NR_gettid); } + + void gpr_log(const char* file, int line, gpr_log_severity severity, + const char* format, ...) { +@@ -70,7 +70,7 @@ void gpr_default_log(gpr_log_func_args* args) { + gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME); + struct tm tm; + static __thread long tid = 0; +- if (tid == 0) tid = gettid(); ++ if (tid == 0) tid = sys_gettid(); + + timer = static_cast(now.tv_sec); + final_slash = strrchr(args->file, '/'); +diff --git a/src/core/lib/gpr/log_posix.cc b/src/core/lib/gpr/log_posix.cc +index b6edc14ab6b..2f7c6ce3760 100644 +--- a/src/core/lib/gpr/log_posix.cc ++++ b/src/core/lib/gpr/log_posix.cc +@@ -31,7 +31,7 @@ + #include + #include + +-static intptr_t gettid(void) { return (intptr_t)pthread_self(); } ++static intptr_t sys_gettid(void) { return (intptr_t)pthread_self(); } + + void gpr_log(const char* file, int line, gpr_log_severity severity, + const char* format, ...) { +@@ -86,7 +86,7 @@ void gpr_default_log(gpr_log_func_args* args) { + char* prefix; + gpr_asprintf(&prefix, "%s%s.%09d %7" PRIdPTR " %s:%d]", + gpr_log_severity_string(args->severity), time_buffer, +- (int)(now.tv_nsec), gettid(), display_file, args->line); ++ (int)(now.tv_nsec), sys_gettid(), display_file, args->line); + + fprintf(stderr, "%-70s %s\n", prefix, args->message); + gpr_free(prefix); +diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc +index 08116b3ab53..76f59844312 100644 +--- a/src/core/lib/iomgr/ev_epollex_linux.cc ++++ b/src/core/lib/iomgr/ev_epollex_linux.cc +@@ -1102,7 +1102,7 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, + } + + #ifndef NDEBUG +-static long gettid(void) { return syscall(__NR_gettid); } ++static long sys_gettid(void) { return syscall(__NR_gettid); } + #endif + + /* pollset->mu lock must be held by the caller before calling this. +@@ -1122,7 +1122,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, + #define WORKER_PTR (&worker) + #endif + #ifndef NDEBUG +- WORKER_PTR->originator = gettid(); ++ WORKER_PTR->originator = sys_gettid(); + #endif + if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) { + gpr_log(GPR_INFO, diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 15dbd7cb0e8c..b80d6f0bfe22 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -617,6 +617,8 @@ def _com_github_grpc_grpc(): "@envoy//bazel:grpc-protoinfo-2.patch", # Pre-integration of https://github.com/grpc/grpc/pull/19860 "@envoy//bazel:grpc-protoinfo-3.patch", + # Pre-integration of https://github.com/grpc/grpc/pull/18950 + "@envoy//bazel:grpc-rename-gettid.patch", ], patch_args = ["-p1"], ) From e674640f70a33d9bf235696ebcb194a5d8009794 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 28 Aug 2019 00:57:36 -0700 Subject: [PATCH 467/542] build: link C++ stdlib dynamically in sanitizer runs (#8019) Description: Sanitizers doesn't support static link, reverts #7929 and link lib(std)c++ dynamically in sanitizer runs. Addresses test issue for #4251. Added workaround in ASAN for #7647. Risk Level: Low (test only) Testing: CI, local libc++ runs Docs Changes: N/A Release Notes: N/A Fixes #7928 --- .bazelrc | 24 ++++++----- bazel/BUILD | 31 +++++++++++++- bazel/envoy_binary.bzl | 3 +- bazel/envoy_internal.bzl | 7 ++++ bazel/envoy_test.bzl | 5 ++- bazel/io_opentracing_cpp.patch | 17 ++++++++ bazel/repositories.bzl | 7 +++- .../configs/clang/bazel_0.28.1/cc/BUILD | 1 - .../clang_libcxx/bazel_0.28.1/cc/BUILD | 1 - .../configs/gcc/bazel_0.28.1/cc/BUILD | 3 +- bazel/toolchains/configs/versions.bzl | 6 +-- bazel/toolchains/rbe_toolchains_config.bzl | 6 +-- test/exe/BUILD | 5 +-- test/integration/fake_upstream.cc | 42 ++++++------------- 14 files changed, 99 insertions(+), 59 deletions(-) mode change 100755 => 100644 bazel/BUILD create mode 100644 bazel/io_opentracing_cpp.patch diff --git a/.bazelrc b/.bazelrc index 6b73d8942498..5235dc6265ac 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,7 +16,7 @@ build --experimental_local_memory_estimate build --experimental_strict_action_env=true build --host_force_python=PY2 build --action_env=BAZEL_LINKLIBS=-l%:libstdc++.a -build --action_env=BAZEL_LINKOPTS=-lm:-static-libgcc +build --action_env=BAZEL_LINKOPTS=-lm build --host_javabase=@bazel_tools//tools/jdk:remote_jdk11 build --javabase=@bazel_tools//tools/jdk:remote_jdk11 @@ -28,19 +28,21 @@ build --action_env=CC build --action_env=CXX build --action_env=PATH +# Common flags for sanitizers +build:sanitizer --define tcmalloc=disabled +build:sanitizer --linkopt -ldl +build:sanitizer --build_tag_filters=-no_san +build:sanitizer --test_tag_filters=-no_san + # Basic ASAN/UBSAN that works for gcc -build:asan --action_env=BAZEL_LINKLIBS= -build:asan --action_env=BAZEL_LINKOPTS=-lstdc++:-lm +build:asan --config=sanitizer +# ASAN install its signal handler, disable ours so the stacktrace will be printed by ASAN +build:asan --define signal_trace=disabled build:asan --define ENVOY_CONFIG_ASAN=1 build:asan --copt -fsanitize=address,undefined build:asan --linkopt -fsanitize=address,undefined build:asan --copt -fno-sanitize=vptr build:asan --linkopt -fno-sanitize=vptr -build:asan --linkopt -ldl -build:asan --define tcmalloc=disabled -build:asan --build_tag_filters=-no_asan -build:asan --test_tag_filters=-no_asan -build:asan --define signal_trace=disabled build:asan --copt -DADDRESS_SANITIZER=1 build:asan --copt -D__SANITIZE_ADDRESS__ build:asan --test_env=ASAN_OPTIONS=handle_abort=1:allow_addr2line=true:check_initialization_order=true:strict_init_order=true:detect_odr_violation=1 @@ -62,21 +64,20 @@ build:macos-asan --copt -DGRPC_BAZEL_BUILD build:macos-asan --dynamic_mode=off # Clang TSAN +build:clang-tsan --config=sanitizer build:clang-tsan --define ENVOY_CONFIG_TSAN=1 build:clang-tsan --copt -fsanitize=thread build:clang-tsan --linkopt -fsanitize=thread build:clang-tsan --linkopt -fuse-ld=lld -build:clang-tsan --linkopt -static-libsan -build:clang-tsan --define tcmalloc=disabled # Needed due to https://github.com/libevent/libevent/issues/777 build:clang-tsan --copt -DEVENT__DISABLE_DEBUG_MODE # Clang MSAN - broken today since we need to rebuild lib[std]c++ and external deps with MSAN # support (see https://github.com/envoyproxy/envoy/issues/443). +build:clang-msan --config=sanitizer build:clang-msan --define ENVOY_CONFIG_MSAN=1 build:clang-msan --copt -fsanitize=memory build:clang-msan --linkopt -fsanitize=memory -build:clang-msan --define tcmalloc=disabled build:clang-msan --copt -fsanitize-memory-track-origins=2 # Clang with libc++ @@ -108,6 +109,7 @@ build:rbe-toolchain-clang-libc++ --config=rbe-toolchain build:rbe-toolchain-clang-libc++ --crosstool_top=@rbe_ubuntu_clang_libcxx//cc:toolchain build:rbe-toolchain-clang-libc++ --extra_toolchains=@rbe_ubuntu_clang_libcxx//config:cc-toolchain build:rbe-toolchain-clang-libc++ --action_env=CC=clang --action_env=CXX=clang++ --action_env=PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin +build:rbe-toolchain-clang-libc++ --define force_libcpp=enabled build:rbe-toolchain-gcc --config=rbe-toolchain build:rbe-toolchain-gcc --crosstool_top=@rbe_ubuntu_gcc//cc:toolchain diff --git a/bazel/BUILD b/bazel/BUILD old mode 100755 new mode 100644 index e60a4ae9c06c..202ecfc9d381 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -1,11 +1,14 @@ licenses(["notice"]) # Apache 2 -package(default_visibility = ["//visibility:public"]) +load("//bazel:envoy_build_system.bzl", "envoy_package") + +envoy_package() + +load("//bazel:envoy_internal.bzl", "envoy_select_force_libcpp") exports_files([ "gen_sh_test_runner.sh", "sh_test_wrapper.sh", - "cc_wrapper.py", ]) genrule( @@ -37,6 +40,25 @@ genrule( stamp = 1, ) +# A target to optionally link C++ standard library dynamically in sanitizer runs. +# TSAN doesn't support libc/libstdc++ static linking per doc: +# http://releases.llvm.org/8.0.1/tools/clang/docs/ThreadSanitizer.html +cc_library( + name = "dynamic_stdlib", + linkopts = envoy_select_force_libcpp( + ["-lc++"], + ["-lstdc++"], + ), +) + +cc_library( + name = "static_stdlib", + linkopts = select({ + "//bazel:linux": ["-static-libgcc"], + "//conditions:default": [], + }), +) + config_setting( name = "windows_opt_build", values = { @@ -81,6 +103,11 @@ config_setting( values = {"define": "ENVOY_CONFIG_ASAN=1"}, ) +config_setting( + name = "tsan_build", + values = {"define": "ENVOY_CONFIG_TSAN=1"}, +) + config_setting( name = "coverage_build", values = {"define": "ENVOY_CONFIG_COVERAGE=1"}, diff --git a/bazel/envoy_binary.bzl b/bazel/envoy_binary.bzl index edcdd09ae03b..8a9d396dcfbc 100644 --- a/bazel/envoy_binary.bzl +++ b/bazel/envoy_binary.bzl @@ -4,6 +4,7 @@ load( ":envoy_internal.bzl", "envoy_copts", "envoy_external_dep_path", + "envoy_stdlib_deps", "tcmalloc_external_dep", ) @@ -24,7 +25,7 @@ def envoy_cc_binary( if stamped: linkopts = linkopts + _envoy_stamped_linkopts() deps = deps + _envoy_stamped_deps() - deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + envoy_stdlib_deps() native.cc_binary( name = name, srcs = srcs, diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 2562390311de..5dca0a028f82 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -89,6 +89,13 @@ def envoy_select_force_libcpp(if_libcpp, default = None): "//conditions:default": default or [], }) +def envoy_stdlib_deps(): + return select({ + "@envoy//bazel:asan_build": ["@envoy//bazel:dynamic_stdlib"], + "@envoy//bazel:tsan_build": ["@envoy//bazel:dynamic_stdlib"], + "//conditions:default": ["@envoy//bazel:static_stdlib"], + }) + # Dependencies on tcmalloc_and_profiler should be wrapped with this function. def tcmalloc_external_dep(repository): return select({ diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index 4e7e91e0070b..e6985f92a1f3 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -8,6 +8,7 @@ load( "envoy_external_dep_path", "envoy_linkstatic", "envoy_select_force_libcpp", + "envoy_stdlib_deps", "tcmalloc_external_dep", ) @@ -80,7 +81,7 @@ def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs): test_lib_name = name + "_lib" envoy_cc_test_library( name = test_lib_name, - deps = deps + ["//test/fuzz:fuzz_runner_lib"], + deps = deps + ["//test/fuzz:fuzz_runner_lib", "//bazel:dynamic_stdlib"], **kwargs ) native.cc_test( @@ -163,7 +164,7 @@ def envoy_cc_test( linkopts = _envoy_test_linkopts(), linkstatic = envoy_linkstatic(), malloc = tcmalloc_external_dep(repository), - deps = [ + deps = envoy_stdlib_deps() + [ ":" + name + "_lib_internal_only", repository + "//test:main", ], diff --git a/bazel/io_opentracing_cpp.patch b/bazel/io_opentracing_cpp.patch new file mode 100644 index 000000000000..6389489d65a0 --- /dev/null +++ b/bazel/io_opentracing_cpp.patch @@ -0,0 +1,17 @@ +diff --git a/src/dynamic_load_unix.cpp b/src/dynamic_load_unix.cpp +index 17e08fd..d25e0c8 100644 +--- a/src/dynamic_load_unix.cpp ++++ b/src/dynamic_load_unix.cpp +@@ -35,7 +35,11 @@ DynamicallyLoadTracingLibrary(const char* shared_library, + std::string& error_message) noexcept try { + dlerror(); // Clear any existing error. + +- const auto handle = dlopen(shared_library, RTLD_NOW | RTLD_LOCAL); ++ const auto handle = dlopen(shared_library, RTLD_NOW | RTLD_LOCAL ++#ifdef __SANITIZE_ADDRESS__ ++ | RTLD_NODELETE ++#endif ++ ); + if (handle == nullptr) { + error_message = dlerror(); + return make_unexpected(dynamic_load_failure_error); diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index b80d6f0bfe22..4f2a28fab99d 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -335,7 +335,12 @@ def _com_github_nghttp2_nghttp2(): ) def _io_opentracing_cpp(): - _repository_impl("io_opentracing_cpp") + _repository_impl( + name = "io_opentracing_cpp", + patch_args = ["-p1"], + # Workaround for LSAN false positive in https://github.com/envoyproxy/envoy/issues/7647 + patches = ["@envoy//bazel:io_opentracing_cpp.patch"], + ) native.bind( name = "opentracing", actual = "@io_opentracing_cpp//:opentracing", diff --git a/bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD b/bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD index 1726bdef2c05..015b58ead104 100755 --- a/bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD +++ b/bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD @@ -116,7 +116,6 @@ cc_toolchain_config( "-Wl,-z,relro,-z,now", "-B/usr/lib/llvm-8/bin", "-lm", - "-static-libgcc", "-fuse-ld=lld"], link_libs = ["-l:libstdc++.a"], opt_link_flags = ["-Wl,--gc-sections"], diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD index dae58f58c97a..8a2ac6331467 100755 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD @@ -114,7 +114,6 @@ cc_toolchain_config( "-Wl,-z,relro,-z,now", "-B/usr/lib/llvm-8/bin", "-lm", - "-static-libgcc", "-pthread", "-fuse-ld=lld"], link_libs = ["-l:libc++.a", diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD index e936d4b91522..443b34aa3eff 100755 --- a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD +++ b/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD @@ -115,8 +115,7 @@ cc_toolchain_config( "-Wl,-z,relro,-z,now", "-B/usr/bin", "-pass-exit-codes", - "-lm", - "-static-libgcc"], + "-lm"], link_libs = ["-l:libstdc++.a"], opt_link_flags = ["-Wl,--gc-sections"], unfiltered_compile_flags = ["-fno-canonical-system-headers", diff --git a/bazel/toolchains/configs/versions.bzl b/bazel/toolchains/configs/versions.bzl index b7fee4d50322..c21f70f9cb14 100644 --- a/bazel/toolchains/configs/versions.bzl +++ b/bazel/toolchains/configs/versions.bzl @@ -1,9 +1,9 @@ # Generated file, do not modify by hand # Generated by 'rbe_ubuntu_gcc_gen' rbe_autoconfig rule """Definitions to be used in rbe_repo attr of an rbe_autoconf rule """ -toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "clang") -toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx") -toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-static-libgcc", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc") +toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "clang") +toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx") +toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc") _TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1,toolchain_config_spec2] _BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.1": ["clang", "clang_libcxx", "gcc"]} LATEST = "sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" diff --git a/bazel/toolchains/rbe_toolchains_config.bzl b/bazel/toolchains/rbe_toolchains_config.bzl index fd7210db1f87..96c38c79e1f5 100644 --- a/bazel/toolchains/rbe_toolchains_config.bzl +++ b/bazel/toolchains/rbe_toolchains_config.bzl @@ -10,7 +10,7 @@ _CONFIGS_OUTPUT_BASE = "bazel/toolchains/configs" _CLANG_ENV = { "BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", - "BAZEL_LINKOPTS": "-lm:-static-libgcc:-fuse-ld=lld", + "BAZEL_LINKOPTS": "-lm:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", @@ -20,7 +20,7 @@ _CLANG_ENV = { _CLANG_LIBCXX_ENV = dicts.add(_CLANG_ENV, { "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", - "BAZEL_LINKOPTS": "-lm:-static-libgcc:-pthread:-fuse-ld=lld", + "BAZEL_LINKOPTS": "-lm:-pthread:-fuse-ld=lld", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++", }) @@ -28,7 +28,7 @@ _CLANG_LIBCXX_ENV = dicts.add(_CLANG_ENV, { _GCC_ENV = { "BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", - "BAZEL_LINKOPTS": "-lm:-static-libgcc", + "BAZEL_LINKOPTS": "-lm", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", diff --git a/test/exe/BUILD b/test/exe/BUILD index 12235850ca35..8ed8e3334dcd 100644 --- a/test/exe/BUILD +++ b/test/exe/BUILD @@ -24,9 +24,8 @@ envoy_sh_test( srcs = ["envoy_static_test.sh"], coverage = False, data = ["//source/exe:envoy-static"], - # NOTE: In some environments, ASAN causes dynamic linking no matter what, so don't run this - # test when doing ASAN. - tags = ["no_asan"], + # Sanitizers doesn't like statically linked lib(std)c++ and libgcc, skip this test in that context. + tags = ["no_san"], ) envoy_sh_test( diff --git a/test/integration/fake_upstream.cc b/test/integration/fake_upstream.cc index b30deb3aee70..f55806361987 100644 --- a/test/integration/fake_upstream.cc +++ b/test/integration/fake_upstream.cc @@ -73,30 +73,22 @@ void FakeStream::decodeMetadata(Http::MetadataMapPtr&& metadata_map_ptr) { } void FakeStream::encode100ContinueHeaders(const Http::HeaderMapImpl& headers) { - // TSan complains about thread-safety of std::shared_ptr when linked against libc++. - // See: https://github.com/envoyproxy/envoy/pull/7929 - std::unique_ptr headers_copy( + std::shared_ptr headers_copy( new Http::HeaderMapImpl(static_cast(headers))); - parent_.connection().dispatcher().post([this, headers = headers_copy.release()]() -> void { - encoder_.encode100ContinueHeaders(*headers); - delete headers; - }); + parent_.connection().dispatcher().post( + [this, headers_copy]() -> void { encoder_.encode100ContinueHeaders(*headers_copy); }); } void FakeStream::encodeHeaders(const Http::HeaderMapImpl& headers, bool end_stream) { - // TSan complains about thread-safety of std::shared_ptr when linked against libc++. - // See: https://github.com/envoyproxy/envoy/pull/7929 - std::unique_ptr headers_copy( + std::shared_ptr headers_copy( new Http::HeaderMapImpl(static_cast(headers))); if (add_served_by_header_) { headers_copy->addCopy(Http::LowerCaseString("x-served-by"), parent_.connection().localAddress()->asString()); } - parent_.connection().dispatcher().post( - [this, headers = headers_copy.release(), end_stream]() -> void { - encoder_.encodeHeaders(*headers, end_stream); - delete headers; - }); + parent_.connection().dispatcher().post([this, headers_copy, end_stream]() -> void { + encoder_.encodeHeaders(*headers_copy, end_stream); + }); } void FakeStream::encodeData(absl::string_view data, bool end_stream) { @@ -114,24 +106,16 @@ void FakeStream::encodeData(uint64_t size, bool end_stream) { } void FakeStream::encodeData(Buffer::Instance& data, bool end_stream) { - // TSan complains about thread-safety of std::shared_ptr when linked against libc++. - // See: https://github.com/envoyproxy/envoy/pull/7929 - std::unique_ptr data_copy(new Buffer::OwnedImpl(data)); - parent_.connection().dispatcher().post([this, data = data_copy.release(), end_stream]() -> void { - encoder_.encodeData(*data, end_stream); - delete data; - }); + std::shared_ptr data_copy(new Buffer::OwnedImpl(data)); + parent_.connection().dispatcher().post( + [this, data_copy, end_stream]() -> void { encoder_.encodeData(*data_copy, end_stream); }); } void FakeStream::encodeTrailers(const Http::HeaderMapImpl& trailers) { - // TSan complains about thread-safety of std::shared_ptr when linked against libc++. - // See: https://github.com/envoyproxy/envoy/pull/7929 - std::unique_ptr trailers_copy( + std::shared_ptr trailers_copy( new Http::HeaderMapImpl(static_cast(trailers))); - parent_.connection().dispatcher().post([this, trailers = trailers_copy.release()]() -> void { - encoder_.encodeTrailers(*trailers); - delete trailers; - }); + parent_.connection().dispatcher().post( + [this, trailers_copy]() -> void { encoder_.encodeTrailers(*trailers_copy); }); } void FakeStream::encodeResetStream() { From b020b63d1ae05fa009a0581e0ecf135cc3e77726 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 28 Aug 2019 09:38:01 -0400 Subject: [PATCH 468/542] test: cleaning up test runtime (#8012) Using the new runtime utility to clean up a bunch of test gorp. Yay utils! Risk Level: n/a (test only) Testing: tests pass Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- test/common/http/http1/BUILD | 1 + test/common/http/http1/codec_impl_test.cc | 46 ++----------------- test/common/http/http2/BUILD | 1 + test/common/http/http2/codec_impl_test.cc | 30 ++---------- test/extensions/filters/http/buffer/BUILD | 1 + .../filters/http/buffer/buffer_filter_test.cc | 27 ++--------- test/test_common/test_runtime.h | 1 - 7 files changed, 15 insertions(+), 92 deletions(-) diff --git a/test/common/http/http1/BUILD b/test/common/http/http1/BUILD index bbc146417792..b59485efb6ae 100644 --- a/test/common/http/http1/BUILD +++ b/test/common/http/http1/BUILD @@ -28,6 +28,7 @@ envoy_cc_test( "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:logging_lib", + "//test/test_common:test_runtime_lib", ], ) diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index f2b04b6e088a..6329e8bdc1bf 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -12,15 +12,10 @@ #include "test/mocks/buffer/mocks.h" #include "test/mocks/http/mocks.h" -#include "test/mocks/init/mocks.h" -#include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" -#include "test/mocks/protobuf/mocks.h" -#include "test/mocks/runtime/mocks.h" -#include "test/mocks/thread_local/mocks.h" #include "test/test_common/logging.h" #include "test/test_common/printers.h" -#include "test/test_common/utility.h" +#include "test/test_common/test_runtime.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -38,17 +33,6 @@ namespace Http1 { class Http1ServerConnectionImplTest : public testing::Test { public: - Http1ServerConnectionImplTest() : api_(Api::createApiForTest()) { - envoy::config::bootstrap::v2::LayeredRuntime config; - config.add_layers()->mutable_admin_layer(); - - // Create a runtime loader, so that tests can manually manipulate runtime - // guarded features. - loader_ = std::make_unique(Runtime::LoaderPtr{ - new Runtime::LoaderImpl(dispatcher_, tls_, config, local_info_, init_manager_, store_, - generator_, validation_visitor_, *api_)}); - } - void initialize() { codec_ = std::make_unique(connection_, store_, callbacks_, codec_settings_, max_request_headers_kb_); @@ -65,15 +49,7 @@ class Http1ServerConnectionImplTest : public testing::Test { protected: uint32_t max_request_headers_kb_{Http::DEFAULT_MAX_REQUEST_HEADERS_KB}; - Event::MockDispatcher dispatcher_; - NiceMock tls_; Stats::IsolatedStoreImpl store_; - Runtime::MockRandomGenerator generator_; - Api::ApiPtr api_; - NiceMock local_info_; - Init::MockManager init_manager_; - NiceMock validation_visitor_; - std::unique_ptr loader_; }; void Http1ServerConnectionImplTest::expect400(Protocol p, bool allow_absolute_url, @@ -366,6 +342,7 @@ TEST_F(Http1ServerConnectionImplTest, HostHeaderTranslation) { // Ensures that requests with invalid HTTP header values are not rejected // when the runtime guard is not enabled for the feature. TEST_F(Http1ServerConnectionImplTest, HeaderInvalidCharsRuntimeGuard) { + TestScopedRuntime scoped_runtime; // When the runtime-guarded feature is NOT enabled, invalid header values // should be accepted by the codec. Runtime::LoaderSingleton::getExisting()->mergeValues( @@ -384,6 +361,7 @@ TEST_F(Http1ServerConnectionImplTest, HeaderInvalidCharsRuntimeGuard) { // Ensures that requests with invalid HTTP header values are properly rejected // when the runtime guard is enabled for the feature. TEST_F(Http1ServerConnectionImplTest, HeaderInvalidCharsRejection) { + TestScopedRuntime scoped_runtime; // When the runtime-guarded feature is enabled, invalid header values // should result in a rejection. Runtime::LoaderSingleton::getExisting()->mergeValues( @@ -850,16 +828,6 @@ TEST_F(Http1ServerConnectionImplTest, WatermarkTest) { class Http1ClientConnectionImplTest : public testing::Test { public: - Http1ClientConnectionImplTest() : api_(Api::createApiForTest()) { - envoy::config::bootstrap::v2::LayeredRuntime config; - - // Create a runtime loader, so that tests can manually manipulate runtime - // guarded features. - loader_ = std::make_unique(Runtime::LoaderPtr{ - new Runtime::LoaderImpl(dispatcher_, tls_, config, local_info_, init_manager_, store_, - generator_, validation_visitor_, *api_)}); - } - void initialize() { codec_ = std::make_unique(connection_, store_, callbacks_); } @@ -869,15 +837,7 @@ class Http1ClientConnectionImplTest : public testing::Test { std::unique_ptr codec_; protected: - Event::MockDispatcher dispatcher_; - NiceMock tls_; Stats::IsolatedStoreImpl store_; - Runtime::MockRandomGenerator generator_; - Api::ApiPtr api_; - NiceMock local_info_; - Init::MockManager init_manager_; - NiceMock validation_visitor_; - std::unique_ptr loader_; }; TEST_F(Http1ClientConnectionImplTest, SimpleGet) { diff --git a/test/common/http/http2/BUILD b/test/common/http/http2/BUILD index ae51864b72d9..3a93f99146e4 100644 --- a/test/common/http/http2/BUILD +++ b/test/common/http/http2/BUILD @@ -30,6 +30,7 @@ envoy_cc_test( "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/common/http/http2/codec_impl_test.cc b/test/common/http/http2/codec_impl_test.cc index e738c140aff0..7592ec0985d0 100644 --- a/test/common/http/http2/codec_impl_test.cc +++ b/test/common/http/http2/codec_impl_test.cc @@ -11,12 +11,9 @@ #include "test/common/http/common.h" #include "test/common/http/http2/http2_frame.h" #include "test/mocks/http/mocks.h" -#include "test/mocks/init/mocks.h" -#include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" -#include "test/mocks/protobuf/mocks.h" -#include "test/mocks/thread_local/mocks.h" #include "test/test_common/printers.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "codec_impl_test_util.h" @@ -169,17 +166,7 @@ class Http2CodecImplTest : public ::testing::TestWithParam(GetParam()), ::testing::get<1>(GetParam())), - api_(Api::createApiForTest()) { - envoy::config::bootstrap::v2::LayeredRuntime config; - config.add_layers()->mutable_admin_layer(); - - // Create a runtime loader, so that tests can manually manipulate runtime - // guarded features. - loader_ = std::make_unique( - std::make_unique(dispatcher_, tls_, config, local_info_, init_manager_, - store_, generator_, validation_visitor_, *api_)); - } + : Http2CodecImplTestFixture(::testing::get<0>(GetParam()), ::testing::get<1>(GetParam())) {} protected: void priorityFlood() { @@ -242,16 +229,9 @@ class Http2CodecImplTest : public ::testing::TestWithParam tls_; - Stats::IsolatedStoreImpl store_; - Runtime::MockRandomGenerator generator_; - Api::ApiPtr api_; - NiceMock local_info_; - Init::MockManager init_manager_; - NiceMock validation_visitor_; - std::unique_ptr loader_; + // Make sure the test fixture has a fake runtime, for the tests which use + // Runtime::LoaderSingleton::getExisting()->mergeValues(...) + TestScopedRuntime scoped_runtime_; }; TEST_P(Http2CodecImplTest, ShutdownNotice) { diff --git a/test/extensions/filters/http/buffer/BUILD b/test/extensions/filters/http/buffer/BUILD index 0cbbdb99fea3..64c60dcba9c7 100644 --- a/test/extensions/filters/http/buffer/BUILD +++ b/test/extensions/filters/http/buffer/BUILD @@ -27,6 +27,7 @@ envoy_extension_cc_test( "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/thread_local:thread_local_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:test_runtime_lib", ], ) diff --git a/test/extensions/filters/http/buffer/buffer_filter_test.cc b/test/extensions/filters/http/buffer/buffer_filter_test.cc index 12b715f5d29f..7f8a46899cd6 100644 --- a/test/extensions/filters/http/buffer/buffer_filter_test.cc +++ b/test/extensions/filters/http/buffer/buffer_filter_test.cc @@ -12,12 +12,8 @@ #include "test/mocks/buffer/mocks.h" #include "test/mocks/http/mocks.h" -#include "test/mocks/init/mocks.h" -#include "test/mocks/local_info/mocks.h" -#include "test/mocks/protobuf/mocks.h" -#include "test/mocks/runtime/mocks.h" -#include "test/mocks/thread_local/mocks.h" #include "test/test_common/printers.h" +#include "test/test_common/test_runtime.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -42,16 +38,8 @@ class BufferFilterTest : public testing::Test { return std::make_shared(proto_config); } - BufferFilterTest() : config_(setupConfig()), filter_(config_), api_(Api::createApiForTest()) { + BufferFilterTest() : config_(setupConfig()), filter_(config_) { filter_.setDecoderFilterCallbacks(callbacks_); - - // Create a runtime loader, so that tests can manually manipulate runtime - // guarded features. - envoy::config::bootstrap::v2::LayeredRuntime config; - config.add_layers()->mutable_admin_layer(); - loader_ = std::make_unique(Runtime::LoaderPtr{ - new Runtime::LoaderImpl(dispatcher_, tls_, config, local_info_, init_manager_, store_, - generator_, validation_visitor_, *api_)}); } void routeLocalConfig(const Router::RouteSpecificFilterConfig* route_settings, @@ -66,15 +54,8 @@ class BufferFilterTest : public testing::Test { NiceMock callbacks_; BufferFilterConfigSharedPtr config_; BufferFilter filter_; - Event::MockDispatcher dispatcher_; - NiceMock tls_; - Stats::IsolatedStoreImpl store_; - Runtime::MockRandomGenerator generator_; - Api::ApiPtr api_; - NiceMock local_info_; - Init::MockManager init_manager_; - NiceMock validation_visitor_; - std::unique_ptr loader_; + // Create a runtime loader, so that tests can manually manipulate runtime guarded features. + TestScopedRuntime scoped_runtime; }; TEST_F(BufferFilterTest, HeaderOnlyRequest) { diff --git a/test/test_common/test_runtime.h b/test/test_common/test_runtime.h index ba9900578fa7..ca796ca23f65 100644 --- a/test/test_common/test_runtime.h +++ b/test/test_common/test_runtime.h @@ -25,7 +25,6 @@ namespace Envoy { -// TODO(alyssawilk) move existing runtime tests over to using this. class TestScopedRuntime { public: TestScopedRuntime() : api_(Api::createApiForTest()) { From 64243c97628369ceb365b4da6d73b43dd8bccba5 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 28 Aug 2019 11:04:24 -0400 Subject: [PATCH 469/542] test: improved coverage and handling of deprecated config (#8057) Making ENVOY_DISABLE_DEPRECATED_FEATURES work for unit tests without runtime configured. Fixing up a handful of unit tests to remove legacy code or use the handy DEPRECATED_FEATURE_TEST macro Adding back coverage of cors.enabled() and redis.catch_all_route() Risk Level: Low (test only) Testing: new unit tests Docs Changes: n/a Release Notes: n/a Fixes #8013 Fixes #7548 Signed-off-by: Alyssa Wilk --- source/common/protobuf/utility.cc | 4 ++ test/common/router/config_impl_test.cc | 52 +++++++++-------- test/config/integration/server.yaml | 17 ++++-- test/config_test/example_configs_test.cc | 2 +- .../http/cors/cors_filter_integration_test.cc | 27 +++++++++ .../network/dubbo_proxy/conn_manager_test.cc | 3 +- .../network/dubbo_proxy/route_matcher_test.cc | 6 +- .../network/redis_proxy/config_test.cc | 57 +++++++++++++++++-- .../thrift_proxy/route_matcher_test.cc | 3 +- test/integration/integration_admin_test.cc | 8 +-- test/server/BUILD | 1 + .../invalid_legacy_runtime_bootstrap.yaml | 4 ++ test/server/invalid_runtime_bootstrap.yaml | 10 ++-- test/server/server_test.cc | 5 ++ 14 files changed, 153 insertions(+), 46 deletions(-) create mode 100644 test/server/invalid_legacy_runtime_bootstrap.yaml diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index 33185a00fc5e..4df4964b768f 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -192,7 +192,11 @@ void MessageUtil::checkForDeprecation(const Protobuf::Message& message, Runtime: continue; } +#ifdef ENVOY_DISABLE_DEPRECATED_FEATURES + bool warn_only = false; +#else bool warn_only = true; +#endif absl::string_view filename = filenameFromPath(field->file()->name()); // Allow runtime to be null both to not crash if this is called before server initialization, // and so proto validation works in context where runtime singleton is not set up (e.g. diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index ebed75c45251..ff6aeb4555d6 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -116,7 +116,8 @@ class ConfigImplTestBase { class RouteMatcherTest : public testing::Test, public ConfigImplTestBase {}; -TEST_F(RouteMatcherTest, TestRoutes) { +// TODO(alyssawilk) go through all these tests and update or duplicate. +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutes)) { const std::string yaml = R"EOF( virtual_hosts: - name: www2 @@ -617,7 +618,7 @@ TEST_F(RouteMatcherTest, TestRoutesWithWildcardAndDefaultOnly) { config.route(genHeaders("example.com", "/", "GET"), 0)->routeEntry()->clusterName()); } -TEST_F(RouteMatcherTest, TestRoutesWithInvalidRegex) { +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutesWithInvalidRegex)) { std::string invalid_route = R"EOF( virtual_hosts: - name: regex @@ -1077,7 +1078,7 @@ name: foo } } -TEST_F(RouteMatcherTest, Priority) { +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(Priority)) { const std::string yaml = R"EOF( virtual_hosts: - name: local_service @@ -1164,7 +1165,7 @@ TEST_F(RouteMatcherTest, NoAutoRewriteAndAutoRewriteHeader) { EnvoyException); } -TEST_F(RouteMatcherTest, HeaderMatchedRouting) { +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(HeaderMatchedRouting)) { const std::string yaml = R"EOF( virtual_hosts: - name: local_service @@ -1288,7 +1289,7 @@ TEST_F(RouteMatcherTest, HeaderMatchedRouting) { } // Verify the fixes for https://github.com/envoyproxy/envoy/issues/2406 -TEST_F(RouteMatcherTest, InvalidHeaderMatchedRoutingConfig) { +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(InvalidHeaderMatchedRoutingConfig)) { std::string value_with_regex_chars = R"EOF( virtual_hosts: - name: local_service @@ -1323,7 +1324,7 @@ TEST_F(RouteMatcherTest, InvalidHeaderMatchedRoutingConfig) { EnvoyException, "Invalid regex"); } -TEST_F(RouteMatcherTest, QueryParamMatchedRouting) { +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(QueryParamMatchedRouting)) { const std::string yaml = R"EOF( virtual_hosts: - name: local_service @@ -1438,7 +1439,7 @@ TEST_F(RouteMatcherTest, QueryParamMatchedRouting) { } // Verify the fixes for https://github.com/envoyproxy/envoy/issues/2406 -TEST_F(RouteMatcherTest, InvalidQueryParamMatchedRoutingConfig) { +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(InvalidQueryParamMatchedRoutingConfig)) { std::string value_with_regex_chars = R"EOF( virtual_hosts: - name: local_service @@ -2249,7 +2250,7 @@ TEST_F(RouteMatcherTest, ClusterNotFoundResponseCodeConfig404) { config.route(headers, 0)->routeEntry()->clusterNotFoundResponseCode()); } -TEST_F(RouteMatcherTest, Shadow) { +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(Shadow)) { const std::string yaml = R"EOF( virtual_hosts: - name: www2 @@ -2307,7 +2308,7 @@ TEST_F(RouteMatcherTest, Shadow) { class RouteConfigurationV2 : public testing::Test, public ConfigImplTestBase {}; -TEST_F(RouteConfigurationV2, RequestMirrorPolicy) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RequestMirrorPolicy)) { const std::string yaml = R"EOF( name: foo virtual_hosts: @@ -4264,7 +4265,7 @@ TEST_F(RoutePropertyTest, excludeVHRateLimits) { EXPECT_TRUE(config_ptr->route(headers, 0)->routeEntry()->includeVirtualHostRateLimits()); } -TEST_F(RoutePropertyTest, TestVHostCorsConfig) { +TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestVHostCorsConfig)) { const std::string yaml = R"EOF( virtual_hosts: - name: "default" @@ -4326,7 +4327,7 @@ TEST_F(RoutePropertyTest, TestVHostCorsConfig) { EXPECT_EQ(cors_policy->allowCredentials(), true); } -TEST_F(RoutePropertyTest, TestRouteCorsConfig) { +TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestRouteCorsConfig)) { const std::string yaml = R"EOF( virtual_hosts: - name: "default" @@ -4379,7 +4380,7 @@ TEST_F(RoutePropertyTest, TestRouteCorsConfig) { EXPECT_EQ(cors_policy->allowCredentials(), true); } -TEST_F(RoutePropertyTest, TestVHostCorsLegacyConfig) { +TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestVHostCorsLegacyConfig)) { const std::string yaml = R"EOF( virtual_hosts: - name: default @@ -4418,7 +4419,7 @@ TEST_F(RoutePropertyTest, TestVHostCorsLegacyConfig) { EXPECT_EQ(cors_policy->allowCredentials(), true); } -TEST_F(RoutePropertyTest, TestRouteCorsLegacyConfig) { +TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestRouteCorsLegacyConfig)) { const std::string yaml = R"EOF( virtual_hosts: - name: default @@ -4909,7 +4910,7 @@ name: foo Envoy::EnvoyException, "Cannot create a Baz when metadata is empty."); } -TEST_F(RouteConfigurationV2, RouteConfigGetters) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RouteConfigGetters)) { const std::string yaml = R"EOF( name: foo virtual_hosts: @@ -4948,7 +4949,7 @@ name: foo EXPECT_EQ("foo", route_entry->virtualHost().routeConfig().name()); } -TEST_F(RouteConfigurationV2, RouteTracingConfig) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RouteTracingConfig)) { const std::string yaml = R"EOF( name: foo virtual_hosts: @@ -5003,7 +5004,7 @@ name: foo } // Test to check Prefix Rewrite for redirects -TEST_F(RouteConfigurationV2, RedirectPrefixRewrite) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RedirectPrefixRewrite)) { std::string RedirectPrefixRewrite = R"EOF( name: AllRedirects virtual_hosts: @@ -5190,7 +5191,7 @@ name: AllRedirects } } -TEST_F(RouteMatcherTest, HeaderMatchedRoutingV2) { +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(HeaderMatchedRoutingV2)) { const std::string yaml = R"EOF( name: foo virtual_hosts: @@ -5365,7 +5366,8 @@ name: foo } } -TEST_F(RouteConfigurationV2, RegexPrefixWithNoRewriteWorksWhenPathChanged) { +TEST_F(RouteConfigurationV2, + DEPRECATED_FEATURE_TEST(RegexPrefixWithNoRewriteWorksWhenPathChanged)) { // Setup regex route entry. the regex is trivial, that's ok as we only want to test that // path change works. @@ -5396,7 +5398,7 @@ name: RegexNoMatch } } -TEST_F(RouteConfigurationV2, NoIdleTimeout) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(NoIdleTimeout)) { const std::string NoIdleTimeout = R"EOF( name: NoIdleTimeout virtual_hosts: @@ -5414,7 +5416,7 @@ name: NoIdleTimeout EXPECT_EQ(absl::nullopt, route_entry->idleTimeout()); } -TEST_F(RouteConfigurationV2, ZeroIdleTimeout) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(ZeroIdleTimeout)) { const std::string ZeroIdleTimeout = R"EOF( name: ZeroIdleTimeout virtual_hosts: @@ -5433,7 +5435,7 @@ name: ZeroIdleTimeout EXPECT_EQ(0, route_entry->idleTimeout().value().count()); } -TEST_F(RouteConfigurationV2, ExplicitIdleTimeout) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(ExplicitIdleTimeout)) { const std::string ExplicitIdleTimeout = R"EOF( name: ExplicitIdleTimeout virtual_hosts: @@ -5453,7 +5455,7 @@ name: ExplicitIdleTimeout EXPECT_EQ(7 * 1000, route_entry->idleTimeout().value().count()); } -TEST_F(RouteConfigurationV2, RetriableStatusCodes) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RetriableStatusCodes)) { const std::string ExplicitIdleTimeout = R"EOF( name: RetriableStatusCodes virtual_hosts: @@ -5475,7 +5477,7 @@ name: RetriableStatusCodes EXPECT_EQ(expected_codes, retry_policy.retriableStatusCodes()); } -TEST_F(RouteConfigurationV2, UpgradeConfigs) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(UpgradeConfigs)) { const std::string UpgradeYaml = R"EOF( name: RetriableStatusCodes virtual_hosts: @@ -5499,7 +5501,7 @@ name: RetriableStatusCodes EXPECT_FALSE(upgrade_map.find("disabled")->second); } -TEST_F(RouteConfigurationV2, DuplicateUpgradeConfigs) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(DuplicateUpgradeConfigs)) { const std::string yaml = R"EOF( name: RetriableStatusCodes virtual_hosts: @@ -5522,7 +5524,7 @@ name: RetriableStatusCodes // Verifies that we're creating a new instance of the retry plugins on each call instead of always // returning the same one. -TEST_F(RouteConfigurationV2, RetryPluginsAreNotReused) { +TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RetryPluginsAreNotReused)) { const std::string ExplicitIdleTimeout = R"EOF( name: RetriableStatusCodes virtual_hosts: diff --git a/test/config/integration/server.yaml b/test/config/integration/server.yaml index 88d34049619b..95a9a439efa4 100644 --- a/test/config/integration/server.yaml +++ b/test/config/integration/server.yaml @@ -168,10 +168,19 @@ stats_sinks: "@type": type.googleapis.com/envoy.config.metrics.v2.StatsdSink tcp_cluster_name: statsd watchdog: {} -runtime: - symlink_root: "{{ test_tmpdir }}/test/common/runtime/test_data/current" - subdirectory: envoy - override_subdirectory: envoy_override +layered_runtime: + layers: + - name: root + disk_layer: + symlink_root: "{{ test_tmpdir }}/test/common/runtime/test_data/current" + subdirectory: envoy + - name: override + disk_layer: + symlink_root: "{{ test_tmpdir }}/test/common/runtime/test_data/current" + subdirectory: envoy_override + append_service_cluster: true + - name: admin + admin_layer: {} admin: access_log_path: "/dev/null" profile_path: "{{ test_tmpdir }}/envoy.prof" diff --git a/test/config_test/example_configs_test.cc b/test/config_test/example_configs_test.cc index 6da6d5e55f0a..fef29e44112c 100644 --- a/test/config_test/example_configs_test.cc +++ b/test/config_test/example_configs_test.cc @@ -5,7 +5,7 @@ #include "gtest/gtest.h" namespace Envoy { -TEST(ExampleConfigsTest, All) { +TEST(ExampleConfigsTest, DEPRECATED_FEATURE_TEST(All)) { TestEnvironment::exec( {TestEnvironment::runfilesPath("test/config_test/example_configs_test_setup.sh")}); diff --git a/test/extensions/filters/http/cors/cors_filter_integration_test.cc b/test/extensions/filters/http/cors/cors_filter_integration_test.cc index b313d6162497..06a9120c8c4a 100644 --- a/test/extensions/filters/http/cors/cors_filter_integration_test.cc +++ b/test/extensions/filters/http/cors/cors_filter_integration_test.cc @@ -195,6 +195,33 @@ TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestCorsDisabled)) { }); } +TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestLegacyCorsDisabled)) { + config_helper_.addConfigModifier( + [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm) + -> void { + auto* route_config = hcm.mutable_route_config(); + auto* virtual_host = route_config->mutable_virtual_hosts(0); + auto* route = virtual_host->add_routes(); + route->mutable_match()->set_prefix("/legacy-no-cors"); + route->mutable_route()->set_cluster("cluster_0"); + route->mutable_route()->mutable_cors()->mutable_enabled()->set_value(false); + }); + testNormalRequest( + Http::TestHeaderMapImpl{ + {":method", "OPTIONS"}, + {":path", "/legacy-no-cors/test"}, + {":scheme", "http"}, + {":authority", "test-host"}, + {"access-control-request-method", "GET"}, + {"origin", "test-origin"}, + }, + Http::TestHeaderMapImpl{ + {"server", "envoy"}, + {"content-length", "0"}, + {":status", "200"}, + }); +} + TEST_P(CorsFilterIntegrationTest, DEPRECATED_FEATURE_TEST(TestEncodeHeaders)) { testNormalRequest( Http::TestHeaderMapImpl{ diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index bd9de61eeb2a..f93f40b47f5d 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -1156,7 +1156,8 @@ TEST_F(ConnectionManagerTest, PendingMessageEnd) { EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); } -TEST_F(ConnectionManagerTest, Routing) { +// TODO(alyssawilk) update. +TEST_F(ConnectionManagerTest, DEPRECATED_FEATURE_TEST(Routing)) { const std::string yaml = R"EOF( stat_prefix: test protocol_type: Dubbo diff --git a/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc b/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc index ad648d51fdc4..398743469bcb 100644 --- a/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc @@ -38,7 +38,8 @@ parseDubboProxyFromV2Yaml(const std::string& yaml) { } // namespace -TEST(DubboRouteMatcherTest, RouteByServiceNameWithAnyMethod) { +// TODO(alyssawilk) update. +TEST(DubboRouteMatcherTest, DEPRECATED_FEATURE_TEST(RouteByServiceNameWithAnyMethod)) { { const std::string yaml = R"EOF( name: local_route @@ -291,7 +292,8 @@ interface: org.apache.dubbo.demo.DemoService EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } -TEST(DubboRouteMatcherTest, RouteByMethodWithRegexMatch) { +// TODO(alyssawilk) update. +TEST(DubboRouteMatcherTest, DEPRECATED_FEATURE_TEST(RouteByMethodWithRegexMatch)) { const std::string yaml = R"EOF( name: local_route interface: org.apache.dubbo.demo.DemoService diff --git a/test/extensions/filters/network/redis_proxy/config_test.cc b/test/extensions/filters/network/redis_proxy/config_test.cc index ff348c9c1a06..8c34fb502fd7 100644 --- a/test/extensions/filters/network/redis_proxy/config_test.cc +++ b/test/extensions/filters/network/redis_proxy/config_test.cc @@ -40,7 +40,9 @@ TEST(RedisProxyFilterConfigFactoryTest, NoUpstreamDefined) { TEST(RedisProxyFilterConfigFactoryTest, RedisProxyNoSettings) { const std::string yaml = R"EOF( -cluster: fake_cluster +prefix_routes: + catch_all_route: + cluster: fake_cluster stat_prefix: foo )EOF"; @@ -51,7 +53,9 @@ stat_prefix: foo TEST(RedisProxyFilterConfigFactoryTest, RedisProxyNoOpTimeout) { const std::string yaml = R"EOF( -cluster: fake_cluster +prefix_routes: + catch_all_route: + cluster: fake_cluster stat_prefix: foo settings: {} )EOF"; @@ -61,7 +65,8 @@ settings: {} ProtoValidationException, "embedded message failed validation"); } -TEST(RedisProxyFilterConfigFactoryTest, RedisProxyCorrectProto) { +TEST(RedisProxyFilterConfigFactoryTest, + DEPRECATED_FEATURE_TEST(RedisProxyCorrectProtoLegacyCluster)) { const std::string yaml = R"EOF( cluster: fake_cluster stat_prefix: foo @@ -80,9 +85,53 @@ stat_prefix: foo cb(connection); } +TEST(RedisProxyFilterConfigFactoryTest, + DEPRECATED_FEATURE_TEST(RedisProxyCorrectProtoLegacyCatchAllCluster)) { + const std::string yaml = R"EOF( +prefix_routes: + catch_all_cluster: fake_cluster +stat_prefix: foo +settings: + op_timeout: 0.02s + )EOF"; + + envoy::config::filter::network::redis_proxy::v2::RedisProxy proto_config{}; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + NiceMock context; + RedisProxyFilterConfigFactory factory; + Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); + EXPECT_TRUE(factory.isTerminalFilter()); + Network::MockConnection connection; + EXPECT_CALL(connection, addReadFilter(_)); + cb(connection); +} + +TEST(RedisProxyFilterConfigFactoryTest, RedisProxyCorrectProto) { + const std::string yaml = R"EOF( +prefix_routes: + catch_all_route: + cluster: fake_cluster +stat_prefix: foo +settings: + op_timeout: 0.02s + )EOF"; + + envoy::config::filter::network::redis_proxy::v2::RedisProxy proto_config{}; + TestUtility::loadFromYamlAndValidate(yaml, proto_config); + NiceMock context; + RedisProxyFilterConfigFactory factory; + Network::FilterFactoryCb cb = factory.createFilterFactoryFromProto(proto_config, context); + EXPECT_TRUE(factory.isTerminalFilter()); + Network::MockConnection connection; + EXPECT_CALL(connection, addReadFilter(_)); + cb(connection); +} + TEST(RedisProxyFilterConfigFactoryTest, RedisProxyEmptyProto) { const std::string yaml = R"EOF( -cluster: fake_cluster +prefix_routes: + catch_all_route: + cluster: fake_cluster stat_prefix: foo settings: op_timeout: 0.02s diff --git a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc index d29c89833022..c7efe51ec11c 100644 --- a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc @@ -331,7 +331,8 @@ name: config EXPECT_EQ("cluster1", route->routeEntry()->clusterName()); } -TEST(ThriftRouteMatcherTest, RouteByRegexHeaderMatcher) { +// TODO(alyssawilk) update. +TEST(ThriftRouteMatcherTest, DEPRECATED_FEATURE_TEST(RouteByRegexHeaderMatcher)) { const std::string yaml = R"EOF( name: config routes: diff --git a/test/integration/integration_admin_test.cc b/test/integration/integration_admin_test.cc index 43c3a867bfd6..41085b762d9c 100644 --- a/test/integration/integration_admin_test.cc +++ b/test/integration/integration_admin_test.cc @@ -542,21 +542,21 @@ TEST_P(StatsMatcherIntegrationTest, ExcludePrefixServerDot) { EXPECT_THAT(response_->body(), testing::Not(testing::HasSubstr("server."))); } -TEST_P(StatsMatcherIntegrationTest, ExcludeRequests) { +TEST_P(StatsMatcherIntegrationTest, DEPRECATED_FEATURE_TEST(ExcludeRequests)) { stats_matcher_.mutable_exclusion_list()->add_patterns()->set_regex(".*requests.*"); initialize(); makeRequest(); EXPECT_THAT(response_->body(), testing::Not(testing::HasSubstr("requests"))); } -TEST_P(StatsMatcherIntegrationTest, ExcludeExact) { +TEST_P(StatsMatcherIntegrationTest, DEPRECATED_FEATURE_TEST(ExcludeExact)) { stats_matcher_.mutable_exclusion_list()->add_patterns()->set_exact("server.concurrency"); initialize(); makeRequest(); EXPECT_THAT(response_->body(), testing::Not(testing::HasSubstr("server.concurrency"))); } -TEST_P(StatsMatcherIntegrationTest, ExcludeMultipleExact) { +TEST_P(StatsMatcherIntegrationTest, DEPRECATED_FEATURE_TEST(ExcludeMultipleExact)) { stats_matcher_.mutable_exclusion_list()->add_patterns()->set_exact("server.concurrency"); stats_matcher_.mutable_exclusion_list()->add_patterns()->set_regex(".*live"); initialize(); @@ -569,7 +569,7 @@ TEST_P(StatsMatcherIntegrationTest, ExcludeMultipleExact) { // `listener_manager.listener_create_success` must be instantiated, because BaseIntegrationTest // blocks on its creation (see waitForCounterGe and the suite of waitFor* functions). // If this invariant is changed, this test must be rewritten. -TEST_P(StatsMatcherIntegrationTest, IncludeExact) { +TEST_P(StatsMatcherIntegrationTest, DEPRECATED_FEATURE_TEST(IncludeExact)) { // Stats matching does not play well with LDS, at least in test. See #7215. use_lds_ = false; stats_matcher_.mutable_inclusion_list()->add_patterns()->set_exact( diff --git a/test/server/BUILD b/test/server/BUILD index 7ffe0fcfd36b..e46a0106c687 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -261,6 +261,7 @@ envoy_cc_test( ":invalid_layered_runtime_duplicate_name.yaml", ":invalid_layered_runtime_missing_name.yaml", ":invalid_layered_runtime_no_layer_specifier.yaml", + ":invalid_legacy_runtime_bootstrap.yaml", ":invalid_runtime_bootstrap.yaml", ":node_bootstrap.yaml", ":node_bootstrap_no_admin_port.yaml", diff --git a/test/server/invalid_legacy_runtime_bootstrap.yaml b/test/server/invalid_legacy_runtime_bootstrap.yaml new file mode 100644 index 000000000000..99c67b7d2d9c --- /dev/null +++ b/test/server/invalid_legacy_runtime_bootstrap.yaml @@ -0,0 +1,4 @@ +runtime: + base: + foo: + - bar: baz diff --git a/test/server/invalid_runtime_bootstrap.yaml b/test/server/invalid_runtime_bootstrap.yaml index 99c67b7d2d9c..3ed04a71c3b0 100644 --- a/test/server/invalid_runtime_bootstrap.yaml +++ b/test/server/invalid_runtime_bootstrap.yaml @@ -1,4 +1,6 @@ -runtime: - base: - foo: - - bar: baz +layered_runtime: + layers: + - name: some_static_layer + static_layer: + foo: + - bar: baz diff --git a/test/server/server_test.cc b/test/server/server_test.cc index 1b9b0deeee19..ab0d89ebad28 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -575,6 +575,11 @@ TEST_P(ServerInstanceImplTest, RuntimeNoAdminLayer) { EXPECT_EQ("No admin layer specified", response_body); } +TEST_P(ServerInstanceImplTest, DEPRECATED_FEATURE_TEST(InvalidLegacyBootstrapRuntime)) { + EXPECT_THROW_WITH_MESSAGE(initialize("test/server/invalid_runtime_bootstrap.yaml"), + EnvoyException, "Invalid runtime entry value for foo"); +} + // Validate invalid runtime in bootstrap is rejected. TEST_P(ServerInstanceImplTest, InvalidBootstrapRuntime) { EXPECT_THROW_WITH_MESSAGE(initialize("test/server/invalid_runtime_bootstrap.yaml"), From f8e42ae3456baba1990ad91e3c43fbe22ea9b6ab Mon Sep 17 00:00:00 2001 From: Colin Schoen Date: Wed, 28 Aug 2019 08:35:41 -0700 Subject: [PATCH 470/542] [Docs typo] Remote Executioon -> Remote Execution (#8061) Fixes mispelling of `Executioon` -> `Execution` Signed-off-by: Colin Schoen --- bazel/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bazel/README.md b/bazel/README.md index 24340fdd0fce..b99b9e50621e 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -102,7 +102,7 @@ CI Docker image. ## Building Envoy with Remote Execution -Envoy can also be built with Bazel [Remote Executioon](https://docs.bazel.build/versions/master/remote-execution.html), +Envoy can also be built with Bazel [Remote Execution](https://docs.bazel.build/versions/master/remote-execution.html), part of the CI is running with the hosted [GCP RBE](https://blog.bazel.build/2018/10/05/remote-build-execution.html) service. To build Envoy with a remote build services, run Bazel with your remote build service flags and with `--config=remote-clang`. From f80188ebc4b592754449c740f831123b4de41ab3 Mon Sep 17 00:00:00 2001 From: Bryce Anderson Date: Wed, 28 Aug 2019 09:40:38 -0600 Subject: [PATCH 471/542] api: Fix duplicate java_outer_classname declarations (#8059) The java_outer_classname is unintentionally duplicated in the new udp_listener_config and regex proto files. This changes them to unique names that match the predominant naming scheme. Signed-off-by: Bryce Anderson --- api/envoy/api/v2/listener/udp_listener_config.proto | 2 +- api/envoy/type/matcher/regex.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/envoy/api/v2/listener/udp_listener_config.proto b/api/envoy/api/v2/listener/udp_listener_config.proto index 88a2a35d3cfc..f75383bab232 100644 --- a/api/envoy/api/v2/listener/udp_listener_config.proto +++ b/api/envoy/api/v2/listener/udp_listener_config.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.api.v2.listener; -option java_outer_classname = "ListenerProto"; +option java_outer_classname = "UdpListenerConfigProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.listener"; option go_package = "listener"; diff --git a/api/envoy/type/matcher/regex.proto b/api/envoy/type/matcher/regex.proto index b3b7194441eb..048a576cc8a6 100644 --- a/api/envoy/type/matcher/regex.proto +++ b/api/envoy/type/matcher/regex.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.type.matcher; -option java_outer_classname = "StringProto"; +option java_outer_classname = "RegexProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type.matcher"; option go_package = "matcher"; From b8966cbbfff4566b08eaed7b2b060a6c12eec168 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 28 Aug 2019 13:40:18 -0400 Subject: [PATCH 472/542] http: making the behavior of the response Server header configurable (#8014) Default behavior remains unchanged, but now Envoy can override, override iff there's no server header from upstream, or always leave the server header (or lack thereof) unmodified. Risk Level: low (config guarded change) Testing: new unit tests Docs Changes: n/a Release Notes: inline Fixes #6716 Signed-off-by: Alyssa Wilk --- .../v2/http_connection_manager.proto | 20 ++++- docs/root/intro/version_history.rst | 2 + source/common/http/conn_manager_config.h | 9 +++ source/common/http/conn_manager_impl.cc | 7 +- .../network/http_connection_manager/BUILD | 1 + .../network/http_connection_manager/config.cc | 2 + .../network/http_connection_manager/config.h | 5 ++ source/server/http/admin.h | 3 + .../http/conn_manager_impl_fuzz_test.cc | 5 ++ test/common/http/conn_manager_impl_test.cc | 79 +++++++++++++++++++ test/common/http/conn_manager_utility_test.cc | 2 + .../http_connection_manager/config_test.cc | 62 +++++++++++++++ test/server/http/admin_test.cc | 12 +++ 13 files changed, 207 insertions(+), 2 deletions(-) diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 4d662fc2c524..bf1195731b86 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -25,7 +25,7 @@ import "gogoproto/gogo.proto"; // [#protodoc-title: HTTP connection manager] // HTTP connection manager :ref:`configuration overview `. -// [#comment:next free field: 34] +// [#comment:next free field: 35] message HttpConnectionManager { enum CodecType { option (gogoproto.goproto_enum_prefix) = false; @@ -144,6 +144,24 @@ message HttpConnectionManager { // header in responses. If not set, the default is *envoy*. string server_name = 10; + enum ServerHeaderTransformation { + option (gogoproto.goproto_enum_prefix) = false; + + // Overwrite any Server header with the contents of server_name. + OVERWRITE = 0; + // If no Server header is present, append Server server_name + // If a Server header is present, pass it through. + APPEND_IF_ABSENT = 1; + // Pass through the value of the server header, and do not append a header + // if none is present. + PASS_THROUGH = 2; + } + // Defines the action to be applied to the Server header on the response path. + // By default, Envoy will overwrite the header with the value specified in + // server_name. + ServerHeaderTransformation server_header_transformation = 34 + [(validate.rules).enum.defined_only = true]; + // The maximum request headers size for incoming connections. // If unconfigured, the default max request headers allowed is 60 KiB. // Requests that exceed this limit will receive a 431 response. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 147c65f5ce60..715ec0fa839f 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -27,6 +27,8 @@ Version history * grpc-json: added support for :ref:`ignoring unknown query parameters`. * header to metadata: added :ref:`PROTOBUF_VALUE ` and :ref:`ValueEncode ` to support protobuf Value and Base64 encoding. * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. +* http: changed Envoy to forward existing x-forwarded-proto from upstream trusted proxies. Guarded by `envoy.reloadable_features.trusted_forwarded_proto` which defaults true. +* http: added the ability to configure the behavior of the server response header, via the :ref:`server_header_transformation` field. * http: changed Envoy to forward existing x-forwarded-proto from downstream trusted proxies. Guarded by `envoy.reloadable_features.trusted_forwarded_proto` which defaults true. * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 52ec547ea9d5..b32d8f520e40 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/config/config_provider.h" +#include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" #include "envoy/http/filter.h" #include "envoy/router/rds.h" #include "envoy/stats/scope.h" @@ -170,6 +171,9 @@ class DefaultInternalAddressConfig : public Http::InternalAddressConfig { */ class ConnectionManagerConfig { public: + using HttpConnectionManagerProto = + envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager; + virtual ~ConnectionManagerConfig() = default; /** @@ -265,6 +269,11 @@ class ConnectionManagerConfig { */ virtual const std::string& serverName() PURE; + /** + * @return ServerHeaderTransformation the transformation to apply to Server response headers. + */ + virtual HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() PURE; + /** * @return ConnectionManagerStats& the stats to write to. */ diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 580ba1e6c365..c830114f5107 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1353,7 +1353,12 @@ void ConnectionManagerImpl::ActiveStream::encodeHeaders(ActiveStreamEncoderFilte // Base headers. connection_manager_.config_.dateProvider().setDateHeader(headers); // Following setReference() is safe because serverName() is constant for the life of the listener. - headers.insertServer().value().setReference(connection_manager_.config_.serverName()); + const auto transformation = connection_manager_.config_.serverHeaderTransformation(); + if (transformation == ConnectionManagerConfig::HttpConnectionManagerProto::OVERWRITE || + (transformation == ConnectionManagerConfig::HttpConnectionManagerProto::APPEND_IF_ABSENT && + headers.Server() == nullptr)) { + headers.insertServer().value().setReference(connection_manager_.config_.serverName()); + } ConnectionManagerUtility::mutateResponseHeaders(headers, request_headers_.get(), connection_manager_.config_.via()); diff --git a/source/extensions/filters/network/http_connection_manager/BUILD b/source/extensions/filters/network/http_connection_manager/BUILD index 1f5474a651e2..fbe72b257b15 100644 --- a/source/extensions/filters/network/http_connection_manager/BUILD +++ b/source/extensions/filters/network/http_connection_manager/BUILD @@ -40,5 +40,6 @@ envoy_cc_library( "//source/common/router:scoped_rds_lib", "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", + "@envoy_api//envoy/config/filter/network/http_connection_manager/v2:http_connection_manager_cc", ], ) diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 82fb5274bb96..81a45e934761 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -286,6 +286,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( access_logs_.push_back(current_access_log); } + server_transformation_ = config.server_header_transformation(); + if (!config.server_name().empty()) { server_name_ = config.server_name(); } else { diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index 8e8132e0aa01..0385762236c1 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -116,6 +116,9 @@ class HttpConnectionManagerConfig : Logger::Loggable, return scoped_routes_config_provider_.get(); } const std::string& serverName() override { return server_name_; } + HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() override { + return server_transformation_; + } Http::ConnectionManagerStats& stats() override { return stats_; } Http::ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } bool useRemoteAddress() override { return use_remote_address_; } @@ -166,6 +169,8 @@ class HttpConnectionManagerConfig : Logger::Loggable, CodecType codec_type_; const Http::Http2Settings http2_settings_; const Http::Http1Settings http1_settings_; + HttpConnectionManagerProto::ServerHeaderTransformation server_transformation_{ + HttpConnectionManagerProto::OVERWRITE}; std::string server_name_; Http::TracingConnectionManagerConfigPtr tracing_config_; absl::optional user_agent_; diff --git a/source/server/http/admin.h b/source/server/http/admin.h index ac01902068a7..f54d72fee7ff 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -121,6 +121,9 @@ class AdminImpl : public Admin, return &scoped_route_config_provider_; } const std::string& serverName() override { return Http::DefaultServerString::get(); } + HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() override { + return HttpConnectionManagerProto::OVERWRITE; + } Http::ConnectionManagerStats& stats() override { return stats_; } Http::ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } bool useRemoteAddress() override { return true; } diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 47f151b5b1db..df9a0c2b4b35 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -91,6 +91,9 @@ class FuzzConfig : public ConnectionManagerConfig { return &scoped_route_config_provider_; } const std::string& serverName() override { return server_name_; } + HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() override { + return server_transformation_; + } ConnectionManagerStats& stats() override { return stats_; } ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } bool useRemoteAddress() override { return use_remote_address_; } @@ -124,6 +127,8 @@ class FuzzConfig : public ConnectionManagerConfig { ConnectionManagerImplHelper::RouteConfigProvider route_config_provider_; ConnectionManagerImplHelper::ScopedRouteConfigProvider scoped_route_config_provider_; std::string server_name_; + HttpConnectionManagerProto::ServerHeaderTransformation server_transformation_{ + HttpConnectionManagerProto::OVERWRITE}; Stats::IsolatedStoreImpl fake_stats_; ConnectionManagerStats stats_; ConnectionManagerTracingStats tracing_stats_; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 251c62394907..496b2af6c7ea 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -227,6 +227,21 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan conn_manager_->onData(fake_input, false); } + HeaderMap* sendResponseHeaders(HeaderMapPtr&& response_headers) { + HeaderMap* altered_response_headers = nullptr; + + EXPECT_CALL(*encoder_filters_[0], encodeHeaders(_, _)) + .WillOnce(Invoke([&](HeaderMap& headers, bool) -> FilterHeadersStatus { + altered_response_headers = &headers; + return FilterHeadersStatus::Continue; + })); + EXPECT_CALL(*encoder_filters_[1], encodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::Continue)); + EXPECT_CALL(response_encoder_, encodeHeaders(_, false)); + decoder_filters_[0]->callbacks_->encodeHeaders(std::move(response_headers), false); + return altered_response_headers; + } + void expectOnDestroy() { for (auto filter : decoder_filters_) { EXPECT_CALL(*filter, onDestroy()); @@ -261,6 +276,9 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan return &scoped_route_config_provider_; } const std::string& serverName() override { return server_name_; } + HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() override { + return server_transformation_; + } ConnectionManagerStats& stats() override { return stats_; } ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } bool useRemoteAddress() override { return use_remote_address_; } @@ -301,6 +319,8 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan NiceMock drain_close_; std::unique_ptr conn_manager_; std::string server_name_; + HttpConnectionManagerProto::ServerHeaderTransformation server_transformation_{ + HttpConnectionManagerProto::OVERWRITE}; Network::Address::Ipv4Instance local_address_{"127.0.0.1"}; bool use_remote_address_{true}; Http::DefaultInternalAddressConfig internal_address_config_; @@ -537,6 +557,65 @@ TEST_F(HttpConnectionManagerImplTest, PauseResume100Continue) { decoder_filters_[1]->callbacks_->encodeHeaders(std::move(response_headers), false); } +// By default, Envoy will set the server header to the server name, here "custom-value" +TEST_F(HttpConnectionManagerImplTest, ServerHeaderOverwritten) { + setup(false, "custom-value", false); + setUpEncoderAndDecoder(false, false); + + sendRequestHeadersAndData(); + const HeaderMap* altered_headers = sendResponseHeaders( + HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}, {"server", "foo"}}}); + EXPECT_EQ("custom-value", altered_headers->Server()->value().getStringView()); +} + +// When configured APPEND_IF_ABSENT if the server header is present it will be retained. +TEST_F(HttpConnectionManagerImplTest, ServerHeaderAppendPresent) { + server_transformation_ = HttpConnectionManagerProto::APPEND_IF_ABSENT; + setup(false, "custom-value", false); + setUpEncoderAndDecoder(false, false); + + sendRequestHeadersAndData(); + const HeaderMap* altered_headers = sendResponseHeaders( + HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}, {"server", "foo"}}}); + EXPECT_EQ("foo", altered_headers->Server()->value().getStringView()); +} + +// When configured APPEND_IF_ABSENT if the server header is absent the server name will be set. +TEST_F(HttpConnectionManagerImplTest, ServerHeaderAppendAbsent) { + server_transformation_ = HttpConnectionManagerProto::APPEND_IF_ABSENT; + setup(false, "custom-value", false); + setUpEncoderAndDecoder(false, false); + + sendRequestHeadersAndData(); + const HeaderMap* altered_headers = + sendResponseHeaders(HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}}}); + EXPECT_EQ("custom-value", altered_headers->Server()->value().getStringView()); +} + +// When configured PASS_THROUGH, the server name will pass through. +TEST_F(HttpConnectionManagerImplTest, ServerHeaderPassthroughPresent) { + server_transformation_ = HttpConnectionManagerProto::PASS_THROUGH; + setup(false, "custom-value", false); + setUpEncoderAndDecoder(false, false); + + sendRequestHeadersAndData(); + const HeaderMap* altered_headers = sendResponseHeaders( + HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}, {"server", "foo"}}}); + EXPECT_EQ("foo", altered_headers->Server()->value().getStringView()); +} + +// When configured PASS_THROUGH, the server header will not be added if absent. +TEST_F(HttpConnectionManagerImplTest, ServerHeaderPassthroughAbsent) { + server_transformation_ = HttpConnectionManagerProto::PASS_THROUGH; + setup(false, "custom-value", false); + setUpEncoderAndDecoder(false, false); + + sendRequestHeadersAndData(); + const HeaderMap* altered_headers = + sendResponseHeaders(HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}}}); + EXPECT_TRUE(altered_headers->Server() == nullptr); +} + TEST_F(HttpConnectionManagerImplTest, InvalidPathWithDualFilter) { InSequence s; setup(false, ""); diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 025a21ab1caf..b0c8e005124f 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -64,6 +64,8 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_METHOD0(routeConfigProvider, Router::RouteConfigProvider*()); MOCK_METHOD0(scopedRouteConfigProvider, Config::ConfigProvider*()); MOCK_METHOD0(serverName, const std::string&()); + MOCK_METHOD0(serverHeaderTransformation, + HttpConnectionManagerProto::ServerHeaderTransformation()); MOCK_METHOD0(stats, ConnectionManagerStats&()); MOCK_METHOD0(tracingStats, ConnectionManagerTracingStats&()); MOCK_METHOD0(useRemoteAddress, bool()); diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index c300885c09fb..0b048d28e3dc 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -237,6 +237,8 @@ stat_prefix: router ContainerEq(config.tracingConfig()->request_headers_for_tags_)); EXPECT_EQ(*context_.local_info_.address_, config.localAddress()); EXPECT_EQ("foo", config.serverName()); + EXPECT_EQ(HttpConnectionManagerConfig::HttpConnectionManagerProto::OVERWRITE, + config.serverHeaderTransformation()); EXPECT_EQ(5 * 60 * 1000, config.streamIdleTimeout().count()); } @@ -388,6 +390,66 @@ TEST_F(HttpConnectionManagerConfigTest, DisabledStreamIdleTimeout) { EXPECT_EQ(0, config.streamIdleTimeout().count()); } +TEST_F(HttpConnectionManagerConfigTest, ServerOverwrite) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + server_header_transformation: OVERWRITE + route_config: + name: local_route + http_filters: + - name: envoy.router + )EOF"; + + EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) + .WillOnce(Invoke(&context_.runtime_loader_.snapshot_, + &Runtime::MockSnapshot::featureEnabledDefault)); + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + EXPECT_EQ(HttpConnectionManagerConfig::HttpConnectionManagerProto::OVERWRITE, + config.serverHeaderTransformation()); +} + +TEST_F(HttpConnectionManagerConfigTest, ServerAppendIfAbsent) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + server_header_transformation: APPEND_IF_ABSENT + route_config: + name: local_route + http_filters: + - name: envoy.router + )EOF"; + + EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) + .WillOnce(Invoke(&context_.runtime_loader_.snapshot_, + &Runtime::MockSnapshot::featureEnabledDefault)); + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + EXPECT_EQ(HttpConnectionManagerConfig::HttpConnectionManagerProto::APPEND_IF_ABSENT, + config.serverHeaderTransformation()); +} + +TEST_F(HttpConnectionManagerConfigTest, ServerPassThrough) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + server_header_transformation: PASS_THROUGH + route_config: + name: local_route + http_filters: + - name: envoy.router + )EOF"; + + EXPECT_CALL(context_.runtime_loader_.snapshot_, featureEnabled(_, An())) + .WillOnce(Invoke(&context_.runtime_loader_.snapshot_, + &Runtime::MockSnapshot::featureEnabledDefault)); + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + EXPECT_EQ(HttpConnectionManagerConfig::HttpConnectionManagerProto::PASS_THROUGH, + config.serverHeaderTransformation()); +} + // Validated that by default we don't normalize paths // unless set build flag path_normalization_by_default=true TEST_F(HttpConnectionManagerConfigTest, NormalizePathDefault) { diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index 24dbed7b348e..38b87e72fb7b 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -21,6 +21,7 @@ #include "extensions/transport_sockets/tls/context_config_impl.h" +#include "test/mocks/http/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/environment.h" @@ -89,6 +90,17 @@ class AdminFilterTest : public testing::TestWithParam Date: Wed, 28 Aug 2019 12:35:10 -0700 Subject: [PATCH 473/542] use bazelversion for filter-example too (#8069) Signed-off-by: Lizan Zhou --- ci/build_setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_setup.sh b/ci/build_setup.sh index 53379d9ab8da..9dc53668825c 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -95,6 +95,7 @@ if [ "$1" != "-nofetch" ]; then # This is the hash on https://github.com/envoyproxy/envoy-filter-example.git we pin to. (cd "${ENVOY_FILTER_EXAMPLE_SRCDIR}" && git fetch origin && git checkout -f 1995c1e0eccea84bbb39f64e75ef3e9102d1ae82) sed -e "s|{ENVOY_SRCDIR}|${ENVOY_SRCDIR}|" "${ENVOY_SRCDIR}"/ci/WORKSPACE.filter.example > "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/WORKSPACE + cp -f "${ENVOY_SRCDIR}"/.bazelversion "${ENVOY_FILTER_EXAMPLE_SRCDIR}"/.bazelversion fi # Also setup some space for building Envoy standalone. From c2e8edad54609c752d4594b69252048af84e798e Mon Sep 17 00:00:00 2001 From: Hans Duedal Date: Wed, 28 Aug 2019 23:56:21 +0200 Subject: [PATCH 474/542] grpc-httpjson-transcode: Update for RFC2045 support (#8065) RFC2045 (MIME) Base64 decoding support has been fixed upstream Description: The grpc transcoding filter has been updated to support RFC2045 (MIME) based inputs for protobuf type "Bytes". This is important since Base64 is often using the RFC2045 format for inputs. Also see: grpc-ecosystem/grpc-httpjson-transcoding#34 Risk Level: Low Testing: Integration / Manual Tests Docs Changes: N/A Release Notes: N/A Signed-off-by: Hans Viken Duedal --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 2abff095453a..5dec6ea9bb10 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -212,10 +212,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v3.8.0/protobuf-all-3.8.0.tar.gz"], ), grpc_httpjson_transcoding = dict( - sha256 = "dedd76b0169eb8c72e479529301a1d9b914a4ccb4d2b5ddb4ebe92d63a7b2152", - strip_prefix = "grpc-httpjson-transcoding-64d6ac985360b624d8e95105701b64a3814794cd", - # 2018-12-19 - urls = ["https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/archive/64d6ac985360b624d8e95105701b64a3814794cd.tar.gz"], + sha256 = "a447458b47ea4dc1d31499f555769af437c5d129d988ec1e13d5fdd0a6a36b4e", + strip_prefix = "grpc-httpjson-transcoding-2feabd5d64436e670084091a937855972ee35161", + # 2019-08-28 + urls = ["https://github.com/grpc-ecosystem/grpc-httpjson-transcoding/archive/2feabd5d64436e670084091a937855972ee35161.tar.gz"], ), io_bazel_rules_go = dict( sha256 = "96b1f81de5acc7658e1f5a86d7dc9e1b89bc935d83799b711363a748652c471a", From 5e45d4874db85be56259df2daa8c69af78987fcf Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Thu, 29 Aug 2019 07:26:01 +0900 Subject: [PATCH 475/542] stats: Clean up all calls to Scope::counter() et al in production code. (#7842) * Convert a few more counter() references to use the StatName interface. Signed-off-by: Joshua Marantz --- source/common/stats/symbol_table_impl.h | 11 ++- .../filters/http/fault/fault_filter.cc | 22 +++-- .../filters/http/fault/fault_filter.h | 16 +++- .../extensions/filters/http/ip_tagging/BUILD | 1 + .../http/ip_tagging/ip_tagging_filter.cc | 52 +++++++++- .../http/ip_tagging/ip_tagging_filter.h | 49 +++------- .../filters/network/mongo_proxy/BUILD | 11 +++ .../filters/network/mongo_proxy/config.cc | 7 +- .../network/mongo_proxy/mongo_stats.cc | 47 +++++++++ .../filters/network/mongo_proxy/mongo_stats.h | 56 +++++++++++ .../filters/network/mongo_proxy/proxy.cc | 96 ++++++++++++------- .../filters/network/mongo_proxy/proxy.h | 17 +++- .../stat_sinks/common/statsd/statsd.cc | 6 +- source/extensions/transport_sockets/tls/BUILD | 1 + .../transport_sockets/tls/context_impl.cc | 60 ++++++++++-- .../transport_sockets/tls/context_impl.h | 8 ++ source/server/BUILD | 2 + source/server/guarddog_impl.cc | 9 +- source/server/overload_manager_impl.cc | 24 +++-- .../filters/http/fault/fault_filter_test.cc | 2 +- .../http/ip_tagging/ip_tagging_filter_test.cc | 2 +- .../filters/network/mongo_proxy/proxy_test.cc | 10 +- .../transport_sockets/tls/ssl_socket_test.cc | 3 +- tools/check_format.py | 19 ---- tools/spelling_dictionary.txt | 2 + 25 files changed, 390 insertions(+), 143 deletions(-) create mode 100644 source/extensions/filters/network/mongo_proxy/mongo_stats.cc create mode 100644 source/extensions/filters/network/mongo_proxy/mongo_stats.h diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index 925ff26368a4..dbbb633dcbe8 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -652,10 +652,13 @@ class StatNameSet { void rememberBuiltin(absl::string_view str); /** - * Finds a StatName by name. If 'token' has been remembered as a built-in, then - * no lock is required. Otherwise we first consult dynamic_stat_names_ under a - * lock that's private to the StatNameSet. If that's empty, we need to create - * the StatName in the pool, which requires taking a global lock. + * Finds a StatName by name. If 'token' has been remembered as a built-in, + * then no lock is required. Otherwise we must consult dynamic_stat_names_ + * under a lock that's private to the StatNameSet. If that's empty, we need to + * create the StatName in the pool, which requires taking a global lock, and + * then remember the new StatName in the dynamic_stat_names_. This allows + * subsequent lookups of the same string to take only the set's lock, and not + * the whole symbol-table lock. * * TODO(jmarantz): Potential perf issue here with contention, both on this * set's mutex and also the SymbolTable mutex which must be taken during diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 0649366a7e5c..0fb09f1c95f1 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -78,7 +78,17 @@ FaultFilterConfig::FaultFilterConfig(const envoy::config::filter::http::fault::v Runtime::Loader& runtime, const std::string& stats_prefix, Stats::Scope& scope, TimeSource& time_source) : settings_(fault), runtime_(runtime), stats_(generateStats(stats_prefix, scope)), - stats_prefix_(stats_prefix), scope_(scope), time_source_(time_source) {} + scope_(scope), time_source_(time_source), stat_name_set_(scope.symbolTable()), + aborts_injected_(stat_name_set_.add("aborts_injected")), + delays_injected_(stat_name_set_.add("delays_injected")), + stats_prefix_(stat_name_set_.add(absl::StrCat(stats_prefix, "fault"))) {} + +void FaultFilterConfig::incCounter(absl::string_view downstream_cluster, + Stats::StatName stat_name) { + Stats::SymbolTable::StoragePtr storage = scope_.symbolTable().join( + {stats_prefix_, stat_name_set_.getStatName(downstream_cluster), stat_name}); + scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); +} FaultFilter::FaultFilter(FaultFilterConfigSharedPtr config) : config_(config) {} @@ -279,10 +289,7 @@ uint64_t FaultFilter::abortHttpStatus() { void FaultFilter::recordDelaysInjectedStats() { // Downstream specific stats. if (!downstream_cluster_.empty()) { - const std::string stats_counter = - fmt::format("{}fault.{}.delays_injected", config_->statsPrefix(), downstream_cluster_); - - config_->scope().counter(stats_counter).inc(); + config_->incDelays(downstream_cluster_); } // General stats. All injected faults are considered a single aggregate active fault. @@ -293,10 +300,7 @@ void FaultFilter::recordDelaysInjectedStats() { void FaultFilter::recordAbortsInjectedStats() { // Downstream specific stats. if (!downstream_cluster_.empty()) { - const std::string stats_counter = - fmt::format("{}fault.{}.aborts_injected", config_->statsPrefix(), downstream_cluster_); - - config_->scope().counter(stats_counter).inc(); + config_->incAborts(downstream_cluster_); } // General stats. All injected faults are considered a single aggregate active fault. diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index be6ef435420b..b0d3a0f1bf30 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -16,6 +16,7 @@ #include "common/buffer/watermark_buffer.h" #include "common/common/token_bucket_impl.h" #include "common/http/header_utility.h" +#include "common/stats/symbol_table_impl.h" #include "extensions/filters/common/fault/fault_config.h" @@ -111,20 +112,31 @@ class FaultFilterConfig { Runtime::Loader& runtime() { return runtime_; } FaultFilterStats& stats() { return stats_; } - const std::string& statsPrefix() { return stats_prefix_; } Stats::Scope& scope() { return scope_; } const FaultSettings* settings() { return &settings_; } TimeSource& timeSource() { return time_source_; } + void incDelays(absl::string_view downstream_cluster) { + incCounter(downstream_cluster, delays_injected_); + } + + void incAborts(absl::string_view downstream_cluster) { + incCounter(downstream_cluster, aborts_injected_); + } + private: static FaultFilterStats generateStats(const std::string& prefix, Stats::Scope& scope); + void incCounter(absl::string_view downstream_cluster, Stats::StatName stat_name); const FaultSettings settings_; Runtime::Loader& runtime_; FaultFilterStats stats_; - const std::string stats_prefix_; Stats::Scope& scope_; TimeSource& time_source_; + Stats::StatNameSet stat_name_set_; + const Stats::StatName aborts_injected_; + const Stats::StatName delays_injected_; + const Stats::StatName stats_prefix_; // Includes ".fault". }; using FaultFilterConfigSharedPtr = std::shared_ptr; diff --git a/source/extensions/filters/http/ip_tagging/BUILD b/source/extensions/filters/http/ip_tagging/BUILD index 583893eadb23..bc88c1313356 100644 --- a/source/extensions/filters/http/ip_tagging/BUILD +++ b/source/extensions/filters/http/ip_tagging/BUILD @@ -22,6 +22,7 @@ envoy_cc_library( "//source/common/http:header_map_lib", "//source/common/http:headers_lib", "//source/common/network:lc_trie_lib", + "//source/common/stats:symbol_table_lib", "@envoy_api//envoy/config/filter/http/ip_tagging/v2:ip_tagging_cc", ], ) diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 1d82239c7dfd..d3b2549a87f7 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -10,6 +10,52 @@ namespace Extensions { namespace HttpFilters { namespace IpTagging { +IpTaggingFilterConfig::IpTaggingFilterConfig( + const envoy::config::filter::http::ip_tagging::v2::IPTagging& config, + const std::string& stat_prefix, Stats::Scope& scope, Runtime::Loader& runtime) + : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), + stat_name_set_(scope.symbolTable()), + stats_prefix_(stat_name_set_.add(stat_prefix + "ip_tagging")), + hit_(stat_name_set_.add("hit")), no_hit_(stat_name_set_.add("no_hit")), + total_(stat_name_set_.add("total")) { + + // Once loading IP tags from a file system is supported, the restriction on the size + // of the set should be removed and observability into what tags are loaded needs + // to be implemented. + // TODO(ccaraman): Remove size check once file system support is implemented. + // Work is tracked by issue https://github.com/envoyproxy/envoy/issues/2695. + if (config.ip_tags().empty()) { + throw EnvoyException("HTTP IP Tagging Filter requires ip_tags to be specified."); + } + + std::vector>> tag_data; + tag_data.reserve(config.ip_tags().size()); + for (const auto& ip_tag : config.ip_tags()) { + std::vector cidr_set; + cidr_set.reserve(ip_tag.ip_list().size()); + for (const envoy::api::v2::core::CidrRange& entry : ip_tag.ip_list()) { + + // Currently, CidrRange::create doesn't guarantee that the CidrRanges are valid. + Network::Address::CidrRange cidr_entry = Network::Address::CidrRange::create(entry); + if (cidr_entry.isValid()) { + cidr_set.emplace_back(std::move(cidr_entry)); + } else { + throw EnvoyException( + fmt::format("invalid ip/mask combo '{}/{}' (format is /<# mask bits>)", + entry.address_prefix(), entry.prefix_len().value())); + } + } + tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); + } + trie_ = std::make_unique>(tag_data); +} + +void IpTaggingFilterConfig::incCounter(Stats::StatName name, absl::string_view tag) { + Stats::SymbolTable::StoragePtr storage = + scope_.symbolTable().join({stats_prefix_, stat_name_set_.getStatName(tag), name}); + scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); +} + IpTaggingFilter::IpTaggingFilter(IpTaggingFilterConfigSharedPtr config) : config_(config) {} IpTaggingFilter::~IpTaggingFilter() = default; @@ -42,12 +88,12 @@ Http::FilterHeadersStatus IpTaggingFilter::decodeHeaders(Http::HeaderMap& header // If there are use cases with a large set of tags, a way to opt into these stats // should be exposed and other observability options like logging tags need to be implemented. for (const std::string& tag : tags) { - config_->scope().counter(fmt::format("{}{}.hit", config_->statsPrefix(), tag)).inc(); + config_->incHit(tag); } } else { - config_->scope().counter(fmt::format("{}no_hit", config_->statsPrefix())).inc(); + config_->incNoHit(); } - config_->scope().counter(fmt::format("{}total", config_->statsPrefix())).inc(); + config_->incTotal(); return Http::FilterHeadersStatus::Continue; } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index b79103aab9bd..6cf2b19b0e74 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -14,6 +14,7 @@ #include "common/network/cidr_range.h" #include "common/network/lc_trie.h" +#include "common/stats/symbol_table_impl.h" namespace Envoy { namespace Extensions { @@ -32,46 +33,16 @@ class IpTaggingFilterConfig { public: IpTaggingFilterConfig(const envoy::config::filter::http::ip_tagging::v2::IPTagging& config, const std::string& stat_prefix, Stats::Scope& scope, - Runtime::Loader& runtime) - : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), - stats_prefix_(stat_prefix + "ip_tagging.") { - - // Once loading IP tags from a file system is supported, the restriction on the size - // of the set should be removed and observability into what tags are loaded needs - // to be implemented. - // TODO(ccaraman): Remove size check once file system support is implemented. - // Work is tracked by issue https://github.com/envoyproxy/envoy/issues/2695. - if (config.ip_tags().empty()) { - throw EnvoyException("HTTP IP Tagging Filter requires ip_tags to be specified."); - } - - std::vector>> tag_data; - tag_data.reserve(config.ip_tags().size()); - for (const auto& ip_tag : config.ip_tags()) { - std::vector cidr_set; - cidr_set.reserve(ip_tag.ip_list().size()); - for (const envoy::api::v2::core::CidrRange& entry : ip_tag.ip_list()) { - - // Currently, CidrRange::create doesn't guarantee that the CidrRanges are valid. - Network::Address::CidrRange cidr_entry = Network::Address::CidrRange::create(entry); - if (cidr_entry.isValid()) { - cidr_set.emplace_back(std::move(cidr_entry)); - } else { - throw EnvoyException( - fmt::format("invalid ip/mask combo '{}/{}' (format is /<# mask bits>)", - entry.address_prefix(), entry.prefix_len().value())); - } - } - tag_data.emplace_back(ip_tag.ip_tag_name(), cidr_set); - } - trie_ = std::make_unique>(tag_data); - } + Runtime::Loader& runtime); Runtime::Loader& runtime() { return runtime_; } Stats::Scope& scope() { return scope_; } FilterRequestType requestType() const { return request_type_; } const Network::LcTrie::LcTrie& trie() const { return *trie_; } - const std::string& statsPrefix() const { return stats_prefix_; } + + void incHit(absl::string_view tag) { incCounter(hit_, tag); } + void incNoHit() { incCounter(no_hit_); } + void incTotal() { incCounter(total_); } private: static FilterRequestType requestTypeEnum( @@ -88,10 +59,16 @@ class IpTaggingFilterConfig { } } + void incCounter(Stats::StatName name1, absl::string_view tag = ""); + const FilterRequestType request_type_; Stats::Scope& scope_; Runtime::Loader& runtime_; - const std::string stats_prefix_; + Stats::StatNameSet stat_name_set_; + const Stats::StatName stats_prefix_; + const Stats::StatName hit_; + const Stats::StatName no_hit_; + const Stats::StatName total_; std::unique_ptr> trie_; }; diff --git a/source/extensions/filters/network/mongo_proxy/BUILD b/source/extensions/filters/network/mongo_proxy/BUILD index 36a94de85abc..f2be1d6a9774 100644 --- a/source/extensions/filters/network/mongo_proxy/BUILD +++ b/source/extensions/filters/network/mongo_proxy/BUILD @@ -58,6 +58,7 @@ envoy_cc_library( deps = [ ":codec_interface", ":codec_lib", + ":mongo_stats_lib", ":utility_lib", "//include/envoy/access_log:access_log_interface", "//include/envoy/common:time_interface", @@ -81,6 +82,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "mongo_stats_lib", + srcs = ["mongo_stats.cc"], + hdrs = ["mongo_stats.h"], + deps = [ + "//include/envoy/stats:stats_interface", + "//source/common/stats:symbol_table_lib", + ], +) + envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], diff --git a/source/extensions/filters/network/mongo_proxy/config.cc b/source/extensions/filters/network/mongo_proxy/config.cc index a8989947e75a..24539b6bd953 100644 --- a/source/extensions/filters/network/mongo_proxy/config.cc +++ b/source/extensions/filters/network/mongo_proxy/config.cc @@ -32,12 +32,13 @@ Network::FilterFactoryCb MongoProxyFilterConfigFactory::createFilterFactoryFromP fault_config = std::make_shared(proto_config.delay()); } + auto stats = std::make_shared(context.scope(), stat_prefix); const bool emit_dynamic_metadata = proto_config.emit_dynamic_metadata(); - return [stat_prefix, &context, access_log, fault_config, - emit_dynamic_metadata](Network::FilterManager& filter_manager) -> void { + return [stat_prefix, &context, access_log, fault_config, emit_dynamic_metadata, + stats](Network::FilterManager& filter_manager) -> void { filter_manager.addFilter(std::make_shared( stat_prefix, context.scope(), context.runtime(), access_log, fault_config, - context.drainDecision(), context.dispatcher().timeSource(), emit_dynamic_metadata)); + context.drainDecision(), context.dispatcher().timeSource(), emit_dynamic_metadata, stats)); }; } diff --git a/source/extensions/filters/network/mongo_proxy/mongo_stats.cc b/source/extensions/filters/network/mongo_proxy/mongo_stats.cc new file mode 100644 index 000000000000..11dd1877cefc --- /dev/null +++ b/source/extensions/filters/network/mongo_proxy/mongo_stats.cc @@ -0,0 +1,47 @@ +#include "extensions/filters/network/mongo_proxy/mongo_stats.h" + +#include +#include +#include + +#include "envoy/stats/scope.h" + +#include "common/stats/symbol_table_impl.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace MongoProxy { + +MongoStats::MongoStats(Stats::Scope& scope, const std::string& prefix) + : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), + callsite_(stat_name_set_.add("callsite")), cmd_(stat_name_set_.add("cmd")), + collection_(stat_name_set_.add("collection")), multi_get_(stat_name_set_.add("multi_get")), + reply_num_docs_(stat_name_set_.add("reply_num_docs")), + reply_size_(stat_name_set_.add("reply_size")), + reply_time_ms_(stat_name_set_.add("reply_time_ms")), time_ms_(stat_name_set_.add("time_ms")), + query_(stat_name_set_.add("query")), scatter_get_(stat_name_set_.add("scatter_get")), + total_(stat_name_set_.add("total")) {} + +Stats::SymbolTable::StoragePtr MongoStats::addPrefix(const std::vector& names) { + std::vector names_with_prefix; + names_with_prefix.reserve(1 + names.size()); + names_with_prefix.push_back(prefix_); + names_with_prefix.insert(names_with_prefix.end(), names.begin(), names.end()); + return scope_.symbolTable().join(names_with_prefix); +} + +void MongoStats::incCounter(const std::vector& names) { + const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(names); + scope_.counterFromStatName(Stats::StatName(stat_name_storage.get())).inc(); +} + +void MongoStats::recordHistogram(const std::vector& names, uint64_t sample) { + const Stats::SymbolTable::StoragePtr stat_name_storage = addPrefix(names); + scope_.histogramFromStatName(Stats::StatName(stat_name_storage.get())).recordValue(sample); +} + +} // namespace MongoProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/mongo_proxy/mongo_stats.h b/source/extensions/filters/network/mongo_proxy/mongo_stats.h new file mode 100644 index 000000000000..d27a8478824a --- /dev/null +++ b/source/extensions/filters/network/mongo_proxy/mongo_stats.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include + +#include "envoy/stats/scope.h" + +#include "common/stats/symbol_table_impl.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace MongoProxy { + +class MongoStats { +public: + MongoStats(Stats::Scope& scope, const std::string& prefix); + + void incCounter(const std::vector& names); + void recordHistogram(const std::vector& names, uint64_t sample); + + /** + * Finds or creates a StatName by string, taking a global lock if needed. + * + * TODO(jmarantz): Potential perf issue here with mutex contention for names + * that have not been remembered as builtins in the constructor. + */ + Stats::StatName getStatName(const std::string& str) { return stat_name_set_.getStatName(str); } + +private: + Stats::SymbolTable::StoragePtr addPrefix(const std::vector& names); + + Stats::Scope& scope_; + Stats::StatNameSet stat_name_set_; + +public: + const Stats::StatName prefix_; + const Stats::StatName callsite_; + const Stats::StatName cmd_; + const Stats::StatName collection_; + const Stats::StatName multi_get_; + const Stats::StatName reply_num_docs_; + const Stats::StatName reply_size_; + const Stats::StatName reply_time_ms_; + const Stats::StatName time_ms_; + const Stats::StatName query_; + const Stats::StatName scatter_get_; + const Stats::StatName total_; +}; +using MongoStatsSharedPtr = std::shared_ptr; + +} // namespace MongoProxy +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/mongo_proxy/proxy.cc b/source/extensions/filters/network/mongo_proxy/proxy.cc index be08a04fc638..bc35859113f8 100644 --- a/source/extensions/filters/network/mongo_proxy/proxy.cc +++ b/source/extensions/filters/network/mongo_proxy/proxy.cc @@ -58,11 +58,11 @@ ProxyFilter::ProxyFilter(const std::string& stat_prefix, Stats::Scope& scope, Runtime::Loader& runtime, AccessLogSharedPtr access_log, const Filters::Common::Fault::FaultDelayConfigSharedPtr& fault_config, const Network::DrainDecision& drain_decision, TimeSource& time_source, - bool emit_dynamic_metadata) - : stat_prefix_(stat_prefix), scope_(scope), stats_(generateStats(stat_prefix, scope)), - runtime_(runtime), drain_decision_(drain_decision), access_log_(access_log), - fault_config_(fault_config), time_source_(time_source), - emit_dynamic_metadata_(emit_dynamic_metadata) { + bool emit_dynamic_metadata, const MongoStatsSharedPtr& mongo_stats) + : stat_prefix_(stat_prefix), stats_(generateStats(stat_prefix, scope)), runtime_(runtime), + drain_decision_(drain_decision), access_log_(access_log), fault_config_(fault_config), + time_source_(time_source), emit_dynamic_metadata_(emit_dynamic_metadata), + mongo_stats_(mongo_stats) { if (!runtime_.snapshot().featureEnabled(MongoRuntimeConfig::get().ConnectionLoggingEnabled, 100)) { // If we are not logging at the connection level, just release the shared pointer so that we @@ -145,21 +145,23 @@ void ProxyFilter::decodeQuery(QueryMessagePtr&& message) { ActiveQueryPtr active_query(new ActiveQuery(*this, *message)); if (!active_query->query_info_.command().empty()) { // First field key is the operation. - scope_.counter(fmt::format("{}cmd.{}.total", stat_prefix_, active_query->query_info_.command())) - .inc(); + mongo_stats_->incCounter({mongo_stats_->cmd_, + mongo_stats_->getStatName(active_query->query_info_.command()), + mongo_stats_->total_}); } else { // Normal query, get stats on a per collection basis first. - std::string collection_stat_prefix = - fmt::format("{}collection.{}", stat_prefix_, active_query->query_info_.collection()); QueryMessageInfo::QueryType query_type = active_query->query_info_.type(); - chargeQueryStats(collection_stat_prefix, query_type); + Stats::StatNameVec names; + names.reserve(6); // 2 entries are added by chargeQueryStats(). + names.push_back(mongo_stats_->collection_); + names.push_back(mongo_stats_->getStatName(active_query->query_info_.collection())); + chargeQueryStats(names, query_type); // Callsite stats if we have it. if (!active_query->query_info_.callsite().empty()) { - std::string callsite_stat_prefix = - fmt::format("{}collection.{}.callsite.{}", stat_prefix_, - active_query->query_info_.collection(), active_query->query_info_.callsite()); - chargeQueryStats(callsite_stat_prefix, query_type); + names.push_back(mongo_stats_->callsite_); + names.push_back(mongo_stats_->getStatName(active_query->query_info_.callsite())); + chargeQueryStats(names, query_type); } // Global stats. @@ -176,14 +178,26 @@ void ProxyFilter::decodeQuery(QueryMessagePtr&& message) { active_query_list_.emplace_back(std::move(active_query)); } -void ProxyFilter::chargeQueryStats(const std::string& prefix, +void ProxyFilter::chargeQueryStats(Stats::StatNameVec& names, QueryMessageInfo::QueryType query_type) { - scope_.counter(fmt::format("{}.query.total", prefix)).inc(); + // names come in containing {"collection", collection}. Report stats for 1 or + // 2 variations on this array, and then return with the array in the same + // state it had on entry. Both of these variations by appending {"query", "total"}. + size_t orig_size = names.size(); + ASSERT(names.capacity() - orig_size >= 2); // Ensures the caller has reserved() enough memory. + names.push_back(mongo_stats_->query_); + names.push_back(mongo_stats_->total_); + mongo_stats_->incCounter(names); + + // And now replace "total" with either "scatter_get" or "multi_get" if depending on query_type. if (query_type == QueryMessageInfo::QueryType::ScatterGet) { - scope_.counter(fmt::format("{}.query.scatter_get", prefix)).inc(); + names.back() = mongo_stats_->scatter_get_; + mongo_stats_->incCounter(names); } else if (query_type == QueryMessageInfo::QueryType::MultiGet) { - scope_.counter(fmt::format("{}.query.multi_get", prefix)).inc(); + names.back() = mongo_stats_->multi_get_; + mongo_stats_->incCounter(names); } + names.resize(orig_size); } void ProxyFilter::decodeReply(ReplyMessagePtr&& message) { @@ -208,21 +222,25 @@ void ProxyFilter::decodeReply(ReplyMessagePtr&& message) { } if (!active_query.query_info_.command().empty()) { - std::string stat_prefix = - fmt::format("{}cmd.{}", stat_prefix_, active_query.query_info_.command()); - chargeReplyStats(active_query, stat_prefix, *message); + Stats::StatNameVec names{mongo_stats_->cmd_, + mongo_stats_->getStatName(active_query.query_info_.command())}; + chargeReplyStats(active_query, names, *message); } else { // Collection stats first. - std::string stat_prefix = - fmt::format("{}collection.{}.query", stat_prefix_, active_query.query_info_.collection()); - chargeReplyStats(active_query, stat_prefix, *message); + Stats::StatNameVec names{mongo_stats_->collection_, + mongo_stats_->getStatName(active_query.query_info_.collection()), + mongo_stats_->query_}; + chargeReplyStats(active_query, names, *message); // Callsite stats if we have it. if (!active_query.query_info_.callsite().empty()) { - std::string callsite_stat_prefix = - fmt::format("{}collection.{}.callsite.{}.query", stat_prefix_, - active_query.query_info_.collection(), active_query.query_info_.callsite()); - chargeReplyStats(active_query, callsite_stat_prefix, *message); + // Currently, names == {"collection", collection, "query"} and we are going + // to mutate the array to {"collection", collection, "callsite", callsite, "query"}. + ASSERT(names.size() == 3); + names.back() = mongo_stats_->callsite_; // Replaces "query". + names.push_back(mongo_stats_->getStatName(active_query.query_info_.callsite())); + names.push_back(mongo_stats_->query_); + chargeReplyStats(active_query, names, *message); } } @@ -270,20 +288,26 @@ void ProxyFilter::onDrainClose() { read_callbacks_->connection().close(Network::ConnectionCloseType::FlushWrite); } -void ProxyFilter::chargeReplyStats(ActiveQuery& active_query, const std::string& prefix, +void ProxyFilter::chargeReplyStats(ActiveQuery& active_query, Stats::StatNameVec& names, const ReplyMessage& message) { uint64_t reply_documents_byte_size = 0; for (const Bson::DocumentSharedPtr& document : message.documents()) { reply_documents_byte_size += document->byteSize(); } - scope_.histogram(fmt::format("{}.reply_num_docs", prefix)) - .recordValue(message.documents().size()); - scope_.histogram(fmt::format("{}.reply_size", prefix)).recordValue(reply_documents_byte_size); - scope_.histogram(fmt::format("{}.reply_time_ms", prefix)) - .recordValue(std::chrono::duration_cast( - time_source_.monotonicTime() - active_query.start_time_) - .count()); + // Write 3 different histograms; appending 3 different suffixes to the name + // that was passed in. Here we overwrite the passed-in names, but we restore + // names to its original state upon return. + const size_t orig_size = names.size(); + names.push_back(mongo_stats_->reply_num_docs_); + mongo_stats_->recordHistogram(names, message.documents().size()); + names[orig_size] = mongo_stats_->reply_size_; + mongo_stats_->recordHistogram(names, reply_documents_byte_size); + names[orig_size] = mongo_stats_->reply_time_ms_; + mongo_stats_->recordHistogram(names, std::chrono::duration_cast( + time_source_.monotonicTime() - active_query.start_time_) + .count()); + names.resize(orig_size); } void ProxyFilter::doDecode(Buffer::Instance& buffer) { diff --git a/source/extensions/filters/network/mongo_proxy/proxy.h b/source/extensions/filters/network/mongo_proxy/proxy.h index f81ef86017ef..da85af19f281 100644 --- a/source/extensions/filters/network/mongo_proxy/proxy.h +++ b/source/extensions/filters/network/mongo_proxy/proxy.h @@ -25,6 +25,7 @@ #include "extensions/filters/common/fault/fault_config.h" #include "extensions/filters/network/mongo_proxy/codec.h" +#include "extensions/filters/network/mongo_proxy/mongo_stats.h" #include "extensions/filters/network/mongo_proxy/utility.h" namespace Envoy { @@ -110,7 +111,7 @@ class ProxyFilter : public Network::Filter, AccessLogSharedPtr access_log, const Filters::Common::Fault::FaultDelayConfigSharedPtr& fault_config, const Network::DrainDecision& drain_decision, TimeSource& time_system, - bool emit_dynamic_metadata); + bool emit_dynamic_metadata, const MongoStatsSharedPtr& stats); ~ProxyFilter() override; virtual DecoderPtr createDecoder(DecoderCallbacks& callbacks) PURE; @@ -164,9 +165,17 @@ class ProxyFilter : public Network::Filter, POOL_HISTOGRAM_PREFIX(scope, prefix))}; } - void chargeQueryStats(const std::string& prefix, QueryMessageInfo::QueryType query_type); - void chargeReplyStats(ActiveQuery& active_query, const std::string& prefix, + // Increment counters related to queries. 'names' is passed by non-const + // reference so the implementation can mutate it without copying, though it + // always restores it to its prior state prior to return. + void chargeQueryStats(Stats::StatNameVec& names, QueryMessageInfo::QueryType query_type); + + // Add samples to histograms related to replies. 'names' is passed by + // non-const reference so the implementation can mutate it without copying, + // though it always restores it to its prior state prior to return. + void chargeReplyStats(ActiveQuery& active_query, Stats::StatNameVec& names, const ReplyMessage& message); + void doDecode(Buffer::Instance& buffer); void logMessage(Message& message, bool full); void onDrainClose(); @@ -176,7 +185,6 @@ class ProxyFilter : public Network::Filter, std::unique_ptr decoder_; std::string stat_prefix_; - Stats::Scope& scope_; MongoProxyStats stats_; Runtime::Loader& runtime_; const Network::DrainDecision& drain_decision_; @@ -191,6 +199,7 @@ class ProxyFilter : public Network::Filter, Event::TimerPtr drain_close_timer_; TimeSource& time_source_; const bool emit_dynamic_metadata_; + MongoStatsSharedPtr mongo_stats_; }; class ProdProxyFilter : public ProxyFilter { diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 64d7238bba07..4dc3a5dae872 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -13,6 +13,7 @@ #include "common/common/fmt.h" #include "common/common/utility.h" #include "common/config/utility.h" +#include "common/stats/symbol_table_impl.h" namespace Envoy { namespace Extensions { @@ -99,8 +100,9 @@ TcpStatsdSink::TcpStatsdSink(const LocalInfo::LocalInfo& local_info, Upstream::ClusterManager& cluster_manager, Stats::Scope& scope, const std::string& prefix) : prefix_(prefix.empty() ? Statsd::getDefaultPrefix() : prefix), tls_(tls.allocateSlot()), - cluster_manager_(cluster_manager), cx_overflow_stat_(scope.counter("statsd.cx_overflow")) { - + cluster_manager_(cluster_manager), + cx_overflow_stat_(scope.counterFromStatName( + Stats::StatNameManagedStorage("statsd.cx_overflow", scope.symbolTable()).statName())) { Config::Utility::checkClusterAndLocalInfo("tcp statsd", cluster_name, cluster_manager, local_info); cluster_info_ = cluster_manager.get(cluster_name)->info(); diff --git a/source/extensions/transport_sockets/tls/BUILD b/source/extensions/transport_sockets/tls/BUILD index cb8bb9ec02ea..3345a9d87439 100644 --- a/source/extensions/transport_sockets/tls/BUILD +++ b/source/extensions/transport_sockets/tls/BUILD @@ -101,6 +101,7 @@ envoy_cc_library( "//source/common/common:utility_lib", "//source/common/network:address_lib", "//source/common/protobuf:utility_lib", + "//source/common/stats:symbol_table_lib", "//source/extensions/transport_sockets/tls/private_key:private_key_manager_lib", "@envoy_api//envoy/admin/v2alpha:certs_cc", ], diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index f4795c0db2b1..85928172612e 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -51,7 +51,11 @@ bool cbsContainsU16(CBS& cbs, uint16_t n) { ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& config, TimeSource& time_source) : scope_(scope), stats_(generateStats(scope)), time_source_(time_source), - tls_max_version_(config.maxProtocolVersion()) { + tls_max_version_(config.maxProtocolVersion()), stat_name_set_(scope.symbolTable()), + ssl_ciphers_(stat_name_set_.add("ssl.ciphers")), + ssl_versions_(stat_name_set_.add("ssl.versions")), + ssl_curves_(stat_name_set_.add("ssl.curves")), + ssl_sigalgs_(stat_name_set_.add("ssl.sigalgs")) { const auto tls_certificates = config.tlsCertificates(); tls_contexts_.resize(std::max(static_cast(1), tls_certificates.size())); @@ -369,6 +373,35 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c } parsed_alpn_protocols_ = parseAlpnProtocols(config.alpnProtocols()); + + // To enumerate the required builtin ciphers, curves, algorithms, and + // versions, uncomment '#define LOG_BUILTIN_STAT_NAMES' below, and run + // bazel test //test/extensions/transport_sockets/tls/... --test_output=streamed + // | grep " Builtin ssl." | sort | uniq + // #define LOG_BUILTIN_STAT_NAMES + // + // TODO(#8035): improve tooling to find any other built-ins needed to avoid + // contention. + + // Ciphers + stat_name_set_.rememberBuiltin("AEAD-AES128-GCM-SHA256"); + stat_name_set_.rememberBuiltin("ECDHE-ECDSA-AES128-GCM-SHA256"); + stat_name_set_.rememberBuiltin("ECDHE-RSA-AES128-GCM-SHA256"); + stat_name_set_.rememberBuiltin("ECDHE-RSA-AES128-SHA"); + stat_name_set_.rememberBuiltin("ECDHE-RSA-CHACHA20-POLY1305"); + + // Curves + stat_name_set_.rememberBuiltin("X25519"); + + // Algorithms + stat_name_set_.rememberBuiltin("ecdsa_secp256r1_sha256"); + stat_name_set_.rememberBuiltin("rsa_pss_rsae_sha256"); + + // Versions + stat_name_set_.rememberBuiltin("TLSv1"); + stat_name_set_.rememberBuiltin("TLSv1.1"); + stat_name_set_.rememberBuiltin("TLSv1.2"); + stat_name_set_.rememberBuiltin("TLSv1.3"); } int ServerContextImpl::alpnSelectCallback(const unsigned char** out, unsigned char* outlen, @@ -477,6 +510,18 @@ int ContextImpl::verifyCertificate(X509* cert, const std::vector& v return 1; } +void ContextImpl::incCounter(const Stats::StatName name, absl::string_view value) const { + Stats::SymbolTable& symbol_table = scope_.symbolTable(); + Stats::SymbolTable::StoragePtr storage = + symbol_table.join({name, stat_name_set_.getStatName(value)}); + scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); + +#ifdef LOG_BUILTIN_STAT_NAMES + std::cerr << absl::StrCat("Builtin ", symbol_table.toString(name), ": ", value, "\n") + << std::flush; +#endif +} + void ContextImpl::logHandshake(SSL* ssl) const { stats_.handshake_.inc(); @@ -484,22 +529,19 @@ void ContextImpl::logHandshake(SSL* ssl) const { stats_.session_reused_.inc(); } - const char* cipher = SSL_get_cipher_name(ssl); - scope_.counter(fmt::format("ssl.ciphers.{}", std::string{cipher})).inc(); - - const char* version = SSL_get_version(ssl); - scope_.counter(fmt::format("ssl.versions.{}", std::string{version})).inc(); + incCounter(ssl_ciphers_, SSL_get_cipher_name(ssl)); + incCounter(ssl_versions_, SSL_get_version(ssl)); uint16_t curve_id = SSL_get_curve_id(ssl); if (curve_id) { - const char* curve = SSL_get_curve_name(curve_id); - scope_.counter(fmt::format("ssl.curves.{}", std::string{curve})).inc(); + // Note: in the unit tests, this curve name is always literal "X25519" + incCounter(ssl_curves_, SSL_get_curve_name(curve_id)); } uint16_t sigalg_id = SSL_get_peer_signature_algorithm(ssl); if (sigalg_id) { const char* sigalg = SSL_get_signature_algorithm_name(sigalg_id, 1 /* include curve */); - scope_.counter(fmt::format("ssl.sigalgs.{}", std::string{sigalg})).inc(); + incCounter(ssl_sigalgs_, sigalg); } bssl::UniquePtr cert(SSL_get_peer_certificate(ssl)); diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index ccb984a63efa..ce5007fd5a34 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -12,6 +12,8 @@ #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" +#include "common/stats/symbol_table_impl.h" + #include "extensions/transport_sockets/tls/context_manager_impl.h" #include "absl/synchronization/mutex.h" @@ -126,6 +128,7 @@ class ContextImpl : public virtual Envoy::Ssl::Context { static SslStats generateStats(Stats::Scope& scope); std::string getCaFileName() const { return ca_file_path_; }; + void incCounter(const Stats::StatName name, absl::string_view value) const; Envoy::Ssl::CertificateDetailsPtr certificateDetails(X509* cert, const std::string& path) const; @@ -167,6 +170,11 @@ class ContextImpl : public virtual Envoy::Ssl::Context { std::string cert_chain_file_path_; TimeSource& time_source_; const unsigned tls_max_version_; + mutable Stats::StatNameSet stat_name_set_; + const Stats::StatName ssl_ciphers_; + const Stats::StatName ssl_versions_; + const Stats::StatName ssl_curves_; + const Stats::StatName ssl_sigalgs_; }; using ContextImplSharedPtr = std::shared_ptr; diff --git a/source/server/BUILD b/source/server/BUILD index 5750846d1dca..169f7d3a3cc3 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -107,6 +107,7 @@ envoy_cc_library( "//source/common/common:minimal_logger_lib", "//source/common/common:thread_lib", "//source/common/event:libevent_lib", + "//source/common/stats:symbol_table_lib", ], ) @@ -228,6 +229,7 @@ envoy_cc_library( "//include/envoy/thread_local:thread_local_interface", "//source/common/common:logger_lib", "//source/common/config:utility_lib", + "//source/common/stats:symbol_table_lib", "//source/server:resource_monitor_config_lib", "@envoy_api//envoy/config/overload/v2alpha:overload_cc", ], diff --git a/source/server/guarddog_impl.cc b/source/server/guarddog_impl.cc index a9e3e583e318..5b0b80163478 100644 --- a/source/server/guarddog_impl.cc +++ b/source/server/guarddog_impl.cc @@ -8,6 +8,7 @@ #include "common/common/assert.h" #include "common/common/fmt.h" #include "common/common/lock_guard.h" +#include "common/stats/symbol_table_impl.h" #include "server/watchdog_impl.h" @@ -30,8 +31,12 @@ GuardDogImpl::GuardDogImpl(Stats::Scope& stats_scope, const Server::Configuratio multikillEnabled() ? multi_kill_timeout_ : min_of_nonfatal, min_of_nonfatal}); }()), - watchdog_miss_counter_(stats_scope.counter("server.watchdog_miss")), - watchdog_megamiss_counter_(stats_scope.counter("server.watchdog_mega_miss")), + watchdog_miss_counter_(stats_scope.counterFromStatName( + Stats::StatNameManagedStorage("server.watchdog_miss", stats_scope.symbolTable()) + .statName())), + watchdog_megamiss_counter_(stats_scope.counterFromStatName( + Stats::StatNameManagedStorage("server.watchdog_mega_miss", stats_scope.symbolTable()) + .statName())), dispatcher_(api.allocateDispatcher()), loop_timer_(dispatcher_->createTimer([this]() { step(); })), run_thread_(true) { start(api); diff --git a/source/server/overload_manager_impl.cc b/source/server/overload_manager_impl.cc index 7795802b6c76..484be6f075a9 100644 --- a/source/server/overload_manager_impl.cc +++ b/source/server/overload_manager_impl.cc @@ -5,6 +5,7 @@ #include "common/common/fmt.h" #include "common/config/utility.h" #include "common/protobuf/utility.h" +#include "common/stats/symbol_table_impl.h" #include "server/resource_monitor_config_impl.h" @@ -33,16 +34,25 @@ class ThresholdTriggerImpl : public OverloadAction::Trigger { absl::optional value_; }; -std::string StatsName(const std::string& a, const std::string& b) { - return absl::StrCat("overload.", a, ".", b); +Stats::Counter& makeCounter(Stats::Scope& scope, absl::string_view a, absl::string_view b) { + Stats::StatNameManagedStorage stat_name(absl::StrCat("overload.", a, ".", b), + scope.symbolTable()); + return scope.counterFromStatName(stat_name.statName()); +} + +Stats::Gauge& makeGauge(Stats::Scope& scope, absl::string_view a, absl::string_view b, + Stats::Gauge::ImportMode import_mode) { + Stats::StatNameManagedStorage stat_name(absl::StrCat("overload.", a, ".", b), + scope.symbolTable()); + return scope.gaugeFromStatName(stat_name.statName(), import_mode); } } // namespace OverloadAction::OverloadAction(const envoy::config::overload::v2alpha::OverloadAction& config, Stats::Scope& stats_scope) - : active_gauge_(stats_scope.gauge(StatsName(config.name(), "active"), - Stats::Gauge::ImportMode::Accumulate)) { + : active_gauge_( + makeGauge(stats_scope, config.name(), "active", Stats::Gauge::ImportMode::Accumulate)) { for (const auto& trigger_config : config.triggers()) { TriggerPtr trigger; @@ -213,9 +223,9 @@ OverloadManagerImpl::Resource::Resource(const std::string& name, ResourceMonitor OverloadManagerImpl& manager, Stats::Scope& stats_scope) : name_(name), monitor_(std::move(monitor)), manager_(manager), pending_update_(false), pressure_gauge_( - stats_scope.gauge(StatsName(name, "pressure"), Stats::Gauge::ImportMode::NeverImport)), - failed_updates_counter_(stats_scope.counter(StatsName(name, "failed_updates"))), - skipped_updates_counter_(stats_scope.counter(StatsName(name, "skipped_updates"))) {} + makeGauge(stats_scope, name, "pressure", Stats::Gauge::ImportMode::NeverImport)), + failed_updates_counter_(makeCounter(stats_scope, name, "failed_updates")), + skipped_updates_counter_(makeCounter(stats_scope, name, "skipped_updates")) {} void OverloadManagerImpl::Resource::update() { if (!pending_update_) { diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index d4f8ad2eb90e..f4e0bc1fcb58 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -151,6 +151,7 @@ class FaultFilterTest : public testing::Test { void TestPerFilterConfigFault(const Router::RouteSpecificFilterConfig* route_fault, const Router::RouteSpecificFilterConfig* vhost_fault); + Stats::IsolatedStoreImpl stats_; FaultFilterConfigSharedPtr config_; std::unique_ptr filter_; NiceMock decoder_filter_callbacks_; @@ -158,7 +159,6 @@ class FaultFilterTest : public testing::Test { Http::TestHeaderMapImpl request_headers_; Http::TestHeaderMapImpl response_headers_; Buffer::OwnedImpl data_; - Stats::IsolatedStoreImpl stats_; NiceMock runtime_; Event::MockTimer* timer_{}; Event::SimulatedTimeSystem time_system_; diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 50679f090c42..774f4e54c216 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -52,11 +52,11 @@ request_type: internal ~IpTaggingFilterTest() override { filter_->onDestroy(); } + NiceMock stats_; IpTaggingFilterConfigSharedPtr config_; std::unique_ptr filter_; NiceMock filter_callbacks_; Buffer::OwnedImpl data_; - NiceMock stats_; NiceMock runtime_; }; diff --git a/test/extensions/filters/network/mongo_proxy/proxy_test.cc b/test/extensions/filters/network/mongo_proxy/proxy_test.cc index 01d0c8082332..a70c1a6f279c 100644 --- a/test/extensions/filters/network/mongo_proxy/proxy_test.cc +++ b/test/extensions/filters/network/mongo_proxy/proxy_test.cc @@ -8,6 +8,7 @@ #include "extensions/filters/network/mongo_proxy/bson_impl.h" #include "extensions/filters/network/mongo_proxy/codec_impl.h" +#include "extensions/filters/network/mongo_proxy/mongo_stats.h" #include "extensions/filters/network/mongo_proxy/proxy.h" #include "extensions/filters/network/well_known_names.h" @@ -62,7 +63,7 @@ class TestProxyFilter : public ProxyFilter { class MongoProxyFilterTest : public testing::Test { public: - MongoProxyFilterTest() { setup(); } + MongoProxyFilterTest() : mongo_stats_(std::make_shared(store_, "test")) { setup(); } void setup() { ON_CALL(runtime_.snapshot_, featureEnabled("mongo.proxy_enabled", 100)) @@ -82,9 +83,9 @@ class MongoProxyFilterTest : public testing::Test { } void initializeFilter(bool emit_dynamic_metadata = false) { - filter_ = std::make_unique("test.", store_, runtime_, access_log_, - fault_config_, drain_decision_, - dispatcher_.timeSource(), emit_dynamic_metadata); + filter_ = std::make_unique( + "test.", store_, runtime_, access_log_, fault_config_, drain_decision_, + dispatcher_.timeSource(), emit_dynamic_metadata, mongo_stats_); filter_->initializeReadFilterCallbacks(read_filter_callbacks_); filter_->onNewConnection(); @@ -114,6 +115,7 @@ class MongoProxyFilterTest : public testing::Test { Buffer::OwnedImpl fake_data_; NiceMock store_; + MongoStatsSharedPtr mongo_stats_; NiceMock runtime_; NiceMock dispatcher_; std::shared_ptr file_{ diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 8b6dbb4b0263..4fc6090dde14 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -699,7 +699,8 @@ const std::string testUtilV2(const TestUtilOptionsV2& options) { dispatcher->run(Event::Dispatcher::RunType::Block); if (!options.expectedServerStats().empty()) { - EXPECT_EQ(1UL, server_stats_store.counter(options.expectedServerStats()).value()); + EXPECT_EQ(1UL, server_stats_store.counter(options.expectedServerStats()).value()) + << options.expectedServerStats(); } if (!options.expectedClientStats().empty()) { diff --git a/tools/check_format.py b/tools/check_format.py index 2adcc22e8c35..26b3e621848f 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -42,20 +42,6 @@ "./test/test_common/utility.cc", "./test/test_common/utility.h", "./test/integration/integration.h") -# Files matching these directories can use stats by string for now. These should -# be eliminated but for now we don't want to grow this work. The goal for this -# whitelist is to eliminate it by making code transformations similar to -# https://github.com/envoyproxy/envoy/pull/7573 and others. -# -# TODO(#4196): Eliminate this list completely and then merge #4980. -STAT_FROM_STRING_WHITELIST = ("./source/extensions/filters/http/fault/fault_filter.cc", - "./source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc", - "./source/extensions/filters/network/mongo_proxy/proxy.cc", - "./source/extensions/stat_sinks/common/statsd/statsd.cc", - "./source/extensions/transport_sockets/tls/context_impl.cc", - "./source/server/guarddog_impl.cc", - "./source/server/overload_manager_impl.cc") - # Files in these paths can use MessageLite::SerializeAsString SERIALIZE_AS_STRING_WHITELIST = ("./test/common/protobuf/utility_test.cc", "./test/common/grpc/codec_test.cc") @@ -339,10 +325,6 @@ def whitelistedForJsonStringToMessage(file_path): return file_path in JSON_STRING_TO_MESSAGE_WHITELIST -def whitelistedForStatFromString(file_path): - return file_path in STAT_FROM_STRING_WHITELIST - - def whitelistedForStdRegex(file_path): return file_path.startswith("./test") or file_path in STD_REGEX_WHITELIST @@ -591,7 +573,6 @@ def checkSourceLine(line, file_path, reportError): reportError("Don't use Protobuf::util::JsonStringToMessage, use TestUtility::loadFromJson.") if isInSubdir(file_path, 'source') and file_path.endswith('.cc') and \ - not whitelistedForStatFromString(file_path) and \ ('.counter(' in line or '.gauge(' in line or '.histogram(' in line): reportError("Don't lookup stats by name at runtime; use StatName saved during construction") diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index bb101fa95946..e0ccece9cfc5 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -777,6 +777,7 @@ uint un unacked unary +uncomment unconfigurable unconfigured uncontended @@ -792,6 +793,7 @@ unescaping unindexed uninsantiated uninstantiated +uniq unix unlink unlinked From 7f060b63fee64d4fb9d7f19661f1058fcebb3357 Mon Sep 17 00:00:00 2001 From: Yuchen Dai Date: Wed, 28 Aug 2019 19:10:07 -0700 Subject: [PATCH 476/542] tls_inspector: inline the recv in the onAccept (#7951) Description: As discussed in #7864 this PR is the attempt to peek the socket at the invoke of onAccept. Usually client_hello packet should be in the buffer when tls_inspector is peeking, we could save a poll cycle for this connection. Once we agree on the solution I can apply to http_inspector as well. The expecting latency improvement especially when poll cycle is large. Benchmark: Env: hardware Intel(R) Xeon(R) CPU @ 2.20GHz envoy: concurrency = 1, tls_inspector as listener filter. One tls filter chain, and one plain text filter chain. load background: a [sniper](https://github.com/lubia/sniper) client with concurrency = 5 hitting the server with tls handshake, aiming to hit using the tls_filter chain. The qps is about 170/s Another load client hitting the plain text filter chain but would go through tls_inspector with concurrency = 1 This PR: TransactionTime: 10.3 - 11.0 ms(mean) Master TransactionTime: 12.3 - 12.8 ms(mean) Risk Level: Med (ActiveSocket code is affected to adopt the side effect of onAccept) Testing: Docs Changes: Release Notes: Fixes #7864 Signed-off-by: Yuchen Dai --- .../common/network/io_socket_handle_impl.cc | 6 +- .../listener/tls_inspector/tls_inspector.cc | 78 ++++++++++++------- .../listener/tls_inspector/tls_inspector.h | 12 ++- source/server/connection_handler_impl.cc | 17 +++- ...dr_family_aware_socket_option_impl_test.cc | 1 + test/config_test/config_test.cc | 1 + .../proxy_protocol/proxy_protocol_test.cc | 25 ++++-- .../tls_inspector/tls_inspector_test.cc | 37 +++++++++ test/mocks/api/mocks.cc | 8 +- test/server/BUILD | 1 + test/server/connection_handler_test.cc | 63 ++++++++++++++- test/server/listener_manager_impl_test.cc | 24 +++--- 12 files changed, 217 insertions(+), 56 deletions(-) diff --git a/source/common/network/io_socket_handle_impl.cc b/source/common/network/io_socket_handle_impl.cc index cce957655380..b44ec48fb3fe 100644 --- a/source/common/network/io_socket_handle_impl.cc +++ b/source/common/network/io_socket_handle_impl.cc @@ -26,9 +26,11 @@ IoSocketHandleImpl::~IoSocketHandleImpl() { Api::IoCallUint64Result IoSocketHandleImpl::close() { ASSERT(fd_ != -1); - const int rc = ::close(fd_); + auto& os_syscalls = Api::OsSysCallsSingleton::get(); + const auto& result = os_syscalls.close(fd_); fd_ = -1; - return Api::IoCallUint64Result(rc, Api::IoErrorPtr(nullptr, IoSocketError::deleteIoError)); + return Api::IoCallUint64Result(result.rc_, + Api::IoErrorPtr(nullptr, IoSocketError::deleteIoError)); } bool IoSocketHandleImpl::isOpen() const { return fd_ != -1; } diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc index f5d2a8e4cef1..13b52cdc6ba0 100644 --- a/source/extensions/filters/listener/tls_inspector/tls_inspector.cc +++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.cc @@ -72,23 +72,47 @@ Network::FilterStatus Filter::onAccept(Network::ListenerFilterCallbacks& cb) { ENVOY_LOG(debug, "tls inspector: new connection accepted"); Network::ConnectionSocket& socket = cb.socket(); ASSERT(file_event_ == nullptr); - - file_event_ = cb.dispatcher().createFileEvent( - socket.ioHandle().fd(), - [this](uint32_t events) { - if (events & Event::FileReadyType::Closed) { - config_->stats().connection_closed_.inc(); - done(false); - return; - } - - ASSERT(events == Event::FileReadyType::Read); - onRead(); - }, - Event::FileTriggerType::Edge, Event::FileReadyType::Read | Event::FileReadyType::Closed); - cb_ = &cb; - return Network::FilterStatus::StopIteration; + + ParseState parse_state = onRead(); + switch (parse_state) { + case ParseState::Error: + // As per discussion in https://github.com/envoyproxy/envoy/issues/7864 + // we don't add new enum in FilterStatus so we have to signal the caller + // the new condition. + cb.socket().close(); + return Network::FilterStatus::StopIteration; + case ParseState::Done: + return Network::FilterStatus::Continue; + case ParseState::Continue: + // do nothing but create the event + file_event_ = cb.dispatcher().createFileEvent( + socket.ioHandle().fd(), + [this](uint32_t events) { + if (events & Event::FileReadyType::Closed) { + config_->stats().connection_closed_.inc(); + done(false); + return; + } + + ASSERT(events == Event::FileReadyType::Read); + ParseState parse_state = onRead(); + switch (parse_state) { + case ParseState::Error: + done(false); + break; + case ParseState::Done: + done(true); + break; + case ParseState::Continue: + // do nothing but wait for the next event + break; + } + }, + Event::FileTriggerType::Edge, Event::FileReadyType::Read | Event::FileReadyType::Closed); + return Network::FilterStatus::StopIteration; + } + NOT_REACHED_GCOVR_EXCL_LINE } void Filter::onALPN(const unsigned char* data, unsigned int len) { @@ -122,7 +146,7 @@ void Filter::onServername(absl::string_view name) { clienthello_success_ = true; } -void Filter::onRead() { +ParseState Filter::onRead() { // This receive code is somewhat complicated, because it must be done as a MSG_PEEK because // there is no way for a listener-filter to pass payload data to the ConnectionImpl and filters // that get created later. @@ -141,11 +165,10 @@ void Filter::onRead() { ENVOY_LOG(trace, "tls inspector: recv: {}", result.rc_); if (result.rc_ == -1 && result.errno_ == EAGAIN) { - return; + return ParseState::Continue; } else if (result.rc_ < 0) { config_->stats().read_error_.inc(); - done(false); - return; + return ParseState::Error; } // Because we're doing a MSG_PEEK, data we've seen before gets returned every time, so @@ -154,8 +177,9 @@ void Filter::onRead() { const uint8_t* data = buf_ + read_; const size_t len = result.rc_ - read_; read_ = result.rc_; - parseClientHello(data, len); + return parseClientHello(data, len); } + return ParseState::Continue; } void Filter::done(bool success) { @@ -164,7 +188,7 @@ void Filter::done(bool success) { cb_->continueFilterChain(success); } -void Filter::parseClientHello(const void* data, size_t len) { +ParseState Filter::parseClientHello(const void* data, size_t len) { // Ownership is passed to ssl_ in SSL_set_bio() bssl::UniquePtr bio(BIO_new_mem_buf(data, len)); @@ -185,9 +209,9 @@ void Filter::parseClientHello(const void* data, size_t len) { // We've hit the specified size limit. This is an unreasonably large ClientHello; // indicate failure. config_->stats().client_hello_too_large_.inc(); - done(false); + return ParseState::Error; } - break; + return ParseState::Continue; case SSL_ERROR_SSL: if (clienthello_success_) { config_->stats().tls_found_.inc(); @@ -200,11 +224,9 @@ void Filter::parseClientHello(const void* data, size_t len) { } else { config_->stats().tls_not_found_.inc(); } - done(true); - break; + return ParseState::Done; default: - done(false); - break; + return ParseState::Error; } } diff --git a/source/extensions/filters/listener/tls_inspector/tls_inspector.h b/source/extensions/filters/listener/tls_inspector/tls_inspector.h index cf75fe87e954..ee353ce9ef08 100644 --- a/source/extensions/filters/listener/tls_inspector/tls_inspector.h +++ b/source/extensions/filters/listener/tls_inspector/tls_inspector.h @@ -36,6 +36,14 @@ struct TlsInspectorStats { ALL_TLS_INSPECTOR_STATS(GENERATE_COUNTER_STRUCT) }; +enum class ParseState { + // Parse result is out. It could be tls or not. + Done, + // Parser expects more data. + Continue, + // Parser reports unrecoverable error. + Error +}; /** * Global configuration for TLS inspector. */ @@ -68,8 +76,8 @@ class Filter : public Network::ListenerFilter, Logger::Loggable std::string { return api_->fileSystem().fileReadToEnd(file); })); + ON_CALL(os_sys_calls_, close(_)).WillByDefault(Return(Api::SysCallIntResult{0, 0})); // Here we setup runtime to mimic the actual deprecated feature list used in the // production code. Note that this test is actually more strict than production because diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index cdc225696c3d..1f4d8a750c85 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -287,7 +287,10 @@ TEST_P(ProxyProtocolTest, errorRecv_2) { const ssize_t rc = ::readv(fd, iov, iovcnt); return Api::SysCallSizeResult{rc, errno}; })); - + EXPECT_CALL(os_sys_calls, close(_)).Times(AnyNumber()).WillRepeatedly(Invoke([](int fd) { + const int rc = ::close(fd); + return Api::SysCallIntResult{rc, errno}; + })); connect(false); write(buffer, sizeof(buffer)); @@ -316,7 +319,10 @@ TEST_P(ProxyProtocolTest, errorFIONREAD_1) { const ssize_t rc = ::readv(fd, iov, iovcnt); return Api::SysCallSizeResult{rc, errno}; })); - + EXPECT_CALL(os_sys_calls, close(_)).Times(AnyNumber()).WillRepeatedly(Invoke([](int fd) { + const int rc = ::close(fd); + return Api::SysCallIntResult{rc, errno}; + })); connect(false); write(buffer, sizeof(buffer)); @@ -527,7 +533,10 @@ TEST_P(ProxyProtocolTest, v2ParseExtensionsIoctlError) { const ssize_t rc = ::readv(fd, iov, iovcnt); return Api::SysCallSizeResult{rc, errno}; })); - + EXPECT_CALL(os_sys_calls, close(_)).Times(AnyNumber()).WillRepeatedly(Invoke([](int fd) { + const int rc = ::close(fd); + return Api::SysCallIntResult{rc, errno}; + })); connect(false); write(buffer, sizeof(buffer)); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); @@ -656,7 +665,10 @@ TEST_P(ProxyProtocolTest, v2Fragmented3Error) { const ssize_t rc = ::readv(fd, iov, iovcnt); return Api::SysCallSizeResult{rc, errno}; })); - + EXPECT_CALL(os_sys_calls, close(_)).Times(AnyNumber()).WillRepeatedly(Invoke([](int fd) { + const int rc = ::close(fd); + return Api::SysCallIntResult{rc, errno}; + })); connect(false); write(buffer, 17); @@ -702,7 +714,10 @@ TEST_P(ProxyProtocolTest, v2Fragmented4Error) { const ssize_t rc = ::readv(fd, iov, iovcnt); return Api::SysCallSizeResult{rc, errno}; })); - + EXPECT_CALL(os_sys_calls, close(_)).Times(AnyNumber()).WillRepeatedly(Invoke([](int fd) { + const int rc = ::close(fd); + return Api::SysCallIntResult{rc, errno}; + })); connect(false); write(buffer, 10); dispatcher_->run(Event::Dispatcher::RunType::NonBlock); diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc index 63cff699cffe..d9793c98a8e6 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc @@ -43,6 +43,13 @@ class TlsInspectorTest : public testing::Test { EXPECT_CALL(cb_, dispatcher()).WillRepeatedly(ReturnRef(dispatcher_)); EXPECT_CALL(socket_, ioHandle()).WillRepeatedly(ReturnRef(*io_handle_)); + // Prepare the first recv attempt during + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce( + Invoke([](int fd, void* buffer, size_t length, int flag) -> Api::SysCallSizeResult { + ENVOY_LOG_MISC(error, "In mock syscall recv {} {} {} {}", fd, buffer, length, flag); + return Api::SysCallSizeResult{static_cast(0), 0}; + })); EXPECT_CALL(dispatcher_, createFileEvent_(_, _, Event::FileTriggerType::Edge, Event::FileReadyType::Read | Event::FileReadyType::Closed)) @@ -231,6 +238,36 @@ TEST_F(TlsInspectorTest, NotSsl) { EXPECT_EQ(1, cfg_->stats().tls_not_found_.value()); } +TEST_F(TlsInspectorTest, InlineReadSucceed) { + filter_ = std::make_unique(cfg_); + + EXPECT_CALL(cb_, socket()).WillRepeatedly(ReturnRef(socket_)); + EXPECT_CALL(cb_, dispatcher()).WillRepeatedly(ReturnRef(dispatcher_)); + EXPECT_CALL(socket_, ioHandle()).WillRepeatedly(ReturnRef(*io_handle_)); + const std::vector alpn_protos = {absl::string_view("h2")}; + const std::string servername("example.com"); + std::vector client_hello = Tls::Test::generateClientHello(servername, "\x02h2"); + + EXPECT_CALL(os_sys_calls_, recv(42, _, _, MSG_PEEK)) + .WillOnce(Invoke( + [&client_hello](int fd, void* buffer, size_t length, int flag) -> Api::SysCallSizeResult { + ENVOY_LOG_MISC(trace, "In mock syscall recv {} {} {} {}", fd, buffer, length, flag); + ASSERT(length >= client_hello.size()); + memcpy(buffer, client_hello.data(), client_hello.size()); + return Api::SysCallSizeResult{ssize_t(client_hello.size()), 0}; + })); + + // No event is created if the inline recv parse the hello. + EXPECT_CALL(dispatcher_, + createFileEvent_(_, _, Event::FileTriggerType::Edge, + Event::FileReadyType::Read | Event::FileReadyType::Closed)) + .Times(0); + + EXPECT_CALL(socket_, setRequestedServerName(Eq(servername))); + EXPECT_CALL(socket_, setRequestedApplicationProtocols(alpn_protos)); + EXPECT_CALL(socket_, setDetectedTransportProtocol(absl::string_view("tls"))); + EXPECT_EQ(Network::FilterStatus::Continue, filter_->onAccept(cb_)); +} } // namespace } // namespace TlsInspector } // namespace ListenerFilters diff --git a/test/mocks/api/mocks.cc b/test/mocks/api/mocks.cc index 77f6a4463dd3..0132f89b272e 100644 --- a/test/mocks/api/mocks.cc +++ b/test/mocks/api/mocks.cc @@ -7,6 +7,7 @@ #include "gtest/gtest.h" using testing::_; +using testing::Invoke; using testing::Return; namespace Envoy { @@ -26,7 +27,12 @@ Event::DispatcherPtr MockApi::allocateDispatcher(Buffer::WatermarkFactoryPtr&& w return Event::DispatcherPtr{allocateDispatcher_(std::move(watermark_factory), time_system_)}; } -MockOsSysCalls::MockOsSysCalls() = default; +MockOsSysCalls::MockOsSysCalls() { + ON_CALL(*this, close(_)).WillByDefault(Invoke([](int fd) { + const int rc = ::close(fd); + return SysCallIntResult{rc, errno}; + })); +} MockOsSysCalls::~MockOsSysCalls() = default; diff --git a/test/server/BUILD b/test/server/BUILD index e46a0106c687..821e257bd2d7 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -58,6 +58,7 @@ envoy_cc_test( "//test/mocks/network:network_mocks", "//test/mocks/server:server_mocks", "//test/test_common:network_utility_lib", + "//test/test_common:threadsafe_singleton_injector_lib", ], ) diff --git a/test/server/connection_handler_test.cc b/test/server/connection_handler_test.cc index 092ff2780649..eb53c6261cfa 100644 --- a/test/server/connection_handler_test.cc +++ b/test/server/connection_handler_test.cc @@ -3,6 +3,7 @@ #include "common/common/utility.h" #include "common/network/address_impl.h" +#include "common/network/io_socket_handle_impl.h" #include "common/network/raw_buffer_socket.h" #include "common/network/utility.h" @@ -11,6 +12,7 @@ #include "test/mocks/network/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/network_utility.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -27,7 +29,6 @@ using testing::ReturnRef; namespace Envoy { namespace Server { namespace { - class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable { public: ConnectionHandlerTest() @@ -110,6 +111,8 @@ class ConnectionHandlerTest : public testing::Test, protected Logger::Loggable factory_; std::list listeners_; const Network::FilterChainSharedPtr filter_chain_; + NiceMock os_sys_calls_; + TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; }; TEST_F(ConnectionHandlerTest, RemoveListener) { @@ -611,9 +614,11 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeout) { .WillOnce(Invoke([&](Network::ListenerFilterCallbacks&) -> Network::FilterStatus { return Network::FilterStatus::StopIteration; })); + Network::MockConnectionSocket* accepted_socket = new NiceMock(); + Network::IoSocketHandleImpl io_handle{42}; + EXPECT_CALL(*accepted_socket, ioHandle()).WillRepeatedly(ReturnRef(io_handle)); Event::MockTimer* timeout = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000), _)); - Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}, true); Stats::Gauge& downstream_pre_cx_active = stats_store_.gauge("downstream_pre_cx_active", Stats::Gauge::ImportMode::Accumulate); @@ -659,9 +664,11 @@ TEST_F(ConnectionHandlerTest, ContinueOnListenerFilterTimeout) { .WillOnce(Invoke([&](Network::ListenerFilterCallbacks&) -> Network::FilterStatus { return Network::FilterStatus::StopIteration; })); + Network::MockConnectionSocket* accepted_socket = new NiceMock(); + Network::IoSocketHandleImpl io_handle{42}; + EXPECT_CALL(*accepted_socket, ioHandle()).WillRepeatedly(ReturnRef(io_handle)); Event::MockTimer* timeout = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000), _)); - Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}, true); Stats::Gauge& downstream_pre_cx_active = stats_store_.gauge("downstream_pre_cx_active", Stats::Gauge::ImportMode::Accumulate); @@ -708,9 +715,12 @@ TEST_F(ConnectionHandlerTest, ListenerFilterTimeoutResetOnSuccess) { listener_filter_cb = &cb; return Network::FilterStatus::StopIteration; })); + Network::MockConnectionSocket* accepted_socket = new NiceMock(); + Network::IoSocketHandleImpl io_handle{42}; + EXPECT_CALL(*accepted_socket, ioHandle()).WillRepeatedly(ReturnRef(io_handle)); + Event::MockTimer* timeout = new Event::MockTimer(&dispatcher_); EXPECT_CALL(*timeout, enableTimer(std::chrono::milliseconds(15000), _)); - Network::MockConnectionSocket* accepted_socket = new NiceMock(); listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}, true); EXPECT_CALL(manager_, findFilterChain(_)).WillOnce(Return(nullptr)); @@ -755,6 +765,51 @@ TEST_F(ConnectionHandlerTest, ListenerFilterDisabledTimeout) { EXPECT_CALL(*listener, onDestroy()); } +// Listener Filter could close socket in the context of listener callback. +TEST_F(ConnectionHandlerTest, ListenerFilterReportError) { + InSequence s; + + TestListener* test_listener = addListener(1, true, false, "test_listener"); + Network::MockListener* listener = new Network::MockListener(); + Network::ListenerCallbacks* listener_callbacks; + EXPECT_CALL(dispatcher_, createListener_(_, _, _, false)) + .WillOnce(Invoke( + [&](Network::Socket&, Network::ListenerCallbacks& cb, bool, bool) -> Network::Listener* { + listener_callbacks = &cb; + return listener; + })); + EXPECT_CALL(test_listener->socket_, localAddress()); + handler_->addListener(*test_listener); + + Network::MockListenerFilter* first_filter = new Network::MockListenerFilter(); + Network::MockListenerFilter* last_filter = new Network::MockListenerFilter(); + + EXPECT_CALL(factory_, createListenerFilterChain(_)) + .WillRepeatedly(Invoke([&](Network::ListenerFilterManager& manager) -> bool { + manager.addAcceptFilter(Network::ListenerFilterPtr{first_filter}); + manager.addAcceptFilter(Network::ListenerFilterPtr{last_filter}); + return true; + })); + // The first filter close the socket + EXPECT_CALL(*first_filter, onAccept(_)) + .WillOnce(Invoke([&](Network::ListenerFilterCallbacks& cb) -> Network::FilterStatus { + cb.socket().close(); + return Network::FilterStatus::StopIteration; + })); + // The last filter won't be invoked + EXPECT_CALL(*last_filter, onAccept(_)).Times(0); + Network::MockConnectionSocket* accepted_socket = new NiceMock(); + listener_callbacks->onAccept(Network::ConnectionSocketPtr{accepted_socket}, true); + + dispatcher_.clearDeferredDeleteList(); + // Make sure the error leads to no listener timer created. + EXPECT_CALL(dispatcher_, createTimer_(_)).Times(0); + // Make sure we never try to match the filer chain since listener filter doesn't complete. + EXPECT_CALL(manager_, findFilterChain(_)).Times(0); + + EXPECT_CALL(*listener, onDestroy()); +} + // Ensure an exception is thrown if there are no filters registered for a UDP listener TEST_F(ConnectionHandlerTest, UdpListenerNoFilterThrowsException) { InSequence s; diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 768468f8c96e..e5b570919e19 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -176,6 +176,7 @@ class ListenerManagerImplWithRealFiltersTest : public ListenerManagerImplTest { socket_ = std::make_unique>(); local_address_.reset(new Network::Address::Ipv4Instance("127.0.0.1", 1234)); remote_address_.reset(new Network::Address::Ipv4Instance("127.0.0.1", 1234)); + EXPECT_CALL(os_sys_calls_, close(_)).WillRepeatedly(Return(Api::SysCallIntResult{0, errno})); } const Network::FilterChain* @@ -265,11 +266,9 @@ class ListenerManagerImplWithRealFiltersTest : public ListenerManagerImplTest { const envoy::api::v2::core::SocketOption::SocketState& expected_state, const Network::SocketOptionName& expected_option, int expected_value, uint32_t expected_num_options = 1) { - NiceMock os_sys_calls; - TestThreadsafeSingletonInjector os_calls(&os_sys_calls); if (expected_option.has_value()) { expectCreateListenSocket(expected_state, expected_num_options); - expectSetsockopt(os_sys_calls, expected_option.level(), expected_option.option(), + expectSetsockopt(os_sys_calls_, expected_option.level(), expected_option.option(), expected_value, expected_num_options); manager_->addOrUpdateListener(listener, "", true); EXPECT_EQ(1U, manager_->listeners().size()); @@ -280,6 +279,10 @@ class ListenerManagerImplWithRealFiltersTest : public ListenerManagerImplTest { } } +protected: + NiceMock os_sys_calls_; + TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; + private: std::unique_ptr socket_; Network::Address::InstanceConstSharedPtr local_address_; @@ -390,9 +393,8 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, UdpAddress) { EXPECT_CALL(server_.random_, uuid()); EXPECT_CALL(listener_factory_, createListenSocket(_, Network::Address::SocketType::Datagram, _, true)); - NiceMock os_sys_calls; - TestThreadsafeSingletonInjector os_calls(&os_sys_calls); - EXPECT_CALL(os_sys_calls, setsockopt_(_, _, _, _, _)).Times(testing::AtLeast(1)); + EXPECT_CALL(os_sys_calls_, setsockopt_(_, _, _, _, _)).Times(testing::AtLeast(1)); + EXPECT_CALL(os_sys_calls_, close(_)).WillRepeatedly(Return(Api::SysCallIntResult{0, errno})); manager_->addOrUpdateListener(listener_proto, "", true); EXPECT_EQ(1u, manager_->listeners().size()); } @@ -694,6 +696,7 @@ drain_type: default ON_CALL(os_sys_calls, socket(AF_INET, _, 0)).WillByDefault(Return(Api::SysCallIntResult{5, 0})); ON_CALL(os_sys_calls, socket(AF_INET6, _, 0)).WillByDefault(Return(Api::SysCallIntResult{-1, 0})); + ON_CALL(os_sys_calls, close(_)).WillByDefault(Return(Api::SysCallIntResult{0, 0})); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); @@ -726,6 +729,7 @@ drain_type: default ON_CALL(os_sys_calls, socket(AF_INET, _, 0)).WillByDefault(Return(Api::SysCallIntResult{-1, 0})); ON_CALL(os_sys_calls, socket(AF_INET6, _, 0)).WillByDefault(Return(Api::SysCallIntResult{5, 0})); + ON_CALL(os_sys_calls, close(_)).WillByDefault(Return(Api::SysCallIntResult{0, 0})); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, true)); @@ -2926,7 +2930,6 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, TransparentFreebindListenerDisabl TEST_F(ListenerManagerImplWithRealFiltersTest, TransparentListenerEnabled) { auto listener = createIPv4Listener("TransparentListener"); listener.mutable_transparent()->set_value(true); - testSocketOption(listener, envoy::api::v2::core::SocketOption::STATE_PREBIND, ENVOY_SOCKET_IP_TRANSPARENT, /* expected_value */ 1, /* expected_num_options */ 2); @@ -2961,9 +2964,6 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, FastOpenListenerEnabled) { } TEST_F(ListenerManagerImplWithRealFiltersTest, LiteralSockoptListenerEnabled) { - NiceMock os_sys_calls; - TestThreadsafeSingletonInjector os_calls(&os_sys_calls); - const envoy::api::v2::Listener listener = parseListenerFromV2Yaml(R"EOF( name: SockoptsListener address: @@ -2981,11 +2981,11 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, LiteralSockoptListenerEnabled) { expectCreateListenSocket(envoy::api::v2::core::SocketOption::STATE_PREBIND, /* expected_num_options */ 3); - expectSetsockopt(os_sys_calls, + expectSetsockopt(os_sys_calls_, /* expected_sockopt_level */ 1, /* expected_sockopt_name */ 2, /* expected_value */ 3); - expectSetsockopt(os_sys_calls, + expectSetsockopt(os_sys_calls_, /* expected_sockopt_level */ 4, /* expected_sockopt_name */ 5, /* expected_value */ 6); From 0fde42edabd4b076e745a3153a7a2cd8ed64416f Mon Sep 17 00:00:00 2001 From: Dmitri Dolguikh Date: Wed, 28 Aug 2019 19:13:17 -0700 Subject: [PATCH 477/542] Fixes gcc 8.3.1 build failure due to FilterChainBenchmarkFixture::SetUp hiding base-class virtual functions (#8071) Description: I'm seeing "bazel-out/k8-fastbuild/bin/external/com_github_google_benchmark/_virtual_includes/benchmark/benchmark/benchmark.h:1071:16: error: 'virtual void benchmark::Fixture::SetUp(benchmark::State&)' was hidden" when running tests. This resolves the issue with hiding of the base-class functions. Risk Level: low Testing: Docs Changes: Release Notes: Signed-off-by: Dmitri Dolguikh --- test/server/filter_chain_benchmark_test.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/test/server/filter_chain_benchmark_test.cc b/test/server/filter_chain_benchmark_test.cc index 0517635414c7..1ce0577c141e 100644 --- a/test/server/filter_chain_benchmark_test.cc +++ b/test/server/filter_chain_benchmark_test.cc @@ -149,6 +149,7 @@ const char YamlSingleDstPortBottom[] = R"EOF( class FilterChainBenchmarkFixture : public benchmark::Fixture { public: + using Fixture::SetUp; void SetUp(const ::benchmark::State& state) override { int64_t input_size = state.range(0); std::vector port_chains; From 9ac491af4ae700fde1f8135717f7dea0b407627a Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 29 Aug 2019 09:12:05 -0400 Subject: [PATCH 478/542] test: fix ups for various deprecated fields (#8068) Takeaways: we've lost the ability to do empty regex (which was covered in router tests and is proto constraint validated on the new safe regex) as well as negative lookahead (also covered in tests) along with a host of other things conveniently documented as not supported here: https://github.com/google/re2/wiki/Syntax Otherwise split up a bunch of tests, duplicated and tagged a bunch of tests, and cleaning up after we finally can remove deprecated fields again will be an order of magnitude easier. Also fixing a dup relnote from #8014 Risk Level: n/a (test only) Testing: yes. yes there is. Docs Changes: no Release Notes: no Signed-off-by: Alyssa Wilk --- docs/root/intro/version_history.rst | 1 - test/common/router/config_impl_test.cc | 458 ++++++++++++++---- .../network/dubbo_proxy/conn_manager_test.cc | 7 +- .../network/dubbo_proxy/route_matcher_test.cc | 26 +- .../thrift_proxy/route_matcher_test.cc | 7 +- 5 files changed, 402 insertions(+), 97 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 715ec0fa839f..c3598a5c7c7a 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -29,7 +29,6 @@ Version history * http: added the ability to reject HTTP/1.1 requests with invalid HTTP header values, using the runtime feature `envoy.reloadable_features.strict_header_validation`. * http: changed Envoy to forward existing x-forwarded-proto from upstream trusted proxies. Guarded by `envoy.reloadable_features.trusted_forwarded_proto` which defaults true. * http: added the ability to configure the behavior of the server response header, via the :ref:`server_header_transformation` field. -* http: changed Envoy to forward existing x-forwarded-proto from downstream trusted proxies. Guarded by `envoy.reloadable_features.trusted_forwarded_proto` which defaults true. * http: added the ability to :ref:`merge adjacent slashes` in the path. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index ff6aeb4555d6..717bf35651a7 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -116,8 +116,169 @@ class ConfigImplTestBase { class RouteMatcherTest : public testing::Test, public ConfigImplTestBase {}; -// TODO(alyssawilk) go through all these tests and update or duplicate. -TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutes)) { +// When removing legacy fields this test can be removed. +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestLegacyRoutes)) { + const std::string yaml = R"EOF( +virtual_hosts: +- name: regex + domains: + - bat.com + routes: + - match: + regex: "/t[io]c" + route: + cluster: clock + - match: + safe_regex: + google_re2: {} + regex: "/baa+" + route: + cluster: sheep + - match: + regex: ".*/\\d{3}$" + route: + cluster: three_numbers + prefix_rewrite: "/rewrote" + - match: + regex: ".*" + route: + cluster: regex_default +- name: regex2 + domains: + - bat2.com + routes: + - match: + regex: '' + route: + cluster: nothingness + - match: + regex: ".*" + route: + cluster: regex_default +- name: default + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: instant-server + timeout: 30s + virtual_clusters: + - pattern: "^/rides$" + method: POST + name: ride_request + - pattern: "^/rides/\\d+$" + method: PUT + name: update_ride + - pattern: "^/users/\\d+/chargeaccounts$" + method: POST + name: cc_add + - pattern: "^/users/\\d+/chargeaccounts/(?!validate)\\w+$" + method: PUT + name: cc_add + - pattern: "^/users$" + method: POST + name: create_user_login + - pattern: "^/users/\\d+$" + method: PUT + name: update_user + )EOF"; + + NiceMock stream_info; + TestConfigImpl config(parseRouteConfigurationFromV2Yaml(yaml), factory_context_, true); + + // Regular Expression matching + EXPECT_EQ("clock", + config.route(genHeaders("bat.com", "/tic", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("clock", + config.route(genHeaders("bat.com", "/toc", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("regex_default", + config.route(genHeaders("bat.com", "/tac", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("regex_default", + config.route(genHeaders("bat.com", "", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("regex_default", + config.route(genHeaders("bat.com", "/tick", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("regex_default", + config.route(genHeaders("bat.com", "/tic/toc", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("sheep", + config.route(genHeaders("bat.com", "/baa", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ( + "sheep", + config.route(genHeaders("bat.com", "/baaaaaaaaaaaa", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("regex_default", + config.route(genHeaders("bat.com", "/ba", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("nothingness", + config.route(genHeaders("bat2.com", "", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("regex_default", + config.route(genHeaders("bat2.com", "/foo", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ("regex_default", + config.route(genHeaders("bat2.com", " ", "GET"), 0)->routeEntry()->clusterName()); + + // Regular Expression matching with query string params + EXPECT_EQ( + "clock", + config.route(genHeaders("bat.com", "/tic?tac=true", "GET"), 0)->routeEntry()->clusterName()); + EXPECT_EQ( + "regex_default", + config.route(genHeaders("bat.com", "/tac?tic=true", "GET"), 0)->routeEntry()->clusterName()); + + // Virtual cluster testing. + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/rides", "GET"); + EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/rides/blah", "POST"); + EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/rides", "POST"); + EXPECT_EQ("ride_request", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/rides/123", "PUT"); + EXPECT_EQ("update_ride", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/rides/123/456", "POST"); + EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = + genHeaders("api.lyft.com", "/users/123/chargeaccounts", "POST"); + EXPECT_EQ("cc_add", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = + genHeaders("api.lyft.com", "/users/123/chargeaccounts/hello123", "PUT"); + EXPECT_EQ("cc_add", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = + genHeaders("api.lyft.com", "/users/123/chargeaccounts/validate", "PUT"); + EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/foo/bar", "PUT"); + EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/users", "POST"); + EXPECT_EQ("create_user_login", + virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/users/123", "PUT"); + EXPECT_EQ("update_user", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } + { + Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/something/else", "GET"); + EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); + } +} + +TEST_F(RouteMatcherTest, TestRoutes) { const std::string yaml = R"EOF( virtual_hosts: - name: www2 @@ -172,7 +333,9 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutes)) { - bat.com routes: - match: - regex: "/t[io]c" + safe_regex: + google_re2: {} + regex: "/t[io]c" route: cluster: clock - match: @@ -182,12 +345,16 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutes)) { route: cluster: sheep - match: - regex: ".*/\\d{3}$" + safe_regex: + google_re2: {} + regex: ".*/\\d{3}$" route: cluster: three_numbers prefix_rewrite: "/rewrote" - match: - regex: ".*" + safe_regex: + google_re2: {} + regex: ".*" route: cluster: regex_default - name: regex2 @@ -195,11 +362,9 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutes)) { - bat2.com routes: - match: - regex: '' - route: - cluster: nothingness - - match: - regex: ".*" + safe_regex: + google_re2: {} + regex: ".*" route: cluster: regex_default - name: default @@ -280,23 +445,45 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutes)) { cluster: instant-server timeout: 30s virtual_clusters: - - pattern: "^/rides$" - method: POST + - headers: + - name: ":path" + safe_regex_match: + google_re2: {} + regex: "^/rides$" + - name: ":method" + exact_match: POST name: ride_request - - pattern: "^/rides/\\d+$" - method: PUT + - headers: + - name: ":path" + safe_regex_match: + google_re2: {} + regex: "^/rides/\\d+$" + - name: ":method" + exact_match: PUT name: update_ride - - pattern: "^/users/\\d+/chargeaccounts$" - method: POST - name: cc_add - - pattern: "^/users/\\d+/chargeaccounts/(?!validate)\\w+$" - method: PUT + - headers: + - name: ":path" + safe_regex_match: + google_re2: {} + regex: "^/users/\\d+/chargeaccounts$" + - name: ":method" + exact_match: POST name: cc_add - - pattern: "^/users$" - method: POST + - headers: + - name: ":path" + safe_regex_match: + google_re2: {} + regex: "^/users$" + - name: ":method" + exact_match: POST name: create_user_login - - pattern: "^/users/\\d+$" - method: PUT + - headers: + - name: ":path" + safe_regex_match: + google_re2: {} + regex: "^/users/\\d+$" + - name: ":method" + exact_match: PUT name: update_user - headers: - name: ":path" @@ -372,8 +559,6 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutes)) { config.route(genHeaders("bat.com", "/baaaaaaaaaaaa", "GET"), 0)->routeEntry()->clusterName()); EXPECT_EQ("regex_default", config.route(genHeaders("bat.com", "/ba", "GET"), 0)->routeEntry()->clusterName()); - EXPECT_EQ("nothingness", - config.route(genHeaders("bat2.com", "", "GET"), 0)->routeEntry()->clusterName()); EXPECT_EQ("regex_default", config.route(genHeaders("bat2.com", "/foo", "GET"), 0)->routeEntry()->clusterName()); EXPECT_EQ("regex_default", @@ -556,21 +741,6 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutes)) { Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/rides/123/456", "POST"); EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); } - { - Http::TestHeaderMapImpl headers = - genHeaders("api.lyft.com", "/users/123/chargeaccounts", "POST"); - EXPECT_EQ("cc_add", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); - } - { - Http::TestHeaderMapImpl headers = - genHeaders("api.lyft.com", "/users/123/chargeaccounts/hello123", "PUT"); - EXPECT_EQ("cc_add", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); - } - { - Http::TestHeaderMapImpl headers = - genHeaders("api.lyft.com", "/users/123/chargeaccounts/validate", "PUT"); - EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); - } { Http::TestHeaderMapImpl headers = genHeaders("api.lyft.com", "/foo/bar", "PUT"); EXPECT_EQ("other", virtualClusterName(config.route(headers, 0)->routeEntry(), headers)); @@ -618,7 +788,8 @@ TEST_F(RouteMatcherTest, TestRoutesWithWildcardAndDefaultOnly) { config.route(genHeaders("example.com", "/", "GET"), 0)->routeEntry()->clusterName()); } -TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutesWithInvalidRegex)) { +// When deprecating regex: this test can be removed. +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutesWithInvalidRegexLegacy)) { std::string invalid_route = R"EOF( virtual_hosts: - name: regex @@ -651,6 +822,46 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(TestRoutesWithInvalidRegex)) { EnvoyException, "Invalid regex '\\^/\\(\\+invalid\\)':"); } +TEST_F(RouteMatcherTest, TestRoutesWithInvalidRegex) { + std::string invalid_route = R"EOF( +virtual_hosts: + - name: regex + domains: ["*"] + routes: + - match: + safe_regex: + google_re2: {} + regex: "/(+invalid)" + route: { cluster: "regex" } + )EOF"; + + std::string invalid_virtual_cluster = R"EOF( +virtual_hosts: + - name: regex + domains: ["*"] + routes: + - match: { prefix: "/" } + route: { cluster: "regex" } + virtual_clusters: + name: "invalid" + headers: + name: "invalid" + safe_regex_match: + google_re2: {} + regex: "^/(+invalid)" + )EOF"; + + NiceMock stream_info; + + EXPECT_THROW_WITH_REGEX( + TestConfigImpl(parseRouteConfigurationFromV2Yaml(invalid_route), factory_context_, true), + EnvoyException, "no argument for repetition operator:"); + + EXPECT_THROW_WITH_REGEX(TestConfigImpl(parseRouteConfigurationFromV2Yaml(invalid_virtual_cluster), + factory_context_, true), + EnvoyException, "no argument for repetition operator"); +} + // Virtual cluster that contains neither pattern nor regex. This must be checked while pattern is // deprecated. TEST_F(RouteMatcherTest, TestRoutesWithInvalidVirtualCluster) { @@ -1078,7 +1289,7 @@ name: foo } } -TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(Priority)) { +TEST_F(RouteMatcherTest, Priority) { const std::string yaml = R"EOF( virtual_hosts: - name: local_service @@ -1094,10 +1305,6 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(Priority)) { prefix: "/bar" route: cluster: local_service_grpc - virtual_clusters: - - pattern: "^/bar$" - method: POST - name: foo )EOF"; TestConfigImpl config(parseRouteConfigurationFromV2Yaml(yaml), factory_context_, true); @@ -1165,7 +1372,7 @@ TEST_F(RouteMatcherTest, NoAutoRewriteAndAutoRewriteHeader) { EnvoyException); } -TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(HeaderMatchedRouting)) { +TEST_F(RouteMatcherTest, HeaderMatchedRouting) { const std::string yaml = R"EOF( virtual_hosts: - name: local_service @@ -1199,7 +1406,9 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(HeaderMatchedRouting)) { prefix: "/" headers: - name: test_header_pattern - regex_match: "^user=test-\\d+$" + safe_regex_match: + google_re2: {} + regex: "^user=test-\\d+$" route: cluster: local_service_with_header_pattern_set_regex - match: @@ -1289,7 +1498,8 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(HeaderMatchedRouting)) { } // Verify the fixes for https://github.com/envoyproxy/envoy/issues/2406 -TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(InvalidHeaderMatchedRoutingConfig)) { +// When removing regex_match this test can be removed entirely. +TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(InvalidHeaderMatchedRoutingConfigLegacy)) { std::string value_with_regex_chars = R"EOF( virtual_hosts: - name: local_service @@ -1324,6 +1534,45 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(InvalidHeaderMatchedRoutingConf EnvoyException, "Invalid regex"); } +// Verify the fixes for https://github.com/envoyproxy/envoy/issues/2406 +TEST_F(RouteMatcherTest, InvalidHeaderMatchedRoutingConfig) { + std::string value_with_regex_chars = R"EOF( +virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + prefix: "/" + headers: + - name: test_header + exact_match: "(+not a regex)" + route: { cluster: "local_service" } + )EOF"; + + std::string invalid_regex = R"EOF( +virtual_hosts: + - name: local_service + domains: ["*"] + routes: + - match: + prefix: "/" + headers: + - name: test_header + safe_regex_match: + google_re2: {} + regex: "(+invalid regex)" + route: { cluster: "local_service" } + )EOF"; + + EXPECT_NO_THROW(TestConfigImpl(parseRouteConfigurationFromV2Yaml(value_with_regex_chars), + factory_context_, true)); + + EXPECT_THROW_WITH_REGEX( + TestConfigImpl(parseRouteConfigurationFromV2Yaml(invalid_regex), factory_context_, true), + EnvoyException, "no argument for repetition operator"); +} + +// When removing value: simply remove that section of the config and the relevant test. TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(QueryParamMatchedRouting)) { const std::string yaml = R"EOF( virtual_hosts: @@ -1438,7 +1687,7 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(QueryParamMatchedRouting)) { } } -// Verify the fixes for https://github.com/envoyproxy/envoy/issues/2406 +// When removing value: this test can be removed. TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(InvalidQueryParamMatchedRoutingConfig)) { std::string value_with_regex_chars = R"EOF( virtual_hosts: @@ -2250,7 +2499,7 @@ TEST_F(RouteMatcherTest, ClusterNotFoundResponseCodeConfig404) { config.route(headers, 0)->routeEntry()->clusterNotFoundResponseCode()); } -TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(Shadow)) { +TEST_F(RouteMatcherTest, Shadow) { const std::string yaml = R"EOF( virtual_hosts: - name: www2 @@ -2268,7 +2517,11 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(Shadow)) { route: request_mirror_policy: cluster: some_cluster2 - runtime_key: foo + runtime_fraction: + default_value: + numerator: 20 + denominator: HUNDRED + runtime_key: foo cluster: www2 - match: prefix: "/baz" @@ -2308,6 +2561,7 @@ TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(Shadow)) { class RouteConfigurationV2 : public testing::Test, public ConfigImplTestBase {}; +// When removing runtime_key: this test can be removed. TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RequestMirrorPolicy)) { const std::string yaml = R"EOF( name: foo @@ -4265,6 +4519,8 @@ TEST_F(RoutePropertyTest, excludeVHRateLimits) { EXPECT_TRUE(config_ptr->route(headers, 0)->routeEntry()->includeVirtualHostRateLimits()); } +// When allow_origin: and allow_origin_regex: are removed, simply remove them +// and the relevant checks below. TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestVHostCorsConfig)) { const std::string yaml = R"EOF( virtual_hosts: @@ -4327,7 +4583,7 @@ TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestVHostCorsConfig)) { EXPECT_EQ(cors_policy->allowCredentials(), true); } -TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestRouteCorsConfig)) { +TEST_F(RoutePropertyTest, TestRouteCorsConfig) { const std::string yaml = R"EOF( virtual_hosts: - name: "default" @@ -4338,7 +4594,8 @@ TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestRouteCorsConfig)) { route: cluster: "ats" cors: - allow_origin: ["test-origin"] + allow_origin_string_match: + - exact: "test-origin" allow_methods: "test-methods" allow_headers: "test-headers" expose_headers: "test-expose-headers" @@ -4380,7 +4637,8 @@ TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestRouteCorsConfig)) { EXPECT_EQ(cors_policy->allowCredentials(), true); } -TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestVHostCorsLegacyConfig)) { +// When allow-origin: is removed, this test can be removed. +TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TTestVHostCorsLegacyConfig)) { const std::string yaml = R"EOF( virtual_hosts: - name: default @@ -4419,6 +4677,7 @@ TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestVHostCorsLegacyConfig)) { EXPECT_EQ(cors_policy->allowCredentials(), true); } +// When allow-origin: is removed, this test can be removed. TEST_F(RoutePropertyTest, DEPRECATED_FEATURE_TEST(TestRouteCorsLegacyConfig)) { const std::string yaml = R"EOF( virtual_hosts: @@ -4910,14 +5169,17 @@ name: foo Envoy::EnvoyException, "Cannot create a Baz when metadata is empty."); } -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RouteConfigGetters)) { +TEST_F(RouteConfigurationV2, RouteConfigGetters) { const std::string yaml = R"EOF( name: foo virtual_hosts: - name: bar domains: ["*"] routes: - - match: { regex: "/rege[xy]" } + - match: + safe_regex: + google_re2: {} + regex: "/rege[xy]" route: { cluster: ww2 } - match: { path: "/exact-path" } route: { cluster: ww2 } @@ -4949,19 +5211,25 @@ name: foo EXPECT_EQ("foo", route_entry->virtualHost().routeConfig().name()); } -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RouteTracingConfig)) { +TEST_F(RouteConfigurationV2, RouteTracingConfig) { const std::string yaml = R"EOF( name: foo virtual_hosts: - name: bar domains: ["*"] routes: - - match: { regex: "/first" } + - match: + safe_regex: + google_re2: {} + regex: "/first" tracing: client_sampling: numerator: 1 route: { cluster: ww2 } - - match: { regex: "/second" } + - match: + safe_regex: + google_re2: {} + regex: "/second" tracing: overall_sampling: numerator: 1 @@ -5004,7 +5272,7 @@ name: foo } // Test to check Prefix Rewrite for redirects -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RedirectPrefixRewrite)) { +TEST_F(RouteConfigurationV2, RedirectPrefixRewrite) { std::string RedirectPrefixRewrite = R"EOF( name: AllRedirects virtual_hosts: @@ -5017,7 +5285,10 @@ name: AllRedirects redirect: { prefix_rewrite: "/new/path/" } - match: { prefix: "/host/prefix" } redirect: { host_redirect: new.lyft.com, prefix_rewrite: "/new/prefix"} - - match: { regex: "/[r][e][g][e][x].*"} + - match: + safe_regex: + google_re2: {} + regex: "/[r][e][g][e][x].*" redirect: { prefix_rewrite: "/new/regex-prefix/" } - match: { prefix: "/http/prefix"} redirect: { prefix_rewrite: "/https/prefix" , https_redirect: true } @@ -5191,7 +5462,7 @@ name: AllRedirects } } -TEST_F(RouteMatcherTest, DEPRECATED_FEATURE_TEST(HeaderMatchedRoutingV2)) { +TEST_F(RouteMatcherTest, HeaderMatchedRoutingV2) { const std::string yaml = R"EOF( name: foo virtual_hosts: @@ -5224,7 +5495,9 @@ name: foo prefix: "/" headers: - name: test_header_pattern - regex_match: "^user=test-\\d+$" + safe_regex_match: + google_re2: {} + regex: "^user=test-\\d+$" route: cluster: local_service_with_header_pattern_set_regex - match: @@ -5366,8 +5639,7 @@ name: foo } } -TEST_F(RouteConfigurationV2, - DEPRECATED_FEATURE_TEST(RegexPrefixWithNoRewriteWorksWhenPathChanged)) { +TEST_F(RouteConfigurationV2, RegexPrefixWithNoRewriteWorksWhenPathChanged) { // Setup regex route entry. the regex is trivial, that's ok as we only want to test that // path change works. @@ -5377,7 +5649,10 @@ name: RegexNoMatch - name: regex domains: [regex.lyft.com] routes: - - match: { regex: "/regex"} + - match: + safe_regex: + google_re2: {} + regex: "/regex" route: { cluster: some-cluster } )EOF"; @@ -5398,14 +5673,17 @@ name: RegexNoMatch } } -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(NoIdleTimeout)) { +TEST_F(RouteConfigurationV2, NoIdleTimeout) { const std::string NoIdleTimeout = R"EOF( name: NoIdleTimeout virtual_hosts: - name: regex domains: [idle.lyft.com] routes: - - match: { regex: "/regex"} + - match: + safe_regex: + google_re2: {} + regex: "/regex" route: cluster: some-cluster )EOF"; @@ -5416,14 +5694,17 @@ name: NoIdleTimeout EXPECT_EQ(absl::nullopt, route_entry->idleTimeout()); } -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(ZeroIdleTimeout)) { +TEST_F(RouteConfigurationV2, ZeroIdleTimeout) { const std::string ZeroIdleTimeout = R"EOF( name: ZeroIdleTimeout virtual_hosts: - name: regex domains: [idle.lyft.com] routes: - - match: { regex: "/regex"} + - match: + safe_regex: + google_re2: {} + regex: "/regex" route: cluster: some-cluster idle_timeout: 0s @@ -5435,14 +5716,17 @@ name: ZeroIdleTimeout EXPECT_EQ(0, route_entry->idleTimeout().value().count()); } -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(ExplicitIdleTimeout)) { +TEST_F(RouteConfigurationV2, ExplicitIdleTimeout) { const std::string ExplicitIdleTimeout = R"EOF( name: ExplicitIdleTimeout virtual_hosts: - name: regex domains: [idle.lyft.com] routes: - - match: { regex: "/regex"} + - match: + safe_regex: + google_re2: {} + regex: "/regex" route: cluster: some-cluster idle_timeout: 7s @@ -5455,14 +5739,17 @@ name: ExplicitIdleTimeout EXPECT_EQ(7 * 1000, route_entry->idleTimeout().value().count()); } -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RetriableStatusCodes)) { +TEST_F(RouteConfigurationV2, RetriableStatusCodes) { const std::string ExplicitIdleTimeout = R"EOF( name: RetriableStatusCodes virtual_hosts: - name: regex domains: [idle.lyft.com] routes: - - match: { regex: "/regex"} + - match: + safe_regex: + google_re2: {} + regex: "/regex" route: cluster: some-cluster retry_policy: @@ -5477,14 +5764,17 @@ name: RetriableStatusCodes EXPECT_EQ(expected_codes, retry_policy.retriableStatusCodes()); } -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(UpgradeConfigs)) { +TEST_F(RouteConfigurationV2, UpgradeConfigs) { const std::string UpgradeYaml = R"EOF( name: RetriableStatusCodes virtual_hosts: - name: regex domains: [idle.lyft.com] routes: - - match: { regex: "/regex"} + - match: + safe_regex: + google_re2: {} + regex: "/regex" route: cluster: some-cluster upgrade_configs: @@ -5501,14 +5791,17 @@ name: RetriableStatusCodes EXPECT_FALSE(upgrade_map.find("disabled")->second); } -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(DuplicateUpgradeConfigs)) { +TEST_F(RouteConfigurationV2, DuplicateUpgradeConfigs) { const std::string yaml = R"EOF( name: RetriableStatusCodes virtual_hosts: - name: regex domains: [idle.lyft.com] routes: - - match: { regex: "/regex"} + - match: + safe_regex: + google_re2: {} + regex: "/regex" route: cluster: some-cluster upgrade_configs: @@ -5524,14 +5817,17 @@ name: RetriableStatusCodes // Verifies that we're creating a new instance of the retry plugins on each call instead of always // returning the same one. -TEST_F(RouteConfigurationV2, DEPRECATED_FEATURE_TEST(RetryPluginsAreNotReused)) { +TEST_F(RouteConfigurationV2, RetryPluginsAreNotReused) { const std::string ExplicitIdleTimeout = R"EOF( name: RetriableStatusCodes virtual_hosts: - name: regex domains: [idle.lyft.com] routes: - - match: { regex: "/regex"} + - match: + safe_regex: + google_re2: {} + regex: "/regex" route: cluster: some-cluster retry_policy: diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index f93f40b47f5d..536e167b0209 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -1156,8 +1156,7 @@ TEST_F(ConnectionManagerTest, PendingMessageEnd) { EXPECT_EQ(1U, store_.gauge("test.request_active", Stats::Gauge::ImportMode::Accumulate).value()); } -// TODO(alyssawilk) update. -TEST_F(ConnectionManagerTest, DEPRECATED_FEATURE_TEST(Routing)) { +TEST_F(ConnectionManagerTest, Routing) { const std::string yaml = R"EOF( stat_prefix: test protocol_type: Dubbo @@ -1169,7 +1168,9 @@ serialization_type: Hessian2 - match: method: name: - regex: "(.*?)" + safe_regex: + google_re2: {} + regex: "(.*?)" route: cluster: user_service_dubbo_server )EOF"; diff --git a/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc b/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc index 398743469bcb..1099862c5417 100644 --- a/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/route_matcher_test.cc @@ -38,8 +38,7 @@ parseDubboProxyFromV2Yaml(const std::string& yaml) { } // namespace -// TODO(alyssawilk) update. -TEST(DubboRouteMatcherTest, DEPRECATED_FEATURE_TEST(RouteByServiceNameWithAnyMethod)) { +TEST(DubboRouteMatcherTest, RouteByServiceNameWithAnyMethod) { { const std::string yaml = R"EOF( name: local_route @@ -48,7 +47,9 @@ interface: org.apache.dubbo.demo.DemoService - match: method: name: - regex: "(.*?)" + safe_regex: + google_re2: {} + regex: "(.*?)" route: cluster: user_service_dubbo_server )EOF"; @@ -95,7 +96,9 @@ group: test - match: method: name: - regex: "(.*?)" + safe_regex: + google_re2: {} + regex: "(.*?)" route: cluster: user_service_dubbo_server )EOF"; @@ -129,7 +132,9 @@ version: 1.0.0 - match: method: name: - regex: "(.*?)" + safe_regex: + google_re2: {} + regex: "(.*?)" route: cluster: user_service_dubbo_server )EOF"; @@ -168,7 +173,9 @@ group: HSF - match: method: name: - regex: "(.*?)" + safe_regex: + google_re2: {} + regex: "(.*?)" route: cluster: user_service_dubbo_server )EOF"; @@ -292,8 +299,7 @@ interface: org.apache.dubbo.demo.DemoService EXPECT_EQ("user_service_dubbo_server", matcher.route(metadata, 0)->routeEntry()->clusterName()); } -// TODO(alyssawilk) update. -TEST(DubboRouteMatcherTest, DEPRECATED_FEATURE_TEST(RouteByMethodWithRegexMatch)) { +TEST(DubboRouteMatcherTest, RouteByMethodWithRegexMatch) { const std::string yaml = R"EOF( name: local_route interface: org.apache.dubbo.demo.DemoService @@ -301,7 +307,9 @@ interface: org.apache.dubbo.demo.DemoService - match: method: name: - regex: "\\d{3}test" + safe_regex: + google_re2: {} + regex: "\\d{3}test" route: cluster: user_service_dubbo_server )EOF"; diff --git a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc index c7efe51ec11c..75b096d15bf0 100644 --- a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc @@ -331,8 +331,7 @@ name: config EXPECT_EQ("cluster1", route->routeEntry()->clusterName()); } -// TODO(alyssawilk) update. -TEST(ThriftRouteMatcherTest, DEPRECATED_FEATURE_TEST(RouteByRegexHeaderMatcher)) { +TEST(ThriftRouteMatcherTest, RouteByRegexHeaderMatcher) { const std::string yaml = R"EOF( name: config routes: @@ -340,7 +339,9 @@ name: config method_name: "method1" headers: - name: "x-version" - regex_match: "0.[5-9]" + safe_regex_match: + google_re2: {} + regex: "0.[5-9]" route: cluster: "cluster1" )EOF"; From 8556a755fba78406813aa9c572c7a7ee9b0b415b Mon Sep 17 00:00:00 2001 From: "Tejasvi (Teju) Nareddy" Date: Thu, 29 Aug 2019 08:49:12 -0700 Subject: [PATCH 479/542] include: add log dependency header to connection_handler.h (#8072) Signed-off-by: Teju Nareddy --- include/envoy/network/connection_handler.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/envoy/network/connection_handler.h b/include/envoy/network/connection_handler.h index 6c24a814db5f..2e5f8057db2f 100644 --- a/include/envoy/network/connection_handler.h +++ b/include/envoy/network/connection_handler.h @@ -9,6 +9,8 @@ #include "envoy/network/listener.h" #include "envoy/ssl/context.h" +#include "spdlog/spdlog.h" + namespace Envoy { namespace Network { From 4f2c5a4630126ed6b42b1cdaac88a8d399faa2a7 Mon Sep 17 00:00:00 2001 From: danzh Date: Thu, 29 Aug 2019 12:07:06 -0400 Subject: [PATCH 480/542] quiche: Update QUICHE dep (#8044) Update QUICHE tar ball to 4abb566fbbc63df8fe7c1ac30b21632b9eb18d0c. Add some new impl's for newly added api. Risk Level: low Testing: using quiche build in tests. Part of #2557 Signed-off-by: Dan Zhang --- bazel/external/quiche.BUILD | 225 ++++++++-- bazel/repository_locations.bzl | 6 +- source/extensions/quic_listeners/quiche/BUILD | 3 + .../quic_listeners/quiche/platform/BUILD | 9 + .../quiche/platform/flags_list.h | 408 +++++++++--------- .../platform/quic_mem_slice_storage_impl.h | 2 + .../quiche/platform/quic_text_utils_impl.h | 4 + .../quiche/platform/spdy_containers_impl.h | 2 + .../quiche/platform/spdy_map_util_impl.h | 18 + test/extensions/quic_listeners/quiche/BUILD | 1 + .../quic_listeners/quiche/platform/BUILD | 13 + 11 files changed, 443 insertions(+), 248 deletions(-) create mode 100644 source/extensions/quic_listeners/quiche/platform/spdy_map_util_impl.h diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index e08d8a5987eb..7f59318c97d5 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -711,6 +711,7 @@ envoy_cc_library( "quiche/spdy/platform/api/spdy_flags.h", "quiche/spdy/platform/api/spdy_logging.h", "quiche/spdy/platform/api/spdy_macros.h", + "quiche/spdy/platform/api/spdy_map_util.h", "quiche/spdy/platform/api/spdy_mem_slice.h", "quiche/spdy/platform/api/spdy_ptr_util.h", "quiche/spdy/platform/api/spdy_string.h", @@ -766,6 +767,16 @@ envoy_cc_library( deps = [":spdy_platform"], ) +envoy_cc_library( + name = "spdy_core_fifo_write_scheduler_lib", + hdrs = ["quiche/spdy/core/fifo_write_scheduler.h"], + repository = "@envoy", + deps = [ + ":spdy_core_write_scheduler_lib", + ":spdy_platform", + ], +) + envoy_cc_library( name = "spdy_core_framer_lib", srcs = [ @@ -849,6 +860,34 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "spdy_core_lifo_write_scheduler_lib", + hdrs = ["quiche/spdy/core/lifo_write_scheduler.h"], + repository = "@envoy", + deps = [ + ":spdy_core_write_scheduler_lib", + ":spdy_platform", + ], +) + +envoy_cc_library( + name = "spdy_core_intrusive_list_lib", + hdrs = ["quiche/spdy/core/spdy_intrusive_list.h"], + repository = "@envoy", +) + +envoy_cc_library( + name = "spdy_core_http2_priority_write_scheduler_lib", + hdrs = ["quiche/spdy/core/http2_priority_write_scheduler.h"], + repository = "@envoy", + deps = [ + ":spdy_core_intrusive_list_lib", + ":spdy_core_protocol_lib", + ":spdy_core_write_scheduler_lib", + ":spdy_platform", + ], +) + envoy_cc_library( name = "spdy_core_hpack_hpack_lib", srcs = [ @@ -976,6 +1015,7 @@ envoy_cc_library( "quiche/quic/platform/api/quic_pcc_sender.h", ], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_core_time_lib", @@ -1021,6 +1061,7 @@ envoy_cc_library( # "quiche/quic/platform/api/quic_test_loopback.h", ], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_platform_export", @@ -1033,6 +1074,7 @@ envoy_cc_library( name = "quic_platform_bbr2_sender", hdrs = ["quiche/quic/platform/api/quic_bbr2_sender.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//source/extensions/quic_listeners/quiche/platform:quic_platform_bbr2_sender_impl_lib"], ) @@ -1040,6 +1082,7 @@ envoy_cc_test_library( name = "quic_platform_epoll_lib", hdrs = ["quiche/quic/platform/api/quic_epoll.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_epoll_impl_lib"], ) @@ -1047,6 +1090,7 @@ envoy_cc_test_library( name = "quic_platform_expect_bug", hdrs = ["quiche/quic/platform/api/quic_expect_bug.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_expect_bug_impl_lib"], ) @@ -1054,6 +1098,7 @@ envoy_cc_library( name = "quic_platform_export", hdrs = ["quiche/quic/platform/api/quic_export.h"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = ["@envoy//source/extensions/quic_listeners/quiche/platform:quic_platform_export_impl_lib"], ) @@ -1062,6 +1107,7 @@ envoy_cc_library( name = "quic_platform_ip_address_family", hdrs = ["quiche/quic/platform/api/quic_ip_address_family.h"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], ) @@ -1071,6 +1117,7 @@ envoy_cc_library( hdrs = ["quiche/quic/platform/api/quic_ip_address.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_platform_base", @@ -1083,6 +1130,7 @@ envoy_cc_test_library( name = "quic_platform_mock_log", hdrs = ["quiche/quic/platform/api/quic_mock_log.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_mock_log_impl_lib"], ) @@ -1090,6 +1138,7 @@ envoy_cc_test_library( name = "quic_platform_port_utils", hdrs = ["quiche/quic/platform/api/quic_port_utils.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_port_utils_impl_lib"], ) @@ -1097,6 +1146,7 @@ envoy_cc_test_library( name = "quic_platform_sleep", hdrs = ["quiche/quic/platform/api/quic_sleep.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_sleep_impl_lib"], ) @@ -1106,6 +1156,7 @@ envoy_cc_library( hdrs = ["quiche/quic/platform/api/quic_socket_address.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_platform_export", @@ -1117,6 +1168,7 @@ envoy_cc_test_library( name = "quic_platform_test", hdrs = ["quiche/quic/platform/api/quic_test.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_test_impl_lib"], ) @@ -1124,6 +1176,7 @@ envoy_cc_test_library( name = "quic_platform_test_output", hdrs = ["quiche/quic/platform/api/quic_test_output.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_test_output_impl_lib"], ) @@ -1131,6 +1184,7 @@ envoy_cc_test_library( name = "quic_platform_system_event_loop", hdrs = ["quiche/quic/platform/api/quic_system_event_loop.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_system_event_loop_impl_lib"], ) @@ -1138,6 +1192,7 @@ envoy_cc_test_library( name = "quic_platform_thread", hdrs = ["quiche/quic/platform/api/quic_thread.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_thread_impl_lib"], ) @@ -1156,6 +1211,7 @@ envoy_cc_library( name = "quic_core_proto_cached_network_parameters_proto_header", hdrs = ["quiche/quic/core/proto/cached_network_parameters_proto.h"], repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_proto_cached_network_parameters_proto_cc"], ) @@ -1174,6 +1230,7 @@ envoy_cc_library( name = "quic_core_proto_source_address_token_proto_header", hdrs = ["quiche/quic/core/proto/source_address_token_proto.h"], repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_proto_source_address_token_proto_cc"], ) @@ -1191,6 +1248,7 @@ envoy_cc_library( name = "quic_core_proto_crypto_server_config_proto_header", hdrs = ["quiche/quic/core/proto/crypto_server_config_proto.h"], repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_proto_crypto_server_config_proto_cc"], ) @@ -1278,6 +1336,7 @@ envoy_cc_library( "quiche/quic/core/quic_simple_buffer_allocator.h", ], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [":quic_platform_export"], ) @@ -1311,6 +1370,8 @@ envoy_cc_library( tags = ["nofips"], deps = [ ":quic_core_bandwidth_lib", + ":quic_core_congestion_control_congestion_control_interface_lib", + ":quic_core_congestion_control_windowed_filter_lib", ":quic_core_packet_number_indexed_queue_lib", ":quic_core_packets_lib", ":quic_core_time_lib", @@ -1903,6 +1964,14 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "quic_core_http_http_constants_lib", + hdrs = ["quiche/quic/core/http/http_constants.h"], + copts = quiche_copt, + repository = "@envoy", + deps = [":quic_core_types_lib"], +) + envoy_cc_library( name = "quic_core_http_client_lib", srcs = [ @@ -2003,6 +2072,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/http/spdy_server_push_utils.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":quic_platform_base", @@ -2039,6 +2109,7 @@ envoy_cc_library( ":quic_core_crypto_crypto_handshake_lib", ":quic_core_error_codes_lib", ":quic_core_http_header_list_lib", + ":quic_core_http_http_constants_lib", ":quic_core_http_http_decoder_lib", ":quic_core_http_http_encoder_lib", ":quic_core_http_spdy_stream_body_buffer_lib", @@ -2230,6 +2301,9 @@ envoy_cc_library( ":quic_core_versions_lib", ":quic_platform", ":quic_platform_socket_address", + ":spdy_core_fifo_write_scheduler_lib", + ":spdy_core_http2_priority_write_scheduler_lib", + ":spdy_core_lifo_write_scheduler_lib", ":spdy_core_priority_write_scheduler_lib", ], ) @@ -2260,29 +2334,13 @@ envoy_cc_library( ) envoy_cc_library( - name = "quic_core_qpack_qpack_instruction_encoder_lib", - srcs = ["quiche/quic/core/qpack/qpack_instruction_encoder.cc"], - hdrs = ["quiche/quic/core/qpack/qpack_instruction_encoder.h"], - copts = quiche_copt, - repository = "@envoy", - deps = [ - ":http2_hpack_huffman_hpack_huffman_encoder_lib", - ":http2_hpack_varint_hpack_varint_encoder_lib", - ":quic_core_qpack_qpack_constants_lib", - ":quic_platform", - ], -) - -envoy_cc_library( - name = "quic_core_qpack_qpack_instruction_decoder_lib", - srcs = ["quiche/quic/core/qpack/qpack_instruction_decoder.cc"], - hdrs = ["quiche/quic/core/qpack/qpack_instruction_decoder.h"], - copts = quiche_copt, + name = "quic_core_qpack_blocking_manager_lib", + srcs = ["quiche/quic/core/qpack/qpack_blocking_manager.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_blocking_manager.h"], repository = "@envoy", + tags = ["nofips"], deps = [ - ":http2_hpack_huffman_hpack_huffman_decoder_lib", - ":http2_hpack_varint_hpack_varint_decoder_lib", - ":quic_core_qpack_qpack_constants_lib", + ":quic_core_types_lib", ":quic_platform_base", ], ) @@ -2293,21 +2351,42 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_constants.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_platform_base"], ) +envoy_cc_library( + name = "quic_core_qpack_qpack_decoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_decoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":quic_core_qpack_qpack_decoder_stream_sender_lib", + ":quic_core_qpack_qpack_encoder_stream_receiver_lib", + ":quic_core_qpack_qpack_header_table_lib", + ":quic_core_qpack_qpack_progressive_decoder_lib", + ":quic_core_types_lib", + ":quic_platform_base", + ], +) + envoy_cc_library( name = "quic_core_qpack_qpack_encoder_lib", srcs = ["quiche/quic/core/qpack/qpack_encoder.cc"], hdrs = ["quiche/quic/core/qpack/qpack_encoder.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ + ":quic_core_qpack_blocking_manager_lib", ":quic_core_qpack_qpack_constants_lib", ":quic_core_qpack_qpack_decoder_stream_receiver_lib", ":quic_core_qpack_qpack_encoder_stream_sender_lib", ":quic_core_qpack_qpack_header_table_lib", ":quic_core_qpack_qpack_instruction_encoder_lib", + ":quic_core_qpack_qpack_required_insert_count_lib", ":quic_core_qpack_value_splitting_header_list_lib", ":quic_core_types_lib", ":quic_platform_base", @@ -2315,49 +2394,76 @@ envoy_cc_library( ) envoy_cc_library( - name = "quic_core_qpack_qpack_progressive_decoder_lib", - srcs = ["quiche/quic/core/qpack/qpack_progressive_decoder.cc"], - hdrs = ["quiche/quic/core/qpack/qpack_progressive_decoder.h"], + name = "quic_core_qpack_qpack_header_table_lib", + srcs = ["quiche/quic/core/qpack/qpack_header_table.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_header_table.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ + ":quic_core_qpack_qpack_static_table_lib", + ":quic_platform_base", + ":spdy_core_hpack_hpack_lib", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_instruction_decoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_instruction_decoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_instruction_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], + deps = [ + ":http2_hpack_huffman_hpack_huffman_decoder_lib", + ":http2_hpack_varint_hpack_varint_decoder_lib", ":quic_core_qpack_qpack_constants_lib", - ":quic_core_qpack_qpack_decoder_stream_sender_lib", - ":quic_core_qpack_qpack_encoder_stream_receiver_lib", - ":quic_core_qpack_qpack_header_table_lib", - ":quic_core_qpack_qpack_instruction_decoder_lib", - ":quic_core_types_lib", ":quic_platform_base", ], ) envoy_cc_library( - name = "quic_core_qpack_qpack_decoder_lib", - srcs = ["quiche/quic/core/qpack/qpack_decoder.cc"], - hdrs = ["quiche/quic/core/qpack/qpack_decoder.h"], + name = "quic_core_qpack_qpack_instruction_encoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_instruction_encoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_instruction_encoder.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], + deps = [ + ":http2_hpack_huffman_hpack_huffman_encoder_lib", + ":http2_hpack_varint_hpack_varint_encoder_lib", + ":quic_core_qpack_qpack_constants_lib", + ":quic_platform", + ], +) + +envoy_cc_library( + name = "quic_core_qpack_qpack_progressive_decoder_lib", + srcs = ["quiche/quic/core/qpack/qpack_progressive_decoder.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_progressive_decoder.h"], + copts = quiche_copt, + repository = "@envoy", + tags = ["nofips"], deps = [ + ":quic_core_qpack_qpack_constants_lib", ":quic_core_qpack_qpack_decoder_stream_sender_lib", ":quic_core_qpack_qpack_encoder_stream_receiver_lib", ":quic_core_qpack_qpack_header_table_lib", - ":quic_core_qpack_qpack_progressive_decoder_lib", + ":quic_core_qpack_qpack_instruction_decoder_lib", + ":quic_core_qpack_qpack_required_insert_count_lib", ":quic_core_types_lib", ":quic_platform_base", ], ) envoy_cc_library( - name = "quic_core_qpack_qpack_header_table_lib", - srcs = ["quiche/quic/core/qpack/qpack_header_table.cc"], - hdrs = ["quiche/quic/core/qpack/qpack_header_table.h"], + name = "quic_core_qpack_qpack_required_insert_count_lib", + srcs = ["quiche/quic/core/qpack/qpack_required_insert_count.cc"], + hdrs = ["quiche/quic/core/qpack/qpack_required_insert_count.h"], copts = quiche_copt, repository = "@envoy", - deps = [ - ":quic_core_qpack_qpack_static_table_lib", - ":quic_platform_base", - ":spdy_core_hpack_hpack_lib", - ], + tags = ["nofips"], + deps = [":quic_platform_base"], ) envoy_cc_library( @@ -2365,6 +2471,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_utils.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_qpack_qpack_stream_sender_delegate_lib"], ) @@ -2374,6 +2481,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_encoder_stream_sender.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_qpack_qpack_constants_lib", ":quic_core_qpack_qpack_instruction_encoder_lib", @@ -2388,6 +2496,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_encoder_stream_receiver.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":http2_decoder_decode_buffer_lib", ":http2_decoder_decode_status_lib", @@ -2404,6 +2513,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_decoder_stream_sender.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_qpack_qpack_constants_lib", ":quic_core_qpack_qpack_instruction_encoder_lib", @@ -2419,6 +2529,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_decoder_stream_receiver.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":http2_decoder_decode_buffer_lib", ":http2_decoder_decode_status_lib", @@ -2436,6 +2547,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_static_table.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_platform_base", ":spdy_core_hpack_hpack_lib", @@ -2447,6 +2559,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_stream_receiver.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_platform_base"], ) @@ -2456,6 +2569,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_decoded_headers_accumulator.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_http_header_list_lib", ":quic_core_qpack_qpack_decoder_lib", @@ -2471,6 +2585,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/value_splitting_header_list.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_platform_base", ":spdy_core_header_block_lib", @@ -2502,6 +2617,7 @@ envoy_cc_library( hdrs = ["quiche/quic/core/qpack/qpack_stream_sender_delegate.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_platform_base"], ) @@ -2767,6 +2883,7 @@ envoy_cc_library( srcs = ["quiche/quic/core/quic_time.cc"], hdrs = ["quiche/quic/core/quic_time.h"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [":quic_platform_base"], ) @@ -2819,10 +2936,12 @@ envoy_cc_library( "quiche/quic/core/quic_types.h", ], copts = quiche_copt, + external_deps = ["ssl"], repository = "@envoy", tags = ["nofips"], visibility = ["//visibility:public"], deps = [ + ":quic_core_crypto_random_lib", ":quic_core_error_codes_lib", ":quic_core_time_lib", ":quic_platform_base", @@ -2915,6 +3034,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/quic_config_peer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_config_lib", ":quic_core_packets_lib", @@ -2928,6 +3048,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/quic_framer_peer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_crypto_encryption_lib", ":quic_core_framer_lib", @@ -2942,6 +3063,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/mock_clock.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_time_lib", ":quic_platform", @@ -2954,6 +3076,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/mock_random.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_crypto_random_lib"], ) @@ -2963,6 +3086,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/quic_packet_generator_peer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_packet_creator_lib", ":quic_core_packet_generator_lib", @@ -2976,6 +3100,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/quic_sent_packet_manager_peer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_congestion_control_congestion_control_interface_lib", ":quic_core_packets_lib", @@ -2990,6 +3115,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/simple_quic_framer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_crypto_encryption_lib", ":quic_core_framer_lib", @@ -3004,6 +3130,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/quic_stream_send_buffer_peer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_stream_send_buffer_lib"], ) @@ -3013,6 +3140,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/quic_stream_peer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_packets_lib", ":quic_core_session_lib", @@ -3043,6 +3171,7 @@ envoy_cc_test_library( copts = quiche_copt, external_deps = ["ssl"], repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_buffer_allocator_lib", ":quic_core_congestion_control_congestion_control_interface_lib", @@ -3086,6 +3215,7 @@ envoy_cc_test_library( hdrs = ["quiche/quic/test_tools/quic_unacked_packet_map_peer.h"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":quic_core_unacked_packet_map_lib"], ) @@ -3103,6 +3233,7 @@ envoy_cc_test_library( "quiche/epoll_server/platform/api/epoll_time.h", ], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:epoll_server_platform_impl_lib"], ) @@ -3124,6 +3255,7 @@ envoy_cc_test_library( }), copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":epoll_server_platform"], ) @@ -3135,6 +3267,7 @@ envoy_cc_library( "quiche/common/platform/api/quiche_unordered_containers.h", ], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = ["@envoy//source/extensions/quic_listeners/quiche/platform:quiche_common_platform_impl_lib"], ) @@ -3143,6 +3276,7 @@ envoy_cc_test_library( name = "quiche_common_platform_test", hdrs = ["quiche/common/platform/api/quiche_test.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quiche_common_platform_test_impl_lib"], ) @@ -3150,6 +3284,7 @@ envoy_cc_library( name = "quiche_common_lib", hdrs = ["quiche/common/simple_linked_hash_map.h"], repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = [":quiche_common_platform"], ) @@ -3162,6 +3297,7 @@ envoy_cc_test( }), copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [":epoll_server_lib"], ) @@ -3170,6 +3306,7 @@ envoy_cc_test( srcs = ["quiche/common/simple_linked_hash_map_test.cc"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quiche_common_lib", ":quiche_common_platform_test", @@ -3183,6 +3320,7 @@ envoy_cc_test( "quiche/http2/test_tools/http2_random_test.cc", ], repository = "@envoy", + tags = ["nofips"], deps = [ ":http2_platform", ":http2_test_tools_random", @@ -3193,6 +3331,7 @@ envoy_cc_test( name = "spdy_platform_api_test", srcs = ["quiche/spdy/platform/api/spdy_string_utils_test.cc"], repository = "@envoy", + tags = ["nofips"], deps = [ ":spdy_platform", ":spdy_platform_test", @@ -3206,6 +3345,7 @@ envoy_cc_library( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], visibility = ["//visibility:public"], deps = ["@envoy//source/extensions/quic_listeners/quiche/platform:quic_platform_mem_slice_span_impl_lib"], ) @@ -3214,6 +3354,7 @@ envoy_cc_test_library( name = "quic_platform_test_mem_slice_vector_lib", hdrs = ["quiche/quic/platform/api/quic_test_mem_slice_vector.h"], repository = "@envoy", + tags = ["nofips"], deps = ["@envoy//test/extensions/quic_listeners/quiche/platform:quic_platform_test_mem_slice_vector_impl_lib"], ) @@ -3230,6 +3371,7 @@ envoy_cc_test( srcs = ["quiche/spdy/core/spdy_header_block_test.cc"], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":spdy_core_header_block_lib", ":spdy_core_test_utils_lib", @@ -3250,6 +3392,7 @@ envoy_cc_test( ], copts = quiche_copt, repository = "@envoy", + tags = ["nofips"], deps = [ ":quic_core_buffer_allocator_lib", ":quic_platform", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 5dec6ea9bb10..6a28c7026cca 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -243,9 +243,9 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/curl/curl/releases/download/curl-7_65_3/curl-7.65.3.tar.gz"], ), com_googlesource_quiche = dict( - # Static snapshot of https://quiche.googlesource.com/quiche/+archive/2a930469533c3b541443488a629fe25cd8ff53d0.tar.gz - sha256 = "fcdebf54c89d839ffa7eefae166c8e4b551c765559db13ff15bff98047f344fb", - urls = ["https://storage.googleapis.com/quiche-envoy-integration/2a930469533c3b541443488a629fe25cd8ff53d0.tar.gz"], + # Static snapshot of https://quiche.googlesource.com/quiche/+archive/4abb566fbbc63df8fe7c1ac30b21632b9eb18d0c.tar.gz + sha256 = "c60bca3cf7f58b91394a89da96080657ff0fbe4d5675be9b21e90da8f68bc06f", + urls = ["https://storage.googleapis.com/quiche-envoy-integration/4abb566fbbc63df8fe7c1ac30b21632b9eb18d0c.tar.gz"], ), com_google_cel_cpp = dict( sha256 = "f027c551d57d38fb9f0b5e4f21a2b0b8663987119e23b1fd8dfcc7588e9a2350", diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index b4b3369fe9bf..c71bb49c6f35 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -40,6 +40,7 @@ envoy_cc_library( srcs = ["envoy_quic_packet_writer.cc"], hdrs = ["envoy_quic_packet_writer.h"], external_deps = ["quiche_quic_platform"], + tags = ["nofips"], deps = [ ":envoy_quic_utils_lib", "@com_googlesource_quiche//:quic_core_packet_writer_interface_lib", @@ -71,6 +72,7 @@ envoy_cc_library( envoy_cc_library( name = "spdy_server_push_utils_for_envoy_lib", srcs = ["spdy_server_push_utils_for_envoy.cc"], + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ "//source/common/common:assert_lib", @@ -82,5 +84,6 @@ envoy_cc_library( name = "envoy_quic_utils_lib", hdrs = ["envoy_quic_utils.h"], external_deps = ["quiche_quic_platform"], + tags = ["nofips"], deps = ["//source/common/network:address_lib"], ) diff --git a/source/extensions/quic_listeners/quiche/platform/BUILD b/source/extensions/quic_listeners/quiche/platform/BUILD index 08c42a1364a8..8fc764386a2f 100644 --- a/source/extensions/quic_listeners/quiche/platform/BUILD +++ b/source/extensions/quic_listeners/quiche/platform/BUILD @@ -94,6 +94,7 @@ envoy_cc_library( envoy_cc_library( name = "quic_platform_export_impl_lib", hdrs = ["quic_export_impl.h"], + tags = ["nofips"], visibility = ["//visibility:public"], ) @@ -104,6 +105,7 @@ envoy_cc_library( "quic_bug_tracker_impl.h", "quic_logging_impl.h", ], + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ "//source/common/common:assert_lib", @@ -153,6 +155,7 @@ envoy_cc_library( "abseil_optional", "googletest", ], + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ ":flags_impl_lib", @@ -191,6 +194,7 @@ envoy_cc_library( "abseil_time", "ssl", ], + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ "//source/common/common:assert_lib", @@ -205,6 +209,7 @@ envoy_cc_library( srcs = ["quic_mem_slice_span_impl.cc"], hdrs = ["quic_mem_slice_span_impl.h"], copts = ["-Wno-unused-parameter"], + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ "//include/envoy/buffer:buffer_interface", @@ -222,6 +227,7 @@ envoy_cc_library( "-Wno-error=invalid-offsetof", "-Wno-unused-parameter", ], + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ "@com_googlesource_quiche//:quic_core_buffer_allocator_lib", @@ -233,6 +239,7 @@ envoy_cc_library( envoy_cc_library( name = "quic_platform_bbr2_sender_impl_lib", hdrs = ["quic_bbr2_sender_impl.h"], + tags = ["nofips"], visibility = ["//visibility:public"], deps = ["@com_googlesource_quiche//:quic_core_congestion_control_bbr_lib"], ) @@ -241,6 +248,7 @@ envoy_cc_library( name = "envoy_quic_clock_lib", srcs = ["envoy_quic_clock.cc"], hdrs = ["envoy_quic_clock.h"], + tags = ["nofips"], visibility = ["//visibility:public"], deps = [ "//include/envoy/event:dispatcher_interface", @@ -277,6 +285,7 @@ envoy_cc_library( "spdy_flags_impl.h", "spdy_logging_impl.h", "spdy_macros_impl.h", + "spdy_map_util_impl.h", "spdy_mem_slice_impl.h", "spdy_ptr_util_impl.h", "spdy_string_impl.h", diff --git a/source/extensions/quic_listeners/quiche/platform/flags_list.h b/source/extensions/quic_listeners/quiche/platform/flags_list.h index 31ba6c3686f5..1b0defc6824d 100644 --- a/source/extensions/quic_listeners/quiche/platform/flags_list.h +++ b/source/extensions/quic_listeners/quiche/platform/flags_list.h @@ -1,5 +1,5 @@ // This file intentionally does not have header guards, it's intended to be -// included multiple times, each time with a different definition of QUIC_FLAG. +// included multiple times, each time with a different definition of QUICHE_FLAG. // NOLINT(namespace-envoy) @@ -18,24 +18,33 @@ QUICHE_FLAG(bool, quic_reloadable_flag_advertise_quic_for_https_for_debugips, fa QUICHE_FLAG(bool, quic_reloadable_flag_advertise_quic_for_https_for_external_users, false, "") -QUICHE_FLAG(bool, quic_reloadable_flag_enable_quic_stateless_reject_support, true, - "Enables server-side support for QUIC stateless rejects.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_active_streams_never_negative, false, + "If true, static streams should never be closed before QuicSession " + "destruction.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_add_upper_limit_of_buffered_control_frames, false, + "If true, close connection if there are too many (> 1000) buffered " + "control frames.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_aggressive_connection_aliveness, false, + "If true, QuicSession::ShouldKeepConnectionAlive() will not consider " + "locally closed streams whose highest byte offset is not received yet.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_allow_backend_set_stream_ttl, false, "If true, check backend response header for X-Response-Ttl. If it is " "provided, the stream TTL is set. A QUIC stream will be immediately " "canceled when tries to write data if this TTL expired.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_allow_client_enabled_bbr_v2, false, + "If true, allow client to enable BBRv2 on server via connection " + "option 'B2ON'.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_alpn_dispatch, false, "Support different QUIC sessions, as indicated by ALPN. Used for QBONE.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_always_reset_short_header_packets, true, - "If true, instead of send encryption none termination packets, send " - "stateless reset in response to short headers.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_app_limited_recovery, false, - "When you're app-limited entering recovery, stay app-limited until " - "you exit recovery in QUIC BBR.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_avoid_empty_frame_after_empty_headers, true, + "If enabled, do not call OnStreamFrame() with empty frame after " + "receiving empty or too large headers with FIN.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_flexible_app_limited, false, "When true and the BBR9 connection option is present, BBR only considers " @@ -60,15 +69,29 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_bbr_startup_rate_reduction, false, "When true, enables the BBS4 and BBS5 connection options, which reduce " "BBR's pacing rate in STARTUP as more losses occur as a fraction of CWND.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_change_default_lumpy_pacing_size_to_two, false, + "If true and --quic_lumpy_pacing_size is 1, QUIC will use a lumpy " + "size of two for pacing.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_clear_queued_packets_on_connection_close, false, + "Calls ClearQueuedPackets after sending a connection close packet") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_bursts, false, + "If true, set burst token to 2 in cwnd bootstrapping experiment.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_cwnd_and_pacing_gains, false, + "If true, uses conservative cwnd gain and pacing gain when cwnd gets " + "bootstrapped.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_debug_wrong_qos, false, "If true, consider getting QoS after stream has been detached as GFE bug.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_default_to_bbr, true, "When true, defaults to BBR congestion control instead of Cubic.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_deprecate_ack_bundling_mode, false, - "If true, stop using AckBundling mode to send ACK, also deprecate " - "ack_queued from QuicConnection.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_default_to_bbr_v2, false, + "If true, use BBRv2 as the default congestion controller. Takes " + "precedence over --quic_default_to_bbr.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_connection_migration_for_udp_proxying, true, "If true, GFE disables connection migration in connection option for " @@ -77,9 +100,21 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_connection_migration_for_udp QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_39, false, "If true, disable QUIC version 39.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_44, true, + "If true, disable QUIC version 44.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_do_not_accept_stop_waiting, false, + "In v44 and above, where STOP_WAITING is never sent, close the " + "connection if it's received.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_donot_reset_ideal_next_packet_send_time, false, "If true, stop resetting ideal_next_packet_send_time_ in pacing sender.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_drop_invalid_small_initial_connection_id, true, + "When true, QuicDispatcher will drop packets that have an initial " + "destination connection ID that is too short, instead of responding " + "with a Version Negotiation packet to reject it.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_eighth_rtt_loss_detection, false, "When true, the LOSS connection option allows for 1/8 RTT of " "reording instead of the current 1/8th threshold which has been " @@ -89,49 +124,57 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_ack_decimation, false, "Default enables QUIC ack decimation and adds a connection option to " "disable it.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_pcc3, false, - "If true, enable experiment for testing PCC congestion-control.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_43, true, - "If true, enable QUIC version 43.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_fifo_write_scheduler, false, + "If true and FIFO connection option is received, write_blocked_streams " + "uses FIFO(stream with smallest ID has highest priority) write scheduler.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_44, true, - "If true, enable version 44 which uses IETF header format.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_lifo_write_scheduler, false, + "If true and LIFO connection option is received, write_blocked_streams " + "uses LIFO(stream with largest ID has highest priority) write scheduler.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_46, true, - "If true, enable QUIC version 46.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_pcc3, false, + "If true, enable experiment for testing PCC congestion-control.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_47, false, "If true, enable QUIC version 47 which adds support for variable " "length connection IDs.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_48, false, + "If true, enable QUIC version 48.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_99, false, "If true, enable version 99.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_enabled, false, "") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_faster_interval_add_in_sequence_buffer, false, - "If true, QuicStreamSequencerBuffer will switch to a new " - "QuicIntervalSet::AddOptimizedForAppend method in OnStreamData().") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_adaptive_time_loss, false, - "Simplify QUIC's adaptive time loss detection to measure the " + "Simplify QUICHE's adaptive time loss detection to measure the " "necessary reordering window for every spurious retransmit.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_has_pending_crypto_data, false, - "If true, QuicSession::HasPendingCryptoData checks whether the " - "crypto stream's send buffer is empty. This flag fixes a bug where " - "the retransmission alarm mode is wrong for the first CHLO packet.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_bbr_cwnd_in_bandwidth_resumption, true, + "If true, adjust congestion window when doing bandwidth resumption in BBR.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_get_packet_header_size, false, + "Fixes quic::GetPacketHeaderSize and callsites when " + "QuicVersionHasLongHeaderLengths is false.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_packets_acked, true, + "If true, when detecting losses, use packets_acked of corresponding " + "packet number space.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_spurious_ack_alarm, false, - "If true, do not schedule ack alarm if should_send_ack is set in the " - "generator.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_rto_retransmission2, false, + "If true, when RTO fires and there is no packet to be RTOed, let " + "connection send.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_termination_packets, false, - "If true, GFE time wait list will send termination packets based on " - "current packet's encryption level.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_handle_staticness_for_spdy_stream, false, + "If true, QuicSpdySession::GetSpdyDataStream() will close the " + "connection if the returned stream is static.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_ignore_tlpr_if_no_pending_stream_data, false, + "If true, ignore TLPR if there is no pending stream data") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_limit_window_updates_in_traces, false, - "Limits the number of window update events recorded in Tracegraf logs.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_inline_getorcreatedynamicstream, false, + "If true, QuicSession::GetOrCreateDynamicStream() is deprecated, and " + "its contents are moved to GetOrCreateStream().") QUICHE_FLAG(bool, quic_reloadable_flag_quic_listener_never_fake_epollout, false, "If true, QuicListener::OnSocketIsWritable will always return false, " @@ -141,20 +184,16 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_listener_never_fake_epollout, false, QUICHE_FLAG(bool, quic_reloadable_flag_quic_log_cert_name_for_empty_sct, true, "If true, log leaf cert subject name into warning log.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_log_is_proxy_in_tcs, false, - "If true, log whether a GFE QUIC server session is UDP proxied and whether " - "it is a health check connection, in transport connection stats.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_logging_frames_in_tracegraf, false, - "If true, populate frames info when logging tracegraf.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_loss_removes_from_inflight, true, + "When true, remove packets from inflight where they're declared " + "lost, rather than in MarkForRetransmission. Also no longer marks " + "handshake packets as no longer inflight when they're retransmitted.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_monotonic_epoll_clock, false, "If true, QuicEpollClock::Now() will monotonically increase.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_client_conn_ver_negotiation, false, - "If true, a client connection would be closed when a version " - "negotiation packet is received. It would be the higher layer's " - "responsibility to do the reconnection.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_negotiate_ack_delay_time, false, + "If true, will negotiate the ACK delay time.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_cloud_domain_sni_lookup_on_missing_sni, false, "Do not attempt to match an empty Server Name Indication (SNI) " @@ -164,68 +203,66 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_dup_experiment_id_2, false, "If true, transport connection stats doesn't report duplicated " "experiments for same connection.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_goaway_for_proxied_port_change, false, - "If true, for proxied quic sessions, GFE will not send a GOAWAY in " - "response to a client port change.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_stream_data_after_reset, false, + "If true, QuicStreamSequencer will not take in new data if the stream is reset.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_v2_scaling_factor, false, "When true, don't use an extra scaling factor when reading packets " - "from QUIC's RX_RING with TPACKET_V2.") + "from QUICHE's RX_RING with TPACKET_V2.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_optimize_inflight_check, false, - "Stop checking QuicUnackedPacketMap::HasUnackedRetransmittableFrames " - "and instead rely on the existing check that bytes_in_flight > 0") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_window_update_on_read_only_stream, false, + "If true, QuicConnection will be closed if a WindowUpdate frame is " + "received on a READ_UNIDIRECTIONAL stream.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_check_toss_on_insertion_failure, false, "If true, enable the code that fixes a race condition for quic udp " "proxying in L0. See b/70036019.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_munge_response_for_healthcheck, true, - "If true, for udp proxy, the health check packets from L1 to L0 will " - "be munged.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_read_packed_strings, true, "If true, QuicProxyDispatcher will prefer to extract client_address " "and server_vip from packed_client_address and packed_server_vip, " "respectively.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_supports_length_prefix, false, + "When true, QuicProxyUtils::GetConnectionId supports length prefixed " + "connection IDs.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_proxy_write_packed_strings, false, "If true, QuicProxyDispatcher will write packed_client_address and " "packed_server_vip in TcpProxyHeaderProto.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_length_prefix_from_packet_info, false, + "When true, QuicDispatcher::MaybeDispatchPacket will use packet_info.use_length_prefix " + "instead of an incorrect local computation.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_record_frontend_service_vip_mapping, false, "If true, for L1 GFE, as requests come in, record frontend service to VIP " "mapping which is used to announce VIP in SHLO for proxied sessions. ") QUICHE_FLAG(bool, quic_reloadable_flag_quic_reject_all_traffic, false, "") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_reject_unprocessable_packets_statelessly, false, + "If true, do not add connection ID of packets with unknown connection ID " + "and no version to time wait list, instead, send appropriate responses " + "depending on the packets' sizes and drop them.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_require_handshake_confirmation, false, "If true, require handshake confirmation for QUIC connections, " "functionally disabling 0-rtt handshakes.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_rpm_decides_when_to_send_acks, false, - "If both this flag and " - "gfe2_reloadable_flag_quic_deprecate_ack_bundling_mode are true, " - "QuicReceivedPacketManager decides when to send ACKs.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_send_timestamps, false, "When the STMP connection option is sent by the client, timestamps " "in the QUIC ACK frame are sent and processed.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_server_push, true, - "If true, enable server push feature on QUIC.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_set_transmission_type_for_next_frame, true, - "If true, QuicPacketCreator::SetTransmissionType will set the " - "transmission type of the next successfully added frame.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_sent_packet_manager_cleanup, false, + "When true, remove obsolete functionality intended to test IETF QUIC " + "recovery.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_simplify_build_connectivity_probing_packet, true, - "If true, simplifies the implementation of " - "QuicFramer::BuildConnectivityProbingPacket().") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_server_push, true, + "If true, enable server push feature on QUICHE.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_stateless_proxy, false, - "If true, UDP proxy will not drop versionless packets, in other " - "words, it will proxy all packets from client.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_simplify_stop_waiting, true, + "If true, do not send STOP_WAITING if no_stop_waiting_frame_.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_stop_reading_when_level_triggered, false, "When true, calling StopReading() on a level-triggered QUIC stream " @@ -237,31 +274,37 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_testonly_default_false, false, QUICHE_FLAG(bool, quic_reloadable_flag_quic_testonly_default_true, true, "A testonly reloadable flag that will always default to true.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_tolerate_reneging, false, - "If true, do not close connection if received largest acked decreases.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_tracegraf_populate_ack_packet_number, false, + "If true, populate packet_number of received ACK in tracegraf.") + +QUICHE_FLAG(bool, quic_reloadable_flag_quic_track_ack_height_in_bandwidth_sampler, false, + "If true, QUIC will track max ack height in BandwidthSampler.") QUICHE_FLAG(bool, quic_reloadable_flag_quic_unified_iw_options, false, "When true, set the initial congestion control window from connection " "options in QuicSentPacketManager rather than TcpCubicSenderBytes.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_cheap_stateless_rejects, true, - "If true, QUIC will use cheap stateless rejects without creating a full " - "connection. Prerequisite: --quic_allow_chlo_buffering has to be true.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_common_stream_check, false, "If true, use common code for checking whether a new stream ID may " "be allocated.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_connection_clock_for_last_ack_time, false, + "If true, QuicFasterStatsGatherer will use a GFEConnectionClock to " + "get the time when the last ack is received.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_header_stage_idle_list2, false, "If true, use header stage idle list for QUIC connections in GFE.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_http2_priority_write_scheduler, false, + "If true and H2PR connection option is received, write_blocked_streams_ " + "uses HTTP2 (tree-style) priority write scheduler.") + QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_leto_key_exchange, false, "If true, QUIC will attempt to use the Leto key exchange service and " "only fall back to local key exchange if that fails.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_new_append_connection_id, false, - "When true QuicFramer will use AppendIetfConnectionIdsNew instead of " - "AppendIetfConnectionId.") +QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_parse_public_header, false, + "When true, QuicDispatcher will always use QuicFramer::ParsePublicHeader") QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_pigeon_sockets, false, "Use USPS Direct Path for QUIC egress.") @@ -270,103 +313,40 @@ QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_quic_time_for_received_timestamp "If true, use QuicClock::Now() for the fallback source of packet " "received time instead of WallNow().") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_uber_loss_algorithm, false, - "If true, use one loss algorithm per encryption level.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_use_uber_received_packet_manager, false, - "If this flag and quic_rpm_decides_when_to_send_acks is true, use uber " - "received packet manager instead of the single received packet manager.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_validate_packet_number_post_decryption, false, - "If true, a QUIC endpoint will valid a received packet number after " - "successfully decrypting the packet.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_eliminate_static_stream_map_3, false, - "If true, static streams in a QuicSession will be stored inside dynamic stream map. " - "static_stream_map will no longer be used.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_simplify_stop_waiting, false, - "Do not send STOP_WAITING if no_stop_waiting_frame_ is true.") - -QUICHE_FLAG(bool, quic_reloadable_flag_send_quic_fallback_server_config_on_leto_error, false, - "If true and using Leto for QUIC shared-key calculations, GFE will react to a failure " - "to contact Leto by sending a REJ containing a fallback ServerConfig, allowing the " - "client to continue the handshake.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_fix_bbr_cwnd_in_bandwidth_resumption, true, - " If true, adjust congestion window when doing bandwidth resumption in BBR.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_cwnd_and_pacing_gains, false, - "If true, uses conservative cwnd gain and pacing gain.") - -QUICHE_FLAG( - bool, quic_reloadable_flag_quic_do_not_accept_stop_waiting, false, - "In v44 and above, where STOP_WAITING is never sent, close the connection if it's received.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_loss_removes_from_inflight, false, - "When true, remove packets from inflight where they're declared lost, rather than in " - "MarkForRetransmission. Also no longer marks handshake packets as no longer inflight " - "when they're retransmitted.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_conservative_bursts, false, - "If true, set burst token to 2 in cwnd bootstrapping experiment.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_deprecate_queued_control_frames, false, - "If true, deprecate queued_control_frames_ from QuicPacketGenerator.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_check_connected_before_flush, false, - "If true, check whether connection is connected before flush.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_ignore_tlpr_if_sending_ping, false, - "If true, ignore TLPR for retransmission delay when sending pings from ping alarm.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_terminate_gquic_connection_as_ietf, false, - "If true, terminate Google QUIC connections similarly as IETF QUIC.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_disable_version_44, false, - "If true, disable QUIC version 44.") - -QUICHE_FLAG( - bool, quic_reloadable_flag_quic_fix_packets_acked, false, - "If true, when detecting losses, use packets_acked of corresponding packet number space.") - -QUICHE_FLAG(bool, quic_reloadable_flag_quic_ignore_tlpr_if_no_pending_stream_data, false, - "If true, ignore TLPR if there is no pending stream data") - -QUICHE_FLAG( - bool, quic_reloadable_flag_quic_drop_invalid_small_initial_connection_id, false, - "When true, QuicDispatcher will drop packets that have an initial destination connection ID " - "that is too short, instead of responding with a Version Negotiation packet to reject it.") - QUICHE_FLAG(bool, quic_reloadable_flag_quic_version_negotiation_grease, false, - "When true, QUIC Version Negotiation packets will randomly include fake versions.") - -QUICHE_FLAG( - bool, quic_reloadable_flag_quic_fix_get_packet_header_size, false, - "Fixes quic::GetPacketHeaderSize and callsites when QuicVersionHasLongHeaderLengths is false.") - -QUICHE_FLAG( - bool, quic_reloadable_flag_quic_change_default_lumpy_pacing_size_to_two, false, - "If true and --quic_lumpy_pacing_size is 1, QUIC will use a lumpy size of two for pacing.") + "When true, QUIC Version Negotiation packets will randomly include " + "fake versions.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_no_window_update_on_read_only_stream, false, - "If true, QuicConnection will be closed if a WindowUpdate frame is received on a " - "READ_UNIDIRECTIONAL stream.") +QUICHE_FLAG(bool, quic_reloadable_flag_send_quic_fallback_server_config_on_leto_error, false, + "If true and using Leto for QUIC shared-key calculations, GFE will react " + "to a failure to contact Leto by sending a REJ containing a fallback " + "ServerConfig, allowing the client to continue the handshake.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_clear_queued_packets_on_connection_close, false, - "Calls ClearQueuedPackets after sending a connection close packet") +QUICHE_FLAG(bool, quic_reloadable_flag_simplify_spdy_quic_https_scheme_detection, false, + "If true, simplify the logic for detecting REQUEST_HAS_HTTPS_SCHEME in " + "NetSpdyRequester::SetRequestUrlAndHost and " + "NetQuicRequester::SetRequestUrlAndHost. Fixes a bug where internal " + "redirects for QUIC connections would be treated as having an http scheme.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_enable_version_48, false, - "If true, enable QUIC version 48.") +QUICHE_FLAG(bool, quic_restart_flag_do_not_create_raw_socket_selector_if_quic_enabled, false, + "If true, do not create the RawSocketSelector in " + "QuicListener::Initialize() if QUIC is disabled by flag.") -QUICHE_FLAG(bool, quic_reloadable_flag_quic_avoid_empty_frame_after_empty_headers, false, - "If enabled, do not call OnStreamFrame() with empty frame after receiving empty or too " - "large headers with FIN.") +QUICHE_FLAG(bool, quic_restart_flag_dont_fetch_quic_private_keys_from_leto, false, + "If true, GFE will not request private keys when fetching QUIC " + "ServerConfigs from Leto.") QUICHE_FLAG(bool, quic_restart_flag_quic_allow_loas_multipacket_chlo, false, "If true, inspects QUIC CHLOs for kLOAS and early creates sessions " "to allow multi-packet CHLOs") +QUICHE_FLAG(bool, quic_restart_flag_quic_connection_id_use_siphash, false, + "When true, QuicConnectionId::Hash uses SipHash instead of XOR.") + +QUICHE_FLAG(bool, quic_restart_flag_quic_dispatcher_hands_chlo_extractor_one_version, false, + "When true, QuicDispatcher will pass the version from the packet to " + "the ChloExtractor instead of all supported versions.") + QUICHE_FLAG(bool, quic_restart_flag_quic_enable_gso_for_udp_egress, false, "If 1) flag is true, 2) UDP egress_method is used in GFE config, and " "3) UDP GSO is supported by the kernel, GFE will use UDP GSO for " @@ -378,9 +358,8 @@ QUICHE_FLAG(bool, quic_restart_flag_quic_enable_sendmmsg_for_udp_egress, false, "gso is not supported by kernel, GFE will use sendmmsg for egress, " "except for UDP proxy.") -QUICHE_FLAG(bool, quic_restart_flag_quic_no_server_conn_ver_negotiation2, false, - "If true, dispatcher passes in a single version when creating a server " - "connection, such that version negotiation is not supported in connection.") +QUICHE_FLAG(bool, quic_restart_flag_quic_no_fallback_for_pigeon_socket, false, + "If true, GFEs using USPS egress will not fallback to raw ip socket.") QUICHE_FLAG(bool, quic_restart_flag_quic_offload_pacing_to_usps2, false, "If true, QUIC offload pacing when using USPS as egress method.") @@ -402,6 +381,10 @@ QUICHE_FLAG(bool, quic_restart_flag_quic_testonly_default_false, false, QUICHE_FLAG(bool, quic_restart_flag_quic_testonly_default_true, true, "A testonly restart flag that will always default to true.") +QUICHE_FLAG(bool, quic_restart_flag_quic_use_allocated_connection_ids, true, + "When true, QuicConnectionId will allocate long connection IDs on " + "the heap instead of inline in the object.") + QUICHE_FLAG(bool, quic_restart_flag_quic_use_leto_for_quic_configs, false, "If true, use Leto to fetch QUIC server configs instead of using the " "seeds from Memento.") @@ -410,27 +393,12 @@ QUICHE_FLAG(bool, quic_restart_flag_quic_use_pigeon_socket_to_backend, false, "If true, create a shared pigeon socket for all quic to backend " "connections and switch to use it after successful handshake.") -QUICHE_FLAG(bool, quic_restart_flag_quic_do_not_override_connection_id, false, - " When true, QuicFramer will not override connection IDs in headers and will instead " - "respect the source/destination direction as expected by IETF QUIC.") - -QUICHE_FLAG(bool, quic_restart_flag_quic_no_framer_object_in_dispatcher, false, - "If true, make QuicDispatcher no longer have an instance of QuicFramer.") - -QUICHE_FLAG( - bool, quic_restart_flag_dont_fetch_quic_private_keys_from_leto, false, - "If true, GFE will not request private keys when fetching QUIC ServerConfigs from Leto.") - -QUICHE_FLAG(bool, quic_restart_flag_quic_use_allocated_connection_ids, false, - "When true, QuicConnectionId will allocate long connection IDs on the heap instead of " - "inline in the object.") - QUICHE_FLAG(bool, quic_allow_chlo_buffering, true, "If true, allows packets to be buffered in anticipation of a " "future CHLO, and allow CHLO packets to be buffered until next " "iteration of the event loop.") -QUICHE_FLAG(bool, quic_disable_pacing_for_perf_tests, false, "If true, disable pacing in QUIC") +QUICHE_FLAG(bool, quic_disable_pacing_for_perf_tests, false, "If true, disable pacing in QUICHE") QUICHE_FLAG(bool, quic_enforce_single_packet_chlo, true, "If true, enforce that QUIC CHLOs fit in one packet") @@ -440,54 +408,86 @@ QUICHE_FLAG(bool, quic_enforce_single_packet_chlo, true, // 200 seconds * 1000 qps = 200000. // Of course, there are usually many queries per QUIC connection, so we allow a // factor of 3 leeway. -QUICHE_FLAG(int64_t, quic_time_wait_list_max_connections, 600000, +QUICHE_FLAG(int64_t, // allow-non-std-int + quic_time_wait_list_max_connections, 600000, "Maximum number of connections on the time-wait list. " "A negative value implies no configured limit.") -QUICHE_FLAG(int64_t, quic_time_wait_list_seconds, 200, +QUICHE_FLAG(int64_t, // allow-non-std-int + quic_time_wait_list_seconds, 200, "Time period for which a given connection_id should live in " "the time-wait state.") QUICHE_FLAG(double, quic_bbr_cwnd_gain, 2.0f, "Congestion window gain for QUIC BBR during PROBE_BW phase.") -QUICHE_FLAG(int32_t, quic_buffered_data_threshold, 8 * 1024, +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_buffered_data_threshold, 8 * 1024, "If buffered data in QUIC stream is less than this " "threshold, buffers all provided data or asks upper layer for more data") -QUICHE_FLAG(int32_t, quic_ietf_draft_version, 0, - "Mechanism to override version label and ALPN for IETF interop.") - -QUICHE_FLAG(int32_t, quic_send_buffer_max_data_slice_size, 4 * 1024, +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_send_buffer_max_data_slice_size, 4 * 1024, "Max size of data slice in bytes for QUIC stream send buffer.") QUICHE_FLAG(bool, quic_supports_tls_handshake, false, "If true, QUIC supports both QUIC Crypto and TLS 1.3 for the " "handshake protocol") -QUICHE_FLAG(int32_t, quic_lumpy_pacing_size, 1, +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_lumpy_pacing_size, 1, "Number of packets that the pacing sender allows in bursts during pacing.") QUICHE_FLAG(double, quic_lumpy_pacing_cwnd_fraction, 0.25f, "Congestion window fraction that the pacing sender allows in bursts " "during pacing.") -QUICHE_FLAG(int32_t, quic_max_pace_time_into_future_ms, 10, +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_max_pace_time_into_future_ms, 10, "Max time that QUIC can pace packets into the future in ms.") QUICHE_FLAG(double, quic_pace_time_into_future_srtt_fraction, 0.125f, // One-eighth smoothed RTT "Smoothed RTT fraction that a connection can pace packets into the future.") +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_ietf_draft_version, 0, + "Mechanism to override version label and ALPN for IETF interop.") + QUICHE_FLAG(bool, quic_export_server_num_packets_per_write_histogram, false, "If true, export number of packets written per write operation histogram.") -QUICHE_FLAG(int64_t, quic_headers_stream_id_in_v99, 0, - "QUIC version 99 will use this stream ID for the headers stream.") - QUICHE_FLAG(bool, quic_disable_version_negotiation_grease_randomness, false, "If true, use predictable version negotiation versions.") +QUICHE_FLAG(int64_t, // allow-non-std-int + quic_max_tracked_packet_count, 10000, "Maximum number of tracked packets.") + +QUICHE_FLAG(bool, quic_prober_uses_length_prefixed_connection_ids, false, + "If true, QuicFramer::WriteClientVersionNegotiationProbePacket uses " + "length-prefixed connection IDs.") + +QUICHE_FLAG(bool, quic_client_convert_http_header_name_to_lowercase, true, + "If true, HTTP request header names sent from QuicSpdyClientBase(and " + "descendents) will be automatically converted to lower case.") + +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_bbr2_default_probe_bw_base_duration_ms, 2000, + "The default minimum duration for BBRv2-native probes, in milliseconds.") + +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_bbr2_default_probe_bw_max_rand_duration_ms, 1000, + "The default upper bound of the random amount of BBRv2-native " + "probes, in milliseconds.") + +QUICHE_FLAG(int32_t, // allow-non-std-int + quic_bbr2_default_probe_rtt_period_ms, 10000, + "The default period for entering PROBE_RTT, in milliseconds.") + +QUICHE_FLAG(double, quic_bbr2_default_loss_threshold, 0.02, + "The default loss threshold for QUIC BBRv2, should be a value " + "between 0 and 1.") + QUICHE_FLAG(bool, http2_reloadable_flag_http2_testonly_default_false, false, "A testonly reloadable flag that will always default to false.") diff --git a/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_storage_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_storage_impl.h index 90c59db2426d..3a43ec7316e1 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_storage_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_mem_slice_storage_impl.h @@ -36,6 +36,8 @@ class QuicMemSliceStorageImpl { QuicMemSliceSpan ToSpan() { return QuicMemSliceSpan(QuicMemSliceSpanImpl(buffer_)); } + void Append(QuicMemSliceImpl mem_slice) { buffer_.move(mem_slice.single_slice_buffer()); } + private: Envoy::Buffer::OwnedImpl buffer_; }; diff --git a/source/extensions/quic_listeners/quiche/platform/quic_text_utils_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_text_utils_impl.h index 42bb24e6828a..e39508adbb29 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_text_utils_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_text_utils_impl.h @@ -69,6 +69,10 @@ class QuicTextUtilsImpl { return std::any_of(data.begin(), data.end(), absl::ascii_isupper); } + static bool IsAllDigits(QuicStringPieceImpl data) { + return std::all_of(data.begin(), data.end(), absl::ascii_isdigit); + } + static std::vector Split(QuicStringPieceImpl data, char delim) { return absl::StrSplit(data, delim); } diff --git a/source/extensions/quic_listeners/quiche/platform/spdy_containers_impl.h b/source/extensions/quic_listeners/quiche/platform/spdy_containers_impl.h index 57d953c939d3..35d08c6183bf 100644 --- a/source/extensions/quic_listeners/quiche/platform/spdy_containers_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/spdy_containers_impl.h @@ -37,4 +37,6 @@ inline size_t SpdyHashStringPairImpl(SpdyStringPieceImpl a, SpdyStringPieceImpl return absl::Hash>()(std::make_pair(a, b)); } +template +using SpdySmallMapImpl = absl::flat_hash_map; } // namespace spdy diff --git a/source/extensions/quic_listeners/quiche/platform/spdy_map_util_impl.h b/source/extensions/quic_listeners/quiche/platform/spdy_map_util_impl.h new file mode 100644 index 000000000000..befd8c7f8c6f --- /dev/null +++ b/source/extensions/quic_listeners/quiche/platform/spdy_map_util_impl.h @@ -0,0 +1,18 @@ +#pragma once + +// NOLINT(namespace-envoy) + +// This file is part of the QUICHE platform implementation, and is not to be +// consumed or referenced directly by other Envoy code. It serves purely as a +// porting layer for QUICHE. + +#include + +namespace spdy { + +template +bool SpdyContainsKeyImpl(const Collection& collection, const Key& key) { + return collection.find(key) != collection.end(); +} + +} // namespace spdy diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index 313ef1d0e808..b588a5c069fa 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -30,6 +30,7 @@ envoy_cc_test( name = "envoy_quic_writer_test", srcs = ["envoy_quic_writer_test.cc"], external_deps = ["quiche_quic_platform"], + tags = ["nofips"], deps = [ "//source/common/network:io_socket_error_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_packet_writer_lib", diff --git a/test/extensions/quic_listeners/quiche/platform/BUILD b/test/extensions/quic_listeners/quiche/platform/BUILD index 1383cfbfbb17..09ef037677b0 100644 --- a/test/extensions/quic_listeners/quiche/platform/BUILD +++ b/test/extensions/quic_listeners/quiche/platform/BUILD @@ -34,6 +34,7 @@ envoy_cc_test( copts = ["-Wno-unused-parameter"], data = ["//test/extensions/transport_sockets/tls/test_data:certs"], external_deps = ["quiche_quic_platform"], + tags = ["nofips"], deps = [ ":quic_platform_epoll_clock_lib", "//source/common/memory:stats_lib", @@ -113,6 +114,7 @@ envoy_cc_test_library( "//bazel:linux": ["quic_epoll_clock.h"], "//conditions:default": [], }), + tags = ["nofips"], deps = [ "@com_googlesource_quiche//:quic_platform", "@com_googlesource_quiche//:quic_platform_epoll_lib", @@ -122,12 +124,14 @@ envoy_cc_test_library( envoy_cc_test_library( name = "quic_platform_epoll_impl_lib", hdrs = ["quic_epoll_impl.h"], + tags = ["nofips"], deps = ["@com_googlesource_quiche//:epoll_server_lib"], ) envoy_cc_test_library( name = "quic_platform_expect_bug_impl_lib", hdrs = ["quic_expect_bug_impl.h"], + tags = ["nofips"], deps = [ "@com_googlesource_quiche//:quic_platform_base", "@com_googlesource_quiche//:quic_platform_mock_log", @@ -137,6 +141,7 @@ envoy_cc_test_library( envoy_cc_test_library( name = "quic_platform_mock_log_impl_lib", hdrs = ["quic_mock_log_impl.h"], + tags = ["nofips"], deps = ["@com_googlesource_quiche//:quic_platform_base"], ) @@ -144,6 +149,7 @@ envoy_cc_test_library( name = "quic_platform_port_utils_impl_lib", srcs = ["quic_port_utils_impl.cc"], hdrs = ["quic_port_utils_impl.h"], + tags = ["nofips"], deps = [ "//source/common/network:utility_lib", "//test/test_common:environment_lib", @@ -153,6 +159,7 @@ envoy_cc_test_library( envoy_cc_test_library( name = "quic_platform_test_mem_slice_vector_impl_lib", hdrs = ["quic_test_mem_slice_vector_impl.h"], + tags = ["nofips"], deps = [ "//include/envoy/buffer:buffer_interface", "@com_googlesource_quiche//:quic_platform_mem_slice_span", @@ -162,17 +169,20 @@ envoy_cc_test_library( envoy_cc_test_library( name = "quic_platform_sleep_impl_lib", hdrs = ["quic_sleep_impl.h"], + tags = ["nofips"], deps = ["@com_googlesource_quiche//:quic_core_time_lib"], ) envoy_cc_test_library( name = "quic_platform_system_event_loop_impl_lib", hdrs = ["quic_system_event_loop_impl.h"], + tags = ["nofips"], ) envoy_cc_test_library( name = "quic_platform_thread_impl_lib", hdrs = ["quic_thread_impl.h"], + tags = ["nofips"], deps = [ "//include/envoy/thread:thread_interface", "//source/common/common:assert_lib", @@ -183,6 +193,7 @@ envoy_cc_test_library( envoy_cc_test_library( name = "quic_platform_test_impl_lib", hdrs = ["quic_test_impl.h"], + tags = ["nofips"], deps = ["//source/common/common:assert_lib"], ) @@ -190,6 +201,7 @@ envoy_cc_test_library( name = "quic_platform_test_output_impl_lib", srcs = ["quic_test_output_impl.cc"], hdrs = ["quic_test_output_impl.h"], + tags = ["nofips"], deps = [ "//source/common/filesystem:filesystem_lib", "@com_googlesource_quiche//:quic_platform_base", @@ -217,6 +229,7 @@ envoy_cc_test_library( envoy_cc_test( name = "envoy_quic_clock_test", srcs = ["envoy_quic_clock_test.cc"], + tags = ["nofips"], deps = [ "//source/extensions/quic_listeners/quiche/platform:envoy_quic_clock_lib", "//test/test_common:simulated_time_system_lib", From 29f199c8625208a42483a1fa2de622c0fa105f49 Mon Sep 17 00:00:00 2001 From: Jyoti Mahapatra <49211422+jyotimahapatra@users.noreply.github.com> Date: Thu, 29 Aug 2019 11:30:50 -0700 Subject: [PATCH 481/542] tools: deprecated field check in Route Checker tool (#8058) We need a way to run the deprecated field check on the RouteConfiguration. Today the schema check tool validates the bootstrap config. This change will help achieve similar functionality on routes served from rds. Risk Level: Low Testing: Manual testing Docs Changes: included Release Notes: included Signed-off-by: Jyoti Mahapatra --- .../install/tools/route_table_check_tool.rst | 10 +++ docs/root/intro/version_history.rst | 1 + test/tools/router_check/router.cc | 9 ++- test/tools/router_check/router.h | 10 ++- test/tools/router_check/router_check.cc | 5 +- .../test/config/HeaderMatchedRouting.yaml | 4 +- .../router_check/test/config/TestRoutes.yaml | 63 ++++++++++++++----- 7 files changed, 83 insertions(+), 19 deletions(-) diff --git a/docs/root/install/tools/route_table_check_tool.rst b/docs/root/install/tools/route_table_check_tool.rst index ac0b523eec09..8acb2ac34a9d 100644 --- a/docs/root/install/tools/route_table_check_tool.rst +++ b/docs/root/install/tools/route_table_check_tool.rst @@ -40,6 +40,16 @@ Usage -p, --useproto Use Proto test file schema + -f, --fail-under + Represents a percent value for route test coverage under which the run should fail. + + --covall + Enables comprehensive code coverage percent calculation taking into account all the possible + asserts. + + --disable-deprecation-check + Disables the deprecation check for RouteConfiguration proto. + -h, --help Displays usage information and exits. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index c3598a5c7c7a..9667356cc3d8 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -45,6 +45,7 @@ Version history * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * router check tool: add coverage reporting & enforcement. * router check tool: add comprehensive coverage reporting. +* router check tool: add deprecated field check. * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. * tracing: added tags for gRPC response status and meesage. diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index 6922d26cea2f..5eeadc7a1bb1 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -63,7 +63,8 @@ ToolConfig::ToolConfig(std::unique_ptr headers, int ran : headers_(std::move(headers)), random_value_(random_value) {} // static -RouterCheckTool RouterCheckTool::create(const std::string& router_config_file) { +RouterCheckTool RouterCheckTool::create(const std::string& router_config_file, + const bool disableDeprecationCheck) { // TODO(hennna): Allow users to load a full config and extract the route configuration from it. envoy::api::v2::RouteConfiguration route_config; auto stats = std::make_unique(); @@ -72,6 +73,9 @@ RouterCheckTool RouterCheckTool::create(const std::string& router_config_file) { auto factory_context = std::make_unique>(); auto config = std::make_unique(route_config, *factory_context, false); + if (!disableDeprecationCheck) { + MessageUtil::checkForDeprecation(route_config, &factory_context->runtime_loader_); + } return RouterCheckTool(std::move(factory_context), std::move(config), std::move(stats), std::move(api), Coverage(route_config)); @@ -439,6 +443,8 @@ Options::Options(int argc, char** argv) { TCLAP::CmdLine cmd("router_check_tool", ' ', "none", true); TCLAP::SwitchArg is_proto("p", "useproto", "Use Proto test file schema", cmd, false); TCLAP::SwitchArg is_detailed("d", "details", "Show detailed test execution results", cmd, false); + TCLAP::SwitchArg disable_deprecation_check("", "disable-deprecation-check", + "Disable deprecated fields check", cmd, false); TCLAP::ValueArg fail_under("f", "fail-under", "Fail if test coverage is under a specified amount", false, 0.0, "float", cmd); @@ -461,6 +467,7 @@ Options::Options(int argc, char** argv) { is_detailed_ = is_detailed.getValue(); fail_under_ = fail_under.getValue(); comprehensive_coverage_ = comprehensive_coverage.getValue(); + disable_deprecation_check_ = disable_deprecation_check.getValue(); if (is_proto_) { config_path_ = config_path.getValue(); diff --git a/test/tools/router_check/router.h b/test/tools/router_check/router.h index 7c81835da7ae..e2156afa1072 100644 --- a/test/tools/router_check/router.h +++ b/test/tools/router_check/router.h @@ -65,10 +65,12 @@ class RouterCheckTool : Logger::Loggable { public: /** * @param router_config_file v2 router config file. + * @param disableDeprecationCheck flag to disable the RouteConfig deprecated field check * @return RouterCheckTool a RouterCheckTool instance with member variables set by the router * config file. * */ - static RouterCheckTool create(const std::string& router_config_file); + static RouterCheckTool create(const std::string& router_config_file, + const bool disableDeprecationCheck); /** * TODO(tonya11en): Use a YAML format for the expected routes. This will require a proto. @@ -198,6 +200,11 @@ class Options { */ bool isDetailed() const { return is_detailed_; } + /** + * @return true if the deprecated field check for RouteConfiguration is disabled. + */ + bool disableDeprecationCheck() const { return disable_deprecation_check_; } + private: std::string test_path_; std::string config_path_; @@ -207,5 +214,6 @@ class Options { bool comprehensive_coverage_; bool is_proto_; bool is_detailed_; + bool disable_deprecation_check_; }; } // namespace Envoy diff --git a/test/tools/router_check/router_check.cc b/test/tools/router_check/router_check.cc index e17f3eecc827..f3856e285d8c 100644 --- a/test/tools/router_check/router_check.cc +++ b/test/tools/router_check/router_check.cc @@ -10,8 +10,9 @@ int main(int argc, char* argv[]) { const bool enforce_coverage = options.failUnder() != 0.0; try { Envoy::RouterCheckTool checktool = - options.isProto() ? Envoy::RouterCheckTool::create(options.configPath()) - : Envoy::RouterCheckTool::create(options.unlabelledConfigPath()); + options.isProto() ? Envoy::RouterCheckTool::create(options.configPath(), + options.disableDeprecationCheck()) + : Envoy::RouterCheckTool::create(options.unlabelledConfigPath(), true); if (options.isDetailed()) { checktool.setShowDetails(); diff --git a/test/tools/router_check/test/config/HeaderMatchedRouting.yaml b/test/tools/router_check/test/config/HeaderMatchedRouting.yaml index f28c96a50a03..5f891bea08d5 100644 --- a/test/tools/router_check/test/config/HeaderMatchedRouting.yaml +++ b/test/tools/router_check/test/config/HeaderMatchedRouting.yaml @@ -30,7 +30,9 @@ virtual_hosts: prefix: / headers: - name: test_header_pattern - regex_match: ^user=test-\d+$ + safe_regex_match: + google_re2: {} + regex: ^user=test-\d+$ route: cluster: local_service_with_header_pattern_set_regex - match: diff --git a/test/tools/router_check/test/config/TestRoutes.yaml b/test/tools/router_check/test/config/TestRoutes.yaml index 34b665fb9fd5..862b6a4da8d5 100644 --- a/test/tools/router_check/test/config/TestRoutes.yaml +++ b/test/tools/router_check/test/config/TestRoutes.yaml @@ -104,26 +104,61 @@ virtual_hosts: timeout: seconds: 30 virtual_clusters: - - pattern: ^/rides$ - method: POST + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/rides$ + - name: :method + exact_match: POST name: ride_request - - pattern: ^/rides/\d+$ - method: PUT + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/rides/\d+$ + - name: :method + exact_match: PUT name: update_ride - - pattern: ^/users/\d+/chargeaccounts$ - method: POST + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/users/\d+/chargeaccounts$ + - name: :method + exact_match: POST name: cc_add - - pattern: ^/users/\d+/chargeaccounts/(?!validate)\w+$ - method: PUT + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/users/\d+/chargeaccounts/[^validate]\w+$ + - name: :method + exact_match: PUT name: cc_add - - pattern: ^/users$ - method: POST + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/users$ + - name: :method + exact_match: POST name: create_user_login - - pattern: ^/users/\d+$ - method: PUT + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/users/\d+$ + - name: :method + exact_match: PUT name: update_user - - pattern: ^/users/\d+/location$ - method: POST + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/users/\d+/location$ + - name: :method + exact_match: POST name: ulu internal_only_headers: - x-lyft-user-id From 6c6e18e5881936405cba72e936ecd6a5b3fe0852 Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Fri, 30 Aug 2019 16:05:12 +0700 Subject: [PATCH 482/542] tracing: Add support for sending data in Zipkin v2 format (#6985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Description: This patch supports sending a list of spans as JSON v2 and protobuf message over HTTP to Zipkin collector. [Sending protobuf](https://github.com/openzipkin/zipkin-api/blob/0.2.1/zipkin.proto) is considered to be more efficient than JSON, even compared to the v2's JSON (https://github.com/openzipkin/zipkin/pull/2589#issuecomment-491642768). This is an effort to rework https://github.com/envoyproxy/envoy/pull/6798. The approach is by serializing the v1 model to both v2 JSON and protobuf. Risk Level: Low, since the default is still HTTP-JSON v1 based on https://github.com/openzipkin/zipkin-api/blob/0.2.2/zipkin-api.yaml. Testing: Unit testing, manual integration test with real Zipkin collector server. Docs Changes: Added Release Notes: Added Fixes: #4839 Signed-off-by: Dhi Aurrahman Signed-off-by: José Carlos Chávez --- api/bazel/repositories.bzl | 25 ++ api/bazel/repository_locations.bzl | 8 + api/envoy/config/trace/v2/trace.proto | 28 +- docs/root/intro/deprecated.rst | 3 + docs/root/intro/version_history.rst | 1 + source/common/http/headers.h | 1 + source/extensions/tracers/zipkin/BUILD | 1 + .../extensions/tracers/zipkin/span_buffer.cc | 217 +++++++++- .../extensions/tracers/zipkin/span_buffer.h | 97 ++++- source/extensions/tracers/zipkin/tracer.cc | 6 +- source/extensions/tracers/zipkin/tracer.h | 10 +- .../tracers/zipkin/tracer_interface.h | 21 + source/extensions/tracers/zipkin/util.cc | 14 +- source/extensions/tracers/zipkin/util.h | 24 ++ .../tracers/zipkin/zipkin_core_constants.h | 3 + .../tracers/zipkin/zipkin_core_types.cc | 3 + .../tracers/zipkin/zipkin_core_types.h | 28 +- .../tracers/zipkin/zipkin_tracer_impl.cc | 69 +-- .../tracers/zipkin/zipkin_tracer_impl.h | 33 +- test/extensions/tracers/zipkin/BUILD | 1 + test/extensions/tracers/zipkin/config_test.cc | 3 +- .../tracers/zipkin/span_buffer_test.cc | 407 ++++++++++++++---- test/extensions/tracers/zipkin/tracer_test.cc | 2 +- .../tracers/zipkin/zipkin_tracer_impl_test.cc | 153 ++++--- 24 files changed, 925 insertions(+), 233 deletions(-) diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index d76337dc1200..7af054b20f4f 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -38,6 +38,11 @@ def api_dependencies(): locations = REPOSITORY_LOCATIONS, build_file_content = KAFKASOURCE_BUILD_CONTENT, ) + envoy_http_archive( + name = "com_github_openzipkin_zipkinapi", + locations = REPOSITORY_LOCATIONS, + build_file_content = ZIPKINAPI_BUILD_CONTENT, + ) GOGOPROTO_BUILD_CONTENT = """ load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") @@ -153,3 +158,23 @@ filegroup( ) """ + +ZIPKINAPI_BUILD_CONTENT = """ + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_go_proto_library") +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") + +api_proto_library( + name = "zipkin", + srcs = [ + "zipkin-jsonv2.proto", + "zipkin.proto", + ], + visibility = ["//visibility:public"], +) + +api_go_proto_library( + name = "zipkin", + proto = ":zipkin", +) +""" diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 2febf71148b7..8d688fd02e22 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -21,6 +21,9 @@ KAFKA_SOURCE_SHA = "ae7a1696c0a0302b43c5b21e515c37e6ecd365941f68a510a7e442eebddf UDPA_GIT_SHA = "4cbdcb9931ca743a915a7c5fda51b2ee793ed157" # Aug 22, 2019 UDPA_SHA256 = "6291d0c0e3a4d5f08057ea7a00ed0b0ec3dd4e5a3b1cf20f803774680b5a806f" +ZIPKINAPI_RELEASE = "0.2.2" # Aug 23, 2019 +ZIPKINAPI_SHA256 = "688c4fe170821dd589f36ec45aaadc03a618a40283bc1f97da8fa11686fc816b" + REPOSITORY_LOCATIONS = dict( bazel_skylib = dict( sha256 = BAZEL_SKYLIB_SHA256, @@ -62,4 +65,9 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "kafka-2.2.0-rc2/clients/src/main/resources/common/message", urls = ["https://github.com/apache/kafka/archive/2.2.0-rc2.zip"], ), + com_github_openzipkin_zipkinapi = dict( + sha256 = ZIPKINAPI_SHA256, + strip_prefix = "zipkin-api-" + ZIPKINAPI_RELEASE, + urls = ["https://github.com/openzipkin/zipkin-api/archive/" + ZIPKINAPI_RELEASE + ".tar.gz"], + ), ) diff --git a/api/envoy/config/trace/v2/trace.proto b/api/envoy/config/trace/v2/trace.proto index 9da6ef4313d8..65c027cd73fe 100644 --- a/api/envoy/config/trace/v2/trace.proto +++ b/api/envoy/config/trace/v2/trace.proto @@ -65,6 +65,7 @@ message LightstepConfig { string access_token_file = 2 [(validate.rules).string.min_bytes = 1]; } +// Configuration for the Zipkin tracer. message ZipkinConfig { // The cluster manager cluster that hosts the Zipkin collectors. Note that the // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster @@ -80,9 +81,34 @@ message ZipkinConfig { // trace instance. The default value is false, which will result in a 64 bit trace id being used. bool trace_id_128bit = 3; - // Determines whether client and server spans will shared the same span id. + // Determines whether client and server spans will share the same span context. // The default value is true. google.protobuf.BoolValue shared_span_context = 4; + + // Available Zipkin collector endpoint versions. + enum CollectorEndpointVersion { + // Zipkin API v1, JSON over HTTP. + // [#comment: The default implementation of Zipkin client before this field is added was only v1 + // and the way user configure this was by not explicitly specifying the version. Consequently, + // before this is added, the corresponding Zipkin collector expected to receive v1 payload. + // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when + // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, + // since in Zipkin realm this v1 version is considered to be not preferable anymore.] + HTTP_JSON_V1 = 0 [deprecated = true]; + + // Zipkin API v2, JSON over HTTP. + HTTP_JSON = 1; + + // Zipkin API v2, protobuf over HTTP. + HTTP_PROTO = 2; + + // [#not-implemented-hide:] + GRPC = 3; + } + + // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be + // used. + CollectorEndpointVersion collector_endpoint_version = 5; } // DynamicOtConfig is used to dynamically load a tracer from a shared library diff --git a/docs/root/intro/deprecated.rst b/docs/root/intro/deprecated.rst index fb7832739383..57fd2dd5dbc3 100644 --- a/docs/root/intro/deprecated.rst +++ b/docs/root/intro/deprecated.rst @@ -31,6 +31,9 @@ Version 1.12.0 (pending) and `present_match` fields. * The :option:`--allow-unknown-fields` command-line option, use :option:`--allow-unknown-static-fields` instead. +* The use of HTTP_JSON_V1 :ref:`Zipkin collector endpoint version + ` or not explicitly + specifying it is deprecated, use HTTP_JSON or HTTP_PROTO instead. Version 1.11.0 (July 11, 2019) ============================== diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 9667356cc3d8..c4e6c2f2ac52 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -45,6 +45,7 @@ Version history * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. * router check tool: add coverage reporting & enforcement. * router check tool: add comprehensive coverage reporting. +* tracing: added support to the Zipkin reporter for sending list of spans as Zipkin JSON v2 and protobuf message over HTTP. * router check tool: add deprecated field check. * tls: added verification of IP address SAN fields in certificates against configured SANs in the certificate validation context. diff --git a/source/common/http/headers.h b/source/common/http/headers.h index b711fb4b142a..b0bcb7cc48ce 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -178,6 +178,7 @@ class HeaderValues { const std::string GrpcWebText{"application/grpc-web-text"}; const std::string GrpcWebTextProto{"application/grpc-web-text+proto"}; const std::string Json{"application/json"}; + const std::string Protobuf{"application/x-protobuf"}; const std::string FormUrlEncoded{"application/x-www-form-urlencoded"}; } ContentTypeValues; diff --git a/source/extensions/tracers/zipkin/BUILD b/source/extensions/tracers/zipkin/BUILD index 77937e6dbbbf..652496900174 100644 --- a/source/extensions/tracers/zipkin/BUILD +++ b/source/extensions/tracers/zipkin/BUILD @@ -57,6 +57,7 @@ envoy_cc_library( "//source/common/singleton:const_singleton", "//source/common/tracing:http_tracer_lib", "//source/extensions/tracers:well_known_names", + "@com_github_openzipkin_zipkinapi//:zipkin_cc", ], ) diff --git a/source/extensions/tracers/zipkin/span_buffer.cc b/source/extensions/tracers/zipkin/span_buffer.cc index 387d851a9f91..66bb96a9463b 100644 --- a/source/extensions/tracers/zipkin/span_buffer.cc +++ b/source/extensions/tracers/zipkin/span_buffer.cc @@ -1,35 +1,224 @@ #include "extensions/tracers/zipkin/span_buffer.h" +#include "common/protobuf/protobuf.h" + +#include "extensions/tracers/zipkin/util.h" +#include "extensions/tracers/zipkin/zipkin_core_constants.h" + +#include "absl/strings/str_join.h" + namespace Envoy { namespace Extensions { namespace Tracers { namespace Zipkin { -// TODO(fabolive): Need to avoid the copy to improve performance. -bool SpanBuffer::addSpan(const Span& span) { - if (span_buffer_.size() == span_buffer_.capacity()) { - // Buffer full +SpanBuffer::SpanBuffer( + const envoy::config::trace::v2::ZipkinConfig::CollectorEndpointVersion& version, + const bool shared_span_context) + : serializer_{makeSerializer(version, shared_span_context)} {} + +SpanBuffer::SpanBuffer( + const envoy::config::trace::v2::ZipkinConfig::CollectorEndpointVersion& version, + const bool shared_span_context, uint64_t size) + : serializer_{makeSerializer(version, shared_span_context)} { + allocateBuffer(size); +} + +bool SpanBuffer::addSpan(Span&& span) { + const auto& annotations = span.annotations(); + if (span_buffer_.size() == span_buffer_.capacity() || annotations.empty() || + annotations.end() == + std::find_if(annotations.begin(), annotations.end(), [](const auto& annotation) { + return annotation.value() == ZipkinCoreConstants::get().CLIENT_SEND || + annotation.value() == ZipkinCoreConstants::get().SERVER_RECV; + })) { + + // Buffer full or invalid span. return false; } + span_buffer_.push_back(std::move(span)); return true; } -std::string SpanBuffer::toStringifiedJsonArray() { - std::string stringified_json_array = "["; +SerializerPtr SpanBuffer::makeSerializer( + const envoy::config::trace::v2::ZipkinConfig::CollectorEndpointVersion& version, + const bool shared_span_context) { + switch (version) { + case envoy::config::trace::v2::ZipkinConfig::HTTP_JSON_V1: + return std::make_unique(); + case envoy::config::trace::v2::ZipkinConfig::HTTP_JSON: + return std::make_unique(shared_span_context); + case envoy::config::trace::v2::ZipkinConfig::HTTP_PROTO: + return std::make_unique(shared_span_context); + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +std::string JsonV1Serializer::serialize(const std::vector& zipkin_spans) { + const std::string serialized_elements = + absl::StrJoin(zipkin_spans, ",", [](std::string* element, Span zipkin_span) { + absl::StrAppend(element, zipkin_span.toJson()); + }); + return absl::StrCat("[", serialized_elements, "]"); +} + +JsonV2Serializer::JsonV2Serializer(const bool shared_span_context) + : shared_span_context_{shared_span_context} {} + +std::string JsonV2Serializer::serialize(const std::vector& zipkin_spans) { + const std::string serialized_elements = + absl::StrJoin(zipkin_spans, ",", [this](std::string* out, const Span& zipkin_span) { + absl::StrAppend(out, + absl::StrJoin(toListOfSpans(zipkin_span), ",", + [](std::string* element, const zipkin::jsonv2::Span& span) { + std::string entry; + Protobuf::util::MessageToJsonString(span, &entry); + absl::StrAppend(element, entry); + })); + }); + return absl::StrCat("[", serialized_elements, "]"); +} - if (pendingSpans()) { - stringified_json_array += span_buffer_[0].toJson(); - const uint64_t size = span_buffer_.size(); - for (uint64_t i = 1; i < size; i++) { - stringified_json_array += ","; - stringified_json_array += span_buffer_[i].toJson(); +const std::vector +JsonV2Serializer::toListOfSpans(const Span& zipkin_span) const { + std::vector spans; + spans.reserve(zipkin_span.annotations().size()); + for (const auto& annotation : zipkin_span.annotations()) { + zipkin::jsonv2::Span span; + + if (annotation.value() == ZipkinCoreConstants::get().CLIENT_SEND) { + span.set_kind(ZipkinCoreConstants::get().KIND_CLIENT); + } else if (annotation.value() == ZipkinCoreConstants::get().SERVER_RECV) { + span.set_shared(shared_span_context_ && zipkin_span.annotations().size() > 1); + span.set_kind(ZipkinCoreConstants::get().KIND_SERVER); + } else { + continue; + } + + if (annotation.isSetEndpoint()) { + span.set_timestamp(annotation.timestamp()); + span.mutable_local_endpoint()->MergeFrom(toProtoEndpoint(annotation.endpoint())); + } + + span.set_trace_id(zipkin_span.traceIdAsHexString()); + if (zipkin_span.isSetParentId()) { + span.set_parent_id(zipkin_span.parentIdAsHexString()); + } + + span.set_id(zipkin_span.idAsHexString()); + span.set_name(zipkin_span.name()); + + if (zipkin_span.isSetDuration()) { + span.set_duration(zipkin_span.duration()); + } + + auto& tags = *span.mutable_tags(); + for (const auto& binary_annotation : zipkin_span.binaryAnnotations()) { + tags[binary_annotation.key()] = binary_annotation.value(); } + + spans.push_back(std::move(span)); + } + return spans; +} + +const zipkin::jsonv2::Endpoint +JsonV2Serializer::toProtoEndpoint(const Endpoint& zipkin_endpoint) const { + zipkin::jsonv2::Endpoint endpoint; + Network::Address::InstanceConstSharedPtr address = zipkin_endpoint.address(); + if (address) { + if (address->ip()->version() == Network::Address::IpVersion::v4) { + endpoint.set_ipv4(address->ip()->addressAsString()); + } else { + endpoint.set_ipv6(address->ip()->addressAsString()); + } + endpoint.set_port(address->ip()->port()); + } + + const std::string& service_name = zipkin_endpoint.serviceName(); + if (!service_name.empty()) { + endpoint.set_service_name(service_name); + } + + return endpoint; +} + +ProtobufSerializer::ProtobufSerializer(const bool shared_span_context) + : shared_span_context_{shared_span_context} {} + +std::string ProtobufSerializer::serialize(const std::vector& zipkin_spans) { + zipkin::proto3::ListOfSpans spans; + for (const Span& zipkin_span : zipkin_spans) { + spans.MergeFrom(toListOfSpans(zipkin_span)); + } + std::string serialized; + spans.SerializeToString(&serialized); + return serialized; +} + +const zipkin::proto3::ListOfSpans ProtobufSerializer::toListOfSpans(const Span& zipkin_span) const { + zipkin::proto3::ListOfSpans spans; + for (const auto& annotation : zipkin_span.annotations()) { + zipkin::proto3::Span span; + if (annotation.value() == ZipkinCoreConstants::get().CLIENT_SEND) { + span.set_kind(zipkin::proto3::Span::CLIENT); + } else if (annotation.value() == ZipkinCoreConstants::get().SERVER_RECV) { + span.set_shared(shared_span_context_ && zipkin_span.annotations().size() > 1); + span.set_kind(zipkin::proto3::Span::SERVER); + } else { + continue; + } + + if (annotation.isSetEndpoint()) { + span.set_timestamp(annotation.timestamp()); + span.mutable_local_endpoint()->MergeFrom(toProtoEndpoint(annotation.endpoint())); + } + + span.set_trace_id(zipkin_span.traceIdAsByteString()); + if (zipkin_span.isSetParentId()) { + span.set_parent_id(zipkin_span.parentIdAsByteString()); + } + + span.set_id(zipkin_span.idAsByteString()); + span.set_name(zipkin_span.name()); + + if (zipkin_span.isSetDuration()) { + span.set_duration(zipkin_span.duration()); + } + + auto& tags = *span.mutable_tags(); + for (const auto& binary_annotation : zipkin_span.binaryAnnotations()) { + tags[binary_annotation.key()] = binary_annotation.value(); + } + + auto* mutable_span = spans.add_spans(); + mutable_span->MergeFrom(span); + } + return spans; +} + +const zipkin::proto3::Endpoint +ProtobufSerializer::toProtoEndpoint(const Endpoint& zipkin_endpoint) const { + zipkin::proto3::Endpoint endpoint; + Network::Address::InstanceConstSharedPtr address = zipkin_endpoint.address(); + if (address) { + if (address->ip()->version() == Network::Address::IpVersion::v4) { + endpoint.set_ipv4(Util::toByteString(address->ip()->ipv4()->address())); + } else { + endpoint.set_ipv6(Util::toByteString(address->ip()->ipv6()->address())); + } + endpoint.set_port(address->ip()->port()); + } + + const std::string& service_name = zipkin_endpoint.serviceName(); + if (!service_name.empty()) { + endpoint.set_service_name(service_name); } - stringified_json_array += "]"; - return stringified_json_array; + return endpoint; } } // namespace Zipkin diff --git a/source/extensions/tracers/zipkin/span_buffer.h b/source/extensions/tracers/zipkin/span_buffer.h index a67479c644a4..a5718600129e 100644 --- a/source/extensions/tracers/zipkin/span_buffer.h +++ b/source/extensions/tracers/zipkin/span_buffer.h @@ -1,7 +1,13 @@ #pragma once +#include "envoy/config/trace/v2/trace.pb.h" + +#include "extensions/tracers/zipkin/tracer_interface.h" #include "extensions/tracers/zipkin/zipkin_core_types.h" +#include "zipkin-jsonv2.pb.h" +#include "zipkin.pb.h" + namespace Envoy { namespace Extensions { namespace Tracers { @@ -16,15 +22,26 @@ class SpanBuffer { /** * Constructor that creates an empty buffer. Space needs to be allocated by invoking * the method allocateBuffer(size). + * + * @param version The selected Zipkin collector version. @see + * api/envoy/config/trace/v2/trace.proto. + * @param shared_span_context To determine whether client and server spans will share the same + * span context. */ - SpanBuffer() = default; + SpanBuffer(const envoy::config::trace::v2::ZipkinConfig::CollectorEndpointVersion& version, + bool shared_span_context); /** * Constructor that initializes a buffer with the given size. * + * @param version The selected Zipkin collector version. @see + * api/envoy/config/trace/v2/trace.proto. + * @param shared_span_context To determine whether client and server spans will share the same + * span context. * @param size The desired buffer size. */ - SpanBuffer(uint64_t size) { allocateBuffer(size); } + SpanBuffer(const envoy::config::trace::v2::ZipkinConfig::CollectorEndpointVersion& version, + bool shared_span_context, uint64_t size); /** * Allocates space for an empty buffer or resizes a previously-allocated one. @@ -40,7 +57,7 @@ class SpanBuffer { * * @return true if the span was successfully added, or false if the buffer was full. */ - bool addSpan(const Span& span); + bool addSpan(Span&& span); /** * Empties the buffer. This method is supposed to be called when all buffered spans @@ -54,14 +71,82 @@ class SpanBuffer { uint64_t pendingSpans() { return span_buffer_.size(); } /** - * @return the contents of the buffer as a stringified array of JSONs, where - * each JSON in the array corresponds to one Zipkin span. + * Serializes std::vector span_buffer_ to std::string as payload for the reporter when the + * reporter does spans flushing. This function does only serialization and does not clear + * span_buffer_. + * + * @return std::string the contents of the buffer, a collection of serialized pending Zipkin + * spans. */ - std::string toStringifiedJsonArray(); + std::string serialize() const { return serializer_->serialize(span_buffer_); } private: + SerializerPtr + makeSerializer(const envoy::config::trace::v2::ZipkinConfig::CollectorEndpointVersion& version, + bool shared_span_context); + // We use a pre-allocated vector to improve performance std::vector span_buffer_; + SerializerPtr serializer_; +}; + +using SpanBufferPtr = std::unique_ptr; + +/** + * JsonV1Serializer implements Zipkin::Serializer that serializes list of Zipkin spans into JSON + * Zipkin v1 array. + */ +class JsonV1Serializer : public Serializer { +public: + JsonV1Serializer() = default; + + /** + * Serialize list of Zipkin spans into Zipkin v1 JSON array. + * @return std::string serialized pending spans as Zipkin v1 JSON array. + */ + std::string serialize(const std::vector& pending_spans) override; +}; + +/** + * JsonV2Serializer implements Zipkin::Serializer that serializes list of Zipkin spans into JSON + * Zipkin v2 array. + */ +class JsonV2Serializer : public Serializer { +public: + JsonV2Serializer(bool shared_span_context); + + /** + * Serialize list of Zipkin spans into Zipkin v2 JSON array. + * @return std::string serialized pending spans as Zipkin v2 JSON array. + */ + std::string serialize(const std::vector& pending_spans) override; + +private: + const std::vector toListOfSpans(const Span& zipkin_span) const; + const zipkin::jsonv2::Endpoint toProtoEndpoint(const Endpoint& zipkin_endpoint) const; + + const bool shared_span_context_; +}; + +/** + * ProtobufSerializer implements Zipkin::Serializer that serializes list of Zipkin spans into + * stringified (SerializeToString) protobuf message. + */ +class ProtobufSerializer : public Serializer { +public: + ProtobufSerializer(bool shared_span_context); + + /** + * Serialize list of Zipkin spans into Zipkin v2 zipkin::proto3::ListOfSpans. + * @return std::string serialized pending spans as Zipkin zipkin::proto3::ListOfSpans. + */ + std::string serialize(const std::vector& pending_spans) override; + +private: + const zipkin::proto3::ListOfSpans toListOfSpans(const Span& zipkin_span) const; + const zipkin::proto3::Endpoint toProtoEndpoint(const Endpoint& zipkin_endpoint) const; + + const bool shared_span_context_; }; } // namespace Zipkin diff --git a/source/extensions/tracers/zipkin/tracer.cc b/source/extensions/tracers/zipkin/tracer.cc index 8b21bef8f23a..aff1d659c12d 100644 --- a/source/extensions/tracers/zipkin/tracer.cc +++ b/source/extensions/tracers/zipkin/tracer.cc @@ -28,7 +28,7 @@ SpanPtr Tracer::startSpan(const Tracing::Config& config, const std::string& span } // Create an all-new span, with no parent id - SpanPtr span_ptr(new Span(time_source_)); + SpanPtr span_ptr = std::make_unique(time_source_); span_ptr->setName(span_name); uint64_t random_number = random_generator_.random(); span_ptr->setId(random_number); @@ -56,8 +56,8 @@ SpanPtr Tracer::startSpan(const Tracing::Config& config, const std::string& span } SpanPtr Tracer::startSpan(const Tracing::Config& config, const std::string& span_name, - SystemTime timestamp, SpanContext& previous_context) { - SpanPtr span_ptr(new Span(time_source_)); + SystemTime timestamp, const SpanContext& previous_context) { + SpanPtr span_ptr = std::make_unique(time_source_); Annotation annotation; uint64_t timestamp_micro; diff --git a/source/extensions/tracers/zipkin/tracer.h b/source/extensions/tracers/zipkin/tracer.h index 190b68631b63..d51e0645844a 100644 --- a/source/extensions/tracers/zipkin/tracer.h +++ b/source/extensions/tracers/zipkin/tracer.h @@ -33,7 +33,7 @@ class Reporter { * * @param span The span that needs action. */ - virtual void reportSpan(const Span& span) PURE; + virtual void reportSpan(Span&& span) PURE; }; using ReporterPtr = std::unique_ptr; @@ -72,6 +72,7 @@ class Tracer : public TracerInterface { * @param config The tracing configuration * @param span_name Name of the new span. * @param start_time The time indicating the beginning of the span. + * @return SpanPtr The root span. */ SpanPtr startSpan(const Tracing::Config&, const std::string& span_name, SystemTime timestamp); @@ -82,12 +83,15 @@ class Tracer : public TracerInterface { * @param span_name Name of the new span. * @param start_time The time indicating the beginning of the span. * @param previous_context The context of the span preceding the one to be created. + * @return SpanPtr The child span. */ SpanPtr startSpan(const Tracing::Config&, const std::string& span_name, SystemTime timestamp, - SpanContext& previous_context); + const SpanContext& previous_context); /** * TracerInterface::reportSpan. + * + * @param span The span to be reported. */ void reportSpan(Span&& span) override; @@ -103,6 +107,8 @@ class Tracer : public TracerInterface { /** * Associates a Reporter object with this Tracer. + * + * @param The span reporter. */ void setReporter(ReporterPtr reporter); diff --git a/source/extensions/tracers/zipkin/tracer_interface.h b/source/extensions/tracers/zipkin/tracer_interface.h index 4c55ab90980b..c56e130e2868 100644 --- a/source/extensions/tracers/zipkin/tracer_interface.h +++ b/source/extensions/tracers/zipkin/tracer_interface.h @@ -1,5 +1,9 @@ #pragma once +#include +#include +#include + #include "envoy/common/pure.h" namespace Envoy { @@ -31,6 +35,23 @@ class TracerInterface { virtual void reportSpan(Span&& span) PURE; }; +/** + * Buffered pending spans serializer. + */ +class Serializer { +public: + virtual ~Serializer() = default; + + /** + * Serialize buffered pending spans. + * + * @return std::string serialized buffered pending spans. + */ + virtual std::string serialize(const std::vector& spans) PURE; +}; + +using SerializerPtr = std::unique_ptr; + } // namespace Zipkin } // namespace Tracers } // namespace Extensions diff --git a/source/extensions/tracers/zipkin/util.cc b/source/extensions/tracers/zipkin/util.cc index d18eff673b04..18eea42daf87 100644 --- a/source/extensions/tracers/zipkin/util.cc +++ b/source/extensions/tracers/zipkin/util.cc @@ -7,6 +7,7 @@ #include "common/common/hex.h" #include "common/common/utility.h" +#include "absl/strings/str_join.h" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" @@ -33,18 +34,7 @@ void Util::mergeJsons(std::string& target, const std::string& source, void Util::addArrayToJson(std::string& target, const std::vector& json_array, const std::string& field_name) { - std::string stringified_json_array = "["; - - if (!json_array.empty()) { - stringified_json_array += json_array[0]; - for (auto it = json_array.begin() + 1; it != json_array.end(); it++) { - stringified_json_array += ","; - stringified_json_array += *it; - } - } - stringified_json_array += "]"; - - mergeJsons(target, stringified_json_array, field_name); + mergeJsons(target, absl::StrCat("[", absl::StrJoin(json_array, ","), "]"), field_name); } uint64_t Util::generateRandom64(TimeSource& time_source) { diff --git a/source/extensions/tracers/zipkin/util.h b/source/extensions/tracers/zipkin/util.h index ce86f73080e9..8b30a155ae04 100644 --- a/source/extensions/tracers/zipkin/util.h +++ b/source/extensions/tracers/zipkin/util.h @@ -5,6 +5,8 @@ #include "envoy/common/time.h" +#include "common/common/byte_order.h" + namespace Envoy { namespace Extensions { namespace Tracers { @@ -48,6 +50,28 @@ class Util { * Returns a randomly-generated 64-bit integer number. */ static uint64_t generateRandom64(TimeSource& time_source); + + /** + * Returns byte string representation of a number. + * + * @param value Number that will be represented in byte string. + * @return std::string byte string representation of a number. + */ + template static std::string toByteString(Type value) { + return std::string(reinterpret_cast(&value), sizeof(Type)); + } + + /** + * Returns big endian byte string representation of a number. + * + * @param value Number that will be represented in byte string. + * @param flip indicates to flip order or not. + * @return std::string byte string representation of a number. + */ + template static std::string toBigEndianByteString(Type value) { + auto bytes = toEndianness(value); + return std::string(reinterpret_cast(&bytes), sizeof(Type)); + } }; } // namespace Zipkin diff --git a/source/extensions/tracers/zipkin/zipkin_core_constants.h b/source/extensions/tracers/zipkin/zipkin_core_constants.h index 7384df786a19..0180aeb8f22c 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_constants.h +++ b/source/extensions/tracers/zipkin/zipkin_core_constants.h @@ -13,6 +13,9 @@ namespace Zipkin { class ZipkinCoreConstantValues { public: + const std::string KIND_CLIENT = "CLIENT"; + const std::string KIND_SERVER = "SERVER"; + const std::string CLIENT_SEND = "cs"; const std::string CLIENT_RECV = "cr"; const std::string SERVER_SEND = "ss"; diff --git a/source/extensions/tracers/zipkin/zipkin_core_types.cc b/source/extensions/tracers/zipkin/zipkin_core_types.cc index 1730e3cd75cd..16c9d688a437 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_types.cc +++ b/source/extensions/tracers/zipkin/zipkin_core_types.cc @@ -241,6 +241,9 @@ void Span::finish() { cr.setTimestamp(stop_timestamp); cr.setValue(ZipkinCoreConstants::get().CLIENT_RECV); annotations_.push_back(std::move(cr)); + } + + if (monotonic_start_time_) { const int64_t monotonic_stop_time = std::chrono::duration_cast( time_source_.monotonicTime().time_since_epoch()) .count(); diff --git a/source/extensions/tracers/zipkin/zipkin_core_types.h b/source/extensions/tracers/zipkin/zipkin_core_types.h index 8c9ff909241f..9de9f4871620 100644 --- a/source/extensions/tracers/zipkin/zipkin_core_types.h +++ b/source/extensions/tracers/zipkin/zipkin_core_types.h @@ -6,11 +6,13 @@ #include "envoy/common/time.h" #include "envoy/network/address.h" +#include "common/common/assert.h" #include "common/common/hex.h" #include "extensions/tracers/zipkin/tracer_interface.h" #include "extensions/tracers/zipkin/util.h" +#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -443,6 +445,11 @@ class Span : public ZipkinBase { */ const std::string idAsHexString() const { return Hex::uint64ToHex(id_); } + /** + * @return the span's id as a byte string. + */ + const std::string idAsByteString() const { return Util::toByteString(id_); } + /** * @return the span's name. */ @@ -460,6 +467,14 @@ class Span : public ZipkinBase { return parent_id_ ? Hex::uint64ToHex(parent_id_.value()) : EMPTY_HEX_STRING_; } + /** + * @return the span's parent id as a byte string. + */ + const std::string parentIdAsByteString() const { + ASSERT(parent_id_); + return Util::toByteString(parent_id_.value()); + } + /** * @return whether or not the debug attribute is set */ @@ -490,10 +505,21 @@ class Span : public ZipkinBase { */ const std::string traceIdAsHexString() const { return trace_id_high_.has_value() - ? Hex::uint64ToHex(trace_id_high_.value()) + Hex::uint64ToHex(trace_id_) + ? absl::StrCat(Hex::uint64ToHex(trace_id_high_.value()), Hex::uint64ToHex(trace_id_)) : Hex::uint64ToHex(trace_id_); } + /** + * @return the span's trace id as a byte string. + */ + const std::string traceIdAsByteString() const { + // https://github.com/openzipkin/zipkin-api/blob/v0.2.1/zipkin.proto#L60-L61. + return trace_id_high_.has_value() + ? absl::StrCat(Util::toBigEndianByteString(trace_id_high_.value()), + Util::toBigEndianByteString(trace_id_)) + : Util::toBigEndianByteString(trace_id_); + } + /** * @return the span's start time (monotonic, used to calculate duration). */ diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc index fbe84ff78662..3b269f742c01 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.cc @@ -56,9 +56,9 @@ void ZipkinSpan::setSampled(bool sampled) { span_.setSampled(sampled); } Tracing::SpanPtr ZipkinSpan::spawnChild(const Tracing::Config& config, const std::string& name, SystemTime start_time) { - SpanContext context(span_); - return Tracing::SpanPtr{ - new ZipkinSpan(*tracer_.startSpan(config, name, start_time, context), tracer_)}; + SpanContext previous_context(span_); + return std::make_unique( + *tracer_.startSpan(config, name, start_time, previous_context), tracer_); } Driver::TlsTracer::TlsTracer(TracerPtr&& tracer, Driver& driver) @@ -76,23 +76,26 @@ Driver::Driver(const envoy::config::trace::v2::ZipkinConfig& zipkin_config, Config::Utility::checkCluster(TracerNames::get().Zipkin, zipkin_config.collector_cluster(), cm_); cluster_ = cm_.get(zipkin_config.collector_cluster())->info(); - std::string collector_endpoint = ZipkinCoreConstants::get().DEFAULT_COLLECTOR_ENDPOINT; + CollectorInfo collector; if (!zipkin_config.collector_endpoint().empty()) { - collector_endpoint = zipkin_config.collector_endpoint(); + collector.endpoint_ = zipkin_config.collector_endpoint(); } - + // The current default version of collector_endpoint_version is HTTP_JSON_V1. + collector.version_ = zipkin_config.collector_endpoint_version(); const bool trace_id_128bit = zipkin_config.trace_id_128bit(); const bool shared_span_context = PROTOBUF_GET_WRAPPED_OR_DEFAULT( zipkin_config, shared_span_context, ZipkinCoreConstants::get().DEFAULT_SHARED_SPAN_CONTEXT); + collector.shared_span_context_ = shared_span_context; - tls_->set([this, collector_endpoint, &random_generator, trace_id_128bit, shared_span_context]( + tls_->set([this, collector, &random_generator, trace_id_128bit, shared_span_context]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { - TracerPtr tracer(new Tracer(local_info_.clusterName(), local_info_.address(), random_generator, - trace_id_128bit, shared_span_context, time_source_)); + TracerPtr tracer = + std::make_unique(local_info_.clusterName(), local_info_.address(), random_generator, + trace_id_128bit, shared_span_context, time_source_); tracer->setReporter( - ReporterImpl::NewInstance(std::ref(*this), std::ref(dispatcher), collector_endpoint)); - return ThreadLocal::ThreadLocalObjectSharedPtr{new TlsTracer(std::move(tracer), *this)}; + ReporterImpl::NewInstance(std::ref(*this), std::ref(dispatcher), collector)); + return std::make_shared(std::move(tracer), *this); }); } @@ -117,16 +120,18 @@ Tracing::SpanPtr Driver::startSpan(const Tracing::Config& config, Http::HeaderMa } } catch (const ExtractorException& e) { - return Tracing::SpanPtr(new Tracing::NullSpan()); + return std::make_unique(); } - ZipkinSpanPtr active_span(new ZipkinSpan(*new_zipkin_span, tracer)); - return active_span; + // Return the active Zipkin span. + return std::make_unique(*new_zipkin_span, tracer); } ReporterImpl::ReporterImpl(Driver& driver, Event::Dispatcher& dispatcher, - const std::string& collector_endpoint) - : driver_(driver), collector_endpoint_(collector_endpoint) { + const CollectorInfo& collector) + : driver_(driver), + collector_(collector), span_buffer_{std::make_unique( + collector.version_, collector.shared_span_context_)} { flush_timer_ = dispatcher.createTimer([this]() -> void { driver_.tracerStats().timer_flushed_.inc(); flushSpans(); @@ -135,24 +140,23 @@ ReporterImpl::ReporterImpl(Driver& driver, Event::Dispatcher& dispatcher, const uint64_t min_flush_spans = driver_.runtime().snapshot().getInteger("tracing.zipkin.min_flush_spans", 5U); - span_buffer_.allocateBuffer(min_flush_spans); + span_buffer_->allocateBuffer(min_flush_spans); enableTimer(); } ReporterPtr ReporterImpl::NewInstance(Driver& driver, Event::Dispatcher& dispatcher, - const std::string& collector_endpoint) { - return ReporterPtr(new ReporterImpl(driver, dispatcher, collector_endpoint)); + const CollectorInfo& collector) { + return std::make_unique(driver, dispatcher, collector); } -// TODO(fabolive): Need to avoid the copy to improve performance. -void ReporterImpl::reportSpan(const Span& span) { - span_buffer_.addSpan(span); +void ReporterImpl::reportSpan(Span&& span) { + span_buffer_->addSpan(std::move(span)); const uint64_t min_flush_spans = driver_.runtime().snapshot().getInteger("tracing.zipkin.min_flush_spans", 5U); - if (span_buffer_.pendingSpans() == min_flush_spans) { + if (span_buffer_->pendingSpans() == min_flush_spans) { flushSpans(); } } @@ -164,18 +168,19 @@ void ReporterImpl::enableTimer() { } void ReporterImpl::flushSpans() { - if (span_buffer_.pendingSpans()) { - driver_.tracerStats().spans_sent_.add(span_buffer_.pendingSpans()); - - const std::string request_body = span_buffer_.toStringifiedJsonArray(); - Http::MessagePtr message(new Http::RequestMessageImpl()); + if (span_buffer_->pendingSpans()) { + driver_.tracerStats().spans_sent_.add(span_buffer_->pendingSpans()); + const std::string request_body = span_buffer_->serialize(); + Http::MessagePtr message = std::make_unique(); message->headers().insertMethod().value().setReference(Http::Headers::get().MethodValues.Post); - message->headers().insertPath().value(collector_endpoint_); + message->headers().insertPath().value(collector_.endpoint_); message->headers().insertHost().value(driver_.cluster()->name()); message->headers().insertContentType().value().setReference( - Http::Headers::get().ContentTypeValues.Json); + collector_.version_ == envoy::config::trace::v2::ZipkinConfig::HTTP_PROTO + ? Http::Headers::get().ContentTypeValues.Protobuf + : Http::Headers::get().ContentTypeValues.Json); - Buffer::InstancePtr body(new Buffer::OwnedImpl()); + Buffer::InstancePtr body = std::make_unique(); body->add(request_body); message->body() = std::move(body); @@ -186,7 +191,7 @@ void ReporterImpl::flushSpans() { .send(std::move(message), *this, Http::AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds(timeout))); - span_buffer_.clear(); + span_buffer_->clear(); } } diff --git a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h index 048b37cd18dc..4cecd015d6a3 100644 --- a/source/extensions/tracers/zipkin/zipkin_tracer_impl.h +++ b/source/extensions/tracers/zipkin/zipkin_tracer_impl.h @@ -11,6 +11,7 @@ #include "extensions/tracers/zipkin/span_buffer.h" #include "extensions/tracers/zipkin/tracer.h" +#include "extensions/tracers/zipkin/zipkin_core_constants.h" namespace Envoy { namespace Extensions { @@ -137,6 +138,23 @@ class Driver : public Tracing::Driver { TimeSource& time_source_; }; +/** + * Information about the Zipkin collector. + */ +struct CollectorInfo { + // The Zipkin collector endpoint/path to receive the collected trace data. e.g. /api/v1/spans if + // HTTP_JSON_V1 or /api/v2/spans otherwise. + std::string endpoint_{ZipkinCoreConstants::get().DEFAULT_COLLECTOR_ENDPOINT}; + + // The version of the collector. This is related to endpoint's supported payload specification and + // transport. Currently it defaults to envoy::config::trace::v2::ZipkinConfig::HTTP_JSON_V1. In + // the future, we will throw when collector_endpoint_version is not specified. + envoy::config::trace::v2::ZipkinConfig::CollectorEndpointVersion version_{ + envoy::config::trace::v2::ZipkinConfig::HTTP_JSON_V1}; + + bool shared_span_context_{ZipkinCoreConstants::get().DEFAULT_SHARED_SPAN_CONTEXT}; +}; + /** * This class derives from the abstract Zipkin::Reporter. * It buffers spans and relies on Http::AsyncClient to send spans to @@ -158,12 +176,11 @@ class ReporterImpl : public Reporter, Http::AsyncClient::Callbacks { * * @param driver ZipkinDriver to be associated with the reporter. * @param dispatcher Controls the timer used to flush buffered spans. - * @param collector_endpoint String representing the Zipkin endpoint to be used + * @param collector holds the endpoint version and path information. * when making HTTP POST requests carrying spans. This value comes from the * Zipkin-related tracing configuration. */ - ReporterImpl(Driver& driver, Event::Dispatcher& dispatcher, - const std::string& collector_endpoint); + ReporterImpl(Driver& driver, Event::Dispatcher& dispatcher, const CollectorInfo& collector); /** * Implementation of Zipkin::Reporter::reportSpan(). @@ -172,7 +189,7 @@ class ReporterImpl : public Reporter, Http::AsyncClient::Callbacks { * * @param span The span to be buffered. */ - void reportSpan(const Span& span) override; + void reportSpan(Span&& span) override; // Http::AsyncClient::Callbacks. // The callbacks below record Zipkin-span-related stats. @@ -184,14 +201,14 @@ class ReporterImpl : public Reporter, Http::AsyncClient::Callbacks { * * @param driver ZipkinDriver to be associated with the reporter. * @param dispatcher Controls the timer used to flush buffered spans. - * @param collector_endpoint String representing the Zipkin endpoint to be used + * @param collector holds the endpoint version and path information. * when making HTTP POST requests carrying spans. This value comes from the * Zipkin-related tracing configuration. * * @return Pointer to the newly-created ZipkinReporter. */ static ReporterPtr NewInstance(Driver& driver, Event::Dispatcher& dispatcher, - const std::string& collector_endpoint); + const CollectorInfo& collector); private: /** @@ -206,8 +223,8 @@ class ReporterImpl : public Reporter, Http::AsyncClient::Callbacks { Driver& driver_; Event::TimerPtr flush_timer_; - SpanBuffer span_buffer_; - const std::string collector_endpoint_; + const CollectorInfo collector_; + SpanBufferPtr span_buffer_; }; } // namespace Zipkin } // namespace Tracers diff --git a/test/extensions/tracers/zipkin/BUILD b/test/extensions/tracers/zipkin/BUILD index 9f98e8c3ba15..a481ea737220 100644 --- a/test/extensions/tracers/zipkin/BUILD +++ b/test/extensions/tracers/zipkin/BUILD @@ -31,6 +31,7 @@ envoy_extension_cc_test( "//source/common/common:utility_lib", "//source/common/network:address_lib", "//source/common/network:utility_lib", + "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_lib", "//source/extensions/tracers/zipkin:zipkin_lib", "//test/mocks:common_lib", diff --git a/test/extensions/tracers/zipkin/config_test.cc b/test/extensions/tracers/zipkin/config_test.cc index b62865dd2601..8b211fa74d10 100644 --- a/test/extensions/tracers/zipkin/config_test.cc +++ b/test/extensions/tracers/zipkin/config_test.cc @@ -49,7 +49,8 @@ TEST(ZipkinTracerConfigTest, ZipkinHttpTracerWithTypedConfig) { typed_config: "@type": type.googleapis.com/envoy.config.trace.v2.ZipkinConfig collector_cluster: fake_cluster - collector_endpoint: /api/v1/spans + collector_endpoint: /api/v2/spans + collector_endpoint_version: HTTP_PROTO )EOF"; envoy::config::trace::v2::Tracing configuration; diff --git a/test/extensions/tracers/zipkin/span_buffer_test.cc b/test/extensions/tracers/zipkin/span_buffer_test.cc index 320b6a909bb0..1ef6e88e6485 100644 --- a/test/extensions/tracers/zipkin/span_buffer_test.cc +++ b/test/extensions/tracers/zipkin/span_buffer_test.cc @@ -1,3 +1,5 @@ +#include "common/network/utility.h" + #include "extensions/tracers/zipkin/span_buffer.h" #include "test/test_common/test_time.h" @@ -10,104 +12,341 @@ namespace Tracers { namespace Zipkin { namespace { -TEST(ZipkinSpanBufferTest, defaultConstructorEndToEnd) { +enum class IpType { V4, V6 }; + +Endpoint createEndpoint(const IpType ip_type) { + Endpoint endpoint; + endpoint.setAddress(ip_type == IpType::V6 + ? Envoy::Network::Utility::parseInternetAddress( + "2001:db8:85a3::8a2e:370:4444", 7334, true) + : Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 8080, false)); + endpoint.setServiceName("service1"); + return endpoint; +} + +Annotation createAnnotation(const absl::string_view value, const IpType ip_type) { + Annotation annotation; + annotation.setValue(value.data()); + annotation.setTimestamp(1566058071601051); + annotation.setEndpoint(createEndpoint(ip_type)); + return annotation; +} + +BinaryAnnotation createTag() { + BinaryAnnotation tag; + tag.setKey("component"); + tag.setValue("proxy"); + return tag; +} + +Span createSpan(const std::vector& annotation_values, const IpType ip_type) { + DangerousDeprecatedTestTime test_time; + Span span(test_time.timeSystem()); + span.setId(1); + span.setTraceId(1); + span.setDuration(100); + std::vector annotations; + annotations.reserve(annotation_values.size()); + for (absl::string_view value : annotation_values) { + annotations.push_back(createAnnotation(value, ip_type)); + } + span.setAnnotations(annotations); + span.setBinaryAnnotations({createTag()}); + return span; +} + +void expectSerializedBuffer(SpanBuffer& buffer, const bool delay_allocation, + const std::vector& expected_list) { DangerousDeprecatedTestTime test_time; - SpanBuffer buffer; EXPECT_EQ(0ULL, buffer.pendingSpans()); - EXPECT_EQ("[]", buffer.toStringifiedJsonArray()); + EXPECT_EQ("[]", buffer.serialize()); + + if (delay_allocation) { + EXPECT_FALSE(buffer.addSpan(createSpan({"cs", "sr"}, IpType::V4))); + buffer.allocateBuffer(expected_list.size() + 1); + } + + // Add span after allocation, but missing required annotations should be false. EXPECT_FALSE(buffer.addSpan(Span(test_time.timeSystem()))); + EXPECT_FALSE(buffer.addSpan(createSpan({"aa"}, IpType::V4))); - buffer.allocateBuffer(2); - EXPECT_EQ(0ULL, buffer.pendingSpans()); - EXPECT_EQ("[]", buffer.toStringifiedJsonArray()); - - buffer.addSpan(Span(test_time.timeSystem())); - EXPECT_EQ(1ULL, buffer.pendingSpans()); - std::string expected_json_array_string = "[{" - R"("traceId":"0000000000000000",)" - R"("name":"",)" - R"("id":"0000000000000000",)" - R"("annotations":[],)" - R"("binaryAnnotations":[])" - "}]"; - EXPECT_EQ(expected_json_array_string, buffer.toStringifiedJsonArray()); + for (uint64_t i = 0; i < expected_list.size(); i++) { + buffer.addSpan(createSpan({"cs", "sr"}, IpType::V4)); + EXPECT_EQ(i + 1, buffer.pendingSpans()); + EXPECT_EQ(expected_list.at(i), buffer.serialize()); + } - buffer.clear(); - EXPECT_EQ(0ULL, buffer.pendingSpans()); - EXPECT_EQ("[]", buffer.toStringifiedJsonArray()); - - buffer.addSpan(Span(test_time.timeSystem())); - buffer.addSpan(Span(test_time.timeSystem())); - expected_json_array_string = "[" - "{" - R"("traceId":"0000000000000000",)" - R"("name":"",)" - R"("id":"0000000000000000",)" - R"("annotations":[],)" - R"("binaryAnnotations":[])" - "}," - "{" - R"("traceId":"0000000000000000",)" - R"("name":"",)" - R"("id":"0000000000000000",)" - R"("annotations":[],)" - R"("binaryAnnotations":[])" - "}" - "]"; - EXPECT_EQ(2ULL, buffer.pendingSpans()); - EXPECT_EQ(expected_json_array_string, buffer.toStringifiedJsonArray()); + // Add a valid span. Valid means can be serialized to v2. + EXPECT_TRUE(buffer.addSpan(createSpan({"cs"}, IpType::V4))); + // While the span is valid, however the buffer is full. + EXPECT_FALSE(buffer.addSpan(createSpan({"cs", "sr"}, IpType::V4))); buffer.clear(); EXPECT_EQ(0ULL, buffer.pendingSpans()); - EXPECT_EQ("[]", buffer.toStringifiedJsonArray()); + EXPECT_EQ("[]", buffer.serialize()); } -TEST(ZipkinSpanBufferTest, sizeConstructorEndtoEnd) { - DangerousDeprecatedTestTime test_time; - SpanBuffer buffer(2); +template std::string serializedMessageToJson(const std::string& serialized) { + Type message; + message.ParseFromString(serialized); + std::string json; + Protobuf::util::MessageToJsonString(message, &json); + return json; +} - EXPECT_EQ(0ULL, buffer.pendingSpans()); - EXPECT_EQ("[]", buffer.toStringifiedJsonArray()); - - buffer.addSpan(Span(test_time.timeSystem())); - EXPECT_EQ(1ULL, buffer.pendingSpans()); - std::string expected_json_array_string = "[{" - R"("traceId":"0000000000000000",)" - R"("name":"",)" - R"("id":"0000000000000000",)" - R"("annotations":[],)" - R"("binaryAnnotations":[])" - "}]"; - EXPECT_EQ(expected_json_array_string, buffer.toStringifiedJsonArray()); +TEST(ZipkinSpanBufferTest, ConstructBuffer) { + const std::string expected1 = R"([{"traceId":"0000000000000001",)" + R"("name":"",)" + R"("id":"0000000000000001",)" + R"("duration":100,)" + R"("annotations":[{"timestamp":1566058071601051,)" + R"("value":"cs",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}},)" + R"({"timestamp":1566058071601051,)" + R"("value":"sr",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}}],)" + R"("binaryAnnotations":[{"key":"component",)" + R"("value":"proxy"}]}])"; - buffer.clear(); - EXPECT_EQ(0ULL, buffer.pendingSpans()); - EXPECT_EQ("[]", buffer.toStringifiedJsonArray()); - - buffer.addSpan(Span(test_time.timeSystem())); - buffer.addSpan(Span(test_time.timeSystem())); - expected_json_array_string = "[" - "{" - R"("traceId":"0000000000000000",)" - R"("name":"",)" - R"("id":"0000000000000000",)" - R"("annotations":[],)" - R"("binaryAnnotations":[])" - "}," - "{" - R"("traceId":"0000000000000000",)" - R"("name":"",)" - R"("id":"0000000000000000",)" - R"("annotations":[],)" - R"("binaryAnnotations":[])" - "}]"; - EXPECT_EQ(2ULL, buffer.pendingSpans()); - EXPECT_EQ(expected_json_array_string, buffer.toStringifiedJsonArray()); + const std::string expected2 = R"([{"traceId":"0000000000000001",)" + R"("name":"",)" + R"("id":"0000000000000001",)" + R"("duration":100,)" + R"("annotations":[{"timestamp":1566058071601051,)" + R"("value":"cs",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}},)" + R"({"timestamp":1566058071601051,)" + R"("value":"sr",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}}],)" + R"("binaryAnnotations":[{"key":"component",)" + R"("value":"proxy"}]},)" + R"({"traceId":"0000000000000001",)" + R"("name":"",)" + R"("id":"0000000000000001",)" + R"("duration":100,)" + R"("annotations":[{"timestamp":1566058071601051,)" + R"("value":"cs",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}},)" + R"({"timestamp":1566058071601051,)" + R"("value":"sr",)" + R"("endpoint":{"ipv4":"1.2.3.4",)" + R"("port":8080,)" + R"("serviceName":"service1"}}],)" + R"("binaryAnnotations":[{"key":"component",)" + R"("value":"proxy"}]}])"; + const bool shared = true; + const bool delay_allocation = true; - buffer.clear(); - EXPECT_EQ(0ULL, buffer.pendingSpans()); - EXPECT_EQ("[]", buffer.toStringifiedJsonArray()); + SpanBuffer buffer1(envoy::config::trace::v2::ZipkinConfig::HTTP_JSON_V1, shared); + expectSerializedBuffer(buffer1, delay_allocation, {expected1, expected2}); + + // Prepare 3 slots, since we will add one more inside the `expectSerializedBuffer` function. + SpanBuffer buffer2(envoy::config::trace::v2::ZipkinConfig::HTTP_JSON_V1, shared, 3); + expectSerializedBuffer(buffer2, !delay_allocation, {expected1, expected2}); +} + +TEST(ZipkinSpanBufferTest, SerializeSpan) { + const bool shared = true; + SpanBuffer buffer1(envoy::config::trace::v2::ZipkinConfig::HTTP_JSON, shared, 2); + buffer1.addSpan(createSpan({"cs"}, IpType::V4)); + EXPECT_EQ("[{" + R"("traceId":"0000000000000001",)" + R"("id":"0000000000000001",)" + R"("kind":"CLIENT",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"1.2.3.4",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]", + buffer1.serialize()); + + SpanBuffer buffer1_v6(envoy::config::trace::v2::ZipkinConfig::HTTP_JSON, shared, 2); + buffer1_v6.addSpan(createSpan({"cs"}, IpType::V6)); + EXPECT_EQ("[{" + R"("traceId":"0000000000000001",)" + R"("id":"0000000000000001",)" + R"("kind":"CLIENT",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv6":"2001:db8:85a3::8a2e:370:4444",)" + R"("port":7334},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]", + buffer1_v6.serialize()); + + SpanBuffer buffer2(envoy::config::trace::v2::ZipkinConfig::HTTP_JSON, shared, 2); + buffer2.addSpan(createSpan({"cs", "sr"}, IpType::V4)); + EXPECT_EQ("[{" + R"("traceId":"0000000000000001",)" + R"("id":"0000000000000001",)" + R"("kind":"CLIENT",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"1.2.3.4",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"}},)" + R"({)" + R"("traceId":"0000000000000001",)" + R"("id":"0000000000000001",)" + R"("kind":"SERVER",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"1.2.3.4",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"},)" + R"("shared":true)" + "}]", + buffer2.serialize()); + + SpanBuffer buffer3(envoy::config::trace::v2::ZipkinConfig::HTTP_JSON, !shared, 2); + buffer3.addSpan(createSpan({"cs", "sr"}, IpType::V4)); + EXPECT_EQ("[{" + R"("traceId":"0000000000000001",)" + R"("id":"0000000000000001",)" + R"("kind":"CLIENT",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"1.2.3.4",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"}},)" + R"({)" + R"("traceId":"0000000000000001",)" + R"("id":"0000000000000001",)" + R"("kind":"SERVER",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"1.2.3.4",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]", + buffer3.serialize()); + + SpanBuffer buffer4(envoy::config::trace::v2::ZipkinConfig::HTTP_PROTO, shared, 2); + buffer4.addSpan(createSpan({"cs"}, IpType::V4)); + EXPECT_EQ("{" + R"("spans":[{)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"CLIENT",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]}", + serializedMessageToJson(buffer4.serialize())); + + SpanBuffer buffer4_v6(envoy::config::trace::v2::ZipkinConfig::HTTP_PROTO, shared, 2); + buffer4_v6.addSpan(createSpan({"cs"}, IpType::V6)); + EXPECT_EQ("{" + R"("spans":[{)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"CLIENT",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv6":"IAENuIWjAAAAAIouA3BERA==",)" + R"("port":7334},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]}", + serializedMessageToJson(buffer4_v6.serialize())); + + SpanBuffer buffer5(envoy::config::trace::v2::ZipkinConfig::HTTP_PROTO, shared, 2); + buffer5.addSpan(createSpan({"cs", "sr"}, IpType::V4)); + EXPECT_EQ("{" + R"("spans":[{)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"CLIENT",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"}},)" + R"({)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"SERVER",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"},)" + R"("shared":true)" + "}]}", + serializedMessageToJson(buffer5.serialize())); + + SpanBuffer buffer6(envoy::config::trace::v2::ZipkinConfig::HTTP_PROTO, !shared, 2); + buffer6.addSpan(createSpan({"cs", "sr"}, IpType::V4)); + EXPECT_EQ("{" + R"("spans":[{)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"CLIENT",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"}},)" + R"({)" + R"("traceId":"AAAAAAAAAAE=",)" + R"("id":"AQAAAAAAAAA=",)" + R"("kind":"SERVER",)" + R"("timestamp":"1566058071601051",)" + R"("duration":"100",)" + R"("localEndpoint":{)" + R"("serviceName":"service1",)" + R"("ipv4":"AQIDBA==",)" + R"("port":8080},)" + R"("tags":{)" + R"("component":"proxy"})" + "}]}", + serializedMessageToJson(buffer6.serialize())); } } // namespace diff --git a/test/extensions/tracers/zipkin/tracer_test.cc b/test/extensions/tracers/zipkin/tracer_test.cc index 04fdbe3e07b4..bc5f9b9e32a9 100644 --- a/test/extensions/tracers/zipkin/tracer_test.cc +++ b/test/extensions/tracers/zipkin/tracer_test.cc @@ -28,7 +28,7 @@ namespace { class TestReporterImpl : public Reporter { public: TestReporterImpl(int value) : value_(value) {} - void reportSpan(const Span& span) override { reported_spans_.push_back(span); } + void reportSpan(Span&& span) override { reported_spans_.push_back(span); } int getValue() { return value_; } std::vector& reportedSpans() { return reported_spans_; } diff --git a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc index b8d18baca67e..5566ed107e27 100644 --- a/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc +++ b/test/extensions/tracers/zipkin/zipkin_tracer_impl_test.cc @@ -56,19 +56,71 @@ class ZipkinDriverTest : public testing::Test { random_, time_source_); } - void setupValidDriver() { + void setupValidDriver(const std::string& version) { EXPECT_CALL(cm_, get(Eq("fake_cluster"))).WillRepeatedly(Return(&cm_.thread_local_cluster_)); - const std::string yaml_string = R"EOF( + const std::string yaml_string = fmt::format(R"EOF( collector_cluster: fake_cluster collector_endpoint: /api/v1/spans - )EOF"; + collector_endpoint_version: {} + )EOF", + version); envoy::config::trace::v2::ZipkinConfig zipkin_config; TestUtility::loadFromYaml(yaml_string, zipkin_config); setup(zipkin_config, true); } + void expectValidFlushSeveralSpans(const std::string& version, const std::string& content_type) { + setupValidDriver(version); + + Http::MockAsyncClientRequest request(&cm_.async_client_); + Http::AsyncClient::Callbacks* callback; + const absl::optional timeout(std::chrono::seconds(5)); + + EXPECT_CALL(cm_.async_client_, + send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(timeout))) + .WillOnce( + Invoke([&](Http::MessagePtr& message, Http::AsyncClient::Callbacks& callbacks, + const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { + callback = &callbacks; + + EXPECT_EQ("/api/v1/spans", message->headers().Path()->value().getStringView()); + EXPECT_EQ("fake_cluster", message->headers().Host()->value().getStringView()); + EXPECT_EQ(content_type, message->headers().ContentType()->value().getStringView()); + + return &request; + })); + + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.min_flush_spans", 5)) + .Times(2) + .WillRepeatedly(Return(2)); + EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.request_timeout", 5000U)) + .WillOnce(Return(5000U)); + + Tracing::SpanPtr first_span = driver_->startSpan( + config_, request_headers_, operation_name_, start_time_, {Tracing::Reason::Sampling, true}); + first_span->finishSpan(); + + Tracing::SpanPtr second_span = driver_->startSpan( + config_, request_headers_, operation_name_, start_time_, {Tracing::Reason::Sampling, true}); + second_span->finishSpan(); + + Http::MessagePtr msg(new Http::ResponseMessageImpl( + Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "202"}}})); + + callback->onSuccess(std::move(msg)); + + EXPECT_EQ(2U, stats_.counter("tracing.zipkin.spans_sent").value()); + EXPECT_EQ(1U, stats_.counter("tracing.zipkin.reports_sent").value()); + EXPECT_EQ(0U, stats_.counter("tracing.zipkin.reports_dropped").value()); + EXPECT_EQ(0U, stats_.counter("tracing.zipkin.reports_failed").value()); + + callback->onFailure(Http::AsyncClient::FailureReason::Reset); + + EXPECT_EQ(1U, stats_.counter("tracing.zipkin.reports_failed").value()); + } + // TODO(#4160): Currently time_system_ is initialized from DangerousDeprecatedTestTime, which uses // real time, not mock-time. When that is switched to use mock-time instead, I think // generateRandom64() may not be as random as we want, and we'll need to inject entropy @@ -133,58 +185,23 @@ TEST_F(ZipkinDriverTest, InitializeDriver) { } TEST_F(ZipkinDriverTest, FlushSeveralSpans) { - setupValidDriver(); - - Http::MockAsyncClientRequest request(&cm_.async_client_); - Http::AsyncClient::Callbacks* callback; - const absl::optional timeout(std::chrono::seconds(5)); - - EXPECT_CALL(cm_.async_client_, - send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(timeout))) - .WillOnce( - Invoke([&](Http::MessagePtr& message, Http::AsyncClient::Callbacks& callbacks, - const Http::AsyncClient::RequestOptions&) -> Http::AsyncClient::Request* { - callback = &callbacks; - - EXPECT_EQ("/api/v1/spans", message->headers().Path()->value().getStringView()); - EXPECT_EQ("fake_cluster", message->headers().Host()->value().getStringView()); - EXPECT_EQ("application/json", - message->headers().ContentType()->value().getStringView()); - - return &request; - })); - - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.min_flush_spans", 5)) - .Times(2) - .WillRepeatedly(Return(2)); - EXPECT_CALL(runtime_.snapshot_, getInteger("tracing.zipkin.request_timeout", 5000U)) - .WillOnce(Return(5000U)); - - Tracing::SpanPtr first_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); - first_span->finishSpan(); - - Tracing::SpanPtr second_span = driver_->startSpan(config_, request_headers_, operation_name_, - start_time_, {Tracing::Reason::Sampling, true}); - second_span->finishSpan(); - - Http::MessagePtr msg(new Http::ResponseMessageImpl( - Http::HeaderMapPtr{new Http::TestHeaderMapImpl{{":status", "202"}}})); - - callback->onSuccess(std::move(msg)); + expectValidFlushSeveralSpans("HTTP_JSON_V1", "application/json"); +} - EXPECT_EQ(2U, stats_.counter("tracing.zipkin.spans_sent").value()); - EXPECT_EQ(1U, stats_.counter("tracing.zipkin.reports_sent").value()); - EXPECT_EQ(0U, stats_.counter("tracing.zipkin.reports_dropped").value()); - EXPECT_EQ(0U, stats_.counter("tracing.zipkin.reports_failed").value()); +TEST_F(ZipkinDriverTest, FlushSeveralSpansHttpJsonV1) { + expectValidFlushSeveralSpans("HTTP_JSON_V1", "application/json"); +} - callback->onFailure(Http::AsyncClient::FailureReason::Reset); +TEST_F(ZipkinDriverTest, FlushSeveralSpansHttpJson) { + expectValidFlushSeveralSpans("HTTP_JSON", "application/json"); +} - EXPECT_EQ(1U, stats_.counter("tracing.zipkin.reports_failed").value()); +TEST_F(ZipkinDriverTest, FlushSeveralSpansHttpProto) { + expectValidFlushSeveralSpans("HTTP_PROTO", "application/x-protobuf"); } TEST_F(ZipkinDriverTest, FlushOneSpanReportFailure) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); Http::MockAsyncClientRequest request(&cm_.async_client_); Http::AsyncClient::Callbacks* callback; @@ -226,7 +243,7 @@ TEST_F(ZipkinDriverTest, FlushOneSpanReportFailure) { } TEST_F(ZipkinDriverTest, FlushSpansTimer) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); const absl::optional timeout(std::chrono::seconds(5)); EXPECT_CALL(cm_.async_client_, @@ -253,7 +270,7 @@ TEST_F(ZipkinDriverTest, FlushSpansTimer) { } TEST_F(ZipkinDriverTest, NoB3ContextSampledTrue) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID)); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID)); @@ -267,7 +284,7 @@ TEST_F(ZipkinDriverTest, NoB3ContextSampledTrue) { } TEST_F(ZipkinDriverTest, NoB3ContextSampledFalse) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID)); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID)); @@ -281,7 +298,7 @@ TEST_F(ZipkinDriverTest, NoB3ContextSampledFalse) { } TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleTrue) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, Hex::uint64ToHex(generateRandom64())); @@ -297,7 +314,7 @@ TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleTrue) { } TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleFalse) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, Hex::uint64ToHex(generateRandom64())); @@ -313,7 +330,7 @@ TEST_F(ZipkinDriverTest, PropagateB3NoSampleDecisionSampleFalse) { } TEST_F(ZipkinDriverTest, PropagateB3NotSampled) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID)); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID)); @@ -335,7 +352,7 @@ TEST_F(ZipkinDriverTest, PropagateB3NotSampled) { } TEST_F(ZipkinDriverTest, PropagateB3NotSampledWithFalse) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID)); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID)); @@ -357,7 +374,7 @@ TEST_F(ZipkinDriverTest, PropagateB3NotSampledWithFalse) { } TEST_F(ZipkinDriverTest, PropagateB3SampledWithTrue) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_SPAN_ID)); EXPECT_EQ(nullptr, request_headers_.get(ZipkinCoreConstants::get().X_B3_TRACE_ID)); @@ -379,7 +396,7 @@ TEST_F(ZipkinDriverTest, PropagateB3SampledWithTrue) { } TEST_F(ZipkinDriverTest, PropagateB3SampleFalse) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, Hex::uint64ToHex(generateRandom64())); @@ -396,7 +413,7 @@ TEST_F(ZipkinDriverTest, PropagateB3SampleFalse) { } TEST_F(ZipkinDriverTest, ZipkinSpanTest) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); // ==== // Test effective setTag() @@ -476,7 +493,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanTest) { } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersTest) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); const std::string trace_id = Hex::uint64ToHex(generateRandom64()); const std::string span_id = Hex::uint64ToHex(generateRandom64()); @@ -500,7 +517,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersTest) { } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersEmptyParentSpanTest) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); // Root span so have same trace and span id const std::string id = Hex::uint64ToHex(generateRandom64()); @@ -521,7 +538,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3HeadersEmptyParentSpanTest) { } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3Headers128TraceIdTest) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); const uint64_t trace_id_high = generateRandom64(); const uint64_t trace_id_low = generateRandom64(); @@ -549,7 +566,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromB3Headers128TraceIdTest) { } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidTraceIdB3HeadersTest) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, std::string("xyz")); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, @@ -563,7 +580,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidTraceIdB3HeadersTest) { } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidSpanIdB3HeadersTest) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, Hex::uint64ToHex(generateRandom64())); @@ -577,7 +594,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidSpanIdB3HeadersTest) { } TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidParentIdB3HeadersTest) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, Hex::uint64ToHex(generateRandom64())); @@ -592,7 +609,7 @@ TEST_F(ZipkinDriverTest, ZipkinSpanContextFromInvalidParentIdB3HeadersTest) { } TEST_F(ZipkinDriverTest, ExplicitlySetSampledFalse) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, start_time_, {Tracing::Reason::Sampling, true}); @@ -609,7 +626,7 @@ TEST_F(ZipkinDriverTest, ExplicitlySetSampledFalse) { } TEST_F(ZipkinDriverTest, ExplicitlySetSampledTrue) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); Tracing::SpanPtr span = driver_->startSpan(config_, request_headers_, operation_name_, start_time_, {Tracing::Reason::Sampling, false}); @@ -626,7 +643,7 @@ TEST_F(ZipkinDriverTest, ExplicitlySetSampledTrue) { } TEST_F(ZipkinDriverTest, DuplicatedHeader) { - setupValidDriver(); + setupValidDriver("HTTP_JSON_V1"); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_TRACE_ID, Hex::uint64ToHex(generateRandom64())); request_headers_.addReferenceKey(ZipkinCoreConstants::get().X_B3_SPAN_ID, From 0b0aa3f81e6e58327966f8ffa49932c5999a8306 Mon Sep 17 00:00:00 2001 From: Jyoti Mahapatra <49211422+jyotimahapatra@users.noreply.github.com> Date: Fri, 30 Aug 2019 09:00:00 -0700 Subject: [PATCH 483/542] Route Checker tool Fix code coverage bug in proto based schema (#8101) Signed-off-by: Jyoti Mahapatra --- test/tools/router_check/router.cc | 36 +++++------ .../ComprehensiveRoutes.golden.proto.json | 63 +++++++++++++++++++ .../test/config/ComprehensiveRoutes.yaml | 9 ++- .../test/config/Weighted.golden.proto.pb_text | 4 +- test/tools/router_check/test/route_tests.sh | 6 ++ test/tools/router_check/validation.proto | 13 ++-- 6 files changed, 103 insertions(+), 28 deletions(-) create mode 100644 test/tools/router_check/test/config/ComprehensiveRoutes.golden.proto.json diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index 5eeadc7a1bb1..fbd9eaed7031 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -224,13 +224,13 @@ bool RouterCheckTool::compareCluster(ToolConfig& tool_config, const std::string& bool RouterCheckTool::compareCluster( ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) { - if (expected.cluster_name().empty()) { + if (!expected.has_cluster_name()) { return true; } if (tool_config.route_ == nullptr) { - return compareResults("", expected.cluster_name(), "cluster_name"); + return compareResults("", expected.cluster_name().value(), "cluster_name"); } - return compareCluster(tool_config, expected.cluster_name()); + return compareCluster(tool_config, expected.cluster_name().value()); } bool RouterCheckTool::compareVirtualCluster(ToolConfig& tool_config, const std::string& expected) { @@ -251,13 +251,13 @@ bool RouterCheckTool::compareVirtualCluster(ToolConfig& tool_config, const std:: bool RouterCheckTool::compareVirtualCluster( ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) { - if (expected.virtual_cluster_name().empty()) { + if (!expected.has_virtual_cluster_name()) { return true; } if (tool_config.route_ == nullptr) { - return compareResults("", expected.virtual_cluster_name(), "virtual_cluster_name"); + return compareResults("", expected.virtual_cluster_name().value(), "virtual_cluster_name"); } - return compareVirtualCluster(tool_config, expected.virtual_cluster_name()); + return compareVirtualCluster(tool_config, expected.virtual_cluster_name().value()); } bool RouterCheckTool::compareVirtualHost(ToolConfig& tool_config, const std::string& expected) { @@ -275,13 +275,13 @@ bool RouterCheckTool::compareVirtualHost(ToolConfig& tool_config, const std::str bool RouterCheckTool::compareVirtualHost( ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) { - if (expected.virtual_host_name().empty()) { + if (!expected.has_virtual_host_name()) { return true; } if (tool_config.route_ == nullptr) { - return compareResults("", expected.virtual_host_name(), "virtual_host_name"); + return compareResults("", expected.virtual_host_name().value(), "virtual_host_name"); } - return compareVirtualHost(tool_config, expected.virtual_host_name()); + return compareVirtualHost(tool_config, expected.virtual_host_name().value()); } bool RouterCheckTool::compareRewritePath(ToolConfig& tool_config, const std::string& expected) { @@ -306,13 +306,13 @@ bool RouterCheckTool::compareRewritePath(ToolConfig& tool_config, const std::str bool RouterCheckTool::compareRewritePath( ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) { - if (expected.path_rewrite().empty()) { + if (!expected.has_path_rewrite()) { return true; } if (tool_config.route_ == nullptr) { - return compareResults("", expected.path_rewrite(), "path_rewrite"); + return compareResults("", expected.path_rewrite().value(), "path_rewrite"); } - return compareRewritePath(tool_config, expected.path_rewrite()); + return compareRewritePath(tool_config, expected.path_rewrite().value()); } bool RouterCheckTool::compareRewriteHost(ToolConfig& tool_config, const std::string& expected) { @@ -337,13 +337,13 @@ bool RouterCheckTool::compareRewriteHost(ToolConfig& tool_config, const std::str bool RouterCheckTool::compareRewriteHost( ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) { - if (expected.host_rewrite().empty()) { + if (!expected.has_host_rewrite()) { return true; } if (tool_config.route_ == nullptr) { - return compareResults("", expected.host_rewrite(), "host_rewrite"); + return compareResults("", expected.host_rewrite().value(), "host_rewrite"); } - return compareRewriteHost(tool_config, expected.host_rewrite()); + return compareRewriteHost(tool_config, expected.host_rewrite().value()); } bool RouterCheckTool::compareRedirectPath(ToolConfig& tool_config, const std::string& expected) { @@ -361,13 +361,13 @@ bool RouterCheckTool::compareRedirectPath(ToolConfig& tool_config, const std::st bool RouterCheckTool::compareRedirectPath( ToolConfig& tool_config, const envoy::RouterCheckToolSchema::ValidationAssert& expected) { - if (expected.path_redirect().empty()) { + if (!expected.has_path_redirect()) { return true; } if (tool_config.route_ == nullptr) { - return compareResults("", expected.path_redirect(), "path_redirect"); + return compareResults("", expected.path_redirect().value(), "path_redirect"); } - return compareRedirectPath(tool_config, expected.path_redirect()); + return compareRedirectPath(tool_config, expected.path_redirect().value()); } bool RouterCheckTool::compareHeaderField( diff --git a/test/tools/router_check/test/config/ComprehensiveRoutes.golden.proto.json b/test/tools/router_check/test/config/ComprehensiveRoutes.golden.proto.json new file mode 100644 index 000000000000..ebc21381c347 --- /dev/null +++ b/test/tools/router_check/test/config/ComprehensiveRoutes.golden.proto.json @@ -0,0 +1,63 @@ +{ + "tests": [ + { + "test_name": "Test 1", + "input": { + "authority": "www.lyft.com", + "path": "/new_endpoint", + "method": "GET" + }, + "validate": { + "cluster_name": "www2", + "virtual_cluster_name": "other", + "virtual_host_name": "www2_host", + "path_rewrite": "/api/new_endpoint", + "host_rewrite": "www.lyft.com", + "path_redirect": "" + } + }, + { + "test_name": "Test 2", + "input": { + "authority": "www.lyft.com", + "path": "/", + "method": "GET" + }, + "validate": { + "cluster_name": "root_www2", + "virtual_cluster_name": "other", + "virtual_host_name": "www2_host", + "path_rewrite": "/", + "host_rewrite": "www.lyft.com", + "path_redirect": "" + } + }, + { + "test_name": "Test 3", + "input": { + "authority": "www.lyft.com", + "path": "/foobar", + "method": "GET" + }, + "validate": { + "cluster_name": "www2", + "virtual_cluster_name": "other", + "virtual_host_name": "www2_host", + "path_rewrite": "/foobar", + "host_rewrite": "www.lyft.com", + "path_redirect": "" + } + }, + { + "test_name": "Test 4", + "input": { + "authority": "www.lyft.com", + "path": "/users/123", + "method": "PUT" + }, + "validate": { + "virtual_cluster_name": "update_user" + } + } + ] +} diff --git a/test/tools/router_check/test/config/ComprehensiveRoutes.yaml b/test/tools/router_check/test/config/ComprehensiveRoutes.yaml index 0613410256ca..6efad99a099e 100644 --- a/test/tools/router_check/test/config/ComprehensiveRoutes.yaml +++ b/test/tools/router_check/test/config/ComprehensiveRoutes.yaml @@ -17,6 +17,11 @@ virtual_hosts: route: cluster: www2 virtual_clusters: - - pattern: ^/users/\d+$ - method: PUT + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/users/\d+$ + - name: :method + exact_match: PUT name: update_user diff --git a/test/tools/router_check/test/config/Weighted.golden.proto.pb_text b/test/tools/router_check/test/config/Weighted.golden.proto.pb_text index c2ab5d18c377..fa7dedcb27c5 100644 --- a/test/tools/router_check/test/config/Weighted.golden.proto.pb_text +++ b/test/tools/router_check/test/config/Weighted.golden.proto.pb_text @@ -8,7 +8,7 @@ tests { method: "GET" } validate: { - path_redirect: "" + path_redirect: { value: "" } } } @@ -21,6 +21,6 @@ tests { random_value: 115 } validate: { - cluster_name: "cluster1" + cluster_name: { value: "cluster1" } } } \ No newline at end of file diff --git a/test/tools/router_check/test/route_tests.sh b/test/tools/router_check/test/route_tests.sh index 0b06cb7425bf..1e85bea3f598 100755 --- a/test/tools/router_check/test/route_tests.sh +++ b/test/tools/router_check/test/route_tests.sh @@ -24,6 +24,12 @@ if [[ "${COVERAGE_OUTPUT}" != *"Current route coverage: "* ]] ; then exit 1 fi +COMP_COVERAGE_CMD="${PATH_BIN} -c ${PATH_CONFIG}/ComprehensiveRoutes.yaml -t ${PATH_CONFIG}/ComprehensiveRoutes.golden.proto.json --details --useproto -f " +COVERAGE_OUTPUT=$($COMP_COVERAGE_CMD "100" "--covall" 2>&1) || echo "${COVERAGE_OUTPUT:-no-output}" +if [[ "${COVERAGE_OUTPUT}" != *"Current route coverage: 100%"* ]] ; then + exit 1 +fi + COMP_COVERAGE_CMD="${PATH_BIN} ${PATH_CONFIG}/ComprehensiveRoutes.yaml ${PATH_CONFIG}/ComprehensiveRoutes.golden.json --details -f " COVERAGE_OUTPUT=$($COMP_COVERAGE_CMD "100" "--covall" 2>&1) || echo "${COVERAGE_OUTPUT:-no-output}" if [[ "${COVERAGE_OUTPUT}" != *"Current route coverage: 100%"* ]] ; then diff --git a/test/tools/router_check/validation.proto b/test/tools/router_check/validation.proto index 9c86153cd3e9..4ff3ab35f434 100644 --- a/test/tools/router_check/validation.proto +++ b/test/tools/router_check/validation.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package envoy.RouterCheckToolSchema; import "envoy/api/v2/core/base.proto"; +import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; // [#protodoc-title: RouterCheckTool Validation] @@ -78,22 +79,22 @@ message ValidationInput { // For example, to test that no cluster match is expected use {“cluster_name”: “”}. message ValidationAssert { // Match the cluster name. - string cluster_name = 1; + google.protobuf.StringValue cluster_name = 1; // Match the virtual cluster name. - string virtual_cluster_name = 2; + google.protobuf.StringValue virtual_cluster_name = 2; // Match the virtual host name. - string virtual_host_name = 3; + google.protobuf.StringValue virtual_host_name = 3; // Match the host header field after rewrite. - string host_rewrite = 4; + google.protobuf.StringValue host_rewrite = 4; // Match the path header field after rewrite. - string path_rewrite = 5; + google.protobuf.StringValue path_rewrite = 5; // Match the returned redirect path. - string path_redirect = 6; + google.protobuf.StringValue path_redirect = 6; // Match the listed header fields. // Examples header fields include the “:path”, “cookie”, and “date” fields. From 7960564746700312b4fec21711c4387794f5ba06 Mon Sep 17 00:00:00 2001 From: Xin Date: Fri, 30 Aug 2019 13:28:27 -0400 Subject: [PATCH 484/542] [hcm] Add scoped RDS routing into HCM (#7762) Description: add Scoped RDS routing logic into HCM. Changes include: * in ActiveStream constructor latch a ScopedConfig impl to the activeStream if SRDS is enabled * in the beginning of ActiveStream::decodeHeaders(headers, end_stream), get routeConfig from latched ScopedConfig impl. This PR is the 3rd in the srds impl PR chain: [#7704, #7451, this]. Risk Level: Medium Testing: unit test and integration tests. Release Notes: Add scoped RDS routing support into HCM. Signed-off-by: Xin Zhuang --- api/envoy/api/v2/srds.proto | 55 ++- .../intro/arch_overview/http/http_routing.rst | 29 ++ docs/root/intro/version_history.rst | 5 +- include/envoy/stream_info/stream_info.h | 2 + source/common/http/BUILD | 1 + source/common/http/conn_manager_impl.cc | 57 ++- source/common/http/conn_manager_impl.h | 7 +- source/common/router/scoped_rds.cc | 2 +- test/common/grpc/grpc_client_integration.h | 26 +- test/common/http/BUILD | 15 +- .../http/conn_manager_impl_fuzz_test.cc | 25 +- test/common/http/conn_manager_impl_test.cc | 282 ++++++++++++++- test/common/router/scoped_rds_test.cc | 2 +- .../scoped_rds_integration_test.cc | 340 +++++++++++++++--- test/mocks/router/mocks.h | 1 + test/test_common/utility.cc | 6 +- test/test_common/utility.h | 4 +- 17 files changed, 719 insertions(+), 140 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index 9038cb1e3257..617fdf9ac644 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -2,36 +2,27 @@ syntax = "proto3"; package envoy.api.v2; -option java_outer_classname = "SrdsProto"; -option java_package = "io.envoyproxy.envoy.api.v2"; -option java_multiple_files = true; -option java_generic_services = true; - import "envoy/api/v2/discovery.proto"; - +import "gogoproto/gogo.proto"; import "google/api/annotations.proto"; - import "validate/validate.proto"; -import "gogoproto/gogo.proto"; +option java_outer_classname = "SrdsProto"; +option java_package = "io.envoyproxy.envoy.api.v2"; +option java_multiple_files = true; +option java_generic_services = true; option (gogoproto.equal_all) = true; // [#protodoc-title: HTTP scoped routing configuration] // * Routing :ref:`architecture overview ` // -// .. attention:: -// -// The Scoped RDS API is not yet fully implemented and *should not* be enabled in -// :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v2.HttpConnectionManager`. -// -// TODO(AndresGuedez): Update :ref:`arch_overview_http_routing` with scoped routing overview and -// configuration details. - // The Scoped Routes Discovery Service (SRDS) API distributes -// :ref:`ScopedRouteConfiguration` resources. Each -// ScopedRouteConfiguration resource represents a "routing scope" containing a mapping that allows -// the HTTP connection manager to dynamically assign a routing table (specified via -// a :ref:`RouteConfiguration` message) to each HTTP request. +// :ref:`ScopedRouteConfiguration` +// resources. Each ScopedRouteConfiguration resource represents a "routing +// scope" containing a mapping that allows the HTTP connection manager to +// dynamically assign a routing table (specified via a +// :ref:`RouteConfiguration` message) to each +// HTTP request. // [#proto-status: experimental] service ScopedRoutesDiscoveryService { rpc StreamScopedRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { @@ -52,9 +43,9 @@ service ScopedRoutesDiscoveryService { // :ref:`Key` to a // :ref:`envoy_api_msg_RouteConfiguration` (identified by its resource name). // -// The HTTP connection manager builds up a table consisting of these Key to RouteConfiguration -// mappings, and looks up the RouteConfiguration to use per request according to the algorithm -// specified in the +// The HTTP connection manager builds up a table consisting of these Key to +// RouteConfiguration mappings, and looks up the RouteConfiguration to use per +// request according to the algorithm specified in the // :ref:`scope_key_builder` // assigned to the HttpConnectionManager. // @@ -104,8 +95,8 @@ service ScopedRoutesDiscoveryService { // Host: foo.com // X-Route-Selector: vip=172.10.10.20 // -// would result in the routing table defined by the `route-config1` RouteConfiguration being -// assigned to the HTTP request/stream. +// would result in the routing table defined by the `route-config1` +// RouteConfiguration being assigned to the HTTP request/stream. // // [#comment:next free field: 4] // [#proto-status: experimental] @@ -115,8 +106,9 @@ message ScopedRouteConfiguration { // Specifies a key which is matched against the output of the // :ref:`scope_key_builder` - // specified in the HttpConnectionManager. The matching is done per HTTP request and is dependent - // on the order of the fragments contained in the Key. + // specified in the HttpConnectionManager. The matching is done per HTTP + // request and is dependent on the order of the fragments contained in the + // Key. message Key { message Fragment { oneof type { @@ -127,14 +119,15 @@ message ScopedRouteConfiguration { } } - // The ordered set of fragments to match against. The order must match the fragments in the - // corresponding + // The ordered set of fragments to match against. The order must match the + // fragments in the corresponding // :ref:`scope_key_builder`. repeated Fragment fragments = 1 [(validate.rules).repeated .min_items = 1]; } - // The resource name to use for a :ref:`envoy_api_msg_DiscoveryRequest` to an RDS server to - // fetch the :ref:`envoy_api_msg_RouteConfiguration` associated with this scope. + // The resource name to use for a :ref:`envoy_api_msg_DiscoveryRequest` to an + // RDS server to fetch the :ref:`envoy_api_msg_RouteConfiguration` associated + // with this scope. string route_configuration_name = 2 [(validate.rules).string.min_bytes = 1]; // The key to match against. diff --git a/docs/root/intro/arch_overview/http/http_routing.rst b/docs/root/intro/arch_overview/http/http_routing.rst index 6a191be26821..574efa611ecf 100644 --- a/docs/root/intro/arch_overview/http/http_routing.rst +++ b/docs/root/intro/arch_overview/http/http_routing.rst @@ -50,6 +50,35 @@ request. The router filter supports the following features: * :ref:`Hash policy ` based routing. * :ref:`Absolute urls ` are supported for non-tls forward proxies. +.. _arch_overview_http_routing_route_scope: + +Route Scope +-------------- + +Scoped routing enables Envoy to put constraints on search space of domains and route rules. +A :ref:`Route Scope` associates a key with a :ref:`route table `. +For each request, a scope key is computed dynamically by the HTTP connection manager to pick the :ref:`route table`. + +The Scoped RDS (SRDS) API contains a set of :ref:`Scopes ` resources, each defining independent routing configuration, +along with a :ref:`ScopeKeyBuilder ` +defining the key construction algorithm used by Envoy to look up the scope corresponding to each request. + +For example, for the following scoped route configuration, Envoy will look into the "addr" header value, split the header value by ";" first, and use the first value for key 'x-foo-key' as the scope key. +If the "addr" header value is "foo=1;x-foo-key=127.0.0.1;x-bar-key=1.1.1.1", then "127.0.0.1" will be computed as the scope key to look up for corresponding route configuration. + +.. code-block:: yaml + + name: scope_by_addr + fragments: + - header_value_extractor: + name: Addr + element_separator: ; + element: + key: x-foo-key + separator: = + +.. _arch_overview_http_routing_route_table: + Route table ----------- diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index c4e6c2f2ac52..653bc4f65d6d 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -43,16 +43,17 @@ Version history ` for more information. * rbac: added conditions to the policy, see :ref:`condition `. * router: added :ref:`rq_retry_skipped_request_not_complete ` counter stat to router stats. +* router: :ref:`Scoped routing ` is supported. * router check tool: add coverage reporting & enforcement. * router check tool: add comprehensive coverage reporting. -* tracing: added support to the Zipkin reporter for sending list of spans as Zipkin JSON v2 and protobuf message over HTTP. * router check tool: add deprecated field check. * tls: added verification of IP address SAN fields in certificates against configured SANs in the +* tracing: added support to the Zipkin reporter for sending list of spans as Zipkin JSON v2 and protobuf message over HTTP. certificate validation context. * tracing: added tags for gRPC response status and meesage. +* upstream: added :ref:`an option ` that allows draining HTTP, TCP connection pools on cluster membership change. * upstream: added network filter chains to upstream connections, see :ref:`filters`. * upstream: use p2c to select hosts for least-requests load balancers if all host weights are the same, even in cases where weights are not equal to 1. -* upstream: added :ref:`an option ` that allows draining HTTP, TCP connection pools on cluster membership change. * zookeeper: parse responses and emit latency stats. 1.11.1 (August 13, 2019) diff --git a/include/envoy/stream_info/stream_info.h b/include/envoy/stream_info/stream_info.h index 1558503ccfe5..442d8a1699a1 100644 --- a/include/envoy/stream_info/stream_info.h +++ b/include/envoy/stream_info/stream_info.h @@ -109,6 +109,8 @@ struct ResponseCodeDetailValues { // The request was rejected because it attempted an unsupported upgrade. const std::string UpgradeFailed = "upgrade_failed"; + // The request was rejected by the HCM because there was no route configuration found. + const std::string RouteConfigurationNotFound = "route_configuration_not_found"; // The request was rejected by the router filter because there was no route found. const std::string RouteNotFound = "route_not_found"; // A direct response was generated by the router filter. diff --git a/source/common/http/BUILD b/source/common/http/BUILD index a41fdf2bb925..df01a16a57db 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -164,6 +164,7 @@ envoy_cc_library( "//include/envoy/router:rds_interface", "//include/envoy/router:scopes_interface", "//include/envoy/runtime:runtime_interface", + "//include/envoy/server:admin_interface", "//include/envoy/server:overload_manager_interface", "//include/envoy/ssl:connection_interface", "//include/envoy/stats:stats_interface", diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index c830114f5107..102373f9cadf 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -12,6 +12,7 @@ #include "envoy/event/dispatcher.h" #include "envoy/network/drain_decision.h" #include "envoy/router/router.h" +#include "envoy/server/admin.h" #include "envoy/ssl/connection.h" #include "envoy/stats/scope.h" #include "envoy/tracing/http_tracer.h" @@ -431,12 +432,27 @@ void ConnectionManagerImpl::chargeTracingStats(const Tracing::Reason& tracing_re ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connection_manager) : connection_manager_(connection_manager), - snapped_route_config_(connection_manager.config_.routeConfigProvider()->config()), stream_id_(connection_manager.random_generator_.random()), request_response_timespan_(new Stats::Timespan( connection_manager_.stats_.named_.downstream_rq_time_, connection_manager_.timeSource())), stream_info_(connection_manager_.codec_->protocol(), connection_manager_.timeSource()), upstream_options_(std::make_shared()) { + // For Server::Admin, no routeConfigProvider or SRDS route provider is used. + ASSERT(dynamic_cast(&connection_manager_.config_) != nullptr || + ((connection_manager.config_.routeConfigProvider() == nullptr && + connection_manager.config_.scopedRouteConfigProvider() != nullptr) || + (connection_manager.config_.routeConfigProvider() != nullptr && + connection_manager.config_.scopedRouteConfigProvider() == nullptr)), + "Either routeConfigProvider or scopedRouteConfigProvider should be set in " + "ConnectionManagerImpl."); + if (connection_manager.config_.routeConfigProvider() != nullptr) { + snapped_route_config_ = connection_manager.config_.routeConfigProvider()->config(); + } else if (connection_manager.config_.scopedRouteConfigProvider() != nullptr) { + snapped_scoped_routes_config_ = + connection_manager_.config_.scopedRouteConfigProvider()->config(); + ASSERT(snapped_scoped_routes_config_ != nullptr, + "Scoped rds provider returns null for scoped routes config."); + } ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); @@ -613,6 +629,17 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(HeaderMapPtr&& headers, ScopeTrackerScopeState scope(this, connection_manager_.read_callbacks_->connection().dispatcher()); request_headers_ = std::move(headers); + // For Admin thread, we don't use routeConfigProvider or SRDS route provider. + if (dynamic_cast(&connection_manager_.config_) == nullptr && + connection_manager_.config_.scopedRouteConfigProvider() != nullptr) { + ASSERT(snapped_route_config_ == nullptr, + "Route config already latched to the active stream when scoped RDS is enabled."); + // We need to snap snapped_route_config_ here as it's used in mutateRequestHeaders later. + if (!snapScopedRouteConfig()) { + return; + } + } + if (Http::Headers::get().MethodValues.Head == request_headers_->Method()->value().getStringView()) { is_head_request_ = true; @@ -1220,10 +1247,36 @@ void ConnectionManagerImpl::startDrainSequence() { drain_timer_->enableTimer(config_.drainTimeout()); } +bool ConnectionManagerImpl::ActiveStream::snapScopedRouteConfig() { + ASSERT(request_headers_ != nullptr, + "Try to snap scoped route config when there is no request headers."); + + snapped_route_config_ = snapped_scoped_routes_config_->getRouteConfig(*request_headers_); + // NOTE: if a RDS subscription hasn't got a RouteConfiguration back, a Router::NullConfigImpl is + // returned, in that case we let it pass. + if (snapped_route_config_ == nullptr) { + ENVOY_STREAM_LOG(trace, "can't find SRDS scope.", *this); + // Stop decoding now. + maybeEndDecode(true); + sendLocalReply(Grpc::Common::hasGrpcContentType(*request_headers_), Http::Code::NotFound, + "route scope not found", nullptr, is_head_request_, absl::nullopt, + StreamInfo::ResponseCodeDetails::get().RouteConfigurationNotFound); + return false; + } + return true; +} + void ConnectionManagerImpl::ActiveStream::refreshCachedRoute() { Router::RouteConstSharedPtr route; if (request_headers_ != nullptr) { - route = snapped_route_config_->route(*request_headers_, stream_id_); + if (dynamic_cast(&connection_manager_.config_) == nullptr && + connection_manager_.config_.scopedRouteConfigProvider() != nullptr) { + // NOTE: re-select scope as well in case the scope key header has been changed by a filter. + snapScopedRouteConfig(); + } + if (snapped_route_config_ != nullptr) { + route = snapped_route_config_->route(*request_headers_, stream_id_); + } } stream_info_.route_entry_ = route ? route->routeEntry() : nullptr; cached_route_ = std::move(route); diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 84dde922406e..661871ea991a 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -496,6 +496,11 @@ class ConnectionManagerImpl : Logger::Loggable, void traceRequest(); + // Updates the snapped_route_config_ if scope found, or ends the stream by + // sending local reply. + // Returns true if scoped route config snapped, false otherwise. + bool snapScopedRouteConfig(); + void refreshCachedRoute(); // Pass on watermark callbacks to watermark subscribers. This boils down to passing watermark @@ -585,7 +590,7 @@ class ConnectionManagerImpl : Logger::Loggable, ConnectionManagerImpl& connection_manager_; Router::ConfigConstSharedPtr snapped_route_config_; - Router::ScopedConfigConstSharedPtr snapped_scoped_route_config_; + Router::ScopedConfigConstSharedPtr snapped_scoped_routes_config_; Tracing::SpanPtr active_span_; const uint64_t stream_id_; StreamEncoder* response_encoder_{}; diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 989e284668c4..9c710540930f 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -316,7 +316,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( *to_remove_repeated.Add() = scoped_route.first; } onConfigUpdate(to_add_repeated, to_remove_repeated, version_info); -} // namespace Router +} ScopedRdsConfigProvider::ScopedRdsConfigProvider( ScopedRdsConfigSubscriptionSharedPtr&& subscription) diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index ff6d4d3a7b53..bdfc0c6ae1ba 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -43,7 +43,6 @@ class GrpcClientIntegrationParamTest : public BaseGrpcClientIntegrationParamTest, public testing::TestWithParam> { public: - ~GrpcClientIntegrationParamTest() override = default; static std::string protocolTestParamsToString( const ::testing::TestParamInfo>& p) { return fmt::format("{}_{}", @@ -54,10 +53,26 @@ class GrpcClientIntegrationParamTest ClientType clientType() const override { return std::get<1>(GetParam()); } }; +class DeltaSotwGrpcClientIntegrationParamTest + : public BaseGrpcClientIntegrationParamTest, + public testing::TestWithParam> { +public: + static std::string protocolTestParamsToString( + const ::testing::TestParamInfo>& + p) { + return fmt::format("{}_{}", + std::get<0>(p.param) == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6", + std::get<1>(p.param) == ClientType::GoogleGrpc ? "GoogleGrpc" : "EnvoyGrpc", + std::get<2>(p.param) ? "Delta" : "StateOfTheWorld"); + } + Network::Address::IpVersion ipVersion() const override { return std::get<0>(GetParam()); } + ClientType clientType() const override { return std::get<1>(GetParam()); } + bool isDelta() { return std::get<2>(GetParam()); } +}; + class DeltaSotwIntegrationParamTest : public testing::TestWithParam> { public: - ~DeltaSotwIntegrationParamTest() override = default; static std::string protocolTestParamsToString( const ::testing::TestParamInfo>& p) { return fmt::format("{}_{}_{}", @@ -84,10 +99,17 @@ class DeltaSotwIntegrationParamTest #define GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc)) +#define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ + testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ + testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc), \ + testing::Bool()) #else #define GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc)) +#define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ + testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ + testing::Values(Grpc::ClientType::EnvoyGrpc), testing::Bool()) #endif // ENVOY_GOOGLE_GRPC #define DELTA_INTEGRATION_PARAMS \ diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 7982b15c462f..1766f89aa0d7 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -129,17 +129,6 @@ envoy_cc_test_library( ], ) -envoy_cc_test_library( - name = "conn_manager_impl_common_lib", - hdrs = ["conn_manager_impl_common.h"], - deps = [ - "//include/envoy/common:time_interface", - "//include/envoy/config:config_provider_interface", - "//include/envoy/router:rds_interface", - "//test/mocks/router:router_mocks", - ], -) - envoy_proto_library( name = "conn_manager_impl_fuzz_proto", srcs = ["conn_manager_impl_fuzz.proto"], @@ -153,7 +142,6 @@ envoy_cc_fuzz_test( srcs = ["conn_manager_impl_fuzz_test.cc"], corpus = "conn_manager_impl_corpus", deps = [ - ":conn_manager_impl_common_lib", ":conn_manager_impl_fuzz_proto_cc", "//source/common/common:empty_string", "//source/common/http:conn_manager_lib", @@ -167,6 +155,7 @@ envoy_cc_fuzz_test( "//test/mocks/http:http_mocks", "//test/mocks/local_info:local_info_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/router:router_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/ssl:ssl_mocks", "//test/mocks/tracing:tracing_mocks", @@ -180,7 +169,6 @@ envoy_cc_test( name = "conn_manager_impl_test", srcs = ["conn_manager_impl_test.cc"], deps = [ - ":conn_manager_impl_common_lib", "//include/envoy/access_log:access_log_interface", "//include/envoy/buffer:buffer_interface", "//include/envoy/event:dispatcher_interface", @@ -207,6 +195,7 @@ envoy_cc_test( "//test/mocks/http:http_mocks", "//test/mocks/local_info:local_info_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/router:router_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/server:server_mocks", "//test/mocks/ssl:ssl_mocks", diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index df9a0c2b4b35..60a097dec870 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -21,7 +21,6 @@ #include "common/network/utility.h" #include "common/stats/symbol_table_creator.h" -#include "test/common/http/conn_manager_impl_common.h" #include "test/common/http/conn_manager_impl_fuzz.pb.h" #include "test/fuzz/fuzz_runner.h" #include "test/fuzz/utility.h" @@ -30,6 +29,7 @@ #include "test/mocks/http/mocks.h" #include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/router/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/ssl/mocks.h" #include "test/mocks/tracing/mocks.h" @@ -47,13 +47,15 @@ namespace Http { class FuzzConfig : public ConnectionManagerConfig { public: FuzzConfig() - : route_config_provider_(time_system_), scoped_route_config_provider_(time_system_), - stats_{{ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER(fake_stats_), POOL_GAUGE(fake_stats_), + : stats_{{ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER(fake_stats_), POOL_GAUGE(fake_stats_), POOL_HISTOGRAM(fake_stats_))}, "", fake_stats_}, tracing_stats_{CONN_MAN_TRACING_STATS(POOL_COUNTER(fake_stats_))}, listener_stats_{CONN_MAN_LISTENER_STATS(POOL_COUNTER(fake_stats_))} { + ON_CALL(route_config_provider_, lastUpdated()).WillByDefault(Return(time_system_.systemTime())); + ON_CALL(scoped_route_config_provider_, lastUpdated()) + .WillByDefault(Return(time_system_.systemTime())); access_logs_.emplace_back(std::make_shared>()); } @@ -86,9 +88,17 @@ class FuzzConfig : public ConnectionManagerConfig { std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; } std::chrono::milliseconds requestTimeout() const override { return request_timeout_; } std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; } - Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { + if (use_srds_) { + return nullptr; + } + return &route_config_provider_; + } Config::ConfigProvider* scopedRouteConfigProvider() override { - return &scoped_route_config_provider_; + if (use_srds_) { + return &scoped_route_config_provider_; + } + return nullptr; } const std::string& serverName() override { return server_name_; } HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() override { @@ -124,8 +134,9 @@ class FuzzConfig : public ConnectionManagerConfig { NiceMock filter_factory_; Event::SimulatedTimeSystem time_system_; SlowDateProviderImpl date_provider_{time_system_}; - ConnectionManagerImplHelper::RouteConfigProvider route_config_provider_; - ConnectionManagerImplHelper::ScopedRouteConfigProvider scoped_route_config_provider_; + bool use_srds_{}; + Router::MockRouteConfigProvider route_config_provider_; + Router::MockScopedRouteConfigProvider scoped_route_config_provider_; std::string server_name_; HttpConnectionManagerProto::ServerHeaderTransformation server_transformation_{ HttpConnectionManagerProto::OVERWRITE}; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 496b2af6c7ea..ebf2c71d59ad 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -26,13 +26,13 @@ #include "extensions/access_loggers/file/file_access_log_impl.h" -#include "test/common/http/conn_manager_impl_common.h" #include "test/mocks/access_log/mocks.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/common.h" #include "test/mocks/http/mocks.h" #include "test/mocks/local_info/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/router/mocks.h" #include "test/mocks/runtime/mocks.h" #include "test/mocks/server/mocks.h" #include "test/mocks/ssl/mocks.h" @@ -69,9 +69,7 @@ namespace Http { class HttpConnectionManagerImplTest : public testing::Test, public ConnectionManagerConfig { public: HttpConnectionManagerImplTest() - : route_config_provider_(test_time_.timeSystem()), - scoped_route_config_provider_(test_time_.timeSystem()), - http_context_(fake_stats_.symbolTable()), access_log_path_("dummy_path"), + : http_context_(fake_stats_.symbolTable()), access_log_path_("dummy_path"), access_logs_{ AccessLog::InstanceSharedPtr{new Extensions::AccessLoggers::File::FileAccessLog( access_log_path_, {}, AccessLog::AccessLogFormatUtils::defaultAccessLogFormatter(), @@ -86,6 +84,10 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan http_context_.setTracer(tracer_); + ON_CALL(route_config_provider_, lastUpdated()) + .WillByDefault(Return(test_time_.timeSystem().systemTime())); + ON_CALL(scoped_route_config_provider_, lastUpdated()) + .WillByDefault(Return(test_time_.timeSystem().systemTime())); // response_encoder_ is not a NiceMock on purpose. This prevents complaining about this // method only. EXPECT_CALL(response_encoder_, getStream()).Times(AtLeast(0)); @@ -95,7 +97,8 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); } - void setup(bool ssl, const std::string& server_name, bool tracing = true) { + void setup(bool ssl, const std::string& server_name, bool tracing = true, bool use_srds = false) { + use_srds_ = use_srds; if (ssl) { ssl_connection_ = std::make_unique(); } @@ -271,9 +274,18 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; } std::chrono::milliseconds requestTimeout() const override { return request_timeout_; } std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; } - Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; } + bool use_srds_{}; + Router::RouteConfigProvider* routeConfigProvider() override { + if (use_srds_) { + return nullptr; + } + return &route_config_provider_; + } Config::ConfigProvider* scopedRouteConfigProvider() override { - return &scoped_route_config_provider_; + if (use_srds_) { + return &scoped_route_config_provider_; + } + return nullptr; } const std::string& serverName() override { return server_name_; } HttpConnectionManagerProto::ServerHeaderTransformation serverHeaderTransformation() override { @@ -302,8 +314,9 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan bool shouldMergeSlashes() const override { return merge_slashes_; } DangerousDeprecatedTestTime test_time_; - ConnectionManagerImplHelper::RouteConfigProvider route_config_provider_; - ConnectionManagerImplHelper::ScopedRouteConfigProvider scoped_route_config_provider_; + NiceMock route_config_provider_; + std::shared_ptr route_config_{new NiceMock()}; + NiceMock scoped_route_config_provider_; NiceMock tracer_; Stats::IsolatedStoreImpl fake_stats_; Http::ContextImpl http_context_; @@ -1890,7 +1903,7 @@ TEST_F(HttpConnectionManagerImplTest, PerStreamIdleTimeoutAfterBidiData) { TEST_F(HttpConnectionManagerImplTest, RequestTimeoutDisabledByDefault) { setup(false, ""); - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, createTimer_).Times(0); conn_manager_->newStream(response_encoder_); })); @@ -1903,7 +1916,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutDisabledIfSetToZero) { request_timeout_ = std::chrono::milliseconds(0); setup(false, ""); - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, createTimer_).Times(0); conn_manager_->newStream(response_encoder_); })); @@ -1916,7 +1929,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutValidlyConfigured) { request_timeout_ = std::chrono::milliseconds(10); setup(false, ""); - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)); @@ -1932,7 +1945,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutCallbackDisarmsAndReturns408 setup(false, ""); std::string response_body; - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); EXPECT_CALL(*request_timer, disableTimer()).Times(AtLeast(1)); @@ -1959,7 +1972,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsNotDisarmedOnIncompleteReq request_timeout_ = std::chrono::milliseconds(10); setup(false, ""); - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); EXPECT_CALL(*request_timer, disableTimer()).Times(0); @@ -1982,7 +1995,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsDisarmedOnCompleteRequestW request_timeout_ = std::chrono::milliseconds(10); setup(false, ""); - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); @@ -2058,7 +2071,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsDisarmedOnEncodeHeaders) { })); EXPECT_CALL(response_encoder_, encodeHeaders(_, _)); - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { Event::MockTimer* request_timer = setUpTimer(); EXPECT_CALL(*request_timer, enableTimer(request_timeout_, _)).Times(1); @@ -2084,7 +2097,7 @@ TEST_F(HttpConnectionManagerImplTest, RequestTimeoutIsDisarmedOnConnectionTermin setup(false, ""); Event::MockTimer* request_timer = setUpTimer(); - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; @@ -4274,7 +4287,7 @@ TEST_F(HttpConnectionManagerImplTest, OverlyLongHeadersRejected) { std::string response_code; std::string response_body; - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; @@ -4299,7 +4312,7 @@ TEST_F(HttpConnectionManagerImplTest, OverlyLongHeadersAcceptedIfConfigured) { max_request_headers_kb_ = 62; setup(false, ""); - EXPECT_CALL(*codec_, dispatch(_)).Times(1).WillOnce(Invoke([&](Buffer::Instance&) -> void { + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); HeaderMapPtr headers{ new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; @@ -4493,5 +4506,236 @@ TEST_F(HttpConnectionManagerImplTest, TestSessionTrace) { } } +// SRDS no scope found. +TEST_F(HttpConnectionManagerImplTest, TestSRDSRouteNotFound) { + setup(false, "", true, true); + + EXPECT_CALL(*static_cast( + scopedRouteConfigProvider()->config().get()), + getRouteConfig(_)) + .WillOnce(Return(nullptr)); + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{ + new TestHeaderMapImpl{{":authority", "host"}, {":method", "GET"}, {":path", "/foo"}}}; + decoder->decodeHeaders(std::move(headers), true); + data.drain(4); + })); + + EXPECT_CALL(response_encoder_, encodeHeaders(_, false)) + .WillOnce(Invoke([](const HeaderMap& headers, bool) -> void { + EXPECT_EQ("404", headers.Status()->value().getStringView()); + })); + + std::string response_body; + EXPECT_CALL(response_encoder_, encodeData(_, true)).WillOnce(AddBufferToString(&response_body)); + + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + EXPECT_EQ(response_body, "route scope not found"); +} + +// SRDS updating scopes affects routing. +TEST_F(HttpConnectionManagerImplTest, TestSRDSUpdate) { + setup(false, "", true, true); + + EXPECT_CALL(*static_cast( + scopedRouteConfigProvider()->config().get()), + getRouteConfig(_)) + .Times(3) + .WillOnce(Return(nullptr)) + .WillOnce(Return(route_config_)) + .WillOnce(Return(route_config_)); // refreshCachedRoute + EXPECT_CALL(*codec_, dispatch(_)) + .Times(2) // Once for no scoped routes, once for scoped routing + .WillRepeatedly(Invoke([&](Buffer::Instance& data) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{ + new TestHeaderMapImpl{{":authority", "host"}, {":method", "GET"}, {":path", "/foo"}}}; + decoder->decodeHeaders(std::move(headers), true); + data.drain(4); + })); + EXPECT_CALL(response_encoder_, encodeHeaders(_, false)) + .WillOnce(Invoke([](const HeaderMap& headers, bool) -> void { + EXPECT_EQ("404", headers.Status()->value().getStringView()); + })); + + std::string response_body; + EXPECT_CALL(response_encoder_, encodeData(_, true)).WillOnce(AddBufferToString(&response_body)); + + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + EXPECT_EQ(response_body, "route scope not found"); + + // Now route config provider returns something. + setupFilterChain(1, 0); // Recreate the chain for second stream. + const std::string fake_cluster1_name = "fake_cluster1"; + std::shared_ptr route1 = std::make_shared>(); + EXPECT_CALL(route1->route_entry_, clusterName()).WillRepeatedly(ReturnRef(fake_cluster1_name)); + std::shared_ptr fake_cluster1 = + std::make_shared>(); + EXPECT_CALL(cluster_manager_, get(_)).WillOnce(Return(fake_cluster1.get())); + EXPECT_CALL(*route_config_, route(_, _)).WillOnce(Return(route1)); + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, true)) + .WillOnce(InvokeWithoutArgs([&]() -> FilterHeadersStatus { + EXPECT_EQ(route1, decoder_filters_[0]->callbacks_->route()); + EXPECT_EQ(route1->routeEntry(), decoder_filters_[0]->callbacks_->streamInfo().routeEntry()); + EXPECT_EQ(fake_cluster1->info(), decoder_filters_[0]->callbacks_->clusterInfo()); + return FilterHeadersStatus::StopIteration; + })); + EXPECT_CALL(*decoder_filters_[0], decodeComplete()); + Buffer::OwnedImpl fake_input2("1234"); + conn_manager_->onData(fake_input2, false); +} + +// SRDS Scope header update cause cross-scope reroute. +TEST_F(HttpConnectionManagerImplTest, TestSRDSCrossScopeReroute) { + setup(false, "", true, true); + + std::shared_ptr route_config1 = + std::make_shared>(); + std::shared_ptr route_config2 = + std::make_shared>(); + std::shared_ptr route1 = std::make_shared>(); + std::shared_ptr route2 = std::make_shared>(); + EXPECT_CALL(*route_config1, route(_, _)).WillRepeatedly(Return(route1)); + EXPECT_CALL(*route_config2, route(_, _)).WillRepeatedly(Return(route2)); + EXPECT_CALL(*static_cast( + scopedRouteConfigProvider()->config().get()), + getRouteConfig(_)) + // 1. Snap scoped route config; + // 2. refreshCachedRoute (both in decodeHeaders(headers,end_stream); + // 3. then refreshCachedRoute triggered by decoder_filters_[1]->callbacks_->route(). + .Times(3) + .WillRepeatedly(Invoke([&](const HeaderMap& headers) -> Router::ConfigConstSharedPtr { + auto& test_headers = static_cast(headers); + if (test_headers.get_("scope_key") == "foo") { + return route_config1; + } + return route_config2; + })); + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{new TestHeaderMapImpl{ + {":authority", "host"}, {":method", "GET"}, {"scope_key", "foo"}, {":path", "/foo"}}}; + decoder->decodeHeaders(std::move(headers), false); + data.drain(4); + })); + setupFilterChain(2, 0); + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) + .WillOnce(Invoke([&](Http::HeaderMap& headers, bool) -> FilterHeadersStatus { + EXPECT_EQ(route1, decoder_filters_[0]->callbacks_->route()); + auto& test_headers = static_cast(headers); + // Clear cached route and change scope key to "bar". + decoder_filters_[0]->callbacks_->clearRouteCache(); + test_headers.remove("scope_key"); + test_headers.addCopy("scope_key", "bar"); + return FilterHeadersStatus::Continue; + })); + EXPECT_CALL(*decoder_filters_[1], decodeHeaders(_, false)) + .WillOnce(Invoke([&](Http::HeaderMap& headers, bool) -> FilterHeadersStatus { + auto& test_headers = static_cast(headers); + EXPECT_EQ(test_headers.get_("scope_key"), "bar"); + // Route now switched to route2 as header "scope_key" has changed. + EXPECT_EQ(route2, decoder_filters_[1]->callbacks_->route()); + EXPECT_EQ(route2->routeEntry(), decoder_filters_[1]->callbacks_->streamInfo().routeEntry()); + return FilterHeadersStatus::StopIteration; + })); + + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + +// SRDS scoped RouteConfiguration found and route found. +TEST_F(HttpConnectionManagerImplTest, TestSRDSRouteFound) { + setup(false, "", true, true); + setupFilterChain(1, 0); + + const std::string fake_cluster1_name = "fake_cluster1"; + std::shared_ptr route1 = std::make_shared>(); + EXPECT_CALL(route1->route_entry_, clusterName()).WillRepeatedly(ReturnRef(fake_cluster1_name)); + std::shared_ptr fake_cluster1 = + std::make_shared>(); + EXPECT_CALL(cluster_manager_, get(_)).WillOnce(Return(fake_cluster1.get())); + EXPECT_CALL(*scopedRouteConfigProvider()->config(), getRouteConfig(_)) + // 1. decodeHeaders() snaping route config. + // 2. refreshCachedRoute() later in the same decodeHeaders(). + .Times(2); + EXPECT_CALL( + *static_cast( + scopedRouteConfigProvider()->config()->route_config_.get()), + route(_, _)) + .WillOnce(Return(route1)); + StreamDecoder* decoder = nullptr; + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance& data) -> void { + decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{ + new TestHeaderMapImpl{{":authority", "host"}, {":method", "GET"}, {":path", "/foo"}}}; + decoder->decodeHeaders(std::move(headers), true); + data.drain(4); + })); + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, true)) + .WillOnce(InvokeWithoutArgs([&]() -> FilterHeadersStatus { + EXPECT_EQ(route1, decoder_filters_[0]->callbacks_->route()); + EXPECT_EQ(route1->routeEntry(), decoder_filters_[0]->callbacks_->streamInfo().routeEntry()); + EXPECT_EQ(fake_cluster1->info(), decoder_filters_[0]->callbacks_->clusterInfo()); + return FilterHeadersStatus::StopIteration; + })); + EXPECT_CALL(*decoder_filters_[0], decodeComplete()); + + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); +} + +class HttpConnectionManagerImplDeathTest : public HttpConnectionManagerImplTest { +public: + Router::RouteConfigProvider* routeConfigProvider() override { + return route_config_provider2_.get(); + } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return scoped_route_config_provider2_.get(); + } + + std::shared_ptr route_config_provider2_; + std::shared_ptr scoped_route_config_provider2_; +}; + +// HCM config can only have either RouteConfigProvider or ScopedRoutesConfigProvider. +TEST_F(HttpConnectionManagerImplDeathTest, InvalidConnectionManagerConfig) { + setup(false, ""); + + Buffer::OwnedImpl fake_input("1234"); + EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance&) -> void { + conn_manager_->newStream(response_encoder_); + })); + // Either RDS or SRDS should be set. + EXPECT_DEBUG_DEATH(conn_manager_->onData(fake_input, false), + "Either routeConfigProvider or scopedRouteConfigProvider should be set in " + "ConnectionManagerImpl."); + + route_config_provider2_ = std::make_shared>(); + + // Only route config provider valid. + EXPECT_NO_THROW(conn_manager_->onData(fake_input, false)); + + scoped_route_config_provider2_ = + std::make_shared>(); + // Can't have RDS and SRDS provider in the same time. + EXPECT_DEBUG_DEATH(conn_manager_->onData(fake_input, false), + "Either routeConfigProvider or scopedRouteConfigProvider should be set in " + "ConnectionManagerImpl."); + + route_config_provider2_.reset(); + // Only scoped route config provider valid. + EXPECT_NO_THROW(conn_manager_->onData(fake_input, false)); + +#if !defined(NDEBUG) + EXPECT_CALL(*scoped_route_config_provider2_, getConfig()).WillRepeatedly(Return(nullptr)); + // ASSERT failure when SRDS provider returns a nullptr. + EXPECT_DEBUG_DEATH(conn_manager_->onData(fake_input, false), + "Scoped rds provider returns null for scoped routes config."); +#endif // !defined(NDEBUG) +} + } // namespace Http } // namespace Envoy diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 00936537bc7f..8ee6cfe8aecf 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -247,7 +247,7 @@ route_configuration_name: foo_routes } // Tests that multiple uniquely named non-conflict resources are allowed in config updates. -TEST_F(ScopedRdsTest, MultipleResourcesStow) { +TEST_F(ScopedRdsTest, MultipleResourcesSotw) { setup(); const std::string config_yaml = R"EOF( diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index e14e87d08fa7..1cefe11ac397 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -4,6 +4,7 @@ #include "test/common/grpc/grpc_client_integration.h" #include "test/integration/http_integration.h" +#include "test/test_common/printers.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -12,12 +13,12 @@ namespace Envoy { namespace { class ScopedRdsIntegrationTest : public HttpIntegrationTest, - public Grpc::GrpcClientIntegrationParamTest { + public Grpc::DeltaSotwGrpcClientIntegrationParamTest { protected: struct FakeUpstreamInfo { FakeHttpConnectionPtr connection_; FakeUpstream* upstream_{}; - FakeStreamPtr stream_; + absl::flat_hash_map stream_by_resource_name_; }; ScopedRdsIntegrationTest() @@ -29,7 +30,15 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, } void initialize() override { + // Setup two upstream hosts, one for each cluster. + setUpstreamCount(2); + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + // Add the static cluster to serve SRDS. + auto* cluster_1 = bootstrap.mutable_static_resources()->add_clusters(); + cluster_1->MergeFrom(bootstrap.static_resources().clusters()[0]); + cluster_1->set_name("cluster_1"); + // Add the static cluster to serve SRDS. auto* scoped_rds_cluster = bootstrap.mutable_static_resources()->add_clusters(); scoped_rds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); @@ -50,15 +59,16 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, fragments: - header_value_extractor: name: Addr + element_separator: ; element: key: x-foo-key - separator: ; + separator: = )EOF"; envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder scope_key_builder; TestUtility::loadFromYaml(scope_key_builder_config_yaml, scope_key_builder); auto* scoped_routes = http_connection_manager.mutable_scoped_routes(); - scoped_routes->set_name("foo-scoped-routes"); + scoped_routes->set_name(srds_config_name_); *scoped_routes->mutable_scope_key_builder() = scope_key_builder; envoy::api::v2::core::ApiConfigSource* rds_api_config_source = @@ -72,7 +82,11 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, scoped_routes->mutable_scoped_rds() ->mutable_scoped_rds_config_source() ->mutable_api_config_source(); - srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + if (isDelta()) { + srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); + } else { + srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + } grpc_service = srds_api_config_source->add_grpc_services(); setGrpcService(*grpc_service, "srds_cluster", getScopedRdsFakeUpstream().localAddress()); }); @@ -80,6 +94,41 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, HttpIntegrationTest::initialize(); } + // TODO(stevenzzzz): move these utility methods to base classes to share with other tests. + // Helper that verifies if given headers are in the response header map. + void verifyResponse(IntegrationStreamDecoderPtr response, const std::string& response_code, + const Http::TestHeaderMapImpl& expected_headers, + const std::string& expected_body) { + EXPECT_TRUE(response->complete()); + EXPECT_EQ(response_code, response->headers().Status()->value().getStringView()); + expected_headers.iterate( + [](const Http::HeaderEntry& header, void* context) -> Http::HeaderMap::Iterate { + auto response_headers = static_cast(context); + const Http::HeaderEntry* entry = response_headers->get( + Http::LowerCaseString{std::string(header.key().getStringView())}); + EXPECT_NE(entry, nullptr); + EXPECT_EQ(header.value().getStringView(), entry->value().getStringView()); + return Http::HeaderMap::Iterate::Continue; + }, + const_cast(static_cast(&response->headers()))); + EXPECT_EQ(response->body(), expected_body); + } + + // Helper that sends a request to Envoy, and verifies if Envoy response headers and body size is + // the same as the expected headers map. + void sendRequestAndVerifyResponse(const Http::TestHeaderMapImpl& request_headers, + const int request_size, + const Http::TestHeaderMapImpl& response_headers, + const int response_size, const int backend_idx) { + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = sendRequestAndWaitForResponse(request_headers, request_size, response_headers, + response_size, backend_idx); + verifyResponse(std::move(response), "200", response_headers, std::string(response_size, 'a')); + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(request_size, upstream_request_->bodyLength()); + cleanupUpstreamAndDownstream(); + } + void createUpstreams() override { HttpIntegrationTest::createUpstreams(); // Create the SRDS upstream. @@ -108,38 +157,87 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, resetFakeUpstreamInfo(&scoped_rds_upstream_info_); } - FakeUpstream& getRdsFakeUpstream() const { return *fake_upstreams_[2]; } + FakeUpstream& getRdsFakeUpstream() const { return *fake_upstreams_[3]; } - FakeUpstream& getScopedRdsFakeUpstream() const { return *fake_upstreams_[1]; } + FakeUpstream& getScopedRdsFakeUpstream() const { return *fake_upstreams_[2]; } - void createStream(FakeUpstreamInfo* upstream_info, FakeUpstream& upstream) { - upstream_info->upstream_ = &upstream; - AssertionResult result = - upstream_info->upstream_->waitForHttpConnection(*dispatcher_, upstream_info->connection_); - RELEASE_ASSERT(result, result.message()); - result = upstream_info->connection_->waitForNewStream(*dispatcher_, upstream_info->stream_); + void createStream(FakeUpstreamInfo* upstream_info, FakeUpstream& upstream, + const std::string& resource_name) { + if (upstream_info->upstream_ == nullptr) { + // bind upstream if not yet. + upstream_info->upstream_ = &upstream; + AssertionResult result = + upstream_info->upstream_->waitForHttpConnection(*dispatcher_, upstream_info->connection_); + RELEASE_ASSERT(result, result.message()); + } + if (!upstream_info->stream_by_resource_name_.try_emplace(resource_name, nullptr).second) { + RELEASE_ASSERT(false, + fmt::format("stream with resource name '{}' already exists!", resource_name)); + } + auto result = upstream_info->connection_->waitForNewStream( + *dispatcher_, upstream_info->stream_by_resource_name_[resource_name]); RELEASE_ASSERT(result, result.message()); - upstream_info->stream_->startGrpcStream(); + upstream_info->stream_by_resource_name_[resource_name]->startGrpcStream(); } - void createRdsStream() { createStream(&rds_upstream_info_, getRdsFakeUpstream()); } + void createRdsStream(const std::string& resource_name) { + createStream(&rds_upstream_info_, getRdsFakeUpstream(), resource_name); + } void createScopedRdsStream() { - createStream(&scoped_rds_upstream_info_, getScopedRdsFakeUpstream()); + createStream(&scoped_rds_upstream_info_, getScopedRdsFakeUpstream(), srds_config_name_); } void sendRdsResponse(const std::string& route_config, const std::string& version) { envoy::api::v2::DiscoveryResponse response; response.set_version_info(version); response.set_type_url(Config::TypeUrl::get().RouteConfiguration); - response.add_resources()->PackFrom( - TestUtility::parseYaml(route_config)); - rds_upstream_info_.stream_->sendGrpcMessage(response); + auto route_configuration = + TestUtility::parseYaml(route_config); + response.add_resources()->PackFrom(route_configuration); + ASSERT(rds_upstream_info_.stream_by_resource_name_[route_configuration.name()] != nullptr); + rds_upstream_info_.stream_by_resource_name_[route_configuration.name()]->sendGrpcMessage( + response); + } + + void sendSrdsResponse(const std::vector& sotw_list, + const std::vector& to_add_list, + const std::vector& to_delete_list, + const std::string& version) { + if (isDelta()) { + sendDeltaScopedRdsResponse(to_add_list, to_delete_list, version); + } else { + sendSotwScopedRdsResponse(sotw_list, version); + } + } + + void sendDeltaScopedRdsResponse(const std::vector& to_add_list, + const std::vector& to_delete_list, + const std::string& version) { + ASSERT(scoped_rds_upstream_info_.stream_by_resource_name_[srds_config_name_] != nullptr); + + envoy::api::v2::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_type_url(Config::TypeUrl::get().ScopedRouteConfiguration); + + for (const auto& scope_name : to_delete_list) { + *response.add_removed_resources() = scope_name; + } + for (const auto& resource_proto : to_add_list) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_proto; + TestUtility::loadFromYaml(resource_proto, scoped_route_proto); + auto resource = response.add_resources(); + resource->set_name(scoped_route_proto.name()); + resource->set_version(version); + resource->mutable_resource()->PackFrom(scoped_route_proto); + } + scoped_rds_upstream_info_.stream_by_resource_name_[srds_config_name_]->sendGrpcMessage( + response); } - void sendScopedRdsResponse(const std::vector& resource_protos, - const std::string& version) { - ASSERT(scoped_rds_upstream_info_.stream_ != nullptr); + void sendSotwScopedRdsResponse(const std::vector& resource_protos, + const std::string& version) { + ASSERT(scoped_rds_upstream_info_.stream_by_resource_name_[srds_config_name_] != nullptr); envoy::api::v2::DiscoveryResponse response; response.set_version_info(version); @@ -150,33 +248,29 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, TestUtility::loadFromYaml(resource_proto, scoped_route_proto); response.add_resources()->PackFrom(scoped_route_proto); } - - scoped_rds_upstream_info_.stream_->sendGrpcMessage(response); + scoped_rds_upstream_info_.stream_by_resource_name_[srds_config_name_]->sendGrpcMessage( + response); } + const std::string srds_config_name_{"foo-scoped-routes"}; FakeUpstreamInfo scoped_rds_upstream_info_; FakeUpstreamInfo rds_upstream_info_; }; INSTANTIATE_TEST_SUITE_P(IpVersionsAndGrpcTypes, ScopedRdsIntegrationTest, - GRPC_CLIENT_INTEGRATION_PARAMS); + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // Test that a SRDS DiscoveryResponse is successfully processed. TEST_P(ScopedRdsIntegrationTest, BasicSuccess) { - const std::string scope_route1 = R"EOF( -name: foo_scope1 -route_configuration_name: foo_route1 + const std::string scope_tmpl = R"EOF( +name: {} +route_configuration_name: {} key: fragments: - - string_key: x-foo-key -)EOF"; - const std::string scope_route2 = R"EOF( -name: foo_scope2 -route_configuration_name: foo_route1 -key: - fragments: - - string_key: x-bar-key + - string_key: {} )EOF"; + const std::string scope_route1 = fmt::format(scope_tmpl, "foo_scope1", "foo_route1", "foo-route"); + const std::string scope_route2 = fmt::format(scope_tmpl, "foo_scope2", "foo_route1", "bar-route"); const std::string route_config_tmpl = R"EOF( name: {} @@ -190,35 +284,127 @@ route_configuration_name: foo_route1 on_server_init_function_ = [&]() { createScopedRdsStream(); - sendScopedRdsResponse({scope_route1, scope_route2}, "1"); - createRdsStream(); - sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_1"), "1"); - sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_2"), "2"); + sendSrdsResponse({scope_route1, scope_route2}, {scope_route1, scope_route2}, {}, "1"); + createRdsStream("foo_route1"); + // CreateRdsStream waits for connection which is fired by RDS subscription. + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_0"), "1"); }; initialize(); - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); + registerTestServerPorts({"http"}); + + // No scope key matches "xyz-route". + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = codec_client_->makeHeaderOnlyRequest( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key=xyz-route"}}); + response->waitForEndStream(); + verifyResponse(std::move(response), "404", Http::TestHeaderMapImpl{}, "route scope not found"); + cleanupUpstreamAndDownstream(); + + // Test "foo-route" and 'bar-route' both gets routed to cluster_0. + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 1); + for (const std::string& scope_key : std::vector{"foo-route", "bar-route"}) { + sendRequestAndVerifyResponse( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", fmt::format("x-foo-key={}", scope_key)}}, + 456, Http::TestHeaderMapImpl{{":status", "200"}, {"service", scope_key}}, 123, + /*cluster_0*/ 0); + } + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", + // update_attempt only increase after a response + isDelta() ? 1 : 2); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); // The version gauge should be set to xxHash64("1"). test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", 13237225503670494420UL); - const std::string scope_route3 = R"EOF( -name: foo_scope3 -route_configuration_name: foo_route1 -key: - fragments: - - string_key: x-baz-key -)EOF"; - sendScopedRdsResponse({scope_route3}, "2"); + // Add a new scope scope_route3 with a brand new RouteConfiguration foo_route2. + const std::string scope_route3 = fmt::format(scope_tmpl, "foo_scope3", "foo_route2", "baz-route"); + + sendSrdsResponse({scope_route1, scope_route2, scope_route3}, /*added*/ {scope_route3}, {}, "2"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_attempt", 2); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_1"), "3"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 2); + createRdsStream("foo_route2"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route2.update_attempt", 1); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route2", "cluster_0"), "1"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route2.update_success", 1); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 2); + // The version gauge should be set to xxHash64("2"). test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", 6927017134761466251UL); - test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_attempt", 3); - sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_foo_3"), "3"); - test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 3); - // RDS updates won't affect SRDS. - test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", - 6927017134761466251UL); + // After RDS update, requests within scope 'foo_scope1' or 'foo_scope2' get routed to + // 'cluster_1'. + for (const std::string& scope_key : std::vector{"foo-route", "bar-route"}) { + sendRequestAndVerifyResponse( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", fmt::format("x-foo-key={}", scope_key)}}, + 456, Http::TestHeaderMapImpl{{":status", "200"}, {"service", scope_key}}, 123, + /*cluster_1*/ 1); + } + // Now requests within scope 'foo_scope3' get routed to 'cluster_0'. + sendRequestAndVerifyResponse( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", fmt::format("x-foo-key={}", "baz-route")}}, + 456, Http::TestHeaderMapImpl{{":status", "200"}, {"service", "bluh"}}, 123, + /*cluster_0*/ 0); + + // Delete foo_scope1 and requests within the scope gets 400s. + sendSrdsResponse({scope_route2, scope_route3}, {}, {"foo_scope1"}, "3"); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 3); + codec_client_ = makeHttpConnection(lookupPort("http")); + response = codec_client_->makeHeaderOnlyRequest( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key=foo-route"}}); + response->waitForEndStream(); + verifyResponse(std::move(response), "404", Http::TestHeaderMapImpl{}, "route scope not found"); + cleanupUpstreamAndDownstream(); + // Add a new scope foo_scope4. + const std::string& scope_route4 = + fmt::format(scope_tmpl, "foo_scope4", "foo_route4", "xyz-route"); + sendSrdsResponse({scope_route3, scope_route2, scope_route4}, {scope_route4}, {}, "4"); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 4); + codec_client_ = makeHttpConnection(lookupPort("http")); + response = codec_client_->makeHeaderOnlyRequest( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key=xyz-route"}}); + response->waitForEndStream(); + // Get 404 because RDS hasn't pushed route configuration "foo_route4" yet. + // But scope is found and the Router::NullConfigImpl is returned. + verifyResponse(std::move(response), "404", Http::TestHeaderMapImpl{}, ""); + cleanupUpstreamAndDownstream(); + + // RDS updated foo_route4, requests with scope key "xyz-route" now hit cluster_1. + test_server_->waitForCounterGe("http.config_test.rds.foo_route4.update_attempt", 1); + createRdsStream("foo_route4"); + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route4", "cluster_1"), "3"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route4.update_success", 1); + sendRequestAndVerifyResponse( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key=xyz-route"}}, + 456, Http::TestHeaderMapImpl{{":status", "200"}, {"service", "xyz-route"}}, 123, + /*cluster_1 */ 1); } // Test that a bad config update updates the corresponding stats. @@ -229,16 +415,56 @@ TEST_P(ScopedRdsIntegrationTest, ConfigUpdateFailure) { route_configuration_name: foo_route1 key: fragments: - - string_key: x-foo-key + - string_key: foo )EOF"; on_server_init_function_ = [this, &scope_route1]() { createScopedRdsStream(); - sendScopedRdsResponse({scope_route1}, "1"); + sendSrdsResponse({scope_route1}, {scope_route1}, {}, "1"); }; initialize(); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_rejected", 1); + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = + codec_client_->makeHeaderOnlyRequest(Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key=foo"}}); + response->waitForEndStream(); + verifyResponse(std::move(response), "404", Http::TestHeaderMapImpl{}, "route scope not found"); + cleanupUpstreamAndDownstream(); + + // SRDS update fixed the problem. + const std::string scope_route2 = R"EOF( +name: foo_scope1 +route_configuration_name: foo_route1 +key: + fragments: + - string_key: foo +)EOF"; + sendSrdsResponse({scope_route2}, {scope_route2}, {}, "1"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_attempt", 1); + createRdsStream("foo_route1"); + const std::string route_config_tmpl = R"EOF( + name: {} + virtual_hosts: + - name: integration + domains: ["*"] + routes: + - match: {{ prefix: "/" }} + route: {{ cluster: {} }} +)EOF"; + sendRdsResponse(fmt::format(route_config_tmpl, "foo_route1", "cluster_0"), "1"); + test_server_->waitForCounterGe("http.config_test.rds.foo_route1.update_success", 1); + sendRequestAndVerifyResponse( + Http::TestHeaderMapImpl{{":method", "GET"}, + {":path", "/meh"}, + {":authority", "host"}, + {":scheme", "http"}, + {"Addr", "x-foo-key=foo"}}, + 456, Http::TestHeaderMapImpl{{":status", "200"}, {"service", "bluh"}}, 123, /*cluster_0*/ 0); } } // namespace diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 06e5875c032a..b4e7c4ff7700 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -391,6 +391,7 @@ class MockRouteConfigProvider : public RouteConfigProvider { MOCK_CONST_METHOD0(configInfo, absl::optional()); MOCK_CONST_METHOD0(lastUpdated, SystemTime()); MOCK_METHOD0(onConfigUpdate, void()); + MOCK_CONST_METHOD1(validateConfig, void(const envoy::api::v2::RouteConfiguration&)); std::shared_ptr> route_config_{new NiceMock()}; }; diff --git a/test/test_common/utility.cc b/test/test_common/utility.cc index 8cf850b1a300..c236ac4ce7d4 100644 --- a/test/test_common/utility.cc +++ b/test/test_common/utility.cc @@ -423,9 +423,11 @@ void TestHeaderMapImpl::addCopy(const std::string& key, const std::string& value void TestHeaderMapImpl::remove(const std::string& key) { remove(LowerCaseString(key)); } -std::string TestHeaderMapImpl::get_(const std::string& key) { return get_(LowerCaseString(key)); } +std::string TestHeaderMapImpl::get_(const std::string& key) const { + return get_(LowerCaseString(key)); +} -std::string TestHeaderMapImpl::get_(const LowerCaseString& key) { +std::string TestHeaderMapImpl::get_(const LowerCaseString& key) const { const HeaderEntry* header = get(key); if (!header) { return EMPTY_STRING; diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 4ff0ac413097..b4f048396ac6 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -637,8 +637,8 @@ class TestHeaderMapImpl : public HeaderMapImpl { using HeaderMapImpl::remove; void addCopy(const std::string& key, const std::string& value); void remove(const std::string& key); - std::string get_(const std::string& key); - std::string get_(const LowerCaseString& key); + std::string get_(const std::string& key) const; + std::string get_(const LowerCaseString& key) const; bool has(const std::string& key); bool has(const LowerCaseString& key); }; From 678bf8c2300de4987d20312ac1c4cdeeb3cad2fb Mon Sep 17 00:00:00 2001 From: htuch Date: Fri, 30 Aug 2019 17:20:56 -0400 Subject: [PATCH 485/542] owners: add @asraa and @lambdai to OWNERS. (#8110) * @asraa is joining Envoy OSS security team. * @lambdai is joining Friends of Envoy as v2 xDS point. Signed-off-by: Harvey Tuch --- OWNERS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/OWNERS.md b/OWNERS.md index 8f5898ff1070..317dad396752 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -39,6 +39,7 @@ routing PRs, questions, etc. to the right place. * All maintainers * Piotr Sikora ([PiotrSikora](https://github.com/PiotrSikora)) (piotrsikora@google.com) * Yan Avlasov ([yanavlasov](https://github.com/yanavlasov)) (yavlasov@google.com) +* Asra Ali ([asraa](https://github.com/asraa)) (asraa@google.com) # Emeritus maintainers @@ -60,3 +61,5 @@ matter expert reviews. Feel free to loop them in as needed. * Bazel/build. * Daniel Hochman ([danielhochman](https://github.com/danielhochman)) (dhochman@lyft.com) * Redis, Python, configuration/operational questions. +* Yuchen Dai ([lambdai](https://github.com/lambdai)) (lambdai@google.com) + * v2 xDS, listeners, filter chain discovery service. From dad0f2e239b39448c7e1f6504ddbb4229cf01abe Mon Sep 17 00:00:00 2001 From: htuch Date: Mon, 2 Sep 2019 10:50:31 -0400 Subject: [PATCH 486/542] protobuf: recursively validate unknown fields. (#8094) This PR unifies the recursive traversal of deprecated fields with that of unknown fields. It doesn't deal with moving to a validator visitor model for deprecation; this would be a nice cleanup that we track at https://github.com/envoyproxy/envoy/issues/8092. Risk level: Low Testing: New nested unknown field test added. Fixes #7980 Signed-off-by: Harvey Tuch --- source/common/protobuf/utility.cc | 41 +++++++++++++----------- source/common/protobuf/utility.h | 13 +++----- test/common/protobuf/utility_test.cc | 48 +++++++++++++++++++--------- test/tools/router_check/router.cc | 5 ++- 4 files changed, 64 insertions(+), 43 deletions(-) diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index 4df4964b768f..b52a716f321f 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -88,21 +88,6 @@ ProtoValidationException::ProtoValidationException(const std::string& validation ENVOY_LOG_MISC(debug, "Proto validation error; throwing {}", what()); } -void MessageUtil::checkUnknownFields(const Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor) { - const auto& unknown_fields = message.GetReflection()->GetUnknownFields(message); - // If there are no unknown fields, we're done here. - if (unknown_fields.empty()) { - return; - } - std::string error_msg; - for (int n = 0; n < unknown_fields.field_count(); ++n) { - error_msg += absl::StrCat(n > 0 ? ", " : "", unknown_fields.field(n).number()); - } - validation_visitor.onUnknownField("type " + message.GetTypeName() + " with unknown field set {" + - error_msg + "}"); -} - void MessageUtil::loadFromJson(const std::string& json, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor) { Protobuf::util::JsonParseOptions options; @@ -159,7 +144,7 @@ void MessageUtil::loadFromFile(const std::string& path, Protobuf::Message& messa if (absl::EndsWith(path, FileExtensions::get().ProtoBinary)) { // Attempt to parse the binary format. if (message.ParseFromString(contents)) { - MessageUtil::checkUnknownFields(message, validation_visitor); + MessageUtil::checkForUnexpectedFields(message, validation_visitor); return; } throw EnvoyException("Unable to parse file \"" + path + "\" as a binary protobuf (type " + @@ -180,7 +165,23 @@ void MessageUtil::loadFromFile(const std::string& path, Protobuf::Message& messa } } -void MessageUtil::checkForDeprecation(const Protobuf::Message& message, Runtime::Loader* runtime) { +void MessageUtil::checkForUnexpectedFields(const Protobuf::Message& message, + ProtobufMessage::ValidationVisitor& validation_visitor, + Runtime::Loader* runtime) { + // Reject unknown fields. + const auto& unknown_fields = message.GetReflection()->GetUnknownFields(message); + if (!unknown_fields.empty()) { + std::string error_msg; + for (int n = 0; n < unknown_fields.field_count(); ++n) { + error_msg += absl::StrCat(n > 0 ? ", " : "", unknown_fields.field(n).number()); + } + // We use the validation visitor but have hard coded behavior below for deprecated fields. + // TODO(htuch): Unify the deprecated and unknown visitor handling behind the validation + // visitor pattern. https://github.com/envoyproxy/envoy/issues/8092. + validation_visitor.onUnknownField("type " + message.GetTypeName() + + " with unknown field set {" + error_msg + "}"); + } + const Protobuf::Descriptor* descriptor = message.GetDescriptor(); const Protobuf::Reflection* reflection = message.GetReflection(); for (int i = 0; i < descriptor->field_count(); ++i) { @@ -231,10 +232,12 @@ void MessageUtil::checkForDeprecation(const Protobuf::Message& message, Runtime: if (field->is_repeated()) { const int size = reflection->FieldSize(message, field); for (int j = 0; j < size; ++j) { - checkForDeprecation(reflection->GetRepeatedMessage(message, field, j), runtime); + checkForUnexpectedFields(reflection->GetRepeatedMessage(message, field, j), + validation_visitor, runtime); } } else { - checkForDeprecation(reflection->GetMessage(message, field), runtime); + checkForUnexpectedFields(reflection->GetMessage(message, field), validation_visitor, + runtime); } } } diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index d0451893d479..00ae4bceb66c 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -206,9 +206,6 @@ class MessageUtil { return HashUtil::xxHash64(text); } - static void checkUnknownFields(const Protobuf::Message& message, - ProtobufMessage::ValidationVisitor& validation_visitor); - static void loadFromJson(const std::string& json, Protobuf::Message& message, ProtobufMessage::ValidationVisitor& validation_visitor); static void loadFromJson(const std::string& json, ProtobufWkt::Struct& message); @@ -225,8 +222,9 @@ class MessageUtil { * in disallowed_features in runtime_features.h */ static void - checkForDeprecation(const Protobuf::Message& message, - Runtime::Loader* loader = Runtime::LoaderSingleton::getExisting()); + checkForUnexpectedFields(const Protobuf::Message& message, + ProtobufMessage::ValidationVisitor& validation_visitor, + Runtime::Loader* loader = Runtime::LoaderSingleton::getExisting()); /** * Validate protoc-gen-validate constraints on a given protobuf. @@ -238,9 +236,8 @@ class MessageUtil { template static void validate(const MessageType& message, ProtobufMessage::ValidationVisitor& validation_visitor) { - // Log warnings or throw errors if deprecated fields are in use. - checkForDeprecation(message); - checkUnknownFields(message, validation_visitor); + // Log warnings or throw errors if deprecated fields or unknown fields are in use. + checkForUnexpectedFields(message, validation_visitor); std::string err; if (!Validate(message, &err)) { diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 4371e4ecc9e8..539dfb6a28f6 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -1,8 +1,10 @@ #include +#include "envoy/api/v2/cds.pb.validate.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.validate.h" +#include "common/protobuf/message_validator_impl.h" #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" #include "common/runtime/runtime_impl.h" @@ -145,6 +147,19 @@ TEST_F(ProtobufUtilityTest, DowncastAndValidateUnknownFields) { "unknown field set {1}) has unknown fields"); } +// Validated exception thrown when downcastAndValidate observes a nested unknown field. +TEST_F(ProtobufUtilityTest, DowncastAndValidateUnknownFieldsNested) { + envoy::config::bootstrap::v2::Bootstrap bootstrap; + auto* cluster = bootstrap.mutable_static_resources()->add_clusters(); + cluster->GetReflection()->MutableUnknownFields(cluster)->AddVarint(1, 0); + EXPECT_THROW_WITH_MESSAGE(TestUtility::validate(*cluster), EnvoyException, + "Protobuf message (type envoy.api.v2.Cluster with " + "unknown field set {1}) has unknown fields"); + EXPECT_THROW_WITH_MESSAGE(TestUtility::validate(bootstrap), EnvoyException, + "Protobuf message (type envoy.api.v2.Cluster with " + "unknown field set {1}) has unknown fields"); +} + TEST_F(ProtobufUtilityTest, LoadBinaryProtoFromFile) { envoy::config::bootstrap::v2::Bootstrap bootstrap; bootstrap.mutable_cluster_manager() @@ -494,20 +509,24 @@ class DeprecatedFieldsTest : public testing::Test { NiceMock validation_visitor_; }; +void checkForDeprecation(const Protobuf::Message& message) { + MessageUtil::checkForUnexpectedFields(message, ProtobufMessage::getStrictValidationVisitor()); +} + TEST_F(DeprecatedFieldsTest, NoCrashIfRuntimeMissing) { loader_.reset(); envoy::test::deprecation_test::Base base; base.set_not_deprecated("foo"); // Fatal checks for a non-deprecated field should cause no problem. - MessageUtil::checkForDeprecation(base); + checkForDeprecation(base); } TEST_F(DeprecatedFieldsTest, NoErrorWhenDeprecatedFieldsUnused) { envoy::test::deprecation_test::Base base; base.set_not_deprecated("foo"); // Fatal checks for a non-deprecated field should cause no problem. - MessageUtil::checkForDeprecation(base); + checkForDeprecation(base); EXPECT_EQ(0, runtime_deprecated_feature_use_.value()); } @@ -517,7 +536,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(IndividualFieldDeprecated)) // Non-fatal checks for a deprecated field should log rather than throw an exception. EXPECT_LOG_CONTAINS("warning", "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated'", - MessageUtil::checkForDeprecation(base)); + checkForDeprecation(base)); EXPECT_EQ(1, runtime_deprecated_feature_use_.value()); } @@ -526,7 +545,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(IndividualFieldDisallowed)) envoy::test::deprecation_test::Base base; base.set_is_deprecated_fatal("foo"); EXPECT_THROW_WITH_REGEX( - MessageUtil::checkForDeprecation(base), ProtoValidationException, + checkForDeprecation(base), ProtoValidationException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'"); } @@ -537,7 +556,7 @@ TEST_F(DeprecatedFieldsTest, // Make sure this is set up right. EXPECT_THROW_WITH_REGEX( - MessageUtil::checkForDeprecation(base), ProtoValidationException, + checkForDeprecation(base), ProtoValidationException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'"); // The config will be rejected, so the feature will not be used. EXPECT_EQ(0, runtime_deprecated_feature_use_.value()); @@ -549,7 +568,7 @@ TEST_F(DeprecatedFieldsTest, // Now the same deprecation check should only trigger a warning. EXPECT_LOG_CONTAINS( "warning", "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'", - MessageUtil::checkForDeprecation(base)); + checkForDeprecation(base)); EXPECT_EQ(1, runtime_deprecated_feature_use_.value()); } @@ -559,7 +578,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(DisallowViaRuntime)) { EXPECT_LOG_CONTAINS("warning", "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated'", - MessageUtil::checkForDeprecation(base)); + checkForDeprecation(base)); EXPECT_EQ(1, runtime_deprecated_feature_use_.value()); // Now create a new snapshot with this feature disallowed. @@ -567,7 +586,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(DisallowViaRuntime)) { {{"envoy.deprecated_features.deprecated.proto:is_deprecated", " false"}}); EXPECT_THROW_WITH_REGEX( - MessageUtil::checkForDeprecation(base), ProtoValidationException, + checkForDeprecation(base), ProtoValidationException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated'"); EXPECT_EQ(1, runtime_deprecated_feature_use_.value()); } @@ -582,7 +601,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(MixOfFatalAndWarnings)) { EXPECT_LOG_CONTAINS( "warning", "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated'", { EXPECT_THROW_WITH_REGEX( - MessageUtil::checkForDeprecation(base), ProtoValidationException, + checkForDeprecation(base), ProtoValidationException, "Using deprecated option 'envoy.test.deprecation_test.Base.is_deprecated_fatal'"); }); } @@ -593,7 +612,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(MessageDeprecated)) { base.mutable_deprecated_message(); EXPECT_LOG_CONTAINS( "warning", "Using deprecated option 'envoy.test.deprecation_test.Base.deprecated_message'", - MessageUtil::checkForDeprecation(base)); + checkForDeprecation(base)); EXPECT_EQ(1, runtime_deprecated_feature_use_.value()); } @@ -601,15 +620,14 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(InnerMessageDeprecated)) { envoy::test::deprecation_test::Base base; base.mutable_not_deprecated_message()->set_inner_not_deprecated("foo"); // Checks for a non-deprecated field shouldn't trigger warnings - EXPECT_LOG_NOT_CONTAINS("warning", "Using deprecated option", - MessageUtil::checkForDeprecation(base)); + EXPECT_LOG_NOT_CONTAINS("warning", "Using deprecated option", checkForDeprecation(base)); base.mutable_not_deprecated_message()->set_inner_deprecated("bar"); // Checks for a deprecated sub-message should result in a warning. EXPECT_LOG_CONTAINS( "warning", "Using deprecated option 'envoy.test.deprecation_test.Base.InnerMessage.inner_deprecated'", - MessageUtil::checkForDeprecation(base)); + checkForDeprecation(base)); } // Check that repeated sub-messages get validated. @@ -623,7 +641,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(SubMessageDeprecated)) { EXPECT_LOG_CONTAINS("warning", "Using deprecated option " "'envoy.test.deprecation_test.Base.InnerMessage.inner_deprecated'", - MessageUtil::checkForDeprecation(base)); + checkForDeprecation(base)); } // Check that deprecated repeated messages trigger @@ -635,7 +653,7 @@ TEST_F(DeprecatedFieldsTest, DEPRECATED_FEATURE_TEST(RepeatedMessageDeprecated)) EXPECT_LOG_CONTAINS("warning", "Using deprecated option " "'envoy.test.deprecation_test.Base.deprecated_repeated_message'", - MessageUtil::checkForDeprecation(base)); + checkForDeprecation(base)); } class TimestampUtilTest : public testing::Test, public ::testing::WithParamInterface {}; diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index fbd9eaed7031..e8e2143f4b8f 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -6,6 +6,7 @@ #include #include "common/network/utility.h" +#include "common/protobuf/message_validator_impl.h" #include "common/protobuf/utility.h" #include "common/stream_info/stream_info_impl.h" @@ -74,7 +75,9 @@ RouterCheckTool RouterCheckTool::create(const std::string& router_config_file, auto factory_context = std::make_unique>(); auto config = std::make_unique(route_config, *factory_context, false); if (!disableDeprecationCheck) { - MessageUtil::checkForDeprecation(route_config, &factory_context->runtime_loader_); + MessageUtil::checkForUnexpectedFields(route_config, + ProtobufMessage::getStrictValidationVisitor(), + &factory_context->runtime_loader_); } return RouterCheckTool(std::move(factory_context), std::move(config), std::move(stats), From 0eab93b59eefd3d4ce45f5934186ffdec9e9ff37 Mon Sep 17 00:00:00 2001 From: Cynthia Coan Date: Mon, 2 Sep 2019 20:20:07 -0600 Subject: [PATCH 487/542] Fuzz reuse (#8119) This PR allows the envoy_cc_fuzz_test rule to be used when pulling in envoy. which can be useful when you're writing filters for envoy, and want to reuse the fuzzing architecture envoy has already built. other rules already allow for this (see envoy_cc_test in this same file for example). Risk Level: Low Testing: Testing the Old Rule Still Works It is possible to test the old rules still work (even without specifying a repository), by simply choosing your favorite fuzz test, and choosing to run bazel test on it. For example: bazel test //test/common/router:header_parser_fuzz_test. Any envoy_cc_fuzz_test rule should do. Testing New Rules Work I've done testing inside my own repository, but if you want to create your own test rule you can probably do the following in envoy-filter-example: Checkout envoy-filter-example, and update the envoy submodule to this pr. Follow the directions in: test/fuzz/README.md to define a envoy_cc_fuzz_test rule. Make sure to add a line for: repository = "@envoy" which is the new argument being added. You should be able to run the fuzz test. Signed-off-by: Cynthia Coan --- bazel/envoy_test.bzl | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/bazel/envoy_test.bzl b/bazel/envoy_test.bzl index e6985f92a1f3..73fdff21dc2d 100644 --- a/bazel/envoy_test.bzl +++ b/bazel/envoy_test.bzl @@ -63,8 +63,15 @@ def _envoy_test_linkopts(): }) + envoy_select_force_libcpp([], ["-lstdc++fs", "-latomic"]) # Envoy C++ fuzz test targets. These are not included in coverage runs. -def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs): - if not (corpus.startswith("//") or corpus.startswith(":")): +def envoy_cc_fuzz_test( + name, + corpus, + repository = "", + size = "medium", + deps = [], + tags = [], + **kwargs): + if not (corpus.startswith("//") or corpus.startswith(":") or corpus.startswith("@")): corpus_name = name + "_corpus" corpus = native.glob([corpus + "/**"]) native.filegroup( @@ -81,7 +88,11 @@ def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs): test_lib_name = name + "_lib" envoy_cc_test_library( name = test_lib_name, - deps = deps + ["//test/fuzz:fuzz_runner_lib", "//bazel:dynamic_stdlib"], + deps = deps + [ + repository + "//test/fuzz:fuzz_runner_lib", + repository + "//bazel:dynamic_stdlib", + ], + repository = repository, **kwargs ) native.cc_test( @@ -93,12 +104,13 @@ def envoy_cc_fuzz_test(name, corpus, deps = [], tags = [], **kwargs): data = [corpus_name], # No fuzzing on macOS. deps = select({ - "@envoy//bazel:apple": ["//test:dummy_main"], + "@envoy//bazel:apple": [repository + "//test:dummy_main"], "//conditions:default": [ ":" + test_lib_name, - "//test/fuzz:main", + repository + "//test/fuzz:main", ], }), + size = size, tags = tags, ) From 0b026cfa01287e55fa7b2adafcaba3a106d1f609 Mon Sep 17 00:00:00 2001 From: John Millikin Date: Tue, 3 Sep 2019 11:21:05 +0900 Subject: [PATCH 488/542] Set INCLUDE_DIRECTORIES so libcurl can find local urlapi.h (#8113) Fixes https://github.com/envoyproxy/envoy/issues/8112 Signed-off-by: John Millikin --- bazel/foreign_cc/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 82fcae17b6a6..1a96306c001b 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -75,6 +75,7 @@ envoy_cmake_external( "CURL_HIDDEN_SYMBOLS": "off", "CMAKE_USE_LIBSSH2": "off", "CMAKE_INSTALL_LIBDIR": "lib", + "INCLUDE_DIRECTORIES": "include/curl", }, lib_source = "@com_github_curl//:all", static_libraries = select({ From 911f3b0a08b7a8d724b42caab0b1f7398ab6efc2 Mon Sep 17 00:00:00 2001 From: Xin Date: Mon, 2 Sep 2019 22:26:10 -0400 Subject: [PATCH 489/542] cleanup: move test utility methods in ScopedRdsIntegrationTest to base class HttpIntegrationTest (#8108) Fixes #8050 Risk Level: LOW [refactor only] Signed-off-by: Xin Zhuang --- test/integration/http_integration.cc | 34 ++++++++++++++++++ test/integration/http_integration.h | 14 ++++++++ .../scoped_rds_integration_test.cc | 35 ------------------- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 62d2eb45ba41..62090c09d7c9 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -304,6 +304,40 @@ void HttpIntegrationTest::cleanupUpstreamAndDownstream() { } } +void HttpIntegrationTest::sendRequestAndVerifyResponse( + const Http::TestHeaderMapImpl& request_headers, const int request_size, + const Http::TestHeaderMapImpl& response_headers, const int response_size, + const int backend_idx) { + codec_client_ = makeHttpConnection(lookupPort("http")); + auto response = sendRequestAndWaitForResponse(request_headers, request_size, response_headers, + response_size, backend_idx); + verifyResponse(std::move(response), "200", response_headers, std::string(response_size, 'a')); + + EXPECT_TRUE(upstream_request_->complete()); + EXPECT_EQ(request_size, upstream_request_->bodyLength()); + cleanupUpstreamAndDownstream(); +} + +void HttpIntegrationTest::verifyResponse(IntegrationStreamDecoderPtr response, + const std::string& response_code, + const Http::TestHeaderMapImpl& expected_headers, + const std::string& expected_body) { + EXPECT_TRUE(response->complete()); + EXPECT_EQ(response_code, response->headers().Status()->value().getStringView()); + expected_headers.iterate( + [](const Http::HeaderEntry& header, void* context) -> Http::HeaderMap::Iterate { + auto response_headers = static_cast(context); + const Http::HeaderEntry* entry = + response_headers->get(Http::LowerCaseString{std::string(header.key().getStringView())}); + EXPECT_NE(entry, nullptr); + EXPECT_EQ(header.value().getStringView(), entry->value().getStringView()); + return Http::HeaderMap::Iterate::Continue; + }, + const_cast(static_cast(&response->headers()))); + + EXPECT_EQ(response->body(), expected_body); +} + uint64_t HttpIntegrationTest::waitForNextUpstreamRequest(const std::vector& upstream_indices) { uint64_t upstream_with_request; diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index bbeda044911e..1a2193556e65 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -138,6 +138,20 @@ class HttpIntegrationTest : public BaseIntegrationTest { // Close |codec_client_| and |fake_upstream_connection_| cleanly. void cleanupUpstreamAndDownstream(); + // Verifies the response_headers contains the expected_headers, and response body matches given + // body string. + void verifyResponse(IntegrationStreamDecoderPtr response, const std::string& response_code, + const Http::TestHeaderMapImpl& expected_headers, + const std::string& expected_body); + + // Helper that sends a request to Envoy, and verifies if Envoy response headers and body size is + // the same as the expected headers map. + // Requires the "http" port has been registered. + void sendRequestAndVerifyResponse(const Http::TestHeaderMapImpl& request_headers, + const int request_size, + const Http::TestHeaderMapImpl& response_headers, + const int response_size, const int backend_idx); + // Check for completion of upstream_request_, and a simple "200" response. void checkSimpleRequestSuccess(uint64_t expected_request_size, uint64_t expected_response_size, IntegrationStreamDecoder* response); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 1cefe11ac397..4bcc2a38997b 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -94,41 +94,6 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, HttpIntegrationTest::initialize(); } - // TODO(stevenzzzz): move these utility methods to base classes to share with other tests. - // Helper that verifies if given headers are in the response header map. - void verifyResponse(IntegrationStreamDecoderPtr response, const std::string& response_code, - const Http::TestHeaderMapImpl& expected_headers, - const std::string& expected_body) { - EXPECT_TRUE(response->complete()); - EXPECT_EQ(response_code, response->headers().Status()->value().getStringView()); - expected_headers.iterate( - [](const Http::HeaderEntry& header, void* context) -> Http::HeaderMap::Iterate { - auto response_headers = static_cast(context); - const Http::HeaderEntry* entry = response_headers->get( - Http::LowerCaseString{std::string(header.key().getStringView())}); - EXPECT_NE(entry, nullptr); - EXPECT_EQ(header.value().getStringView(), entry->value().getStringView()); - return Http::HeaderMap::Iterate::Continue; - }, - const_cast(static_cast(&response->headers()))); - EXPECT_EQ(response->body(), expected_body); - } - - // Helper that sends a request to Envoy, and verifies if Envoy response headers and body size is - // the same as the expected headers map. - void sendRequestAndVerifyResponse(const Http::TestHeaderMapImpl& request_headers, - const int request_size, - const Http::TestHeaderMapImpl& response_headers, - const int response_size, const int backend_idx) { - codec_client_ = makeHttpConnection(lookupPort("http")); - auto response = sendRequestAndWaitForResponse(request_headers, request_size, response_headers, - response_size, backend_idx); - verifyResponse(std::move(response), "200", response_headers, std::string(response_size, 'a')); - EXPECT_TRUE(upstream_request_->complete()); - EXPECT_EQ(request_size, upstream_request_->bodyLength()); - cleanupUpstreamAndDownstream(); - } - void createUpstreams() override { HttpIntegrationTest::createUpstreams(); // Create the SRDS upstream. From 1339ed2d4994392be93a3e2f9949afd40451e13a Mon Sep 17 00:00:00 2001 From: Andres Guedez <34292400+AndresGuedez@users.noreply.github.com> Date: Mon, 2 Sep 2019 22:27:50 -0400 Subject: [PATCH 490/542] upstream: fix invalid access of ClusterMap iterator during warming cluster modification (#8106) Risk Level: Medium Testing: New unit test added. Fix verified via --config=asan. Signed-off-by: Andres Guedez --- .../common/upstream/cluster_manager_impl.cc | 19 ++++- source/common/upstream/cluster_manager_impl.h | 6 ++ .../upstream/cluster_manager_impl_test.cc | 72 +++++++++++++++++++ test/common/upstream/utility.h | 18 ++--- 4 files changed, 104 insertions(+), 11 deletions(-) diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index cad4bd02d679..8a1b32cb90d8 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -487,9 +487,22 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::api::v2::Cluster& clust if (existing_active_cluster != active_clusters_.end() || existing_warming_cluster != warming_clusters_.end()) { - // The following init manager remove call is a NOP in the case we are already initialized. It's - // just kept here to avoid additional logic. - init_helper_.removeCluster(*existing_active_cluster->second->cluster_); + if (existing_active_cluster != active_clusters_.end()) { + // The following init manager remove call is a NOP in the case we are already initialized. + // It's just kept here to avoid additional logic. + init_helper_.removeCluster(*existing_active_cluster->second->cluster_); + } else { + // Validate that warming clusters are not added to the init_helper_. + // NOTE: This loop is compiled out in optimized builds. + for (const std::list& cluster_list : + {std::cref(init_helper_.primary_init_clusters_), + std::cref(init_helper_.secondary_init_clusters_)}) { + ASSERT(!std::any_of(cluster_list.begin(), cluster_list.end(), + [&existing_warming_cluster](Cluster* cluster) { + return existing_warming_cluster->second->cluster_.get() == cluster; + })); + } + } cm_stats_.cluster_modified_.inc(); } else { cm_stats_.cluster_added_.inc(); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index e45e9752372f..cb45bb14aca7 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -90,6 +90,9 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { Singleton::Manager& singleton_manager_; }; +// For friend declaration in ClusterManagerInitHelper. +class ClusterManagerImpl; + /** * This is a helper class used during cluster management initialization. Dealing with primary * clusters, secondary clusters, and CDS, is quite complicated, so this makes it easier to test. @@ -129,6 +132,9 @@ class ClusterManagerInitHelper : Logger::Loggable { State state() const { return state_; } private: + // To enable invariant assertions on the cluster lists. + friend ClusterManagerImpl; + void initializeSecondaryClusters(); void maybeFinishInitialize(); void onClusterInit(Cluster& cluster); diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index f9632b12da76..6fbc2e5afea2 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1234,6 +1234,78 @@ TEST_F(ClusterManagerImplTest, RemoveWarmingCluster) { EXPECT_TRUE(Mock::VerifyAndClearExpectations(cluster1.get())); } +TEST_F(ClusterManagerImplTest, ModifyWarmingCluster) { + time_system_.setSystemTime(std::chrono::milliseconds(1234567891234)); + create(defaultConfig()); + + InSequence s; + ReadyWatcher initialized; + EXPECT_CALL(initialized, ready()); + cluster_manager_->setInitializedCb([&]() -> void { initialized.ready(); }); + + // Add a "fake_cluster" in warming state. + std::shared_ptr cluster1 = + std::make_shared>(); + EXPECT_CALL(factory_, clusterFromProto_(_, _, _, _)) + .WillOnce(Return(std::make_pair(cluster1, nullptr))); + EXPECT_CALL(*cluster1, initializePhase()).Times(0); + EXPECT_CALL(*cluster1, initialize(_)); + EXPECT_TRUE( + cluster_manager_->addOrUpdateCluster(defaultStaticCluster("fake_cluster"), "version1")); + checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 0 /*active*/, 1 /*warming*/); + EXPECT_EQ(nullptr, cluster_manager_->get("fake_cluster")); + checkConfigDump(R"EOF( + dynamic_warming_clusters: + - version_info: "version1" + cluster: + name: "fake_cluster" + type: STATIC + connect_timeout: 0.25s + hosts: + - socket_address: + address: "127.0.0.1" + port_value: 11001 + last_updated: + seconds: 1234567891 + nanos: 234000000 + )EOF"); + + // Update the warming cluster that was just added. + std::shared_ptr cluster2 = + std::make_shared>(); + EXPECT_CALL(factory_, clusterFromProto_(_, _, _, _)) + .WillOnce(Return(std::make_pair(cluster2, nullptr))); + EXPECT_CALL(*cluster2, initializePhase()).Times(0); + EXPECT_CALL(*cluster2, initialize(_)); + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster( + parseClusterFromV2Json(fmt::sprintf(kDefaultStaticClusterTmpl, "fake_cluster", + R"EOF( +"socket_address": { + "address": "127.0.0.1", + "port_value": 11002 +})EOF")), + "version2")); + checkStats(1 /*added*/, 1 /*modified*/, 0 /*removed*/, 0 /*active*/, 1 /*warming*/); + checkConfigDump(R"EOF( + dynamic_warming_clusters: + - version_info: "version2" + cluster: + name: "fake_cluster" + type: STATIC + connect_timeout: 0.25s + hosts: + - socket_address: + address: "127.0.0.1" + port_value: 11002 + last_updated: + seconds: 1234567891 + nanos: 234000000 + )EOF"); + + EXPECT_TRUE(Mock::VerifyAndClearExpectations(cluster1.get())); + EXPECT_TRUE(Mock::VerifyAndClearExpectations(cluster2.get())); +} + // Verify that shutting down the cluster manager destroys warming clusters. TEST_F(ClusterManagerImplTest, ShutdownWithWarming) { create(defaultConfig()); diff --git a/test/common/upstream/utility.h b/test/common/upstream/utility.h index b5da2071d09b..b41b9cbfd280 100644 --- a/test/common/upstream/utility.h +++ b/test/common/upstream/utility.h @@ -15,8 +15,7 @@ namespace Envoy { namespace Upstream { namespace { -inline std::string defaultStaticClusterJson(const std::string& name) { - return fmt::sprintf(R"EOF( +constexpr static const char* kDefaultStaticClusterTmpl = R"EOF( { "name": "%s", "connect_timeout": "0.250s", @@ -24,15 +23,18 @@ inline std::string defaultStaticClusterJson(const std::string& name) { "lb_policy": "round_robin", "hosts": [ { - "socket_address": { - "address": "127.0.0.1", - "port_value": 11001 - } + %s, } ] } - )EOF", - name); + )EOF"; + +inline std::string defaultStaticClusterJson(const std::string& name) { + return fmt::sprintf(kDefaultStaticClusterTmpl, name, R"EOF( +"socket_address": { + "address": "127.0.0.1", + "port_value": 11001 +})EOF"); } inline envoy::config::bootstrap::v2::Bootstrap From b28edcae3da369ed9969796f89118c1f898a27d9 Mon Sep 17 00:00:00 2001 From: jaychenatr <54647402+jaychenatr@users.noreply.github.com> Date: Mon, 2 Sep 2019 19:29:05 -0700 Subject: [PATCH 491/542] api:Add a flag to disable overprovisioning in ClusterLoadAssignment (#8080) * api:Add a flag to disable overprovisioning in ClusterLoadAssignment Signed-off-by: Jie Chen * api:Add [#next-major-version and [#not-implemented-hide to the comment for field of disable_overprovisioning in ClusterLoadAssignment Signed-off-by: Jie Chen * api:Refine comments for the new added bool flag as suggested. Signed-off-by: Jie Chen --- api/envoy/api/v2/eds.proto | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/api/envoy/api/v2/eds.proto b/api/envoy/api/v2/eds.proto index d680ef7ea5aa..1719ad6d2c34 100644 --- a/api/envoy/api/v2/eds.proto +++ b/api/envoy/api/v2/eds.proto @@ -117,6 +117,17 @@ message ClusterLoadAssignment { // are considered stale and should be marked unhealthy. // Defaults to 0 which means endpoints never go stale. google.protobuf.Duration endpoint_stale_after = 4 [(validate.rules).duration.gt.seconds = 0]; + + // The flag to disable overprovisioning. If it is set to true, + // :ref:`overprovisioning factor + // ` will be ignored + // and Envoy will not perform graceful failover between priority levels or + // localities as endpoints become unhealthy. Otherwise Envoy will perform + // graceful failover as :ref:`overprovisioning factor + // ` suggests. + // [#next-major-version: Unify with overprovisioning config as a single message.] + // [#not-implemented-hide:] + bool disable_overprovisioning = 5; } // Load balancing policy settings. From 085d72b490c124a02849812798f5513a8df9ae72 Mon Sep 17 00:00:00 2001 From: htuch Date: Tue, 3 Sep 2019 14:49:12 -0400 Subject: [PATCH 492/542] api: clone v2[alpha] to v3alpha. (#8125) This patch establishes a v3alpha baseline API, by doing a simple copy of v2[alpha] dirs and some sed-style heuristic fixups of BUILD dependencies and proto package namespaces. The objective is provide a baseline which we can compare the output from tooling described in #8083 in later PRs, providing smaller visual diffs. The core philosophy of the API migration is that every step will be captured in a script (at least until the last manual steps), api/migration/v3alpha.sh. This script will capture deterministic migration steps, allowing v2[alpha] to continue to be updated until we finalize v3. There is likely to be significant changes, e.g. in addition to the work scoped for v3, we might want to reduce the amount of API churn by referring back to v2 protos where it makes sense. This will be done via tooling in later PRs. Part of #8083. Risk level: Low Testing: build @envoy_api//... Signed-off-by: Harvey Tuch --- api/envoy/admin/v3alpha/BUILD | 76 + api/envoy/admin/v3alpha/certs.proto | 57 + api/envoy/admin/v3alpha/clusters.proto | 143 ++ api/envoy/admin/v3alpha/config_dump.proto | 266 ++++ api/envoy/admin/v3alpha/listeners.proto | 28 + api/envoy/admin/v3alpha/memory.proto | 37 + api/envoy/admin/v3alpha/metrics.proto | 26 + api/envoy/admin/v3alpha/mutex_stats.proto | 28 + api/envoy/admin/v3alpha/server_info.proto | 137 ++ api/envoy/admin/v3alpha/tap.proto | 20 + api/envoy/api/v3alpha/BUILD | 171 ++ api/envoy/api/v3alpha/README.md | 9 + api/envoy/api/v3alpha/auth/BUILD | 35 + api/envoy/api/v3alpha/auth/cert.proto | 407 +++++ api/envoy/api/v3alpha/cds.proto | 660 ++++++++ api/envoy/api/v3alpha/cluster/BUILD | 48 + .../api/v3alpha/cluster/circuit_breaker.proto | 70 + api/envoy/api/v3alpha/cluster/filter.proto | 29 + .../v3alpha/cluster/outlier_detection.proto | 117 ++ api/envoy/api/v3alpha/core/BUILD | 136 ++ api/envoy/api/v3alpha/core/address.proto | 121 ++ api/envoy/api/v3alpha/core/base.proto | 292 ++++ .../api/v3alpha/core/config_source.proto | 126 ++ api/envoy/api/v3alpha/core/grpc_service.proto | 173 ++ api/envoy/api/v3alpha/core/health_check.proto | 271 ++++ api/envoy/api/v3alpha/core/http_uri.proto | 54 + api/envoy/api/v3alpha/core/protocol.proto | 157 ++ api/envoy/api/v3alpha/discovery.proto | 230 +++ api/envoy/api/v3alpha/eds.proto | 135 ++ api/envoy/api/v3alpha/endpoint/BUILD | 49 + api/envoy/api/v3alpha/endpoint/endpoint.proto | 129 ++ .../api/v3alpha/endpoint/load_report.proto | 148 ++ api/envoy/api/v3alpha/lds.proto | 206 +++ api/envoy/api/v3alpha/listener/BUILD | 41 + api/envoy/api/v3alpha/listener/listener.proto | 210 +++ .../listener/udp_listener_config.proto | 31 + api/envoy/api/v3alpha/ratelimit/BUILD | 14 + .../api/v3alpha/ratelimit/ratelimit.proto | 66 + api/envoy/api/v3alpha/rds.proto | 135 ++ api/envoy/api/v3alpha/route/BUILD | 28 + api/envoy/api/v3alpha/route/route.proto | 1404 +++++++++++++++++ api/envoy/api/v3alpha/srds.proto | 135 ++ api/envoy/config/accesslog/v3alpha/BUILD | 22 + api/envoy/config/accesslog/v3alpha/als.proto | 64 + api/envoy/config/accesslog/v3alpha/file.proto | 32 + api/envoy/config/bootstrap/v3alpha/BUILD | 40 + .../config/bootstrap/v3alpha/bootstrap.proto | 318 ++++ .../dynamic_forward_proxy/v3alpha/BUILD | 11 + .../v3alpha/cluster.proto | 24 + .../dynamic_forward_proxy/v3alpha/BUILD | 12 + .../v3alpha/dns_cache.proto | 69 + api/envoy/config/common/tap/v3alpha/BUILD | 13 + .../config/common/tap/v3alpha/common.proto | 51 + .../config/filter/accesslog/v3alpha/BUILD | 28 + .../filter/accesslog/v3alpha/accesslog.proto | 248 +++ api/envoy/config/filter/fault/v3alpha/BUILD | 13 + .../config/filter/fault/v3alpha/fault.proto | 84 + .../http/adaptive_concurrency/v3alpha/BUILD | 11 + .../v3alpha/adaptive_concurrency.proto | 11 + .../config/filter/http/buffer/v3alpha/BUILD | 8 + .../filter/http/buffer/v3alpha/buffer.proto | 36 + .../config/filter/http/csrf/v3alpha/BUILD | 12 + .../filter/http/csrf/v3alpha/csrf.proto | 51 + .../http/dynamic_forward_proxy/v3alpha/BUILD | 11 + .../v3alpha/dynamic_forward_proxy.proto | 24 + .../filter/http/ext_authz/v3alpha/BUILD | 15 + .../http/ext_authz/v3alpha/ext_authz.proto | 209 +++ .../config/filter/http/fault/v3alpha/BUILD | 13 + .../filter/http/fault/v3alpha/fault.proto | 115 ++ .../config/filter/http/gzip/v3alpha/BUILD | 8 + .../filter/http/gzip/v3alpha/gzip.proto | 75 + .../http/header_to_metadata/v3alpha/BUILD | 9 + .../v3alpha/header_to_metadata.proto | 92 ++ .../filter/http/health_check/v3alpha/BUILD | 21 + .../health_check/v3alpha/health_check.proto | 44 + .../filter/http/ip_tagging/v3alpha/BUILD | 9 + .../http/ip_tagging/v3alpha/ip_tagging.proto | 53 + .../filter/http/jwt_authn/v3alpha/BUILD | 23 + .../filter/http/jwt_authn/v3alpha/README.md | 66 + .../http/jwt_authn/v3alpha/config.proto | 467 ++++++ .../config/filter/http/lua/v3alpha/BUILD | 8 + .../config/filter/http/lua/v3alpha/lua.proto | 21 + .../filter/http/rate_limit/v3alpha/BUILD | 11 + .../http/rate_limit/v3alpha/rate_limit.proto | 60 + .../config/filter/http/rbac/v3alpha/BUILD | 9 + .../filter/http/rbac/v3alpha/rbac.proto | 38 + .../config/filter/http/router/v3alpha/BUILD | 15 + .../filter/http/router/v3alpha/router.proto | 67 + .../config/filter/http/squash/v3alpha/BUILD | 8 + .../filter/http/squash/v3alpha/squash.proto | 55 + .../config/filter/http/tap/v3alpha/BUILD | 11 + .../config/filter/http/tap/v3alpha/tap.proto | 21 + .../filter/http/transcoder/v3alpha/BUILD | 13 + .../http/transcoder/v3alpha/transcoder.proto | 123 ++ .../network/client_ssl_auth/v3alpha/BUILD | 9 + .../v3alpha/client_ssl_auth.proto | 41 + .../filter/network/ext_authz/v3alpha/BUILD | 9 + .../network/ext_authz/v3alpha/ext_authz.proto | 35 + .../http_connection_manager/v3alpha/BUILD | 31 + .../v3alpha/http_connection_manager.proto | 599 +++++++ .../filter/network/mongo_proxy/v3alpha/BUILD | 9 + .../mongo_proxy/v3alpha/mongo_proxy.proto | 36 + .../filter/network/rate_limit/v3alpha/BUILD | 12 + .../rate_limit/v3alpha/rate_limit.proto | 47 + .../config/filter/network/rbac/v3alpha/BUILD | 9 + .../filter/network/rbac/v3alpha/rbac.proto | 52 + .../filter/network/redis_proxy/v3alpha/BUILD | 12 + .../redis_proxy/v3alpha/redis_proxy.proto | 236 +++ .../filter/network/tcp_proxy/v3alpha/BUILD | 23 + .../network/tcp_proxy/v3alpha/tcp_proxy.proto | 146 ++ .../config/grpc_credential/v3alpha/BUILD | 27 + .../grpc_credential/v3alpha/aws_iam.proto | 29 + .../v3alpha/file_based_metadata.proto | 28 + .../config/health_checker/redis/v3alpha/BUILD | 8 + .../health_checker/redis/v3alpha/redis.proto | 19 + api/envoy/config/metrics/v3alpha/BUILD | 43 + .../metrics/v3alpha/metrics_service.proto | 21 + api/envoy/config/metrics/v3alpha/stats.proto | 331 ++++ api/envoy/config/overload/v3alpha/BUILD | 14 + .../config/overload/v3alpha/overload.proto | 78 + api/envoy/config/ratelimit/v3alpha/BUILD | 20 + api/envoy/config/ratelimit/v3alpha/rls.proto | 26 + api/envoy/config/rbac/v3alpha/BUILD | 36 + api/envoy/config/rbac/v3alpha/rbac.proto | 215 +++ .../resource_monitor/fixed_heap/v3alpha/BUILD | 9 + .../fixed_heap/v3alpha/fixed_heap.proto | 19 + .../injected_resource/v3alpha/BUILD | 9 + .../v3alpha/injected_resource.proto | 20 + api/envoy/config/trace/v3alpha/BUILD | 24 + api/envoy/config/trace/v3alpha/trace.proto | 204 +++ .../transport_socket/alts/v3alpha/BUILD | 11 + .../transport_socket/alts/v3alpha/alts.proto | 24 + .../config/transport_socket/tap/v3alpha/BUILD | 12 + .../transport_socket/tap/v3alpha/tap.proto | 26 + api/envoy/data/accesslog/v3alpha/BUILD | 24 + .../data/accesslog/v3alpha/accesslog.proto | 356 +++++ api/envoy/data/cluster/v3alpha/BUILD | 11 + .../v3alpha/outlier_detection_event.proto | 102 ++ api/envoy/data/core/v3alpha/BUILD | 15 + .../core/v3alpha/health_check_event.proto | 85 + api/envoy/data/tap/v3alpha/BUILD | 37 + api/envoy/data/tap/v3alpha/common.proto | 31 + api/envoy/data/tap/v3alpha/http.proto | 60 + api/envoy/data/tap/v3alpha/transport.proto | 97 ++ api/envoy/data/tap/v3alpha/wrapper.proto | 34 + api/envoy/service/accesslog/v3alpha/BUILD | 23 + api/envoy/service/accesslog/v3alpha/als.proto | 71 + api/envoy/service/auth/v3alpha/BUILD | 28 + .../auth/v3alpha/attribute_context.proto | 154 ++ .../service/auth/v3alpha/external_auth.proto | 77 + api/envoy/service/discovery/v3alpha/BUILD | 75 + api/envoy/service/discovery/v3alpha/ads.proto | 38 + api/envoy/service/discovery/v3alpha/hds.proto | 127 ++ .../service/discovery/v3alpha/rtds.proto | 50 + api/envoy/service/discovery/v3alpha/sds.proto | 34 + api/envoy/service/load_stats/v3alpha/BUILD | 22 + .../service/load_stats/v3alpha/lrs.proto | 82 + api/envoy/service/metrics/v3alpha/BUILD | 24 + .../metrics/v3alpha/metrics_service.proto | 41 + api/envoy/service/ratelimit/v3alpha/BUILD | 24 + api/envoy/service/ratelimit/v3alpha/rls.proto | 95 ++ api/envoy/service/tap/v3alpha/BUILD | 36 + api/envoy/service/tap/v3alpha/common.proto | 200 +++ api/envoy/service/tap/v3alpha/tap.proto | 50 + api/envoy/service/tap/v3alpha/tapds.proto | 44 + api/envoy/service/trace/v3alpha/BUILD | 23 + .../service/trace/v3alpha/trace_service.proto | 46 + api/migration/v3alpha.sh | 6 + tools/api/clone.sh | 62 + 169 files changed, 14370 insertions(+) create mode 100644 api/envoy/admin/v3alpha/BUILD create mode 100644 api/envoy/admin/v3alpha/certs.proto create mode 100644 api/envoy/admin/v3alpha/clusters.proto create mode 100644 api/envoy/admin/v3alpha/config_dump.proto create mode 100644 api/envoy/admin/v3alpha/listeners.proto create mode 100644 api/envoy/admin/v3alpha/memory.proto create mode 100644 api/envoy/admin/v3alpha/metrics.proto create mode 100644 api/envoy/admin/v3alpha/mutex_stats.proto create mode 100644 api/envoy/admin/v3alpha/server_info.proto create mode 100644 api/envoy/admin/v3alpha/tap.proto create mode 100644 api/envoy/api/v3alpha/BUILD create mode 100644 api/envoy/api/v3alpha/README.md create mode 100644 api/envoy/api/v3alpha/auth/BUILD create mode 100644 api/envoy/api/v3alpha/auth/cert.proto create mode 100644 api/envoy/api/v3alpha/cds.proto create mode 100644 api/envoy/api/v3alpha/cluster/BUILD create mode 100644 api/envoy/api/v3alpha/cluster/circuit_breaker.proto create mode 100644 api/envoy/api/v3alpha/cluster/filter.proto create mode 100644 api/envoy/api/v3alpha/cluster/outlier_detection.proto create mode 100644 api/envoy/api/v3alpha/core/BUILD create mode 100644 api/envoy/api/v3alpha/core/address.proto create mode 100644 api/envoy/api/v3alpha/core/base.proto create mode 100644 api/envoy/api/v3alpha/core/config_source.proto create mode 100644 api/envoy/api/v3alpha/core/grpc_service.proto create mode 100644 api/envoy/api/v3alpha/core/health_check.proto create mode 100644 api/envoy/api/v3alpha/core/http_uri.proto create mode 100644 api/envoy/api/v3alpha/core/protocol.proto create mode 100644 api/envoy/api/v3alpha/discovery.proto create mode 100644 api/envoy/api/v3alpha/eds.proto create mode 100644 api/envoy/api/v3alpha/endpoint/BUILD create mode 100644 api/envoy/api/v3alpha/endpoint/endpoint.proto create mode 100644 api/envoy/api/v3alpha/endpoint/load_report.proto create mode 100644 api/envoy/api/v3alpha/lds.proto create mode 100644 api/envoy/api/v3alpha/listener/BUILD create mode 100644 api/envoy/api/v3alpha/listener/listener.proto create mode 100644 api/envoy/api/v3alpha/listener/udp_listener_config.proto create mode 100644 api/envoy/api/v3alpha/ratelimit/BUILD create mode 100644 api/envoy/api/v3alpha/ratelimit/ratelimit.proto create mode 100644 api/envoy/api/v3alpha/rds.proto create mode 100644 api/envoy/api/v3alpha/route/BUILD create mode 100644 api/envoy/api/v3alpha/route/route.proto create mode 100644 api/envoy/api/v3alpha/srds.proto create mode 100644 api/envoy/config/accesslog/v3alpha/BUILD create mode 100644 api/envoy/config/accesslog/v3alpha/als.proto create mode 100644 api/envoy/config/accesslog/v3alpha/file.proto create mode 100644 api/envoy/config/bootstrap/v3alpha/BUILD create mode 100644 api/envoy/config/bootstrap/v3alpha/bootstrap.proto create mode 100644 api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/BUILD create mode 100644 api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/cluster.proto create mode 100644 api/envoy/config/common/dynamic_forward_proxy/v3alpha/BUILD create mode 100644 api/envoy/config/common/dynamic_forward_proxy/v3alpha/dns_cache.proto create mode 100644 api/envoy/config/common/tap/v3alpha/BUILD create mode 100644 api/envoy/config/common/tap/v3alpha/common.proto create mode 100644 api/envoy/config/filter/accesslog/v3alpha/BUILD create mode 100644 api/envoy/config/filter/accesslog/v3alpha/accesslog.proto create mode 100644 api/envoy/config/filter/fault/v3alpha/BUILD create mode 100644 api/envoy/config/filter/fault/v3alpha/fault.proto create mode 100644 api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto create mode 100644 api/envoy/config/filter/http/buffer/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/buffer/v3alpha/buffer.proto create mode 100644 api/envoy/config/filter/http/csrf/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/csrf/v3alpha/csrf.proto create mode 100644 api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/dynamic_forward_proxy.proto create mode 100644 api/envoy/config/filter/http/ext_authz/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto create mode 100644 api/envoy/config/filter/http/fault/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/fault/v3alpha/fault.proto create mode 100644 api/envoy/config/filter/http/gzip/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/gzip/v3alpha/gzip.proto create mode 100644 api/envoy/config/filter/http/header_to_metadata/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/header_to_metadata/v3alpha/header_to_metadata.proto create mode 100644 api/envoy/config/filter/http/health_check/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/health_check/v3alpha/health_check.proto create mode 100644 api/envoy/config/filter/http/ip_tagging/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/ip_tagging/v3alpha/ip_tagging.proto create mode 100644 api/envoy/config/filter/http/jwt_authn/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/jwt_authn/v3alpha/README.md create mode 100644 api/envoy/config/filter/http/jwt_authn/v3alpha/config.proto create mode 100644 api/envoy/config/filter/http/lua/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/lua/v3alpha/lua.proto create mode 100644 api/envoy/config/filter/http/rate_limit/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto create mode 100644 api/envoy/config/filter/http/rbac/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/rbac/v3alpha/rbac.proto create mode 100644 api/envoy/config/filter/http/router/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/router/v3alpha/router.proto create mode 100644 api/envoy/config/filter/http/squash/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/squash/v3alpha/squash.proto create mode 100644 api/envoy/config/filter/http/tap/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/tap/v3alpha/tap.proto create mode 100644 api/envoy/config/filter/http/transcoder/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/transcoder/v3alpha/transcoder.proto create mode 100644 api/envoy/config/filter/network/client_ssl_auth/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto create mode 100644 api/envoy/config/filter/network/ext_authz/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/ext_authz/v3alpha/ext_authz.proto create mode 100644 api/envoy/config/filter/network/http_connection_manager/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto create mode 100644 api/envoy/config/filter/network/mongo_proxy/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/mongo_proxy/v3alpha/mongo_proxy.proto create mode 100644 api/envoy/config/filter/network/rate_limit/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto create mode 100644 api/envoy/config/filter/network/rbac/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/rbac/v3alpha/rbac.proto create mode 100644 api/envoy/config/filter/network/redis_proxy/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto create mode 100644 api/envoy/config/filter/network/tcp_proxy/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto create mode 100644 api/envoy/config/grpc_credential/v3alpha/BUILD create mode 100644 api/envoy/config/grpc_credential/v3alpha/aws_iam.proto create mode 100644 api/envoy/config/grpc_credential/v3alpha/file_based_metadata.proto create mode 100644 api/envoy/config/health_checker/redis/v3alpha/BUILD create mode 100644 api/envoy/config/health_checker/redis/v3alpha/redis.proto create mode 100644 api/envoy/config/metrics/v3alpha/BUILD create mode 100644 api/envoy/config/metrics/v3alpha/metrics_service.proto create mode 100644 api/envoy/config/metrics/v3alpha/stats.proto create mode 100644 api/envoy/config/overload/v3alpha/BUILD create mode 100644 api/envoy/config/overload/v3alpha/overload.proto create mode 100644 api/envoy/config/ratelimit/v3alpha/BUILD create mode 100644 api/envoy/config/ratelimit/v3alpha/rls.proto create mode 100644 api/envoy/config/rbac/v3alpha/BUILD create mode 100644 api/envoy/config/rbac/v3alpha/rbac.proto create mode 100644 api/envoy/config/resource_monitor/fixed_heap/v3alpha/BUILD create mode 100644 api/envoy/config/resource_monitor/fixed_heap/v3alpha/fixed_heap.proto create mode 100644 api/envoy/config/resource_monitor/injected_resource/v3alpha/BUILD create mode 100644 api/envoy/config/resource_monitor/injected_resource/v3alpha/injected_resource.proto create mode 100644 api/envoy/config/trace/v3alpha/BUILD create mode 100644 api/envoy/config/trace/v3alpha/trace.proto create mode 100644 api/envoy/config/transport_socket/alts/v3alpha/BUILD create mode 100644 api/envoy/config/transport_socket/alts/v3alpha/alts.proto create mode 100644 api/envoy/config/transport_socket/tap/v3alpha/BUILD create mode 100644 api/envoy/config/transport_socket/tap/v3alpha/tap.proto create mode 100644 api/envoy/data/accesslog/v3alpha/BUILD create mode 100644 api/envoy/data/accesslog/v3alpha/accesslog.proto create mode 100644 api/envoy/data/cluster/v3alpha/BUILD create mode 100644 api/envoy/data/cluster/v3alpha/outlier_detection_event.proto create mode 100644 api/envoy/data/core/v3alpha/BUILD create mode 100644 api/envoy/data/core/v3alpha/health_check_event.proto create mode 100644 api/envoy/data/tap/v3alpha/BUILD create mode 100644 api/envoy/data/tap/v3alpha/common.proto create mode 100644 api/envoy/data/tap/v3alpha/http.proto create mode 100644 api/envoy/data/tap/v3alpha/transport.proto create mode 100644 api/envoy/data/tap/v3alpha/wrapper.proto create mode 100644 api/envoy/service/accesslog/v3alpha/BUILD create mode 100644 api/envoy/service/accesslog/v3alpha/als.proto create mode 100644 api/envoy/service/auth/v3alpha/BUILD create mode 100644 api/envoy/service/auth/v3alpha/attribute_context.proto create mode 100644 api/envoy/service/auth/v3alpha/external_auth.proto create mode 100644 api/envoy/service/discovery/v3alpha/BUILD create mode 100644 api/envoy/service/discovery/v3alpha/ads.proto create mode 100644 api/envoy/service/discovery/v3alpha/hds.proto create mode 100644 api/envoy/service/discovery/v3alpha/rtds.proto create mode 100644 api/envoy/service/discovery/v3alpha/sds.proto create mode 100644 api/envoy/service/load_stats/v3alpha/BUILD create mode 100644 api/envoy/service/load_stats/v3alpha/lrs.proto create mode 100644 api/envoy/service/metrics/v3alpha/BUILD create mode 100644 api/envoy/service/metrics/v3alpha/metrics_service.proto create mode 100644 api/envoy/service/ratelimit/v3alpha/BUILD create mode 100644 api/envoy/service/ratelimit/v3alpha/rls.proto create mode 100644 api/envoy/service/tap/v3alpha/BUILD create mode 100644 api/envoy/service/tap/v3alpha/common.proto create mode 100644 api/envoy/service/tap/v3alpha/tap.proto create mode 100644 api/envoy/service/tap/v3alpha/tapds.proto create mode 100644 api/envoy/service/trace/v3alpha/BUILD create mode 100644 api/envoy/service/trace/v3alpha/trace_service.proto create mode 100755 api/migration/v3alpha.sh create mode 100755 tools/api/clone.sh diff --git a/api/envoy/admin/v3alpha/BUILD b/api/envoy/admin/v3alpha/BUILD new file mode 100644 index 000000000000..71b0790b69a9 --- /dev/null +++ b/api/envoy/admin/v3alpha/BUILD @@ -0,0 +1,76 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "config_dump", + srcs = ["config_dump.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha:cds", + "//envoy/api/v3alpha:lds", + "//envoy/api/v3alpha:rds", + "//envoy/api/v3alpha:srds", + "//envoy/api/v3alpha/auth:cert", + "//envoy/config/bootstrap/v3alpha:bootstrap", + ], +) + +api_proto_library_internal( + name = "clusters", + srcs = ["clusters.proto"], + visibility = ["//visibility:public"], + deps = [ + ":metrics", + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:health_check", + "//envoy/type:percent", + ], +) + +api_proto_library_internal( + name = "listeners", + srcs = ["listeners.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha/core:address", + ], +) + +api_proto_library_internal( + name = "metrics", + srcs = ["metrics.proto"], + visibility = ["//visibility:public"], +) + +api_proto_library_internal( + name = "memory", + srcs = ["memory.proto"], + visibility = ["//visibility:public"], +) + +api_proto_library_internal( + name = "mutex_stats", + srcs = ["mutex_stats.proto"], + visibility = ["//visibility:public"], +) + +api_proto_library_internal( + name = "certs", + srcs = ["certs.proto"], + visibility = ["//visibility:public"], +) + +api_proto_library_internal( + name = "server_info", + srcs = ["server_info.proto"], + visibility = ["//visibility:public"], +) + +api_proto_library_internal( + name = "tap", + srcs = ["tap.proto"], + deps = [ + "//envoy/service/tap/v3alpha:common", + ], +) diff --git a/api/envoy/admin/v3alpha/certs.proto b/api/envoy/admin/v3alpha/certs.proto new file mode 100644 index 000000000000..e34fd36d992b --- /dev/null +++ b/api/envoy/admin/v3alpha/certs.proto @@ -0,0 +1,57 @@ +syntax = "proto3"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "CertsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +import "google/protobuf/timestamp.proto"; + +// [#protodoc-title: Certificates] + +// Proto representation of certificate details. Admin endpoint uses this wrapper for `/certs` to +// display certificate information. See :ref:`/certs ` for more +// information. +message Certificates { + // List of certificates known to an Envoy. + repeated Certificate certificates = 1; +} + +message Certificate { + + // Details of CA certificate. + repeated CertificateDetails ca_cert = 1; + + // Details of Certificate Chain + repeated CertificateDetails cert_chain = 2; +} + +message CertificateDetails { + // Path of the certificate. + string path = 1; + + // Certificate Serial Number. + string serial_number = 2; + + // List of Subject Alternate names. + repeated SubjectAlternateName subject_alt_names = 3; + + // Minimum of days until expiration of certificate and it's chain. + uint64 days_until_expiration = 4; + + // Indicates the time from which the certificate is valid. + google.protobuf.Timestamp valid_from = 5; + + // Indicates the time at which the certificate expires. + google.protobuf.Timestamp expiration_time = 6; +} + +message SubjectAlternateName { + + // Subject Alternate Name. + oneof name { + string dns = 1; + string uri = 2; + } +} diff --git a/api/envoy/admin/v3alpha/clusters.proto b/api/envoy/admin/v3alpha/clusters.proto new file mode 100644 index 000000000000..093448d9f82c --- /dev/null +++ b/api/envoy/admin/v3alpha/clusters.proto @@ -0,0 +1,143 @@ +syntax = "proto3"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "ClustersProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +import "envoy/admin/v3alpha/metrics.proto"; +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/core/health_check.proto"; +import "envoy/type/percent.proto"; + +// [#protodoc-title: Clusters] + +// Admin endpoint uses this wrapper for `/clusters` to display cluster status information. +// See :ref:`/clusters ` for more information. +message Clusters { + // Mapping from cluster name to each cluster's status. + repeated ClusterStatus cluster_statuses = 1; +} + +// Details an individual cluster's current status. +message ClusterStatus { + // Name of the cluster. + string name = 1; + + // Denotes whether this cluster was added via API or configured statically. + bool added_via_api = 2; + + // The success rate threshold used in the last interval. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *false*, all errors: externally and locally generated were used to calculate the threshold. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *true*, only externally generated errors were used to calculate the threshold. + // The threshold is used to eject hosts based on their success rate. See + // :ref:`Cluster outlier detection ` documentation for details. + // + // Note: this field may be omitted in any of the three following cases: + // + // 1. There were not enough hosts with enough request volume to proceed with success rate based + // outlier ejection. + // 2. The threshold is computed to be < 0 because a negative value implies that there was no + // threshold for that interval. + // 3. Outlier detection is not enabled for this cluster. + envoy.type.Percent success_rate_ejection_threshold = 3; + + // Mapping from host address to the host's current status. + repeated HostStatus host_statuses = 4; + + // The success rate threshold used in the last interval when only locally originated failures were + // taken into account and externally originated errors were treated as success. + // This field should be interpretted only when + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *true*. The threshold is used to eject hosts based on their success rate. + // See :ref:`Cluster outlier detection ` documentation for + // details. + // + // Note: this field may be omitted in any of the three following cases: + // + // 1. There were not enough hosts with enough request volume to proceed with success rate based + // outlier ejection. + // 2. The threshold is computed to be < 0 because a negative value implies that there was no + // threshold for that interval. + // 3. Outlier detection is not enabled for this cluster. + envoy.type.Percent local_origin_success_rate_ejection_threshold = 5; +} + +// Current state of a particular host. +message HostStatus { + // Address of this host. + envoy.api.v3alpha.core.Address address = 1; + + // List of stats specific to this host. + repeated SimpleMetric stats = 2; + + // The host's current health status. + HostHealthStatus health_status = 3; + + // Request success rate for this host over the last calculated interval. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *false*, all errors: externally and locally generated were used in success rate + // calculation. If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *true*, only externally generated errors were used in success rate calculation. + // See :ref:`Cluster outlier detection ` documentation for + // details. + // + // Note: the message will not be present if host did not have enough request volume to calculate + // success rate or the cluster did not have enough hosts to run through success rate outlier + // ejection. + envoy.type.Percent success_rate = 4; + + // The host's weight. If not configured, the value defaults to 1. + uint32 weight = 5; + + // The hostname of the host, if applicable. + string hostname = 6; + + // The host's priority. If not configured, the value defaults to 0 (highest priority). + uint32 priority = 7; + + // Request success rate for this host over the last calculated + // interval when only locally originated errors are taken into account and externally originated + // errors were treated as success. + // This field should be interpretted only when + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *true*. + // See :ref:`Cluster outlier detection ` documentation for + // details. + // + // Note: the message will not be present if host did not have enough request volume to calculate + // success rate or the cluster did not have enough hosts to run through success rate outlier + // ejection. + envoy.type.Percent local_origin_success_rate = 8; +} + +// Health status for a host. +message HostHealthStatus { + // The host is currently failing active health checks. + bool failed_active_health_check = 1; + + // The host is currently considered an outlier and has been ejected. + bool failed_outlier_check = 2; + + // The host is currently being marked as degraded through active health checking. + bool failed_active_degraded_check = 4; + + // The host has been removed from service discovery, but is being stabilized due to active + // health checking. + bool pending_dynamic_removal = 5; + + // The host has not yet been health checked. + bool pending_active_hc = 6; + + // Health status as reported by EDS. Note: only HEALTHY and UNHEALTHY are currently supported + // here. + // TODO(mrice32): pipe through remaining EDS health status possibilities. + envoy.api.v3alpha.core.HealthStatus eds_health_status = 3; +} diff --git a/api/envoy/admin/v3alpha/config_dump.proto b/api/envoy/admin/v3alpha/config_dump.proto new file mode 100644 index 000000000000..e909b1adfa72 --- /dev/null +++ b/api/envoy/admin/v3alpha/config_dump.proto @@ -0,0 +1,266 @@ +syntax = "proto3"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "ConfigDumpProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +import "envoy/api/v3alpha/auth/cert.proto"; +import "envoy/api/v3alpha/cds.proto"; +import "envoy/api/v3alpha/lds.proto"; +import "envoy/api/v3alpha/rds.proto"; +import "envoy/api/v3alpha/srds.proto"; +import "envoy/config/bootstrap/v3alpha/bootstrap.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/timestamp.proto"; + +import "gogoproto/gogo.proto"; + +// [#protodoc-title: ConfigDump] + +// The :ref:`/config_dump ` admin endpoint uses this wrapper +// message to maintain and serve arbitrary configuration information from any component in Envoy. +message ConfigDump { + // This list is serialized and dumped in its entirety at the + // :ref:`/config_dump ` endpoint. + // + // The following configurations are currently supported and will be dumped in the order given + // below: + // + // * *bootstrap*: :ref:`BootstrapConfigDump ` + // * *clusters*: :ref:`ClustersConfigDump ` + // * *listeners*: :ref:`ListenersConfigDump ` + // * *routes*: :ref:`RoutesConfigDump ` + repeated google.protobuf.Any configs = 1; +} + +// This message describes the bootstrap configuration that Envoy was started with. This includes +// any CLI overrides that were merged. Bootstrap configuration information can be used to recreate +// the static portions of an Envoy configuration by reusing the output as the bootstrap +// configuration for another Envoy. +message BootstrapConfigDump { + envoy.config.bootstrap.v3alpha.Bootstrap bootstrap = 1; + + // The timestamp when the BootstrapConfig was last updated. + google.protobuf.Timestamp last_updated = 2; +} + +// Envoy's listener manager fills this message with all currently known listeners. Listener +// configuration information can be used to recreate an Envoy configuration by populating all +// listeners as static listeners or by returning them in a LDS response. +message ListenersConfigDump { + // This is the :ref:`version_info ` in the + // last processed LDS discovery response. If there are only static bootstrap listeners, this field + // will be "". + string version_info = 1; + + // Describes a statically loaded listener. + message StaticListener { + // The listener config. + envoy.api.v3alpha.Listener listener = 1; + + // The timestamp when the Listener was last updated. + google.protobuf.Timestamp last_updated = 2; + } + + // Describes a dynamically loaded cluster via the LDS API. + message DynamicListener { + // This is the per-resource version information. This version is currently taken from the + // :ref:`version_info ` field at the time + // that the listener was loaded. In the future, discrete per-listener versions may be supported + // by the API. + string version_info = 1; + + // The listener config. + envoy.api.v3alpha.Listener listener = 2; + + // The timestamp when the Listener was last updated. + google.protobuf.Timestamp last_updated = 3; + } + + // The statically loaded listener configs. + repeated StaticListener static_listeners = 2; + + // The dynamically loaded active listeners. These are listeners that are available to service + // data plane traffic. + repeated DynamicListener dynamic_active_listeners = 3; + + // The dynamically loaded warming listeners. These are listeners that are currently undergoing + // warming in preparation to service data plane traffic. Note that if attempting to recreate an + // Envoy configuration from a configuration dump, the warming listeners should generally be + // discarded. + repeated DynamicListener dynamic_warming_listeners = 4; + + // The dynamically loaded draining listeners. These are listeners that are currently undergoing + // draining in preparation to stop servicing data plane traffic. Note that if attempting to + // recreate an Envoy configuration from a configuration dump, the draining listeners should + // generally be discarded. + repeated DynamicListener dynamic_draining_listeners = 5; +} + +// Envoy's cluster manager fills this message with all currently known clusters. Cluster +// configuration information can be used to recreate an Envoy configuration by populating all +// clusters as static clusters or by returning them in a CDS response. +message ClustersConfigDump { + // This is the :ref:`version_info ` in the + // last processed CDS discovery response. If there are only static bootstrap clusters, this field + // will be "". + string version_info = 1; + + // Describes a statically loaded cluster. + message StaticCluster { + // The cluster config. + envoy.api.v3alpha.Cluster cluster = 1; + + // The timestamp when the Cluster was last updated. + google.protobuf.Timestamp last_updated = 2; + } + + // Describes a dynamically loaded cluster via the CDS API. + message DynamicCluster { + // This is the per-resource version information. This version is currently taken from the + // :ref:`version_info ` field at the time + // that the cluster was loaded. In the future, discrete per-cluster versions may be supported by + // the API. + string version_info = 1; + + // The cluster config. + envoy.api.v3alpha.Cluster cluster = 2; + + // The timestamp when the Cluster was last updated. + google.protobuf.Timestamp last_updated = 3; + } + + // The statically loaded cluster configs. + repeated StaticCluster static_clusters = 2; + + // The dynamically loaded active clusters. These are clusters that are available to service + // data plane traffic. + repeated DynamicCluster dynamic_active_clusters = 3; + + // The dynamically loaded warming clusters. These are clusters that are currently undergoing + // warming in preparation to service data plane traffic. Note that if attempting to recreate an + // Envoy configuration from a configuration dump, the warming clusters should generally be + // discarded. + repeated DynamicCluster dynamic_warming_clusters = 4; +} + +// Envoy's RDS implementation fills this message with all currently loaded routes, as described by +// their RouteConfiguration objects. Static routes configured in the bootstrap configuration are +// separated from those configured dynamically via RDS. Route configuration information can be used +// to recreate an Envoy configuration by populating all routes as static routes or by returning them +// in RDS responses. +message RoutesConfigDump { + message StaticRouteConfig { + // The route config. + envoy.api.v3alpha.RouteConfiguration route_config = 1; + + // The timestamp when the Route was last updated. + google.protobuf.Timestamp last_updated = 2; + } + + message DynamicRouteConfig { + // This is the per-resource version information. This version is currently taken from the + // :ref:`version_info ` field at the time that + // the route configuration was loaded. + string version_info = 1; + + // The route config. + envoy.api.v3alpha.RouteConfiguration route_config = 2; + + // The timestamp when the Route was last updated. + google.protobuf.Timestamp last_updated = 3; + } + + // The statically loaded route configs. + repeated StaticRouteConfig static_route_configs = 2; + + // The dynamically loaded route configs. + repeated DynamicRouteConfig dynamic_route_configs = 3; +} + +// Envoy's scoped RDS implementation fills this message with all currently loaded route +// configuration scopes (defined via ScopedRouteConfigurationsSet protos). This message lists both +// the scopes defined inline with the higher order object (i.e., the HttpConnectionManager) and the +// dynamically obtained scopes via the SRDS API. +message ScopedRoutesConfigDump { + message InlineScopedRouteConfigs { + // The name assigned to the scoped route configurations. + string name = 1; + + // The scoped route configurations. + repeated envoy.api.v3alpha.ScopedRouteConfiguration scoped_route_configs = 2; + + // The timestamp when the scoped route config set was last updated. + google.protobuf.Timestamp last_updated = 3; + } + + message DynamicScopedRouteConfigs { + // The name assigned to the scoped route configurations. + string name = 1; + + // This is the per-resource version information. This version is currently taken from the + // :ref:`version_info ` field at the time that + // the scoped routes configuration was loaded. + string version_info = 2; + + // The scoped route configurations. + repeated envoy.api.v3alpha.ScopedRouteConfiguration scoped_route_configs = 3; + + // The timestamp when the scoped route config set was last updated. + google.protobuf.Timestamp last_updated = 4; + } + + // The statically loaded scoped route configs. + repeated InlineScopedRouteConfigs inline_scoped_route_configs = 1; + + // The dynamically loaded scoped route configs. + repeated DynamicScopedRouteConfigs dynamic_scoped_route_configs = 2; +} + +// Envoys SDS implementation fills this message with all secrets fetched dynamically via SDS. +message SecretsConfigDump { + // DynamicSecret contains secret information fetched via SDS. + message DynamicSecret { + // The name assigned to the secret. + string name = 1; + + // This is the per-resource version information. + string version_info = 2; + + // The timestamp when the secret was last updated. + google.protobuf.Timestamp last_updated = 3; + + // The actual secret information. + // Security sensitive information is redacted (replaced with "[redacted]") for + // private keys and passwords in TLS certificates. + envoy.api.v3alpha.auth.Secret secret = 4; + } + + // StaticSecret specifies statically loaded secret in bootstrap. + message StaticSecret { + // The name assigned to the secret. + string name = 1; + + // The timestamp when the secret was last updated. + google.protobuf.Timestamp last_updated = 2; + + // The actual secret information. + // Security sensitive information is redacted (replaced with "[redacted]") for + // private keys and passwords in TLS certificates. + envoy.api.v3alpha.auth.Secret secret = 3; + } + + // The statically loaded secrets. + repeated StaticSecret static_secrets = 1; + + // The dynamically loaded active secrets. These are secrets that are available to service + // clusters or listeners. + repeated DynamicSecret dynamic_active_secrets = 2; + + // The dynamically loaded warming secrets. These are secrets that are currently undergoing + // warming in preparation to service clusters or listeners. + repeated DynamicSecret dynamic_warming_secrets = 3; +} diff --git a/api/envoy/admin/v3alpha/listeners.proto b/api/envoy/admin/v3alpha/listeners.proto new file mode 100644 index 000000000000..5e4d121b1fb2 --- /dev/null +++ b/api/envoy/admin/v3alpha/listeners.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "ListenersProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +import "envoy/api/v3alpha/core/address.proto"; + +// [#protodoc-title: Listeners] + +// Admin endpoint uses this wrapper for `/listeners` to display listener status information. +// See :ref:`/listeners ` for more information. +message Listeners { + // List of listener statuses. + repeated ListenerStatus listener_statuses = 1; +} + +// Details an individual listener's current status. +message ListenerStatus { + // Name of the listener + string name = 1; + + // The actual local address that the listener is listening on. If a listener was configured + // to listen on port 0, then this address has the port that was allocated by the OS. + envoy.api.v3alpha.core.Address local_address = 2; +} diff --git a/api/envoy/admin/v3alpha/memory.proto b/api/envoy/admin/v3alpha/memory.proto new file mode 100644 index 000000000000..4c17be034e47 --- /dev/null +++ b/api/envoy/admin/v3alpha/memory.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "MemoryProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +// [#protodoc-title: Memory] + +// Proto representation of the internal memory consumption of an Envoy instance. These represent +// values extracted from an internal TCMalloc instance. For more information, see the section of the +// docs entitled ["Generic Tcmalloc Status"](https://gperftools.github.io/gperftools/tcmalloc.html). +message Memory { + + // The number of bytes allocated by the heap for Envoy. This is an alias for + // `generic.current_allocated_bytes`. + uint64 allocated = 1; + + // The number of bytes reserved by the heap but not necessarily allocated. This is an alias for + // `generic.heap_size`. + uint64 heap_size = 2; + + // The number of bytes in free, unmapped pages in the page heap. These bytes always count towards + // virtual memory usage, and depending on the OS, typically do not count towards physical memory + // usage. This is an alias for `tcmalloc.pageheap_unmapped_bytes`. + uint64 pageheap_unmapped = 3; + + // The number of bytes in free, mapped pages in the page heap. These bytes always count towards + // virtual memory usage, and unless the underlying memory is swapped out by the OS, they also + // count towards physical memory usage. This is an alias for `tcmalloc.pageheap_free_bytes`. + uint64 pageheap_free = 4; + + // The amount of memory used by the TCMalloc thread caches (for small objects). This is an alias + // for `tcmalloc.current_total_thread_cache_bytes`. + uint64 total_thread_cache = 5; +} diff --git a/api/envoy/admin/v3alpha/metrics.proto b/api/envoy/admin/v3alpha/metrics.proto new file mode 100644 index 000000000000..5a52ff2648b4 --- /dev/null +++ b/api/envoy/admin/v3alpha/metrics.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "MetricsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +// [#protodoc-title: Metrics] + +// Proto representation of an Envoy Counter or Gauge value. +message SimpleMetric { + enum Type { + COUNTER = 0; + GAUGE = 1; + } + + // Type of the metric represented. + Type type = 1; + + // Current metric value. + uint64 value = 2; + + // Name of the metric. + string name = 3; +} diff --git a/api/envoy/admin/v3alpha/mutex_stats.proto b/api/envoy/admin/v3alpha/mutex_stats.proto new file mode 100644 index 000000000000..72350dea8d77 --- /dev/null +++ b/api/envoy/admin/v3alpha/mutex_stats.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "MutexStatsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +// [#protodoc-title: MutexStats] + +// Proto representation of the statistics collected upon absl::Mutex contention, if Envoy is run +// under :option:`--enable-mutex-tracing`. For more information, see the `absl::Mutex` +// [docs](https://abseil.io/about/design/mutex#extra-features). +// +// *NB*: The wait cycles below are measured by `absl::base_internal::CycleClock`, and may not +// correspond to core clock frequency. For more information, see the `CycleClock` +// [docs](https://github.com/abseil/abseil-cpp/blob/master/absl/base/internal/cycleclock.h). +message MutexStats { + + // The number of individual mutex contentions which have occurred since startup. + uint64 num_contentions = 1; + + // The length of the current contention wait cycle. + uint64 current_wait_cycles = 2; + + // The lifetime total of all contention wait cycles. + uint64 lifetime_wait_cycles = 3; +} diff --git a/api/envoy/admin/v3alpha/server_info.proto b/api/envoy/admin/v3alpha/server_info.proto new file mode 100644 index 000000000000..a259a87f1651 --- /dev/null +++ b/api/envoy/admin/v3alpha/server_info.proto @@ -0,0 +1,137 @@ +syntax = "proto3"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "ServerInfoProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +import "google/protobuf/duration.proto"; + +// [#protodoc-title: Server State] + +// Proto representation of the value returned by /server_info, containing +// server version/server status information. +message ServerInfo { + // Server version. + string version = 1; + + enum State { + // Server is live and serving traffic. + LIVE = 0; + // Server is draining listeners in response to external health checks failing. + DRAINING = 1; + // Server has not yet completed cluster manager initialization. + PRE_INITIALIZING = 2; + // Server is running the cluster manager initialization callbacks (e.g., RDS). + INITIALIZING = 3; + } + + // State of the server. + State state = 2; + + // Uptime since current epoch was started. + google.protobuf.Duration uptime_current_epoch = 3; + + // Uptime since the start of the first epoch. + google.protobuf.Duration uptime_all_epochs = 4; + + // Hot restart version. + string hot_restart_version = 5; + + // Command line options the server is currently running with. + CommandLineOptions command_line_options = 6; +} + +message CommandLineOptions { + // See :option:`--base-id` for details. + uint64 base_id = 1; + + // See :option:`--concurrency` for details. + uint32 concurrency = 2; + + // See :option:`--config-path` for details. + string config_path = 3; + + // See :option:`--config-yaml` for details. + string config_yaml = 4; + + // See :option:`--allow-unknown-static-fields` for details. + bool allow_unknown_static_fields = 5; + + // See :option:`--reject-unknown-dynamic-fields` for details. + bool reject_unknown_dynamic_fields = 26; + + // See :option:`--admin-address-path` for details. + string admin_address_path = 6; + + enum IpVersion { + v4 = 0; + v6 = 1; + } + + // See :option:`--local-address-ip-version` for details. + IpVersion local_address_ip_version = 7; + + // See :option:`--log-level` for details. + string log_level = 8; + + // See :option:`--component-log-level` for details. + string component_log_level = 9; + + // See :option:`--log-format` for details. + string log_format = 10; + + // See :option:`--log-path` for details. + string log_path = 11; + + reserved 12; + + // See :option:`--service-cluster` for details. + string service_cluster = 13; + + // See :option:`--service-node` for details. + string service_node = 14; + + // See :option:`--service-zone` for details. + string service_zone = 15; + + // See :option:`--file-flush-interval-msec` for details. + google.protobuf.Duration file_flush_interval = 16; + + // See :option:`--drain-time-s` for details. + google.protobuf.Duration drain_time = 17; + + // See :option:`--parent-shutdown-time-s` for details. + google.protobuf.Duration parent_shutdown_time = 18; + + enum Mode { + // Validate configs and then serve traffic normally. + Serve = 0; + + // Validate configs and exit. + Validate = 1; + + // Completely load and initialize the config, and then exit without running the listener loop. + InitOnly = 2; + } + + // See :option:`--mode` for details. + Mode mode = 19; + + // max_stats and max_obj_name_len are now unused and have no effect. + uint64 max_stats = 20 [deprecated = true]; + uint64 max_obj_name_len = 21 [deprecated = true]; + + // See :option:`--disable-hot-restart` for details. + bool disable_hot_restart = 22; + + // See :option:`--enable-mutex-tracing` for details. + bool enable_mutex_tracing = 23; + + // See :option:`--restart-epoch` for details. + uint32 restart_epoch = 24; + + // See :option:`--cpuset-threads` for details. + bool cpuset_threads = 25; +} diff --git a/api/envoy/admin/v3alpha/tap.proto b/api/envoy/admin/v3alpha/tap.proto new file mode 100644 index 000000000000..b6fd6a85f567 --- /dev/null +++ b/api/envoy/admin/v3alpha/tap.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +import "envoy/service/tap/v3alpha/common.proto"; +import "validate/validate.proto"; + +package envoy.admin.v3alpha; + +option java_outer_classname = "TapProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.admin.v3alpha"; + +// The /tap admin request body that is used to configure an active tap session. +message TapRequest { + // The opaque configuration ID used to match the configuration to a loaded extension. + // A tap extension configures a similar opaque ID that is used to match. + string config_id = 1 [(validate.rules).string.min_bytes = 1]; + + // The tap configuration to load. + service.tap.v3alpha.TapConfig tap_config = 2 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/api/v3alpha/BUILD b/api/envoy/api/v3alpha/BUILD new file mode 100644 index 000000000000..0e2892e87e69 --- /dev/null +++ b/api/envoy/api/v3alpha/BUILD @@ -0,0 +1,171 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +# Friends of core API packages - filters, services, service configs. +# Package //envoy/api/v3alpha contains xDS and discovery definitions that should +# be in //envoy/service/discovery, but remain here for backwards compatibility. +package_group( + name = "friends", + packages = [ + "//envoy/admin/...", + "//envoy/api/v3alpha", + "//envoy/config/...", + "//envoy/data/...", + "//envoy/service/...", + ], +) + +api_proto_library_internal( + name = "discovery", + srcs = ["discovery.proto"], + visibility = [":friends"], + deps = ["//envoy/api/v3alpha/core:base"], +) + +api_go_proto_library( + name = "discovery", + proto = ":discovery", + deps = ["//envoy/api/v3alpha/core:base_go_proto"], +) + +api_proto_library_internal( + name = "eds", + srcs = ["eds.proto"], + has_services = 1, + visibility = [":friends"], + deps = [ + ":discovery", + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:health_check", + "//envoy/api/v3alpha/endpoint", + "//envoy/type:percent", + ], +) + +api_go_grpc_library( + name = "eds", + proto = ":eds", + deps = [ + ":discovery_go_proto", + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:health_check_go_proto", + "//envoy/api/v3alpha/endpoint:endpoint_go_proto", + "//envoy/type:percent_go_proto", + ], +) + +api_proto_library_internal( + name = "cds", + srcs = ["cds.proto"], + has_services = 1, + visibility = [":friends"], + deps = [ + ":discovery", + ":eds", + "//envoy/api/v3alpha/auth:cert", + "//envoy/api/v3alpha/cluster:circuit_breaker", + "//envoy/api/v3alpha/cluster:filter", + "//envoy/api/v3alpha/cluster:outlier_detection", + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:config_source", + "//envoy/api/v3alpha/core:health_check", + "//envoy/api/v3alpha/core:protocol", + "//envoy/api/v3alpha/endpoint", + "//envoy/type:percent", + ], +) + +api_go_grpc_library( + name = "cds", + proto = ":cds", + deps = [ + ":discovery_go_proto", + ":eds_go_grpc", + "//envoy/api/v3alpha/auth:cert_go_proto", + "//envoy/api/v3alpha/cluster:circuit_breaker_go_proto", + "//envoy/api/v3alpha/cluster:filter_go_proto", + "//envoy/api/v3alpha/cluster:outlier_detection_go_proto", + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:config_source_go_proto", + "//envoy/api/v3alpha/core:health_check_go_proto", + "//envoy/api/v3alpha/core:protocol_go_proto", + "//envoy/api/v3alpha/endpoint:endpoint_go_proto", + "//envoy/type:percent_go_proto", + ], +) + +api_proto_library_internal( + name = "lds", + srcs = ["lds.proto"], + has_services = 1, + visibility = [":friends"], + deps = [ + ":discovery", + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/listener", + "//envoy/api/v3alpha/listener:udp_listener_config", + ], +) + +api_go_grpc_library( + name = "lds", + proto = ":lds", + deps = [ + ":discovery_go_proto", + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/listener:listener_go_proto", + "//envoy/api/v3alpha/listener:udp_listener_config_go_proto", + ], +) + +api_proto_library_internal( + name = "rds", + srcs = ["rds.proto"], + has_services = 1, + visibility = [":friends"], + deps = [ + ":discovery", + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:config_source", + "//envoy/api/v3alpha/route", + ], +) + +api_go_grpc_library( + name = "rds", + proto = ":rds", + deps = [ + ":discovery_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:config_source_go_proto", + "//envoy/api/v3alpha/route:route_go_proto", + ], +) + +api_proto_library_internal( + name = "srds", + srcs = ["srds.proto"], + has_services = 1, + visibility = [":friends"], + deps = [ + ":discovery", + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/route", + ], +) + +api_go_grpc_library( + name = "srds", + proto = ":srds", + deps = [ + ":discovery_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + ], +) diff --git a/api/envoy/api/v3alpha/README.md b/api/envoy/api/v3alpha/README.md new file mode 100644 index 000000000000..984be690a103 --- /dev/null +++ b/api/envoy/api/v3alpha/README.md @@ -0,0 +1,9 @@ +Protocol buffer definitions for xDS and top-level resource API messages. + +Package group `//envoy/api/v2:friends` enumerates all consumers of the shared +API messages. That includes package envoy.api.v2 itself, which contains several +xDS definitions. Default visibility for all shared definitions should be set to +`//envoy/api/v2:friends`. + +Additionally, packages envoy.api.v2.core and envoy.api.v2.auth are also +consumed throughout the subpackages of `//envoy/api/v2`. diff --git a/api/envoy/api/v3alpha/auth/BUILD b/api/envoy/api/v3alpha/auth/BUILD new file mode 100644 index 000000000000..f206a35f97f2 --- /dev/null +++ b/api/envoy/api/v3alpha/auth/BUILD @@ -0,0 +1,35 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +package_group( + name = "friends", + includes = [ + "//envoy/api/v3alpha:friends", + ], + packages = [ + "//envoy/api/v3alpha/cluster", + "//envoy/api/v3alpha/endpoint", + "//envoy/api/v3alpha/listener", + "//envoy/api/v3alpha/route", + ], +) + +api_proto_library_internal( + name = "cert", + srcs = ["cert.proto"], + visibility = [":friends"], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:config_source", + ], +) + +api_go_proto_library( + name = "cert", + proto = ":cert", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:config_source_go_proto", + ], +) diff --git a/api/envoy/api/v3alpha/auth/cert.proto b/api/envoy/api/v3alpha/auth/cert.proto new file mode 100644 index 000000000000..925453074ac1 --- /dev/null +++ b/api/envoy/api/v3alpha/auth/cert.proto @@ -0,0 +1,407 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.auth; + +option java_outer_classname = "CertProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.auth"; +option go_package = "auth"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/core/config_source.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Common TLS configuration] + +message TlsParameters { + enum TlsProtocol { + // Envoy will choose the optimal TLS version. + TLS_AUTO = 0; + + // TLS 1.0 + TLSv1_0 = 1; + + // TLS 1.1 + TLSv1_1 = 2; + + // TLS 1.2 + TLSv1_2 = 3; + + // TLS 1.3 + TLSv1_3 = 4; + } + + // Minimum TLS protocol version. By default, it's ``TLSv1_0``. + TlsProtocol tls_minimum_protocol_version = 1 [(validate.rules).enum.defined_only = true]; + + // Maximum TLS protocol version. By default, it's ``TLSv1_3`` for servers in non-FIPS builds, and + // ``TLSv1_2`` for clients and for servers using :ref:`BoringSSL FIPS `. + TlsProtocol tls_maximum_protocol_version = 2 [(validate.rules).enum.defined_only = true]; + + // If specified, the TLS listener will only support the specified `cipher list + // `_ + // when negotiating TLS 1.0-1.2 (this setting has no effect when negotiating TLS 1.3). If not + // specified, the default list will be used. + // + // In non-FIPS builds, the default cipher list is: + // + // .. code-block:: none + // + // [ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305] + // [ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305] + // ECDHE-ECDSA-AES128-SHA + // ECDHE-RSA-AES128-SHA + // AES128-GCM-SHA256 + // AES128-SHA + // ECDHE-ECDSA-AES256-GCM-SHA384 + // ECDHE-RSA-AES256-GCM-SHA384 + // ECDHE-ECDSA-AES256-SHA + // ECDHE-RSA-AES256-SHA + // AES256-GCM-SHA384 + // AES256-SHA + // + // In builds using :ref:`BoringSSL FIPS `, the default cipher list is: + // + // .. code-block:: none + // + // ECDHE-ECDSA-AES128-GCM-SHA256 + // ECDHE-RSA-AES128-GCM-SHA256 + // ECDHE-ECDSA-AES128-SHA + // ECDHE-RSA-AES128-SHA + // AES128-GCM-SHA256 + // AES128-SHA + // ECDHE-ECDSA-AES256-GCM-SHA384 + // ECDHE-RSA-AES256-GCM-SHA384 + // ECDHE-ECDSA-AES256-SHA + // ECDHE-RSA-AES256-SHA + // AES256-GCM-SHA384 + // AES256-SHA + repeated string cipher_suites = 3; + + // If specified, the TLS connection will only support the specified ECDH + // curves. If not specified, the default curves will be used. + // + // In non-FIPS builds, the default curves are: + // + // .. code-block:: none + // + // X25519 + // P-256 + // + // In builds using :ref:`BoringSSL FIPS `, the default curve is: + // + // .. code-block:: none + // + // P-256 + repeated string ecdh_curves = 4; +} + +// BoringSSL private key method configuration. The private key methods are used for external +// (potentially asynchronous) signing and decryption operations. Some use cases for private key +// methods would be TPM support and TLS acceleration. +message PrivateKeyProvider { + // Private key method provider name. The name must match a + // supported private key method provider type. + string provider_name = 1 [(validate.rules).string.min_bytes = 1]; + + // Private key method provider specific configuration. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} + +message TlsCertificate { + // The TLS certificate chain. + core.DataSource certificate_chain = 1; + + // The TLS private key. + core.DataSource private_key = 2; + + // BoringSSL private key method provider. This is an alternative to :ref:`private_key + // ` field. This can't be + // marked as ``oneof`` due to API compatibility reasons. Setting both :ref:`private_key + // ` and + // :ref:`private_key_provider + // ` fields will result in an + // error. + PrivateKeyProvider private_key_provider = 6; + + // The password to decrypt the TLS private key. If this field is not set, it is assumed that the + // TLS private key is not password encrypted. + core.DataSource password = 3; + + // [#not-implemented-hide:] + core.DataSource ocsp_staple = 4; + + // [#not-implemented-hide:] + repeated core.DataSource signed_certificate_timestamp = 5; +} + +message TlsSessionTicketKeys { + // Keys for encrypting and decrypting TLS session tickets. The + // first key in the array contains the key to encrypt all new sessions created by this context. + // All keys are candidates for decrypting received tickets. This allows for easy rotation of keys + // by, for example, putting the new key first, and the previous key second. + // + // If :ref:`session_ticket_keys ` + // is not specified, the TLS library will still support resuming sessions via tickets, but it will + // use an internally-generated and managed key, so sessions cannot be resumed across hot restarts + // or on different hosts. + // + // Each key must contain exactly 80 bytes of cryptographically-secure random data. For + // example, the output of ``openssl rand 80``. + // + // .. attention:: + // + // Using this feature has serious security considerations and risks. Improper handling of keys + // may result in loss of secrecy in connections, even if ciphers supporting perfect forward + // secrecy are used. See https://www.imperialviolet.org/2013/06/27/botchingpfs.html for some + // discussion. To minimize the risk, you must: + // + // * Keep the session ticket keys at least as secure as your TLS certificate private keys + // * Rotate session ticket keys at least daily, and preferably hourly + // * Always generate keys using a cryptographically-secure random data source + repeated core.DataSource keys = 1 [(validate.rules).repeated .min_items = 1]; +} + +message CertificateValidationContext { + // TLS certificate data containing certificate authority certificates to use in verifying + // a presented peer certificate (e.g. server certificate for clusters or client certificate + // for listeners). If not specified and a peer certificate is presented it will not be + // verified. By default, a client certificate is optional, unless one of the additional + // options (:ref:`require_client_certificate + // `, + // :ref:`verify_certificate_spki + // `, + // :ref:`verify_certificate_hash + // `, or + // :ref:`verify_subject_alt_name + // `) is also + // specified. + // + // It can optionally contain certificate revocation lists, in which case Envoy will verify + // that the presented peer certificate has not been revoked by one of the included CRLs. + // + // See :ref:`the TLS overview ` for a list of common + // system CA locations. + core.DataSource trusted_ca = 1; + + // An optional list of base64-encoded SHA-256 hashes. If specified, Envoy will verify that the + // SHA-256 of the DER-encoded Subject Public Key Information (SPKI) of the presented certificate + // matches one of the specified values. + // + // A base64-encoded SHA-256 of the Subject Public Key Information (SPKI) of the certificate + // can be generated with the following command: + // + // .. code-block:: bash + // + // $ openssl x509 -in path/to/client.crt -noout -pubkey \ + // | openssl pkey -pubin -outform DER \ + // | openssl dgst -sha256 -binary \ + // | openssl enc -base64 + // NvqYIYSbgK2vCJpQhObf77vv+bQWtc5ek5RIOwPiC9A= + // + // This is the format used in HTTP Public Key Pinning. + // + // When both: + // :ref:`verify_certificate_hash + // ` and + // :ref:`verify_certificate_spki + // ` are specified, + // a hash matching value from either of the lists will result in the certificate being accepted. + // + // .. attention:: + // + // This option is preferred over :ref:`verify_certificate_hash + // `, + // because SPKI is tied to a private key, so it doesn't change when the certificate + // is renewed using the same private key. + repeated string verify_certificate_spki = 3 + [(validate.rules).repeated .items.string = {min_bytes: 44, max_bytes: 44}]; + + // An optional list of hex-encoded SHA-256 hashes. If specified, Envoy will verify that + // the SHA-256 of the DER-encoded presented certificate matches one of the specified values. + // + // A hex-encoded SHA-256 of the certificate can be generated with the following command: + // + // .. code-block:: bash + // + // $ openssl x509 -in path/to/client.crt -outform DER | openssl dgst -sha256 | cut -d" " -f2 + // df6ff72fe9116521268f6f2dd4966f51df479883fe7037b39f75916ac3049d1a + // + // A long hex-encoded and colon-separated SHA-256 (a.k.a. "fingerprint") of the certificate + // can be generated with the following command: + // + // .. code-block:: bash + // + // $ openssl x509 -in path/to/client.crt -noout -fingerprint -sha256 | cut -d"=" -f2 + // DF:6F:F7:2F:E9:11:65:21:26:8F:6F:2D:D4:96:6F:51:DF:47:98:83:FE:70:37:B3:9F:75:91:6A:C3:04:9D:1A + // + // Both of those formats are acceptable. + // + // When both: + // :ref:`verify_certificate_hash + // ` and + // :ref:`verify_certificate_spki + // ` are specified, + // a hash matching value from either of the lists will result in the certificate being accepted. + repeated string verify_certificate_hash = 2 + [(validate.rules).repeated .items.string = {min_bytes: 64, max_bytes: 95}]; + + // An optional list of Subject Alternative Names. If specified, Envoy will verify that the + // Subject Alternative Name of the presented certificate matches one of the specified values. + // + // .. attention:: + // + // Subject Alternative Names are easily spoofable and verifying only them is insecure, + // therefore this option must be used together with :ref:`trusted_ca + // `. + repeated string verify_subject_alt_name = 4; + + // [#not-implemented-hide:] Must present a signed time-stamped OCSP response. + google.protobuf.BoolValue require_ocsp_staple = 5; + + // [#not-implemented-hide:] Must present signed certificate time-stamp. + google.protobuf.BoolValue require_signed_certificate_timestamp = 6; + + // An optional `certificate revocation list + // `_ + // (in PEM format). If specified, Envoy will verify that the presented peer + // certificate has not been revoked by this CRL. If this DataSource contains + // multiple CRLs, all of them will be used. + core.DataSource crl = 7; + + // If specified, Envoy will not reject expired certificates. + bool allow_expired_certificate = 8; +} + +// TLS context shared by both client and server TLS contexts. +message CommonTlsContext { + // TLS protocol versions, cipher suites etc. + TlsParameters tls_params = 1; + + // :ref:`Multiple TLS certificates ` can be associated with the + // same context to allow both RSA and ECDSA certificates. + // + // Only a single TLS certificate is supported in client contexts. In server contexts, the first + // RSA certificate is used for clients that only support RSA and the first ECDSA certificate is + // used for clients that support ECDSA. + repeated TlsCertificate tls_certificates = 2; + + // Configs for fetching TLS certificates via SDS API. + repeated SdsSecretConfig tls_certificate_sds_secret_configs = 6 + [(validate.rules).repeated .max_items = 1]; + + message CombinedCertificateValidationContext { + // How to validate peer certificates. + CertificateValidationContext default_validation_context = 1 + [(validate.rules).message.required = true]; + + // Config for fetching validation context via SDS API. + SdsSecretConfig validation_context_sds_secret_config = 2 + [(validate.rules).message.required = true]; + }; + + oneof validation_context_type { + // How to validate peer certificates. + CertificateValidationContext validation_context = 3; + + // Config for fetching validation context via SDS API. + SdsSecretConfig validation_context_sds_secret_config = 7; + + // Combined certificate validation context holds a default CertificateValidationContext + // and SDS config. When SDS server returns dynamic CertificateValidationContext, both dynamic + // and default CertificateValidationContext are merged into a new CertificateValidationContext + // for validation. This merge is done by Message::MergeFrom(), so dynamic + // CertificateValidationContext overwrites singular fields in default + // CertificateValidationContext, and concatenates repeated fields to default + // CertificateValidationContext, and logical OR is applied to boolean fields. + CombinedCertificateValidationContext combined_validation_context = 8; + } + + // Supplies the list of ALPN protocols that the listener should expose. In + // practice this is likely to be set to one of two values (see the + // :ref:`codec_type + // ` + // parameter in the HTTP connection manager for more information): + // + // * "h2,http/1.1" If the listener is going to support both HTTP/2 and HTTP/1.1. + // * "http/1.1" If the listener is only going to support HTTP/1.1. + // + // There is no default for this parameter. If empty, Envoy will not expose ALPN. + repeated string alpn_protocols = 4; + + reserved 5; +} + +message UpstreamTlsContext { + // Common TLS context settings. + CommonTlsContext common_tls_context = 1; + + // SNI string to use when creating TLS backend connections. + string sni = 2 [(validate.rules).string.max_bytes = 255]; + + // If true, server-initiated TLS renegotiation will be allowed. + // + // .. attention:: + // + // TLS renegotiation is considered insecure and shouldn't be used unless absolutely necessary. + bool allow_renegotiation = 3; + + // Maximum number of session keys (Pre-Shared Keys for TLSv1.3+, Session IDs and Session Tickets + // for TLSv1.2 and older) to store for the purpose of session resumption. + // + // Defaults to 1, setting this to 0 disables session resumption. + google.protobuf.UInt32Value max_session_keys = 4; +} + +message DownstreamTlsContext { + // Common TLS context settings. + CommonTlsContext common_tls_context = 1; + + // If specified, Envoy will reject connections without a valid client + // certificate. + google.protobuf.BoolValue require_client_certificate = 2; + + // If specified, Envoy will reject connections without a valid and matching SNI. + // [#not-implemented-hide:] + google.protobuf.BoolValue require_sni = 3; + + oneof session_ticket_keys_type { + // TLS session ticket key settings. + TlsSessionTicketKeys session_ticket_keys = 4; + + // [#not-implemented-hide:] + SdsSecretConfig session_ticket_keys_sds_secret_config = 5; + } +} + +// [#proto-status: experimental] +message SdsSecretConfig { + // Name (FQDN, UUID, SPKI, SHA256, etc.) by which the secret can be uniquely referred to. + // When both name and config are specified, then secret can be fetched and/or reloaded via SDS. + // When only name is specified, then secret will be loaded from static resources [V2-API-DIFF]. + string name = 1; + core.ConfigSource sds_config = 2; +} + +// [#proto-status: experimental] +message Secret { + // Name (FQDN, UUID, SPKI, SHA256, etc.) by which the secret can be uniquely referred to. + string name = 1; + oneof type { + TlsCertificate tls_certificate = 2; + TlsSessionTicketKeys session_ticket_keys = 3; + CertificateValidationContext validation_context = 4; + } +} diff --git a/api/envoy/api/v3alpha/cds.proto b/api/envoy/api/v3alpha/cds.proto new file mode 100644 index 000000000000..50b7adaf996e --- /dev/null +++ b/api/envoy/api/v3alpha/cds.proto @@ -0,0 +1,660 @@ +syntax = "proto3"; + +package envoy.api.v3alpha; + +option java_outer_classname = "CdsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha"; + +option java_generic_services = true; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/auth/cert.proto"; +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/core/config_source.proto"; +import "envoy/api/v3alpha/discovery.proto"; +import "envoy/api/v3alpha/core/health_check.proto"; +import "envoy/api/v3alpha/core/protocol.proto"; +import "envoy/api/v3alpha/cluster/circuit_breaker.proto"; +import "envoy/api/v3alpha/cluster/filter.proto"; +import "envoy/api/v3alpha/cluster/outlier_detection.proto"; +import "envoy/api/v3alpha/eds.proto"; +import "envoy/type/percent.proto"; + +import "google/api/annotations.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; +option (gogoproto.stable_marshaler_all) = true; + +// Return list of all clusters this proxy will load balance to. +service ClusterDiscoveryService { + rpc StreamClusters(stream DiscoveryRequest) returns (stream DiscoveryResponse) { + } + + rpc DeltaClusters(stream DeltaDiscoveryRequest) returns (stream DeltaDiscoveryResponse) { + } + + rpc FetchClusters(DiscoveryRequest) returns (DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:clusters" + body: "*" + }; + } +} + +// [#protodoc-title: Clusters] + +// Configuration for a single upstream cluster. +// [#comment:next free field: 41] +message Cluster { + // Supplies the name of the cluster which must be unique across all clusters. + // The cluster name is used when emitting + // :ref:`statistics ` if :ref:`alt_stat_name + // ` is not provided. + // Any ``:`` in the cluster name will be converted to ``_`` when emitting statistics. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // An optional alternative to the cluster name to be used while emitting stats. + // Any ``:`` in the name will be converted to ``_`` when emitting statistics. This should not be + // confused with :ref:`Router Filter Header + // `. + string alt_stat_name = 28; + + // Refer to :ref:`service discovery type ` + // for an explanation on each type. + enum DiscoveryType { + // Refer to the :ref:`static discovery type` + // for an explanation. + STATIC = 0; + + // Refer to the :ref:`strict DNS discovery + // type` + // for an explanation. + STRICT_DNS = 1; + + // Refer to the :ref:`logical DNS discovery + // type` + // for an explanation. + LOGICAL_DNS = 2; + + // Refer to the :ref:`service discovery type` + // for an explanation. + EDS = 3; + + // Refer to the :ref:`original destination discovery + // type` + // for an explanation. + ORIGINAL_DST = 4; + } + + // Extended cluster type. + message CustomClusterType { + // The type of the cluster to instantiate. The name must match a supported cluster type. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Cluster specific configuration which depends on the cluster being instantiated. + // See the supported cluster for further documentation. + google.protobuf.Any typed_config = 2; + } + + oneof cluster_discovery_type { + // The :ref:`service discovery type ` + // to use for resolving the cluster. + DiscoveryType type = 2 [(validate.rules).enum.defined_only = true]; + + // The custom cluster type. + CustomClusterType cluster_type = 38; + } + + // Only valid when discovery type is EDS. + message EdsClusterConfig { + // Configuration for the source of EDS updates for this Cluster. + core.ConfigSource eds_config = 1; + + // Optional alternative to cluster name to present to EDS. This does not + // have the same restrictions as cluster name, i.e. it may be arbitrary + // length. + string service_name = 2; + } + // Configuration to use for EDS updates for the Cluster. + EdsClusterConfig eds_cluster_config = 3; + + // The timeout for new network connections to hosts in the cluster. + google.protobuf.Duration connect_timeout = 4 + [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + + // Soft limit on size of the cluster’s connections read and write buffers. If + // unspecified, an implementation defined default is applied (1MiB). + google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5; + + // Refer to :ref:`load balancer type ` architecture + // overview section for information on each type. + enum LbPolicy { + // Refer to the :ref:`round robin load balancing + // policy` + // for an explanation. + ROUND_ROBIN = 0; + + // Refer to the :ref:`least request load balancing + // policy` + // for an explanation. + LEAST_REQUEST = 1; + + // Refer to the :ref:`ring hash load balancing + // policy` + // for an explanation. + RING_HASH = 2; + + // Refer to the :ref:`random load balancing + // policy` + // for an explanation. + RANDOM = 3; + + // Refer to the :ref:`original destination load balancing + // policy` + // for an explanation. + // + // .. attention:: + // + // **This load balancing policy is deprecated**. Use CLUSTER_PROVIDED instead. + // + ORIGINAL_DST_LB = 4 [deprecated = true]; + + // Refer to the :ref:`Maglev load balancing policy` + // for an explanation. + MAGLEV = 5; + + // This load balancer type must be specified if the configured cluster provides a cluster + // specific load balancer. Consult the configured cluster's documentation for whether to set + // this option or not. + CLUSTER_PROVIDED = 6; + } + // The :ref:`load balancer type ` to use + // when picking a host in the cluster. + LbPolicy lb_policy = 6 [(validate.rules).enum.defined_only = true]; + + // If the service discovery type is + // :ref:`STATIC`, + // :ref:`STRICT_DNS` + // or :ref:`LOGICAL_DNS`, + // then hosts is required. + // + // .. attention:: + // + // **This field is deprecated**. Set the + // :ref:`load_assignment` field instead. + // + repeated core.Address hosts = 7; + + // Setting this is required for specifying members of + // :ref:`STATIC`, + // :ref:`STRICT_DNS` + // or :ref:`LOGICAL_DNS` clusters. + // This field supersedes :ref:`hosts` field. + // [#comment:TODO(dio): Deprecate the hosts field and add it to :ref:`deprecated log` + // once load_assignment is implemented.] + // + // .. attention:: + // + // Setting this allows non-EDS cluster types to contain embedded EDS equivalent + // :ref:`endpoint assignments`. + // Setting this overrides :ref:`hosts` values. + // + ClusterLoadAssignment load_assignment = 33; + + // Optional :ref:`active health checking ` + // configuration for the cluster. If no + // configuration is specified no health checking will be done and all cluster + // members will be considered healthy at all times. + repeated core.HealthCheck health_checks = 8; + + // Optional maximum requests for a single upstream connection. This parameter + // is respected by both the HTTP/1.1 and HTTP/2 connection pool + // implementations. If not specified, there is no limit. Setting this + // parameter to 1 will effectively disable keep alive. + google.protobuf.UInt32Value max_requests_per_connection = 9; + + // Optional :ref:`circuit breaking ` for the cluster. + cluster.CircuitBreakers circuit_breakers = 10; + + // The TLS configuration for connections to the upstream cluster. If no TLS + // configuration is specified, TLS will not be used for new connections. + // + // .. attention:: + // + // Server certificate verification is not enabled by default. Configure + // :ref:`trusted_ca` to enable + // verification. + auth.UpstreamTlsContext tls_context = 11; + + reserved 12; + + // Additional options when handling HTTP requests. These options will be applicable to both + // HTTP1 and HTTP2 requests. + core.HttpProtocolOptions common_http_protocol_options = 29; + + // Additional options when handling HTTP1 requests. + core.Http1ProtocolOptions http_protocol_options = 13; + + // Even if default HTTP2 protocol options are desired, this field must be + // set so that Envoy will assume that the upstream supports HTTP/2 when + // making new HTTP connection pool connections. Currently, Envoy only + // supports prior knowledge for upstream connections. Even if TLS is used + // with ALPN, `http2_protocol_options` must be specified. As an aside this allows HTTP/2 + // connections to happen over plain text. + core.Http2ProtocolOptions http2_protocol_options = 14; + + // The extension_protocol_options field is used to provide extension-specific protocol options + // for upstream connections. The key should match the extension filter name, such as + // "envoy.filters.network.thrift_proxy". See the extension's documentation for details on + // specific options. + map extension_protocol_options = 35; + + // The extension_protocol_options field is used to provide extension-specific protocol options + // for upstream connections. The key should match the extension filter name, such as + // "envoy.filters.network.thrift_proxy". See the extension's documentation for details on + // specific options. + map typed_extension_protocol_options = 36; + + reserved 15; + + // If the DNS refresh rate is specified and the cluster type is either + // :ref:`STRICT_DNS`, + // or :ref:`LOGICAL_DNS`, + // this value is used as the cluster’s DNS refresh + // rate. If this setting is not specified, the value defaults to 5000ms. For + // cluster types other than + // :ref:`STRICT_DNS` + // and :ref:`LOGICAL_DNS` + // this setting is ignored. + google.protobuf.Duration dns_refresh_rate = 16 + [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + + // Optional configuration for setting cluster's DNS refresh rate. If the value is set to true, + // cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS + // resolution. + bool respect_dns_ttl = 39; + + // When V4_ONLY is selected, the DNS resolver will only perform a lookup for + // addresses in the IPv4 family. If V6_ONLY is selected, the DNS resolver will + // only perform a lookup for addresses in the IPv6 family. If AUTO is + // specified, the DNS resolver will first perform a lookup for addresses in + // the IPv6 family and fallback to a lookup for addresses in the IPv4 family. + // For cluster types other than + // :ref:`STRICT_DNS` and + // :ref:`LOGICAL_DNS`, + // this setting is + // ignored. + enum DnsLookupFamily { + AUTO = 0; + V4_ONLY = 1; + V6_ONLY = 2; + } + + // The DNS IP address resolution policy. If this setting is not specified, the + // value defaults to + // :ref:`AUTO`. + DnsLookupFamily dns_lookup_family = 17 [(validate.rules).enum.defined_only = true]; + + // If DNS resolvers are specified and the cluster type is either + // :ref:`STRICT_DNS`, + // or :ref:`LOGICAL_DNS`, + // this value is used to specify the cluster’s dns resolvers. + // If this setting is not specified, the value defaults to the default + // resolver, which uses /etc/resolv.conf for configuration. For cluster types + // other than + // :ref:`STRICT_DNS` + // and :ref:`LOGICAL_DNS` + // this setting is ignored. + repeated core.Address dns_resolvers = 18; + + // If specified, outlier detection will be enabled for this upstream cluster. + // Each of the configuration values can be overridden via + // :ref:`runtime values `. + cluster.OutlierDetection outlier_detection = 19; + + // The interval for removing stale hosts from a cluster type + // :ref:`ORIGINAL_DST`. + // Hosts are considered stale if they have not been used + // as upstream destinations during this interval. New hosts are added + // to original destination clusters on demand as new connections are + // redirected to Envoy, causing the number of hosts in the cluster to + // grow over time. Hosts that are not stale (they are actively used as + // destinations) are kept in the cluster, which allows connections to + // them remain open, saving the latency that would otherwise be spent + // on opening new connections. If this setting is not specified, the + // value defaults to 5000ms. For cluster types other than + // :ref:`ORIGINAL_DST` + // this setting is ignored. + google.protobuf.Duration cleanup_interval = 20 + [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + + // Optional configuration used to bind newly established upstream connections. + // This overrides any bind_config specified in the bootstrap proto. + // If the address and port are empty, no bind will be performed. + core.BindConfig upstream_bind_config = 21; + + // Optionally divide the endpoints in this cluster into subsets defined by + // endpoint metadata and selected by route and weighted cluster metadata. + message LbSubsetConfig { + + // If NO_FALLBACK is selected, a result + // equivalent to no healthy hosts is reported. If ANY_ENDPOINT is selected, + // any cluster endpoint may be returned (subject to policy, health checks, + // etc). If DEFAULT_SUBSET is selected, load balancing is performed over the + // endpoints matching the values from the default_subset field. + enum LbSubsetFallbackPolicy { + NO_FALLBACK = 0; + ANY_ENDPOINT = 1; + DEFAULT_SUBSET = 2; + } + + // The behavior used when no endpoint subset matches the selected route's + // metadata. The value defaults to + // :ref:`NO_FALLBACK`. + LbSubsetFallbackPolicy fallback_policy = 1 [(validate.rules).enum.defined_only = true]; + + // Specifies the default subset of endpoints used during fallback if + // fallback_policy is + // :ref:`DEFAULT_SUBSET`. + // Each field in default_subset is + // compared to the matching LbEndpoint.Metadata under the *envoy.lb* + // namespace. It is valid for no hosts to match, in which case the behavior + // is the same as a fallback_policy of + // :ref:`NO_FALLBACK`. + google.protobuf.Struct default_subset = 2; + + // Specifications for subsets. + message LbSubsetSelector { + // List of keys to match with the weighted cluster metadata. + repeated string keys = 1; + // The behavior used when no endpoint subset matches the selected route's + // metadata. + LbSubsetSelectorFallbackPolicy fallback_policy = 2 + [(validate.rules).enum.defined_only = true]; + + // Allows to override top level fallback policy per selector. + enum LbSubsetSelectorFallbackPolicy { + // If NOT_DEFINED top level config fallback policy is used instead. + NOT_DEFINED = 0; + // If NO_FALLBACK is selected, a result equivalent to no healthy hosts is reported. + NO_FALLBACK = 1; + // If ANY_ENDPOINT is selected, any cluster endpoint may be returned + // (subject to policy, health checks, etc). + ANY_ENDPOINT = 2; + // If DEFAULT_SUBSET is selected, load balancing is performed over the + // endpoints matching the values from the default_subset field. + DEFAULT_SUBSET = 3; + } + } + + // For each entry, LbEndpoint.Metadata's + // *envoy.lb* namespace is traversed and a subset is created for each unique + // combination of key and value. For example: + // + // .. code-block:: json + // + // { "subset_selectors": [ + // { "keys": [ "version" ] }, + // { "keys": [ "stage", "hardware_type" ] } + // ]} + // + // A subset is matched when the metadata from the selected route and + // weighted cluster contains the same keys and values as the subset's + // metadata. The same host may appear in multiple subsets. + repeated LbSubsetSelector subset_selectors = 3; + + // If true, routing to subsets will take into account the localities and locality weights of the + // endpoints when making the routing decision. + // + // There are some potential pitfalls associated with enabling this feature, as the resulting + // traffic split after applying both a subset match and locality weights might be undesirable. + // + // Consider for example a situation in which you have 50/50 split across two localities X/Y + // which have 100 hosts each without subsetting. If the subset LB results in X having only 1 + // host selected but Y having 100, then a lot more load is being dumped on the single host in X + // than originally anticipated in the load balancing assignment delivered via EDS. + bool locality_weight_aware = 4; + + // When used with locality_weight_aware, scales the weight of each locality by the ratio + // of hosts in the subset vs hosts in the original subset. This aims to even out the load + // going to an individual locality if said locality is disproportionally affected by the + // subset predicate. + bool scale_locality_weight = 5; + + // If true, when a fallback policy is configured and its corresponding subset fails to find + // a host this will cause any host to be selected instead. + // + // This is useful when using the default subset as the fallback policy, given the default + // subset might become empty. With this option enabled, if that happens the LB will attempt + // to select a host from the entire cluster. + bool panic_mode_any = 6; + + // If true, metadata specified for a metadata key will be matched against the corresponding + // endpoint metadata if the endpoint metadata matches the value exactly OR it is a list value + // and any of the elements in the list matches the criteria. + bool list_as_any = 7; + } + + // Configuration for load balancing subsetting. + LbSubsetConfig lb_subset_config = 22; + + // Specific configuration for the LeastRequest load balancing policy. + message LeastRequestLbConfig { + // The number of random healthy hosts from which the host with the fewest active requests will + // be chosen. Defaults to 2 so that we perform two-choice selection if the field is not set. + google.protobuf.UInt32Value choice_count = 1 [(validate.rules).uint32.gte = 2]; + } + + // Specific configuration for the :ref:`RingHash` + // load balancing policy. + message RingHashLbConfig { + // Minimum hash ring size. The larger the ring is (that is, the more hashes there are for each + // provided host) the better the request distribution will reflect the desired weights. Defaults + // to 1024 entries, and limited to 8M entries. See also + // :ref:`maximum_ring_size`. + google.protobuf.UInt64Value minimum_ring_size = 1 [(validate.rules).uint64.lte = 8388608]; + + reserved 2; + + // The hash function used to hash hosts onto the ketama ring. + enum HashFunction { + // Use `xxHash `_, this is the default hash function. + XX_HASH = 0; + // Use `MurmurHash2 `_, this is compatible with + // std:hash in GNU libstdc++ 3.4.20 or above. This is typically the case when compiled + // on Linux and not macOS. + MURMUR_HASH_2 = 1; + } + + // The hash function used to hash hosts onto the ketama ring. The value defaults to + // :ref:`XX_HASH`. + HashFunction hash_function = 3 [(validate.rules).enum.defined_only = true]; + + // Maximum hash ring size. Defaults to 8M entries, and limited to 8M entries, but can be lowered + // to further constrain resource use. See also + // :ref:`minimum_ring_size`. + google.protobuf.UInt64Value maximum_ring_size = 4 [(validate.rules).uint64.lte = 8388608]; + } + + // Specific configuration for the + // :ref:`Original Destination ` + // load balancing policy. + message OriginalDstLbConfig { + // When true, :ref:`x-envoy-original-dst-host + // ` can be used to override destination + // address. + // + // .. attention:: + // + // This header isn't sanitized by default, so enabling this feature allows HTTP clients to + // route traffic to arbitrary hosts and/or ports, which may have serious security + // consequences. + bool use_http_header = 1; + } + + // Optional configuration for the load balancing algorithm selected by + // LbPolicy. Currently only + // :ref:`RING_HASH` and + // :ref:`LEAST_REQUEST` + // has additional configuration options. + // Specifying ring_hash_lb_config or least_request_lb_config without setting the corresponding + // LbPolicy will generate an error at runtime. + oneof lb_config { + // Optional configuration for the Ring Hash load balancing policy. + RingHashLbConfig ring_hash_lb_config = 23; + // Optional configuration for the Original Destination load balancing policy. + OriginalDstLbConfig original_dst_lb_config = 34; + // Optional configuration for the LeastRequest load balancing policy. + LeastRequestLbConfig least_request_lb_config = 37; + } + + // Common configuration for all load balancer implementations. + message CommonLbConfig { + // Configures the :ref:`healthy panic threshold `. + // If not specified, the default is 50%. + // To disable panic mode, set to 0%. + // + // .. note:: + // The specified percent will be truncated to the nearest 1%. + envoy.type.Percent healthy_panic_threshold = 1; + // Configuration for :ref:`zone aware routing + // `. + message ZoneAwareLbConfig { + // Configures percentage of requests that will be considered for zone aware routing + // if zone aware routing is configured. If not specified, the default is 100%. + // * :ref:`runtime values `. + // * :ref:`Zone aware routing support `. + envoy.type.Percent routing_enabled = 1; + // Configures minimum upstream cluster size required for zone aware routing + // If upstream cluster size is less than specified, zone aware routing is not performed + // even if zone aware routing is configured. If not specified, the default is 6. + // * :ref:`runtime values `. + // * :ref:`Zone aware routing support `. + google.protobuf.UInt64Value min_cluster_size = 2; + } + // Configuration for :ref:`locality weighted load balancing + // ` + message LocalityWeightedLbConfig { + } + oneof locality_config_specifier { + ZoneAwareLbConfig zone_aware_lb_config = 2; + LocalityWeightedLbConfig locality_weighted_lb_config = 3; + } + // If set, all health check/weight/metadata updates that happen within this duration will be + // merged and delivered in one shot when the duration expires. The start of the duration is when + // the first update happens. This is useful for big clusters, with potentially noisy deploys + // that might trigger excessive CPU usage due to a constant stream of healthcheck state changes + // or metadata updates. The first set of updates to be seen apply immediately (e.g.: a new + // cluster). Please always keep in mind that the use of sandbox technologies may change this + // behavior. + // + // If this is not set, we default to a merge window of 1000ms. To disable it, set the merge + // window to 0. + // + // Note: merging does not apply to cluster membership changes (e.g.: adds/removes); this is + // because merging those updates isn't currently safe. See + // https://github.com/envoyproxy/envoy/pull/3941. + google.protobuf.Duration update_merge_window = 4; + + // If set to true, Envoy will not consider new hosts when computing load balancing weights until + // they have been health checked for the first time. This will have no effect unless + // active health checking is also configured. + // + // Ignoring a host means that for any load balancing calculations that adjust weights based + // on the ratio of eligible hosts and total hosts (priority spillover, locality weighting and + // panic mode) Envoy will exclude these hosts in the denominator. + // + // For example, with hosts in two priorities P0 and P1, where P0 looks like + // {healthy, unhealthy (new), unhealthy (new)} + // and where P1 looks like + // {healthy, healthy} + // all traffic will still hit P0, as 1 / (3 - 2) = 1. + // + // Enabling this will allow scaling up the number of hosts for a given cluster without entering + // panic mode or triggering priority spillover, assuming the hosts pass the first health check. + // + // If panic mode is triggered, new hosts are still eligible for traffic; they simply do not + // contribute to the calculation when deciding whether panic mode is enabled or not. + bool ignore_new_hosts_until_first_hc = 5; + + // If set to `true`, the cluster manager will drain all existing + // connections to upstream hosts whenever hosts are added or removed from the cluster. + bool close_connections_on_host_set_change = 6; + } + + // Common configuration for all load balancer implementations. + CommonLbConfig common_lb_config = 27; + + // Optional custom transport socket implementation to use for upstream connections. + core.TransportSocket transport_socket = 24; + + // The Metadata field can be used to provide additional information about the + // cluster. It can be used for stats, logging, and varying filter behavior. + // Fields should use reverse DNS notation to denote which entity within Envoy + // will need the information. For instance, if the metadata is intended for + // the Router filter, the filter name should be specified as *envoy.router*. + core.Metadata metadata = 25; + + enum ClusterProtocolSelection { + // Cluster can only operate on one of the possible upstream protocols (HTTP1.1, HTTP2). + // If :ref:`http2_protocol_options ` are + // present, HTTP2 will be used, otherwise HTTP1.1 will be used. + USE_CONFIGURED_PROTOCOL = 0; + // Use HTTP1.1 or HTTP2, depending on which one is used on the downstream connection. + USE_DOWNSTREAM_PROTOCOL = 1; + } + + // Determines how Envoy selects the protocol used to speak to upstream hosts. + ClusterProtocolSelection protocol_selection = 26; + + // Optional options for upstream connections. + envoy.api.v3alpha.UpstreamConnectionOptions upstream_connection_options = 30; + + // If an upstream host becomes unhealthy (as determined by the configured health checks + // or outlier detection), immediately close all connections to the failed host. + // + // .. note:: + // + // This is currently only supported for connections created by tcp_proxy. + // + // .. note:: + // + // The current implementation of this feature closes all connections immediately when + // the unhealthy status is detected. If there are a large number of connections open + // to an upstream host that becomes unhealthy, Envoy may spend a substantial amount of + // time exclusively closing these connections, and not processing any other traffic. + bool close_connections_on_host_health_failure = 31; + + // If this cluster uses EDS or STRICT_DNS to configure its hosts, immediately drain + // connections from any hosts that are removed from service discovery. + // + // This only affects behavior for hosts that are being actively health checked. + // If this flag is not set to true, Envoy will wait until the hosts fail active health + // checking before removing it from the cluster. + bool drain_connections_on_host_removal = 32; + + // An optional list of network filters that make up the filter chain for + // outgoing connections made by the cluster. Order matters as the filters are + // processed sequentially as connection events happen. + repeated cluster.Filter filters = 40; +} + +// An extensible structure containing the address Envoy should bind to when +// establishing upstream connections. +message UpstreamBindConfig { + // The address Envoy should bind to when establishing upstream connections. + core.Address source_address = 1; +} + +message UpstreamConnectionOptions { + // If set then set SO_KEEPALIVE on the socket to enable TCP Keepalives. + core.TcpKeepalive tcp_keepalive = 1; +} diff --git a/api/envoy/api/v3alpha/cluster/BUILD b/api/envoy/api/v3alpha/cluster/BUILD new file mode 100644 index 000000000000..942701221a37 --- /dev/null +++ b/api/envoy/api/v3alpha/cluster/BUILD @@ -0,0 +1,48 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "circuit_breaker", + srcs = ["circuit_breaker.proto"], + visibility = [ + "//envoy/api/v3alpha:__pkg__", + ], + deps = [ + "//envoy/api/v3alpha/core:base", + ], +) + +api_go_proto_library( + name = "circuit_breaker", + proto = ":circuit_breaker", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + ], +) + +api_proto_library_internal( + name = "outlier_detection", + srcs = ["outlier_detection.proto"], + visibility = [ + "//envoy/api/v3alpha:__pkg__", + ], +) + +api_go_proto_library( + name = "outlier_detection", + proto = ":outlier_detection", +) + +api_proto_library_internal( + name = "filter", + srcs = ["filter.proto"], + visibility = [ + "//envoy/api/v3alpha:__pkg__", + ], +) + +api_go_proto_library( + name = "filter", + proto = ":filter", +) diff --git a/api/envoy/api/v3alpha/cluster/circuit_breaker.proto b/api/envoy/api/v3alpha/cluster/circuit_breaker.proto new file mode 100644 index 000000000000..39f4f77c5ddd --- /dev/null +++ b/api/envoy/api/v3alpha/cluster/circuit_breaker.proto @@ -0,0 +1,70 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.cluster; + +option java_outer_classname = "CircuitBreakerProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.cluster"; +option go_package = "cluster"; +option csharp_namespace = "Envoy.Api.V2.ClusterNS"; +option ruby_package = "Envoy.Api.V2.ClusterNS"; + +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/wrappers.proto"; + +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Circuit breakers] + +// :ref:`Circuit breaking` settings can be +// specified individually for each defined priority. +message CircuitBreakers { + + // A Thresholds defines CircuitBreaker settings for a + // :ref:`RoutingPriority`. + message Thresholds { + // The :ref:`RoutingPriority` + // the specified CircuitBreaker settings apply to. + // [#comment:TODO(htuch): add (validate.rules).enum.defined_only = true once + // https://github.com/lyft/protoc-gen-validate/issues/42 is resolved.] + core.RoutingPriority priority = 1; + + // The maximum number of connections that Envoy will make to the upstream + // cluster. If not specified, the default is 1024. + google.protobuf.UInt32Value max_connections = 2; + + // The maximum number of pending requests that Envoy will allow to the + // upstream cluster. If not specified, the default is 1024. + google.protobuf.UInt32Value max_pending_requests = 3; + + // The maximum number of parallel requests that Envoy will make to the + // upstream cluster. If not specified, the default is 1024. + google.protobuf.UInt32Value max_requests = 4; + + // The maximum number of parallel retries that Envoy will allow to the + // upstream cluster. If not specified, the default is 3. + google.protobuf.UInt32Value max_retries = 5; + + // If track_remaining is true, then stats will be published that expose + // the number of resources remaining until the circuit breakers open. If + // not specified, the default is false. + bool track_remaining = 6; + + // The maximum number of connection pools per cluster that Envoy will concurrently support at + // once. If not specified, the default is unlimited. Set this for clusters which create a + // large number of connection pools. See + // :ref:`Circuit Breaking ` for + // more details. + google.protobuf.UInt32Value max_connection_pools = 7; + } + + // If multiple :ref:`Thresholds` + // are defined with the same :ref:`RoutingPriority`, + // the first one in the list is used. If no Thresholds is defined for a given + // :ref:`RoutingPriority`, the default values + // are used. + repeated Thresholds thresholds = 1; +} diff --git a/api/envoy/api/v3alpha/cluster/filter.proto b/api/envoy/api/v3alpha/cluster/filter.proto new file mode 100644 index 000000000000..ab1a3d13747b --- /dev/null +++ b/api/envoy/api/v3alpha/cluster/filter.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.cluster; + +option java_outer_classname = "FilterProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.cluster"; +option csharp_namespace = "Envoy.Api.V2.ClusterNS"; +option ruby_package = "Envoy.Api.V2.ClusterNS"; + +import "google/protobuf/any.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Upstream filters] +// +// Upstream filters apply to the connections to the upstream cluster hosts. +message Filter { + // The name of the filter to instantiate. The name must match a + // :ref:`supported filter `. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Filter specific configuration which depends on the filter being + // instantiated. See the supported filters for further documentation. + google.protobuf.Any typed_config = 2; +} diff --git a/api/envoy/api/v3alpha/cluster/outlier_detection.proto b/api/envoy/api/v3alpha/cluster/outlier_detection.proto new file mode 100644 index 000000000000..6600b566d1bd --- /dev/null +++ b/api/envoy/api/v3alpha/cluster/outlier_detection.proto @@ -0,0 +1,117 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.cluster; + +option java_outer_classname = "OutlierDetectionProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.cluster"; +option csharp_namespace = "Envoy.Api.V2.ClusterNS"; +option ruby_package = "Envoy.Api.V2.ClusterNS"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Outlier detection] + +// See the :ref:`architecture overview ` for +// more information on outlier detection. +message OutlierDetection { + // The number of consecutive 5xx responses or local origin errors that are mapped + // to 5xx error codes before a consecutive 5xx ejection + // occurs. Defaults to 5. + google.protobuf.UInt32Value consecutive_5xx = 1; + + // The time interval between ejection analysis sweeps. This can result in + // both new ejections as well as hosts being returned to service. Defaults + // to 10000ms or 10s. + google.protobuf.Duration interval = 2 [(validate.rules).duration.gt = {}]; + + // The base time that a host is ejected for. The real time is equal to the + // base time multiplied by the number of times the host has been ejected. + // Defaults to 30000ms or 30s. + google.protobuf.Duration base_ejection_time = 3 [(validate.rules).duration.gt = {}]; + + // The maximum % of an upstream cluster that can be ejected due to outlier + // detection. Defaults to 10% but will eject at least one host regardless of the value. + google.protobuf.UInt32Value max_ejection_percent = 4 [(validate.rules).uint32.lte = 100]; + + // The % chance that a host will be actually ejected when an outlier status + // is detected through consecutive 5xx. This setting can be used to disable + // ejection or to ramp it up slowly. Defaults to 100. + google.protobuf.UInt32Value enforcing_consecutive_5xx = 5 [(validate.rules).uint32.lte = 100]; + + // The % chance that a host will be actually ejected when an outlier status + // is detected through success rate statistics. This setting can be used to + // disable ejection or to ramp it up slowly. Defaults to 100. + google.protobuf.UInt32Value enforcing_success_rate = 6 [(validate.rules).uint32.lte = 100]; + + // The number of hosts in a cluster that must have enough request volume to + // detect success rate outliers. If the number of hosts is less than this + // setting, outlier detection via success rate statistics is not performed + // for any host in the cluster. Defaults to 5. + google.protobuf.UInt32Value success_rate_minimum_hosts = 7; + + // The minimum number of total requests that must be collected in one + // interval (as defined by the interval duration above) to include this host + // in success rate based outlier detection. If the volume is lower than this + // setting, outlier detection via success rate statistics is not performed + // for that host. Defaults to 100. + google.protobuf.UInt32Value success_rate_request_volume = 8; + + // This factor is used to determine the ejection threshold for success rate + // outlier ejection. The ejection threshold is the difference between the + // mean success rate, and the product of this factor and the standard + // deviation of the mean success rate: mean - (stdev * + // success_rate_stdev_factor). This factor is divided by a thousand to get a + // double. That is, if the desired factor is 1.9, the runtime value should + // be 1900. Defaults to 1900. + google.protobuf.UInt32Value success_rate_stdev_factor = 9; + + // The number of consecutive gateway failures (502, 503, 504 status codes) + // before a consecutive gateway failure ejection occurs. Defaults to 5. + google.protobuf.UInt32Value consecutive_gateway_failure = 10; + + // The % chance that a host will be actually ejected when an outlier status + // is detected through consecutive gateway failures. This setting can be + // used to disable ejection or to ramp it up slowly. Defaults to 0. + google.protobuf.UInt32Value enforcing_consecutive_gateway_failure = 11 + [(validate.rules).uint32.lte = 100]; + + // Determines whether to distinguish local origin failures from external errors. If set to true + // the following configuration parameters are taken into account: + // :ref:`consecutive_local_origin_failure`, + // :ref:`enforcing_consecutive_local_origin_failure` + // and + // :ref:`enforcing_local_origin_success_rate`. + // Defaults to false. + bool split_external_local_origin_errors = 12; + + // The number of consecutive locally originated failures before ejection + // occurs. Defaults to 5. Parameter takes effect only when + // :ref:`split_external_local_origin_errors` + // is set to true. + google.protobuf.UInt32Value consecutive_local_origin_failure = 13; + + // The % chance that a host will be actually ejected when an outlier status + // is detected through consecutive locally originated failures. This setting can be + // used to disable ejection or to ramp it up slowly. Defaults to 100. + // Parameter takes effect only when + // :ref:`split_external_local_origin_errors` + // is set to true. + google.protobuf.UInt32Value enforcing_consecutive_local_origin_failure = 14 + [(validate.rules).uint32.lte = 100]; + + // The % chance that a host will be actually ejected when an outlier status + // is detected through success rate statistics for locally originated errors. + // This setting can be used to disable ejection or to ramp it up slowly. Defaults to 100. + // Parameter takes effect only when + // :ref:`split_external_local_origin_errors` + // is set to true. + google.protobuf.UInt32Value enforcing_local_origin_success_rate = 15 + [(validate.rules).uint32.lte = 100]; +} diff --git a/api/envoy/api/v3alpha/core/BUILD b/api/envoy/api/v3alpha/core/BUILD new file mode 100644 index 000000000000..cfc6bd83ca78 --- /dev/null +++ b/api/envoy/api/v3alpha/core/BUILD @@ -0,0 +1,136 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +package_group( + name = "friends", + includes = [ + "//envoy/api/v3alpha:friends", + ], + packages = [ + "//envoy/api/v3alpha/auth", + "//envoy/api/v3alpha/cluster", + "//envoy/api/v3alpha/endpoint", + "//envoy/api/v3alpha/listener", + "//envoy/api/v3alpha/route", + ], +) + +api_proto_library_internal( + name = "address", + srcs = ["address.proto"], + visibility = [ + ":friends", + ], + deps = [":base"], +) + +api_go_proto_library( + name = "address", + proto = ":address", + deps = [":base_go_proto"], +) + +api_proto_library_internal( + name = "base", + srcs = ["base.proto"], + visibility = [ + ":friends", + ], + deps = [ + ":http_uri", + "//envoy/type:percent", + ], +) + +api_go_proto_library( + name = "base", + proto = ":base", + deps = [ + ":http_uri_go_proto", + "//envoy/type:percent_go_proto", + ], +) + +api_proto_library_internal( + name = "health_check", + srcs = ["health_check.proto"], + visibility = [ + ":friends", + ], + deps = [ + ":base", + "//envoy/type:range", + ], +) + +api_go_proto_library( + name = "health_check", + proto = ":health_check", + deps = [ + ":base_go_proto", + "//envoy/type:range_go_proto", + ], +) + +api_proto_library_internal( + name = "config_source", + srcs = ["config_source.proto"], + visibility = [ + ":friends", + ], + deps = [ + ":base", + ":grpc_service", + ], +) + +api_go_proto_library( + name = "config_source", + proto = ":config_source", + deps = [ + ":base_go_proto", + ":grpc_service_go_proto", + ], +) + +api_go_proto_library( + name = "http_uri", + proto = ":http_uri", +) + +api_proto_library_internal( + name = "http_uri", + srcs = ["http_uri.proto"], + visibility = [ + ":friends", + ], +) + +api_proto_library_internal( + name = "grpc_service", + srcs = ["grpc_service.proto"], + visibility = [ + ":friends", + ], + deps = [":base"], +) + +api_go_proto_library( + name = "grpc_service", + proto = ":grpc_service", + deps = [":base_go_proto"], +) + +api_proto_library_internal( + name = "protocol", + srcs = ["protocol.proto"], + visibility = [ + ":friends", + ], +) + +api_go_proto_library( + name = "protocol", + proto = ":protocol", +) diff --git a/api/envoy/api/v3alpha/core/address.proto b/api/envoy/api/v3alpha/core/address.proto new file mode 100644 index 000000000000..5cec89b11d8e --- /dev/null +++ b/api/envoy/api/v3alpha/core/address.proto @@ -0,0 +1,121 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.core; + +option java_outer_classname = "AddressProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; + +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Network addresses] + +message Pipe { + // Unix Domain Socket path. On Linux, paths starting with '@' will use the + // abstract namespace. The starting '@' is replaced by a null byte by Envoy. + // Paths starting with '@' will result in an error in environments other than + // Linux. + string path = 1 [(validate.rules).string.min_bytes = 1]; +} + +message SocketAddress { + enum Protocol { + option (gogoproto.goproto_enum_prefix) = false; + TCP = 0; + // [#not-implemented-hide:] + UDP = 1; + } + Protocol protocol = 1 [(validate.rules).enum.defined_only = true]; + // The address for this socket. :ref:`Listeners ` will bind + // to the address. An empty address is not allowed. Specify ``0.0.0.0`` or ``::`` + // to bind to any address. [#comment:TODO(zuercher) reinstate when implemented: + // It is possible to distinguish a Listener address via the prefix/suffix matching + // in :ref:`FilterChainMatch `.] When used + // within an upstream :ref:`BindConfig `, the address + // controls the source address of outbound connections. For :ref:`clusters + // `, the cluster type determines whether the + // address must be an IP (*STATIC* or *EDS* clusters) or a hostname resolved by DNS + // (*STRICT_DNS* or *LOGICAL_DNS* clusters). Address resolution can be customized + // via :ref:`resolver_name `. + string address = 2 [(validate.rules).string.min_bytes = 1]; + oneof port_specifier { + option (validate.required) = true; + uint32 port_value = 3 [(validate.rules).uint32.lte = 65535]; + // This is only valid if :ref:`resolver_name + // ` is specified below and the + // named resolver is capable of named port resolution. + string named_port = 4; + } + // The name of the custom resolver. This must have been registered with Envoy. If + // this is empty, a context dependent default applies. If the address is a concrete + // IP address, no resolution will occur. If address is a hostname this + // should be set for resolution other than DNS. Specifying a custom resolver with + // *STRICT_DNS* or *LOGICAL_DNS* will generate an error at runtime. + string resolver_name = 5; + + // When binding to an IPv6 address above, this enables `IPv4 compatibility + // `_. Binding to ``::`` will + // allow both IPv4 and IPv6 connections, with peer IPv4 addresses mapped into + // IPv6 space as ``::FFFF:``. + bool ipv4_compat = 6; +} + +message TcpKeepalive { + // Maximum number of keepalive probes to send without response before deciding + // the connection is dead. Default is to use the OS level configuration (unless + // overridden, Linux defaults to 9.) + google.protobuf.UInt32Value keepalive_probes = 1; + // The number of seconds a connection needs to be idle before keep-alive probes + // start being sent. Default is to use the OS level configuration (unless + // overridden, Linux defaults to 7200s (ie 2 hours.) + google.protobuf.UInt32Value keepalive_time = 2; + // The number of seconds between keep-alive probes. Default is to use the OS + // level configuration (unless overridden, Linux defaults to 75s.) + google.protobuf.UInt32Value keepalive_interval = 3; +} + +message BindConfig { + // The address to bind to when creating a socket. + SocketAddress source_address = 1 [(validate.rules).message.required = true]; + + // Whether to set the *IP_FREEBIND* option when creating the socket. When this + // flag is set to true, allows the :ref:`source_address + // ` to be an IP address + // that is not configured on the system running Envoy. When this flag is set + // to false, the option *IP_FREEBIND* is disabled on the socket. When this + // flag is not set (default), the socket is not modified, i.e. the option is + // neither enabled nor disabled. + google.protobuf.BoolValue freebind = 2; + + // Additional socket options that may not be present in Envoy source code or + // precompiled binaries. + repeated SocketOption socket_options = 3; +} + +// Addresses specify either a logical or physical address and port, which are +// used to tell Envoy where to bind/listen, connect to upstream and find +// management servers. +message Address { + oneof address { + option (validate.required) = true; + + SocketAddress socket_address = 1; + Pipe pipe = 2; + } +} + +// CidrRange specifies an IP Address and a prefix length to construct +// the subnet mask for a `CIDR `_ range. +message CidrRange { + // IPv4 or IPv6 address, e.g. ``192.0.0.0`` or ``2001:db8::``. + string address_prefix = 1 [(validate.rules).string.min_bytes = 1]; + // Length of prefix, e.g. 0, 32. + google.protobuf.UInt32Value prefix_len = 2 [(validate.rules).uint32.lte = 128]; +} diff --git a/api/envoy/api/v3alpha/core/base.proto b/api/envoy/api/v3alpha/core/base.proto new file mode 100644 index 000000000000..0661d99ec546 --- /dev/null +++ b/api/envoy/api/v3alpha/core/base.proto @@ -0,0 +1,292 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.core; + +option java_outer_classname = "BaseProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; +option go_package = "core"; + +import "envoy/api/v3alpha/core/http_uri.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +import "envoy/type/percent.proto"; + +option (gogoproto.equal_all) = true; +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: Common types] + +// Identifies location of where either Envoy runs or where upstream hosts run. +message Locality { + // Region this :ref:`zone ` belongs to. + string region = 1; + + // Defines the local service zone where Envoy is running. Though optional, it + // should be set if discovery service routing is used and the discovery + // service exposes :ref:`zone data `, + // either in this message or via :option:`--service-zone`. The meaning of zone + // is context dependent, e.g. `Availability Zone (AZ) + // `_ + // on AWS, `Zone `_ on + // GCP, etc. + string zone = 2; + + // When used for locality of upstream hosts, this field further splits zone + // into smaller chunks of sub-zones so they can be load balanced + // independently. + string sub_zone = 3; +} + +// Identifies a specific Envoy instance. The node identifier is presented to the +// management server, which may use this identifier to distinguish per Envoy +// configuration for serving. +message Node { + // An opaque node identifier for the Envoy node. This also provides the local + // service node name. It should be set if any of the following features are + // used: :ref:`statsd `, :ref:`CDS + // `, and :ref:`HTTP tracing + // `, either in this message or via + // :option:`--service-node`. + string id = 1; + + // Defines the local service cluster name where Envoy is running. Though + // optional, it should be set if any of the following features are used: + // :ref:`statsd `, :ref:`health check cluster + // verification `, + // :ref:`runtime override directory `, + // :ref:`user agent addition + // `, + // :ref:`HTTP global rate limiting `, + // :ref:`CDS `, and :ref:`HTTP tracing + // `, either in this message or via + // :option:`--service-cluster`. + string cluster = 2; + + // Opaque metadata extending the node identifier. Envoy will pass this + // directly to the management server. + google.protobuf.Struct metadata = 3; + + // Locality specifying where the Envoy instance is running. + Locality locality = 4; + + // This is motivated by informing a management server during canary which + // version of Envoy is being tested in a heterogeneous fleet. This will be set + // by Envoy in management server RPCs. + string build_version = 5; +} + +// Metadata provides additional inputs to filters based on matched listeners, +// filter chains, routes and endpoints. It is structured as a map, usually from +// filter name (in reverse DNS format) to metadata specific to the filter. Metadata +// key-values for a filter are merged as connection and request handling occurs, +// with later values for the same key overriding earlier values. +// +// An example use of metadata is providing additional values to +// http_connection_manager in the envoy.http_connection_manager.access_log +// namespace. +// +// Another example use of metadata is to per service config info in cluster metadata, which may get +// consumed by multiple filters. +// +// For load balancing, Metadata provides a means to subset cluster endpoints. +// Endpoints have a Metadata object associated and routes contain a Metadata +// object to match against. There are some well defined metadata used today for +// this purpose: +// +// * ``{"envoy.lb": {"canary": }}`` This indicates the canary status of an +// endpoint and is also used during header processing +// (x-envoy-upstream-canary) and for stats purposes. +message Metadata { + // Key is the reverse DNS filter name, e.g. com.acme.widget. The envoy.* + // namespace is reserved for Envoy's built-in filters. + map filter_metadata = 1; +} + +// Runtime derived uint32 with a default when not specified. +message RuntimeUInt32 { + // Default value if runtime value is not available. + uint32 default_value = 2; + + // Runtime key to get value for comparison. This value is used if defined. + string runtime_key = 3 [(validate.rules).string.min_bytes = 1]; +} + +// Envoy supports :ref:`upstream priority routing +// ` both at the route and the virtual +// cluster level. The current priority implementation uses different connection +// pool and circuit breaking settings for each priority level. This means that +// even for HTTP/2 requests, two physical connections will be used to an +// upstream host. In the future Envoy will likely support true HTTP/2 priority +// over a single upstream connection. +enum RoutingPriority { + DEFAULT = 0; + HIGH = 1; +} + +// HTTP request method. +enum RequestMethod { + option (gogoproto.goproto_enum_prefix) = false; + METHOD_UNSPECIFIED = 0; + GET = 1; + HEAD = 2; + POST = 3; + PUT = 4; + DELETE = 5; + CONNECT = 6; + OPTIONS = 7; + TRACE = 8; + PATCH = 9; +} + +// Header name/value pair. +message HeaderValue { + // Header name. + string key = 1 [(validate.rules).string = {min_bytes: 1, max_bytes: 16384}]; + + // Header value. + // + // The same :ref:`format specifier ` as used for + // :ref:`HTTP access logging ` applies here, however + // unknown header values are replaced with the empty string instead of `-`. + string value = 2 [(validate.rules).string.max_bytes = 16384]; +} + +// Header name/value pair plus option to control append behavior. +message HeaderValueOption { + // Header name/value pair that this option applies to. + HeaderValue header = 1 [(validate.rules).message.required = true]; + + // Should the value be appended? If true (default), the value is appended to + // existing values. + google.protobuf.BoolValue append = 2; +} + +// Wrapper for a set of headers. +message HeaderMap { + repeated HeaderValue headers = 1; +} + +// Data source consisting of either a file or an inline value. +message DataSource { + oneof specifier { + option (validate.required) = true; + + // Local filesystem data source. + string filename = 1 [(validate.rules).string.min_bytes = 1]; + + // Bytes inlined in the configuration. + bytes inline_bytes = 2 [(validate.rules).bytes.min_len = 1]; + + // String inlined in the configuration. + string inline_string = 3 [(validate.rules).string.min_bytes = 1]; + } +} + +// The message specifies how to fetch data from remote and how to verify it. +message RemoteDataSource { + // The HTTP URI to fetch the remote data. + HttpUri http_uri = 1 [(validate.rules).message.required = true]; + + // SHA256 string for verifying data. + string sha256 = 2 [(validate.rules).string.min_bytes = 1]; +} + +// Async data source which support async data fetch. +message AsyncDataSource { + oneof specifier { + option (validate.required) = true; + + // Local async data source. + DataSource local = 1; + + // Remote async data source. + RemoteDataSource remote = 2; + } +} + +// Configuration for transport socket in :ref:`listeners ` and +// :ref:`clusters `. If the configuration is +// empty, a default transport socket implementation and configuration will be +// chosen based on the platform and existence of tls_context. +message TransportSocket { + // The name of the transport socket to instantiate. The name must match a supported transport + // socket implementation. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Implementation specific configuration which depends on the implementation being instantiated. + // See the supported transport socket implementations for further documentation. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} + +// Generic socket option message. This would be used to set socket options that +// might not exist in upstream kernels or precompiled Envoy binaries. +message SocketOption { + // An optional name to give this socket option for debugging, etc. + // Uniqueness is not required and no special meaning is assumed. + string description = 1; + // Corresponding to the level value passed to setsockopt, such as IPPROTO_TCP + int64 level = 2; + // The numeric name as passed to setsockopt + int64 name = 3; + oneof value { + option (validate.required) = true; + + // Because many sockopts take an int value. + int64 int_value = 4; + // Otherwise it's a byte buffer. + bytes buf_value = 5; + } + enum SocketState { + option (gogoproto.goproto_enum_prefix) = false; + // Socket options are applied after socket creation but before binding the socket to a port + STATE_PREBIND = 0; + // Socket options are applied after binding the socket to a port but before calling listen() + STATE_BOUND = 1; + // Socket options are applied after calling listen() + STATE_LISTENING = 2; + } + // The state in which the option will be applied. When used in BindConfig + // STATE_PREBIND is currently the only valid value. + SocketState state = 6 [(validate.rules).enum.defined_only = true]; +} + +// Runtime derived FractionalPercent with defaults for when the numerator or denominator is not +// specified via a runtime key. +message RuntimeFractionalPercent { + // Default value if the runtime value's for the numerator/denominator keys are not available. + envoy.type.FractionalPercent default_value = 1 [(validate.rules).message.required = true]; + + // Runtime key for a YAML representation of a FractionalPercent. + string runtime_key = 2; +} + +// Identifies a specific ControlPlane instance that Envoy is connected to. +message ControlPlane { + // An opaque control plane identifier that uniquely identifies an instance + // of control plane. This can be used to identify which control plane instance, + // the Envoy is connected to. + string identifier = 1; +} + +// Identifies the direction of the traffic relative to the local Envoy. +enum TrafficDirection { + // Default option is unspecified. + UNSPECIFIED = 0; + + // The transport is used for incoming traffic. + INBOUND = 1; + + // The transport is used for outgoing traffic. + OUTBOUND = 2; +} diff --git a/api/envoy/api/v3alpha/core/config_source.proto b/api/envoy/api/v3alpha/core/config_source.proto new file mode 100644 index 000000000000..58ef7daeb0a1 --- /dev/null +++ b/api/envoy/api/v3alpha/core/config_source.proto @@ -0,0 +1,126 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.core; + +option java_outer_classname = "ConfigSourceProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; + +import "envoy/api/v3alpha/core/grpc_service.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Configuration sources] + +// API configuration source. This identifies the API type and cluster that Envoy +// will use to fetch an xDS API. +message ApiConfigSource { + // APIs may be fetched via either REST or gRPC. + enum ApiType { + // Ideally this would be 'reserved 0' but one can't reserve the default + // value. Instead we throw an exception if this is ever used. + UNSUPPORTED_REST_LEGACY = 0 [deprecated = true]; + // REST-JSON v2 API. The `canonical JSON encoding + // `_ for + // the v2 protos is used. + REST = 1; + // gRPC v2 API. + GRPC = 2; + // Using the delta xDS gRPC service, i.e. DeltaDiscovery{Request,Response} + // rather than Discovery{Request,Response}. Rather than sending Envoy the entire state + // with every update, the xDS server only sends what has changed since the last update. + // + // DELTA_GRPC is not yet entirely implemented! Initially, only CDS is available. + // Do not use for other xDSes. TODO(fredlas) update/remove this warning when appropriate. + DELTA_GRPC = 3; + } + ApiType api_type = 1 [(validate.rules).enum.defined_only = true]; + // Cluster names should be used only with REST. If > 1 + // cluster is defined, clusters will be cycled through if any kind of failure + // occurs. + // + // .. note:: + // + // The cluster with name ``cluster_name`` must be statically defined and its + // type must not be ``EDS``. + repeated string cluster_names = 2; + + // Multiple gRPC services be provided for GRPC. If > 1 cluster is defined, + // services will be cycled through if any kind of failure occurs. + repeated GrpcService grpc_services = 4; + + // For REST APIs, the delay between successive polls. + google.protobuf.Duration refresh_delay = 3 [(gogoproto.stdduration) = true]; + + // For REST APIs, the request timeout. If not set, a default value of 1s will be used. + google.protobuf.Duration request_timeout = 5 + [(validate.rules).duration.gt.seconds = 0, (gogoproto.stdduration) = true]; + + // For GRPC APIs, the rate limit settings. If present, discovery requests made by Envoy will be + // rate limited. + RateLimitSettings rate_limit_settings = 6; + + // Skip the node identifier in subsequent discovery requests for streaming gRPC config types. + bool set_node_on_first_message_only = 7; +} + +// Aggregated Discovery Service (ADS) options. This is currently empty, but when +// set in :ref:`ConfigSource ` can be used to +// specify that ADS is to be used. +message AggregatedConfigSource { +} + +// Rate Limit settings to be applied for discovery requests made by Envoy. +message RateLimitSettings { + // Maximum number of tokens to be used for rate limiting discovery request calls. If not set, a + // default value of 100 will be used. + google.protobuf.UInt32Value max_tokens = 1; + + // Rate at which tokens will be filled per second. If not set, a default fill rate of 10 tokens + // per second will be used. + google.protobuf.DoubleValue fill_rate = 2 [(validate.rules).double.gt = 0.0]; +} + +// Configuration for :ref:`listeners `, :ref:`clusters +// `, :ref:`routes +// `, :ref:`endpoints +// ` etc. may either be sourced from the +// filesystem or from an xDS API source. Filesystem configs are watched with +// inotify for updates. +message ConfigSource { + oneof config_source_specifier { + option (validate.required) = true; + // Path on the filesystem to source and watch for configuration updates. + // + // .. note:: + // + // The path to the source must exist at config load time. + // + // .. note:: + // + // Envoy will only watch the file path for *moves.* This is because in general only moves + // are atomic. The same method of swapping files as is demonstrated in the + // :ref:`runtime documentation ` can be used here also. + string path = 1; + // API configuration source. + ApiConfigSource api_config_source = 2; + // When set, ADS will be used to fetch resources. The ADS API configuration + // source in the bootstrap configuration is used. + AggregatedConfigSource ads = 3; + } + + // When this timeout is specified, Envoy will wait no longer than the specified time for first + // config response on this xDS subscription during the :ref:`initialization process + // `. After reaching the timeout, Envoy will move to the next + // initialization phase, even if the first config is not delivered yet. The timer is activated + // when the xDS API subscription starts, and is disarmed on first config update or on error. 0 + // means no timeout - Envoy will wait indefinitely for the first xDS config (unless another + // timeout applies). The default is 15s. + google.protobuf.Duration initial_fetch_timeout = 4; +} diff --git a/api/envoy/api/v3alpha/core/grpc_service.proto b/api/envoy/api/v3alpha/core/grpc_service.proto new file mode 100644 index 000000000000..931a4cb8926f --- /dev/null +++ b/api/envoy/api/v3alpha/core/grpc_service.proto @@ -0,0 +1,173 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.core; + +option java_outer_classname = "GrpcServiceProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; + +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/empty.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: gRPC services] + +// gRPC service configuration. This is used by :ref:`ApiConfigSource +// ` and filter configurations. +message GrpcService { + message EnvoyGrpc { + // The name of the upstream gRPC cluster. SSL credentials will be supplied + // in the :ref:`Cluster ` :ref:`tls_context + // `. + string cluster_name = 1 [(validate.rules).string.min_bytes = 1]; + } + + // [#proto-status: draft] + message GoogleGrpc { + // The target URI when using the `Google C++ gRPC client + // `_. SSL credentials will be supplied in + // :ref:`channel_credentials `. + string target_uri = 1 [(validate.rules).string.min_bytes = 1]; + + // See https://grpc.io/grpc/cpp/structgrpc_1_1_ssl_credentials_options.html. + message SslCredentials { + // PEM encoded server root certificates. + DataSource root_certs = 1; + + // PEM encoded client private key. + DataSource private_key = 2; + + // PEM encoded client certificate chain. + DataSource cert_chain = 3; + } + + // Local channel credentials. Only UDS is supported for now. + // See https://github.com/grpc/grpc/pull/15909. + message GoogleLocalCredentials { + } + + // See https://grpc.io/docs/guides/auth.html#credential-types to understand Channel and Call + // credential types. + message ChannelCredentials { + oneof credential_specifier { + option (validate.required) = true; + SslCredentials ssl_credentials = 1; + + // https://grpc.io/grpc/cpp/namespacegrpc.html#a6beb3ac70ff94bd2ebbd89b8f21d1f61 + google.protobuf.Empty google_default = 2; + + GoogleLocalCredentials local_credentials = 3; + } + } + + ChannelCredentials channel_credentials = 2; + + message CallCredentials { + message ServiceAccountJWTAccessCredentials { + string json_key = 1; + uint64 token_lifetime_seconds = 2; + } + + message GoogleIAMCredentials { + string authorization_token = 1; + string authority_selector = 2; + } + + message MetadataCredentialsFromPlugin { + string name = 1; + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } + } + + oneof credential_specifier { + option (validate.required) = true; + + // Access token credentials. + // https://grpc.io/grpc/cpp/namespacegrpc.html#ad3a80da696ffdaea943f0f858d7a360d. + string access_token = 1; + + // Google Compute Engine credentials. + // https://grpc.io/grpc/cpp/namespacegrpc.html#a6beb3ac70ff94bd2ebbd89b8f21d1f61 + google.protobuf.Empty google_compute_engine = 2; + + // Google refresh token credentials. + // https://grpc.io/grpc/cpp/namespacegrpc.html#a96901c997b91bc6513b08491e0dca37c. + string google_refresh_token = 3; + + // Service Account JWT Access credentials. + // https://grpc.io/grpc/cpp/namespacegrpc.html#a92a9f959d6102461f66ee973d8e9d3aa. + ServiceAccountJWTAccessCredentials service_account_jwt_access = 4; + + // Google IAM credentials. + // https://grpc.io/grpc/cpp/namespacegrpc.html#a9fc1fc101b41e680d47028166e76f9d0. + GoogleIAMCredentials google_iam = 5; + + // Custom authenticator credentials. + // https://grpc.io/grpc/cpp/namespacegrpc.html#a823c6a4b19ffc71fb33e90154ee2ad07. + // https://grpc.io/docs/guides/auth.html#extending-grpc-to-support-other-authentication-mechanisms. + MetadataCredentialsFromPlugin from_plugin = 6; + } + } + + // A set of call credentials that can be composed with `channel credentials + // `_. + repeated CallCredentials call_credentials = 3; + + // The human readable prefix to use when emitting statistics for the gRPC + // service. + // + // .. csv-table:: + // :header: Name, Type, Description + // :widths: 1, 1, 2 + // + // streams_total, Counter, Total number of streams opened + // streams_closed_, Counter, Total streams closed with + string stat_prefix = 4 [(validate.rules).string.min_bytes = 1]; + + // The name of the Google gRPC credentials factory to use. This must have been registered with + // Envoy. If this is empty, a default credentials factory will be used that sets up channel + // credentials based on other configuration parameters. + string credentials_factory_name = 5; + + // Additional configuration for site-specific customizations of the Google + // gRPC library. + google.protobuf.Struct config = 6; + } + + oneof target_specifier { + option (validate.required) = true; + + // Envoy's in-built gRPC client. + // See the :ref:`gRPC services overview ` + // documentation for discussion on gRPC client selection. + EnvoyGrpc envoy_grpc = 1; + + // `Google C++ gRPC client `_ + // See the :ref:`gRPC services overview ` + // documentation for discussion on gRPC client selection. + GoogleGrpc google_grpc = 2; + } + + // The timeout for the gRPC request. This is the timeout for a specific + // request. + google.protobuf.Duration timeout = 3; + + // Field 4 reserved due to moving credentials inside the GoogleGrpc message + reserved 4; + + // Additional metadata to include in streams initiated to the GrpcService. + // This can be used for scenarios in which additional ad hoc authorization + // headers (e.g. `x-foo-bar: baz-key`) are to be injected. + repeated HeaderValue initial_metadata = 5; +} diff --git a/api/envoy/api/v3alpha/core/health_check.proto b/api/envoy/api/v3alpha/core/health_check.proto new file mode 100644 index 000000000000..1a9508fb6490 --- /dev/null +++ b/api/envoy/api/v3alpha/core/health_check.proto @@ -0,0 +1,271 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.core; + +option java_outer_classname = "HealthCheckProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/type/range.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Health check] +// * Health checking :ref:`architecture overview `. +// * If health checking is configured for a cluster, additional statistics are emitted. They are +// documented :ref:`here `. + +message HealthCheck { + // The time to wait for a health check response. If the timeout is reached the + // health check attempt will be considered a failure. + google.protobuf.Duration timeout = 1 [ + (validate.rules).duration = { + required: true, + gt: {seconds: 0} + }, + (gogoproto.stdduration) = true + ]; + + // The interval between health checks. + google.protobuf.Duration interval = 2 [ + (validate.rules).duration = { + required: true, + gt: {seconds: 0} + }, + (gogoproto.stdduration) = true + ]; + + // An optional jitter amount in milliseconds. If specified, Envoy will start health + // checking after for a random time in ms between 0 and initial_jitter. This only + // applies to the first health check. + google.protobuf.Duration initial_jitter = 20; + + // An optional jitter amount in milliseconds. If specified, during every + // interval Envoy will add interval_jitter to the wait time. + google.protobuf.Duration interval_jitter = 3; + + // An optional jitter amount as a percentage of interval_ms. If specified, + // during every interval Envoy will add interval_ms * + // interval_jitter_percent / 100 to the wait time. + // + // If interval_jitter_ms and interval_jitter_percent are both set, both of + // them will be used to increase the wait time. + uint32 interval_jitter_percent = 18; + + // The number of unhealthy health checks required before a host is marked + // unhealthy. Note that for *http* health checking if a host responds with 503 + // this threshold is ignored and the host is considered unhealthy immediately. + google.protobuf.UInt32Value unhealthy_threshold = 4; + + // The number of healthy health checks required before a host is marked + // healthy. Note that during startup, only a single successful health check is + // required to mark a host healthy. + google.protobuf.UInt32Value healthy_threshold = 5; + + // [#not-implemented-hide:] Non-serving port for health checking. + google.protobuf.UInt32Value alt_port = 6; + + // Reuse health check connection between health checks. Default is true. + google.protobuf.BoolValue reuse_connection = 7; + + // Describes the encoding of the payload bytes in the payload. + message Payload { + oneof payload { + option (validate.required) = true; + + // Hex encoded payload. E.g., "000000FF". + string text = 1 [(validate.rules).string.min_bytes = 1]; + + // [#not-implemented-hide:] Binary payload. + bytes binary = 2; + } + } + + // [#comment:next free field: 10] + message HttpHealthCheck { + // The value of the host header in the HTTP health check request. If + // left empty (default value), the name of the cluster this health check is associated + // with will be used. + string host = 1; + + // Specifies the HTTP path that will be requested during health checking. For example + // */healthcheck*. + string path = 2 [(validate.rules).string.min_bytes = 1]; + + // [#not-implemented-hide:] HTTP specific payload. + Payload send = 3; + + // [#not-implemented-hide:] HTTP specific response. + Payload receive = 4; + + // An optional service name parameter which is used to validate the identity of + // the health checked cluster. See the :ref:`architecture overview + // ` for more information. + string service_name = 5; + + // Specifies a list of HTTP headers that should be added to each request that is sent to the + // health checked cluster. For more information, including details on header value syntax, see + // the documentation on :ref:`custom request headers + // `. + repeated core.HeaderValueOption request_headers_to_add = 6 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of HTTP headers that should be removed from each request that is sent to the + // health checked cluster. + repeated string request_headers_to_remove = 8; + + // If set, health checks will be made using http/2. + bool use_http2 = 7; + + // Specifies a list of HTTP response statuses considered healthy. If provided, replaces default + // 200-only policy - 200 must be included explicitly as needed. Ranges follow half-open + // semantics of :ref:`Int64Range `. + repeated envoy.type.Int64Range expected_statuses = 9; + } + + message TcpHealthCheck { + // Empty payloads imply a connect-only health check. + Payload send = 1; + + // When checking the response, “fuzzy” matching is performed such that each + // binary block must be found, and in the order specified, but not + // necessarily contiguous. + repeated Payload receive = 2; + } + + message RedisHealthCheck { + // If set, optionally perform ``EXISTS `` instead of ``PING``. A return value + // from Redis of 0 (does not exist) is considered a passing healthcheck. A return value other + // than 0 is considered a failure. This allows the user to mark a Redis instance for maintenance + // by setting the specified key to any value and waiting for traffic to drain. + string key = 1; + } + + // `grpc.health.v1.Health + // `_-based + // healthcheck. See `gRPC doc `_ + // for details. + message GrpcHealthCheck { + // An optional service name parameter which will be sent to gRPC service in + // `grpc.health.v1.HealthCheckRequest + // `_. + // message. See `gRPC health-checking overview + // `_ for more information. + string service_name = 1; + + // The value of the :authority header in the gRPC health check request. If + // left empty (default value), the name of the cluster this health check is associated + // with will be used. + string authority = 2; + } + + // Custom health check. + message CustomHealthCheck { + // The registered name of the custom health checker. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // A custom health checker specific configuration which depends on the custom health checker + // being instantiated. See :api:`envoy/config/health_checker` for reference. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } + } + + oneof health_checker { + option (validate.required) = true; + + // HTTP health check. + HttpHealthCheck http_health_check = 8; + + // TCP health check. + TcpHealthCheck tcp_health_check = 9; + + // gRPC health check. + GrpcHealthCheck grpc_health_check = 11; + + // Custom health check. + CustomHealthCheck custom_health_check = 13; + } + + reserved 10; // redis_health_check is deprecated by :ref:`custom_health_check + // ` + reserved "redis_health_check"; + + // The "no traffic interval" is a special health check interval that is used when a cluster has + // never had traffic routed to it. This lower interval allows cluster information to be kept up to + // date, without sending a potentially large amount of active health checking traffic for no + // reason. Once a cluster has been used for traffic routing, Envoy will shift back to using the + // standard health check interval that is defined. Note that this interval takes precedence over + // any other. + // + // The default value for "no traffic interval" is 60 seconds. + google.protobuf.Duration no_traffic_interval = 12 [(validate.rules).duration.gt = {}]; + + // The "unhealthy interval" is a health check interval that is used for hosts that are marked as + // unhealthy. As soon as the host is marked as healthy, Envoy will shift back to using the + // standard health check interval that is defined. + // + // The default value for "unhealthy interval" is the same as "interval". + google.protobuf.Duration unhealthy_interval = 14 [(validate.rules).duration.gt = {}]; + + // The "unhealthy edge interval" is a special health check interval that is used for the first + // health check right after a host is marked as unhealthy. For subsequent health checks + // Envoy will shift back to using either "unhealthy interval" if present or the standard health + // check interval that is defined. + // + // The default value for "unhealthy edge interval" is the same as "unhealthy interval". + google.protobuf.Duration unhealthy_edge_interval = 15 [(validate.rules).duration.gt = {}]; + + // The "healthy edge interval" is a special health check interval that is used for the first + // health check right after a host is marked as healthy. For subsequent health checks + // Envoy will shift back to using the standard health check interval that is defined. + // + // The default value for "healthy edge interval" is the same as the default interval. + google.protobuf.Duration healthy_edge_interval = 16 [(validate.rules).duration.gt = {}]; + + // Specifies the path to the :ref:`health check event log `. + // If empty, no event log will be written. + string event_log_path = 17; + + // If set to true, health check failure events will always be logged. If set to false, only the + // initial health check failure event will be logged. + // The default value is false. + bool always_log_health_check_failures = 19; +} + +// Endpoint health status. +enum HealthStatus { + // The health status is not known. This is interpreted by Envoy as *HEALTHY*. + UNKNOWN = 0; + + // Healthy. + HEALTHY = 1; + + // Unhealthy. + UNHEALTHY = 2; + + // Connection draining in progress. E.g., + // ``_ + // or + // ``_. + // This is interpreted by Envoy as *UNHEALTHY*. + DRAINING = 3; + + // Health check timed out. This is part of HDS and is interpreted by Envoy as + // *UNHEALTHY*. + TIMEOUT = 4; + + // Degraded. + DEGRADED = 5; +} diff --git a/api/envoy/api/v3alpha/core/http_uri.proto b/api/envoy/api/v3alpha/core/http_uri.proto new file mode 100644 index 000000000000..b47533f2066a --- /dev/null +++ b/api/envoy/api/v3alpha/core/http_uri.proto @@ -0,0 +1,54 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.core; + +option java_outer_classname = "HttpUriProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; + +import "google/protobuf/duration.proto"; +import "gogoproto/gogo.proto"; + +import "validate/validate.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: HTTP Service URI ] + +// Envoy external URI descriptor +message HttpUri { + // The HTTP server URI. It should be a full FQDN with protocol, host and path. + // + // Example: + // + // .. code-block:: yaml + // + // uri: https://www.googleapis.com/oauth2/v1/certs + // + string uri = 1 [(validate.rules).string.min_bytes = 1]; + + // Specify how `uri` is to be fetched. Today, this requires an explicit + // cluster, but in the future we may support dynamic cluster creation or + // inline DNS resolution. See `issue + // `_. + oneof http_upstream_type { + option (validate.required) = true; + // A cluster is created in the Envoy "cluster_manager" config + // section. This field specifies the cluster name. + // + // Example: + // + // .. code-block:: yaml + // + // cluster: jwks_cluster + // + string cluster = 2 [(validate.rules).string.min_bytes = 1]; + } + + // Sets the maximum duration in milliseconds that a response can take to arrive upon request. + google.protobuf.Duration timeout = 3 [ + (validate.rules).duration.gte = {}, + (validate.rules).duration.required = true, + (gogoproto.stdduration) = true + ]; +} diff --git a/api/envoy/api/v3alpha/core/protocol.proto b/api/envoy/api/v3alpha/core/protocol.proto new file mode 100644 index 000000000000..63d0f6920d83 --- /dev/null +++ b/api/envoy/api/v3alpha/core/protocol.proto @@ -0,0 +1,157 @@ +// [#protodoc-title: Protocol options] + +syntax = "proto3"; + +package envoy.api.v3alpha.core; + +option java_outer_classname = "ProtocolProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Protocol options] + +// [#not-implemented-hide:] +message TcpProtocolOptions { +} + +message HttpProtocolOptions { + // The idle timeout for upstream connection pool connections. The idle timeout is defined as the + // period in which there are no active requests. If not set, there is no idle timeout. When the + // idle timeout is reached the connection will be closed. Note that request based timeouts mean + // that HTTP/2 PINGs will not keep the connection alive. + google.protobuf.Duration idle_timeout = 1 [(gogoproto.stdduration) = true]; +} + +message Http1ProtocolOptions { + // Handle HTTP requests with absolute URLs in the requests. These requests + // are generally sent by clients to forward/explicit proxies. This allows clients to configure + // envoy as their HTTP proxy. In Unix, for example, this is typically done by setting the + // *http_proxy* environment variable. + google.protobuf.BoolValue allow_absolute_url = 1; + + // Handle incoming HTTP/1.0 and HTTP 0.9 requests. + // This is off by default, and not fully standards compliant. There is support for pre-HTTP/1.1 + // style connect logic, dechunking, and handling lack of client host iff + // *default_host_for_http_10* is configured. + bool accept_http_10 = 2; + + // A default host for HTTP/1.0 requests. This is highly suggested if *accept_http_10* is true as + // Envoy does not otherwise support HTTP/1.0 without a Host header. + // This is a no-op if *accept_http_10* is not true. + string default_host_for_http_10 = 3; +} + +// [#comment:next free field: 13] +message Http2ProtocolOptions { + // `Maximum table size `_ + // (in octets) that the encoder is permitted to use for the dynamic HPACK table. Valid values + // range from 0 to 4294967295 (2^32 - 1) and defaults to 4096. 0 effectively disables header + // compression. + google.protobuf.UInt32Value hpack_table_size = 1; + + // `Maximum concurrent streams `_ + // allowed for peer on one HTTP/2 connection. Valid values range from 1 to 2147483647 (2^31 - 1) + // and defaults to 2147483647. + google.protobuf.UInt32Value max_concurrent_streams = 2 + [(validate.rules).uint32 = {gte: 1, lte: 2147483647}]; + + // `Initial stream-level flow-control window + // `_ size. Valid values range from 65535 + // (2^16 - 1, HTTP/2 default) to 2147483647 (2^31 - 1, HTTP/2 maximum) and defaults to 268435456 + // (256 * 1024 * 1024). + // + // NOTE: 65535 is the initial window size from HTTP/2 spec. We only support increasing the default + // window size now, so it's also the minimum. + + // This field also acts as a soft limit on the number of bytes Envoy will buffer per-stream in the + // HTTP/2 codec buffers. Once the buffer reaches this pointer, watermark callbacks will fire to + // stop the flow of data to the codec buffers. + google.protobuf.UInt32Value initial_stream_window_size = 3 + [(validate.rules).uint32 = {gte: 65535, lte: 2147483647}]; + + // Similar to *initial_stream_window_size*, but for connection-level flow-control + // window. Currently, this has the same minimum/maximum/default as *initial_stream_window_size*. + google.protobuf.UInt32Value initial_connection_window_size = 4 + [(validate.rules).uint32 = {gte: 65535, lte: 2147483647}]; + + // Allows proxying Websocket and other upgrades over H2 connect. + bool allow_connect = 5; + + // [#not-implemented-hide:] Hiding until envoy has full metadata support. + // Still under implementation. DO NOT USE. + // + // Allows metadata. See [metadata + // docs](https://github.com/envoyproxy/envoy/blob/master/source/docs/h2_metadata.md) for more + // information. + bool allow_metadata = 6; + + // Limit the number of pending outbound downstream frames of all types (frames that are waiting to + // be written into the socket). Exceeding this limit triggers flood mitigation and connection is + // terminated. The ``http2.outbound_flood`` stat tracks the number of terminated connections due + // to flood mitigation. The default limit is 10000. + // [#comment:TODO: implement same limits for upstream outbound frames as well.] + google.protobuf.UInt32Value max_outbound_frames = 7 [(validate.rules).uint32 = {gte: 1}]; + + // Limit the number of pending outbound downstream frames of types PING, SETTINGS and RST_STREAM, + // preventing high memory utilization when receiving continuous stream of these frames. Exceeding + // this limit triggers flood mitigation and connection is terminated. The + // ``http2.outbound_control_flood`` stat tracks the number of terminated connections due to flood + // mitigation. The default limit is 1000. + // [#comment:TODO: implement same limits for upstream outbound frames as well.] + google.protobuf.UInt32Value max_outbound_control_frames = 8 [(validate.rules).uint32 = {gte: 1}]; + + // Limit the number of consecutive inbound frames of types HEADERS, CONTINUATION and DATA with an + // empty payload and no end stream flag. Those frames have no legitimate use and are abusive, but + // might be a result of a broken HTTP/2 implementation. The `http2.inbound_empty_frames_flood`` + // stat tracks the number of connections terminated due to flood mitigation. + // Setting this to 0 will terminate connection upon receiving first frame with an empty payload + // and no end stream flag. The default limit is 1. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_consecutive_inbound_frames_with_empty_payload = 9; + + // Limit the number of inbound PRIORITY frames allowed per each opened stream. If the number + // of PRIORITY frames received over the lifetime of connection exceeds the value calculated + // using this formula:: + // + // max_inbound_priority_frames_per_stream * (1 + inbound_streams) + // + // the connection is terminated. The ``http2.inbound_priority_frames_flood`` stat tracks + // the number of connections terminated due to flood mitigation. The default limit is 100. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_inbound_priority_frames_per_stream = 10; + + // Limit the number of inbound WINDOW_UPDATE frames allowed per DATA frame sent. If the number + // of WINDOW_UPDATE frames received over the lifetime of connection exceeds the value calculated + // using this formula:: + // + // 1 + 2 * (inbound_streams + + // max_inbound_window_update_frames_per_data_frame_sent * outbound_data_frames) + // + // the connection is terminated. The ``http2.inbound_priority_frames_flood`` stat tracks + // the number of connections terminated due to flood mitigation. The default limit is 10. + // Setting this to 1 should be enough to support HTTP/2 implementations with basic flow control, + // but more complex implementations that try to estimate available bandwidth require at least 2. + // [#comment:TODO: implement same limits for upstream inbound frames as well.] + google.protobuf.UInt32Value max_inbound_window_update_frames_per_data_frame_sent = 11 + [(validate.rules).uint32 = {gte: 1}]; + + // Allows invalid HTTP messaging and headers. When this option is disabled (default), then + // the whole HTTP/2 connection is terminated upon receiving invalid HEADERS frame. However, + // when this option is enabled, only the offending stream is terminated. + // + // See [RFC7540, sec. 8.1](https://tools.ietf.org/html/rfc7540#section-8.1) for details. + bool stream_error_on_invalid_http_messaging = 12; +} + +// [#not-implemented-hide:] +message GrpcProtocolOptions { + Http2ProtocolOptions http2_protocol_options = 1; +} diff --git a/api/envoy/api/v3alpha/discovery.proto b/api/envoy/api/v3alpha/discovery.proto new file mode 100644 index 000000000000..87433f0dca27 --- /dev/null +++ b/api/envoy/api/v3alpha/discovery.proto @@ -0,0 +1,230 @@ +syntax = "proto3"; + +package envoy.api.v3alpha; + +option java_outer_classname = "DiscoveryProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/any.proto"; +import "google/rpc/status.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: Common discovery API components] + +// A DiscoveryRequest requests a set of versioned resources of the same type for +// a given Envoy node on some API. +message DiscoveryRequest { + // The version_info provided in the request messages will be the version_info + // received with the most recent successfully processed response or empty on + // the first request. It is expected that no new request is sent after a + // response is received until the Envoy instance is ready to ACK/NACK the new + // configuration. ACK/NACK takes place by returning the new API config version + // as applied or the previous API config version respectively. Each type_url + // (see below) has an independent version associated with it. + string version_info = 1; + + // The node making the request. + core.Node node = 2; + + // List of resources to subscribe to, e.g. list of cluster names or a route + // configuration name. If this is empty, all resources for the API are + // returned. LDS/CDS may have empty resource_names, which will cause all + // resources for the Envoy instance to be returned. The LDS and CDS responses + // will then imply a number of resources that need to be fetched via EDS/RDS, + // which will be explicitly enumerated in resource_names. + repeated string resource_names = 3; + + // Type of the resource that is being requested, e.g. + // "type.googleapis.com/envoy.api.v3alpha.ClusterLoadAssignment". This is implicit + // in requests made via singleton xDS APIs such as CDS, LDS, etc. but is + // required for ADS. + string type_url = 4; + + // nonce corresponding to DiscoveryResponse being ACK/NACKed. See above + // discussion on version_info and the DiscoveryResponse nonce comment. This + // may be empty if no nonce is available, e.g. at startup or for non-stream + // xDS implementations. + string response_nonce = 5; + + // This is populated when the previous :ref:`DiscoveryResponse ` + // failed to update configuration. The *message* field in *error_details* provides the Envoy + // internal exception related to the failure. It is only intended for consumption during manual + // debugging, the string provided is not guaranteed to be stable across Envoy versions. + google.rpc.Status error_detail = 6; +} + +message DiscoveryResponse { + // The version of the response data. + string version_info = 1; + + // The response resources. These resources are typed and depend on the API being called. + repeated google.protobuf.Any resources = 2; + + // [#not-implemented-hide:] + // Canary is used to support two Envoy command line flags: + // + // * --terminate-on-canary-transition-failure. When set, Envoy is able to + // terminate if it detects that configuration is stuck at canary. Consider + // this example sequence of updates: + // - Management server applies a canary config successfully. + // - Management server rolls back to a production config. + // - Envoy rejects the new production config. + // Since there is no sensible way to continue receiving configuration + // updates, Envoy will then terminate and apply production config from a + // clean slate. + // * --dry-run-canary. When set, a canary response will never be applied, only + // validated via a dry run. + bool canary = 3; + + // Type URL for resources. Identifies the xDS API when muxing over ADS. + // Must be consistent with the type_url in the 'resources' repeated Any (if non-empty). + string type_url = 4; + + // For gRPC based subscriptions, the nonce provides a way to explicitly ack a + // specific DiscoveryResponse in a following DiscoveryRequest. Additional + // messages may have been sent by Envoy to the management server for the + // previous version on the stream prior to this DiscoveryResponse, that were + // unprocessed at response send time. The nonce allows the management server + // to ignore any further DiscoveryRequests for the previous version until a + // DiscoveryRequest bearing the nonce. The nonce is optional and is not + // required for non-stream based xDS implementations. + string nonce = 5; + + // [#not-implemented-hide:] + // The control plane instance that sent the response. + core.ControlPlane control_plane = 6; +} + +// DeltaDiscoveryRequest and DeltaDiscoveryResponse are used in a new gRPC +// endpoint for Delta xDS. +// +// With Delta xDS, the DeltaDiscoveryResponses do not need to include a full +// snapshot of the tracked resources. Instead, DeltaDiscoveryResponses are a +// diff to the state of a xDS client. +// In Delta XDS there are per-resource versions, which allow tracking state at +// the resource granularity. +// An xDS Delta session is always in the context of a gRPC bidirectional +// stream. This allows the xDS server to keep track of the state of xDS clients +// connected to it. +// +// In Delta xDS the nonce field is required and used to pair +// DeltaDiscoveryResponse to a DeltaDiscoveryRequest ACK or NACK. +// Optionally, a response message level system_version_info is present for +// debugging purposes only. +// +// DeltaDiscoveryRequest plays two independent roles. Any DeltaDiscoveryRequest +// can be either or both of: [1] informing the server of what resources the +// client has gained/lost interest in (using resource_names_subscribe and +// resource_names_unsubscribe), or [2] (N)ACKing an earlier resource update from +// the server (using response_nonce, with presence of error_detail making it a NACK). +// Additionally, the first message (for a given type_url) of a reconnected gRPC stream +// has a third role: informing the server of the resources (and their versions) +// that the client already possesses, using the initial_resource_versions field. +// +// As with state-of-the-world, when multiple resource types are multiplexed (ADS), +// all requests/acknowledgments/updates are logically walled off by type_url: +// a Cluster ACK exists in a completely separate world from a prior Route NACK. +// In particular, initial_resource_versions being sent at the "start" of every +// gRPC stream actually entails a message for each type_url, each with its own +// initial_resource_versions. +message DeltaDiscoveryRequest { + // The node making the request. + core.Node node = 1; + + // Type of the resource that is being requested, e.g. + // "type.googleapis.com/envoy.api.v3alpha.ClusterLoadAssignment". + string type_url = 2; + + // DeltaDiscoveryRequests allow the client to add or remove individual + // resources to the set of tracked resources in the context of a stream. + // All resource names in the resource_names_subscribe list are added to the + // set of tracked resources and all resource names in the resource_names_unsubscribe + // list are removed from the set of tracked resources. + // + // *Unlike* state-of-the-world xDS, an empty resource_names_subscribe or + // resource_names_unsubscribe list simply means that no resources are to be + // added or removed to the resource list. + // *Like* state-of-the-world xDS, the server must send updates for all tracked + // resources, but can also send updates for resources the client has not subscribed to. + // + // NOTE: the server must respond with all resources listed in resource_names_subscribe, + // even if it believes the client has the most recent version of them. The reason: + // the client may have dropped them, but then regained interest before it had a chance + // to send the unsubscribe message. See DeltaSubscriptionStateTest.RemoveThenAdd. + // + // These two fields can be set in any DeltaDiscoveryRequest, including ACKs + // and initial_resource_versions. + // + // A list of Resource names to add to the list of tracked resources. + repeated string resource_names_subscribe = 3; + + // A list of Resource names to remove from the list of tracked resources. + repeated string resource_names_unsubscribe = 4; + + // Informs the server of the versions of the resources the xDS client knows of, to enable the + // client to continue the same logical xDS session even in the face of gRPC stream reconnection. + // It will not be populated: [1] in the very first stream of a session, since the client will + // not yet have any resources, [2] in any message after the first in a stream (for a given + // type_url), since the server will already be correctly tracking the client's state. + // (In ADS, the first message *of each type_url* of a reconnected stream populates this map.) + // The map's keys are names of xDS resources known to the xDS client. + // The map's values are opaque resource versions. + map initial_resource_versions = 5; + + // When the DeltaDiscoveryRequest is a ACK or NACK message in response + // to a previous DeltaDiscoveryResponse, the response_nonce must be the + // nonce in the DeltaDiscoveryResponse. + // Otherwise response_nonce must be omitted. + string response_nonce = 6; + + // This is populated when the previous :ref:`DiscoveryResponse ` + // failed to update configuration. The *message* field in *error_details* + // provides the Envoy internal exception related to the failure. + google.rpc.Status error_detail = 7; +} + +message DeltaDiscoveryResponse { + // The version of the response data (used for debugging). + string system_version_info = 1; + + // The response resources. These are typed resources, whose types must match + // the type_url field. + repeated Resource resources = 2; + + // field id 3 IS available! + + // Type URL for resources. Identifies the xDS API when muxing over ADS. + // Must be consistent with the type_url in the Any within 'resources' if 'resources' is non-empty. + string type_url = 4; + + // Resources names of resources that have be deleted and to be removed from the xDS Client. + // Removed resources for missing resources can be ignored. + repeated string removed_resources = 6; + + // The nonce provides a way for DeltaDiscoveryRequests to uniquely + // reference a DeltaDiscoveryResponse when (N)ACKing. The nonce is required. + string nonce = 5; +} + +message Resource { + // The resource's name, to distinguish it from others of the same type of resource. + string name = 3; + + // [#not-implemented-hide:] + // The aliases are a list of other names that this resource can go by. + repeated string aliases = 4; + + // The resource level version. It allows xDS to track the state of individual + // resources. + string version = 1; + + // The resource being tracked. + google.protobuf.Any resource = 2; +} diff --git a/api/envoy/api/v3alpha/eds.proto b/api/envoy/api/v3alpha/eds.proto new file mode 100644 index 000000000000..7ba8592eb793 --- /dev/null +++ b/api/envoy/api/v3alpha/eds.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; + +package envoy.api.v3alpha; + +option java_outer_classname = "EdsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha"; + +option java_generic_services = true; + +import "envoy/api/v3alpha/discovery.proto"; +import "envoy/api/v3alpha/endpoint/endpoint.proto"; +import "envoy/type/percent.proto"; + +import "google/api/annotations.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; +import "google/protobuf/wrappers.proto"; +import "google/protobuf/duration.proto"; + +option (gogoproto.equal_all) = true; +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: EDS] +// Endpoint discovery :ref:`architecture overview ` + +service EndpointDiscoveryService { + // The resource_names field in DiscoveryRequest specifies a list of clusters + // to subscribe to updates for. + rpc StreamEndpoints(stream DiscoveryRequest) returns (stream DiscoveryResponse) { + } + + rpc DeltaEndpoints(stream DeltaDiscoveryRequest) returns (stream DeltaDiscoveryResponse) { + } + + rpc FetchEndpoints(DiscoveryRequest) returns (DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:endpoints" + body: "*" + }; + } +} + +// Each route from RDS will map to a single cluster or traffic split across +// clusters using weights expressed in the RDS WeightedCluster. +// +// With EDS, each cluster is treated independently from a LB perspective, with +// LB taking place between the Localities within a cluster and at a finer +// granularity between the hosts within a locality. The percentage of traffic +// for each endpoint is determined by both its load_balancing_weight, and the +// load_balancing_weight of its locality. First, a locality will be selected, +// then an endpoint within that locality will be chose based on its weight. +message ClusterLoadAssignment { + // Name of the cluster. This will be the :ref:`service_name + // ` value if specified + // in the cluster :ref:`EdsClusterConfig + // `. + string cluster_name = 1 [(validate.rules).string.min_bytes = 1]; + + // List of endpoints to load balance to. + repeated endpoint.LocalityLbEndpoints endpoints = 2; + + // Map of named endpoints that can be referenced in LocalityLbEndpoints. + map named_endpoints = 5; + + // Load balancing policy settings. + message Policy { + reserved 1; + + message DropOverload { + // Identifier for the policy specifying the drop. + string category = 1 [(validate.rules).string.min_bytes = 1]; + + // Percentage of traffic that should be dropped for the category. + envoy.type.FractionalPercent drop_percentage = 2; + } + // Action to trim the overall incoming traffic to protect the upstream + // hosts. This action allows protection in case the hosts are unable to + // recover from an outage, or unable to autoscale or unable to handle + // incoming traffic volume for any reason. + // + // At the client each category is applied one after the other to generate + // the 'actual' drop percentage on all outgoing traffic. For example: + // + // .. code-block:: json + // + // { "drop_overloads": [ + // { "category": "throttle", "drop_percentage": 60 } + // { "category": "lb", "drop_percentage": 50 } + // ]} + // + // The actual drop percentages applied to the traffic at the clients will be + // "throttle"_drop = 60% + // "lb"_drop = 20% // 50% of the remaining 'actual' load, which is 40%. + // actual_outgoing_load = 20% // remaining after applying all categories. + repeated DropOverload drop_overloads = 2; + + // Priority levels and localities are considered overprovisioned with this + // factor (in percentage). This means that we don't consider a priority + // level or locality unhealthy until the percentage of healthy hosts + // multiplied by the overprovisioning factor drops below 100. + // With the default value 140(1.4), Envoy doesn't consider a priority level + // or a locality unhealthy until their percentage of healthy hosts drops + // below 72%. For example: + // + // .. code-block:: json + // + // { "overprovisioning_factor": 100 } + // + // Read more at :ref:`priority levels ` and + // :ref:`localities `. + google.protobuf.UInt32Value overprovisioning_factor = 3 [(validate.rules).uint32.gt = 0]; + + // The max time until which the endpoints from this assignment can be used. + // If no new assignments are received before this time expires the endpoints + // are considered stale and should be marked unhealthy. + // Defaults to 0 which means endpoints never go stale. + google.protobuf.Duration endpoint_stale_after = 4 [(validate.rules).duration.gt.seconds = 0]; + + // The flag to disable overprovisioning. If it is set to true, + // :ref:`overprovisioning factor + // ` will be ignored + // and Envoy will not perform graceful failover between priority levels or + // localities as endpoints become unhealthy. Otherwise Envoy will perform + // graceful failover as :ref:`overprovisioning factor + // ` suggests. + // [#next-major-version: Unify with overprovisioning config as a single message.] + // [#not-implemented-hide:] + bool disable_overprovisioning = 5; + } + + // Load balancing policy settings. + Policy policy = 4; +} diff --git a/api/envoy/api/v3alpha/endpoint/BUILD b/api/envoy/api/v3alpha/endpoint/BUILD new file mode 100644 index 000000000000..1630438b13f6 --- /dev/null +++ b/api/envoy/api/v3alpha/endpoint/BUILD @@ -0,0 +1,49 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "endpoint", + srcs = ["endpoint.proto"], + visibility = ["//envoy/api/v3alpha:friends"], + deps = [ + "//envoy/api/v3alpha/auth:cert", + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:config_source", + "//envoy/api/v3alpha/core:health_check", + "//envoy/api/v3alpha/core:protocol", + ], +) + +api_go_proto_library( + name = "endpoint", + proto = ":endpoint", + deps = [ + "//envoy/api/v3alpha/auth:cert_go_proto", + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:config_source_go_proto", + "//envoy/api/v3alpha/core:health_check_go_proto", + "//envoy/api/v3alpha/core:protocol_go_proto", + ], +) + +api_proto_library_internal( + name = "load_report", + srcs = ["load_report.proto"], + visibility = ["//envoy/api/v3alpha:friends"], + deps = [ + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + ], +) + +api_go_proto_library( + name = "load_report", + proto = ":load_report", + deps = [ + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + ], +) diff --git a/api/envoy/api/v3alpha/endpoint/endpoint.proto b/api/envoy/api/v3alpha/endpoint/endpoint.proto new file mode 100644 index 000000000000..4bb1b57e8710 --- /dev/null +++ b/api/envoy/api/v3alpha/endpoint/endpoint.proto @@ -0,0 +1,129 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.endpoint; + +option java_outer_classname = "EndpointProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.endpoint"; +option go_package = "endpoint"; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/core/health_check.proto"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Endpoints] + +// Upstream host identifier. +message Endpoint { + // The upstream host address. + // + // .. attention:: + // + // The form of host address depends on the given cluster type. For STATIC or EDS, + // it is expected to be a direct IP address (or something resolvable by the + // specified :ref:`resolver ` + // in the Address). For LOGICAL or STRICT DNS, it is expected to be hostname, + // and will be resolved via DNS. + core.Address address = 1; + + // The optional health check configuration. + message HealthCheckConfig { + // Optional alternative health check port value. + // + // By default the health check address port of an upstream host is the same + // as the host's serving address port. This provides an alternative health + // check port. Setting this with a non-zero value allows an upstream host + // to have different health check address port. + uint32 port_value = 1 [(validate.rules).uint32.lte = 65535]; + } + + // The optional health check configuration is used as configuration for the + // health checker to contact the health checked host. + // + // .. attention:: + // + // This takes into effect only for upstream clusters with + // :ref:`active health checking ` enabled. + HealthCheckConfig health_check_config = 2; +} + +// An Endpoint that Envoy can route traffic to. +message LbEndpoint { + // Upstream host identifier or a named reference. + oneof host_identifier { + Endpoint endpoint = 1; + string endpoint_name = 5; + } + + // Optional health status when known and supplied by EDS server. + core.HealthStatus health_status = 2; + + // The endpoint metadata specifies values that may be used by the load + // balancer to select endpoints in a cluster for a given request. The filter + // name should be specified as *envoy.lb*. An example boolean key-value pair + // is *canary*, providing the optional canary status of the upstream host. + // This may be matched against in a route's + // :ref:`RouteAction ` metadata_match field + // to subset the endpoints considered in cluster load balancing. + core.Metadata metadata = 3; + + // The optional load balancing weight of the upstream host; at least 1. + // Envoy uses the load balancing weight in some of the built in load + // balancers. The load balancing weight for an endpoint is divided by the sum + // of the weights of all endpoints in the endpoint's locality to produce a + // percentage of traffic for the endpoint. This percentage is then further + // weighted by the endpoint's locality's load balancing weight from + // LocalityLbEndpoints. If unspecified, each host is presumed to have equal + // weight in a locality. + google.protobuf.UInt32Value load_balancing_weight = 4 [(validate.rules).uint32 = {gte: 1}]; +} + +// A group of endpoints belonging to a Locality. +// One can have multiple LocalityLbEndpoints for a locality, but this is +// generally only done if the different groups need to have different load +// balancing weights or different priorities. +message LocalityLbEndpoints { + // Identifies location of where the upstream hosts run. + core.Locality locality = 1; + + // The group of endpoints belonging to the locality specified. + repeated LbEndpoint lb_endpoints = 2; + + // Optional: Per priority/region/zone/sub_zone weight; at least 1. The load + // balancing weight for a locality is divided by the sum of the weights of all + // localities at the same priority level to produce the effective percentage + // of traffic for the locality. + // + // Locality weights are only considered when :ref:`locality weighted load + // balancing ` is + // configured. These weights are ignored otherwise. If no weights are + // specified when locality weighted load balancing is enabled, the locality is + // assigned no load. + google.protobuf.UInt32Value load_balancing_weight = 3 [(validate.rules).uint32 = {gte: 1}]; + + // Optional: the priority for this LocalityLbEndpoints. If unspecified this will + // default to the highest priority (0). + // + // Under usual circumstances, Envoy will only select endpoints for the highest + // priority (0). In the event all endpoints for a particular priority are + // unavailable/unhealthy, Envoy will fail over to selecting endpoints for the + // next highest priority group. + // + // Priorities should range from 0 (highest) to N (lowest) without skipping. + uint32 priority = 5 [(validate.rules).uint32 = {lte: 128}]; + + // Optional: Per locality proximity value which indicates how close this + // locality is from the source locality. This value only provides ordering + // information (lower the value, closer it is to the source locality). + // This will be consumed by load balancing schemes that need proximity order + // to determine where to route the requests. + // [#not-implemented-hide:] + google.protobuf.UInt32Value proximity = 6; +} diff --git a/api/envoy/api/v3alpha/endpoint/load_report.proto b/api/envoy/api/v3alpha/endpoint/load_report.proto new file mode 100644 index 000000000000..7e4aed3ba103 --- /dev/null +++ b/api/envoy/api/v3alpha/endpoint/load_report.proto @@ -0,0 +1,148 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.endpoint; + +option java_outer_classname = "LoadReportProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.endpoint"; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// These are stats Envoy reports to GLB every so often. Report frequency is +// defined by +// :ref:`LoadStatsResponse.load_reporting_interval`. +// Stats per upstream region/zone and optionally per subzone. +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. +message UpstreamLocalityStats { + // Name of zone, region and optionally endpoint group these metrics were + // collected from. Zone and region names could be empty if unknown. + core.Locality locality = 1; + + // The total number of requests successfully completed by the endpoints in the + // locality. + uint64 total_successful_requests = 2; + + // The total number of unfinished requests + uint64 total_requests_in_progress = 3; + + // The total number of requests that failed due to errors at the endpoint, + // aggregated over all endpoints in the locality. + uint64 total_error_requests = 4; + + // The total number of requests that were issued by this Envoy since + // the last report. This information is aggregated over all the + // upstream endpoints in the locality. + uint64 total_issued_requests = 8; + + // Stats for multi-dimensional load balancing. + repeated EndpointLoadMetricStats load_metric_stats = 5; + + // Endpoint granularity stats information for this locality. This information + // is populated if the Server requests it by setting + // :ref:`LoadStatsResponse.report_endpoint_granularity`. + repeated UpstreamEndpointStats upstream_endpoint_stats = 7; + + // [#not-implemented-hide:] The priority of the endpoint group these metrics + // were collected from. + uint32 priority = 6; +} + +message UpstreamEndpointStats { + // Upstream host address. + core.Address address = 1; + + // Opaque and implementation dependent metadata of the + // endpoint. Envoy will pass this directly to the management server. + google.protobuf.Struct metadata = 6; + + // The total number of requests successfully completed by the endpoints in the + // locality. These include non-5xx responses for HTTP, where errors + // originate at the client and the endpoint responded successfully. For gRPC, + // the grpc-status values are those not covered by total_error_requests below. + uint64 total_successful_requests = 2; + + // The total number of unfinished requests for this endpoint. + uint64 total_requests_in_progress = 3; + + // The total number of requests that failed due to errors at the endpoint. + // For HTTP these are responses with 5xx status codes and for gRPC the + // grpc-status values: + // + // - DeadlineExceeded + // - Unimplemented + // - Internal + // - Unavailable + // - Unknown + // - DataLoss + uint64 total_error_requests = 4; + + // The total number of requests that were issued to this endpoint + // since the last report. A single TCP connection, HTTP or gRPC + // request or stream is counted as one request. + uint64 total_issued_requests = 7; + + // Stats for multi-dimensional load balancing. + repeated EndpointLoadMetricStats load_metric_stats = 5; +} + +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. +message EndpointLoadMetricStats { + // Name of the metric; may be empty. + string metric_name = 1; + + // Number of calls that finished and included this metric. + uint64 num_requests_finished_with_metric = 2; + + // Sum of metric values across all calls that finished with this metric for + // load_reporting_interval. + double total_metric_value = 3; +} + +// Per cluster load stats. Envoy reports these stats a management server in a +// :ref:`LoadStatsRequest` +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. +// Next ID: 7 +message ClusterStats { + // The name of the cluster. + string cluster_name = 1 [(validate.rules).string.min_bytes = 1]; + + // The eds_cluster_config service_name of the cluster. + // It's possible that two clusters send the same service_name to EDS, + // in that case, the management server is supposed to do aggregation on the load reports. + string cluster_service_name = 6; + + // Need at least one. + repeated UpstreamLocalityStats upstream_locality_stats = 2 + [(validate.rules).repeated .min_items = 1]; + + // Cluster-level stats such as total_successful_requests may be computed by + // summing upstream_locality_stats. In addition, below there are additional + // cluster-wide stats. + // + // The total number of dropped requests. This covers requests + // deliberately dropped by the drop_overload policy and circuit breaking. + uint64 total_dropped_requests = 3; + + message DroppedRequests { + // Identifier for the policy specifying the drop. + string category = 1 [(validate.rules).string.min_bytes = 1]; + // Total number of deliberately dropped requests for the category. + uint64 dropped_count = 2; + } + // Information about deliberately dropped requests for each category specified + // in the DropOverload policy. + repeated DroppedRequests dropped_requests = 5; + + // Period over which the actual load report occurred. This will be guaranteed to include every + // request reported. Due to system load and delays between the *LoadStatsRequest* sent from Envoy + // and the *LoadStatsResponse* message sent from the management server, this may be longer than + // the requested load reporting interval in the *LoadStatsResponse*. + google.protobuf.Duration load_report_interval = 4; +} diff --git a/api/envoy/api/v3alpha/lds.proto b/api/envoy/api/v3alpha/lds.proto new file mode 100644 index 000000000000..d9976d6c0e3c --- /dev/null +++ b/api/envoy/api/v3alpha/lds.proto @@ -0,0 +1,206 @@ +syntax = "proto3"; + +package envoy.api.v3alpha; + +option java_outer_classname = "LdsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha"; + +option java_generic_services = true; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/discovery.proto"; +import "envoy/api/v3alpha/listener/listener.proto"; +import "envoy/api/v3alpha/listener/udp_listener_config.proto"; + +import "google/api/annotations.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Listener] +// Listener :ref:`configuration overview ` + +// The Envoy instance initiates an RPC at startup to discover a list of +// listeners. Updates are delivered via streaming from the LDS server and +// consist of a complete update of all listeners. Existing connections will be +// allowed to drain from listeners that are no longer present. +service ListenerDiscoveryService { + rpc DeltaListeners(stream DeltaDiscoveryRequest) returns (stream DeltaDiscoveryResponse) { + } + + rpc StreamListeners(stream DiscoveryRequest) returns (stream DiscoveryResponse) { + } + + rpc FetchListeners(DiscoveryRequest) returns (DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:listeners" + body: "*" + }; + } +} + +// [#comment:next free field: 19] +message Listener { + // The unique name by which this listener is known. If no name is provided, + // Envoy will allocate an internal UUID for the listener. If the listener is to be dynamically + // updated or removed via :ref:`LDS ` a unique name must be provided. + string name = 1; + + // The address that the listener should listen on. In general, the address must be unique, though + // that is governed by the bind rules of the OS. E.g., multiple listeners can listen on port 0 on + // Linux as the actual port will be allocated by the OS. + core.Address address = 2 [(validate.rules).message.required = true]; + + // A list of filter chains to consider for this listener. The + // :ref:`FilterChain ` with the most specific + // :ref:`FilterChainMatch ` criteria is used on a + // connection. + // + // Example using SNI for filter chain selection can be found in the + // :ref:`FAQ entry `. + repeated listener.FilterChain filter_chains = 3; + + // If a connection is redirected using *iptables*, the port on which the proxy + // receives it might be different from the original destination address. When this flag is set to + // true, the listener hands off redirected connections to the listener associated with the + // original destination address. If there is no listener associated with the original destination + // address, the connection is handled by the listener that receives it. Defaults to false. + // + // .. attention:: + // + // This field is deprecated. Use :ref:`an original_dst ` + // :ref:`listener filter ` instead. + // + // Note that hand off to another listener is *NOT* performed without this flag. Once + // :ref:`FilterChainMatch ` is implemented this flag + // will be removed, as filter chain matching can be used to select a filter chain based on the + // restored destination address. + google.protobuf.BoolValue use_original_dst = 4 [deprecated = true]; + + // Soft limit on size of the listener’s new connection read and write buffers. + // If unspecified, an implementation defined default is applied (1MiB). + google.protobuf.UInt32Value per_connection_buffer_limit_bytes = 5; + + // Listener metadata. + core.Metadata metadata = 6; + + // [#not-implemented-hide:] + message DeprecatedV1 { + // Whether the listener should bind to the port. A listener that doesn't + // bind can only receive connections redirected from other listeners that + // set use_original_dst parameter to true. Default is true. + // + // [V2-API-DIFF] This is deprecated in v2, all Listeners will bind to their + // port. An additional filter chain must be created for every original + // destination port this listener may redirect to in v2, with the original + // port specified in the FilterChainMatch destination_port field. + // + // [#comment:TODO(PiotrSikora): Remove this once verified that we no longer need it.] + google.protobuf.BoolValue bind_to_port = 1; + } + + // [#not-implemented-hide:] + DeprecatedV1 deprecated_v1 = 7; + + enum DrainType { + // Drain in response to calling /healthcheck/fail admin endpoint (along with the health check + // filter), listener removal/modification, and hot restart. + DEFAULT = 0; + // Drain in response to listener removal/modification and hot restart. This setting does not + // include /healthcheck/fail. This setting may be desirable if Envoy is hosting both ingress + // and egress listeners. + MODIFY_ONLY = 1; + } + + // The type of draining to perform at a listener-wide level. + DrainType drain_type = 8; + + // Listener filters have the opportunity to manipulate and augment the connection metadata that + // is used in connection filter chain matching, for example. These filters are run before any in + // :ref:`filter_chains `. Order matters as the + // filters are processed sequentially right after a socket has been accepted by the listener, and + // before a connection is created. + // UDP Listener filters can be specified when the protocol in the listener socket address in + // :ref:`protocol ` is :ref:'UDP + // `. + // UDP listeners currently support a single filter. + repeated listener.ListenerFilter listener_filters = 9; + + // The timeout to wait for all listener filters to complete operation. If the timeout is reached, + // the accepted socket is closed without a connection being created unless + // `continue_on_listener_filters_timeout` is set to true. Specify 0 to disable the + // timeout. If not specified, a default timeout of 15s is used. + google.protobuf.Duration listener_filters_timeout = 15 [(gogoproto.stdduration) = true]; + + // Whether a connection should be created when listener filters timeout. Default is false. + // + // .. attention:: + // + // Some listener filters, such as :ref:`Proxy Protocol filter + // `, should not be used with this option. It will cause + // unexpected behavior when a connection is created. + bool continue_on_listener_filters_timeout = 17; + + // Whether the listener should be set as a transparent socket. + // When this flag is set to true, connections can be redirected to the listener using an + // *iptables* *TPROXY* target, in which case the original source and destination addresses and + // ports are preserved on accepted connections. This flag should be used in combination with + // :ref:`an original_dst ` :ref:`listener filter + // ` to mark the connections' local addresses as + // "restored." This can be used to hand off each redirected connection to another listener + // associated with the connection's destination address. Direct connections to the socket without + // using *TPROXY* cannot be distinguished from connections redirected using *TPROXY* and are + // therefore treated as if they were redirected. + // When this flag is set to false, the listener's socket is explicitly reset as non-transparent. + // Setting this flag requires Envoy to run with the *CAP_NET_ADMIN* capability. + // When this flag is not set (default), the socket is not modified, i.e. the transparent option + // is neither set nor reset. + google.protobuf.BoolValue transparent = 10; + + // Whether the listener should set the *IP_FREEBIND* socket option. When this + // flag is set to true, listeners can be bound to an IP address that is not + // configured on the system running Envoy. When this flag is set to false, the + // option *IP_FREEBIND* is disabled on the socket. When this flag is not set + // (default), the socket is not modified, i.e. the option is neither enabled + // nor disabled. + google.protobuf.BoolValue freebind = 11; + + // Additional socket options that may not be present in Envoy source code or + // precompiled binaries. + repeated core.SocketOption socket_options = 13; + + // Whether the listener should accept TCP Fast Open (TFO) connections. + // When this flag is set to a value greater than 0, the option TCP_FASTOPEN is enabled on + // the socket, with a queue length of the specified size + // (see `details in RFC7413 `_). + // When this flag is set to 0, the option TCP_FASTOPEN is disabled on the socket. + // When this flag is not set (default), the socket is not modified, + // i.e. the option is neither enabled nor disabled. + // + // On Linux, the net.ipv4.tcp_fastopen kernel parameter must include flag 0x2 to enable + // TCP_FASTOPEN. + // See `ip-sysctl.txt `_. + // + // On macOS, only values of 0, 1, and unset are valid; other values may result in an error. + // To set the queue length on macOS, set the net.inet.tcp.fastopen_backlog kernel parameter. + google.protobuf.UInt32Value tcp_fast_open_queue_length = 12; + + reserved 14; + + // Specifies the intended direction of the traffic relative to the local Envoy. + core.TrafficDirection traffic_direction = 16; + + // If the protocol in the listener socket address in :ref:`protocol + // ` is :ref:'UDP + // `, this field specifies the actual udp listener to create, + // i.e. :ref:`udp_listener_name + // ` = "raw_udp_listener" for + // creating a packet-oriented UDP listener. If not present, treat it as "raw_udp_listener". + listener.UdpListenerConfig udp_listener_config = 18; +} diff --git a/api/envoy/api/v3alpha/listener/BUILD b/api/envoy/api/v3alpha/listener/BUILD new file mode 100644 index 000000000000..693ead54dde0 --- /dev/null +++ b/api/envoy/api/v3alpha/listener/BUILD @@ -0,0 +1,41 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "listener", + srcs = ["listener.proto"], + visibility = ["//envoy/api/v3alpha:friends"], + deps = [ + "//envoy/api/v3alpha/auth:cert", + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + ], +) + +api_go_proto_library( + name = "listener", + proto = ":listener", + deps = [ + "//envoy/api/v3alpha/auth:cert_go_proto", + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + ], +) + +api_proto_library_internal( + name = "udp_listener_config", + srcs = ["udp_listener_config.proto"], + visibility = ["//envoy/api/v3alpha:friends"], + deps = [ + "//envoy/api/v3alpha/core:base", + ], +) + +api_go_proto_library( + name = "udp_listener_config", + proto = ":udp_listener_config", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + ], +) diff --git a/api/envoy/api/v3alpha/listener/listener.proto b/api/envoy/api/v3alpha/listener/listener.proto new file mode 100644 index 000000000000..2aa7146a822c --- /dev/null +++ b/api/envoy/api/v3alpha/listener/listener.proto @@ -0,0 +1,210 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.listener; + +option java_outer_classname = "ListenerProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.listener"; +option go_package = "listener"; +option csharp_namespace = "Envoy.Api.V2.ListenerNS"; +option ruby_package = "Envoy::Api::V2::ListenerNS"; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/auth/cert.proto"; +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Listener components] +// Listener :ref:`configuration overview ` + +message Filter { + // The name of the filter to instantiate. The name must match a + // :ref:`supported filter `. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Filter specific configuration which depends on the filter being + // instantiated. See the supported filters for further documentation. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 4; + } + + reserved 3; +} + +// Specifies the match criteria for selecting a specific filter chain for a +// listener. +// +// In order for a filter chain to be selected, *ALL* of its criteria must be +// fulfilled by the incoming connection, properties of which are set by the +// networking stack and/or listener filters. +// +// The following order applies: +// +// 1. Destination port. +// 2. Destination IP address. +// 3. Server name (e.g. SNI for TLS protocol), +// 4. Transport protocol. +// 5. Application protocols (e.g. ALPN for TLS protocol). +// 6. Source type (e.g. any, local or external network). +// 7. Source IP address. +// 8. Source port. +// +// For criteria that allow ranges or wildcards, the most specific value in any +// of the configured filter chains that matches the incoming connection is going +// to be used (e.g. for SNI ``www.example.com`` the most specific match would be +// ``www.example.com``, then ``*.example.com``, then ``*.com``, then any filter +// chain without ``server_names`` requirements). +// +// [#comment: Implemented rules are kept in the preference order, with deprecated fields +// listed at the end, because that's how we want to list them in the docs. +// +// [#comment:TODO(PiotrSikora): Add support for configurable precedence of the rules] +message FilterChainMatch { + // Optional destination port to consider when use_original_dst is set on the + // listener in determining a filter chain match. + google.protobuf.UInt32Value destination_port = 8 [(validate.rules).uint32 = {gte: 1, lte: 65535}]; + + // If non-empty, an IP address and prefix length to match addresses when the + // listener is bound to 0.0.0.0/:: or when use_original_dst is specified. + repeated core.CidrRange prefix_ranges = 3; + + // If non-empty, an IP address and suffix length to match addresses when the + // listener is bound to 0.0.0.0/:: or when use_original_dst is specified. + // [#not-implemented-hide:] + string address_suffix = 4; + + // [#not-implemented-hide:] + google.protobuf.UInt32Value suffix_len = 5; + + enum ConnectionSourceType { + // Any connection source matches. + ANY = 0; + // Match a connection originating from the same host. + LOCAL = 1; + // Match a connection originating from a different host. + EXTERNAL = 2; + } + + // Specifies the connection source IP match type. Can be any, local or external network. + ConnectionSourceType source_type = 12 [(validate.rules).enum.defined_only = true]; + + // The criteria is satisfied if the source IP address of the downstream + // connection is contained in at least one of the specified subnets. If the + // parameter is not specified or the list is empty, the source IP address is + // ignored. + repeated core.CidrRange source_prefix_ranges = 6; + + // The criteria is satisfied if the source port of the downstream connection + // is contained in at least one of the specified ports. If the parameter is + // not specified, the source port is ignored. + repeated uint32 source_ports = 7 [(validate.rules).repeated .items.uint32 = {gte: 1, lte: 65535}]; + + // If non-empty, a list of server names (e.g. SNI for TLS protocol) to consider when determining + // a filter chain match. Those values will be compared against the server names of a new + // connection, when detected by one of the listener filters. + // + // The server name will be matched against all wildcard domains, i.e. ``www.example.com`` + // will be first matched against ``www.example.com``, then ``*.example.com``, then ``*.com``. + // + // Note that partial wildcards are not supported, and values like ``*w.example.com`` are invalid. + // + // .. attention:: + // + // See the :ref:`FAQ entry ` on how to configure SNI for more + // information. + repeated string server_names = 11; + + // If non-empty, a transport protocol to consider when determining a filter chain match. + // This value will be compared against the transport protocol of a new connection, when + // it's detected by one of the listener filters. + // + // Suggested values include: + // + // * ``raw_buffer`` - default, used when no transport protocol is detected, + // * ``tls`` - set by :ref:`envoy.listener.tls_inspector ` + // when TLS protocol is detected. + string transport_protocol = 9; + + // If non-empty, a list of application protocols (e.g. ALPN for TLS protocol) to consider when + // determining a filter chain match. Those values will be compared against the application + // protocols of a new connection, when detected by one of the listener filters. + // + // Suggested values include: + // + // * ``http/1.1`` - set by :ref:`envoy.listener.tls_inspector + // `, + // * ``h2`` - set by :ref:`envoy.listener.tls_inspector ` + // + // .. attention:: + // + // Currently, only :ref:`TLS Inspector ` provides + // application protocol detection based on the requested + // `ALPN `_ values. + // + // However, the use of ALPN is pretty much limited to the HTTP/2 traffic on the Internet, + // and matching on values other than ``h2`` is going to lead to a lot of false negatives, + // unless all connecting clients are known to use ALPN. + repeated string application_protocols = 10; + + reserved 1; + reserved "sni_domains"; +} + +// A filter chain wraps a set of match criteria, an option TLS context, a set of filters, and +// various other parameters. +message FilterChain { + // The criteria to use when matching a connection to this filter chain. + FilterChainMatch filter_chain_match = 1; + + // The TLS context for this filter chain. + auth.DownstreamTlsContext tls_context = 2; + + // A list of individual network filters that make up the filter chain for + // connections established with the listener. Order matters as the filters are + // processed sequentially as connection events happen. Note: If the filter + // list is empty, the connection will close by default. + repeated Filter filters = 3; + + // Whether the listener should expect a PROXY protocol V1 header on new + // connections. If this option is enabled, the listener will assume that that + // remote address of the connection is the one specified in the header. Some + // load balancers including the AWS ELB support this option. If the option is + // absent or set to false, Envoy will use the physical peer address of the + // connection as the remote address. + google.protobuf.BoolValue use_proxy_proto = 4; + + // [#not-implemented-hide:] filter chain metadata. + core.Metadata metadata = 5; + + // See :ref:`base.TransportSocket` description. + core.TransportSocket transport_socket = 6; + + // [#not-implemented-hide:] The unique name (or empty) by which this filter chain is known. If no + // name is provided, Envoy will allocate an internal UUID for the filter chain. If the filter + // chain is to be dynamically updated or removed via FCDS a unique name must be provided. + string name = 7; +} + +message ListenerFilter { + // The name of the filter to instantiate. The name must match a + // :ref:`supported filter `. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Filter specific configuration which depends on the filter being instantiated. + // See the supported filters for further documentation. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} diff --git a/api/envoy/api/v3alpha/listener/udp_listener_config.proto b/api/envoy/api/v3alpha/listener/udp_listener_config.proto new file mode 100644 index 000000000000..763a08a93ad3 --- /dev/null +++ b/api/envoy/api/v3alpha/listener/udp_listener_config.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.listener; + +option java_outer_classname = "UdpListenerConfigProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.listener"; +option go_package = "listener"; +option csharp_namespace = "Envoy.Api.V2.ListenerNS"; +option ruby_package = "Envoy::Api::V2::ListenerNS"; + +import "google/protobuf/struct.proto"; +import "google/protobuf/any.proto"; + +// [#protodoc-title: Udp Listener Config] +// Listener :ref:`configuration overview ` + +message UdpListenerConfig { + // Used to look up UDP listener factory, matches "raw_udp_listener" or + // "quic_listener" to create a specific udp listener. + // If not specified, treat as "raw_udp_listener". + string udp_listener_name = 1; + + // Used to create a specific listener factory. To some factory, e.g. + // "raw_udp_listener", config is not needed. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} diff --git a/api/envoy/api/v3alpha/ratelimit/BUILD b/api/envoy/api/v3alpha/ratelimit/BUILD new file mode 100644 index 000000000000..b08c1fc029a0 --- /dev/null +++ b/api/envoy/api/v3alpha/ratelimit/BUILD @@ -0,0 +1,14 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "ratelimit", + srcs = ["ratelimit.proto"], + visibility = ["//envoy/api/v3alpha:friends"], +) + +api_go_proto_library( + name = "ratelimit", + proto = ":ratelimit", +) diff --git a/api/envoy/api/v3alpha/ratelimit/ratelimit.proto b/api/envoy/api/v3alpha/ratelimit/ratelimit.proto new file mode 100644 index 000000000000..c10bfef83b98 --- /dev/null +++ b/api/envoy/api/v3alpha/ratelimit/ratelimit.proto @@ -0,0 +1,66 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.ratelimit; + +option java_outer_classname = "RatelimitProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.ratelimit"; +option go_package = "ratelimit"; + +import "validate/validate.proto"; + +// [#protodoc-title: Common rate limit components] + +// A RateLimitDescriptor is a list of hierarchical entries that are used by the service to +// determine the final rate limit key and overall allowed limit. Here are some examples of how +// they might be used for the domain "envoy". +// +// .. code-block:: cpp +// +// ["authenticated": "false"], ["remote_address": "10.0.0.1"] +// +// What it does: Limits all unauthenticated traffic for the IP address 10.0.0.1. The +// configuration supplies a default limit for the *remote_address* key. If there is a desire to +// raise the limit for 10.0.0.1 or block it entirely it can be specified directly in the +// configuration. +// +// .. code-block:: cpp +// +// ["authenticated": "false"], ["path": "/foo/bar"] +// +// What it does: Limits all unauthenticated traffic globally for a specific path (or prefix if +// configured that way in the service). +// +// .. code-block:: cpp +// +// ["authenticated": "false"], ["path": "/foo/bar"], ["remote_address": "10.0.0.1"] +// +// What it does: Limits unauthenticated traffic to a specific path for a specific IP address. +// Like (1) we can raise/block specific IP addresses if we want with an override configuration. +// +// .. code-block:: cpp +// +// ["authenticated": "true"], ["client_id": "foo"] +// +// What it does: Limits all traffic for an authenticated client "foo" +// +// .. code-block:: cpp +// +// ["authenticated": "true"], ["client_id": "foo"], ["path": "/foo/bar"] +// +// What it does: Limits traffic to a specific path for an authenticated client "foo" +// +// The idea behind the API is that (1)/(2)/(3) and (4)/(5) can be sent in 1 request if desired. +// This enables building complex application scenarios with a generic backend. +message RateLimitDescriptor { + message Entry { + // Descriptor key. + string key = 1 [(validate.rules).string.min_bytes = 1]; + + // Descriptor value. + string value = 2 [(validate.rules).string.min_bytes = 1]; + } + + // Descriptor entries. + repeated Entry entries = 1 [(validate.rules).repeated .min_items = 1]; +} diff --git a/api/envoy/api/v3alpha/rds.proto b/api/envoy/api/v3alpha/rds.proto new file mode 100644 index 000000000000..ed20b34e7cca --- /dev/null +++ b/api/envoy/api/v3alpha/rds.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; + +package envoy.api.v3alpha; + +option java_outer_classname = "RdsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha"; + +option java_generic_services = true; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/core/config_source.proto"; +import "envoy/api/v3alpha/discovery.proto"; +import "envoy/api/v3alpha/route/route.proto"; + +import "google/api/annotations.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: HTTP route configuration] +// * Routing :ref:`architecture overview ` +// * HTTP :ref:`router filter ` + +// The resource_names field in DiscoveryRequest specifies a route configuration. +// This allows an Envoy configuration with multiple HTTP listeners (and +// associated HTTP connection manager filters) to use different route +// configurations. Each listener will bind its HTTP connection manager filter to +// a route table via this identifier. +service RouteDiscoveryService { + rpc StreamRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { + } + + rpc DeltaRoutes(stream DeltaDiscoveryRequest) returns (stream DeltaDiscoveryResponse) { + } + + rpc FetchRoutes(DiscoveryRequest) returns (DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:routes" + body: "*" + }; + } +} + +// Virtual Host Discovery Service (VHDS) is used to dynamically update the list of virtual hosts for +// a given RouteConfiguration. If VHDS is configured a virtual host list update will be triggered +// during the processing of an HTTP request if a route for the request cannot be resolved. The +// :ref:`resource_names_subscribe ` +// field contains a list of virtual host names or aliases to track. The contents of an alias would +// be the contents of a *host* or *authority* header used to make an http request. An xDS server +// will match an alias to a virtual host based on the content of :ref:`domains' +// ` field. The *resource_names_unsubscribe* field contains +// a list of virtual host names that have been :ref:`unsubscribed ` +// from the routing table associated with the RouteConfiguration. +service VirtualHostDiscoveryService { + rpc DeltaVirtualHosts(stream DeltaDiscoveryRequest) returns (stream DeltaDiscoveryResponse) { + } +} + +// [#comment:next free field: 10] +message RouteConfiguration { + // The name of the route configuration. For example, it might match + // :ref:`route_config_name + // ` + // in :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v3alpha.Rds`. + string name = 1; + + // An array of virtual hosts that make up the route table. + repeated route.VirtualHost virtual_hosts = 2; + + // An array of virtual hosts will be dynamically loaded via the VHDS API. + // Both *virtual_hosts* and *vhds* fields will be used when present. *virtual_hosts* can be used + // for a base routing table or for infrequently changing virtual hosts. *vhds* is used for + // on-demand discovery of virtual hosts. The contents of these two fields will be merged to + // generate a routing table for a given RouteConfiguration, with *vhds* derived configuration + // taking precedence. + // [#not-implemented-hide:] + Vhds vhds = 9; + + // Optionally specifies a list of HTTP headers that the connection manager + // will consider to be internal only. If they are found on external requests they will be cleaned + // prior to filter invocation. See :ref:`config_http_conn_man_headers_x-envoy-internal` for more + // information. + repeated string internal_only_headers = 3; + + // Specifies a list of HTTP headers that should be added to each response that + // the connection manager encodes. Headers specified at this level are applied + // after headers from any enclosed :ref:`envoy_api_msg_route.VirtualHost` or + // :ref:`envoy_api_msg_route.RouteAction`. For more information, including details on + // header value syntax, see the documentation on :ref:`custom request headers + // `. + repeated core.HeaderValueOption response_headers_to_add = 4 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of HTTP headers that should be removed from each response + // that the connection manager encodes. + repeated string response_headers_to_remove = 5; + + // Specifies a list of HTTP headers that should be added to each request + // routed by the HTTP connection manager. Headers specified at this level are + // applied after headers from any enclosed :ref:`envoy_api_msg_route.VirtualHost` or + // :ref:`envoy_api_msg_route.RouteAction`. For more information, including details on + // header value syntax, see the documentation on :ref:`custom request headers + // `. + repeated core.HeaderValueOption request_headers_to_add = 6 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of HTTP headers that should be removed from each request + // routed by the HTTP connection manager. + repeated string request_headers_to_remove = 8; + + // An optional boolean that specifies whether the clusters that the route + // table refers to will be validated by the cluster manager. If set to true + // and a route refers to a non-existent cluster, the route table will not + // load. If set to false and a route refers to a non-existent cluster, the + // route table will load and the router filter will return a 404 if the route + // is selected at runtime. This setting defaults to true if the route table + // is statically defined via the :ref:`route_config + // ` + // option. This setting default to false if the route table is loaded dynamically via the + // :ref:`rds + // ` + // option. Users may wish to override the default behavior in certain cases (for example when + // using CDS with a static route table). + google.protobuf.BoolValue validate_clusters = 7; +} + +// [#not-implemented-hide:] +message Vhds { + // Configuration source specifier for VHDS. + envoy.api.v3alpha.core.ConfigSource config_source = 1 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/api/v3alpha/route/BUILD b/api/envoy/api/v3alpha/route/BUILD new file mode 100644 index 000000000000..0b660893c5d4 --- /dev/null +++ b/api/envoy/api/v3alpha/route/BUILD @@ -0,0 +1,28 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "route", + srcs = ["route.proto"], + visibility = ["//envoy/api/v3alpha:friends"], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/type:percent", + "//envoy/type:range", + "//envoy/type/matcher:regex", + "//envoy/type/matcher:string", + ], +) + +api_go_proto_library( + name = "route", + proto = ":route", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/type:percent_go_proto", + "//envoy/type:range_go_proto", + "//envoy/type/matcher:regex_go_proto", + "//envoy/type/matcher:string_go_proto", + ], +) diff --git a/api/envoy/api/v3alpha/route/route.proto b/api/envoy/api/v3alpha/route/route.proto new file mode 100644 index 000000000000..963d94f1b022 --- /dev/null +++ b/api/envoy/api/v3alpha/route/route.proto @@ -0,0 +1,1404 @@ +syntax = "proto3"; + +package envoy.api.v3alpha.route; + +option java_outer_classname = "RouteProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.api.v3alpha.route"; +option go_package = "route"; +option java_generic_services = true; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/type/matcher/regex.proto"; +import "envoy/type/matcher/string.proto"; +import "envoy/type/percent.proto"; +import "envoy/type/range.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: HTTP route] +// * Routing :ref:`architecture overview ` +// * HTTP :ref:`router filter ` + +// The top level element in the routing configuration is a virtual host. Each virtual host has +// a logical name as well as a set of domains that get routed to it based on the incoming request's +// host header. This allows a single listener to service multiple top level domain path trees. Once +// a virtual host is selected based on the domain, the routes are processed in order to see which +// upstream cluster to route to or whether to perform a redirect. +// [#comment:next free field: 17] +message VirtualHost { + // The logical name of the virtual host. This is used when emitting certain + // statistics but is not relevant for routing. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // A list of domains (host/authority header) that will be matched to this + // virtual host. Wildcard hosts are supported in the suffix or prefix form. + // + // Domain search order: + // 1. Exact domain names: ``www.foo.com``. + // 2. Suffix domain wildcards: ``*.foo.com`` or ``*-bar.foo.com``. + // 3. Prefix domain wildcards: ``foo.*`` or ``foo-*``. + // 4. Special wildcard ``*`` matching any domain. + // + // .. note:: + // + // The wildcard will not match the empty string. + // e.g. ``*-bar.foo.com`` will match ``baz-bar.foo.com`` but not ``-bar.foo.com``. + // The longest wildcards match first. + // Only a single virtual host in the entire route configuration can match on ``*``. A domain + // must be unique across all virtual hosts or the config will fail to load. + repeated string domains = 2 [(validate.rules).repeated .min_items = 1]; + + // The list of routes that will be matched, in order, for incoming requests. + // The first route that matches will be used. + repeated Route routes = 3; + + enum TlsRequirementType { + // No TLS requirement for the virtual host. + NONE = 0; + + // External requests must use TLS. If a request is external and it is not + // using TLS, a 301 redirect will be sent telling the client to use HTTPS. + EXTERNAL_ONLY = 1; + + // All requests must use TLS. If a request is not using TLS, a 301 redirect + // will be sent telling the client to use HTTPS. + ALL = 2; + } + + // Specifies the type of TLS enforcement the virtual host expects. If this option is not + // specified, there is no TLS requirement for the virtual host. + TlsRequirementType require_tls = 4; + + // A list of virtual clusters defined for this virtual host. Virtual clusters + // are used for additional statistics gathering. + repeated VirtualCluster virtual_clusters = 5; + + // Specifies a set of rate limit configurations that will be applied to the + // virtual host. + repeated RateLimit rate_limits = 6; + + // Specifies a list of HTTP headers that should be added to each request + // handled by this virtual host. Headers specified at this level are applied + // after headers from enclosed :ref:`envoy_api_msg_route.Route` and before headers from the + // enclosing :ref:`envoy_api_msg_RouteConfiguration`. For more information, including + // details on header value syntax, see the documentation on :ref:`custom request headers + // `. + repeated core.HeaderValueOption request_headers_to_add = 7 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of HTTP headers that should be removed from each request + // handled by this virtual host. + repeated string request_headers_to_remove = 13; + + // Specifies a list of HTTP headers that should be added to each response + // handled by this virtual host. Headers specified at this level are applied + // after headers from enclosed :ref:`envoy_api_msg_route.Route` and before headers from the + // enclosing :ref:`envoy_api_msg_RouteConfiguration`. For more information, including + // details on header value syntax, see the documentation on :ref:`custom request headers + // `. + repeated core.HeaderValueOption response_headers_to_add = 10 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of HTTP headers that should be removed from each response + // handled by this virtual host. + repeated string response_headers_to_remove = 11; + + // Indicates that the virtual host has a CORS policy. + CorsPolicy cors = 8; + + reserved 9; + + // The per_filter_config field can be used to provide virtual host-specific + // configurations for filters. The key should match the filter name, such as + // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter + // specific; see the :ref:`HTTP filter documentation ` + // for if and how it is utilized. + map per_filter_config = 12; + + // The per_filter_config field can be used to provide virtual host-specific + // configurations for filters. The key should match the filter name, such as + // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter + // specific; see the :ref:`HTTP filter documentation ` + // for if and how it is utilized. + map typed_per_filter_config = 15; + + // Decides whether the :ref:`x-envoy-attempt-count + // ` header should be included + // in the upstream request. Setting this option will cause it to override any existing header + // value, so in the case of two Envoys on the request path with this option enabled, the upstream + // will see the attempt count as perceived by the second Envoy. Defaults to false. + // This header is unaffected by the + // :ref:`suppress_envoy_headers + // ` flag. + bool include_request_attempt_count = 14; + + // Indicates the retry policy for all routes in this virtual host. Note that setting a + // route level entry will take precedence over this config and it'll be treated + // independently (e.g.: values are not inherited). + RetryPolicy retry_policy = 16; + + // Indicates the hedge policy for all routes in this virtual host. Note that setting a + // route level entry will take precedence over this config and it'll be treated + // independently (e.g.: values are not inherited). + HedgePolicy hedge_policy = 17; +} + +// A route is both a specification of how to match a request as well as an indication of what to do +// next (e.g., redirect, forward, rewrite, etc.). +// +// .. attention:: +// +// Envoy supports routing on HTTP method via :ref:`header matching +// `. +// [#comment:next free field: 15] +message Route { + // Name for the route. + string name = 14; + + // Route matching parameters. + RouteMatch match = 1 [(validate.rules).message.required = true]; + + oneof action { + option (validate.required) = true; + + // Route request to some upstream cluster. + RouteAction route = 2; + + // Return a redirect. + RedirectAction redirect = 3; + + // Return an arbitrary HTTP response directly, without proxying. + DirectResponseAction direct_response = 7; + } + + // The Metadata field can be used to provide additional information + // about the route. It can be used for configuration, stats, and logging. + // The metadata should go under the filter namespace that will need it. + // For instance, if the metadata is intended for the Router filter, + // the filter name should be specified as *envoy.router*. + core.Metadata metadata = 4; + + // Decorator for the matched route. + Decorator decorator = 5; + + reserved 6; + + // The per_filter_config field can be used to provide route-specific + // configurations for filters. The key should match the filter name, such as + // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter + // specific; see the :ref:`HTTP filter documentation ` for + // if and how it is utilized. + map per_filter_config = 8; + + // The per_filter_config field can be used to provide route-specific + // configurations for filters. The key should match the filter name, such as + // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter + // specific; see the :ref:`HTTP filter documentation ` for + // if and how it is utilized. + map typed_per_filter_config = 13; + + // Specifies a set of headers that will be added to requests matching this + // route. Headers specified at this level are applied before headers from the + // enclosing :ref:`envoy_api_msg_route.VirtualHost` and + // :ref:`envoy_api_msg_RouteConfiguration`. For more information, including details on + // header value syntax, see the documentation on :ref:`custom request headers + // `. + repeated core.HeaderValueOption request_headers_to_add = 9 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of HTTP headers that should be removed from each request + // matching this route. + repeated string request_headers_to_remove = 12; + + // Specifies a set of headers that will be added to responses to requests + // matching this route. Headers specified at this level are applied before + // headers from the enclosing :ref:`envoy_api_msg_route.VirtualHost` and + // :ref:`envoy_api_msg_RouteConfiguration`. For more information, including + // details on header value syntax, see the documentation on + // :ref:`custom request headers `. + repeated core.HeaderValueOption response_headers_to_add = 10 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of HTTP headers that should be removed from each response + // to requests matching this route. + repeated string response_headers_to_remove = 11; + + // Presence of the object defines whether the connection manager's tracing configuration + // is overridden by this route specific instance. + Tracing tracing = 15; +} + +// Compared to the :ref:`cluster ` field that specifies a +// single upstream cluster as the target of a request, the :ref:`weighted_clusters +// ` option allows for specification of +// multiple upstream clusters along with weights that indicate the percentage of +// traffic to be forwarded to each cluster. The router selects an upstream cluster based on the +// weights. +// [#comment:next free field: 11] +message WeightedCluster { + message ClusterWeight { + // Name of the upstream cluster. The cluster must exist in the + // :ref:`cluster manager configuration `. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // An integer between 0 and :ref:`total_weight + // `. When a request matches the route, + // the choice of an upstream cluster is determined by its weight. The sum of weights across all + // entries in the clusters array must add up to the total_weight, which defaults to 100. + google.protobuf.UInt32Value weight = 2; + + // Optional endpoint metadata match criteria used by the subset load balancer. Only endpoints in + // the upstream cluster with metadata matching what is set in this field will be considered for + // load balancing. Note that this will be merged with what's provided in :ref: + // `RouteAction.MetadataMatch `, with values + // here taking precedence. The filter name should be specified as *envoy.lb*. + core.Metadata metadata_match = 3; + + // Specifies a list of headers to be added to requests when this cluster is selected + // through the enclosing :ref:`envoy_api_msg_route.RouteAction`. + // Headers specified at this level are applied before headers from the enclosing + // :ref:`envoy_api_msg_route.Route`, :ref:`envoy_api_msg_route.VirtualHost`, and + // :ref:`envoy_api_msg_RouteConfiguration`. For more information, including details on + // header value syntax, see the documentation on :ref:`custom request headers + // `. + repeated core.HeaderValueOption request_headers_to_add = 4 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of HTTP headers that should be removed from each request when + // this cluster is selected through the enclosing :ref:`envoy_api_msg_route.RouteAction`. + repeated string request_headers_to_remove = 9; + + // Specifies a list of headers to be added to responses when this cluster is selected + // through the enclosing :ref:`envoy_api_msg_route.RouteAction`. + // Headers specified at this level are applied before headers from the enclosing + // :ref:`envoy_api_msg_route.Route`, :ref:`envoy_api_msg_route.VirtualHost`, and + // :ref:`envoy_api_msg_RouteConfiguration`. For more information, including details on + // header value syntax, see the documentation on :ref:`custom request headers + // `. + repeated core.HeaderValueOption response_headers_to_add = 5 + [(validate.rules).repeated .max_items = 1000]; + + // Specifies a list of headers to be removed from responses when this cluster is selected + // through the enclosing :ref:`envoy_api_msg_route.RouteAction`. + repeated string response_headers_to_remove = 6; + + reserved 7; + + // The per_filter_config field can be used to provide weighted cluster-specific + // configurations for filters. The key should match the filter name, such as + // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter + // specific; see the :ref:`HTTP filter documentation ` + // for if and how it is utilized. + map per_filter_config = 8; + + // The per_filter_config field can be used to provide weighted cluster-specific + // configurations for filters. The key should match the filter name, such as + // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter + // specific; see the :ref:`HTTP filter documentation ` + // for if and how it is utilized. + map typed_per_filter_config = 10; + } + + // Specifies one or more upstream clusters associated with the route. + repeated ClusterWeight clusters = 1 [(validate.rules).repeated .min_items = 1]; + + // Specifies the total weight across all clusters. The sum of all cluster weights must equal this + // value, which must be greater than 0. Defaults to 100. + google.protobuf.UInt32Value total_weight = 3 [(validate.rules).uint32.gte = 1]; + + // Specifies the runtime key prefix that should be used to construct the + // runtime keys associated with each cluster. When the *runtime_key_prefix* is + // specified, the router will look for weights associated with each upstream + // cluster under the key *runtime_key_prefix* + "." + *cluster[i].name* where + // *cluster[i]* denotes an entry in the clusters array field. If the runtime + // key for the cluster does not exist, the value specified in the + // configuration file will be used as the default weight. See the :ref:`runtime documentation + // ` for how key names map to the underlying implementation. + string runtime_key_prefix = 2; +} + +message RouteMatch { + oneof path_specifier { + option (validate.required) = true; + + // If specified, the route is a prefix rule meaning that the prefix must + // match the beginning of the *:path* header. + string prefix = 1; + + // If specified, the route is an exact path rule meaning that the path must + // exactly match the *:path* header once the query string is removed. + string path = 2; + + // If specified, the route is a regular expression rule meaning that the + // regex must match the *:path* header once the query string is removed. The entire path + // (without the query string) must match the regex. The rule will not match if only a + // subsequence of the *:path* header matches the regex. The regex grammar is defined `here + // `_. + // + // Examples: + // + // * The regex */b[io]t* matches the path */bit* + // * The regex */b[io]t* matches the path */bot* + // * The regex */b[io]t* does not match the path */bite* + // * The regex */b[io]t* does not match the path */bit/bot* + // + // .. attention:: + // This field has been deprecated in favor of `safe_regex` as it is not safe for use with + // untrusted input in all cases. + string regex = 3 [(validate.rules).string.max_bytes = 1024, deprecated = true]; + + // If specified, the route is a regular expression rule meaning that the + // regex must match the *:path* header once the query string is removed. The entire path + // (without the query string) must match the regex. The rule will not match if only a + // subsequence of the *:path* header matches the regex. + // + // [#next-major-version: In the v3 API we should redo how path specification works such + // that we utilize StringMatcher, and additionally have consistent options around whether we + // strip query strings, do a case sensitive match, etc. In the interim it will be too disruptive + // to deprecate the existing options. We should even consider whether we want to do away with + // path_specifier entirely and just rely on a set of header matchers which can already match + // on :path, etc. The issue with that is it is unclear how to generically deal with query string + // stripping. This needs more thought.] + type.matcher.RegexMatcher safe_regex = 10 [(validate.rules).message.required = true]; + } + + // Indicates that prefix/path matching should be case insensitive. The default + // is true. + google.protobuf.BoolValue case_sensitive = 4; + + reserved 5; + + // Indicates that the route should additionally match on a runtime key. Every time the route + // is considered for a match, it must also fall under the percentage of matches indicated by + // this field. For some fraction N/D, a random number in the range [0,D) is selected. If the + // number is <= the value of the numerator N, or if the key is not present, the default + // value, the router continues to evaluate the remaining match criteria. A runtime_fraction + // route configuration can be used to roll out route changes in a gradual manner without full + // code/config deploys. Refer to the :ref:`traffic shifting + // ` docs for additional documentation. + // + // .. note:: + // + // Parsing this field is implemented such that the runtime key's data may be represented + // as a FractionalPercent proto represented as JSON/YAML and may also be represented as an + // integer with the assumption that the value is an integral percentage out of 100. For + // instance, a runtime key lookup returning the value "42" would parse as a FractionalPercent + // whose numerator is 42 and denominator is HUNDRED. This preserves legacy semantics. + core.RuntimeFractionalPercent runtime_fraction = 9; + + // Specifies a set of headers that the route should match on. The router will + // check the request’s headers against all the specified headers in the route + // config. A match will happen if all the headers in the route are present in + // the request with the same values (or based on presence if the value field + // is not in the config). + repeated HeaderMatcher headers = 6; + + // Specifies a set of URL query parameters on which the route should + // match. The router will check the query string from the *path* header + // against all the specified query parameters. If the number of specified + // query parameters is nonzero, they all must match the *path* header's + // query string for a match to occur. + repeated QueryParameterMatcher query_parameters = 7; + + message GrpcRouteMatchOptions { + } + + // If specified, only gRPC requests will be matched. The router will check + // that the content-type header has a application/grpc or one of the various + // application/grpc+ values. + GrpcRouteMatchOptions grpc = 8; +} + +// [#comment:next free field: 11] +message CorsPolicy { + // Specifies the origins that will be allowed to do CORS requests. + // + // An origin is allowed if either allow_origin or allow_origin_regex match. + // + // .. attention:: + // This field has been deprecated in favor of `allow_origin_string_match`. + repeated string allow_origin = 1 [deprecated = true]; + + // Specifies regex patterns that match allowed origins. + // + // An origin is allowed if either allow_origin or allow_origin_regex match. + // + // .. attention:: + // This field has been deprecated in favor of `allow_origin_string_match` as it is not safe for + // use with untrusted input in all cases. + repeated string allow_origin_regex = 8 + [(validate.rules).repeated .items.string.max_bytes = 1024, deprecated = true]; + + // Specifies string patterns that match allowed origins. An origin is allowed if any of the + // string matchers match. + repeated type.matcher.StringMatcher allow_origin_string_match = 11; + + // Specifies the content for the *access-control-allow-methods* header. + string allow_methods = 2; + + // Specifies the content for the *access-control-allow-headers* header. + string allow_headers = 3; + + // Specifies the content for the *access-control-expose-headers* header. + string expose_headers = 4; + + // Specifies the content for the *access-control-max-age* header. + string max_age = 5; + + // Specifies whether the resource allows credentials. + google.protobuf.BoolValue allow_credentials = 6; + + oneof enabled_specifier { + // Specifies if CORS is enabled. Defaults to true. Only effective on route. + // + // .. attention:: + // + // **This field is deprecated**. Set the + // :ref:`filter_enabled` field instead. + google.protobuf.BoolValue enabled = 7 [deprecated = true]; + + // Specifies if CORS is enabled. + // + // More information on how this can be controlled via runtime can be found + // :ref:`here `. + // + // .. note:: + // + // This field defaults to 100/:ref:`HUNDRED + // `. + core.RuntimeFractionalPercent filter_enabled = 9; + } + + // Specifies if CORS policies are evaluated and tracked when filter is off but + // does not enforce any policies. + // + // More information on how this can be controlled via runtime can be found + // :ref:`here `. + // + // .. note:: + // + // This field defaults to 100/:ref:`HUNDRED + // `. + core.RuntimeFractionalPercent shadow_enabled = 10; +} + +// [#comment:next free field: 30] +message RouteAction { + oneof cluster_specifier { + option (validate.required) = true; + + // Indicates the upstream cluster to which the request should be routed + // to. + string cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // Envoy will determine the cluster to route to by reading the value of the + // HTTP header named by cluster_header from the request headers. If the + // header is not found or the referenced cluster does not exist, Envoy will + // return a 404 response. + // + // .. attention:: + // + // Internally, Envoy always uses the HTTP/2 *:authority* header to represent the HTTP/1 + // *Host* header. Thus, if attempting to match on *Host*, match on *:authority* instead. + string cluster_header = 2 [(validate.rules).string.min_bytes = 1]; + + // Multiple upstream clusters can be specified for a given route. The + // request is routed to one of the upstream clusters based on weights + // assigned to each cluster. See + // :ref:`traffic splitting ` + // for additional documentation. + WeightedCluster weighted_clusters = 3; + } + + enum ClusterNotFoundResponseCode { + // HTTP status code - 503 Service Unavailable. + SERVICE_UNAVAILABLE = 0; + + // HTTP status code - 404 Not Found. + NOT_FOUND = 1; + } + + // The HTTP status code to use when configured cluster is not found. + // The default response code is 503 Service Unavailable. + ClusterNotFoundResponseCode cluster_not_found_response_code = 20 + [(validate.rules).enum.defined_only = true]; + + // Optional endpoint metadata match criteria used by the subset load balancer. Only endpoints + // in the upstream cluster with metadata matching what's set in this field will be considered + // for load balancing. If using :ref:`weighted_clusters + // `, metadata will be merged, with values + // provided there taking precedence. The filter name should be specified as *envoy.lb*. + core.Metadata metadata_match = 4; + + // Indicates that during forwarding, the matched prefix (or path) should be + // swapped with this value. This option allows application URLs to be rooted + // at a different path from those exposed at the reverse proxy layer. The router filter will + // place the original path before rewrite into the :ref:`x-envoy-original-path + // ` header. + // + // .. attention:: + // + // Pay careful attention to the use of trailing slashes in the + // :ref:`route's match ` prefix value. + // Stripping a prefix from a path requires multiple Routes to handle all cases. For example, + // rewriting */prefix* to */* and */prefix/etc* to */etc* cannot be done in a single + // :ref:`Route `, as shown by the below config entries: + // + // .. code-block:: yaml + // + // - match: + // prefix: "/prefix/" + // route: + // prefix_rewrite: "/" + // - match: + // prefix: "/prefix" + // route: + // prefix_rewrite: "/" + // + // Having above entries in the config, requests to */prefix* will be stripped to */*, while + // requests to */prefix/etc* will be stripped to */etc*. + string prefix_rewrite = 5; + + oneof host_rewrite_specifier { + // Indicates that during forwarding, the host header will be swapped with + // this value. + string host_rewrite = 6; + + // Indicates that during forwarding, the host header will be swapped with + // the hostname of the upstream host chosen by the cluster manager. This + // option is applicable only when the destination cluster for a route is of + // type *strict_dns* or *logical_dns*. Setting this to true with other cluster + // types has no effect. + google.protobuf.BoolValue auto_host_rewrite = 7; + + // Indicates that during forwarding, the host header will be swapped with the content of given + // downstream or :ref:`custom ` header. + // If header value is empty, host header is left intact. + // + // .. attention:: + // + // Pay attention to the potential security implications of using this option. Provided header + // must come from trusted source. + string auto_host_rewrite_header = 29; + } + + // Specifies the upstream timeout for the route. If not specified, the default is 15s. This + // spans between the point at which the entire downstream request (i.e. end-of-stream) has been + // processed and when the upstream response has been completely processed. A value of 0 will + // disable the route's timeout. + // + // .. note:: + // + // This timeout includes all retries. See also + // :ref:`config_http_filters_router_x-envoy-upstream-rq-timeout-ms`, + // :ref:`config_http_filters_router_x-envoy-upstream-rq-per-try-timeout-ms`, and the + // :ref:`retry overview `. + google.protobuf.Duration timeout = 8 [(gogoproto.stdduration) = true]; + + // Specifies the idle timeout for the route. If not specified, there is no per-route idle timeout, + // although the connection manager wide :ref:`stream_idle_timeout + // ` + // will still apply. A value of 0 will completely disable the route's idle timeout, even if a + // connection manager stream idle timeout is configured. + // + // The idle timeout is distinct to :ref:`timeout + // `, which provides an upper bound + // on the upstream response time; :ref:`idle_timeout + // ` instead bounds the amount + // of time the request's stream may be idle. + // + // After header decoding, the idle timeout will apply on downstream and + // upstream request events. Each time an encode/decode event for headers or + // data is processed for the stream, the timer will be reset. If the timeout + // fires, the stream is terminated with a 408 Request Timeout error code if no + // upstream response header has been received, otherwise a stream reset + // occurs. + google.protobuf.Duration idle_timeout = 24 [(gogoproto.stdduration) = true]; + + // Indicates that the route has a retry policy. Note that if this is set, + // it'll take precedence over the virtual host level retry policy entirely + // (e.g.: policies are not merged, most internal one becomes the enforced policy). + RetryPolicy retry_policy = 9; + + // The router is capable of shadowing traffic from one cluster to another. The current + // implementation is "fire and forget," meaning Envoy will not wait for the shadow cluster to + // respond before returning the response from the primary cluster. All normal statistics are + // collected for the shadow cluster making this feature useful for testing. + // + // During shadowing, the host/authority header is altered such that *-shadow* is appended. This is + // useful for logging. For example, *cluster1* becomes *cluster1-shadow*. + message RequestMirrorPolicy { + // Specifies the cluster that requests will be mirrored to. The cluster must + // exist in the cluster manager configuration. + string cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // If not specified, all requests to the target cluster will be mirrored. If + // specified, Envoy will lookup the runtime key to get the % of requests to + // mirror. Valid values are from 0 to 10000, allowing for increments of + // 0.01% of requests to be mirrored. If the runtime key is specified in the + // configuration but not present in runtime, 0 is the default and thus 0% of + // requests will be mirrored. + // + // .. attention:: + // + // **This field is deprecated**. Set the + // :ref:`runtime_fraction + // ` field instead. + string runtime_key = 2 [deprecated = true]; + + // If both :ref:`runtime_key + // ` and this field are not + // specified, all requests to the target cluster will be mirrored. + // + // If specified, this field takes precedence over the `runtime_key` field and requests must also + // fall under the percentage of matches indicated by this field. + // + // For some fraction N/D, a random number in the range [0,D) is selected. If the + // number is <= the value of the numerator N, or if the key is not present, the default + // value, the request will be mirrored. + // + // .. note:: + // + // Parsing this field is implemented such that the runtime key's data may be represented + // as a :ref:`FractionalPercent ` proto represented + // as JSON/YAML and may also be represented as an integer with the assumption that the value + // is an integral percentage out of 100. For instance, a runtime key lookup returning the + // value "42" would parse as a `FractionalPercent` whose numerator is 42 and denominator is + // HUNDRED. This is behaviour is different to that of the deprecated `runtime_key` field, + // where the implicit denominator is 10000. + core.RuntimeFractionalPercent runtime_fraction = 3; + } + + // Indicates that the route has a request mirroring policy. + RequestMirrorPolicy request_mirror_policy = 10; + + // Optionally specifies the :ref:`routing priority `. + // [#comment:TODO(htuch): add (validate.rules).enum.defined_only = true once + // https://github.com/lyft/protoc-gen-validate/issues/42 is resolved.] + core.RoutingPriority priority = 11; + + reserved 12; + reserved 18; + reserved 19; + + // Specifies a set of rate limit configurations that could be applied to the + // route. + repeated RateLimit rate_limits = 13; + + // Specifies if the rate limit filter should include the virtual host rate + // limits. By default, if the route configured rate limits, the virtual host + // :ref:`rate_limits ` are not applied to the + // request. + google.protobuf.BoolValue include_vh_rate_limits = 14; + + // Specifies the route's hashing policy if the upstream cluster uses a hashing :ref:`load balancer + // `. + message HashPolicy { + message Header { + // The name of the request header that will be used to obtain the hash + // key. If the request header is not present, no hash will be produced. + string header_name = 1 [(validate.rules).string.min_bytes = 1]; + } + + // Envoy supports two types of cookie affinity: + // + // 1. Passive. Envoy takes a cookie that's present in the cookies header and + // hashes on its value. + // + // 2. Generated. Envoy generates and sets a cookie with an expiration (TTL) + // on the first request from the client in its response to the client, + // based on the endpoint the request gets sent to. The client then + // presents this on the next and all subsequent requests. The hash of + // this is sufficient to ensure these requests get sent to the same + // endpoint. The cookie is generated by hashing the source and + // destination ports and addresses so that multiple independent HTTP2 + // streams on the same connection will independently receive the same + // cookie, even if they arrive at the Envoy simultaneously. + message Cookie { + // The name of the cookie that will be used to obtain the hash key. If the + // cookie is not present and ttl below is not set, no hash will be + // produced. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // If specified, a cookie with the TTL will be generated if the cookie is + // not present. If the TTL is present and zero, the generated cookie will + // be a session cookie. + google.protobuf.Duration ttl = 2 [(gogoproto.stdduration) = true]; + + // The name of the path for the cookie. If no path is specified here, no path + // will be set for the cookie. + string path = 3; + } + + message ConnectionProperties { + // Hash on source IP address. + bool source_ip = 1; + } + + oneof policy_specifier { + option (validate.required) = true; + + // Header hash policy. + Header header = 1; + + // Cookie hash policy. + Cookie cookie = 2; + + // Connection properties hash policy. + ConnectionProperties connection_properties = 3; + } + + // The flag that shortcircuits the hash computing. This field provides a + // 'fallback' style of configuration: "if a terminal policy doesn't work, + // fallback to rest of the policy list", it saves time when the terminal + // policy works. + // + // If true, and there is already a hash computed, ignore rest of the + // list of hash polices. + // For example, if the following hash methods are configured: + // + // ========= ======== + // specifier terminal + // ========= ======== + // Header A true + // Header B false + // Header C false + // ========= ======== + // + // The generateHash process ends if policy "header A" generates a hash, as + // it's a terminal policy. + bool terminal = 4; + } + + // Specifies a list of hash policies to use for ring hash load balancing. Each + // hash policy is evaluated individually and the combined result is used to + // route the request. The method of combination is deterministic such that + // identical lists of hash policies will produce the same hash. Since a hash + // policy examines specific parts of a request, it can fail to produce a hash + // (i.e. if the hashed header is not present). If (and only if) all configured + // hash policies fail to generate a hash, no hash will be produced for + // the route. In this case, the behavior is the same as if no hash policies + // were specified (i.e. the ring hash load balancer will choose a random + // backend). If a hash policy has the "terminal" attribute set to true, and + // there is already a hash generated, the hash is returned immediately, + // ignoring the rest of the hash policy list. + repeated HashPolicy hash_policy = 15; + + reserved 16; + reserved 22; + + // Indicates that the route has a CORS policy. + CorsPolicy cors = 17; + + reserved 21; + + // If present, and the request is a gRPC request, use the + // `grpc-timeout header `_, + // or its default value (infinity) instead of + // :ref:`timeout `, but limit the applied timeout + // to the maximum value specified here. If configured as 0, the maximum allowed timeout for + // gRPC requests is infinity. If not configured at all, the `grpc-timeout` header is not used + // and gRPC requests time out like any other requests using + // :ref:`timeout ` or its default. + // This can be used to prevent unexpected upstream request timeouts due to potentially long + // time gaps between gRPC request and response in gRPC streaming mode. + google.protobuf.Duration max_grpc_timeout = 23 [(gogoproto.stdduration) = true]; + + // If present, Envoy will adjust the timeout provided by the `grpc-timeout` header by subtracting + // the provided duration from the header. This is useful in allowing Envoy to set its global + // timeout to be less than that of the deadline imposed by the calling client, which makes it more + // likely that Envoy will handle the timeout instead of having the call canceled by the client. + // The offset will only be applied if the provided grpc_timeout is greater than the offset. This + // ensures that the offset will only ever decrease the timeout and never set it to 0 (meaning + // infinity). + google.protobuf.Duration grpc_timeout_offset = 28 [(gogoproto.stdduration) = true]; + + // Allows enabling and disabling upgrades on a per-route basis. + // This overrides any enabled/disabled upgrade filter chain specified in the + // HttpConnectionManager + // :ref:upgrade_configs` + // ` + // but does not affect any custom filter chain specified there. + message UpgradeConfig { + // The case-insensitive name of this upgrade, e.g. "websocket". + // For each upgrade type present in upgrade_configs, requests with + // Upgrade: [upgrade_type] will be proxied upstream. + string upgrade_type = 1; + // Determines if upgrades are available on this route. Defaults to true. + google.protobuf.BoolValue enabled = 2; + }; + repeated UpgradeConfig upgrade_configs = 25; + + // Configures :ref:`internal redirect ` behavior. + enum InternalRedirectAction { + PASS_THROUGH_INTERNAL_REDIRECT = 0; + HANDLE_INTERNAL_REDIRECT = 1; + } + InternalRedirectAction internal_redirect_action = 26; + + // Indicates that the route has a hedge policy. Note that if this is set, + // it'll take precedence over the virtual host level hedge policy entirely + // (e.g.: policies are not merged, most internal one becomes the enforced policy). + HedgePolicy hedge_policy = 27; +} + +// HTTP retry :ref:`architecture overview `. +// [#comment:next free field: 9] +message RetryPolicy { + // Specifies the conditions under which retry takes place. These are the same + // conditions documented for :ref:`config_http_filters_router_x-envoy-retry-on` and + // :ref:`config_http_filters_router_x-envoy-retry-grpc-on`. + string retry_on = 1; + + // Specifies the allowed number of retries. This parameter is optional and + // defaults to 1. These are the same conditions documented for + // :ref:`config_http_filters_router_x-envoy-max-retries`. + google.protobuf.UInt32Value num_retries = 2; + + // Specifies a non-zero upstream timeout per retry attempt. This parameter is optional. The + // same conditions documented for + // :ref:`config_http_filters_router_x-envoy-upstream-rq-per-try-timeout-ms` apply. + // + // .. note:: + // + // If left unspecified, Envoy will use the global + // :ref:`route timeout ` for the request. + // Consequently, when using a :ref:`5xx ` based + // retry policy, a request that times out will not be retried as the total timeout budget + // would have been exhausted. + google.protobuf.Duration per_try_timeout = 3 [(gogoproto.stdduration) = true]; + + message RetryPriority { + string name = 1 [(validate.rules).string.min_bytes = 1]; + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } + } + + // Specifies an implementation of a RetryPriority which is used to determine the + // distribution of load across priorities used for retries. Refer to + // :ref:`retry plugin configuration ` for more details. + RetryPriority retry_priority = 4; + + message RetryHostPredicate { + string name = 1 [(validate.rules).string.min_bytes = 1]; + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } + } + + // Specifies a collection of RetryHostPredicates that will be consulted when selecting a host + // for retries. If any of the predicates reject the host, host selection will be reattempted. + // Refer to :ref:`retry plugin configuration ` for more + // details. + repeated RetryHostPredicate retry_host_predicate = 5; + + // The maximum number of times host selection will be reattempted before giving up, at which + // point the host that was last selected will be routed to. If unspecified, this will default to + // retrying once. + int64 host_selection_retry_max_attempts = 6; + + // HTTP status codes that should trigger a retry in addition to those specified by retry_on. + repeated uint32 retriable_status_codes = 7; + + message RetryBackOff { + // Specifies the base interval between retries. This parameter is required and must be greater + // than zero. Values less than 1 ms are rounded up to 1 ms. + // See :ref:`config_http_filters_router_x-envoy-max-retries` for a discussion of Envoy's + // back-off algorithm. + google.protobuf.Duration base_interval = 1 [ + (validate.rules).duration = { + required: true, + gt: {seconds: 0} + }, + (gogoproto.stdduration) = true + ]; + + // Specifies the maximum interval between retries. This parameter is optional, but must be + // greater than or equal to the `base_interval` if set. The default is 10 times the + // `base_interval`. See :ref:`config_http_filters_router_x-envoy-max-retries` for a discussion + // of Envoy's back-off algorithm. + google.protobuf.Duration max_interval = 2 + [(validate.rules).duration.gt = {seconds: 0}, (gogoproto.stdduration) = true]; + } + + // Specifies parameters that control retry back off. This parameter is optional, in which case the + // default base interval is 25 milliseconds or, if set, the current value of the + // `upstream.base_retry_backoff_ms` runtime parameter. The default maximum interval is 10 times + // the base interval. The documentation for :ref:`config_http_filters_router_x-envoy-max-retries` + // describes Envoy's back-off algorithm. + RetryBackOff retry_back_off = 8; +} + +// HTTP request hedging :ref:`architecture overview `. +message HedgePolicy { + // Specifies the number of initial requests that should be sent upstream. + // Must be at least 1. + // Defaults to 1. + // [#not-implemented-hide:] + google.protobuf.UInt32Value initial_requests = 1 [(validate.rules).uint32.gte = 1]; + + // Specifies a probability that an additional upstream request should be sent + // on top of what is specified by initial_requests. + // Defaults to 0. + // [#not-implemented-hide:] + envoy.type.FractionalPercent additional_request_chance = 2; + + // Indicates that a hedged request should be sent when the per-try timeout + // is hit. This will only occur if the retry policy also indicates that a + // timed out request should be retried. + // Once a timed out request is retried due to per try timeout, the router + // filter will ensure that it is not retried again even if the returned + // response headers would otherwise be retried according the specified + // :ref:`RetryPolicy `. + // Defaults to false. + bool hedge_on_per_try_timeout = 3; +} + +message RedirectAction { + // When the scheme redirection take place, the following rules apply: + // 1. If the source URI scheme is `http` and the port is explicitly + // set to `:80`, the port will be removed after the redirection + // 2. If the source URI scheme is `https` and the port is explicitly + // set to `:443`, the port will be removed after the redirection + oneof scheme_rewrite_specifier { + // The scheme portion of the URL will be swapped with "https". + bool https_redirect = 4; + // The scheme portion of the URL will be swapped with this value. + string scheme_redirect = 7; + } + // The host portion of the URL will be swapped with this value. + string host_redirect = 1; + // The port value of the URL will be swapped with this value. + uint32 port_redirect = 8; + + oneof path_rewrite_specifier { + // The path portion of the URL will be swapped with this value. + string path_redirect = 2; + + // Indicates that during redirection, the matched prefix (or path) + // should be swapped with this value. This option allows redirect URLs be dynamically created + // based on the request. + // + // .. attention:: + // + // Pay attention to the use of trailing slashes as mentioned in + // :ref:`RouteAction's prefix_rewrite `. + string prefix_rewrite = 5; + } + + enum RedirectResponseCode { + // Moved Permanently HTTP Status Code - 301. + MOVED_PERMANENTLY = 0; + + // Found HTTP Status Code - 302. + FOUND = 1; + + // See Other HTTP Status Code - 303. + SEE_OTHER = 2; + + // Temporary Redirect HTTP Status Code - 307. + TEMPORARY_REDIRECT = 3; + + // Permanent Redirect HTTP Status Code - 308. + PERMANENT_REDIRECT = 4; + } + + // The HTTP status code to use in the redirect response. The default response + // code is MOVED_PERMANENTLY (301). + RedirectResponseCode response_code = 3 [(validate.rules).enum.defined_only = true]; + + // Indicates that during redirection, the query portion of the URL will + // be removed. Default value is false. + bool strip_query = 6; +} + +message DirectResponseAction { + // Specifies the HTTP response status to be returned. + uint32 status = 1 [(validate.rules).uint32 = {gte: 100, lt: 600}]; + + // Specifies the content of the response body. If this setting is omitted, + // no body is included in the generated response. + // + // .. note:: + // + // Headers can be specified using *response_headers_to_add* in the enclosing + // :ref:`envoy_api_msg_route.Route`, :ref:`envoy_api_msg_RouteConfiguration` or + // :ref:`envoy_api_msg_route.VirtualHost`. + core.DataSource body = 2; +} + +message Decorator { + // The operation name associated with the request matched to this route. If tracing is + // enabled, this information will be used as the span name reported for this request. + // + // .. note:: + // + // For ingress (inbound) requests, or egress (outbound) responses, this value may be overridden + // by the :ref:`x-envoy-decorator-operation + // ` header. + string operation = 1 [(validate.rules).string.min_bytes = 1]; +} + +message Tracing { + + // Target percentage of requests managed by this HTTP connection manager that will be force + // traced if the :ref:`x-client-trace-id ` + // header is set. This field is a direct analog for the runtime variable + // 'tracing.client_sampling' in the :ref:`HTTP Connection Manager + // `. + // Default: 100% + envoy.type.FractionalPercent client_sampling = 1; + + // Target percentage of requests managed by this HTTP connection manager that will be randomly + // selected for trace generation, if not requested by the client or not forced. This field is + // a direct analog for the runtime variable 'tracing.random_sampling' in the + // :ref:`HTTP Connection Manager `. + // Default: 100% + envoy.type.FractionalPercent random_sampling = 2; + + // Target percentage of requests managed by this HTTP connection manager that will be traced + // after all other sampling checks have been applied (client-directed, force tracing, random + // sampling). This field functions as an upper limit on the total configured sampling rate. For + // instance, setting client_sampling to 100% but overall_sampling to 1% will result in only 1% + // of client requests with the appropriate headers to be force traced. This field is a direct + // analog for the runtime variable 'tracing.global_enabled' in the + // :ref:`HTTP Connection Manager `. + // Default: 100% + envoy.type.FractionalPercent overall_sampling = 3; +} + +// A virtual cluster is a way of specifying a regex matching rule against +// certain important endpoints such that statistics are generated explicitly for +// the matched requests. The reason this is useful is that when doing +// prefix/path matching Envoy does not always know what the application +// considers to be an endpoint. Thus, it’s impossible for Envoy to generically +// emit per endpoint statistics. However, often systems have highly critical +// endpoints that they wish to get “perfect” statistics on. Virtual cluster +// statistics are perfect in the sense that they are emitted on the downstream +// side such that they include network level failures. +// +// Documentation for :ref:`virtual cluster statistics `. +// +// .. note:: +// +// Virtual clusters are a useful tool, but we do not recommend setting up a virtual cluster for +// every application endpoint. This is both not easily maintainable and as well the matching and +// statistics output are not free. +message VirtualCluster { + // Specifies a regex pattern to use for matching requests. The entire path of the request + // must match the regex. The regex grammar used is defined `here + // `_. + // + // Examples: + // + // * The regex */rides/\d+* matches the path */rides/0* + // * The regex */rides/\d+* matches the path */rides/123* + // * The regex */rides/\d+* does not match the path */rides/123/456* + // + // .. attention:: + // This field has been deprecated in favor of `headers` as it is not safe for use with + // untrusted input in all cases. + string pattern = 1 [(validate.rules).string.max_bytes = 1024, deprecated = true]; + + // Specifies a list of header matchers to use for matching requests. Each specified header must + // match. The pseudo-headers `:path` and `:method` can be used to match the request path and + // method, respectively. + repeated HeaderMatcher headers = 4; + + // Specifies the name of the virtual cluster. The virtual cluster name as well + // as the virtual host name are used when emitting statistics. The statistics are emitted by the + // router filter and are documented :ref:`here `. + string name = 2 [(validate.rules).string.min_bytes = 1]; + + // Optionally specifies the HTTP method to match on. For example GET, PUT, + // etc. + // + // .. attention:: + // This field has been deprecated in favor of `headers`. + core.RequestMethod method = 3 [deprecated = true]; +} + +// Global rate limiting :ref:`architecture overview `. +message RateLimit { + // Refers to the stage set in the filter. The rate limit configuration only + // applies to filters with the same stage number. The default stage number is + // 0. + // + // .. note:: + // + // The filter supports a range of 0 - 10 inclusively for stage numbers. + google.protobuf.UInt32Value stage = 1 [(validate.rules).uint32.lte = 10]; + + // The key to be set in runtime to disable this rate limit configuration. + string disable_key = 2; + + message Action { + // The following descriptor entry is appended to the descriptor: + // + // .. code-block:: cpp + // + // ("source_cluster", "") + // + // is derived from the :option:`--service-cluster` option. + message SourceCluster { + } + + // The following descriptor entry is appended to the descriptor: + // + // .. code-block:: cpp + // + // ("destination_cluster", "") + // + // Once a request matches against a route table rule, a routed cluster is determined by one of + // the following :ref:`route table configuration ` + // settings: + // + // * :ref:`cluster ` indicates the upstream cluster + // to route to. + // * :ref:`weighted_clusters ` + // chooses a cluster randomly from a set of clusters with attributed weight. + // * :ref:`cluster_header ` indicates which + // header in the request contains the target cluster. + message DestinationCluster { + } + + // The following descriptor entry is appended when a header contains a key that matches the + // *header_name*: + // + // .. code-block:: cpp + // + // ("", "") + message RequestHeaders { + // The header name to be queried from the request headers. The header’s + // value is used to populate the value of the descriptor entry for the + // descriptor_key. + string header_name = 1 [(validate.rules).string.min_bytes = 1]; + + // The key to use in the descriptor entry. + string descriptor_key = 2 [(validate.rules).string.min_bytes = 1]; + } + + // The following descriptor entry is appended to the descriptor and is populated using the + // trusted address from :ref:`x-forwarded-for `: + // + // .. code-block:: cpp + // + // ("remote_address", "") + message RemoteAddress { + } + + // The following descriptor entry is appended to the descriptor: + // + // .. code-block:: cpp + // + // ("generic_key", "") + message GenericKey { + // The value to use in the descriptor entry. + string descriptor_value = 1 [(validate.rules).string.min_bytes = 1]; + } + + // The following descriptor entry is appended to the descriptor: + // + // .. code-block:: cpp + // + // ("header_match", "") + message HeaderValueMatch { + // The value to use in the descriptor entry. + string descriptor_value = 1 [(validate.rules).string.min_bytes = 1]; + + // If set to true, the action will append a descriptor entry when the + // request matches the headers. If set to false, the action will append a + // descriptor entry when the request does not match the headers. The + // default value is true. + google.protobuf.BoolValue expect_match = 2; + + // Specifies a set of headers that the rate limit action should match + // on. The action will check the request’s headers against all the + // specified headers in the config. A match will happen if all the + // headers in the config are present in the request with the same values + // (or based on presence if the value field is not in the config). + repeated HeaderMatcher headers = 3 [(validate.rules).repeated .min_items = 1]; + } + + oneof action_specifier { + option (validate.required) = true; + + // Rate limit on source cluster. + SourceCluster source_cluster = 1; + + // Rate limit on destination cluster. + DestinationCluster destination_cluster = 2; + + // Rate limit on request headers. + RequestHeaders request_headers = 3; + + // Rate limit on remote address. + RemoteAddress remote_address = 4; + + // Rate limit on a generic key. + GenericKey generic_key = 5; + + // Rate limit on the existence of request headers. + HeaderValueMatch header_value_match = 6; + } + } + + // A list of actions that are to be applied for this rate limit configuration. + // Order matters as the actions are processed sequentially and the descriptor + // is composed by appending descriptor entries in that sequence. If an action + // cannot append a descriptor entry, no descriptor is generated for the + // configuration. See :ref:`composing actions + // ` for additional documentation. + repeated Action actions = 3 [(validate.rules).repeated .min_items = 1]; +} + +// .. attention:: +// +// Internally, Envoy always uses the HTTP/2 *:authority* header to represent the HTTP/1 *Host* +// header. Thus, if attempting to match on *Host*, match on *:authority* instead. +// +// .. attention:: +// +// To route on HTTP method, use the special HTTP/2 *:method* header. This works for both +// HTTP/1 and HTTP/2 as Envoy normalizes headers. E.g., +// +// .. code-block:: json +// +// { +// "name": ":method", +// "exact_match": "POST" +// } +// +// .. attention:: +// In the absence of any header match specifier, match will default to :ref:`present_match +// `. i.e, a request that has the :ref:`name +// ` header will match, regardless of the header's +// value. +// +// [#next-major-version: HeaderMatcher should be refactored to use StringMatcher.] +message HeaderMatcher { + // Specifies the name of the header in the request. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + reserved 2; // value deprecated by :ref:`exact_match + // ` + + reserved 3; // regex deprecated by :ref:`regex_match + // ` + + // Specifies how the header match will be performed to route the request. + oneof header_match_specifier { + // If specified, header match will be performed based on the value of the header. + string exact_match = 4; + + // If specified, this regex string is a regular expression rule which implies the entire request + // header value must match the regex. The rule will not match if only a subsequence of the + // request header value matches the regex. The regex grammar used in the value field is defined + // `here `_. + // + // Examples: + // + // * The regex *\d{3}* matches the value *123* + // * The regex *\d{3}* does not match the value *1234* + // * The regex *\d{3}* does not match the value *123.456* + // + // .. attention:: + // This field has been deprecated in favor of `safe_regex_match` as it is not safe for use + // with untrusted input in all cases. + string regex_match = 5 [(validate.rules).string.max_bytes = 1024, deprecated = true]; + + // If specified, this regex string is a regular expression rule which implies the entire request + // header value must match the regex. The rule will not match if only a subsequence of the + // request header value matches the regex. + type.matcher.RegexMatcher safe_regex_match = 11; + + // If specified, header match will be performed based on range. + // The rule will match if the request header value is within this range. + // The entire request header value must represent an integer in base 10 notation: consisting of + // an optional plus or minus sign followed by a sequence of digits. The rule will not match if + // the header value does not represent an integer. Match will fail for empty values, floating + // point numbers or if only a subsequence of the header value is an integer. + // + // Examples: + // + // * For range [-10,0), route will match for header value -1, but not for 0, "somestring", 10.9, + // "-1somestring" + envoy.type.Int64Range range_match = 6; + + // If specified, header match will be performed based on whether the header is in the + // request. + bool present_match = 7; + + // If specified, header match will be performed based on the prefix of the header value. + // Note: empty prefix is not allowed, please use present_match instead. + // + // Examples: + // + // * The prefix *abcd* matches the value *abcdxyz*, but not for *abcxyz*. + string prefix_match = 9 [(validate.rules).string.min_bytes = 1]; + + // If specified, header match will be performed based on the suffix of the header value. + // Note: empty suffix is not allowed, please use present_match instead. + // + // Examples: + // + // * The suffix *abcd* matches the value *xyzabcd*, but not for *xyzbcd*. + string suffix_match = 10 [(validate.rules).string.min_bytes = 1]; + } + + // If specified, the match result will be inverted before checking. Defaults to false. + // + // Examples: + // + // * The regex *\d{3}* does not match the value *1234*, so it will match when inverted. + // * The range [-10,0) will match the value -1, so it will not match when inverted. + bool invert_match = 8; +} + +// Query parameter matching treats the query string of a request's :path header +// as an ampersand-separated list of keys and/or key=value elements. +message QueryParameterMatcher { + // Specifies the name of a key that must be present in the requested + // *path*'s query string. + string name = 1 [(validate.rules).string = {min_bytes: 1, max_bytes: 1024}]; + + // Specifies the value of the key. If the value is absent, a request + // that contains the key in its query string will match, whether the + // key appears with a value (e.g., "?debug=true") or not (e.g., "?debug") + // + // ..attention:: + // This field is deprecated. Use an `exact` match inside the `string_match` field. + string value = 3 [deprecated = true]; + + // Specifies whether the query parameter value is a regular expression. + // Defaults to false. The entire query parameter value (i.e., the part to + // the right of the equals sign in "key=value") must match the regex. + // E.g., the regex "\d+$" will match "123" but not "a123" or "123a". + // + // ..attention:: + // This field is deprecated. Use a `safe_regex` match inside the `string_match` field. + google.protobuf.BoolValue regex = 4 [deprecated = true]; + + oneof query_parameter_match_specifier { + // Specifies whether a query parameter value should match against a string. + type.matcher.StringMatcher string_match = 5 [(validate.rules).message.required = true]; + + // Specifies whether a query parameter should be present. + bool present_match = 6; + } +} diff --git a/api/envoy/api/v3alpha/srds.proto b/api/envoy/api/v3alpha/srds.proto new file mode 100644 index 000000000000..22ad6e675683 --- /dev/null +++ b/api/envoy/api/v3alpha/srds.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; + +package envoy.api.v3alpha; + +import "envoy/api/v3alpha/discovery.proto"; +import "gogoproto/gogo.proto"; +import "google/api/annotations.proto"; +import "validate/validate.proto"; + +option java_outer_classname = "SrdsProto"; +option java_package = "io.envoyproxy.envoy.api.v3alpha"; +option java_multiple_files = true; +option java_generic_services = true; +option (gogoproto.equal_all) = true; + +// [#protodoc-title: HTTP scoped routing configuration] +// * Routing :ref:`architecture overview ` +// +// The Scoped Routes Discovery Service (SRDS) API distributes +// :ref:`ScopedRouteConfiguration` +// resources. Each ScopedRouteConfiguration resource represents a "routing +// scope" containing a mapping that allows the HTTP connection manager to +// dynamically assign a routing table (specified via a +// :ref:`RouteConfiguration` message) to each +// HTTP request. +// [#proto-status: experimental] +service ScopedRoutesDiscoveryService { + rpc StreamScopedRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { + } + + rpc DeltaScopedRoutes(stream DeltaDiscoveryRequest) returns (stream DeltaDiscoveryResponse) { + } + + rpc FetchScopedRoutes(DiscoveryRequest) returns (DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:scoped-routes" + body: "*" + }; + } +} + +// Specifies a routing scope, which associates a +// :ref:`Key` to a +// :ref:`envoy_api_msg_RouteConfiguration` (identified by its resource name). +// +// The HTTP connection manager builds up a table consisting of these Key to +// RouteConfiguration mappings, and looks up the RouteConfiguration to use per +// request according to the algorithm specified in the +// :ref:`scope_key_builder` +// assigned to the HttpConnectionManager. +// +// For example, with the following configurations (in YAML): +// +// HttpConnectionManager config: +// +// .. code:: +// +// ... +// scoped_routes: +// name: foo-scoped-routes +// scope_key_builder: +// fragments: +// - header_value_extractor: +// name: X-Route-Selector +// element_separator: , +// element: +// separator: = +// key: vip +// +// ScopedRouteConfiguration resources (specified statically via +// :ref:`scoped_route_configurations_list` +// or obtained dynamically via SRDS): +// +// .. code:: +// +// (1) +// name: route-scope1 +// route_configuration_name: route-config1 +// key: +// fragments: +// - string_key: 172.10.10.20 +// +// (2) +// name: route-scope2 +// route_configuration_name: route-config2 +// key: +// fragments: +// - string_key: 172.20.20.30 +// +// A request from a client such as: +// +// .. code:: +// +// GET / HTTP/1.1 +// Host: foo.com +// X-Route-Selector: vip=172.10.10.20 +// +// would result in the routing table defined by the `route-config1` +// RouteConfiguration being assigned to the HTTP request/stream. +// +// [#comment:next free field: 4] +// [#proto-status: experimental] +message ScopedRouteConfiguration { + // The name assigned to the routing scope. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Specifies a key which is matched against the output of the + // :ref:`scope_key_builder` + // specified in the HttpConnectionManager. The matching is done per HTTP + // request and is dependent on the order of the fragments contained in the + // Key. + message Key { + message Fragment { + oneof type { + option (validate.required) = true; + + // A string to match against. + string string_key = 1; + } + } + + // The ordered set of fragments to match against. The order must match the + // fragments in the corresponding + // :ref:`scope_key_builder`. + repeated Fragment fragments = 1 [(validate.rules).repeated .min_items = 1]; + } + + // The resource name to use for a :ref:`envoy_api_msg_DiscoveryRequest` to an + // RDS server to fetch the :ref:`envoy_api_msg_RouteConfiguration` associated + // with this scope. + string route_configuration_name = 2 [(validate.rules).string.min_bytes = 1]; + + // The key to match against. + Key key = 3 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/accesslog/v3alpha/BUILD b/api/envoy/config/accesslog/v3alpha/BUILD new file mode 100644 index 000000000000..4f5da73ee424 --- /dev/null +++ b/api/envoy/config/accesslog/v3alpha/BUILD @@ -0,0 +1,22 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "als", + srcs = ["als.proto"], + deps = [ + "//envoy/api/v3alpha/core:grpc_service", + ], +) + +api_proto_library_internal( + name = "file", + srcs = ["file.proto"], +) + +api_go_proto_library( + name = "als", + proto = ":als", + deps = ["//envoy/api/v3alpha/core:grpc_service_go_proto"], +) diff --git a/api/envoy/config/accesslog/v3alpha/als.proto b/api/envoy/config/accesslog/v3alpha/als.proto new file mode 100644 index 000000000000..a194d1449e4b --- /dev/null +++ b/api/envoy/config/accesslog/v3alpha/als.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package envoy.config.accesslog.v3alpha; + +option java_outer_classname = "AlsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.accesslog.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/grpc_service.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: gRPC Access Log Service (ALS)] + +// Configuration for the built-in *envoy.http_grpc_access_log* +// :ref:`AccessLog `. This configuration +// will populate :ref:`StreamAccessLogsMessage.http_logs +// `. +message HttpGrpcAccessLogConfig { + CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message.required = true]; + + // Additional request headers to log in :ref:`HTTPRequestProperties.request_headers + // `. + repeated string additional_request_headers_to_log = 2; + + // Additional response headers to log in :ref:`HTTPResponseProperties.response_headers + // `. + repeated string additional_response_headers_to_log = 3; + + // Additional response trailers to log in :ref:`HTTPResponseProperties.response_trailers + // `. + repeated string additional_response_trailers_to_log = 4; +} + +// Configuration for the built-in *envoy.tcp_grpc_access_log* type. This configuration will +// populate *StreamAccessLogsMessage.tcp_logs*. +message TcpGrpcAccessLogConfig { + CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message.required = true]; +} + +// Common configuration for gRPC access logs. +message CommonGrpcAccessLogConfig { + // The friendly name of the access log to be returned in :ref:`StreamAccessLogsMessage.Identifier + // `. This allows the + // access log server to differentiate between different access logs coming from the same Envoy. + string log_name = 1 [(validate.rules).string.min_bytes = 1]; + + // The gRPC service for the access log service. + envoy.api.v3alpha.core.GrpcService grpc_service = 2 [(validate.rules).message.required = true]; + + // Interval for flushing access logs to the gRPC stream. Logger will flush requests every time + // this interval is elapsed, or when batch size limit is hit, whichever comes first. Defaults to + // 1 second. + google.protobuf.Duration buffer_flush_interval = 3 [(validate.rules).duration.gt = {}]; + + // Soft size limit in bytes for access log entries buffer. Logger will buffer requests until + // this limit it hit, or every time flush interval is elapsed, whichever comes first. Setting it + // to zero effectively disables the batching. Defaults to 16384. + google.protobuf.UInt32Value buffer_size_bytes = 4; +} diff --git a/api/envoy/config/accesslog/v3alpha/file.proto b/api/envoy/config/accesslog/v3alpha/file.proto new file mode 100644 index 000000000000..b07658bc9275 --- /dev/null +++ b/api/envoy/config/accesslog/v3alpha/file.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package envoy.config.accesslog.v3alpha; + +option java_outer_classname = "FileProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.accesslog.v3alpha"; +option go_package = "v2"; + +import "validate/validate.proto"; +import "google/protobuf/struct.proto"; + +// [#protodoc-title: File access log] + +// Custom configuration for an :ref:`AccessLog +// ` that writes log entries directly to a +// file. Configures the built-in *envoy.file_access_log* AccessLog. +message FileAccessLog { + // A path to a local file to which to write the access log entries. + string path = 1 [(validate.rules).string.min_bytes = 1]; + + // Access log format. Envoy supports :ref:`custom access log formats + // ` as well as a :ref:`default format + // `. + oneof access_log_format { + // Access log :ref:`format string` + string format = 2; + + // Access log :ref:`format dictionary` + google.protobuf.Struct json_format = 3; + } +} diff --git a/api/envoy/config/bootstrap/v3alpha/BUILD b/api/envoy/config/bootstrap/v3alpha/BUILD new file mode 100644 index 000000000000..d148021c741a --- /dev/null +++ b/api/envoy/config/bootstrap/v3alpha/BUILD @@ -0,0 +1,40 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "bootstrap", + srcs = ["bootstrap.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha:cds", + "//envoy/api/v3alpha:lds", + "//envoy/api/v3alpha/auth:cert", + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:config_source", + "//envoy/config/metrics/v3alpha:metrics_service", + "//envoy/config/metrics/v3alpha:stats", + "//envoy/config/overload/v3alpha:overload", + "//envoy/config/ratelimit/v3alpha:rls", + "//envoy/config/trace/v3alpha:trace", + ], +) + +api_go_proto_library( + name = "bootstrap", + proto = ":bootstrap", + deps = [ + "//envoy/api/v3alpha:cds_go_grpc", + "//envoy/api/v3alpha:lds_go_grpc", + "//envoy/api/v3alpha/auth:cert_go_proto", + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:config_source_go_proto", + "//envoy/config/metrics/v3alpha:metrics_service_go_proto", + "//envoy/config/metrics/v3alpha:stats_go_proto", + "//envoy/config/overload/v3alpha:overload_go_proto", + "//envoy/config/ratelimit/v3alpha:rls_go_grpc", + "//envoy/config/trace/v3alpha:trace_go_proto", + ], +) diff --git a/api/envoy/config/bootstrap/v3alpha/bootstrap.proto b/api/envoy/config/bootstrap/v3alpha/bootstrap.proto new file mode 100644 index 000000000000..57157a4ae3f3 --- /dev/null +++ b/api/envoy/config/bootstrap/v3alpha/bootstrap.proto @@ -0,0 +1,318 @@ +// [#protodoc-title: Bootstrap] +// This proto is supplied via the :option:`-c` CLI flag and acts as the root +// of the Envoy v2 configuration. See the :ref:`v2 configuration overview +// ` for more detail. + +syntax = "proto3"; + +package envoy.config.bootstrap.v3alpha; + +option java_outer_classname = "BootstrapProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.bootstrap.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/auth/cert.proto"; +import "envoy/api/v3alpha/core/config_source.proto"; +import "envoy/api/v3alpha/cds.proto"; +import "envoy/api/v3alpha/lds.proto"; +import "envoy/config/trace/v3alpha/trace.proto"; +import "envoy/config/metrics/v3alpha/stats.proto"; +import "envoy/config/overload/v3alpha/overload.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// Bootstrap :ref:`configuration overview `. +message Bootstrap { + // Node identity to present to the management server and for instance + // identification purposes (e.g. in generated headers). + envoy.api.v3alpha.core.Node node = 1; + + message StaticResources { + // Static :ref:`Listeners `. These listeners are + // available regardless of LDS configuration. + repeated envoy.api.v3alpha.Listener listeners = 1; + + // If a network based configuration source is specified for :ref:`cds_config + // `, it's + // necessary to have some initial cluster definitions available to allow Envoy to know how to + // speak to the management server. These cluster definitions may not use :ref:`EDS + // ` (i.e. they should be static IP or DNS-based). + repeated envoy.api.v3alpha.Cluster clusters = 2; + + // These static secrets can be used by :ref:`SdsSecretConfig + // ` + repeated envoy.api.v3alpha.auth.Secret secrets = 3; + } + // Statically specified resources. + StaticResources static_resources = 2; + + message DynamicResources { + // All :ref:`Listeners ` are provided by a single + // :ref:`LDS ` configuration source. + envoy.api.v3alpha.core.ConfigSource lds_config = 1; + + // All post-bootstrap :ref:`Cluster ` definitions are + // provided by a single :ref:`CDS ` + // configuration source. + envoy.api.v3alpha.core.ConfigSource cds_config = 2; + + // A single :ref:`ADS ` source may be optionally + // specified. This must have :ref:`api_type + // ` :ref:`GRPC + // `. Only + // :ref:`ConfigSources ` that have + // the :ref:`ads ` field set will be + // streamed on the ADS channel. + envoy.api.v3alpha.core.ApiConfigSource ads_config = 3; + + reserved 4; + } + // xDS configuration sources. + DynamicResources dynamic_resources = 3; + + // Configuration for the cluster manager which owns all upstream clusters + // within the server. + ClusterManager cluster_manager = 4; + + // Health discovery service config option. + // (:ref:`core.ApiConfigSource `) + envoy.api.v3alpha.core.ApiConfigSource hds_config = 14; + + // Optional file system path to search for startup flag files. + string flags_path = 5; + + // Optional set of stats sinks. + repeated envoy.config.metrics.v3alpha.StatsSink stats_sinks = 6; + + // Configuration for internal processing of stats. + envoy.config.metrics.v3alpha.StatsConfig stats_config = 13; + + // Optional duration between flushes to configured stats sinks. For + // performance reasons Envoy latches counters and only flushes counters and + // gauges at a periodic interval. If not specified the default is 5000ms (5 + // seconds). + // Duration must be at least 1ms and at most 5 min. + google.protobuf.Duration stats_flush_interval = 7 [ + (validate.rules).duration = { + lt: {seconds: 300}, + gte: {nanos: 1000000} + }, + (gogoproto.stdduration) = true + ]; + + // Optional watchdog configuration. + Watchdog watchdog = 8; + + // Configuration for an external tracing provider. If not specified, no + // tracing will be performed. + envoy.config.trace.v3alpha.Tracing tracing = 9; + + reserved 10; + + // Configuration for the runtime configuration provider (deprecated). If not + // specified, a “null” provider will be used which will result in all defaults + // being used. + Runtime runtime = 11 [deprecated = true]; + + // Configuration for the runtime configuration provider. If not + // specified, a “null” provider will be used which will result in all defaults + // being used. + LayeredRuntime layered_runtime = 17; + + // Configuration for the local administration HTTP server. + Admin admin = 12; + + // Optional overload manager configuration. + envoy.config.overload.v3alpha.OverloadManager overload_manager = 15; + + // Enable :ref:`stats for event dispatcher `, defaults to false. + // Note that this records a value for each iteration of the event loop on every thread. This + // should normally be minimal overhead, but when using + // :ref:`statsd `, it will send each observed + // value over the wire individually because the statsd protocol doesn't have any way to represent + // a histogram summary. Be aware that this can be a very large volume of data. + bool enable_dispatcher_stats = 16; + + // Optional string which will be used in lieu of x-envoy in prefixing headers. + // + // For example, if this string is present and set to X-Foo, then x-envoy-retry-on will be + // transformed into x-foo-retry-on etc. + // + // Note this applies to the headers Envoy will generate, the headers Envoy will sanitize, and the + // headers Envoy will trust for core code and core extensions only. Be VERY careful making + // changes to this string, especially in multi-layer Envoy deployments or deployments using + // extensions which are not upstream. + string header_prefix = 18; +} + +// Administration interface :ref:`operations documentation +// `. +message Admin { + // The path to write the access log for the administration server. If no + // access log is desired specify ‘/dev/null’. This is only required if + // :ref:`address ` is set. + string access_log_path = 1; + + // The cpu profiler output path for the administration server. If no profile + // path is specified, the default is ‘/var/log/envoy/envoy.prof’. + string profile_path = 2; + + // The TCP address that the administration server will listen on. + // If not specified, Envoy will not start an administration server. + envoy.api.v3alpha.core.Address address = 3; + + // Additional socket options that may not be present in Envoy source code or + // precompiled binaries. + repeated envoy.api.v3alpha.core.SocketOption socket_options = 4; +} + +// Cluster manager :ref:`architecture overview `. +message ClusterManager { + // Name of the local cluster (i.e., the cluster that owns the Envoy running + // this configuration). In order to enable :ref:`zone aware routing + // ` this option must be set. + // If *local_cluster_name* is defined then :ref:`clusters + // ` must be defined in the :ref:`Bootstrap + // static cluster resources + // `. This is + // unrelated to the :option:`--service-cluster` option which does not `affect zone aware routing + // `_. + string local_cluster_name = 1; + + message OutlierDetection { + // Specifies the path to the outlier event log. + string event_log_path = 1; + } + // Optional global configuration for outlier detection. + OutlierDetection outlier_detection = 2; + + // Optional configuration used to bind newly established upstream connections. + // This may be overridden on a per-cluster basis by upstream_bind_config in the cds_config. + envoy.api.v3alpha.core.BindConfig upstream_bind_config = 3; + + // A management server endpoint to stream load stats to via + // *StreamLoadStats*. This must have :ref:`api_type + // ` :ref:`GRPC + // `. + envoy.api.v3alpha.core.ApiConfigSource load_stats_config = 4; +} + +// Envoy process watchdog configuration. When configured, this monitors for +// nonresponsive threads and kills the process after the configured thresholds. +message Watchdog { + // The duration after which Envoy counts a nonresponsive thread in the + // *server.watchdog_miss* statistic. If not specified the default is 200ms. + google.protobuf.Duration miss_timeout = 1; + + // The duration after which Envoy counts a nonresponsive thread in the + // *server.watchdog_mega_miss* statistic. If not specified the default is + // 1000ms. + google.protobuf.Duration megamiss_timeout = 2; + + // If a watched thread has been nonresponsive for this duration, assume a + // programming error and kill the entire Envoy process. Set to 0 to disable + // kill behavior. If not specified the default is 0 (disabled). + google.protobuf.Duration kill_timeout = 3; + + // If at least two watched threads have been nonresponsive for at least this + // duration assume a true deadlock and kill the entire Envoy process. Set to 0 + // to disable this behavior. If not specified the default is 0 (disabled). + google.protobuf.Duration multikill_timeout = 4; +} + +// Runtime :ref:`configuration overview ` (deprecated). +message Runtime { + // The implementation assumes that the file system tree is accessed via a + // symbolic link. An atomic link swap is used when a new tree should be + // switched to. This parameter specifies the path to the symbolic link. Envoy + // will watch the location for changes and reload the file system tree when + // they happen. If this parameter is not set, there will be no disk based + // runtime. + string symlink_root = 1; + + // Specifies the subdirectory to load within the root directory. This is + // useful if multiple systems share the same delivery mechanism. Envoy + // configuration elements can be contained in a dedicated subdirectory. + string subdirectory = 2; + + // Specifies an optional subdirectory to load within the root directory. If + // specified and the directory exists, configuration values within this + // directory will override those found in the primary subdirectory. This is + // useful when Envoy is deployed across many different types of servers. + // Sometimes it is useful to have a per service cluster directory for runtime + // configuration. See below for exactly how the override directory is used. + string override_subdirectory = 3; + + // Static base runtime. This will be :ref:`overridden + // ` by other runtime layers, e.g. + // disk or admin. This follows the :ref:`runtime protobuf JSON representation + // encoding `. + google.protobuf.Struct base = 4; +} + +message RuntimeLayer { + // :ref:`Disk runtime ` layer. + message DiskLayer { + // The implementation assumes that the file system tree is accessed via a + // symbolic link. An atomic link swap is used when a new tree should be + // switched to. This parameter specifies the path to the symbolic link. + // Envoy will watch the location for changes and reload the file system tree + // when they happen. See documentation on runtime :ref:`atomicity + // ` for further details on how reloads are + // treated. + string symlink_root = 1; + + // Specifies the subdirectory to load within the root directory. This is + // useful if multiple systems share the same delivery mechanism. Envoy + // configuration elements can be contained in a dedicated subdirectory. + string subdirectory = 3; + + // :ref:`Append ` the + // service cluster to the path under symlink root. + bool append_service_cluster = 2; + } + + // :ref:`Admin console runtime ` layer. + message AdminLayer { + } + + // :ref:`Runtime Discovery Service (RTDS) ` layer. + message RtdsLayer { + // Resource to subscribe to at *rtds_config* for the RTDS layer. + string name = 1; + + // RTDS configuration source. + envoy.api.v3alpha.core.ConfigSource rtds_config = 2; + } + + // Descriptive name for the runtime layer. This is only used for the runtime + // :http:get:`/runtime` output. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + oneof layer_specifier { + // :ref:`Static runtime ` layer. + // This follows the :ref:`runtime protobuf JSON representation encoding + // `. Unlike static xDS resources, this static + // layer is overridable by later layers in the runtime virtual filesystem. + option (validate.required) = true; + + google.protobuf.Struct static_layer = 2; + DiskLayer disk_layer = 3; + AdminLayer admin_layer = 4; + RtdsLayer rtds_layer = 5; + } +} + +// Runtime :ref:`configuration overview `. +message LayeredRuntime { + // The :ref:`layers ` of the runtime. This is ordered + // such that later layers in the list overlay earlier entries. + repeated RuntimeLayer layers = 1; +} diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/BUILD b/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..50d0aa2354eb --- /dev/null +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "cluster", + srcs = ["cluster.proto"], + deps = [ + "//envoy/config/common/dynamic_forward_proxy/v3alpha:dns_cache", + ], +) diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/cluster.proto b/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/cluster.proto new file mode 100644 index 000000000000..baed68d3b1ac --- /dev/null +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/cluster.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.config.cluster.dynamic_forward_proxy.v3alpha; + +option java_outer_classname = "DynamicForwardProxyClusterProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.cluster.dynamic_forward_proxy.v3alpha"; +option go_package = "v2alpha"; + +import "envoy/config/common/dynamic_forward_proxy/v3alpha/dns_cache.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Dynamic forward proxy cluster configuration] + +// Configuration for the dynamic forward proxy cluster. See the :ref:`architecture overview +// ` for more information. +message ClusterConfig { + // The DNS cache configuration that the cluster will attach to. Note this configuration must + // match that of associated :ref:`dynamic forward proxy HTTP filter configuration + // `. + common.dynamic_forward_proxy.v3alpha.DnsCacheConfig dns_cache_config = 1 + [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/common/dynamic_forward_proxy/v3alpha/BUILD b/api/envoy/config/common/dynamic_forward_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..bdd23e86de9f --- /dev/null +++ b/api/envoy/config/common/dynamic_forward_proxy/v3alpha/BUILD @@ -0,0 +1,12 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "dns_cache", + srcs = ["dns_cache.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha:cds", + ], +) diff --git a/api/envoy/config/common/dynamic_forward_proxy/v3alpha/dns_cache.proto b/api/envoy/config/common/dynamic_forward_proxy/v3alpha/dns_cache.proto new file mode 100644 index 000000000000..7b8a67be4333 --- /dev/null +++ b/api/envoy/config/common/dynamic_forward_proxy/v3alpha/dns_cache.proto @@ -0,0 +1,69 @@ +syntax = "proto3"; + +package envoy.config.common.dynamic_forward_proxy.v3alpha; + +option java_outer_classname = "DnsCacheProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.common.dynamic_forward_proxy.v3alpha"; + +import "envoy/api/v3alpha/cds.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Dynamic forward proxy common configuration] + +// Configuration for the dynamic forward proxy DNS cache. See the :ref:`architecture overview +// ` for more information. +message DnsCacheConfig { + // The name of the cache. Multiple named caches allow independent dynamic forward proxy + // configurations to operate within a single Envoy process using different configurations. All + // configurations with the same name *must* otherwise have the same settings when referenced + // from different configuration components. Configuration will fail to load if this is not + // the case. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // The DNS lookup family to use during resolution. + // + // [#comment:TODO(mattklein123): Figure out how to support IPv4/IPv6 "happy eyeballs" mode. The + // way this might work is a new lookup family which returns both IPv4 and IPv6 addresses, and + // then configures a host to have a primary and fall back address. With this, we could very + // likely build a "happy eyeballs" connection pool which would race the primary / fall back + // address and return the one that wins. This same method could potentially also be used for + // QUIC to TCP fall back.] + api.v3alpha.Cluster.DnsLookupFamily dns_lookup_family = 2 + [(validate.rules).enum.defined_only = true]; + + // The DNS refresh rate for currently cached DNS hosts. If not specified defaults to 60s. + // + // .. note: + // + // The returned DNS TTL is not currently used to alter the refresh rate. This feature will be + // added in a future change. + google.protobuf.Duration dns_refresh_rate = 3 [(validate.rules).duration.gt = {}]; + + // The TTL for hosts that are unused. Hosts that have not been used in the configured time + // interval will be purged. If not specified defaults to 5m. + // + // .. note: + // + // The TTL is only checked at the time of DNS refresh, as specified by *dns_refresh_rate*. This + // means that if the configured TTL is shorter than the refresh rate the host may not be removed + // immediately. + // + // .. note: + // + // The TTL has no relation to DNS TTL and is only used to control Envoy's resource usage. + google.protobuf.Duration host_ttl = 4 [(validate.rules).duration.gt = {}]; + + // The maximum number of hosts that the cache will hold. If not specified defaults to 1024. + // + // .. note: + // + // The implementation is approximate and enforced independently on each worker thread, thus + // it is possible for the maximum hosts in the cache to go slightly above the configured + // value depending on timing. This is similar to how other circuit breakers work. + google.protobuf.UInt32Value max_hosts = 5 [(validate.rules).uint32.gt = 0]; +} diff --git a/api/envoy/config/common/tap/v3alpha/BUILD b/api/envoy/config/common/tap/v3alpha/BUILD new file mode 100644 index 000000000000..673a602800af --- /dev/null +++ b/api/envoy/config/common/tap/v3alpha/BUILD @@ -0,0 +1,13 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "common", + srcs = ["common.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha/core:config_source", + "//envoy/service/tap/v3alpha:common", + ], +) diff --git a/api/envoy/config/common/tap/v3alpha/common.proto b/api/envoy/config/common/tap/v3alpha/common.proto new file mode 100644 index 000000000000..4e5debddbc51 --- /dev/null +++ b/api/envoy/config/common/tap/v3alpha/common.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +import "envoy/service/tap/v3alpha/common.proto"; +import "envoy/api/v3alpha/core/config_source.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +package envoy.config.common.tap.v3alpha; + +option java_outer_classname = "CommonProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.common.tap.v3alpha"; + +// [#protodoc-title: Common tap extension configuration] + +// Common configuration for all tap extensions. +message CommonExtensionConfig { + + // [#not-implemented-hide:] + message TapDSConfig { + // Configuration for the source of TapDS updates for this Cluster. + envoy.api.v3alpha.core.ConfigSource config_source = 1 + [(validate.rules).message.required = true]; + + // Tap config to request from XDS server. + string name = 2 [(validate.rules).string.min_bytes = 1]; + } + + oneof config_type { + option (validate.required) = true; + + // If specified, the tap filter will be configured via an admin handler. + AdminConfig admin_config = 1; + + // If specified, the tap filter will be configured via a static configuration that cannot be + // changed. + service.tap.v3alpha.TapConfig static_config = 2; + + // [#not-implemented-hide:] Configuration to use for TapDS updates for the filter. + TapDSConfig tapds_config = 3; + } +} + +// Configuration for the admin handler. See :ref:`here ` for +// more information. +message AdminConfig { + // Opaque configuration ID. When requests are made to the admin handler, the passed opaque ID is + // matched to the configured filter opaque ID to determine which filter to configure. + string config_id = 1 [(validate.rules).string.min_bytes = 1]; +} diff --git a/api/envoy/config/filter/accesslog/v3alpha/BUILD b/api/envoy/config/filter/accesslog/v3alpha/BUILD new file mode 100644 index 000000000000..3f241bc5e10b --- /dev/null +++ b/api/envoy/config/filter/accesslog/v3alpha/BUILD @@ -0,0 +1,28 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "accesslog", + srcs = ["accesslog.proto"], + visibility = [ + "//envoy/config/filter/http/router/v3alpha:__pkg__", + "//envoy/config/filter/network/http_connection_manager/v3alpha:__pkg__", + "//envoy/config/filter/network/tcp_proxy/v3alpha:__pkg__", + ], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/route", + "//envoy/type:percent", + ], +) + +api_go_proto_library( + name = "accesslog", + proto = ":accesslog", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/route:route_go_proto", + "//envoy/type:percent_go_proto", + ], +) diff --git a/api/envoy/config/filter/accesslog/v3alpha/accesslog.proto b/api/envoy/config/filter/accesslog/v3alpha/accesslog.proto new file mode 100644 index 000000000000..381e7bdf9a87 --- /dev/null +++ b/api/envoy/config/filter/accesslog/v3alpha/accesslog.proto @@ -0,0 +1,248 @@ +syntax = "proto3"; + +package envoy.config.filter.accesslog.v3alpha; + +option java_outer_classname = "AccesslogProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.accesslog.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/route/route.proto"; +import "envoy/type/percent.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Common access log types] + +message AccessLog { + // The name of the access log implementation to instantiate. The name must + // match a statically registered access log. Current built-in loggers include: + // + // #. "envoy.file_access_log" + // #. "envoy.http_grpc_access_log" + // #. "envoy.tcp_grpc_access_log" + string name = 1; + + // Filter which is used to determine if the access log needs to be written. + AccessLogFilter filter = 2; + + // Custom configuration that depends on the access log being instantiated. Built-in + // configurations include: + // + // #. "envoy.file_access_log": :ref:`FileAccessLog + // ` + // #. "envoy.http_grpc_access_log": :ref:`HttpGrpcAccessLogConfig + // ` + // #. "envoy.tcp_grpc_access_log": :ref:`TcpGrpcAccessLogConfig + // ` + oneof config_type { + google.protobuf.Struct config = 3; + + google.protobuf.Any typed_config = 4; + } +} + +message AccessLogFilter { + oneof filter_specifier { + option (validate.required) = true; + + // Status code filter. + StatusCodeFilter status_code_filter = 1; + + // Duration filter. + DurationFilter duration_filter = 2; + + // Not health check filter. + NotHealthCheckFilter not_health_check_filter = 3; + + // Traceable filter. + TraceableFilter traceable_filter = 4; + + // Runtime filter. + RuntimeFilter runtime_filter = 5; + + // And filter. + AndFilter and_filter = 6; + + // Or filter. + OrFilter or_filter = 7; + + // Header filter. + HeaderFilter header_filter = 8; + + // Response flag filter. + ResponseFlagFilter response_flag_filter = 9; + + // gRPC status filter. + GrpcStatusFilter grpc_status_filter = 10; + + // Extension filter. + ExtensionFilter extension_filter = 11; + } +} + +// Filter on an integer comparison. +message ComparisonFilter { + enum Op { + // = + EQ = 0; + + // >= + GE = 1; + + // <= + LE = 2; + } + + // Comparison operator. + Op op = 1 [(validate.rules).enum.defined_only = true]; + + // Value to compare against. + envoy.api.v3alpha.core.RuntimeUInt32 value = 2; +} + +// Filters on HTTP response/status code. +message StatusCodeFilter { + // Comparison. + ComparisonFilter comparison = 1 [(validate.rules).message.required = true]; +} + +// Filters on total request duration in milliseconds. +message DurationFilter { + // Comparison. + ComparisonFilter comparison = 1 [(validate.rules).message.required = true]; +} + +// Filters for requests that are not health check requests. A health check +// request is marked by the health check filter. +message NotHealthCheckFilter { +} + +// Filters for requests that are traceable. See the tracing overview for more +// information on how a request becomes traceable. +message TraceableFilter { +} + +// Filters for random sampling of requests. +message RuntimeFilter { + // Runtime key to get an optional overridden numerator for use in the *percent_sampled* field. + // If found in runtime, this value will replace the default numerator. + string runtime_key = 1 [(validate.rules).string.min_bytes = 1]; + + // The default sampling percentage. If not specified, defaults to 0% with denominator of 100. + envoy.type.FractionalPercent percent_sampled = 2; + + // By default, sampling pivots on the header + // :ref:`x-request-id` being present. If + // :ref:`x-request-id` is present, the filter will + // consistently sample across multiple hosts based on the runtime key value and the value + // extracted from :ref:`x-request-id`. If it is + // missing, or *use_independent_randomness* is set to true, the filter will randomly sample based + // on the runtime key value alone. *use_independent_randomness* can be used for logging kill + // switches within complex nested :ref:`AndFilter + // ` and :ref:`OrFilter + // ` blocks that are easier to reason + // about from a probability perspective (i.e., setting to true will cause the filter to behave + // like an independent random variable when composed within logical operator filters). + bool use_independent_randomness = 3; +} + +// Performs a logical “and” operation on the result of each filter in filters. +// Filters are evaluated sequentially and if one of them returns false, the +// filter returns false immediately. +message AndFilter { + repeated AccessLogFilter filters = 1 [(validate.rules).repeated .min_items = 2]; +} + +// Performs a logical “or” operation on the result of each individual filter. +// Filters are evaluated sequentially and if one of them returns true, the +// filter returns true immediately. +message OrFilter { + repeated AccessLogFilter filters = 2 [(validate.rules).repeated .min_items = 2]; +} + +// Filters requests based on the presence or value of a request header. +message HeaderFilter { + // Only requests with a header which matches the specified HeaderMatcher will pass the filter + // check. + envoy.api.v3alpha.route.HeaderMatcher header = 1 [(validate.rules).message.required = true]; +} + +// Filters requests that received responses with an Envoy response flag set. +// A list of the response flags can be found +// in the access log formatter :ref:`documentation`. +message ResponseFlagFilter { + // Only responses with the any of the flags listed in this field will be logged. + // This field is optional. If it is not specified, then any response flag will pass + // the filter check. + repeated string flags = 1 [(validate.rules).repeated .items.string = { + in: [ + "LH", + "UH", + "UT", + "LR", + "UR", + "UF", + "UC", + "UO", + "NR", + "DI", + "FI", + "RL", + "UAEX", + "RLSE", + "DC", + "URX", + "SI", + "IH" + ] + }]; +} + +// Filters gRPC requests based on their response status. If a gRPC status is not provided, the +// filter will infer the status from the HTTP status code. +message GrpcStatusFilter { + enum Status { + OK = 0; + CANCELED = 1; + UNKNOWN = 2; + INVALID_ARGUMENT = 3; + DEADLINE_EXCEEDED = 4; + NOT_FOUND = 5; + ALREADY_EXISTS = 6; + PERMISSION_DENIED = 7; + RESOURCE_EXHAUSTED = 8; + FAILED_PRECONDITION = 9; + ABORTED = 10; + OUT_OF_RANGE = 11; + UNIMPLEMENTED = 12; + INTERNAL = 13; + UNAVAILABLE = 14; + DATA_LOSS = 15; + UNAUTHENTICATED = 16; + } + + // Logs only responses that have any one of the gRPC statuses in this field. + repeated Status statuses = 1 [(validate.rules).repeated .items.enum.defined_only = true]; + + // If included and set to true, the filter will instead block all responses with a gRPC status or + // inferred gRPC status enumerated in statuses, and allow all other responses. + bool exclude = 2; +} + +// Extension filter is statically registered at runtime. +message ExtensionFilter { + // The name of the filter implementation to instantiate. The name must + // match a statically registered filter. + string name = 1; + + // Custom configuration that depends on the filter being instantiated. + oneof config_type { + google.protobuf.Struct config = 2; + google.protobuf.Any typed_config = 3; + } +} diff --git a/api/envoy/config/filter/fault/v3alpha/BUILD b/api/envoy/config/filter/fault/v3alpha/BUILD new file mode 100644 index 000000000000..22e3bec56ca3 --- /dev/null +++ b/api/envoy/config/filter/fault/v3alpha/BUILD @@ -0,0 +1,13 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "fault", + srcs = ["fault.proto"], + visibility = [ + "//envoy/config/filter/http/fault/v3alpha:__pkg__", + "//envoy/config/filter/network/mongo_proxy/v3alpha:__pkg__", + ], + deps = ["//envoy/type:percent"], +) diff --git a/api/envoy/config/filter/fault/v3alpha/fault.proto b/api/envoy/config/filter/fault/v3alpha/fault.proto new file mode 100644 index 000000000000..b54a063e7665 --- /dev/null +++ b/api/envoy/config/filter/fault/v3alpha/fault.proto @@ -0,0 +1,84 @@ +syntax = "proto3"; + +package envoy.config.filter.fault.v3alpha; + +option java_outer_classname = "FaultProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.fault.v3alpha"; +option go_package = "v2"; + +import "envoy/type/percent.proto"; + +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Common fault injection types] + +// Delay specification is used to inject latency into the +// HTTP/gRPC/Mongo/Redis operation or delay proxying of TCP connections. +message FaultDelay { + // Fault delays are controlled via an HTTP header (if applicable). See the + // :ref:`http fault filter ` documentation for + // more information. + message HeaderDelay { + } + + enum FaultDelayType { + // Unused and deprecated. + FIXED = 0; + } + + // Unused and deprecated. Will be removed in the next release. + FaultDelayType type = 1 [deprecated = true]; + + reserved 2; + + oneof fault_delay_secifier { + option (validate.required) = true; + + // Add a fixed delay before forwarding the operation upstream. See + // https://developers.google.com/protocol-buffers/docs/proto3#json for + // the JSON/YAML Duration mapping. For HTTP/Mongo/Redis, the specified + // delay will be injected before a new request/operation. For TCP + // connections, the proxying of the connection upstream will be delayed + // for the specified period. This is required if type is FIXED. + google.protobuf.Duration fixed_delay = 3 + [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + + // Fault delays are controlled via an HTTP header (if applicable). + HeaderDelay header_delay = 5; + } + + // The percentage of operations/connections/requests on which the delay will be injected. + type.FractionalPercent percentage = 4; +} + +// Describes a rate limit to be applied. +message FaultRateLimit { + // Describes a fixed/constant rate limit. + message FixedLimit { + // The limit supplied in KiB/s. + uint64 limit_kbps = 1 [(validate.rules).uint64.gte = 1]; + } + + // Rate limits are controlled via an HTTP header (if applicable). See the + // :ref:`http fault filter ` documentation for + // more information. + message HeaderLimit { + } + + oneof limit_type { + option (validate.required) = true; + + // A fixed rate limit. + FixedLimit fixed_limit = 1; + + // Rate limits are controlled via an HTTP header (if applicable). + HeaderLimit header_limit = 3; + } + + // The percentage of operations/connections/requests on which the rate limit will be injected. + type.FractionalPercent percentage = 2; +} diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD new file mode 100644 index 000000000000..aa2b0634739c --- /dev/null +++ b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "adaptive_concurrency", + srcs = ["adaptive_concurrency.proto"], + deps = [ + "//envoy/api/v3alpha/core:base", + ], +) diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto new file mode 100644 index 000000000000..17bac55800ce --- /dev/null +++ b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package envoy.config.filter.http.adaptive_concurrency.v3alpha; + +option java_package = "io.envoyproxy.envoy.config.filter.http.adaptive_concurrency.v3alpha"; +option java_outer_classname = "AdaptiveConcurrencyProto"; +option java_multiple_files = true; +option go_package = "v2alpha"; + +message AdaptiveConcurrency { +} diff --git a/api/envoy/config/filter/http/buffer/v3alpha/BUILD b/api/envoy/config/filter/http/buffer/v3alpha/BUILD new file mode 100644 index 000000000000..e59429af9ace --- /dev/null +++ b/api/envoy/config/filter/http/buffer/v3alpha/BUILD @@ -0,0 +1,8 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "buffer", + srcs = ["buffer.proto"], +) diff --git a/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto b/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto new file mode 100644 index 000000000000..a948493b2450 --- /dev/null +++ b/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package envoy.config.filter.http.buffer.v3alpha; + +option java_outer_classname = "BufferProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.buffer.v3alpha"; +option go_package = "v2"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Buffer] +// Buffer :ref:`configuration overview `. + +message Buffer { + reserved 2; // formerly max_request_time + + // The maximum request size that the filter will buffer before the connection + // manager will stop buffering and return a 413 response. + google.protobuf.UInt32Value max_request_bytes = 1 [(validate.rules).uint32.gt = 0]; +} + +message BufferPerRoute { + oneof override { + option (validate.required) = true; + + // Disable the buffer filter for this particular vhost or route. + bool disabled = 1 [(validate.rules).bool.const = true]; + + // Override the global configuration of the filter with this new config. + Buffer buffer = 2 [(validate.rules).message.required = true]; + } +} diff --git a/api/envoy/config/filter/http/csrf/v3alpha/BUILD b/api/envoy/config/filter/http/csrf/v3alpha/BUILD new file mode 100644 index 000000000000..b5da684c54b4 --- /dev/null +++ b/api/envoy/config/filter/http/csrf/v3alpha/BUILD @@ -0,0 +1,12 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "csrf", + srcs = ["csrf.proto"], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/type/matcher:string", + ], +) diff --git a/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto b/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto new file mode 100644 index 000000000000..5eaa14c567d7 --- /dev/null +++ b/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +package envoy.config.filter.http.csrf.v3alpha; + +option java_outer_classname = "CsrfPolicyProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.csrf.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/type/matcher/string.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: CSRF] +// Cross-Site Request Forgery :ref:`configuration overview `. + +// CSRF filter config. +message CsrfPolicy { + // Specifies if CSRF is enabled. + // + // More information on how this can be controlled via runtime can be found + // :ref:`here `. + // + // .. note:: + // + // This field defaults to 100/:ref:`HUNDRED + // `. + envoy.api.v3alpha.core.RuntimeFractionalPercent filter_enabled = 1 + [(validate.rules).message.required = true]; + + // Specifies that CSRF policies will be evaluated and tracked, but not enforced. + // This is intended to be used when filter_enabled is off. + // + // More information on how this can be controlled via runtime can be found + // :ref:`here `. + // + // .. note:: + // + // This field defaults to 100/:ref:`HUNDRED + // `. + envoy.api.v3alpha.core.RuntimeFractionalPercent shadow_enabled = 2; + + // Specifies additional source origins that will be allowed in addition to + // the destination origin. + // + // More information on how this can be configured via runtime can be found + // :ref:`here `. + repeated envoy.type.matcher.StringMatcher additional_origins = 3; +} diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/BUILD b/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..f09166ba8129 --- /dev/null +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "dynamic_forward_proxy", + srcs = ["dynamic_forward_proxy.proto"], + deps = [ + "//envoy/config/common/dynamic_forward_proxy/v3alpha:dns_cache", + ], +) diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/dynamic_forward_proxy.proto b/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/dynamic_forward_proxy.proto new file mode 100644 index 000000000000..0fab44d63db5 --- /dev/null +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/dynamic_forward_proxy.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.config.filter.http.dynamic_forward_proxy.v3alpha; + +option java_outer_classname = "DynamicForwardProxyProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.dynamic_forward_proxy.v3alpha"; +option go_package = "v2alpha"; + +import "envoy/config/common/dynamic_forward_proxy/v3alpha/dns_cache.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Dynamic forward proxy] + +// Configuration for the dynamic forward proxy HTTP filter. See the :ref:`architecture overview +// ` for more information. +message FilterConfig { + // The DNS cache configuration that the filter will attach to. Note this configuration must + // match that of associated :ref:`dynamic forward proxy cluster configuration + // `. + common.dynamic_forward_proxy.v3alpha.DnsCacheConfig dns_cache_config = 1 + [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/filter/http/ext_authz/v3alpha/BUILD b/api/envoy/config/filter/http/ext_authz/v3alpha/BUILD new file mode 100644 index 000000000000..39f9e44bb382 --- /dev/null +++ b/api/envoy/config/filter/http/ext_authz/v3alpha/BUILD @@ -0,0 +1,15 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "ext_authz", + srcs = ["ext_authz.proto"], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:grpc_service", + "//envoy/api/v3alpha/core:http_uri", + "//envoy/type:http_status", + "//envoy/type/matcher:string", + ], +) diff --git a/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto new file mode 100644 index 000000000000..af6f3c4866e5 --- /dev/null +++ b/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto @@ -0,0 +1,209 @@ +syntax = "proto3"; + +package envoy.config.filter.http.ext_authz.v3alpha; + +option java_outer_classname = "ExtAuthzProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.ext_authz.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/core/grpc_service.proto"; +import "envoy/api/v3alpha/core/http_uri.proto"; + +import "envoy/type/http_status.proto"; +import "envoy/type/matcher/string.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: External Authorization] +// External Authorization :ref:`configuration overview `. + +message ExtAuthz { + // External authorization service configuration. + oneof services { + // gRPC service configuration (default timeout: 200ms). + envoy.api.v3alpha.core.GrpcService grpc_service = 1; + + // HTTP service configuration (default timeout: 200ms). + HttpService http_service = 3; + } + + // Changes filter's behaviour on errors: + // + // 1. When set to true, the filter will *accept* client request even if the communication with + // the authorization service has failed, or if the authorization service has returned a HTTP 5xx + // error. + // + // 2. When set to false, ext-authz will *reject* client requests and return a *Forbidden* + // response if the communication with the authorization service has failed, or if the + // authorization service has returned a HTTP 5xx error. + // + // Note that errors can be *always* tracked in the :ref:`stats + // `. + bool failure_mode_allow = 2; + + // Sets the package version the gRPC service should use. This is particularly + // useful when transitioning from alpha to release versions assuming that both definitions are + // semantically compatible. Deprecation note: This field is deprecated and should only be used for + // version upgrade. See release notes for more details. + bool use_alpha = 4 [deprecated = true]; + + // Enables filter to buffer the client request body and send it within the authorization request. + // A ``x-envoy-auth-partial-body: false|true`` metadata header will be added to the authorization + // request message indicating if the body data is partial. + BufferSettings with_request_body = 5; + + // Clears route cache in order to allow the external authorization service to correctly affect + // routing decisions. Filter clears all cached routes when: + // + // 1. The field is set to *true*. + // + // 2. The status returned from the authorization service is a HTTP 200 or gRPC 0. + // + // 3. At least one *authorization response header* is added to the client request, or is used for + // altering another client request header. + // + bool clear_route_cache = 6; + + // Sets the HTTP status that is returned to the client when there is a network error between the + // filter and the authorization server. The default status is HTTP 403 Forbidden. + envoy.type.HttpStatus status_on_error = 7; + + // Specifies a list of metadata namespaces whose values, if present, will be passed to the + // ext_authz service as an opaque *protobuf::Struct*. + // + // For example, if the *jwt_authn* filter is used and :ref:`payload_in_metadata + // ` is set, + // then the following will pass the jwt payload to the authorization server. + // + // .. code-block:: yaml + // + // metadata_context_namespaces: + // - envoy.filters.http.jwt_authn + // + repeated string metadata_context_namespaces = 8; +} + +// Configuration for buffering the request data. +message BufferSettings { + // Sets the maximum size of a message body that the filter will hold in memory. Envoy will return + // *HTTP 413* and will *not* initiate the authorization process when buffer reaches the number + // set in this field. Note that this setting will have precedence over :ref:`failure_mode_allow + // `. + uint32 max_request_bytes = 1 [(validate.rules).uint32.gt = 0]; + + // When this field is true, Envoy will buffer the message until *max_request_bytes* is reached. + // The authorization request will be dispatched and no 413 HTTP error will be returned by the + // filter. + bool allow_partial_message = 2; +} + +// HttpService is used for raw HTTP communication between the filter and the authorization service. +// When configured, the filter will parse the client request and use these attributes to call the +// authorization server. Depending on the response, the filter may reject or accept the client +// request. Note that in any of these events, metadata can be added, removed or overridden by the +// filter: +// +// *On authorization request*, a list of allowed request headers may be supplied. See +// :ref:`allowed_headers +// ` +// for details. Additional headers metadata may be added to the authorization request. See +// :ref:`headers_to_add +// ` for +// details. +// +// On authorization response status HTTP 200 OK, the filter will allow traffic to the upstream and +// additional headers metadata may be added to the original client request. See +// :ref:`allowed_upstream_headers +// ` +// for details. +// +// On other authorization response statuses, the filter will not allow traffic. Additional headers +// metadata as well as body may be added to the client's response. See :ref:`allowed_client_headers +// ` +// for details. +message HttpService { + // Sets the HTTP server URI which the authorization requests must be sent to. + envoy.api.v3alpha.core.HttpUri server_uri = 1; + + // Sets a prefix to the value of authorization request header *Path*. + string path_prefix = 2; + + reserved 3; + reserved 4; + reserved 5; + reserved 6; + + // Settings used for controlling authorization request metadata. + AuthorizationRequest authorization_request = 7; + + // Settings used for controlling authorization response metadata. + AuthorizationResponse authorization_response = 8; +} + +message AuthorizationRequest { + // Authorization request will include the client request headers that have a correspondent match + // in the :ref:`list `. Note that in addition to the + // user's supplied matchers: + // + // 1. *Host*, *Method*, *Path* and *Content-Length* are automatically included to the list. + // + // 2. *Content-Length* will be set to 0 and the request to the authorization service will not have + // a message body. + // + envoy.type.matcher.ListStringMatcher allowed_headers = 1; + + // Sets a list of headers that will be included to the request to authorization service. Note that + // client request of the same key will be overridden. + repeated envoy.api.v3alpha.core.HeaderValue headers_to_add = 2; +} + +message AuthorizationResponse { + // When this :ref:`list ` is set, authorization + // response headers that have a correspondent match will be added to the original client request. + // Note that coexistent headers will be overridden. + envoy.type.matcher.ListStringMatcher allowed_upstream_headers = 1; + + // When this :ref:`list `. is set, authorization + // response headers that have a correspondent match will be added to the client's response. Note + // that when this list is *not* set, all the authorization response headers, except *Authority + // (Host)* will be in the response to the client. When a header is included in this list, *Path*, + // *Status*, *Content-Length*, *WWWAuthenticate* and *Location* are automatically added. + envoy.type.matcher.ListStringMatcher allowed_client_headers = 2; +} + +// Extra settings on a per virtualhost/route/weighted-cluster level. +message ExtAuthzPerRoute { + oneof override { + option (validate.required) = true; + + // Disable the ext auth filter for this particular vhost or route. + // If disabled is specified in multiple per-filter-configs, the most specific one will be used. + bool disabled = 1 [(validate.rules).bool.const = true]; + + // Check request settings for this route. + CheckSettings check_settings = 2 [(validate.rules).message.required = true]; + } +} + +// Extra settings for the check request. You can use this to provide extra context for the +// external authorization server on specific virtual hosts \ routes. For example, adding a context +// extension on the virtual host level can give the ext-authz server information on what virtual +// host is used without needing to parse the host header. If CheckSettings is specified in multiple +// per-filter-configs, they will be merged in order, and the result will be used. +message CheckSettings { + // Context extensions to set on the CheckRequest's + // :ref:`AttributeContext.context_extensions` + // + // Merge semantics for this field are such that keys from more specific configs override. + // + // .. note:: + // + // These settings are only applied to a filter configured with a + // :ref:`grpc_service`. + map context_extensions = 1; +} diff --git a/api/envoy/config/filter/http/fault/v3alpha/BUILD b/api/envoy/config/filter/http/fault/v3alpha/BUILD new file mode 100644 index 000000000000..1fd5632c088d --- /dev/null +++ b/api/envoy/config/filter/http/fault/v3alpha/BUILD @@ -0,0 +1,13 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "fault", + srcs = ["fault.proto"], + deps = [ + "//envoy/api/v3alpha/route", + "//envoy/config/filter/fault/v3alpha:fault", + "//envoy/type:percent", + ], +) diff --git a/api/envoy/config/filter/http/fault/v3alpha/fault.proto b/api/envoy/config/filter/http/fault/v3alpha/fault.proto new file mode 100644 index 000000000000..f654ec17f617 --- /dev/null +++ b/api/envoy/config/filter/http/fault/v3alpha/fault.proto @@ -0,0 +1,115 @@ +syntax = "proto3"; + +package envoy.config.filter.http.fault.v3alpha; + +option java_outer_classname = "FaultProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.fault.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/route/route.proto"; +import "envoy/config/filter/fault/v3alpha/fault.proto"; +import "envoy/type/percent.proto"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Fault Injection] +// Fault Injection :ref:`configuration overview `. + +message FaultAbort { + reserved 1; + + oneof error_type { + option (validate.required) = true; + + // HTTP status code to use to abort the HTTP request. + uint32 http_status = 2 [(validate.rules).uint32 = {gte: 200, lt: 600}]; + } + + // The percentage of requests/operations/connections that will be aborted with the error code + // provided. + type.FractionalPercent percentage = 3; +} + +message HTTPFault { + // If specified, the filter will inject delays based on the values in the + // object. + filter.fault.v3alpha.FaultDelay delay = 1; + + // If specified, the filter will abort requests based on the values in + // the object. At least *abort* or *delay* must be specified. + FaultAbort abort = 2; + + // Specifies the name of the (destination) upstream cluster that the + // filter should match on. Fault injection will be restricted to requests + // bound to the specific upstream cluster. + string upstream_cluster = 3; + + // Specifies a set of headers that the filter should match on. The fault + // injection filter can be applied selectively to requests that match a set of + // headers specified in the fault filter config. The chances of actual fault + // injection further depend on the value of the :ref:`percentage + // ` field. + // The filter will check the request's headers against all the specified + // headers in the filter config. A match will happen if all the headers in the + // config are present in the request with the same values (or based on + // presence if the *value* field is not in the config). + repeated envoy.api.v3alpha.route.HeaderMatcher headers = 4; + + // Faults are injected for the specified list of downstream hosts. If this + // setting is not set, faults are injected for all downstream nodes. + // Downstream node name is taken from :ref:`the HTTP + // x-envoy-downstream-service-node + // ` header and compared + // against downstream_nodes list. + repeated string downstream_nodes = 5; + + // The maximum number of faults that can be active at a single time via the configured fault + // filter. Note that because this setting can be overridden at the route level, it's possible + // for the number of active faults to be greater than this value (if injected via a different + // route). If not specified, defaults to unlimited. This setting can be overridden via + // `runtime ` and any faults that are not injected + // due to overflow will be indicated via the `faults_overflow + // ` stat. + // + // .. attention:: + // Like other :ref:`circuit breakers ` in Envoy, this is a fuzzy + // limit. It's possible for the number of active faults to rise slightly above the configured + // amount due to the implementation details. + google.protobuf.UInt32Value max_active_faults = 6; + + // The response rate limit to be applied to the response body of the stream. When configured, + // the percentage can be overridden by the :ref:`fault.http.rate_limit.response_percent + // ` runtime key. + // + // .. attention:: + // This is a per-stream limit versus a connection level limit. This means that concurrent streams + // will each get an independent limit. + filter.fault.v3alpha.FaultRateLimit response_rate_limit = 7; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.delay.fixed_delay_percent + string delay_percent_runtime = 8; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.abort.abort_percent + string abort_percent_runtime = 9; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.delay.fixed_duration_ms + string delay_duration_runtime = 10; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.abort.http_status + string abort_http_status_runtime = 11; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.max_active_faults + string max_active_faults_runtime = 12; + + // The runtime key to override the :ref:`default ` + // runtime. The default is: fault.http.rate_limit.response_percent + string response_rate_limit_percent_runtime = 13; +} diff --git a/api/envoy/config/filter/http/gzip/v3alpha/BUILD b/api/envoy/config/filter/http/gzip/v3alpha/BUILD new file mode 100644 index 000000000000..e34d73c51c21 --- /dev/null +++ b/api/envoy/config/filter/http/gzip/v3alpha/BUILD @@ -0,0 +1,8 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "gzip", + srcs = ["gzip.proto"], +) diff --git a/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto b/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto new file mode 100644 index 000000000000..5b5c6d6d1df7 --- /dev/null +++ b/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto @@ -0,0 +1,75 @@ +syntax = "proto3"; + +package envoy.config.filter.http.gzip.v3alpha; + +option java_outer_classname = "GzipProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.gzip.v3alpha"; +option go_package = "v2"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Gzip] +// Gzip :ref:`configuration overview `. + +message Gzip { + // Value from 1 to 9 that controls the amount of internal memory used by zlib. Higher values + // use more memory, but are faster and produce better compression results. The default value is 5. + google.protobuf.UInt32Value memory_level = 1 [(validate.rules).uint32 = {gte: 1, lte: 9}]; + + // Minimum response length, in bytes, which will trigger compression. The default value is 30. + google.protobuf.UInt32Value content_length = 2 [(validate.rules).uint32.gte = 30]; + + message CompressionLevel { + enum Enum { + DEFAULT = 0; + BEST = 1; + SPEED = 2; + } + } + + // A value used for selecting the zlib compression level. This setting will affect speed and + // amount of compression applied to the content. "BEST" provides higher compression at the cost of + // higher latency, "SPEED" provides lower compression with minimum impact on response time. + // "DEFAULT" provides an optimal result between speed and compression. This field will be set to + // "DEFAULT" if not specified. + CompressionLevel.Enum compression_level = 3 [(validate.rules).enum.defined_only = true]; + + enum CompressionStrategy { + DEFAULT = 0; + FILTERED = 1; + HUFFMAN = 2; + RLE = 3; + } + + // A value used for selecting the zlib compression strategy which is directly related to the + // characteristics of the content. Most of the time "DEFAULT" will be the best choice, though + // there are situations which changing this parameter might produce better results. For example, + // run-length encoding (RLE) is typically used when the content is known for having sequences + // which same data occurs many consecutive times. For more information about each strategy, please + // refer to zlib manual. + CompressionStrategy compression_strategy = 4 [(validate.rules).enum.defined_only = true]; + + // Set of strings that allows specifying which mime-types yield compression; e.g., + // application/json, text/html, etc. When this field is not defined, compression will be applied + // to the following mime-types: "application/javascript", "application/json", + // "application/xhtml+xml", "image/svg+xml", "text/css", "text/html", "text/plain", "text/xml". + repeated string content_type = 6 [(validate.rules).repeated = {max_items: 50}]; + + // If true, disables compression when the response contains an etag header. When it is false, the + // filter will preserve weak etags and remove the ones that require strong validation. + bool disable_on_etag_header = 7; + + // If true, removes accept-encoding from the request headers before dispatching it to the upstream + // so that responses do not get compressed before reaching the filter. + bool remove_accept_encoding_header = 8; + + // Value from 9 to 15 that represents the base two logarithmic of the compressor's window size. + // Larger window results in better compression at the expense of memory usage. The default is 12 + // which will produce a 4096 bytes window. For more details about this parameter, please refer to + // zlib manual > deflateInit2. + google.protobuf.UInt32Value window_bits = 9 [(validate.rules).uint32 = {gte: 9, lte: 15}]; +} diff --git a/api/envoy/config/filter/http/header_to_metadata/v3alpha/BUILD b/api/envoy/config/filter/http/header_to_metadata/v3alpha/BUILD new file mode 100644 index 000000000000..3f8503acbe65 --- /dev/null +++ b/api/envoy/config/filter/http/header_to_metadata/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "header_to_metadata", + srcs = ["header_to_metadata.proto"], + deps = [], +) diff --git a/api/envoy/config/filter/http/header_to_metadata/v3alpha/header_to_metadata.proto b/api/envoy/config/filter/http/header_to_metadata/v3alpha/header_to_metadata.proto new file mode 100644 index 000000000000..927574a5a721 --- /dev/null +++ b/api/envoy/config/filter/http/header_to_metadata/v3alpha/header_to_metadata.proto @@ -0,0 +1,92 @@ +syntax = "proto3"; + +package envoy.config.filter.http.header_to_metadata.v3alpha; + +option java_outer_classname = "HeaderToMetadataProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.header_to_metadata.v3alpha"; +option go_package = "v2"; + +import "validate/validate.proto"; + +// [#protodoc-title: Header-To-Metadata Filter] +// +// The configuration for transforming headers into metadata. This is useful +// for matching load balancer subsets, logging, etc. +// +// Header to Metadata :ref:`configuration overview `. + +message Config { + enum ValueType { + STRING = 0; + NUMBER = 1; + + // The value is a serialized `protobuf.Value + // `_. + PROTOBUF_VALUE = 2; + } + + // ValueEncode defines the encoding algorithm. + enum ValueEncode { + // The value is not encoded. + NONE = 0; + + // The value is encoded in `Base64 `_. + // Note: this is mostly used for STRING and PROTOBUF_VALUE to escape the + // non-ASCII characters in the header. + BASE64 = 1; + } + + message KeyValuePair { + // The namespace — if this is empty, the filter's namespace will be used. + string metadata_namespace = 1; + + // The key to use within the namespace. + string key = 2 [(validate.rules).string.min_bytes = 1]; + + // The value to pair with the given key. + // + // When used for a `on_header_present` case, if value is non-empty it'll be used + // instead of the header value. If both are empty, no metadata is added. + // + // When used for a `on_header_missing` case, a non-empty value must be provided + // otherwise no metadata is added. + string value = 3; + + // The value's type — defaults to string. + ValueType type = 4; + + // How is the value encoded, default is NONE (not encoded). + // The value will be decoded accordingly before storing to metadata. + ValueEncode encode = 5; + } + + // A Rule defines what metadata to apply when a header is present or missing. + message Rule { + // The header that triggers this rule — required. + string header = 1 [(validate.rules).string.min_bytes = 1]; + + // If the header is present, apply this metadata KeyValuePair. + // + // If the value in the KeyValuePair is non-empty, it'll be used instead + // of the header value. + KeyValuePair on_header_present = 2; + + // If the header is not present, apply this metadata KeyValuePair. + // + // The value in the KeyValuePair must be set, since it'll be used in lieu + // of the missing header value. + KeyValuePair on_header_missing = 3; + + // Whether or not to remove the header after a rule is applied. + // + // This prevents headers from leaking. + bool remove = 4; + } + + // The list of rules to apply to requests. + repeated Rule request_rules = 1; + + // The list of rules to apply to responses. + repeated Rule response_rules = 2; +} diff --git a/api/envoy/config/filter/http/health_check/v3alpha/BUILD b/api/envoy/config/filter/http/health_check/v3alpha/BUILD new file mode 100644 index 000000000000..89e6eb3af702 --- /dev/null +++ b/api/envoy/config/filter/http/health_check/v3alpha/BUILD @@ -0,0 +1,21 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "health_check", + srcs = ["health_check.proto"], + deps = [ + "//envoy/api/v3alpha/route", + "//envoy/type:percent", + ], +) + +api_go_proto_library( + name = "health_check", + proto = ":health_check", + deps = [ + "//envoy/api/v3alpha/route:route_go_proto", + "//envoy/type:percent_go_proto", + ], +) diff --git a/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto b/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto new file mode 100644 index 000000000000..31fcdfffaa80 --- /dev/null +++ b/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; + +package envoy.config.filter.http.health_check.v3alpha; + +option java_outer_classname = "HealthCheckProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.health_check.v3alpha"; +option go_package = "v2"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "envoy/api/v3alpha/route/route.proto"; +import "envoy/type/percent.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: Health check] +// Health check :ref:`configuration overview `. + +message HealthCheck { + // Specifies whether the filter operates in pass through mode or not. + google.protobuf.BoolValue pass_through_mode = 1 [(validate.rules).message.required = true]; + + reserved 2; + reserved "endpoint"; + + // If operating in pass through mode, the amount of time in milliseconds + // that the filter should cache the upstream response. + google.protobuf.Duration cache_time = 3 [(gogoproto.stdduration) = true]; + + // If operating in non-pass-through mode, specifies a set of upstream cluster + // names and the minimum percentage of servers in each of those clusters that + // must be healthy or degraded in order for the filter to return a 200. + map cluster_min_healthy_percentages = 4; + + // Specifies a set of health check request headers to match on. The health check filter will + // check a request’s headers against all the specified headers. To specify the health check + // endpoint, set the ``:path`` header to match on. + repeated envoy.api.v3alpha.route.HeaderMatcher headers = 5; +} diff --git a/api/envoy/config/filter/http/ip_tagging/v3alpha/BUILD b/api/envoy/config/filter/http/ip_tagging/v3alpha/BUILD new file mode 100644 index 000000000000..5b34fcd9c458 --- /dev/null +++ b/api/envoy/config/filter/http/ip_tagging/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "ip_tagging", + srcs = ["ip_tagging.proto"], + deps = ["//envoy/api/v3alpha/core:address"], +) diff --git a/api/envoy/config/filter/http/ip_tagging/v3alpha/ip_tagging.proto b/api/envoy/config/filter/http/ip_tagging/v3alpha/ip_tagging.proto new file mode 100644 index 000000000000..e305800a5fc8 --- /dev/null +++ b/api/envoy/config/filter/http/ip_tagging/v3alpha/ip_tagging.proto @@ -0,0 +1,53 @@ +syntax = "proto3"; + +package envoy.config.filter.http.ip_tagging.v3alpha; + +option java_outer_classname = "IpTaggingProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.ip_tagging.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/address.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: IP tagging] +// IP tagging :ref:`configuration overview `. + +message IPTagging { + + // The type of requests the filter should apply to. The supported types + // are internal, external or both. The + // :ref:`x-forwarded-for` header is + // used to determine if a request is internal and will result in + // :ref:`x-envoy-internal` + // being set. The filter defaults to both, and it will apply to all request types. + enum RequestType { + // Both external and internal requests will be tagged. This is the default value. + BOTH = 0; + + // Only internal requests will be tagged. + INTERNAL = 1; + + // Only external requests will be tagged. + EXTERNAL = 2; + } + + // The type of request the filter should apply to. + RequestType request_type = 1 [(validate.rules).enum.defined_only = true]; + + // Supplies the IP tag name and the IP address subnets. + message IPTag { + // Specifies the IP tag name to apply. + string ip_tag_name = 1; + + // A list of IP address subnets that will be tagged with + // ip_tag_name. Both IPv4 and IPv6 are supported. + repeated envoy.api.v3alpha.core.CidrRange ip_list = 2; + } + + // [#comment:TODO(ccaraman): Extend functionality to load IP tags from file system. + // Tracked by issue https://github.com/envoyproxy/envoy/issues/2695] + // The set of IP tags for the filter. + repeated IPTag ip_tags = 4 [(validate.rules).repeated .min_items = 1]; +} diff --git a/api/envoy/config/filter/http/jwt_authn/v3alpha/BUILD b/api/envoy/config/filter/http/jwt_authn/v3alpha/BUILD new file mode 100644 index 000000000000..2970da93f467 --- /dev/null +++ b/api/envoy/config/filter/http/jwt_authn/v3alpha/BUILD @@ -0,0 +1,23 @@ +licenses(["notice"]) # Apache 2 + +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +api_proto_library_internal( + name = "jwt_authn", + srcs = ["config.proto"], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:http_uri", + "//envoy/api/v3alpha/route", + ], +) + +api_go_proto_library( + name = "jwt_authn", + proto = ":jwt_authn", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:http_uri_go_proto", + "//envoy/api/v3alpha/route:route_go_proto", + ], +) diff --git a/api/envoy/config/filter/http/jwt_authn/v3alpha/README.md b/api/envoy/config/filter/http/jwt_authn/v3alpha/README.md new file mode 100644 index 000000000000..c390a4d5ce50 --- /dev/null +++ b/api/envoy/config/filter/http/jwt_authn/v3alpha/README.md @@ -0,0 +1,66 @@ +# JWT Authentication HTTP filter config + +## Overview + +1. The proto file in this folder defines an HTTP filter config for "jwt_authn" filter. + +2. This filter will verify the JWT in the HTTP request as: + - The signature should be valid + - JWT should not be expired + - Issuer and audiences are valid and specified in the filter config. + +3. [JWK](https://tools.ietf.org/html/rfc7517#appendix-A) is needed to verify JWT signature. It can be fetched from a remote server or read from a local file. If the JWKS is fetched remotely, it will be cached by the filter. + +3. If a JWT is valid, the user is authenticated and the request will be forwarded to the backend server. If a JWT is not valid, the request will be rejected with an error message. + +## The locations to extract JWT + +JWT will be extracted from the HTTP headers or query parameters. The default location is the HTTP header: +``` +Authorization: Bearer +``` +The next default location is in the query parameter as: +``` +?access_token= +``` + +If a custom location is desired, `from_headers` or `from_params` can be used to specify custom locations to extract JWT. + +## HTTP header to pass successfully verified JWT + +If a JWT is valid, its payload will be passed to the backend in a new HTTP header specified in `forward_payload_header` field. Its value is base64 encoded JWT payload in JSON. + + +## Further header options + +In addition to the `name` field, which specifies the HTTP header name, +the `from_headers` section can specify an optional `value_prefix` value, as in: + +```yaml + from_headers: + - name: bespoke + value_prefix: jwt_value +``` + +The above will cause the jwt_authn filter to look for the JWT in the `bespoke` header, following the tag `jwt_value`. + +Any non-JWT characters (i.e., anything _other than_ alphanumerics, `_`, `-`, and `.`) will be skipped, +and all following, contiguous, JWT-legal chars will be taken as the JWT. + +This means all of the following will return a JWT of `eyJFbnZveSI6ICJyb2NrcyJ9.e30.c2lnbmVk`: + +```text +bespoke: jwt_value=eyJFbnZveSI6ICJyb2NrcyJ9.e30.c2lnbmVk + +bespoke: {"jwt_value": "eyJFbnZveSI6ICJyb2NrcyJ9.e30.c2lnbmVk"} + +bespoke: beta:true,jwt_value:"eyJFbnZveSI6ICJyb2NrcyJ9.e30.c2lnbmVk",trace=1234 +``` + +The header `name` may be `Authorization`. + +The `value_prefix` must match exactly, i.e., case-sensitively. +If the `value_prefix` is not found, the header is skipped: not considered as a source for a JWT token. + +If there are no JWT-legal characters after the `value_prefix`, the entire string after it +is taken to be the JWT token. This is unlikely to succeed; the error will reported by the JWT parser. \ No newline at end of file diff --git a/api/envoy/config/filter/http/jwt_authn/v3alpha/config.proto b/api/envoy/config/filter/http/jwt_authn/v3alpha/config.proto new file mode 100644 index 000000000000..9c7f10c4adbc --- /dev/null +++ b/api/envoy/config/filter/http/jwt_authn/v3alpha/config.proto @@ -0,0 +1,467 @@ + +syntax = "proto3"; + +package envoy.config.filter.http.jwt_authn.v3alpha; + +option java_outer_classname = "ConfigProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.jwt_authn.v3alpha"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/core/http_uri.proto"; +import "envoy/api/v3alpha/route/route.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: JWT Authentication] +// JWT Authentication :ref:`configuration overview `. + +// Please see following for JWT authentication flow: +// +// * `JSON Web Token (JWT) `_ +// * `The OAuth 2.0 Authorization Framework `_ +// * `OpenID Connect `_ +// +// A JwtProvider message specifies how a JSON Web Token (JWT) can be verified. It specifies: +// +// * issuer: the principal that issues the JWT. It has to match the one from the token. +// * allowed audiences: the ones in the token have to be listed here. +// * how to fetch public key JWKS to verify the token signature. +// * how to extract JWT token in the request. +// * how to pass successfully verified token payload. +// +// Example: +// +// .. code-block:: yaml +// +// issuer: https://example.com +// audiences: +// - bookstore_android.apps.googleusercontent.com +// - bookstore_web.apps.googleusercontent.com +// remote_jwks: +// http_uri: +// uri: https://example.com/.well-known/jwks.json +// cluster: example_jwks_cluster +// cache_duration: +// seconds: 300 +// +message JwtProvider { + // Specify the `principal `_ that issued + // the JWT, usually a URL or an email address. + // + // Example: https://securetoken.google.com + // Example: 1234567-compute@developer.gserviceaccount.com + // + string issuer = 1 [(validate.rules).string.min_bytes = 1]; + + // The list of JWT `audiences `_ are + // allowed to access. A JWT containing any of these audiences will be accepted. If not specified, + // will not check audiences in the token. + // + // Example: + // + // .. code-block:: yaml + // + // audiences: + // - bookstore_android.apps.googleusercontent.com + // - bookstore_web.apps.googleusercontent.com + // + repeated string audiences = 2; + + // `JSON Web Key Set (JWKS) `_ is needed to + // validate signature of a JWT. This field specifies where to fetch JWKS. + oneof jwks_source_specifier { + option (validate.required) = true; + + // JWKS can be fetched from remote server via HTTP/HTTPS. This field specifies the remote HTTP + // URI and how the fetched JWKS should be cached. + // + // Example: + // + // .. code-block:: yaml + // + // remote_jwks: + // http_uri: + // uri: https://www.googleapis.com/oauth2/v1/certs + // cluster: jwt.www.googleapis.com|443 + // cache_duration: + // seconds: 300 + // + RemoteJwks remote_jwks = 3; + + // JWKS is in local data source. It could be either in a local file or embedded in the + // inline_string. + // + // Example: local file + // + // .. code-block:: yaml + // + // local_jwks: + // filename: /etc/envoy/jwks/jwks1.txt + // + // Example: inline_string + // + // .. code-block:: yaml + // + // local_jwks: + // inline_string: ACADADADADA + // + envoy.api.v3alpha.core.DataSource local_jwks = 4; + } + + // If false, the JWT is removed in the request after a success verification. If true, the JWT is + // not removed in the request. Default value is false. + bool forward = 5; + + // Two fields below define where to extract the JWT from an HTTP request. + // + // If no explicit location is specified, the following default locations are tried in order: + // + // 1. The Authorization header using the `Bearer schema + // `_. Example:: + // + // Authorization: Bearer . + // + // 2. `access_token `_ query parameter. + // + // Multiple JWTs can be verified for a request. Each JWT has to be extracted from the locations + // its provider specified or from the default locations. + // + // Specify the HTTP headers to extract JWT token. For examples, following config: + // + // .. code-block:: yaml + // + // from_headers: + // - name: x-goog-iap-jwt-assertion + // + // can be used to extract token from header:: + // + // x-goog-iap-jwt-assertion: . + // + repeated JwtHeader from_headers = 6; + + // JWT is sent in a query parameter. `jwt_params` represents the query parameter names. + // + // For example, if config is: + // + // .. code-block:: yaml + // + // from_params: + // - jwt_token + // + // The JWT format in query parameter is:: + // + // /path?jwt_token= + // + repeated string from_params = 7; + + // This field specifies the header name to forward a successfully verified JWT payload to the + // backend. The forwarded data is:: + // + // base64_encoded(jwt_payload_in_JSON) + // + // If it is not specified, the payload will not be forwarded. + string forward_payload_header = 8; + + // If non empty, successfully verified JWT payloads will be written to StreamInfo DynamicMetadata + // in the format as: *namespace* is the jwt_authn filter name as **envoy.filters.http.jwt_authn** + // The value is the *protobuf::Struct*. The value of this field will be the key for its *fields* + // and the value is the *protobuf::Struct* converted from JWT JSON payload. + // + // For example, if payload_in_metadata is *my_payload*: + // + // .. code-block:: yaml + // + // envoy.filters.http.jwt_authn: + // my_payload: + // iss: https://example.com + // sub: test@example.com + // aud: https://example.com + // exp: 1501281058 + // + string payload_in_metadata = 9; +} + +// This message specifies how to fetch JWKS from remote and how to cache it. +message RemoteJwks { + // The HTTP URI to fetch the JWKS. For example: + // + // .. code-block:: yaml + // + // http_uri: + // uri: https://www.googleapis.com/oauth2/v1/certs + // cluster: jwt.www.googleapis.com|443 + // + envoy.api.v3alpha.core.HttpUri http_uri = 1; + + // Duration after which the cached JWKS should be expired. If not specified, default cache + // duration is 5 minutes. + google.protobuf.Duration cache_duration = 2; +} + +// This message specifies a header location to extract JWT token. +message JwtHeader { + // The HTTP header name. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // The value prefix. The value format is "value_prefix" + // For example, for "Authorization: Bearer ", value_prefix="Bearer " with a space at the + // end. + string value_prefix = 2; +} + +// Specify a required provider with audiences. +message ProviderWithAudiences { + // Specify a required provider name. + string provider_name = 1; + + // This field overrides the one specified in the JwtProvider. + repeated string audiences = 2; +} + +// This message specifies a Jwt requirement. An empty message means JWT verification is not +// required. Here are some config examples: +// +// .. code-block:: yaml +// +// # Example 1: not required with an empty message +// +// # Example 2: require A +// provider_name: provider-A +// +// # Example 3: require A or B +// requires_any: +// requirements: +// - provider_name: provider-A +// - provider_name: provider-B +// +// # Example 4: require A and B +// requires_all: +// requirements: +// - provider_name: provider-A +// - provider_name: provider-B +// +// # Example 5: require A and (B or C) +// requires_all: +// requirements: +// - provider_name: provider-A +// - requires_any: +// requirements: +// - provider_name: provider-B +// - provider_name: provider-C +// +// # Example 6: require A or (B and C) +// requires_any: +// requirements: +// - provider_name: provider-A +// - requires_all: +// requirements: +// - provider_name: provider-B +// - provider_name: provider-C +// +message JwtRequirement { + oneof requires_type { + // Specify a required provider name. + string provider_name = 1; + + // Specify a required provider with audiences. + ProviderWithAudiences provider_and_audiences = 2; + + // Specify list of JwtRequirement. Their results are OR-ed. + // If any one of them passes, the result is passed. + JwtRequirementOrList requires_any = 3; + + // Specify list of JwtRequirement. Their results are AND-ed. + // All of them must pass, if one of them fails or missing, it fails. + JwtRequirementAndList requires_all = 4; + + // The requirement is always satisfied even if JWT is missing or the JWT + // verification fails. A typical usage is: this filter is used to only verify + // JWTs and pass the verified JWT payloads to another filter, the other filter + // will make decision. In this mode, all JWT tokens will be verified. + google.protobuf.Empty allow_missing_or_failed = 5; + } +} + +// This message specifies a list of RequiredProvider. +// Their results are OR-ed; if any one of them passes, the result is passed +message JwtRequirementOrList { + // Specify a list of JwtRequirement. + repeated JwtRequirement requirements = 1 [(validate.rules).repeated .min_items = 2]; +} + +// This message specifies a list of RequiredProvider. +// Their results are AND-ed; all of them must pass, if one of them fails or missing, it fails. +message JwtRequirementAndList { + // Specify a list of JwtRequirement. + repeated JwtRequirement requirements = 1 [(validate.rules).repeated .min_items = 2]; +} + +// This message specifies a Jwt requirement for a specific Route condition. +// Example 1: +// +// .. code-block:: yaml +// +// - match: +// prefix: /healthz +// +// In above example, "requires" field is empty for /healthz prefix match, +// it means that requests matching the path prefix don't require JWT authentication. +// +// Example 2: +// +// .. code-block:: yaml +// +// - match: +// prefix: / +// requires: { provider_name: provider-A } +// +// In above example, all requests matched the path prefix require jwt authentication +// from "provider-A". +message RequirementRule { + // The route matching parameter. Only when the match is satisfied, the "requires" field will + // apply. + // + // For example: following match will match all requests. + // + // .. code-block:: yaml + // + // match: + // prefix: / + // + envoy.api.v3alpha.route.RouteMatch match = 1 [(validate.rules).message.required = true]; + + // Specify a Jwt Requirement. Please detail comment in message JwtRequirement. + JwtRequirement requires = 2; +} + +// This message specifies Jwt requirements based on stream_info.filterState. +// This FilterState should use `Router::StringAccessor` object to set a string value. +// Other HTTP filters can use it to specify Jwt requirements dynamically. +// +// Example: +// +// .. code-block:: yaml +// +// name: jwt_selector +// requires: +// issuer_1: +// provider_name: issuer1 +// issuer_2: +// provider_name: issuer2 +// +// If a filter set "jwt_selector" with "issuer_1" to FilterState for a request, +// jwt_authn filter will use JwtRequirement{"provider_name": "issuer1"} to verify. +message FilterStateRule { + // The filter state name to retrieve the `Router::StringAccessor` object. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // A map of string keys to requirements. The string key is the string value + // in the FilterState with the name specified in the *name* field above. + map requires = 3; +} + +// This is the Envoy HTTP filter config for JWT authentication. +// +// For example: +// +// .. code-block:: yaml +// +// providers: +// provider1: +// issuer: issuer1 +// audiences: +// - audience1 +// - audience2 +// remote_jwks: +// http_uri: +// uri: https://example.com/.well-known/jwks.json +// cluster: example_jwks_cluster +// provider2: +// issuer: issuer2 +// local_jwks: +// inline_string: jwks_string +// +// rules: +// # Not jwt verification is required for /health path +// - match: +// prefix: /health +// +// # Jwt verification for provider1 is required for path prefixed with "prefix" +// - match: +// prefix: /prefix +// requires: +// provider_name: provider1 +// +// # Jwt verification for either provider1 or provider2 is required for all other requests. +// - match: +// prefix: / +// requires: +// requires_any: +// requirements: +// - provider_name: provider1 +// - provider_name: provider2 +// +message JwtAuthentication { + // Map of provider names to JwtProviders. + // + // .. code-block:: yaml + // + // providers: + // provider1: + // issuer: issuer1 + // audiences: + // - audience1 + // - audience2 + // remote_jwks: + // http_uri: + // uri: https://example.com/.well-known/jwks.json + // cluster: example_jwks_cluster + // provider2: + // issuer: provider2 + // local_jwks: + // inline_string: jwks_string + // + map providers = 1; + + // Specifies requirements based on the route matches. The first matched requirement will be + // applied. If there are overlapped match conditions, please put the most specific match first. + // + // Examples + // + // .. code-block:: yaml + // + // rules: + // - match: + // prefix: /healthz + // - match: + // prefix: /baz + // requires: + // provider_name: provider1 + // - match: + // prefix: /foo + // requires: + // requires_any: + // requirements: + // - provider_name: provider1 + // - provider_name: provider2 + // - match: + // prefix: /bar + // requires: + // requires_all: + // requirements: + // - provider_name: provider1 + // - provider_name: provider2 + // + repeated RequirementRule rules = 2; + + // This message specifies Jwt requirements based on stream_info.filterState. + // Other HTTP filters can use it to specify Jwt requirements dynamically. + // The *rules* field above is checked first, if it could not find any matches, + // check this one. + FilterStateRule filter_state_rules = 3; +} diff --git a/api/envoy/config/filter/http/lua/v3alpha/BUILD b/api/envoy/config/filter/http/lua/v3alpha/BUILD new file mode 100644 index 000000000000..6daf0c82f174 --- /dev/null +++ b/api/envoy/config/filter/http/lua/v3alpha/BUILD @@ -0,0 +1,8 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "lua", + srcs = ["lua.proto"], +) diff --git a/api/envoy/config/filter/http/lua/v3alpha/lua.proto b/api/envoy/config/filter/http/lua/v3alpha/lua.proto new file mode 100644 index 000000000000..ff586ca2429e --- /dev/null +++ b/api/envoy/config/filter/http/lua/v3alpha/lua.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package envoy.config.filter.http.lua.v3alpha; + +option java_outer_classname = "LuaProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.lua.v3alpha"; +option go_package = "v2"; + +import "validate/validate.proto"; + +// [#protodoc-title: Lua] +// Lua :ref:`configuration overview `. + +message Lua { + // The Lua code that Envoy will execute. This can be a very small script that + // further loads code from disk if desired. Note that if JSON configuration is used, the code must + // be properly escaped. YAML configuration may be easier to read since YAML supports multi-line + // strings so complex scripts can be easily expressed inline in the configuration. + string inline_code = 1 [(validate.rules).string.min_bytes = 1]; +} diff --git a/api/envoy/config/filter/http/rate_limit/v3alpha/BUILD b/api/envoy/config/filter/http/rate_limit/v3alpha/BUILD new file mode 100644 index 000000000000..e131d3a92263 --- /dev/null +++ b/api/envoy/config/filter/http/rate_limit/v3alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "rate_limit", + srcs = ["rate_limit.proto"], + deps = [ + "//envoy/config/ratelimit/v3alpha:rls", + ], +) diff --git a/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto b/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto new file mode 100644 index 000000000000..69e8d389bc4b --- /dev/null +++ b/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +package envoy.config.filter.http.rate_limit.v3alpha; + +option java_outer_classname = "RateLimitProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.rate_limit.v3alpha"; +option go_package = "v2"; + +import "envoy/config/ratelimit/v3alpha/rls.proto"; + +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Rate limit] +// Rate limit :ref:`configuration overview `. + +message RateLimit { + // The rate limit domain to use when calling the rate limit service. + string domain = 1 [(validate.rules).string.min_bytes = 1]; + + // Specifies the rate limit configurations to be applied with the same + // stage number. If not set, the default stage number is 0. + // + // .. note:: + // + // The filter supports a range of 0 - 10 inclusively for stage numbers. + uint32 stage = 2 [(validate.rules).uint32.lte = 10]; + + // The type of requests the filter should apply to. The supported + // types are *internal*, *external* or *both*. A request is considered internal if + // :ref:`x-envoy-internal` is set to true. If + // :ref:`x-envoy-internal` is not set or false, a + // request is considered external. The filter defaults to *both*, and it will apply to all request + // types. + string request_type = 3; + + // The timeout in milliseconds for the rate limit service RPC. If not + // set, this defaults to 20ms. + google.protobuf.Duration timeout = 4 [(gogoproto.stdduration) = true]; + + // The filter's behaviour in case the rate limiting service does + // not respond back. When it is set to true, Envoy will not allow traffic in case of + // communication failure between rate limiting service and the proxy. + // Defaults to false. + bool failure_mode_deny = 5; + + // Specifies whether a `RESOURCE_EXHAUSTED` gRPC code must be returned instead + // of the default `UNAVAILABLE` gRPC code for a rate limited gRPC call. The + // HTTP code will be 200 for a gRPC response. + bool rate_limited_as_resource_exhausted = 6; + + // Configuration for an external rate limit service provider. If not + // specified, any calls to the rate limit service will immediately return + // success. + envoy.config.ratelimit.v3alpha.RateLimitServiceConfig rate_limit_service = 7 + [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/filter/http/rbac/v3alpha/BUILD b/api/envoy/config/filter/http/rbac/v3alpha/BUILD new file mode 100644 index 000000000000..a6ee42cf7893 --- /dev/null +++ b/api/envoy/config/filter/http/rbac/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "rbac", + srcs = ["rbac.proto"], + deps = ["//envoy/config/rbac/v3alpha:rbac"], +) diff --git a/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto b/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto new file mode 100644 index 000000000000..8ec8989652aa --- /dev/null +++ b/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +package envoy.config.filter.http.rbac.v3alpha; + +option java_outer_classname = "RbacProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.rbac.v3alpha"; +option go_package = "v2"; + +import "envoy/config/rbac/v3alpha/rbac.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: RBAC] +// Role-Based Access Control :ref:`configuration overview `. + +// RBAC filter config. +message RBAC { + // Specify the RBAC rules to be applied globally. + // If absent, no enforcing RBAC policy will be applied. + config.rbac.v3alpha.RBAC rules = 1; + + // Shadow rules are not enforced by the filter (i.e., returning a 403) + // but will emit stats and logs and can be used for rule testing. + // If absent, no shadow RBAC policy will be applied. + config.rbac.v3alpha.RBAC shadow_rules = 2; +} + +message RBACPerRoute { + reserved 1; + + reserved "disabled"; + + // Override the global configuration of the filter with this new config. + // If absent, the global RBAC policy will be disabled for this route. + RBAC rbac = 2; +} diff --git a/api/envoy/config/filter/http/router/v3alpha/BUILD b/api/envoy/config/filter/http/router/v3alpha/BUILD new file mode 100644 index 000000000000..f0b6c100d445 --- /dev/null +++ b/api/envoy/config/filter/http/router/v3alpha/BUILD @@ -0,0 +1,15 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "router", + srcs = ["router.proto"], + deps = ["//envoy/config/filter/accesslog/v3alpha:accesslog"], +) + +api_go_proto_library( + name = "router", + proto = ":router", + deps = ["//envoy/config/filter/accesslog/v3alpha:accesslog_go_proto"], +) diff --git a/api/envoy/config/filter/http/router/v3alpha/router.proto b/api/envoy/config/filter/http/router/v3alpha/router.proto new file mode 100644 index 000000000000..92efe315c6ff --- /dev/null +++ b/api/envoy/config/filter/http/router/v3alpha/router.proto @@ -0,0 +1,67 @@ +syntax = "proto3"; + +package envoy.config.filter.http.router.v3alpha; + +option java_outer_classname = "RouterProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.router.v3alpha"; +option go_package = "v2"; + +import "envoy/config/filter/accesslog/v3alpha/accesslog.proto"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Router] +// Router :ref:`configuration overview `. + +message Router { + // Whether the router generates dynamic cluster statistics. Defaults to + // true. Can be disabled in high performance scenarios. + google.protobuf.BoolValue dynamic_stats = 1; + + // Whether to start a child span for egress routed calls. This can be + // useful in scenarios where other filters (auth, ratelimit, etc.) make + // outbound calls and have child spans rooted at the same ingress + // parent. Defaults to false. + bool start_child_span = 2; + + // Configuration for HTTP upstream logs emitted by the router. Upstream logs + // are configured in the same way as access logs, but each log entry represents + // an upstream request. Presuming retries are configured, multiple upstream + // requests may be made for each downstream (inbound) request. + repeated envoy.config.filter.accesslog.v3alpha.AccessLog upstream_log = 3; + + // Do not add any additional *x-envoy-* headers to requests or responses. This + // only affects the :ref:`router filter generated *x-envoy-* headers + // `, other Envoy filters and the HTTP + // connection manager may continue to set *x-envoy-* headers. + bool suppress_envoy_headers = 4; + + // Specifies a list of HTTP headers to strictly validate. Envoy will reject a + // request and respond with HTTP status 400 if the request contains an invalid + // value for any of the headers listed in this field. Strict header checking + // is only supported for the following headers: + // + // Value must be a ','-delimited list (i.e. no spaces) of supported retry + // policy values: + // + // * :ref:`config_http_filters_router_x-envoy-retry-grpc-on` + // * :ref:`config_http_filters_router_x-envoy-retry-on` + // + // Value must be an integer: + // + // * :ref:`config_http_filters_router_x-envoy-max-retries` + // * :ref:`config_http_filters_router_x-envoy-upstream-rq-timeout-ms` + // * :ref:`config_http_filters_router_x-envoy-upstream-rq-per-try-timeout-ms` + repeated string strict_check_headers = 5 [(validate.rules).repeated .items.string = { + in: [ + "x-envoy-upstream-rq-timeout-ms", + "x-envoy-upstream-rq-per-try-timeout-ms", + "x-envoy-max-retries", + "x-envoy-retry-grpc-on", + "x-envoy-retry-on" + ] + }]; +} diff --git a/api/envoy/config/filter/http/squash/v3alpha/BUILD b/api/envoy/config/filter/http/squash/v3alpha/BUILD new file mode 100644 index 000000000000..86bd4e8cfb65 --- /dev/null +++ b/api/envoy/config/filter/http/squash/v3alpha/BUILD @@ -0,0 +1,8 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "squash", + srcs = ["squash.proto"], +) diff --git a/api/envoy/config/filter/http/squash/v3alpha/squash.proto b/api/envoy/config/filter/http/squash/v3alpha/squash.proto new file mode 100644 index 000000000000..43a62af98c1c --- /dev/null +++ b/api/envoy/config/filter/http/squash/v3alpha/squash.proto @@ -0,0 +1,55 @@ +syntax = "proto3"; + +package envoy.config.filter.http.squash.v3alpha; + +option java_outer_classname = "SquashProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.squash.v3alpha"; +option go_package = "v2"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Squash] +// Squash :ref:`configuration overview `. + +// [#proto-status: experimental] +message Squash { + // The name of the cluster that hosts the Squash server. + string cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // When the filter requests the Squash server to create a DebugAttachment, it will use this + // structure as template for the body of the request. It can contain reference to environment + // variables in the form of '{{ ENV_VAR_NAME }}'. These can be used to provide the Squash server + // with more information to find the process to attach the debugger to. For example, in a + // Istio/k8s environment, this will contain information on the pod: + // + // .. code-block:: json + // + // { + // "spec": { + // "attachment": { + // "pod": "{{ POD_NAME }}", + // "namespace": "{{ POD_NAMESPACE }}" + // }, + // "match_request": true + // } + // } + // + // (where POD_NAME, POD_NAMESPACE are configured in the pod via the Downward API) + google.protobuf.Struct attachment_template = 2; + + // The timeout for individual requests sent to the Squash cluster. Defaults to 1 second. + google.protobuf.Duration request_timeout = 3 [(gogoproto.stdduration) = true]; + + // The total timeout Squash will delay a request and wait for it to be attached. Defaults to 60 + // seconds. + google.protobuf.Duration attachment_timeout = 4 [(gogoproto.stdduration) = true]; + + // Amount of time to poll for the status of the attachment object in the Squash server + // (to check if has been attached). Defaults to 1 second. + google.protobuf.Duration attachment_poll_period = 5 [(gogoproto.stdduration) = true]; +} diff --git a/api/envoy/config/filter/http/tap/v3alpha/BUILD b/api/envoy/config/filter/http/tap/v3alpha/BUILD new file mode 100644 index 000000000000..a2af23059be6 --- /dev/null +++ b/api/envoy/config/filter/http/tap/v3alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "tap", + srcs = ["tap.proto"], + deps = [ + "//envoy/config/common/tap/v3alpha:common", + ], +) diff --git a/api/envoy/config/filter/http/tap/v3alpha/tap.proto b/api/envoy/config/filter/http/tap/v3alpha/tap.proto new file mode 100644 index 000000000000..e92c7d229f5e --- /dev/null +++ b/api/envoy/config/filter/http/tap/v3alpha/tap.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +import "envoy/config/common/tap/v3alpha/common.proto"; + +import "validate/validate.proto"; + +package envoy.config.filter.http.tap.v3alpha; + +option java_outer_classname = "TapProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.tap.v3alpha"; + +// [#protodoc-title: Tap] +// Tap :ref:`configuration overview `. + +// Top level configuration for the tap filter. +message Tap { + // Common configuration for the HTTP tap filter. + common.tap.v3alpha.CommonExtensionConfig common_config = 1 + [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/filter/http/transcoder/v3alpha/BUILD b/api/envoy/config/filter/http/transcoder/v3alpha/BUILD new file mode 100644 index 000000000000..c1a845bcd96e --- /dev/null +++ b/api/envoy/config/filter/http/transcoder/v3alpha/BUILD @@ -0,0 +1,13 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "transcoder", + srcs = ["transcoder.proto"], +) + +api_go_proto_library( + name = "transcoder", + proto = ":transcoder", +) diff --git a/api/envoy/config/filter/http/transcoder/v3alpha/transcoder.proto b/api/envoy/config/filter/http/transcoder/v3alpha/transcoder.proto new file mode 100644 index 000000000000..078ac52473ac --- /dev/null +++ b/api/envoy/config/filter/http/transcoder/v3alpha/transcoder.proto @@ -0,0 +1,123 @@ +syntax = "proto3"; + +package envoy.config.filter.http.transcoder.v3alpha; + +option java_outer_classname = "TranscoderProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.transcoder.v3alpha"; +option go_package = "v2"; + +import "validate/validate.proto"; + +// [#protodoc-title: gRPC-JSON transcoder] +// gRPC-JSON transcoder :ref:`configuration overview `. + +message GrpcJsonTranscoder { + oneof descriptor_set { + option (validate.required) = true; + + // Supplies the filename of + // :ref:`the proto descriptor set ` for the gRPC + // services. + string proto_descriptor = 1; + + // Supplies the binary content of + // :ref:`the proto descriptor set ` for the gRPC + // services. + bytes proto_descriptor_bin = 4; + } + + // A list of strings that + // supplies the fully qualified service names (i.e. "package_name.service_name") that + // the transcoder will translate. If the service name doesn't exist in ``proto_descriptor``, + // Envoy will fail at startup. The ``proto_descriptor`` may contain more services than + // the service names specified here, but they won't be translated. + repeated string services = 2 [(validate.rules).repeated .min_items = 1]; + + message PrintOptions { + // Whether to add spaces, line breaks and indentation to make the JSON + // output easy to read. Defaults to false. + bool add_whitespace = 1; + + // Whether to always print primitive fields. By default primitive + // fields with default values will be omitted in JSON output. For + // example, an int32 field set to 0 will be omitted. Setting this flag to + // true will override the default behavior and print primitive fields + // regardless of their values. Defaults to false. + bool always_print_primitive_fields = 2; + + // Whether to always print enums as ints. By default they are rendered + // as strings. Defaults to false. + bool always_print_enums_as_ints = 3; + + // Whether to preserve proto field names. By default protobuf will + // generate JSON field names using the ``json_name`` option, or lower camel case, + // in that order. Setting this flag will preserve the original field names. Defaults to false. + bool preserve_proto_field_names = 4; + }; + + // Control options for response JSON. These options are passed directly to + // `JsonPrintOptions `_. + PrintOptions print_options = 3; + + // Whether to keep the incoming request route after the outgoing headers have been transformed to + // the match the upstream gRPC service. Note: This means that routes for gRPC services that are + // not transcoded cannot be used in combination with *match_incoming_request_route*. + bool match_incoming_request_route = 5; + + // A list of query parameters to be ignored for transcoding method mapping. + // By default, the transcoder filter will not transcode a request if there are any + // unknown/invalid query parameters. + // + // Example : + // + // .. code-block:: proto + // + // service Bookstore { + // rpc GetShelf(GetShelfRequest) returns (Shelf) { + // option (google.api.http) = { + // get: "/shelves/{shelf}" + // }; + // } + // } + // + // message GetShelfRequest { + // int64 shelf = 1; + // } + // + // message Shelf {} + // + // The request ``/shelves/100?foo=bar`` will not be mapped to ``GetShelf``` because variable + // binding for ``foo`` is not defined. Adding ``foo`` to ``ignored_query_parameters`` will allow + // the same request to be mapped to ``GetShelf``. + repeated string ignored_query_parameters = 6; + + // Whether to route methods without the ``google.api.http`` option. + // + // Example : + // + // .. code-block:: proto + // + // package bookstore; + // + // service Bookstore { + // rpc GetShelf(GetShelfRequest) returns (Shelf) {} + // } + // + // message GetShelfRequest { + // int64 shelf = 1; + // } + // + // message Shelf {} + // + // The client could ``post`` a json body ``{"shelf": 1234}`` with the path of + // ``/bookstore.Bookstore/GetShelfRequest`` to call ``GetShelfRequest``. + bool auto_mapping = 7; + + // Whether to ignore query parameters that cannot be mapped to a corresponding + // protobuf field. Use this if you cannot control the query parameters and do + // not know them beforehand. Otherwise use ``ignored_query_parameters``. + // Defaults to false. + bool ignore_unknown_query_parameters = 8; +} diff --git a/api/envoy/config/filter/network/client_ssl_auth/v3alpha/BUILD b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/BUILD new file mode 100644 index 000000000000..bece14103bbe --- /dev/null +++ b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "client_ssl_auth", + srcs = ["client_ssl_auth.proto"], + deps = ["//envoy/api/v3alpha/core:address"], +) diff --git a/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto new file mode 100644 index 000000000000..a0ea3bf0bfaa --- /dev/null +++ b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package envoy.config.filter.network.client_ssl_auth.v3alpha; + +option java_outer_classname = "ClientSslAuthProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.client_ssl_auth.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/address.proto"; +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Client TLS authentication] +// Client TLS authentication +// :ref:`configuration overview `. + +message ClientSSLAuth { + // The :ref:`cluster manager ` cluster that runs + // the authentication service. The filter will connect to the service every 60s to fetch the list + // of principals. The service must support the expected :ref:`REST API + // `. + string auth_api_cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // The prefix to use when emitting :ref:`statistics + // `. + string stat_prefix = 2 [(validate.rules).string.min_bytes = 1]; + + // Time in milliseconds between principal refreshes from the + // authentication service. Default is 60000 (60s). The actual fetch time + // will be this value plus a random jittered value between + // 0-refresh_delay_ms milliseconds. + google.protobuf.Duration refresh_delay = 3 [(gogoproto.stdduration) = true]; + + // An optional list of IP address and subnet masks that should be white + // listed for access by the filter. If no list is provided, there is no + // IP white list. + repeated envoy.api.v3alpha.core.CidrRange ip_white_list = 4; +} diff --git a/api/envoy/config/filter/network/ext_authz/v3alpha/BUILD b/api/envoy/config/filter/network/ext_authz/v3alpha/BUILD new file mode 100644 index 000000000000..839724af13b4 --- /dev/null +++ b/api/envoy/config/filter/network/ext_authz/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "ext_authz", + srcs = ["ext_authz.proto"], + deps = ["//envoy/api/v3alpha/core:grpc_service"], +) diff --git a/api/envoy/config/filter/network/ext_authz/v3alpha/ext_authz.proto b/api/envoy/config/filter/network/ext_authz/v3alpha/ext_authz.proto new file mode 100644 index 000000000000..99c0c7239753 --- /dev/null +++ b/api/envoy/config/filter/network/ext_authz/v3alpha/ext_authz.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +package envoy.config.filter.network.ext_authz.v3alpha; + +option java_outer_classname = "ExtAuthzProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.ext_authz.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/grpc_service.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Network External Authorization ] +// The network layer external authorization service configuration +// :ref:`configuration overview `. + +// External Authorization filter calls out to an external service over the +// gRPC Authorization API defined by +// :ref:`CheckRequest `. +// A failed check will cause this filter to close the TCP connection. +message ExtAuthz { + // The prefix to use when emitting statistics. + string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + // The external authorization gRPC service configuration. + // The default timeout is set to 200ms by this filter. + envoy.api.v3alpha.core.GrpcService grpc_service = 2; + + // The filter's behaviour in case the external authorization service does + // not respond back. When it is set to true, Envoy will also allow traffic in case of + // communication failure between authorization service and the proxy. + // Defaults to false. + bool failure_mode_allow = 3; +} diff --git a/api/envoy/config/filter/network/http_connection_manager/v3alpha/BUILD b/api/envoy/config/filter/network/http_connection_manager/v3alpha/BUILD new file mode 100644 index 000000000000..300b3c8e671a --- /dev/null +++ b/api/envoy/config/filter/network/http_connection_manager/v3alpha/BUILD @@ -0,0 +1,31 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "http_connection_manager", + srcs = ["http_connection_manager.proto"], + deps = [ + "//envoy/api/v3alpha:rds", + "//envoy/api/v3alpha:srds", + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:config_source", + "//envoy/api/v3alpha/core:protocol", + "//envoy/config/filter/accesslog/v3alpha:accesslog", + "//envoy/type:percent", + ], +) + +api_go_proto_library( + name = "http_connection_manager", + proto = ":http_connection_manager", + deps = [ + "//envoy/api/v3alpha:rds_go_grpc", + "//envoy/api/v3alpha:srds_go_grpc", + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:config_source_go_proto", + "//envoy/api/v3alpha/core:protocol_go_proto", + "//envoy/config/filter/accesslog/v3alpha:accesslog_go_proto", + "//envoy/type:percent_go_proto", + ], +) diff --git a/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto new file mode 100644 index 000000000000..57e529b2164a --- /dev/null +++ b/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto @@ -0,0 +1,599 @@ +syntax = "proto3"; + +package envoy.config.filter.network.http_connection_manager.v3alpha; + +option java_outer_classname = "HttpConnectionManagerProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.http_connection_manager.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/config_source.proto"; +import "envoy/api/v3alpha/core/protocol.proto"; +import "envoy/api/v3alpha/rds.proto"; +import "envoy/api/v3alpha/srds.proto"; +import "envoy/config/filter/accesslog/v3alpha/accesslog.proto"; +import "envoy/type/percent.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: HTTP connection manager] +// HTTP connection manager :ref:`configuration overview `. + +// [#comment:next free field: 35] +message HttpConnectionManager { + enum CodecType { + option (gogoproto.goproto_enum_prefix) = false; + + // For every new connection, the connection manager will determine which + // codec to use. This mode supports both ALPN for TLS listeners as well as + // protocol inference for plaintext listeners. If ALPN data is available, it + // is preferred, otherwise protocol inference is used. In almost all cases, + // this is the right option to choose for this setting. + AUTO = 0; + + // The connection manager will assume that the client is speaking HTTP/1.1. + HTTP1 = 1; + + // The connection manager will assume that the client is speaking HTTP/2 + // (Envoy does not require HTTP/2 to take place over TLS or to use ALPN. + // Prior knowledge is allowed). + HTTP2 = 2; + } + + // Supplies the type of codec that the connection manager should use. + CodecType codec_type = 1 [(validate.rules).enum.defined_only = true]; + + // The human readable prefix to use when emitting statistics for the + // connection manager. See the :ref:`statistics documentation ` for + // more information. + string stat_prefix = 2 [(validate.rules).string.min_bytes = 1]; + + oneof route_specifier { + option (validate.required) = true; + + // The connection manager’s route table will be dynamically loaded via the RDS API. + Rds rds = 3; + + // The route table for the connection manager is static and is specified in this property. + envoy.api.v3alpha.RouteConfiguration route_config = 4; + + // A route table will be dynamically assigned to each request based on request attributes + // (e.g., the value of a header). The "routing scopes" (i.e., route tables) and "scope keys" are + // specified in this message. + ScopedRoutes scoped_routes = 31; + } + + // A list of individual HTTP filters that make up the filter chain for + // requests made to the connection manager. Order matters as the filters are + // processed sequentially as request events happen. + repeated HttpFilter http_filters = 5; + + // Whether the connection manager manipulates the :ref:`config_http_conn_man_headers_user-agent` + // and :ref:`config_http_conn_man_headers_downstream-service-cluster` headers. See the linked + // documentation for more information. Defaults to false. + google.protobuf.BoolValue add_user_agent = 6; + + message Tracing { + // [#comment:TODO(kyessenov): Align this field with listener traffic direction field.] + enum OperationName { + option (gogoproto.goproto_enum_prefix) = false; + + // The HTTP listener is used for ingress/incoming requests. + INGRESS = 0; + + // The HTTP listener is used for egress/outgoing requests. + EGRESS = 1; + } + + // The span name will be derived from this field. + OperationName operation_name = 1 [(validate.rules).enum.defined_only = true]; + + // A list of header names used to create tags for the active span. The header name is used to + // populate the tag name, and the header value is used to populate the tag value. The tag is + // created if the specified header name is present in the request's headers. + repeated string request_headers_for_tags = 2; + + // Target percentage of requests managed by this HTTP connection manager that will be force + // traced if the :ref:`x-client-trace-id ` + // header is set. This field is a direct analog for the runtime variable + // 'tracing.client_sampling' in the :ref:`HTTP Connection Manager + // `. + // Default: 100% + envoy.type.Percent client_sampling = 3; + + // Target percentage of requests managed by this HTTP connection manager that will be randomly + // selected for trace generation, if not requested by the client or not forced. This field is + // a direct analog for the runtime variable 'tracing.random_sampling' in the + // :ref:`HTTP Connection Manager `. + // Default: 100% + envoy.type.Percent random_sampling = 4; + + // Target percentage of requests managed by this HTTP connection manager that will be traced + // after all other sampling checks have been applied (client-directed, force tracing, random + // sampling). This field functions as an upper limit on the total configured sampling rate. For + // instance, setting client_sampling to 100% but overall_sampling to 1% will result in only 1% + // of client requests with the appropriate headers to be force traced. This field is a direct + // analog for the runtime variable 'tracing.global_enabled' in the + // :ref:`HTTP Connection Manager `. + // Default: 100% + envoy.type.Percent overall_sampling = 5; + + // Whether to annotate spans with additional data. If true, spans will include logs for stream + // events. + bool verbose = 6; + } + + // Presence of the object defines whether the connection manager + // emits :ref:`tracing ` data to the :ref:`configured tracing provider + // `. + Tracing tracing = 7; + + // Additional HTTP/1 settings that are passed to the HTTP/1 codec. + envoy.api.v3alpha.core.Http1ProtocolOptions http_protocol_options = 8; + + // Additional HTTP/2 settings that are passed directly to the HTTP/2 codec. + envoy.api.v3alpha.core.Http2ProtocolOptions http2_protocol_options = 9; + + // An optional override that the connection manager will write to the server + // header in responses. If not set, the default is *envoy*. + string server_name = 10; + + enum ServerHeaderTransformation { + option (gogoproto.goproto_enum_prefix) = false; + + // Overwrite any Server header with the contents of server_name. + OVERWRITE = 0; + // If no Server header is present, append Server server_name + // If a Server header is present, pass it through. + APPEND_IF_ABSENT = 1; + // Pass through the value of the server header, and do not append a header + // if none is present. + PASS_THROUGH = 2; + } + // Defines the action to be applied to the Server header on the response path. + // By default, Envoy will overwrite the header with the value specified in + // server_name. + ServerHeaderTransformation server_header_transformation = 34 + [(validate.rules).enum.defined_only = true]; + + // The maximum request headers size for incoming connections. + // If unconfigured, the default max request headers allowed is 60 KiB. + // Requests that exceed this limit will receive a 431 response. + // The max configurable limit is 96 KiB, based on current implementation + // constraints. + google.protobuf.UInt32Value max_request_headers_kb = 29 + [(validate.rules).uint32.gt = 0, (validate.rules).uint32.lte = 96]; + + // The idle timeout for connections managed by the connection manager. The + // idle timeout is defined as the period in which there are no active + // requests. If not set, there is no idle timeout. When the idle timeout is + // reached the connection will be closed. If the connection is an HTTP/2 + // connection a drain sequence will occur prior to closing the connection. See + // :ref:`drain_timeout + // `. + google.protobuf.Duration idle_timeout = 11 [(gogoproto.stdduration) = true]; + + // The stream idle timeout for connections managed by the connection manager. + // If not specified, this defaults to 5 minutes. The default value was selected + // so as not to interfere with any smaller configured timeouts that may have + // existed in configurations prior to the introduction of this feature, while + // introducing robustness to TCP connections that terminate without a FIN. + // + // This idle timeout applies to new streams and is overridable by the + // :ref:`route-level idle_timeout + // `. Even on a stream in + // which the override applies, prior to receipt of the initial request + // headers, the :ref:`stream_idle_timeout + // ` + // applies. Each time an encode/decode event for headers or data is processed + // for the stream, the timer will be reset. If the timeout fires, the stream + // is terminated with a 408 Request Timeout error code if no upstream response + // header has been received, otherwise a stream reset occurs. + // + // Note that it is possible to idle timeout even if the wire traffic for a stream is non-idle, due + // to the granularity of events presented to the connection manager. For example, while receiving + // very large request headers, it may be the case that there is traffic regularly arriving on the + // wire while the connection manage is only able to observe the end-of-headers event, hence the + // stream may still idle timeout. + // + // A value of 0 will completely disable the connection manager stream idle + // timeout, although per-route idle timeout overrides will continue to apply. + google.protobuf.Duration stream_idle_timeout = 24 [(gogoproto.stdduration) = true]; + + // A timeout for idle requests managed by the connection manager. + // The timer is activated when the request is initiated, and is disarmed when the last byte of the + // request is sent upstream (i.e. all decoding filters have processed the request), OR when the + // response is initiated. If not specified or set to 0, this timeout is disabled. + google.protobuf.Duration request_timeout = 28 [(gogoproto.stdduration) = true]; + + // The time that Envoy will wait between sending an HTTP/2 “shutdown + // notification” (GOAWAY frame with max stream ID) and a final GOAWAY frame. + // This is used so that Envoy provides a grace period for new streams that + // race with the final GOAWAY frame. During this grace period, Envoy will + // continue to accept new streams. After the grace period, a final GOAWAY + // frame is sent and Envoy will start refusing new streams. Draining occurs + // both when a connection hits the idle timeout or during general server + // draining. The default grace period is 5000 milliseconds (5 seconds) if this + // option is not specified. + google.protobuf.Duration drain_timeout = 12 [(gogoproto.stdduration) = true]; + + // The delayed close timeout is for downstream connections managed by the HTTP connection manager. + // It is defined as a grace period after connection close processing has been locally initiated + // during which Envoy will wait for the peer to close (i.e., a TCP FIN/RST is received by Envoy + // from the downstream connection) prior to Envoy closing the socket associated with that + // connection. + // NOTE: This timeout is enforced even when the socket associated with the downstream connection + // is pending a flush of the write buffer. However, any progress made writing data to the socket + // will restart the timer associated with this timeout. This means that the total grace period for + // a socket in this state will be + // +. + // + // Delaying Envoy's connection close and giving the peer the opportunity to initiate the close + // sequence mitigates a race condition that exists when downstream clients do not drain/process + // data in a connection's receive buffer after a remote close has been detected via a socket + // write(). This race leads to such clients failing to process the response code sent by Envoy, + // which could result in erroneous downstream processing. + // + // If the timeout triggers, Envoy will close the connection's socket. + // + // The default timeout is 1000 ms if this option is not specified. + // + // .. NOTE:: + // To be useful in avoiding the race condition described above, this timeout must be set + // to *at least* +<100ms to account for + // a reasonsable "worst" case processing time for a full iteration of Envoy's event loop>. + // + // .. WARNING:: + // A value of 0 will completely disable delayed close processing. When disabled, the downstream + // connection's socket will be closed immediately after the write flush is completed or will + // never close if the write flush does not complete. + google.protobuf.Duration delayed_close_timeout = 26 [(gogoproto.stdduration) = true]; + + // Configuration for :ref:`HTTP access logs ` + // emitted by the connection manager. + repeated envoy.config.filter.accesslog.v3alpha.AccessLog access_log = 13; + + // If set to true, the connection manager will use the real remote address + // of the client connection when determining internal versus external origin and manipulating + // various headers. If set to false or absent, the connection manager will use the + // :ref:`config_http_conn_man_headers_x-forwarded-for` HTTP header. See the documentation for + // :ref:`config_http_conn_man_headers_x-forwarded-for`, + // :ref:`config_http_conn_man_headers_x-envoy-internal`, and + // :ref:`config_http_conn_man_headers_x-envoy-external-address` for more information. + google.protobuf.BoolValue use_remote_address = 14; + + // The number of additional ingress proxy hops from the right side of the + // :ref:`config_http_conn_man_headers_x-forwarded-for` HTTP header to trust when + // determining the origin client's IP address. The default is zero if this option + // is not specified. See the documentation for + // :ref:`config_http_conn_man_headers_x-forwarded-for` for more information. + uint32 xff_num_trusted_hops = 19; + + message InternalAddressConfig { + // Whether unix socket addresses should be considered internal. + bool unix_sockets = 1; + } + + // Configures what network addresses are considered internal for stats and header sanitation + // purposes. If unspecified, only RFC1918 IP addresses will be considered internal. + // See the documentation for :ref:`config_http_conn_man_headers_x-envoy-internal` for more + // information about internal/external addresses. + InternalAddressConfig internal_address_config = 25; + + // If set, Envoy will not append the remote address to the + // :ref:`config_http_conn_man_headers_x-forwarded-for` HTTP header. This may be used in + // conjunction with HTTP filters that explicitly manipulate XFF after the HTTP connection manager + // has mutated the request headers. While :ref:`use_remote_address + // ` + // will also suppress XFF addition, it has consequences for logging and other + // Envoy uses of the remote address, so *skip_xff_append* should be used + // when only an elision of XFF addition is intended. + bool skip_xff_append = 21; + + // Via header value to append to request and response headers. If this is + // empty, no via header will be appended. + string via = 22; + + // Whether the connection manager will generate the :ref:`x-request-id + // ` header if it does not exist. This defaults to + // true. Generating a random UUID4 is expensive so in high throughput scenarios where this feature + // is not desired it can be disabled. + google.protobuf.BoolValue generate_request_id = 15; + + // Whether the connection manager will keep the :ref:`x-request-id + // ` header if passed for a request that is edge + // (Edge request is the request from external clients to front Envoy) and not reset it, which + // is the current Envoy behaviour. This defaults to false. + bool preserve_external_request_id = 32; + + // How to handle the :ref:`config_http_conn_man_headers_x-forwarded-client-cert` (XFCC) HTTP + // header. + enum ForwardClientCertDetails { + option (gogoproto.goproto_enum_prefix) = false; + + // Do not send the XFCC header to the next hop. This is the default value. + SANITIZE = 0; + + // When the client connection is mTLS (Mutual TLS), forward the XFCC header + // in the request. + FORWARD_ONLY = 1; + + // When the client connection is mTLS, append the client certificate + // information to the request’s XFCC header and forward it. + APPEND_FORWARD = 2; + + // When the client connection is mTLS, reset the XFCC header with the client + // certificate information and send it to the next hop. + SANITIZE_SET = 3; + + // Always forward the XFCC header in the request, regardless of whether the + // client connection is mTLS. + ALWAYS_FORWARD_ONLY = 4; + }; + + // How to handle the :ref:`config_http_conn_man_headers_x-forwarded-client-cert` (XFCC) HTTP + // header. + ForwardClientCertDetails forward_client_cert_details = 16 + [(validate.rules).enum.defined_only = true]; + + // [#comment:next free field: 7] + message SetCurrentClientCertDetails { + // Whether to forward the subject of the client cert. Defaults to false. + google.protobuf.BoolValue subject = 1; + + reserved 2; // san deprecated by uri + + // Whether to forward the entire client cert in URL encoded PEM format. This will appear in the + // XFCC header comma separated from other values with the value Cert="PEM". + // Defaults to false. + bool cert = 3; + + // Whether to forward the entire client cert chain (including the leaf cert) in URL encoded PEM + // format. This will appear in the XFCC header comma separated from other values with the value + // Chain="PEM". + // Defaults to false. + bool chain = 6; + + // Whether to forward the DNS type Subject Alternative Names of the client cert. + // Defaults to false. + bool dns = 4; + + // Whether to forward the URI type Subject Alternative Name of the client cert. Defaults to + // false. + bool uri = 5; + }; + + // This field is valid only when :ref:`forward_client_cert_details + // ` + // is APPEND_FORWARD or SANITIZE_SET and the client connection is mTLS. It specifies the fields in + // the client certificate to be forwarded. Note that in the + // :ref:`config_http_conn_man_headers_x-forwarded-client-cert` header, *Hash* is always set, and + // *By* is always set when the client certificate presents the URI type Subject Alternative Name + // value. + SetCurrentClientCertDetails set_current_client_cert_details = 17; + + // If proxy_100_continue is true, Envoy will proxy incoming "Expect: + // 100-continue" headers upstream, and forward "100 Continue" responses + // downstream. If this is false or not set, Envoy will instead strip the + // "Expect: 100-continue" header, and send a "100 Continue" response itself. + bool proxy_100_continue = 18; + + // If + // :ref:`use_remote_address + // ` + // is true and represent_ipv4_remote_address_as_ipv4_mapped_ipv6 is true and the remote address is + // an IPv4 address, the address will be mapped to IPv6 before it is appended to *x-forwarded-for*. + // This is useful for testing compatibility of upstream services that parse the header value. For + // example, 50.0.0.1 is represented as ::FFFF:50.0.0.1. See `IPv4-Mapped IPv6 Addresses + // `_ for details. This will also affect the + // :ref:`config_http_conn_man_headers_x-envoy-external-address` header. See + // :ref:`http_connection_manager.represent_ipv4_remote_address_as_ipv4_mapped_ipv6 + // ` for runtime + // control. + // [#not-implemented-hide:] + bool represent_ipv4_remote_address_as_ipv4_mapped_ipv6 = 20; + + // The configuration for HTTP upgrades. + // For each upgrade type desired, an UpgradeConfig must be added. + // + // .. warning:: + // + // The current implementation of upgrade headers does not handle + // multi-valued upgrade headers. Support for multi-valued headers may be + // added in the future if needed. + // + // .. warning:: + // The current implementation of upgrade headers does not work with HTTP/2 + // upstreams. + message UpgradeConfig { + // The case-insensitive name of this upgrade, e.g. "websocket". + // For each upgrade type present in upgrade_configs, requests with + // Upgrade: [upgrade_type] + // will be proxied upstream. + string upgrade_type = 1; + // If present, this represents the filter chain which will be created for + // this type of upgrade. If no filters are present, the filter chain for + // HTTP connections will be used for this upgrade type. + repeated HttpFilter filters = 2; + // Determines if upgrades are enabled or disabled by default. Defaults to true. + // This can be overridden on a per-route basis with :ref:`cluster + // ` as documented in the + // :ref:`upgrade documentation `. + google.protobuf.BoolValue enabled = 3; + }; + repeated UpgradeConfig upgrade_configs = 23; + + reserved 27; + + // Should paths be normalized according to RFC 3986 before any processing of + // requests by HTTP filters or routing? This affects the upstream *:path* header + // as well. For paths that fail this check, Envoy will respond with 400 to + // paths that are malformed. This defaults to false currently but will default + // true in the future. When not specified, this value may be overridden by the + // runtime variable + // :ref:`http_connection_manager.normalize_path`. + // See `Normalization and Comparison ` + // for details of normalization. + // Note that Envoy does not perform + // `case normalization ` + google.protobuf.BoolValue normalize_path = 30; + + // Determines if adjacent slashes in the path are merged into one before any processing of + // requests by HTTP filters or routing. This affects the upstream *:path* header as well. Without + // setting this option, incoming requests with path `//dir///file` will not match against route + // with `prefix` match set to `/dir`. Defaults to `false`. Note that slash merging is not part of + // `HTTP spec ` and is provided for convenience. + bool merge_slashes = 33; +} + +message Rds { + // Configuration source specifier for RDS. + envoy.api.v3alpha.core.ConfigSource config_source = 1 [(validate.rules).message.required = true]; + + // The name of the route configuration. This name will be passed to the RDS + // API. This allows an Envoy configuration with multiple HTTP listeners (and + // associated HTTP connection manager filters) to use different route + // configurations. + string route_config_name = 2 [(validate.rules).string.min_bytes = 1]; +} + +// This message is used to work around the limitations with 'oneof' and repeated fields. +message ScopedRouteConfigurationsList { + repeated envoy.api.v3alpha.ScopedRouteConfiguration scoped_route_configurations = 1 + [(validate.rules).repeated .min_items = 1]; +} + +message ScopedRoutes { + // The name assigned to the scoped routing configuration. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Specifies the mechanism for constructing "scope keys" based on HTTP request attributes. These + // keys are matched against a set of :ref:`Key` + // objects assembled from :ref:`ScopedRouteConfiguration` + // messages distributed via SRDS (the Scoped Route Discovery Service) or assigned statically via + // :ref:`scoped_route_configurations_list`. + // + // Upon receiving a request's headers, the Router will build a key using the algorithm specified + // by this message. This key will be used to look up the routing table (i.e., the + // :ref:`RouteConfiguration`) to use for the request. + message ScopeKeyBuilder { + // Specifies the mechanism for constructing key fragments which are composed into scope keys. + message FragmentBuilder { + // Specifies how the value of a header should be extracted. + // The following example maps the structure of a header to the fields in this message. + // + // .. code:: + // + // <0> <1> <-- index + // X-Header: a=b;c=d + // | || | + // | || \----> + // | || + // | |\----> + // | | + // | \----> + // | + // \----> + // + // Each 'a=b' key-value pair constitutes an 'element' of the header field. + message HeaderValueExtractor { + // The name of the header field to extract the value from. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // The element separator (e.g., ';' separates 'a;b;c;d'). + // Default: empty string. This causes the entirety of the header field to be extracted. + // If this field is set to an empty string and 'index' is used in the oneof below, 'index' + // must be set to 0. + string element_separator = 2; + + // Specifies a header field's key value pair to match on. + message KvElement { + // The separator between key and value (e.g., '=' separates 'k=v;...'). + // If an element is an empty string, the element is ignored. + // If an element contains no separator, the whole element is parsed as key and the + // fragment value is an empty string. + // If there are multiple values for a matched key, the first value is returned. + string separator = 1 [(validate.rules).string.min_bytes = 1]; + + // The key to match on. + string key = 2 [(validate.rules).string.min_bytes = 1]; + } + + oneof extract_type { + // Specifies the zero based index of the element to extract. + // Note Envoy concatenates multiple values of the same header key into a comma separated + // string, the splitting always happens after the concatenation. + uint32 index = 3; + + // Specifies the key value pair to extract the value from. + KvElement element = 4; + } + } + + oneof type { + option (validate.required) = true; + + // Specifies how a header field's value should be extracted. + HeaderValueExtractor header_value_extractor = 1; + } + } + + // The final scope key consists of the ordered union of these fragments. + repeated FragmentBuilder fragments = 1 [(validate.rules).repeated .min_items = 1]; + } + + // The algorithm to use for constructing a scope key for each request. + ScopeKeyBuilder scope_key_builder = 2 [(validate.rules).message.required = true]; + + // Configuration source specifier for RDS. + // This config source is used to subscribe to RouteConfiguration resources specified in + // ScopedRouteConfiguration messages. + envoy.api.v3alpha.core.ConfigSource rds_config_source = 3 + [(validate.rules).message.required = true]; + + oneof config_specifier { + option (validate.required) = true; + + // The set of routing scopes corresponding to the HCM. A scope is assigned to a request by + // matching a key constructed from the request's attributes according to the algorithm specified + // by the + // :ref:`ScopeKeyBuilder` + // in this message. + ScopedRouteConfigurationsList scoped_route_configurations_list = 4; + + // The set of routing scopes associated with the HCM will be dynamically loaded via the SRDS + // API. A scope is assigned to a request by matching a key constructed from the request's + // attributes according to the algorithm specified by the + // :ref:`ScopeKeyBuilder` + // in this message. + ScopedRds scoped_rds = 5; + } +} + +message ScopedRds { + // Configuration source specifier for scoped RDS. + envoy.api.v3alpha.core.ConfigSource scoped_rds_config_source = 1 + [(validate.rules).message.required = true]; +} + +message HttpFilter { + // The name of the filter to instantiate. The name must match a + // :ref:`supported filter `. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Filter specific configuration which depends on the filter being instantiated. See the supported + // filters for further documentation. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 4; + } + + reserved 3; +} diff --git a/api/envoy/config/filter/network/mongo_proxy/v3alpha/BUILD b/api/envoy/config/filter/network/mongo_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..a2c09e709030 --- /dev/null +++ b/api/envoy/config/filter/network/mongo_proxy/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "mongo_proxy", + srcs = ["mongo_proxy.proto"], + deps = ["//envoy/config/filter/fault/v3alpha:fault"], +) diff --git a/api/envoy/config/filter/network/mongo_proxy/v3alpha/mongo_proxy.proto b/api/envoy/config/filter/network/mongo_proxy/v3alpha/mongo_proxy.proto new file mode 100644 index 000000000000..9149b433e372 --- /dev/null +++ b/api/envoy/config/filter/network/mongo_proxy/v3alpha/mongo_proxy.proto @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package envoy.config.filter.network.mongo_proxy.v3alpha; + +option java_outer_classname = "MongoProxyProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.mongo_proxy.v3alpha"; +option go_package = "v2"; + +import "envoy/config/filter/fault/v3alpha/fault.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Mongo proxy] +// MongoDB :ref:`configuration overview `. + +message MongoProxy { + // The human readable prefix to use when emitting :ref:`statistics + // `. + string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + // The optional path to use for writing Mongo access logs. If not access log + // path is specified no access logs will be written. Note that access log is + // also gated :ref:`runtime `. + string access_log = 2; + + // Inject a fixed delay before proxying a Mongo operation. Delays are + // applied to the following MongoDB operations: Query, Insert, GetMore, + // and KillCursors. Once an active delay is in progress, all incoming + // data up until the timer event fires will be a part of the delay. + envoy.config.filter.fault.v3alpha.FaultDelay delay = 3; + + // Flag to specify whether :ref:`dynamic metadata + // ` should be emitted. Defaults to false. + bool emit_dynamic_metadata = 4; +} diff --git a/api/envoy/config/filter/network/rate_limit/v3alpha/BUILD b/api/envoy/config/filter/network/rate_limit/v3alpha/BUILD new file mode 100644 index 000000000000..9dc17266721c --- /dev/null +++ b/api/envoy/config/filter/network/rate_limit/v3alpha/BUILD @@ -0,0 +1,12 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "rate_limit", + srcs = ["rate_limit.proto"], + deps = [ + "//envoy/api/v3alpha/ratelimit", + "//envoy/config/ratelimit/v3alpha:rls", + ], +) diff --git a/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto b/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto new file mode 100644 index 000000000000..a0edc98e561d --- /dev/null +++ b/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package envoy.config.filter.network.rate_limit.v3alpha; + +option java_outer_classname = "RateLimitProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.rate_limit.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/ratelimit/ratelimit.proto"; +import "envoy/config/ratelimit/v3alpha/rls.proto"; + +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Rate limit] +// Rate limit :ref:`configuration overview `. + +message RateLimit { + // The prefix to use when emitting :ref:`statistics `. + string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + // The rate limit domain to use in the rate limit service request. + string domain = 2 [(validate.rules).string.min_bytes = 1]; + + // The rate limit descriptor list to use in the rate limit service request. + repeated envoy.api.v3alpha.ratelimit.RateLimitDescriptor descriptors = 3 + [(validate.rules).repeated .min_items = 1]; + + // The timeout in milliseconds for the rate limit service RPC. If not + // set, this defaults to 20ms. + google.protobuf.Duration timeout = 4 [(gogoproto.stdduration) = true]; + + // The filter's behaviour in case the rate limiting service does + // not respond back. When it is set to true, Envoy will not allow traffic in case of + // communication failure between rate limiting service and the proxy. + // Defaults to false. + bool failure_mode_deny = 5; + + // Configuration for an external rate limit service provider. If not + // specified, any calls to the rate limit service will immediately return + // success. + envoy.config.ratelimit.v3alpha.RateLimitServiceConfig rate_limit_service = 6 + [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/filter/network/rbac/v3alpha/BUILD b/api/envoy/config/filter/network/rbac/v3alpha/BUILD new file mode 100644 index 000000000000..a6ee42cf7893 --- /dev/null +++ b/api/envoy/config/filter/network/rbac/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "rbac", + srcs = ["rbac.proto"], + deps = ["//envoy/config/rbac/v3alpha:rbac"], +) diff --git a/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto b/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto new file mode 100644 index 000000000000..5c2114cd6063 --- /dev/null +++ b/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto @@ -0,0 +1,52 @@ +syntax = "proto3"; + +package envoy.config.filter.network.rbac.v3alpha; + +option java_outer_classname = "RbacProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.rbac.v3alpha"; +option go_package = "v2"; + +import "envoy/config/rbac/v3alpha/rbac.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: RBAC] +// Role-Based Access Control :ref:`configuration overview `. + +// RBAC network filter config. +// +// Header should not be used in rules/shadow_rules in RBAC network filter as +// this information is only available in :ref:`RBAC http filter `. +message RBAC { + // Specify the RBAC rules to be applied globally. + // If absent, no enforcing RBAC policy will be applied. + config.rbac.v3alpha.RBAC rules = 1; + + // Shadow rules are not enforced by the filter but will emit stats and logs + // and can be used for rule testing. + // If absent, no shadow RBAC policy will be applied. + config.rbac.v3alpha.RBAC shadow_rules = 2; + + // The prefix to use when emitting statistics. + string stat_prefix = 3 [(validate.rules).string.min_bytes = 1]; + + enum EnforcementType { + // Apply RBAC policies when the first byte of data arrives on the connection. + ONE_TIME_ON_FIRST_BYTE = 0; + + // Continuously apply RBAC policies as data arrives. Use this mode when + // using RBAC with message oriented protocols such as Mongo, MySQL, Kafka, + // etc. when the protocol decoders emit dynamic metadata such as the + // resources being accessed and the operations on the resources. + CONTINUOUS = 1; + }; + + // RBAC enforcement strategy. By default RBAC will be enforced only once + // when the first byte of data arrives from the downstream. When used in + // conjunction with filters that emit dynamic metadata after decoding + // every payload (e.g., Mongo, MySQL, Kafka) set the enforcement type to + // CONTINUOUS to enforce RBAC policies on every message boundary. + EnforcementType enforcement_type = 4; +} diff --git a/api/envoy/config/filter/network/redis_proxy/v3alpha/BUILD b/api/envoy/config/filter/network/redis_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..ef7cc5683f0c --- /dev/null +++ b/api/envoy/config/filter/network/redis_proxy/v3alpha/BUILD @@ -0,0 +1,12 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "redis_proxy", + srcs = ["redis_proxy.proto"], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/type:percent", + ], +) diff --git a/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto new file mode 100644 index 000000000000..1bda0ab7c466 --- /dev/null +++ b/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto @@ -0,0 +1,236 @@ +syntax = "proto3"; + +package envoy.config.filter.network.redis_proxy.v3alpha; + +option java_outer_classname = "RedisProxyProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.redis_proxy.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Redis Proxy] +// Redis Proxy :ref:`configuration overview `. + +message RedisProxy { + // The prefix to use when emitting :ref:`statistics `. + string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + // Name of cluster from cluster manager. See the :ref:`configuration section + // ` of the architecture overview for recommendations on + // configuring the backing cluster. + // + // .. attention:: + // + // This field is deprecated. Use a :ref:`catch_all + // route` + // instead. + string cluster = 2 [deprecated = true]; + + // Redis connection pool settings. + message ConnPoolSettings { + // Per-operation timeout in milliseconds. The timer starts when the first + // command of a pipeline is written to the backend connection. Each response received from Redis + // resets the timer since it signifies that the next command is being processed by the backend. + // The only exception to this behavior is when a connection to a backend is not yet established. + // In that case, the connect timeout on the cluster will govern the timeout until the connection + // is ready. + google.protobuf.Duration op_timeout = 1 + [(validate.rules).duration.required = true, (gogoproto.stdduration) = true]; + + // Use hash tagging on every redis key to guarantee that keys with the same hash tag will be + // forwarded to the same upstream. The hash key used for determining the upstream in a + // consistent hash ring configuration will be computed from the hash tagged key instead of the + // whole key. The algorithm used to compute the hash tag is identical to the `redis-cluster + // implementation `_. + // + // Examples: + // + // * '{user1000}.following' and '{user1000}.followers' **will** be sent to the same upstream + // * '{user1000}.following' and '{user1001}.following' **might** be sent to the same upstream + bool enable_hashtagging = 2; + + // Accept `moved and ask redirection + // `_ errors from upstream + // redis servers, and retry commands to the specified target server. The target server does not + // need to be known to the cluster manager. If the command cannot be redirected, then the + // original error is passed downstream unchanged. By default, this support is not enabled. + bool enable_redirection = 3; + + // Maximum size of encoded request buffer before flush is triggered and encoded requests + // are sent upstream. If this is unset, the buffer flushes whenever it receives data + // and performs no batching. + // This feature makes it possible for multiple clients to send requests to Envoy and have + // them batched- for example if one is running several worker processes, each with its own + // Redis connection. There is no benefit to using this with a single downstream process. + // Recommended size (if enabled) is 1024 bytes. + uint32 max_buffer_size_before_flush = 4; + + // The encoded request buffer is flushed N milliseconds after the first request has been + // encoded, unless the buffer size has already exceeded `max_buffer_size_before_flush`. + // If `max_buffer_size_before_flush` is not set, this flush timer is not used. Otherwise, + // the timer should be set according to the number of clients, overall request rate and + // desired maximum latency for a single command. For example, if there are many requests + // being batched together at a high rate, the buffer will likely be filled before the timer + // fires. Alternatively, if the request rate is lower the buffer will not be filled as often + // before the timer fires. + // If `max_buffer_size_before_flush` is set, but `buffer_flush_timeout` is not, the latter + // defaults to 3ms. + google.protobuf.Duration buffer_flush_timeout = 5 [(gogoproto.stdduration) = true]; + + // `max_upstream_unknown_connections` controls how many upstream connections to unknown hosts + // can be created at any given time by any given worker thread (see `enable_redirection` for + // more details). If the host is unknown and a connection cannot be created due to enforcing + // this limit, then redirection will fail and the original redirection error will be passed + // downstream unchanged. This limit defaults to 100. + google.protobuf.UInt32Value max_upstream_unknown_connections = 6; + + // ReadPolicy controls how Envoy routes read commands to Redis nodes. This is currently + // supported for Redis Cluster. All ReadPolicy settings except MASTER may return stale data + // because replication is asynchronous and requires some delay. You need to ensure that your + // application can tolerate stale data. + enum ReadPolicy { + // Default mode. Read from the current master node. + MASTER = 0; + // Read from the master, but if it is unavailable, read from replica nodes. + PREFER_MASTER = 1; + // Read from replica nodes. If multiple replica nodes are present within a shard, a random + // node is selected. Healthy nodes have precedent over unhealthy nodes. + REPLICA = 2; + // Read from the replica nodes (similar to REPLICA), but if all replicas are unavailable (not + // present or unhealthy), read from the master. + PREFER_REPLICA = 3; + // Read from any node of the cluster. A random node is selected among the master and replicas, + // healthy nodes have precedent over unhealthy nodes. + ANY = 4; + } + + // Read policy. The default is to read from the master. + ReadPolicy read_policy = 7 [(validate.rules).enum.defined_only = true]; + } + + // Network settings for the connection pool to the upstream clusters. + ConnPoolSettings settings = 3 [(validate.rules).message.required = true]; + + // Indicates that latency stat should be computed in microseconds. By default it is computed in + // milliseconds. + bool latency_in_micros = 4; + + message PrefixRoutes { + message Route { + // String prefix that must match the beginning of the keys. Envoy will always favor the + // longest match. + string prefix = 1; + + // Indicates if the prefix needs to be removed from the key when forwarded. + bool remove_prefix = 2; + + // Upstream cluster to forward the command to. + string cluster = 3 [(validate.rules).string.min_bytes = 1]; + + // The router is capable of shadowing traffic from one cluster to another. The current + // implementation is "fire and forget," meaning Envoy will not wait for the shadow cluster to + // respond before returning the response from the primary cluster. All normal statistics are + // collected for the shadow cluster making this feature useful for testing. + message RequestMirrorPolicy { + // Specifies the cluster that requests will be mirrored to. The cluster must + // exist in the cluster manager configuration. + string cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // If not specified or the runtime key is not present, all requests to the target cluster + // will be mirrored. + // + // If specified, Envoy will lookup the runtime key to get the percentage of requests to the + // mirror. + // + // Parsing this field is implemented such that the runtime key's data may be represented + // as a :ref:`FractionalPercent ` proto represented + // as JSON/YAML and may also be represented as an integer with the assumption that the value + // is an integral percentage out of 100. For instance, a runtime key lookup returning the + // value "42" would parse as a `FractionalPercent` whose numerator is 42 and denominator is + // HUNDRED. + envoy.api.v3alpha.core.RuntimeFractionalPercent runtime_fraction = 2; + + // Set this to TRUE to only mirror write commands, this is effectively replicating the + // writes in a "fire and forget" manner. + bool exclude_read_commands = 3; + } + + // Indicates that the route has a request mirroring policy. + repeated RequestMirrorPolicy request_mirror_policy = 4; + } + + // List of prefix routes. + repeated Route routes = 1; + + // Indicates that prefix matching should be case insensitive. + bool case_insensitive = 2; + + // Optional catch-all route to forward commands that doesn't match any of the routes. The + // catch-all route becomes required when no routes are specified. + // .. attention:: + // + // This field is deprecated. Use a :ref:`catch_all + // route` + // instead. + string catch_all_cluster = 3 [deprecated = true]; + + // Optional catch-all route to forward commands that doesn't match any of the routes. The + // catch-all route becomes required when no routes are specified. + Route catch_all_route = 4; + } + + // List of **unique** prefixes used to separate keys from different workloads to different + // clusters. Envoy will always favor the longest match first in case of overlap. A catch-all + // cluster can be used to forward commands when there is no match. Time complexity of the + // lookups are in O(min(longest key prefix, key length)). + // + // Example: + // + // .. code-block:: yaml + // + // prefix_routes: + // routes: + // - prefix: "ab" + // cluster: "cluster_a" + // - prefix: "abc" + // cluster: "cluster_b" + // + // When using the above routes, the following prefixes would be sent to: + // + // * 'get abc:users' would retrive the key 'abc:users' from cluster_b. + // * 'get ab:users' would retrive the key 'ab:users' from cluster_a. + // * 'get z:users' would return a NoUpstreamHost error. A :ref:`catch-all + // route` + // would have retrieved the key from that cluster instead. + // + // See the :ref:`configuration section + // ` of the architecture overview for recommendations on + // configuring the backing clusters. + PrefixRoutes prefix_routes = 5; + + // Authenticate Redis client connections locally by forcing downstream clients to issue a 'Redis + // AUTH command `_ with this password before enabling any other + // command. If an AUTH command's password matches this password, an "OK" response will be returned + // to the client. If the AUTH command password does not match this password, then an "ERR invalid + // password" error will be returned. If any other command is received before AUTH when this + // password is set, then a "NOAUTH Authentication required." error response will be sent to the + // client. If an AUTH command is received when the password is not set, then an "ERR Client sent + // AUTH, but no password is set" error will be returned. + envoy.api.v3alpha.core.DataSource downstream_auth_password = 6; +} + +// RedisProtocolOptions specifies Redis upstream protocol options. This object is used in +// :ref:`extension_protocol_options`, keyed +// by the name `envoy.redis_proxy`. +message RedisProtocolOptions { + // Upstream server password as defined by the `requirepass directive + // `_ in the server's configuration file. + envoy.api.v3alpha.core.DataSource auth_password = 1; +} diff --git a/api/envoy/config/filter/network/tcp_proxy/v3alpha/BUILD b/api/envoy/config/filter/network/tcp_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..a9ea8de3bf1e --- /dev/null +++ b/api/envoy/config/filter/network/tcp_proxy/v3alpha/BUILD @@ -0,0 +1,23 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "tcp_proxy", + srcs = ["tcp_proxy.proto"], + deps = [ + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + "//envoy/config/filter/accesslog/v3alpha:accesslog", + ], +) + +api_go_proto_library( + name = "tcp_proxy", + proto = ":tcp_proxy", + deps = [ + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/config/filter/accesslog/v3alpha:accesslog_go_proto", + ], +) diff --git a/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto b/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto new file mode 100644 index 000000000000..f2597a3ab361 --- /dev/null +++ b/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto @@ -0,0 +1,146 @@ +syntax = "proto3"; + +package envoy.config.filter.network.tcp_proxy.v3alpha; + +option java_outer_classname = "TcpProxyProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.tcp_proxy.v3alpha"; +option go_package = "v2"; + +import "envoy/config/filter/accesslog/v3alpha/accesslog.proto"; +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: TCP Proxy] +// TCP Proxy :ref:`configuration overview `. + +message TcpProxy { + // The prefix to use when emitting :ref:`statistics + // `. + string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + oneof cluster_specifier { + option (validate.required) = true; + + // The upstream cluster to connect to. + // + string cluster = 2; + + // Multiple upstream clusters can be specified for a given route. The + // request is routed to one of the upstream clusters based on weights + // assigned to each cluster. + WeightedCluster weighted_clusters = 10; + } + + // Optional endpoint metadata match criteria. Only endpoints in the upstream + // cluster with metadata matching that set in metadata_match will be + // considered. The filter name should be specified as *envoy.lb*. + envoy.api.v3alpha.core.Metadata metadata_match = 9; + + // The idle timeout for connections managed by the TCP proxy filter. The idle timeout + // is defined as the period in which there are no bytes sent or received on either + // the upstream or downstream connection. If not set, connections will never be closed + // by the TCP proxy due to being idle. + google.protobuf.Duration idle_timeout = 8 + [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + + // [#not-implemented-hide:] The idle timeout for connections managed by the TCP proxy + // filter. The idle timeout is defined as the period in which there is no + // active traffic. If not set, there is no idle timeout. When the idle timeout + // is reached the connection will be closed. The distinction between + // downstream_idle_timeout/upstream_idle_timeout provides a means to set + // timeout based on the last byte sent on the downstream/upstream connection. + google.protobuf.Duration downstream_idle_timeout = 3; + + // [#not-implemented-hide:] + google.protobuf.Duration upstream_idle_timeout = 4; + + // Configuration for :ref:`access logs ` + // emitted by the this tcp_proxy. + repeated envoy.config.filter.accesslog.v3alpha.AccessLog access_log = 5; + + // [#not-implemented-hide:] Deprecated. + // TCP Proxy filter configuration using V1 format. + message DeprecatedV1 { + // A TCP proxy route consists of a set of optional L4 criteria and the + // name of a cluster. If a downstream connection matches all the + // specified criteria, the cluster in the route is used for the + // corresponding upstream connection. Routes are tried in the order + // specified until a match is found. If no match is found, the connection + // is closed. A route with no criteria is valid and always produces a + // match. + message TCPRoute { + // The cluster to connect to when a the downstream network connection + // matches the specified criteria. + string cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // An optional list of IP address subnets in the form + // “ip_address/xx”. The criteria is satisfied if the destination IP + // address of the downstream connection is contained in at least one of + // the specified subnets. If the parameter is not specified or the list + // is empty, the destination IP address is ignored. The destination IP + // address of the downstream connection might be different from the + // addresses on which the proxy is listening if the connection has been + // redirected. + repeated envoy.api.v3alpha.core.CidrRange destination_ip_list = 2; + + // An optional string containing a comma-separated list of port numbers + // or ranges. The criteria is satisfied if the destination port of the + // downstream connection is contained in at least one of the specified + // ranges. If the parameter is not specified, the destination port is + // ignored. The destination port address of the downstream connection + // might be different from the port on which the proxy is listening if + // the connection has been redirected. + string destination_ports = 3; + + // An optional list of IP address subnets in the form + // “ip_address/xx”. The criteria is satisfied if the source IP address + // of the downstream connection is contained in at least one of the + // specified subnets. If the parameter is not specified or the list is + // empty, the source IP address is ignored. + repeated envoy.api.v3alpha.core.CidrRange source_ip_list = 4; + + // An optional string containing a comma-separated list of port numbers + // or ranges. The criteria is satisfied if the source port of the + // downstream connection is contained in at least one of the specified + // ranges. If the parameter is not specified, the source port is + // ignored. + string source_ports = 5; + } + + // The route table for the filter. All filter instances must have a route + // table, even if it is empty. + repeated TCPRoute routes = 1 [(validate.rules).repeated .min_items = 1]; + } + + // [#not-implemented-hide:] Deprecated. + DeprecatedV1 deprecated_v1 = 6 [deprecated = true]; + + // The maximum number of unsuccessful connection attempts that will be made before + // giving up. If the parameter is not specified, 1 connection attempt will be made. + google.protobuf.UInt32Value max_connect_attempts = 7 [(validate.rules).uint32.gte = 1]; + + // Allows for specification of multiple upstream clusters along with weights + // that indicate the percentage of traffic to be forwarded to each cluster. + // The router selects an upstream cluster based on these weights. + message WeightedCluster { + message ClusterWeight { + // Name of the upstream cluster. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // When a request matches the route, the choice of an upstream cluster is + // determined by its weight. The sum of weights across all entries in the + // clusters array determines the total weight. + uint32 weight = 2 [(validate.rules).uint32.gte = 1]; + } + + // Specifies one or more upstream clusters associated with the route. + repeated ClusterWeight clusters = 1 [(validate.rules).repeated .min_items = 1]; + } +} diff --git a/api/envoy/config/grpc_credential/v3alpha/BUILD b/api/envoy/config/grpc_credential/v3alpha/BUILD new file mode 100644 index 000000000000..2f6736732881 --- /dev/null +++ b/api/envoy/config/grpc_credential/v3alpha/BUILD @@ -0,0 +1,27 @@ +licenses(["notice"]) # Apache 2 + +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +api_proto_library_internal( + name = "file_based_metadata", + srcs = ["file_based_metadata.proto"], + deps = ["//envoy/api/v3alpha/core:base"], +) + +api_go_proto_library( + name = "file_based_metadata", + proto = ":file_based_metadata", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + ], +) + +api_proto_library_internal( + name = "aws_iam", + srcs = ["aws_iam.proto"], +) + +api_go_proto_library( + name = "aws_iam", + proto = ":aws_iam", +) diff --git a/api/envoy/config/grpc_credential/v3alpha/aws_iam.proto b/api/envoy/config/grpc_credential/v3alpha/aws_iam.proto new file mode 100644 index 000000000000..33921db6d69a --- /dev/null +++ b/api/envoy/config/grpc_credential/v3alpha/aws_iam.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +// [#protodoc-title: Grpc Credentials AWS IAM] +// Configuration for AWS IAM Grpc Credentials Plugin + +package envoy.config.grpc_credential.v3alpha; + +option java_outer_classname = "AwsIamProto"; +option java_package = "io.envoyproxy.envoy.config.grpc_credential.v3alpha"; +option java_multiple_files = true; +option go_package = "v2alpha"; + +import "validate/validate.proto"; + +message AwsIamConfig { + // The `service namespace + // `_ + // of the Grpc endpoint. + // + // Example: appmesh + string service_name = 1 [(validate.rules).string.min_bytes = 1]; + + // The `region `_ hosting the Grpc + // endpoint. If unspecified, the extension will use the value in the ``AWS_REGION`` environment + // variable. + // + // Example: us-west-2 + string region = 2; +} diff --git a/api/envoy/config/grpc_credential/v3alpha/file_based_metadata.proto b/api/envoy/config/grpc_credential/v3alpha/file_based_metadata.proto new file mode 100644 index 000000000000..2886921b3415 --- /dev/null +++ b/api/envoy/config/grpc_credential/v3alpha/file_based_metadata.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +// [#protodoc-title: Grpc Credentials File Based Metadata] +// Configuration for File Based Metadata Grpc Credentials Plugin + +package envoy.config.grpc_credential.v3alpha; + +option java_outer_classname = "FileBasedMetadataProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.grpc_credential.v3alpha"; +option go_package = "v2alpha"; + +import "envoy/api/v3alpha/core/base.proto"; + +message FileBasedMetadataConfig { + + // Location or inline data of secret to use for authentication of the Google gRPC connection + // this secret will be attached to a header of the gRPC connection + envoy.api.v3alpha.core.DataSource secret_data = 1; + + // Metadata header key to use for sending the secret data + // if no header key is set, "authorization" header will be used + string header_key = 2; + + // Prefix to prepend to the secret in the metadata header + // if no prefix is set, the default is to use no prefix + string header_prefix = 3; +} diff --git a/api/envoy/config/health_checker/redis/v3alpha/BUILD b/api/envoy/config/health_checker/redis/v3alpha/BUILD new file mode 100644 index 000000000000..239d1f224fc6 --- /dev/null +++ b/api/envoy/config/health_checker/redis/v3alpha/BUILD @@ -0,0 +1,8 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "redis", + srcs = ["redis.proto"], +) diff --git a/api/envoy/config/health_checker/redis/v3alpha/redis.proto b/api/envoy/config/health_checker/redis/v3alpha/redis.proto new file mode 100644 index 000000000000..234da40d56ba --- /dev/null +++ b/api/envoy/config/health_checker/redis/v3alpha/redis.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package envoy.config.health_checker.redis.v3alpha; + +option java_outer_classname = "RedisProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.health_checker.redis.v3alpha"; +option go_package = "v2"; + +// [#protodoc-title: Redis] +// Redis health checker :ref:`configuration overview `. + +message Redis { + // If set, optionally perform ``EXISTS `` instead of ``PING``. A return value + // from Redis of 0 (does not exist) is considered a passing healthcheck. A return value other + // than 0 is considered a failure. This allows the user to mark a Redis instance for maintenance + // by setting the specified key to any value and waiting for traffic to drain. + string key = 1; +} diff --git a/api/envoy/config/metrics/v3alpha/BUILD b/api/envoy/config/metrics/v3alpha/BUILD new file mode 100644 index 000000000000..39d0b79654d0 --- /dev/null +++ b/api/envoy/config/metrics/v3alpha/BUILD @@ -0,0 +1,43 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "metrics_service", + srcs = ["metrics_service.proto"], + visibility = [ + "//envoy/config/bootstrap/v3alpha:__pkg__", + ], + deps = [ + "//envoy/api/v3alpha/core:grpc_service", + ], +) + +api_go_proto_library( + name = "metrics_service", + proto = ":metrics_service", + deps = [ + "//envoy/api/v3alpha/core:grpc_service_go_proto", + ], +) + +api_proto_library_internal( + name = "stats", + srcs = ["stats.proto"], + visibility = [ + "//envoy/config/bootstrap/v3alpha:__pkg__", + ], + deps = [ + "//envoy/api/v3alpha/core:address", + "//envoy/type/matcher:string", + ], +) + +api_go_proto_library( + name = "stats", + proto = ":stats", + deps = [ + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/type/matcher:string_go_proto", + ], +) diff --git a/api/envoy/config/metrics/v3alpha/metrics_service.proto b/api/envoy/config/metrics/v3alpha/metrics_service.proto new file mode 100644 index 000000000000..392ceb8d6fed --- /dev/null +++ b/api/envoy/config/metrics/v3alpha/metrics_service.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +// [#protodoc-title: Metrics service] + +package envoy.config.metrics.v3alpha; + +option java_outer_classname = "MetricsServiceProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.metrics.v3alpha"; + +import "envoy/api/v3alpha/core/grpc_service.proto"; + +import "validate/validate.proto"; + +// Metrics Service is configured as a built-in *envoy.metrics_service* :ref:`StatsSink +// `. This opaque configuration will be used to +// create Metrics Service. +message MetricsServiceConfig { + // The upstream gRPC cluster that hosts the metrics service. + envoy.api.v3alpha.core.GrpcService grpc_service = 1 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/metrics/v3alpha/stats.proto b/api/envoy/config/metrics/v3alpha/stats.proto new file mode 100644 index 000000000000..91324ed0ef61 --- /dev/null +++ b/api/envoy/config/metrics/v3alpha/stats.proto @@ -0,0 +1,331 @@ +// [#protodoc-title: Stats] +// Statistics :ref:`architecture overview `. + +syntax = "proto3"; + +package envoy.config.metrics.v3alpha; + +option java_outer_classname = "StatsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.metrics.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/type/matcher/string.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +// Configuration for pluggable stats sinks. +message StatsSink { + // The name of the stats sink to instantiate. The name must match a supported + // stats sink. The built-in stats sinks are: + // + // * :ref:`envoy.statsd ` + // * :ref:`envoy.dog_statsd ` + // * :ref:`envoy.metrics_service ` + // * :ref:`envoy.stat_sinks.hystrix ` + // + // Sinks optionally support tagged/multiple dimensional metrics. + string name = 1; + + // Stats sink specific configuration which depends on the sink being instantiated. See + // :ref:`StatsdSink ` for an example. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} + +// Statistics configuration such as tagging. +message StatsConfig { + // Each stat name is iteratively processed through these tag specifiers. + // When a tag is matched, the first capture group is removed from the name so + // later :ref:`TagSpecifiers ` cannot match + // that same portion of the match. + repeated TagSpecifier stats_tags = 1; + + // Use all default tag regexes specified in Envoy. These can be combined with + // custom tags specified in :ref:`stats_tags + // `. They will be processed before + // the custom tags. + // + // .. note:: + // + // If any default tags are specified twice, the config will be considered + // invalid. + // + // See :repo:`well_known_names.h ` for a list of the + // default tags in Envoy. + // + // If not provided, the value is assumed to be true. + google.protobuf.BoolValue use_all_default_tags = 2; + + // Inclusion/exclusion matcher for stat name creation. If not provided, all stats are instantiated + // as normal. Preventing the instantiation of certain families of stats can improve memory + // performance for Envoys running especially large configs. + StatsMatcher stats_matcher = 3; +} + +// Configuration for disabling stat instantiation. +message StatsMatcher { + // The instantiation of stats is unrestricted by default. If the goal is to configure Envoy to + // instantiate all stats, there is no need to construct a StatsMatcher. + // + // However, StatsMatcher can be used to limit the creation of families of stats in order to + // conserve memory. Stats can either be disabled entirely, or they can be + // limited by either an exclusion or an inclusion list of :ref:`StringMatcher + // ` protos: + // + // * If `reject_all` is set to `true`, no stats will be instantiated. If `reject_all` is set to + // `false`, all stats will be instantiated. + // + // * If an exclusion list is supplied, any stat name matching *any* of the StringMatchers in the + // list will not instantiate. + // + // * If an inclusion list is supplied, no stats will instantiate, except those matching *any* of + // the StringMatchers in the list. + // + // + // A StringMatcher can be used to match against an exact string, a suffix / prefix, or a regex. + // **NB:** For performance reasons, it is highly recommended to use a prefix- or suffix-based + // matcher rather than a regex-based matcher. + // + // Example 1. Excluding all stats. + // + // .. code-block:: json + // + // { + // "statsMatcher": { + // "rejectAll": "true" + // } + // } + // + // Example 2. Excluding all cluster-specific stats, but not cluster-manager stats: + // + // .. code-block:: json + // + // { + // "statsMatcher": { + // "exclusionList": { + // "patterns": [ + // { + // "prefix": "cluster." + // } + // ] + // } + // } + // } + // + // Example 3. Including only manager-related stats: + // + // .. code-block:: json + // + // { + // "statsMatcher": { + // "inclusionList": { + // "patterns": [ + // { + // "prefix": "cluster_manager." + // }, + // { + // "prefix": "listener_manager." + // } + // ] + // } + // } + // } + // + + oneof stats_matcher { + option (validate.required) = true; + + // If `reject_all` is true, then all stats are disabled. If `reject_all` is false, then all + // stats are enabled. + bool reject_all = 1; + + // Exclusive match. All stats are enabled except for those matching one of the supplied + // StringMatcher protos. + envoy.type.matcher.ListStringMatcher exclusion_list = 2; + + // Inclusive match. No stats are enabled except for those matching one of the supplied + // StringMatcher protos. + envoy.type.matcher.ListStringMatcher inclusion_list = 3; + }; +} + +// Designates a tag name and value pair. The value may be either a fixed value +// or a regex providing the value via capture groups. The specified tag will be +// unconditionally set if a fixed value, otherwise it will only be set if one +// or more capture groups in the regex match. +message TagSpecifier { + // Attaches an identifier to the tag values to identify the tag being in the + // sink. Envoy has a set of default names and regexes to extract dynamic + // portions of existing stats, which can be found in :repo:`well_known_names.h + // ` in the Envoy repository. If a :ref:`tag_name + // ` is provided in the config and + // neither :ref:`regex ` or + // :ref:`fixed_value ` were + // specified, Envoy will attempt to find that name in its set of defaults and use the accompanying + // regex. + // + // .. note:: + // + // It is invalid to specify the same tag name twice in a config. + string tag_name = 1; + + oneof tag_value { + // Designates a tag to strip from the tag extracted name and provide as a named + // tag value for all statistics. This will only occur if any part of the name + // matches the regex provided with one or more capture groups. + // + // The first capture group identifies the portion of the name to remove. The + // second capture group (which will normally be nested inside the first) will + // designate the value of the tag for the statistic. If no second capture + // group is provided, the first will also be used to set the value of the tag. + // All other capture groups will be ignored. + // + // Example 1. a stat name ``cluster.foo_cluster.upstream_rq_timeout`` and + // one tag specifier: + // + // .. code-block:: json + // + // { + // "tag_name": "envoy.cluster_name", + // "regex": "^cluster\.((.+?)\.)" + // } + // + // Note that the regex will remove ``foo_cluster.`` making the tag extracted + // name ``cluster.upstream_rq_timeout`` and the tag value for + // ``envoy.cluster_name`` will be ``foo_cluster`` (note: there will be no + // ``.`` character because of the second capture group). + // + // Example 2. a stat name + // ``http.connection_manager_1.user_agent.ios.downstream_cx_total`` and two + // tag specifiers: + // + // .. code-block:: json + // + // [ + // { + // "tag_name": "envoy.http_user_agent", + // "regex": "^http(?=\.).*?\.user_agent\.((.+?)\.)\w+?$" + // }, + // { + // "tag_name": "envoy.http_conn_manager_prefix", + // "regex": "^http\.((.*?)\.)" + // } + // ] + // + // The two regexes of the specifiers will be processed in the definition order. + // + // The first regex will remove ``ios.``, leaving the tag extracted name + // ``http.connection_manager_1.user_agent.downstream_cx_total``. The tag + // ``envoy.http_user_agent`` will be added with tag value ``ios``. + // + // The second regex will remove ``connection_manager_1.`` from the tag + // extracted name produced by the first regex + // ``http.connection_manager_1.user_agent.downstream_cx_total``, leaving + // ``http.user_agent.downstream_cx_total`` as the tag extracted name. The tag + // ``envoy.http_conn_manager_prefix`` will be added with the tag value + // ``connection_manager_1``. + string regex = 2 [(validate.rules).string.max_bytes = 1024]; + + // Specifies a fixed tag value for the ``tag_name``. + string fixed_value = 3; + } +} + +// Stats configuration proto schema for built-in *envoy.statsd* sink. This sink does not support +// tagged metrics. +message StatsdSink { + oneof statsd_specifier { + option (validate.required) = true; + + // The UDP address of a running `statsd `_ + // compliant listener. If specified, statistics will be flushed to this + // address. + envoy.api.v3alpha.core.Address address = 1; + + // The name of a cluster that is running a TCP `statsd + // `_ compliant listener. If specified, + // Envoy will connect to this cluster to flush statistics. + string tcp_cluster_name = 2; + } + // Optional custom prefix for StatsdSink. If + // specified, this will override the default prefix. + // For example: + // + // .. code-block:: json + // + // { + // "prefix" : "envoy-prod" + // } + // + // will change emitted stats to + // + // .. code-block:: cpp + // + // envoy-prod.test_counter:1|c + // envoy-prod.test_timer:5|ms + // + // Note that the default prefix, "envoy", will be used if a prefix is not + // specified. + // + // Stats with default prefix: + // + // .. code-block:: cpp + // + // envoy.test_counter:1|c + // envoy.test_timer:5|ms + string prefix = 3; +} + +// Stats configuration proto schema for built-in *envoy.dog_statsd* sink. +// The sink emits stats with `DogStatsD `_ +// compatible tags. Tags are configurable via :ref:`StatsConfig +// `. +// [#comment:next free field: 3] +message DogStatsdSink { + oneof dog_statsd_specifier { + option (validate.required) = true; + + // The UDP address of a running DogStatsD compliant listener. If specified, + // statistics will be flushed to this address. + envoy.api.v3alpha.core.Address address = 1; + } + + reserved 2; + + // Optional custom metric name prefix. See :ref:`StatsdSink's prefix field + // ` for more details. + string prefix = 3; +} + +// Stats configuration proto schema for built-in *envoy.stat_sinks.hystrix* sink. +// The sink emits stats in `text/event-stream +// `_ +// formatted stream for use by `Hystrix dashboard +// `_. +// +// Note that only a single HystrixSink should be configured. +// +// Streaming is started through an admin endpoint :http:get:`/hystrix_event_stream`. +message HystrixSink { + // The number of buckets the rolling statistical window is divided into. + // + // Each time the sink is flushed, all relevant Envoy statistics are sampled and + // added to the rolling window (removing the oldest samples in the window + // in the process). The sink then outputs the aggregate statistics across the + // current rolling window to the event stream(s). + // + // rolling_window(ms) = stats_flush_interval(ms) * num_of_buckets + // + // More detailed explanation can be found in `Hystrix wiki + // `_. + int64 num_buckets = 1; +} diff --git a/api/envoy/config/overload/v3alpha/BUILD b/api/envoy/config/overload/v3alpha/BUILD new file mode 100644 index 000000000000..bfffb5639ca7 --- /dev/null +++ b/api/envoy/config/overload/v3alpha/BUILD @@ -0,0 +1,14 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "overload", + srcs = ["overload.proto"], + visibility = ["//visibility:public"], +) + +api_go_proto_library( + name = "overload", + proto = ":overload", +) diff --git a/api/envoy/config/overload/v3alpha/overload.proto b/api/envoy/config/overload/v3alpha/overload.proto new file mode 100644 index 000000000000..474c7677002b --- /dev/null +++ b/api/envoy/config/overload/v3alpha/overload.proto @@ -0,0 +1,78 @@ +syntax = "proto3"; + +package envoy.config.overload.v3alpha; + +option java_outer_classname = "OverloadProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.overload.v3alpha"; +option go_package = "v2alpha"; + +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/struct.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Overload Manager] + +// The Overload Manager provides an extensible framework to protect Envoy instances +// from overload of various resources (memory, cpu, file descriptors, etc). +// It monitors a configurable set of resources and notifies registered listeners +// when triggers related to those resources fire. + +message ResourceMonitor { + // The name of the resource monitor to instantiate. Must match a registered + // resource monitor type. The built-in resource monitors are: + // + // * :ref:`envoy.resource_monitors.fixed_heap + // ` + // * :ref:`envoy.resource_monitors.injected_resource + // ` + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Configuration for the resource monitor being instantiated. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} + +message ThresholdTrigger { + // If the resource pressure is greater than or equal to this value, the trigger + // will fire. + double value = 1 [(validate.rules).double = {gte: 0, lte: 1}]; +} + +message Trigger { + // The name of the resource this is a trigger for. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + oneof trigger_oneof { + option (validate.required) = true; + ThresholdTrigger threshold = 2; + } +} + +message OverloadAction { + // The name of the overload action. This is just a well-known string that listeners can + // use for registering callbacks. Custom overload actions should be named using reverse + // DNS to ensure uniqueness. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // A set of triggers for this action. If any of these triggers fire the overload action + // is activated. Listeners are notified when the overload action transitions from + // inactivated to activated, or vice versa. + repeated Trigger triggers = 2 [(validate.rules).repeated .min_items = 1]; +} + +message OverloadManager { + // The interval for refreshing resource usage. + google.protobuf.Duration refresh_interval = 1; + + // The set of resources to monitor. + repeated ResourceMonitor resource_monitors = 2 [(validate.rules).repeated .min_items = 1]; + + // The set of overload actions. + repeated OverloadAction actions = 3; +} diff --git a/api/envoy/config/ratelimit/v3alpha/BUILD b/api/envoy/config/ratelimit/v3alpha/BUILD new file mode 100644 index 000000000000..571a768dde4b --- /dev/null +++ b/api/envoy/config/ratelimit/v3alpha/BUILD @@ -0,0 +1,20 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "rls", + srcs = ["rls.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha/core:grpc_service", + ], +) + +api_go_grpc_library( + name = "rls", + proto = ":rls", + deps = [ + "//envoy/api/v3alpha/core:grpc_service_go_proto", + ], +) diff --git a/api/envoy/config/ratelimit/v3alpha/rls.proto b/api/envoy/config/ratelimit/v3alpha/rls.proto new file mode 100644 index 000000000000..67ac6479cd23 --- /dev/null +++ b/api/envoy/config/ratelimit/v3alpha/rls.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package envoy.config.ratelimit.v3alpha; + +option java_outer_classname = "RlsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.ratelimit.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/grpc_service.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Rate limit service] + +// Rate limit :ref:`configuration overview `. +message RateLimitServiceConfig { + reserved 1; + + // Specifies the gRPC service that hosts the rate limit service. The client + // will connect to this cluster when it needs to make rate limit service + // requests. + envoy.api.v3alpha.core.GrpcService grpc_service = 2 [(validate.rules).message.required = true]; + + reserved 3; +} diff --git a/api/envoy/config/rbac/v3alpha/BUILD b/api/envoy/config/rbac/v3alpha/BUILD new file mode 100644 index 000000000000..89f98c97d481 --- /dev/null +++ b/api/envoy/config/rbac/v3alpha/BUILD @@ -0,0 +1,36 @@ +licenses(["notice"]) # Apache 2 + +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +api_proto_library_internal( + name = "rbac", + srcs = ["rbac.proto"], + external_cc_proto_deps = [ + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_cc_proto", + ], + external_proto_deps = [ + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", + ], + external_py_proto_deps = [ + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_py_proto", + ], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/route", + "//envoy/type/matcher:metadata", + "//envoy/type/matcher:string", + ], +) + +api_go_proto_library( + name = "rbac", + proto = ":rbac", + deps = [ + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/route:route_go_proto", + "//envoy/type/matcher:metadata_go_proto", + "//envoy/type/matcher:string_go_proto", + "@com_google_googleapis//google/api/expr/v1alpha1:cel_go_proto", + ], +) diff --git a/api/envoy/config/rbac/v3alpha/rbac.proto b/api/envoy/config/rbac/v3alpha/rbac.proto new file mode 100644 index 000000000000..d299c384da90 --- /dev/null +++ b/api/envoy/config/rbac/v3alpha/rbac.proto @@ -0,0 +1,215 @@ +syntax = "proto3"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/route/route.proto"; +import "envoy/type/matcher/metadata.proto"; +import "envoy/type/matcher/string.proto"; + +import "google/api/expr/v1alpha1/syntax.proto"; + +package envoy.config.rbac.v3alpha; + +option java_outer_classname = "RbacProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.rbac.v3alpha"; +option go_package = "v2"; + +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: Role Based Access Control (RBAC)] + +// Role Based Access Control (RBAC) provides service-level and method-level access control for a +// service. RBAC policies are additive. The policies are examined in order. A request is allowed +// once a matching policy is found (suppose the `action` is ALLOW). +// +// Here is an example of RBAC configuration. It has two policies: +// +// * Service account "cluster.local/ns/default/sa/admin" has full access to the service, and so +// does "cluster.local/ns/default/sa/superuser". +// +// * Any user can read ("GET") the service at paths with prefix "/products", so long as the +// destination port is either 80 or 443. +// +// .. code-block:: yaml +// +// action: ALLOW +// policies: +// "service-admin": +// permissions: +// - any: true +// principals: +// - authenticated: +// principal_name: +// exact: "cluster.local/ns/default/sa/admin" +// - authenticated: +// principal_name: +// exact: "cluster.local/ns/default/sa/superuser" +// "product-viewer": +// permissions: +// - and_rules: +// rules: +// - header: { name: ":method", exact_match: "GET" } +// - header: { name: ":path", regex_match: "/products(/.*)?" } +// - or_rules: +// rules: +// - destination_port: 80 +// - destination_port: 443 +// principals: +// - any: true +// +message RBAC { + // Should we do safe-list or block-list style access control? + enum Action { + // The policies grant access to principals. The rest is denied. This is safe-list style + // access control. This is the default type. + ALLOW = 0; + + // The policies deny access to principals. The rest is allowed. This is block-list style + // access control. + DENY = 1; + } + + // The action to take if a policy matches. The request is allowed if and only if: + // + // * `action` is "ALLOWED" and at least one policy matches + // * `action` is "DENY" and none of the policies match + Action action = 1; + + // Maps from policy name to policy. A match occurs when at least one policy matches the request. + map policies = 2; +} + +// Policy specifies a role and the principals that are assigned/denied the role. A policy matches if +// and only if at least one of its permissions match the action taking place AND at least one of its +// principals match the downstream AND the condition is true if specified. +message Policy { + // Required. The set of permissions that define a role. Each permission is matched with OR + // semantics. To match all actions for this policy, a single Permission with the `any` field set + // to true should be used. + repeated Permission permissions = 1 [(validate.rules).repeated .min_items = 1]; + + // Required. The set of principals that are assigned/denied the role based on “action”. Each + // principal is matched with OR semantics. To match all downstreams for this policy, a single + // Principal with the `any` field set to true should be used. + repeated Principal principals = 2 [(validate.rules).repeated .min_items = 1]; + + // An optional symbolic expression specifying an access control condition. + // The condition is combined with AND semantics. + google.api.expr.v1alpha1.Expr condition = 3; +} + +// Permission defines an action (or actions) that a principal can take. +message Permission { + + // Used in the `and_rules` and `or_rules` fields in the `rule` oneof. Depending on the context, + // each are applied with the associated behavior. + message Set { + repeated Permission rules = 1 [(validate.rules).repeated .min_items = 1]; + } + + oneof rule { + option (validate.required) = true; + + // A set of rules that all must match in order to define the action. + Set and_rules = 1; + + // A set of rules where at least one must match in order to define the action. + Set or_rules = 2; + + // When any is set, it matches any action. + bool any = 3 [(validate.rules).bool.const = true]; + + // A header (or pseudo-header such as :path or :method) on the incoming HTTP request. Only + // available for HTTP request. + envoy.api.v3alpha.route.HeaderMatcher header = 4; + + // A CIDR block that describes the destination IP. + envoy.api.v3alpha.core.CidrRange destination_ip = 5; + + // A port number that describes the destination port connecting to. + uint32 destination_port = 6 [(validate.rules).uint32.lte = 65535]; + + // Metadata that describes additional information about the action. + envoy.type.matcher.MetadataMatcher metadata = 7; + + // Negates matching the provided permission. For instance, if the value of `not_rule` would + // match, this permission would not match. Conversely, if the value of `not_rule` would not + // match, this permission would match. + Permission not_rule = 8; + + // The request server from the client's connection request. This is + // typically TLS SNI. + // + // .. attention:: + // + // The behavior of this field may be affected by how Envoy is configured + // as explained below. + // + // * If the :ref:`TLS Inspector ` + // filter is not added, and if a `FilterChainMatch` is not defined for + // the :ref:`server name `, + // a TLS connection's requested SNI server name will be treated as if it + // wasn't present. + // + // * A :ref:`listener filter ` may + // overwrite a connection's requested server name within Envoy. + // + // Please refer to :ref:`this FAQ entry ` to learn to + // setup SNI. + envoy.type.matcher.StringMatcher requested_server_name = 9; + } +} + +// Principal defines an identity or a group of identities for a downstream subject. +message Principal { + + // Used in the `and_ids` and `or_ids` fields in the `identifier` oneof. Depending on the context, + // each are applied with the associated behavior. + message Set { + repeated Principal ids = 1 [(validate.rules).repeated .min_items = 1]; + } + + // Authentication attributes for a downstream. + message Authenticated { + reserved 1; + reserved "name"; + + // The name of the principal. If set, The URI SAN or DNS SAN in that order is used from the + // certificate, otherwise the subject field is used. If unset, it applies to any user that is + // authenticated. + envoy.type.matcher.StringMatcher principal_name = 2; + } + + oneof identifier { + option (validate.required) = true; + + // A set of identifiers that all must match in order to define the downstream. + Set and_ids = 1; + + // A set of identifiers at least one must match in order to define the downstream. + Set or_ids = 2; + + // When any is set, it matches any downstream. + bool any = 3 [(validate.rules).bool.const = true]; + + // Authenticated attributes that identify the downstream. + Authenticated authenticated = 4; + + // A CIDR block that describes the downstream IP. + envoy.api.v3alpha.core.CidrRange source_ip = 5; + + // A header (or pseudo-header such as :path or :method) on the incoming HTTP request. Only + // available for HTTP request. + envoy.api.v3alpha.route.HeaderMatcher header = 6; + + // Metadata that describes additional information about the principal. + envoy.type.matcher.MetadataMatcher metadata = 7; + + // Negates matching the provided principal. For instance, if the value of `not_id` would match, + // this principal would not match. Conversely, if the value of `not_id` would not match, this + // principal would match. + Principal not_id = 8; + } +} diff --git a/api/envoy/config/resource_monitor/fixed_heap/v3alpha/BUILD b/api/envoy/config/resource_monitor/fixed_heap/v3alpha/BUILD new file mode 100644 index 000000000000..363d90f11808 --- /dev/null +++ b/api/envoy/config/resource_monitor/fixed_heap/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "fixed_heap", + srcs = ["fixed_heap.proto"], + visibility = ["//visibility:public"], +) diff --git a/api/envoy/config/resource_monitor/fixed_heap/v3alpha/fixed_heap.proto b/api/envoy/config/resource_monitor/fixed_heap/v3alpha/fixed_heap.proto new file mode 100644 index 000000000000..2bc1baf85243 --- /dev/null +++ b/api/envoy/config/resource_monitor/fixed_heap/v3alpha/fixed_heap.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package envoy.config.resource_monitor.fixed_heap.v3alpha; + +option java_outer_classname = "FixedHeapProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.resource_monitor.fixed_heap.v3alpha"; +option go_package = "v2alpha"; + +import "validate/validate.proto"; + +// [#protodoc-title: Fixed heap] + +// The fixed heap resource monitor reports the Envoy process memory pressure, computed as a +// fraction of currently reserved heap memory divided by a statically configured maximum +// specified in the FixedHeapConfig. +message FixedHeapConfig { + uint64 max_heap_size_bytes = 1 [(validate.rules).uint64.gt = 0]; +} diff --git a/api/envoy/config/resource_monitor/injected_resource/v3alpha/BUILD b/api/envoy/config/resource_monitor/injected_resource/v3alpha/BUILD new file mode 100644 index 000000000000..10abf09e9ef8 --- /dev/null +++ b/api/envoy/config/resource_monitor/injected_resource/v3alpha/BUILD @@ -0,0 +1,9 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "injected_resource", + srcs = ["injected_resource.proto"], + visibility = ["//visibility:public"], +) diff --git a/api/envoy/config/resource_monitor/injected_resource/v3alpha/injected_resource.proto b/api/envoy/config/resource_monitor/injected_resource/v3alpha/injected_resource.proto new file mode 100644 index 000000000000..f5b41ef165c8 --- /dev/null +++ b/api/envoy/config/resource_monitor/injected_resource/v3alpha/injected_resource.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package envoy.config.resource_monitor.injected_resource.v3alpha; + +option java_outer_classname = "InjectedResourceProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.resource_monitor.injected_resource.v3alpha"; +option go_package = "v2alpha"; + +import "validate/validate.proto"; + +// [#protodoc-title: Injected resource] + +// The injected resource monitor allows injecting a synthetic resource pressure into Envoy +// via a text file, which must contain a floating-point number in the range [0..1] representing +// the resource pressure and be updated atomically by a symbolic link swap. +// This is intended primarily for integration tests to force Envoy into an overloaded state. +message InjectedResourceConfig { + string filename = 1 [(validate.rules).string.min_bytes = 1]; +} diff --git a/api/envoy/config/trace/v3alpha/BUILD b/api/envoy/config/trace/v3alpha/BUILD new file mode 100644 index 000000000000..72056b3ad4b6 --- /dev/null +++ b/api/envoy/config/trace/v3alpha/BUILD @@ -0,0 +1,24 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "trace", + srcs = ["trace.proto"], + visibility = [ + "//envoy/config/bootstrap/v3alpha:__pkg__", + ], + deps = [ + "//envoy/api/v3alpha/core:grpc_service", + "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", + ], +) + +api_go_proto_library( + name = "trace", + proto = ":trace", + deps = [ + "//envoy/api/v3alpha/core:grpc_service_go_proto", + "@opencensus_proto//opencensus/proto/trace/v1:trace_and_config_proto_go", + ], +) diff --git a/api/envoy/config/trace/v3alpha/trace.proto b/api/envoy/config/trace/v3alpha/trace.proto new file mode 100644 index 000000000000..2771c1b40f28 --- /dev/null +++ b/api/envoy/config/trace/v3alpha/trace.proto @@ -0,0 +1,204 @@ +// [#protodoc-title: Tracing] +// Tracing :ref:`architecture overview `. + +syntax = "proto3"; + +package envoy.config.trace.v3alpha; + +option java_outer_classname = "TraceProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.trace.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/grpc_service.proto"; +import "opencensus/proto/trace/v1/trace_config.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +// The tracing configuration specifies global +// settings for the HTTP tracer used by Envoy. The configuration is defined by +// the :ref:`Bootstrap ` :ref:`tracing +// ` field. Envoy may support other +// tracers in the future, but right now the HTTP tracer is the only one supported. +message Tracing { + message Http { + // The name of the HTTP trace driver to instantiate. The name must match a + // supported HTTP trace driver. Built-in trace drivers: + // + // - *envoy.lightstep* + // - *envoy.zipkin* + // - *envoy.dynamic.ot* + // - *envoy.tracers.datadog* + // - *envoy.tracers.opencensus* + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Trace driver specific configuration which depends on the driver being instantiated. + // See the trace drivers for examples: + // + // - :ref:`LightstepConfig ` + // - :ref:`ZipkinConfig ` + // - :ref:`DynamicOtConfig ` + // - :ref:`DatadogConfig ` + // - :ref:`OpenCensusConfig ` + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } + } + // Provides configuration for the HTTP tracer. + Http http = 1; +} + +// Configuration for the LightStep tracer. +message LightstepConfig { + // The cluster manager cluster that hosts the LightStep collectors. + string collector_cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // File containing the access token to the `LightStep + // `_ API. + string access_token_file = 2 [(validate.rules).string.min_bytes = 1]; +} + +// Configuration for the Zipkin tracer. +message ZipkinConfig { + // The cluster manager cluster that hosts the Zipkin collectors. Note that the + // Zipkin cluster must be defined in the :ref:`Bootstrap static cluster + // resources `. + string collector_cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // The API endpoint of the Zipkin service where the spans will be sent. When + // using a standard Zipkin installation, the API endpoint is typically + // /api/v1/spans, which is the default value. + string collector_endpoint = 2 [(validate.rules).string.min_bytes = 1]; + + // Determines whether a 128bit trace id will be used when creating a new + // trace instance. The default value is false, which will result in a 64 bit trace id being used. + bool trace_id_128bit = 3; + + // Determines whether client and server spans will share the same span context. + // The default value is true. + google.protobuf.BoolValue shared_span_context = 4; + + // Available Zipkin collector endpoint versions. + enum CollectorEndpointVersion { + // Zipkin API v1, JSON over HTTP. + // [#comment: The default implementation of Zipkin client before this field is added was only v1 + // and the way user configure this was by not explicitly specifying the version. Consequently, + // before this is added, the corresponding Zipkin collector expected to receive v1 payload. + // Hence the motivation of adding HTTP_JSON_V1 as the default is to avoid a breaking change when + // user upgrading Envoy with this change. Furthermore, we also immediately deprecate this field, + // since in Zipkin realm this v1 version is considered to be not preferable anymore.] + HTTP_JSON_V1 = 0 [deprecated = true]; + + // Zipkin API v2, JSON over HTTP. + HTTP_JSON = 1; + + // Zipkin API v2, protobuf over HTTP. + HTTP_PROTO = 2; + + // [#not-implemented-hide:] + GRPC = 3; + } + + // Determines the selected collector endpoint version. By default, the ``HTTP_JSON_V1`` will be + // used. + CollectorEndpointVersion collector_endpoint_version = 5; +} + +// DynamicOtConfig is used to dynamically load a tracer from a shared library +// that implements the `OpenTracing dynamic loading API +// `_. +message DynamicOtConfig { + // Dynamic library implementing the `OpenTracing API + // `_. + string library = 1 [(validate.rules).string.min_bytes = 1]; + + // The configuration to use when creating a tracer from the given dynamic + // library. + google.protobuf.Struct config = 2; +} + +// Configuration for the Datadog tracer. +message DatadogConfig { + // The cluster to use for submitting traces to the Datadog agent. + string collector_cluster = 1 [(validate.rules).string.min_bytes = 1]; + // The name used for the service when traces are generated by envoy. + string service_name = 2 [(validate.rules).string.min_bytes = 1]; +} + +// Configuration for the OpenCensus tracer. +// [#proto-status: experimental] +message OpenCensusConfig { + // Configures tracing, e.g. the sampler, max number of annotations, etc. + opencensus.proto.trace.v1.TraceConfig trace_config = 1; + + // Enables the stdout exporter if set to true. This is intended for debugging + // purposes. + bool stdout_exporter_enabled = 2; + + // Enables the Stackdriver exporter if set to true. The project_id must also + // be set. + bool stackdriver_exporter_enabled = 3; + + // The Cloud project_id to use for Stackdriver tracing. + string stackdriver_project_id = 4; + + // (optional) By default, the Stackdriver exporter will connect to production + // Stackdriver. If stackdriver_address is non-empty, it will instead connect + // to this address, which is in the gRPC format: + // https://github.com/grpc/grpc/blob/master/doc/naming.md + string stackdriver_address = 10; + + // Enables the Zipkin exporter if set to true. The url and service name must + // also be set. + bool zipkin_exporter_enabled = 5; + + // The URL to Zipkin, e.g. "http://127.0.0.1:9411/api/v3alpha/spans" + string zipkin_url = 6; + + // Enables the OpenCensus Agent exporter if set to true. The address must also + // be set. + bool ocagent_exporter_enabled = 11; + + // The address of the OpenCensus Agent, if its exporter is enabled, in gRPC + // format: https://github.com/grpc/grpc/blob/master/doc/naming.md + string ocagent_address = 12; + + reserved 7; // Formerly zipkin_service_name. + + enum TraceContext { + // No-op default, no trace context is utilized. + NONE = 0; + + // W3C Trace-Context format "traceparent:" header. + TRACE_CONTEXT = 1; + + // Binary "grpc-trace-bin:" header. + GRPC_TRACE_BIN = 2; + + // "X-Cloud-Trace-Context:" header. + CLOUD_TRACE_CONTEXT = 3; + + // X-B3-* headers. + B3 = 4; + } + + // List of incoming trace context headers we will accept. First one found + // wins. + repeated TraceContext incoming_trace_context = 8; + + // List of outgoing trace context headers we will produce. + repeated TraceContext outgoing_trace_context = 9; +} + +// Configuration structure. +message TraceServiceConfig { + // The upstream gRPC cluster that hosts the metrics service. + envoy.api.v3alpha.core.GrpcService grpc_service = 1 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/transport_socket/alts/v3alpha/BUILD b/api/envoy/config/transport_socket/alts/v3alpha/BUILD new file mode 100644 index 000000000000..7ffc03097000 --- /dev/null +++ b/api/envoy/config/transport_socket/alts/v3alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") + +licenses(["notice"]) # Apache 2 + +api_proto_library( + name = "alts", + srcs = ["alts.proto"], + deps = [ + "//envoy/api/v3alpha/core:base", + ], +) diff --git a/api/envoy/config/transport_socket/alts/v3alpha/alts.proto b/api/envoy/config/transport_socket/alts/v3alpha/alts.proto new file mode 100644 index 000000000000..22684b862614 --- /dev/null +++ b/api/envoy/config/transport_socket/alts/v3alpha/alts.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.config.transport_socket.alts.v3alpha; + +option java_outer_classname = "AltsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.transport_socket.alts.v3alpha"; +option go_package = "v2"; + +// [#protodoc-title: ALTS] + +import "validate/validate.proto"; + +// Configuration for ALTS transport socket. This provides Google's ALTS protocol to Envoy. +// https://cloud.google.com/security/encryption-in-transit/application-layer-transport-security/ +message Alts { + // The location of a handshaker service, this is usually 169.254.169.254:8080 + // on GCE. + string handshaker_service = 1 [(validate.rules).string.min_bytes = 1]; + + // The acceptable service accounts from peer, peers not in the list will be rejected in the + // handshake validation step. If empty, no validation will be performed. + repeated string peer_service_accounts = 2; +} diff --git a/api/envoy/config/transport_socket/tap/v3alpha/BUILD b/api/envoy/config/transport_socket/tap/v3alpha/BUILD new file mode 100644 index 000000000000..8056ad6f17bb --- /dev/null +++ b/api/envoy/config/transport_socket/tap/v3alpha/BUILD @@ -0,0 +1,12 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "tap", + srcs = ["tap.proto"], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/config/common/tap/v3alpha:common", + ], +) diff --git a/api/envoy/config/transport_socket/tap/v3alpha/tap.proto b/api/envoy/config/transport_socket/tap/v3alpha/tap.proto new file mode 100644 index 000000000000..1cca6814c803 --- /dev/null +++ b/api/envoy/config/transport_socket/tap/v3alpha/tap.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package envoy.config.transport_socket.tap.v3alpha; + +option java_outer_classname = "TapProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.transport_socket.tap.v3alpha"; +option go_package = "v2"; + +// [#protodoc-title: Tap] + +import "envoy/config/common/tap/v3alpha/common.proto"; +import "envoy/api/v3alpha/core/base.proto"; + +import "validate/validate.proto"; + +// Configuration for tap transport socket. This wraps another transport socket, providing the +// ability to interpose and record in plain text any traffic that is surfaced to Envoy. +message Tap { + // Common configuration for the tap transport socket. + common.tap.v3alpha.CommonExtensionConfig common_config = 1 + [(validate.rules).message.required = true]; + + // The underlying transport socket being wrapped. + api.v3alpha.core.TransportSocket transport_socket = 2 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/data/accesslog/v3alpha/BUILD b/api/envoy/data/accesslog/v3alpha/BUILD new file mode 100644 index 000000000000..30157958e7fe --- /dev/null +++ b/api/envoy/data/accesslog/v3alpha/BUILD @@ -0,0 +1,24 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "accesslog", + srcs = ["accesslog.proto"], + visibility = [ + "//envoy/service/accesslog/v3alpha:__pkg__", + ], + deps = [ + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + ], +) + +api_go_proto_library( + name = "accesslog", + proto = ":accesslog", + deps = [ + "//envoy/api/v3alpha/core:address_go_proto", + "//envoy/api/v3alpha/core:base_go_proto", + ], +) diff --git a/api/envoy/data/accesslog/v3alpha/accesslog.proto b/api/envoy/data/accesslog/v3alpha/accesslog.proto new file mode 100644 index 000000000000..b4588ecd31ff --- /dev/null +++ b/api/envoy/data/accesslog/v3alpha/accesslog.proto @@ -0,0 +1,356 @@ +syntax = "proto3"; + +package envoy.data.accesslog.v3alpha; + +option java_outer_classname = "AccesslogProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.data.accesslog.v3alpha"; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; +import "gogoproto/gogo.proto"; +import "validate/validate.proto"; + +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: gRPC access logs] +// Envoy access logs describe incoming interaction with Envoy over a fixed +// period of time, and typically cover a single request/response exchange, +// (e.g. HTTP), stream (e.g. over HTTP/gRPC), or proxied connection (e.g. TCP). +// Access logs contain fields defined in protocol-specific protobuf messages. +// +// Except where explicitly declared otherwise, all fields describe +// *downstream* interaction between Envoy and a connected client. +// Fields describing *upstream* interaction will explicitly include ``upstream`` +// in their name. + +message TCPAccessLogEntry { + // Common properties shared by all Envoy access logs. + AccessLogCommon common_properties = 1; + + // Properties of the TCP connection. + ConnectionProperties connection_properties = 2; +} + +message HTTPAccessLogEntry { + // Common properties shared by all Envoy access logs. + AccessLogCommon common_properties = 1; + + // HTTP version + enum HTTPVersion { + PROTOCOL_UNSPECIFIED = 0; + HTTP10 = 1; + HTTP11 = 2; + HTTP2 = 3; + } + HTTPVersion protocol_version = 2; + + // Description of the incoming HTTP request. + HTTPRequestProperties request = 3; + + // Description of the outgoing HTTP response. + HTTPResponseProperties response = 4; +} + +// Defines fields for a connection +message ConnectionProperties { + // Number of bytes received from downstream. + uint64 received_bytes = 1; + + // Number of bytes sent to downstream. + uint64 sent_bytes = 2; +} + +// Defines fields that are shared by all Envoy access logs. +message AccessLogCommon { + // [#not-implemented-hide:] + // This field indicates the rate at which this log entry was sampled. + // Valid range is (0.0, 1.0]. + double sample_rate = 1 [(validate.rules).double.gt = 0.0, (validate.rules).double.lte = 1.0]; + + // This field is the remote/origin address on which the request from the user was received. + // Note: This may not be the physical peer. E.g, if the remote address is inferred from for + // example the x-forwarder-for header, proxy protocol, etc. + envoy.api.v3alpha.core.Address downstream_remote_address = 2; + + // This field is the local/destination address on which the request from the user was received. + envoy.api.v3alpha.core.Address downstream_local_address = 3; + + // If the connection is secure,S this field will contain TLS properties. + TLSProperties tls_properties = 4; + + // The time that Envoy started servicing this request. This is effectively the time that the first + // downstream byte is received. + google.protobuf.Timestamp start_time = 5 [(gogoproto.stdtime) = true]; + + // Interval between the first downstream byte received and the last + // downstream byte received (i.e. time it takes to receive a request). + google.protobuf.Duration time_to_last_rx_byte = 6 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the first upstream byte sent. There may + // by considerable delta between *time_to_last_rx_byte* and this value due to filters. + // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about + // not accounting for kernel socket buffer time, etc. + google.protobuf.Duration time_to_first_upstream_tx_byte = 7 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the last upstream byte sent. There may + // by considerable delta between *time_to_last_rx_byte* and this value due to filters. + // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about + // not accounting for kernel socket buffer time, etc. + google.protobuf.Duration time_to_last_upstream_tx_byte = 8 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the first upstream + // byte received (i.e. time it takes to start receiving a response). + google.protobuf.Duration time_to_first_upstream_rx_byte = 9 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the last upstream + // byte received (i.e. time it takes to receive a complete response). + google.protobuf.Duration time_to_last_upstream_rx_byte = 10 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the first downstream byte sent. + // There may be a considerable delta between the *time_to_first_upstream_rx_byte* and this field + // due to filters. Additionally, the same caveats apply as documented in + // *time_to_last_downstream_tx_byte* about not accounting for kernel socket buffer time, etc. + google.protobuf.Duration time_to_first_downstream_tx_byte = 11 [(gogoproto.stdduration) = true]; + + // Interval between the first downstream byte received and the last downstream byte sent. + // Depending on protocol, buffering, windowing, filters, etc. there may be a considerable delta + // between *time_to_last_upstream_rx_byte* and this field. Note also that this is an approximate + // time. In the current implementation it does not include kernel socket buffer time. In the + // current implementation it also does not include send window buffering inside the HTTP/2 codec. + // In the future it is likely that work will be done to make this duration more accurate. + google.protobuf.Duration time_to_last_downstream_tx_byte = 12 [(gogoproto.stdduration) = true]; + + // The upstream remote/destination address that handles this exchange. This does not include + // retries. + envoy.api.v3alpha.core.Address upstream_remote_address = 13; + + // The upstream local/origin address that handles this exchange. This does not include retries. + envoy.api.v3alpha.core.Address upstream_local_address = 14; + + // The upstream cluster that *upstream_remote_address* belongs to. + string upstream_cluster = 15; + + // Flags indicating occurrences during request/response processing. + ResponseFlags response_flags = 16; + + // All metadata encountered during request processing, including endpoint + // selection. + // + // This can be used to associate IDs attached to the various configurations + // used to process this request with the access log entry. For example, a + // route created from a higher level forwarding rule with some ID can place + // that ID in this field and cross reference later. It can also be used to + // determine if a canary endpoint was used or not. + envoy.api.v3alpha.core.Metadata metadata = 17; + + // If upstream connection failed due to transport socket (e.g. TLS handshake), provides the + // failure reason from the transport socket. The format of this field depends on the configured + // upstream transport socket. Common TLS failures are in + // :ref:`TLS trouble shooting `. + string upstream_transport_failure_reason = 18; + + // The name of the route + string route_name = 19; +} + +// Flags indicating occurrences during request/response processing. +message ResponseFlags { + // Indicates local server healthcheck failed. + bool failed_local_healthcheck = 1; + + // Indicates there was no healthy upstream. + bool no_healthy_upstream = 2; + + // Indicates an there was an upstream request timeout. + bool upstream_request_timeout = 3; + + // Indicates local codec level reset was sent on the stream. + bool local_reset = 4; + + // Indicates remote codec level reset was received on the stream. + bool upstream_remote_reset = 5; + + // Indicates there was a local reset by a connection pool due to an initial connection failure. + bool upstream_connection_failure = 6; + + // Indicates the stream was reset due to an upstream connection termination. + bool upstream_connection_termination = 7; + + // Indicates the stream was reset because of a resource overflow. + bool upstream_overflow = 8; + + // Indicates no route was found for the request. + bool no_route_found = 9; + + // Indicates that the request was delayed before proxying. + bool delay_injected = 10; + + // Indicates that the request was aborted with an injected error code. + bool fault_injected = 11; + + // Indicates that the request was rate-limited locally. + bool rate_limited = 12; + + message Unauthorized { + // Reasons why the request was unauthorized + enum Reason { + REASON_UNSPECIFIED = 0; + // The request was denied by the external authorization service. + EXTERNAL_SERVICE = 1; + } + + Reason reason = 1; + } + + // Indicates if the request was deemed unauthorized and the reason for it. + Unauthorized unauthorized_details = 13; + + // Indicates that the request was rejected because there was an error in rate limit service. + bool rate_limit_service_error = 14; + + // Indicates the stream was reset due to a downstream connection termination. + bool downstream_connection_termination = 15; + + // Indicates that the upstream retry limit was exceeded, resulting in a downstream error. + bool upstream_retry_limit_exceeded = 16; + + // Indicates that the stream idle timeout was hit, resulting in a downstream 408. + bool stream_idle_timeout = 17; + + // Indicates that the request was rejected because an envoy request header failed strict + // validation. + bool invalid_envoy_request_headers = 18; +} + +// Properties of a negotiated TLS connection. +message TLSProperties { + enum TLSVersion { + VERSION_UNSPECIFIED = 0; + TLSv1 = 1; + TLSv1_1 = 2; + TLSv1_2 = 3; + TLSv1_3 = 4; + } + // Version of TLS that was negotiated. + TLSVersion tls_version = 1; + + // TLS cipher suite negotiated during handshake. The value is a + // four-digit hex code defined by the IANA TLS Cipher Suite Registry + // (e.g. ``009C`` for ``TLS_RSA_WITH_AES_128_GCM_SHA256``). + // + // Here it is expressed as an integer. + google.protobuf.UInt32Value tls_cipher_suite = 2; + + // SNI hostname from handshake. + string tls_sni_hostname = 3; + + message CertificateProperties { + message SubjectAltName { + oneof san { + string uri = 1; + // [#not-implemented-hide:] + string dns = 2; + } + } + + // SANs present in the certificate. + repeated SubjectAltName subject_alt_name = 1; + + // The subject field of the certificate. + string subject = 2; + } + + // Properties of the local certificate used to negotiate TLS. + CertificateProperties local_certificate_properties = 4; + + // Properties of the peer certificate used to negotiate TLS. + CertificateProperties peer_certificate_properties = 5; + + // The TLS session ID. + string tls_session_id = 6; +} + +message HTTPRequestProperties { + // The request method (RFC 7231/2616). + // [#comment:TODO(htuch): add (validate.rules).enum.defined_only = true once + // https://github.com/lyft/protoc-gen-validate/issues/42 is resolved.] + envoy.api.v3alpha.core.RequestMethod request_method = 1; + + // The scheme portion of the incoming request URI. + string scheme = 2; + + // HTTP/2 ``:authority`` or HTTP/1.1 ``Host`` header value. + string authority = 3; + + // The port of the incoming request URI + // (unused currently, as port is composed onto authority). + google.protobuf.UInt32Value port = 4; + + // The path portion from the incoming request URI. + string path = 5; + + // Value of the ``User-Agent`` request header. + string user_agent = 6; + + // Value of the ``Referer`` request header. + string referer = 7; + + // Value of the ``X-Forwarded-For`` request header. + string forwarded_for = 8; + + // Value of the ``X-Request-Id`` request header + // + // This header is used by Envoy to uniquely identify a request. + // It will be generated for all external requests and internal requests that + // do not already have a request ID. + string request_id = 9; + + // Value of the ``X-Envoy-Original-Path`` request header. + string original_path = 10; + + // Size of the HTTP request headers in bytes. + // + // This value is captured from the OSI layer 7 perspective, i.e. it does not + // include overhead from framing or encoding at other networking layers. + uint64 request_headers_bytes = 11; + + // Size of the HTTP request body in bytes. + // + // This value is captured from the OSI layer 7 perspective, i.e. it does not + // include overhead from framing or encoding at other networking layers. + uint64 request_body_bytes = 12; + + // Map of additional headers that have been configured to be logged. + map request_headers = 13; +} + +message HTTPResponseProperties { + // The HTTP response code returned by Envoy. + google.protobuf.UInt32Value response_code = 1; + + // Size of the HTTP response headers in bytes. + // + // This value is captured from the OSI layer 7 perspective, i.e. it does not + // include overhead from framing or encoding at other networking layers. + uint64 response_headers_bytes = 2; + + // Size of the HTTP response body in bytes. + // + // This value is captured from the OSI layer 7 perspective, i.e. it does not + // include overhead from framing or encoding at other networking layers. + uint64 response_body_bytes = 3; + + // Map of additional headers configured to be logged. + map response_headers = 4; + + // Map of trailers configured to be logged. + map response_trailers = 5; + + // The HTTP response code details. + string response_code_details = 6; +} diff --git a/api/envoy/data/cluster/v3alpha/BUILD b/api/envoy/data/cluster/v3alpha/BUILD new file mode 100644 index 000000000000..00edd8294b6f --- /dev/null +++ b/api/envoy/data/cluster/v3alpha/BUILD @@ -0,0 +1,11 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") + +licenses(["notice"]) # Apache 2 + +api_proto_library( + name = "outlier_detection_event", + srcs = ["outlier_detection_event.proto"], + visibility = [ + "//visibility:public", + ], +) diff --git a/api/envoy/data/cluster/v3alpha/outlier_detection_event.proto b/api/envoy/data/cluster/v3alpha/outlier_detection_event.proto new file mode 100644 index 000000000000..48f0e27b86bf --- /dev/null +++ b/api/envoy/data/cluster/v3alpha/outlier_detection_event.proto @@ -0,0 +1,102 @@ +syntax = "proto3"; + +package envoy.data.cluster.v3alpha; + +option java_outer_classname = "OutlierDetectionEventProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.data.cluster.v3alpha"; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Outlier detection logging events] +// :ref:`Outlier detection logging `. + +message OutlierDetectionEvent { + // In case of eject represents type of ejection that took place. + OutlierEjectionType type = 1 [(validate.rules).enum.defined_only = true]; + // Timestamp for event. + google.protobuf.Timestamp timestamp = 2 [(gogoproto.stdtime) = true]; + // The time in seconds since the last action (either an ejection or unejection) took place. + google.protobuf.UInt64Value secs_since_last_action = 3; + // The :ref:`cluster ` that owns the ejected host. + string cluster_name = 4 [(validate.rules).string.min_bytes = 1]; + // The URL of the ejected host. E.g., ``tcp://1.2.3.4:80``. + string upstream_url = 5 [(validate.rules).string.min_bytes = 1]; + // The action that took place. + Action action = 6 [(validate.rules).enum.defined_only = true]; + // If ``action`` is ``eject``, specifies the number of times the host has been ejected (local to + // that Envoy and gets reset if the host gets removed from the upstream cluster for any reason and + // then re-added). + uint32 num_ejections = 7; + // If ``action`` is ``eject``, specifies if the ejection was enforced. ``true`` means the host was + // ejected. ``false`` means the event was logged but the host was not actually ejected. + bool enforced = 8; + + oneof event { + option (validate.required) = true; + OutlierEjectSuccessRate eject_success_rate_event = 9; + OutlierEjectConsecutive eject_consecutive_event = 10; + } +} + +// Type of ejection that took place +enum OutlierEjectionType { + // In case upstream host returns certain number of consecutive 5xx. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *false*, all type of errors are treated as HTTP 5xx errors. + // See :ref:`Cluster outlier detection ` documentation for + // details. + CONSECUTIVE_5XX = 0; + // In case upstream host returns certain number of consecutive gateway errors + CONSECUTIVE_GATEWAY_FAILURE = 1; + // Runs over aggregated success rate statistics from every host in cluster + // and selects hosts for which ratio of successful replies deviates from other hosts + // in the cluster. + // If + // :ref:`outlier_detection.split_external_local_origin_errors` + // is *false*, all errors (externally and locally generated) are used to calculate success rate + // statistics. See :ref:`Cluster outlier detection ` + // documentation for details. + SUCCESS_RATE = 2; + // Consecutive local origin failures: Connection failures, resets, timeouts, etc + // This type of ejection happens only when + // :ref:`outlier_detection.split_external_local_origin_errors` + // is set to *true*. + // See :ref:`Cluster outlier detection ` documentation for + CONSECUTIVE_LOCAL_ORIGIN_FAILURE = 3; + // Runs over aggregated success rate statistics for local origin failures + // for all hosts in the cluster and selects hosts for which success rate deviates from other + // hosts in the cluster. This type of ejection happens only when + // :ref:`outlier_detection.split_external_local_origin_errors` + // is set to *true*. + // See :ref:`Cluster outlier detection ` documentation for + SUCCESS_RATE_LOCAL_ORIGIN = 4; +} + +// Represents possible action applied to upstream host +enum Action { + // In case host was excluded from service + EJECT = 0; + // In case host was brought back into service + UNEJECT = 1; +} + +message OutlierEjectSuccessRate { + // Host’s success rate at the time of the ejection event on a 0-100 range. + uint32 host_success_rate = 1 [(validate.rules).uint32.lte = 100]; + // Average success rate of the hosts in the cluster at the time of the ejection event on a 0-100 + // range. + uint32 cluster_average_success_rate = 2 [(validate.rules).uint32.lte = 100]; + // Success rate ejection threshold at the time of the ejection event. + uint32 cluster_success_rate_ejection_threshold = 3 [(validate.rules).uint32.lte = 100]; +} + +message OutlierEjectConsecutive { +} diff --git a/api/envoy/data/core/v3alpha/BUILD b/api/envoy/data/core/v3alpha/BUILD new file mode 100644 index 000000000000..9e82e3eb1731 --- /dev/null +++ b/api/envoy/data/core/v3alpha/BUILD @@ -0,0 +1,15 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") + +licenses(["notice"]) # Apache 2 + +api_proto_library( + name = "health_check_event", + srcs = ["health_check_event.proto"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + ], +) diff --git a/api/envoy/data/core/v3alpha/health_check_event.proto b/api/envoy/data/core/v3alpha/health_check_event.proto new file mode 100644 index 000000000000..628b6870b64d --- /dev/null +++ b/api/envoy/data/core/v3alpha/health_check_event.proto @@ -0,0 +1,85 @@ +syntax = "proto3"; + +package envoy.data.core.v3alpha; + +option java_outer_classname = "HealthCheckEventProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.data.core.v3alpha"; + +import "envoy/api/v3alpha/core/address.proto"; + +import "google/protobuf/timestamp.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: Health check logging events] +// :ref:`Health check logging `. + +message HealthCheckEvent { + HealthCheckerType health_checker_type = 1 [(validate.rules).enum.defined_only = true]; + envoy.api.v3alpha.core.Address host = 2; + string cluster_name = 3 [(validate.rules).string.min_bytes = 1]; + + oneof event { + option (validate.required) = true; + + // Host ejection. + HealthCheckEjectUnhealthy eject_unhealthy_event = 4; + + // Host addition. + HealthCheckAddHealthy add_healthy_event = 5; + + // Host failure. + HealthCheckFailure health_check_failure_event = 7; + + // Healthy host became degraded. + DegradedHealthyHost degraded_healthy_host = 8; + + // A degraded host returned to being healthy. + NoLongerDegradedHost no_longer_degraded_host = 9; + } + + // Timestamp for event. + google.protobuf.Timestamp timestamp = 6 [(gogoproto.stdtime) = true]; +} + +enum HealthCheckFailureType { + ACTIVE = 0; + PASSIVE = 1; + NETWORK = 2; +} + +enum HealthCheckerType { + HTTP = 0; + TCP = 1; + GRPC = 2; + REDIS = 3; +} + +message HealthCheckEjectUnhealthy { + // The type of failure that caused this ejection. + HealthCheckFailureType failure_type = 1 [(validate.rules).enum.defined_only = true]; +} + +message HealthCheckAddHealthy { + // Whether this addition is the result of the first ever health check on a host, in which case + // the configured :ref:`healthy threshold ` + // is bypassed and the host is immediately added. + bool first_check = 1; +} + +message HealthCheckFailure { + // The type of failure that caused this event. + HealthCheckFailureType failure_type = 1 [(validate.rules).enum.defined_only = true]; + // Whether this event is the result of the first ever health check on a host. + bool first_check = 2; +} + +message DegradedHealthyHost { +} + +message NoLongerDegradedHost { +} diff --git a/api/envoy/data/tap/v3alpha/BUILD b/api/envoy/data/tap/v3alpha/BUILD new file mode 100644 index 000000000000..ab9be74ce98a --- /dev/null +++ b/api/envoy/data/tap/v3alpha/BUILD @@ -0,0 +1,37 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "common", + srcs = ["common.proto"], +) + +api_proto_library_internal( + name = "transport", + srcs = ["transport.proto"], + deps = [ + ":common", + "//envoy/api/v3alpha/core:address", + ], +) + +api_proto_library_internal( + name = "http", + srcs = ["http.proto"], + deps = [ + ":common", + "//envoy/api/v3alpha/core:base", + ], +) + +api_proto_library_internal( + name = "wrapper", + srcs = ["wrapper.proto"], + visibility = ["//visibility:public"], + deps = [ + ":http", + ":transport", + "//envoy/api/v3alpha/core:address", + ], +) diff --git a/api/envoy/data/tap/v3alpha/common.proto b/api/envoy/data/tap/v3alpha/common.proto new file mode 100644 index 000000000000..21da336e7485 --- /dev/null +++ b/api/envoy/data/tap/v3alpha/common.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package envoy.data.tap.v3alpha; + +option java_outer_classname = "CommonProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.data.tap.v3alpha"; + +// [#protodoc-title: Tap common data] + +// Wrapper for tapped body data. This includes HTTP request/response body, transport socket received +// and transmitted data, etc. +message Body { + oneof body_type { + // Body data as bytes. By default, tap body data will be present in this field, as the proto + // `bytes` type can contain any valid byte. + bytes as_bytes = 1; + + // Body data as string. This field is only used when the :ref:`JSON_BODY_AS_STRING + // ` sink + // format type is selected. See the documentation for that option for why this is useful. + string as_string = 2; + } + + // Specifies whether body data has been truncated to fit within the specified + // :ref:`max_buffered_rx_bytes + // ` and + // :ref:`max_buffered_tx_bytes + // ` settings. + bool truncated = 3; +} diff --git a/api/envoy/data/tap/v3alpha/http.proto b/api/envoy/data/tap/v3alpha/http.proto new file mode 100644 index 000000000000..36d82f7a048c --- /dev/null +++ b/api/envoy/data/tap/v3alpha/http.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +package envoy.data.tap.v3alpha; + +option java_outer_classname = "HttpProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.data.tap.v3alpha"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/data/tap/v3alpha/common.proto"; + +// [#protodoc-title: HTTP tap data] + +// A fully buffered HTTP trace message. +message HttpBufferedTrace { + // HTTP message wrapper. + message Message { + // Message headers. + repeated api.v3alpha.core.HeaderValue headers = 1; + + // Message body. + Body body = 2; + + // Message trailers. + repeated api.v3alpha.core.HeaderValue trailers = 3; + } + + // Request message. + Message request = 1; + + // Response message. + Message response = 2; +} + +// A streamed HTTP trace segment. Multiple segments make up a full trace. +message HttpStreamedTraceSegment { + // Trace ID unique to the originating Envoy only. Trace IDs can repeat and should not be used + // for long term stable uniqueness. + uint64 trace_id = 1; + + oneof message_piece { + // Request headers. + api.v3alpha.core.HeaderMap request_headers = 2; + + // Request body chunk. + Body request_body_chunk = 3; + + // Request trailers. + api.v3alpha.core.HeaderMap request_trailers = 4; + + // Response headers. + api.v3alpha.core.HeaderMap response_headers = 5; + + // Response body chunk. + Body response_body_chunk = 6; + + // Response trailers. + api.v3alpha.core.HeaderMap response_trailers = 7; + } +} diff --git a/api/envoy/data/tap/v3alpha/transport.proto b/api/envoy/data/tap/v3alpha/transport.proto new file mode 100644 index 000000000000..3dfb0c6478ba --- /dev/null +++ b/api/envoy/data/tap/v3alpha/transport.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +// [#protodoc-title: Transport tap data] +// Trace format for the tap transport socket extension. This dumps plain text read/write +// sequences on a socket. + +package envoy.data.tap.v3alpha; + +option java_outer_classname = "TransportProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.data.tap.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/data/tap/v3alpha/common.proto"; + +import "google/protobuf/timestamp.proto"; + +// Connection properties. +message Connection { + // Local address. + envoy.api.v3alpha.core.Address local_address = 2; + + // Remote address. + envoy.api.v3alpha.core.Address remote_address = 3; +} + +// Event in a socket trace. +message SocketEvent { + // Timestamp for event. + google.protobuf.Timestamp timestamp = 1; + + // Data read by Envoy from the transport socket. + message Read { + // Binary data read. + Body data = 1; + + // TODO(htuch): Half-close for reads. + } + + // Data written by Envoy to the transport socket. + message Write { + // Binary data written. + Body data = 1; + + // Stream was half closed after this write. + bool end_stream = 2; + } + + // The connection was closed. + message Closed { + // TODO(mattklein123): Close event type. + } + + // Read or write with content as bytes string. + oneof event_selector { + Read read = 2; + Write write = 3; + Closed closed = 4; + } +} + +// Sequence of read/write events that constitute a buffered trace on a socket. +message SocketBufferedTrace { + // Trace ID unique to the originating Envoy only. Trace IDs can repeat and should not be used + // for long term stable uniqueness. Matches connection IDs used in Envoy logs. + uint64 trace_id = 1; + + // Connection properties. + Connection connection = 2; + + // Sequence of observed events. + repeated SocketEvent events = 3; + + // Set to true if read events were truncated due to the :ref:`max_buffered_rx_bytes + // ` setting. + bool read_truncated = 4; + + // Set to true if write events were truncated due to the :ref:`max_buffered_tx_bytes + // ` setting. + bool write_truncated = 5; +} + +// A streamed socket trace segment. Multiple segments make up a full trace. +message SocketStreamedTraceSegment { + // Trace ID unique to the originating Envoy only. Trace IDs can repeat and should not be used + // for long term stable uniqueness. Matches connection IDs used in Envoy logs. + uint64 trace_id = 1; + + oneof message_piece { + // Connection properties. + Connection connection = 2; + + // Socket event. + SocketEvent event = 3; + } +} diff --git a/api/envoy/data/tap/v3alpha/wrapper.proto b/api/envoy/data/tap/v3alpha/wrapper.proto new file mode 100644 index 000000000000..1aff052e90d1 --- /dev/null +++ b/api/envoy/data/tap/v3alpha/wrapper.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +import "envoy/data/tap/v3alpha/http.proto"; +import "envoy/data/tap/v3alpha/transport.proto"; + +import "validate/validate.proto"; + +package envoy.data.tap.v3alpha; + +option java_outer_classname = "WrapperProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.data.tap.v3alpha"; + +// [#protodoc-title: Tap data wrappers] + +// Wrapper for all fully buffered and streamed tap traces that Envoy emits. This is required for +// sending traces over gRPC APIs or more easily persisting binary messages to files. +message TraceWrapper { + oneof trace { + option (validate.required) = true; + + // An HTTP buffered tap trace. + HttpBufferedTrace http_buffered_trace = 1; + + // An HTTP streamed tap trace segment. + HttpStreamedTraceSegment http_streamed_trace_segment = 2; + + // A socket buffered tap trace. + SocketBufferedTrace socket_buffered_trace = 3; + + // A socket streamed tap trace segment. + SocketStreamedTraceSegment socket_streamed_trace_segment = 4; + } +} diff --git a/api/envoy/service/accesslog/v3alpha/BUILD b/api/envoy/service/accesslog/v3alpha/BUILD new file mode 100644 index 000000000000..1a8eab975b56 --- /dev/null +++ b/api/envoy/service/accesslog/v3alpha/BUILD @@ -0,0 +1,23 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "als", + srcs = ["als.proto"], + has_services = 1, + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:grpc_service", + "//envoy/data/accesslog/v3alpha:accesslog", + ], +) + +api_go_grpc_library( + name = "als", + proto = ":als", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/data/accesslog/v3alpha:accesslog_go_proto", + ], +) diff --git a/api/envoy/service/accesslog/v3alpha/als.proto b/api/envoy/service/accesslog/v3alpha/als.proto new file mode 100644 index 000000000000..092d4d17696c --- /dev/null +++ b/api/envoy/service/accesslog/v3alpha/als.proto @@ -0,0 +1,71 @@ +syntax = "proto3"; + +package envoy.service.accesslog.v3alpha; + +option java_outer_classname = "AlsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.accesslog.v3alpha"; +option go_package = "v2"; +option java_generic_services = true; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/data/accesslog/v3alpha/accesslog.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: gRPC Access Log Service (ALS)] + +// Service for streaming access logs from Envoy to an access log server. +service AccessLogService { + // Envoy will connect and send StreamAccessLogsMessage messages forever. It does not expect any + // response to be sent as nothing would be done in the case of failure. The server should + // disconnect if it expects Envoy to reconnect. In the future we may decide to add a different + // API for "critical" access logs in which Envoy will buffer access logs for some period of time + // until it gets an ACK so it could then retry. This API is designed for high throughput with the + // expectation that it might be lossy. + rpc StreamAccessLogs(stream StreamAccessLogsMessage) returns (StreamAccessLogsResponse) { + } +} + +// Empty response for the StreamAccessLogs API. Will never be sent. See below. +message StreamAccessLogsResponse { +} + +// Stream message for the StreamAccessLogs API. Envoy will open a stream to the server and stream +// access logs without ever expecting a response. +message StreamAccessLogsMessage { + message Identifier { + // The node sending the access log messages over the stream. + envoy.api.v3alpha.core.Node node = 1 [(validate.rules).message.required = true]; + + // The friendly name of the log configured in :ref:`CommonGrpcAccessLogConfig + // `. + string log_name = 2 [(validate.rules).string.min_bytes = 1]; + } + + // Identifier data that will only be sent in the first message on the stream. This is effectively + // structured metadata and is a performance optimization. + Identifier identifier = 1; + + // Wrapper for batches of HTTP access log entries. + message HTTPAccessLogEntries { + repeated envoy.data.accesslog.v3alpha.HTTPAccessLogEntry log_entry = 1 + [(validate.rules).repeated .min_items = 1]; + } + + // Wrapper for batches of TCP access log entries. + message TCPAccessLogEntries { + repeated envoy.data.accesslog.v3alpha.TCPAccessLogEntry log_entry = 1 + [(validate.rules).repeated .min_items = 1]; + } + + // Batches of log entries of a single type. Generally speaking, a given stream should only + // ever include one type of log entry. + oneof log_entries { + option (validate.required) = true; + + HTTPAccessLogEntries http_logs = 2; + + TCPAccessLogEntries tcp_logs = 3; + } +} diff --git a/api/envoy/service/auth/v3alpha/BUILD b/api/envoy/service/auth/v3alpha/BUILD new file mode 100644 index 000000000000..6a335f88f949 --- /dev/null +++ b/api/envoy/service/auth/v3alpha/BUILD @@ -0,0 +1,28 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "attribute_context", + srcs = [ + "attribute_context.proto", + ], + deps = [ + "//envoy/api/v3alpha/core:address", + "//envoy/api/v3alpha/core:base", + ], +) + +api_proto_library_internal( + name = "external_auth", + srcs = [ + "external_auth.proto", + ], + has_services = 1, + visibility = ["//visibility:public"], + deps = [ + ":attribute_context", + "//envoy/api/v3alpha/core:base", + "//envoy/type:http_status", + ], +) diff --git a/api/envoy/service/auth/v3alpha/attribute_context.proto b/api/envoy/service/auth/v3alpha/attribute_context.proto new file mode 100644 index 000000000000..de3164583167 --- /dev/null +++ b/api/envoy/service/auth/v3alpha/attribute_context.proto @@ -0,0 +1,154 @@ +syntax = "proto3"; + +package envoy.service.auth.v3alpha; + +option java_outer_classname = "AttributeContextProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.auth.v3alpha"; + +import "envoy/api/v3alpha/core/address.proto"; +import "envoy/api/v3alpha/core/base.proto"; + +import "google/protobuf/timestamp.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: Attribute Context ] + +// See :ref:`network filter configuration overview ` +// and :ref:`HTTP filter configuration overview `. + +// An attribute is a piece of metadata that describes an activity on a network. +// For example, the size of an HTTP request, or the status code of an HTTP response. +// +// Each attribute has a type and a name, which is logically defined as a proto message field +// of the `AttributeContext`. The `AttributeContext` is a collection of individual attributes +// supported by Envoy authorization system. +message AttributeContext { + // This message defines attributes for a node that handles a network request. + // The node can be either a service or an application that sends, forwards, + // or receives the request. Service peers should fill in the `service`, + // `principal`, and `labels` as appropriate. + message Peer { + // The address of the peer, this is typically the IP address. + // It can also be UDS path, or others. + envoy.api.v3alpha.core.Address address = 1; + + // The canonical service name of the peer. + // It should be set to :ref:`the HTTP x-envoy-downstream-service-cluster + // ` + // If a more trusted source of the service name is available through mTLS/secure naming, it + // should be used. + string service = 2; + + // The labels associated with the peer. + // These could be pod labels for Kubernetes or tags for VMs. + // The source of the labels could be an X.509 certificate or other configuration. + map labels = 3; + + // The authenticated identity of this peer. + // For example, the identity associated with the workload such as a service account. + // If an X.509 certificate is used to assert the identity this field should be sourced from + // `URI Subject Alternative Names`, `DNS Subject Alternate Names` or `Subject` in that order. + // The primary identity should be the principal. The principal format is issuer specific. + // + // Example: + // * SPIFFE format is `spiffe://trust-domain/path` + // * Google account format is `https://accounts.google.com/{userid}` + string principal = 4; + } + + // Represents a network request, such as an HTTP request. + message Request { + // The timestamp when the proxy receives the first byte of the request. + google.protobuf.Timestamp time = 1; + + // Represents an HTTP request or an HTTP-like request. + HttpRequest http = 2; + + // More request types are added here as necessary. + } + + // This message defines attributes for an HTTP request. + // HTTP/1.x, HTTP/2, gRPC are all considered as HTTP requests. + message HttpRequest { + // The unique ID for a request, which can be propagated to downstream + // systems. The ID should have low probability of collision + // within a single day for a specific service. + // For HTTP requests, it should be X-Request-ID or equivalent. + string id = 1; + + // The HTTP request method, such as `GET`, `POST`. + string method = 2; + + // The HTTP request headers. If multiple headers share the same key, they + // must be merged according to the HTTP spec. All header keys must be + // lowercased, because HTTP header keys are case-insensitive. + map headers = 3; + + // The request target, as it appears in the first line of the HTTP request. This includes + // the URL path and query-string. No decoding is performed. + string path = 4; + + // The HTTP request `Host` or 'Authority` header value. + string host = 5; + + // The HTTP URL scheme, such as `http` and `https`. + string scheme = 6; + + // This field is always empty, and exists for compatibility reasons. The HTTP URL query is + // included in `path` field. + string query = 7; + + // This field is always empty, and exists for compatibility reasons. The URL fragment is + // not submitted as part of HTTP requests; it is unknowable. + string fragment = 8; + + // The HTTP request size in bytes. If unknown, it must be -1. + int64 size = 9; + + // The network protocol used with the request, such as "HTTP/1.0", "HTTP/1.1", or "HTTP/2". + // + // See :repo:`headers.h:ProtocolStrings ` for a list of all + // possible values. + string protocol = 10; + + // The HTTP request body. + string body = 11; + } + + // The source of a network activity, such as starting a TCP connection. + // In a multi hop network activity, the source represents the sender of the + // last hop. + Peer source = 1; + + // The destination of a network activity, such as accepting a TCP connection. + // In a multi hop network activity, the destination represents the receiver of + // the last hop. + Peer destination = 2; + + // Represents a network request, such as an HTTP request. + Request request = 4; + + // This is analogous to http_request.headers, however these contents will not be sent to the + // upstream server. Context_extensions provide an extension mechanism for sending additional + // information to the auth server without modifying the proto definition. It maps to the + // internal opaque context in the filter chain. + map context_extensions = 10; + + // Dynamic metadata associated with the request. + envoy.api.v3alpha.core.Metadata metadata_context = 11; +} + +// The following items are left out of this proto +// Request.Auth field for jwt tokens +// Request.Api for api management +// Origin peer that originated the request +// Caching Protocol +// request_context return values to inject back into the filter chain +// peer.claims -- from X.509 extensions +// Configuration +// - field mask to send +// - which return values from request_context are copied back +// - which return values are copied into request_headers diff --git a/api/envoy/service/auth/v3alpha/external_auth.proto b/api/envoy/service/auth/v3alpha/external_auth.proto new file mode 100644 index 000000000000..4b7e459a4436 --- /dev/null +++ b/api/envoy/service/auth/v3alpha/external_auth.proto @@ -0,0 +1,77 @@ +syntax = "proto3"; + +package envoy.service.auth.v3alpha; + +option java_outer_classname = "ExternalAuthProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.auth.v3alpha"; +option go_package = "v2"; +option java_generic_services = true; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/type/http_status.proto"; +import "envoy/service/auth/v3alpha/attribute_context.proto"; + +import "google/rpc/status.proto"; +import "validate/validate.proto"; + +// [#protodoc-title: Authorization Service ] + +// The authorization service request messages used by external authorization :ref:`network filter +// ` and :ref:`HTTP filter `. + +// A generic interface for performing authorization check on incoming +// requests to a networked service. +service Authorization { + // Performs authorization check based on the attributes associated with the + // incoming request, and returns status `OK` or not `OK`. + rpc Check(CheckRequest) returns (CheckResponse); +} + +message CheckRequest { + // The request attributes. + AttributeContext attributes = 1; +} + +// HTTP attributes for a denied response. +message DeniedHttpResponse { + // This field allows the authorization service to send a HTTP response status + // code to the downstream client other than 403 (Forbidden). + envoy.type.HttpStatus status = 1 [(validate.rules).message.required = true]; + + // This field allows the authorization service to send HTTP response headers + // to the downstream client. + repeated envoy.api.v3alpha.core.HeaderValueOption headers = 2; + + // This field allows the authorization service to send a response body data + // to the downstream client. + string body = 3; +} + +// HTTP attributes for an ok response. +message OkHttpResponse { + // HTTP entity headers in addition to the original request headers. This allows the authorization + // service to append, to add or to override headers from the original request before + // dispatching it to the upstream. By setting `append` field to `true` in the `HeaderValueOption`, + // the filter will append the correspondent header value to the matched request header. Note that + // by Leaving `append` as false, the filter will either add a new header, or override an existing + // one if there is a match. + repeated envoy.api.v3alpha.core.HeaderValueOption headers = 2; +} + +// Intended for gRPC and Network Authorization servers `only`. +message CheckResponse { + // Status `OK` allows the request. Any other status indicates the request should be denied. + google.rpc.Status status = 1; + + // An message that contains HTTP response attributes. This message is + // used when the authorization service needs to send custom responses to the + // downstream client or, to modify/add request headers being dispatched to the upstream. + oneof http_response { + // Supplies http attributes for a denied response. + DeniedHttpResponse denied_response = 2; + + // Supplies http attributes for an ok response. + OkHttpResponse ok_response = 3; + } +} diff --git a/api/envoy/service/discovery/v3alpha/BUILD b/api/envoy/service/discovery/v3alpha/BUILD new file mode 100644 index 000000000000..d34955c1cb5a --- /dev/null +++ b/api/envoy/service/discovery/v3alpha/BUILD @@ -0,0 +1,75 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "ads", + srcs = ["ads.proto"], + has_services = 1, + deps = [ + "//envoy/api/v3alpha:discovery", + ], +) + +api_go_grpc_library( + name = "ads", + proto = ":ads", + deps = [ + "//envoy/api/v3alpha:discovery_go_proto", + ], +) + +api_proto_library_internal( + name = "hds", + srcs = ["hds.proto"], + has_services = 1, + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:health_check", + "//envoy/api/v3alpha/endpoint", + ], +) + +api_go_grpc_library( + name = "hds", + proto = ":hds", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:health_check_go_proto", + "//envoy/api/v3alpha/endpoint:endpoint_go_proto", + ], +) + +api_proto_library_internal( + name = "sds", + srcs = ["sds.proto"], + has_services = 1, + deps = [ + "//envoy/api/v3alpha:discovery", + ], +) + +api_go_grpc_library( + name = "sds", + proto = ":sds", + deps = [ + "//envoy/api/v3alpha:discovery_go_proto", + ], +) + +api_proto_library_internal( + name = "rtds", + srcs = ["rtds.proto"], + has_services = 1, + deps = [ + "//envoy/api/v3alpha:discovery", + ], +) + +api_go_grpc_library( + name = "rtds", + proto = ":rtds", + deps = [ + "//envoy/api/v3alpha:discovery_go_proto", + ], +) diff --git a/api/envoy/service/discovery/v3alpha/ads.proto b/api/envoy/service/discovery/v3alpha/ads.proto new file mode 100644 index 000000000000..d6b7897ba7a1 --- /dev/null +++ b/api/envoy/service/discovery/v3alpha/ads.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +package envoy.service.discovery.v3alpha; + +option java_outer_classname = "AdsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.discovery.v3alpha"; +option go_package = "v2"; +option java_generic_services = true; + +import "envoy/api/v3alpha/discovery.proto"; + +// [#not-implemented-hide:] Not configuration. Workaround c++ protobuf issue with importing +// services: https://github.com/google/protobuf/issues/4221 +message AdsDummy { +} + +// [#not-implemented-hide:] Discovery services for endpoints, clusters, routes, +// and listeners are retained in the package `envoy.api.v3alpha` for backwards +// compatibility with existing management servers. New development in discovery +// services should proceed in the package `envoy.service.discovery.v3alpha`. + +// See https://github.com/lyft/envoy-api#apis for a description of the role of +// ADS and how it is intended to be used by a management server. ADS requests +// have the same structure as their singleton xDS counterparts, but can +// multiplex many resource types on a single stream. The type_url in the +// DiscoveryRequest/DiscoveryResponse provides sufficient information to recover +// the multiplexed singleton APIs at the Envoy instance and management server. +service AggregatedDiscoveryService { + // This is a gRPC-only API. + rpc StreamAggregatedResources(stream envoy.api.v3alpha.DiscoveryRequest) + returns (stream envoy.api.v3alpha.DiscoveryResponse) { + } + + rpc DeltaAggregatedResources(stream envoy.api.v3alpha.DeltaDiscoveryRequest) + returns (stream envoy.api.v3alpha.DeltaDiscoveryResponse) { + } +} diff --git a/api/envoy/service/discovery/v3alpha/hds.proto b/api/envoy/service/discovery/v3alpha/hds.proto new file mode 100644 index 000000000000..5ec7f491c8f2 --- /dev/null +++ b/api/envoy/service/discovery/v3alpha/hds.proto @@ -0,0 +1,127 @@ +syntax = "proto3"; + +package envoy.service.discovery.v3alpha; + +option java_outer_classname = "HdsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.discovery.v3alpha"; + +option java_generic_services = true; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/core/health_check.proto"; +import "envoy/api/v3alpha/endpoint/endpoint.proto"; + +import "google/api/annotations.proto"; +import "google/protobuf/duration.proto"; + +// [#proto-status: experimental] +// HDS is Health Discovery Service. It compliments Envoy’s health checking +// service by designating this Envoy to be a healthchecker for a subset of hosts +// in the cluster. The status of these health checks will be reported to the +// management server, where it can be aggregated etc and redistributed back to +// Envoy through EDS. +service HealthDiscoveryService { + // 1. Envoy starts up and if its can_healthcheck option in the static + // bootstrap config is enabled, sends HealthCheckRequest to the management + // server. It supplies its capabilities (which protocol it can health check + // with, what zone it resides in, etc.). + // 2. In response to (1), the management server designates this Envoy as a + // healthchecker to health check a subset of all upstream hosts for a given + // cluster (for example upstream Host 1 and Host 2). It streams + // HealthCheckSpecifier messages with cluster related configuration for all + // clusters this Envoy is designated to health check. Subsequent + // HealthCheckSpecifier message will be sent on changes to: + // a. Endpoints to health checks + // b. Per cluster configuration change + // 3. Envoy creates a health probe based on the HealthCheck config and sends + // it to endpoint(ip:port) of Host 1 and 2. Based on the HealthCheck + // configuration Envoy waits upon the arrival of the probe response and + // looks at the content of the response to decide whether the endpoint is + // healthy or not. If a response hasn't been received within the timeout + // interval, the endpoint health status is considered TIMEOUT. + // 4. Envoy reports results back in an EndpointHealthResponse message. + // Envoy streams responses as often as the interval configured by the + // management server in HealthCheckSpecifier. + // 5. The management Server collects health statuses for all endpoints in the + // cluster (for all clusters) and uses this information to construct + // EndpointDiscoveryResponse messages. + // 6. Once Envoy has a list of upstream endpoints to send traffic to, it load + // balances traffic to them without additional health checking. It may + // use inline healthcheck (i.e. consider endpoint UNHEALTHY if connection + // failed to a particular endpoint to account for health status propagation + // delay between HDS and EDS). + // By default, can_healthcheck is true. If can_healthcheck is false, Cluster + // configuration may not contain HealthCheck message. + // TODO(htuch): How is can_healthcheck communicated to CDS to ensure the above + // invariant? + // TODO(htuch): Add @amb67's diagram. + rpc StreamHealthCheck(stream HealthCheckRequestOrEndpointHealthResponse) + returns (stream HealthCheckSpecifier) { + } + + // TODO(htuch): Unlike the gRPC version, there is no stream-based binding of + // request/response. Should we add an identifier to the HealthCheckSpecifier + // to bind with the response? + rpc FetchHealthCheck(HealthCheckRequestOrEndpointHealthResponse) returns (HealthCheckSpecifier) { + option (google.api.http) = { + post: "/v2/discovery:health_check" + body: "*" + }; + } +} + +// Defines supported protocols etc, so the management server can assign proper +// endpoints to healthcheck. +message Capability { + // Different Envoy instances may have different capabilities (e.g. Redis) + // and/or have ports enabled for different protocols. + enum Protocol { + HTTP = 0; + TCP = 1; + REDIS = 2; + } + repeated Protocol health_check_protocols = 1; +} + +message HealthCheckRequest { + envoy.api.v3alpha.core.Node node = 1; + Capability capability = 2; +} + +message EndpointHealth { + envoy.api.v3alpha.endpoint.Endpoint endpoint = 1; + envoy.api.v3alpha.core.HealthStatus health_status = 2; +} + +message EndpointHealthResponse { + repeated EndpointHealth endpoints_health = 1; +} + +message HealthCheckRequestOrEndpointHealthResponse { + oneof request_type { + HealthCheckRequest health_check_request = 1; + EndpointHealthResponse endpoint_health_response = 2; + } +} + +message LocalityEndpoints { + envoy.api.v3alpha.core.Locality locality = 1; + repeated envoy.api.v3alpha.endpoint.Endpoint endpoints = 2; +} + +// The cluster name and locality is provided to Envoy for the endpoints that it +// health checks to support statistics reporting, logging and debugging by the +// Envoy instance (outside of HDS). For maximum usefulness, it should match the +// same cluster structure as that provided by EDS. +message ClusterHealthCheck { + string cluster_name = 1; + repeated envoy.api.v3alpha.core.HealthCheck health_checks = 2; + repeated LocalityEndpoints locality_endpoints = 3; +} + +message HealthCheckSpecifier { + repeated ClusterHealthCheck cluster_health_checks = 1; + // The default is 1 second. + google.protobuf.Duration interval = 2; +} diff --git a/api/envoy/service/discovery/v3alpha/rtds.proto b/api/envoy/service/discovery/v3alpha/rtds.proto new file mode 100644 index 000000000000..5a59cf13f814 --- /dev/null +++ b/api/envoy/service/discovery/v3alpha/rtds.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +package envoy.service.discovery.v3alpha; + +option java_outer_classname = "RtdsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.discovery.v3alpha"; +option java_generic_services = true; + +import "envoy/api/v3alpha/discovery.proto"; + +import "google/api/annotations.proto"; +import "google/protobuf/struct.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Runtime Discovery Service (RTDS)] +// RTDS :ref:`configuration overview ` + +// [#not-implemented-hide:] Not configuration. Workaround c++ protobuf issue with importing +// services: https://github.com/google/protobuf/issues/4221 +message RtdsDummy { +} + +// Discovery service for Runtime resources. +service RuntimeDiscoveryService { + rpc StreamRuntime(stream envoy.api.v3alpha.DiscoveryRequest) + returns (stream envoy.api.v3alpha.DiscoveryResponse) { + } + + rpc DeltaRuntime(stream envoy.api.v3alpha.DeltaDiscoveryRequest) + returns (stream envoy.api.v3alpha.DeltaDiscoveryResponse) { + } + + rpc FetchRuntime(envoy.api.v3alpha.DiscoveryRequest) + returns (envoy.api.v3alpha.DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:runtime" + body: "*" + }; + } +} + +// RTDS resource type. This describes a layer in the runtime virtual filesystem. +message Runtime { + // Runtime resource name. This makes the Runtime a self-describing xDS + // resource. + string name = 1 [(validate.rules).string.min_bytes = 1]; + google.protobuf.Struct layer = 2; +} diff --git a/api/envoy/service/discovery/v3alpha/sds.proto b/api/envoy/service/discovery/v3alpha/sds.proto new file mode 100644 index 000000000000..9f8aa92befa2 --- /dev/null +++ b/api/envoy/service/discovery/v3alpha/sds.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package envoy.service.discovery.v3alpha; + +option java_outer_classname = "SdsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.discovery.v3alpha"; + +import "envoy/api/v3alpha/discovery.proto"; + +import "google/api/annotations.proto"; + +// [#not-implemented-hide:] Not configuration. Workaround c++ protobuf issue with importing +// services: https://github.com/google/protobuf/issues/4221 +message SdsDummy { +} + +service SecretDiscoveryService { + rpc DeltaSecrets(stream envoy.api.v3alpha.DeltaDiscoveryRequest) + returns (stream envoy.api.v3alpha.DeltaDiscoveryResponse) { + } + + rpc StreamSecrets(stream envoy.api.v3alpha.DiscoveryRequest) + returns (stream envoy.api.v3alpha.DiscoveryResponse) { + } + + rpc FetchSecrets(envoy.api.v3alpha.DiscoveryRequest) + returns (envoy.api.v3alpha.DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:secrets" + body: "*" + }; + } +} diff --git a/api/envoy/service/load_stats/v3alpha/BUILD b/api/envoy/service/load_stats/v3alpha/BUILD new file mode 100644 index 000000000000..42c7ce8438da --- /dev/null +++ b/api/envoy/service/load_stats/v3alpha/BUILD @@ -0,0 +1,22 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "lrs", + srcs = ["lrs.proto"], + has_services = 1, + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/endpoint:load_report", + ], +) + +api_go_grpc_library( + name = "lrs", + proto = ":lrs", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/endpoint:load_report_go_proto", + ], +) diff --git a/api/envoy/service/load_stats/v3alpha/lrs.proto b/api/envoy/service/load_stats/v3alpha/lrs.proto new file mode 100644 index 000000000000..81058ed574a7 --- /dev/null +++ b/api/envoy/service/load_stats/v3alpha/lrs.proto @@ -0,0 +1,82 @@ +syntax = "proto3"; + +package envoy.service.load_stats.v3alpha; + +option java_outer_classname = "LrsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.load_stats.v3alpha"; +option go_package = "v2"; +option java_generic_services = true; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/endpoint/load_report.proto"; + +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Load reporting service] + +service LoadReportingService { + // Advanced API to allow for multi-dimensional load balancing by remote + // server. For receiving LB assignments, the steps are: + // 1, The management server is configured with per cluster/zone/load metric + // capacity configuration. The capacity configuration definition is + // outside of the scope of this document. + // 2. Envoy issues a standard {Stream,Fetch}Endpoints request for the clusters + // to balance. + // + // Independently, Envoy will initiate a StreamLoadStats bidi stream with a + // management server: + // 1. Once a connection establishes, the management server publishes a + // LoadStatsResponse for all clusters it is interested in learning load + // stats about. + // 2. For each cluster, Envoy load balances incoming traffic to upstream hosts + // based on per-zone weights and/or per-instance weights (if specified) + // based on intra-zone LbPolicy. This information comes from the above + // {Stream,Fetch}Endpoints. + // 3. When upstream hosts reply, they optionally add header with ASCII representation of EndpointLoadMetricStats. + // 4. Envoy aggregates load reports over the period of time given to it in + // LoadStatsResponse.load_reporting_interval. This includes aggregation + // stats Envoy maintains by itself (total_requests, rpc_errors etc.) as + // well as load metrics from upstream hosts. + // 5. When the timer of load_reporting_interval expires, Envoy sends new + // LoadStatsRequest filled with load reports for each cluster. + // 6. The management server uses the load reports from all reported Envoys + // from around the world, computes global assignment and prepares traffic + // assignment destined for each zone Envoys are located in. Goto 2. + rpc StreamLoadStats(stream LoadStatsRequest) returns (stream LoadStatsResponse) { + } +} + +// A load report Envoy sends to the management server. +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. +message LoadStatsRequest { + // Node identifier for Envoy instance. + envoy.api.v3alpha.core.Node node = 1; + + // A list of load stats to report. + repeated envoy.api.v3alpha.endpoint.ClusterStats cluster_stats = 2; +} + +// The management server sends envoy a LoadStatsResponse with all clusters it +// is interested in learning load stats about. +// [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs. +message LoadStatsResponse { + // Clusters to report stats for. + repeated string clusters = 1 [(validate.rules).repeated .min_items = 1]; + + // The minimum interval of time to collect stats over. This is only a minimum for two reasons: + // 1. There may be some delay from when the timer fires until stats sampling occurs. + // 2. For clusters that were already feature in the previous *LoadStatsResponse*, any traffic + // that is observed in between the corresponding previous *LoadStatsRequest* and this + // *LoadStatsResponse* will also be accumulated and billed to the cluster. This avoids a period + // of inobservability that might otherwise exists between the messages. New clusters are not + // subject to this consideration. + google.protobuf.Duration load_reporting_interval = 2; + + // Set to *true* if the management server supports endpoint granularity + // report. + bool report_endpoint_granularity = 3; +} diff --git a/api/envoy/service/metrics/v3alpha/BUILD b/api/envoy/service/metrics/v3alpha/BUILD new file mode 100644 index 000000000000..1f1bb553cd82 --- /dev/null +++ b/api/envoy/service/metrics/v3alpha/BUILD @@ -0,0 +1,24 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "metrics_service", + srcs = ["metrics_service.proto"], + has_services = 1, + require_py = 0, + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:grpc_service", + "@prometheus_metrics_model//:client_model", + ], +) + +api_go_grpc_library( + name = "metrics_service", + proto = ":metrics_service", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "@prometheus_metrics_model//:client_model_go_proto", + ], +) diff --git a/api/envoy/service/metrics/v3alpha/metrics_service.proto b/api/envoy/service/metrics/v3alpha/metrics_service.proto new file mode 100644 index 000000000000..9a5306553b18 --- /dev/null +++ b/api/envoy/service/metrics/v3alpha/metrics_service.proto @@ -0,0 +1,41 @@ +syntax = "proto3"; + +package envoy.service.metrics.v3alpha; + +option java_outer_classname = "MetricsServiceProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.metrics.v3alpha"; +option go_package = "v2"; +option java_generic_services = true; + +import "envoy/api/v3alpha/core/base.proto"; + +import "metrics.proto"; + +import "validate/validate.proto"; + +// Service for streaming metrics to server that consumes the metrics data. It uses Prometheus metric +// data model as a standard to represent metrics information. +service MetricsService { + // Envoy will connect and send StreamMetricsMessage messages forever. It does not expect any + // response to be sent as nothing would be done in the case of failure. + rpc StreamMetrics(stream StreamMetricsMessage) returns (StreamMetricsResponse) { + } +} + +message StreamMetricsResponse { +} + +message StreamMetricsMessage { + message Identifier { + // The node sending metrics over the stream. + envoy.api.v3alpha.core.Node node = 1 [(validate.rules).message.required = true]; + } + + // Identifier data effectively is a structured metadata. As a performance optimization this will + // only be sent in the first message on the stream. + Identifier identifier = 1; + + // A list of metric entries + repeated io.prometheus.client.MetricFamily envoy_metrics = 2; +} diff --git a/api/envoy/service/ratelimit/v3alpha/BUILD b/api/envoy/service/ratelimit/v3alpha/BUILD new file mode 100644 index 000000000000..19954c5bfcc9 --- /dev/null +++ b/api/envoy/service/ratelimit/v3alpha/BUILD @@ -0,0 +1,24 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "rls", + srcs = ["rls.proto"], + has_services = 1, + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:grpc_service", + "//envoy/api/v3alpha/ratelimit", + ], +) + +api_go_grpc_library( + name = "rls", + proto = ":rls", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "//envoy/api/v3alpha/core:grpc_service_go_proto", + "//envoy/api/v3alpha/ratelimit:ratelimit_go_proto", + ], +) diff --git a/api/envoy/service/ratelimit/v3alpha/rls.proto b/api/envoy/service/ratelimit/v3alpha/rls.proto new file mode 100644 index 000000000000..7bbd2e3ec183 --- /dev/null +++ b/api/envoy/service/ratelimit/v3alpha/rls.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +package envoy.service.ratelimit.v3alpha; + +option java_outer_classname = "RlsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.ratelimit.v3alpha"; +option go_package = "v2"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/ratelimit/ratelimit.proto"; + +import "validate/validate.proto"; + +// [#protodoc-title: Rate Limit Service (RLS)] + +service RateLimitService { + // Determine whether rate limiting should take place. + rpc ShouldRateLimit(RateLimitRequest) returns (RateLimitResponse) { + } +} + +// Main message for a rate limit request. The rate limit service is designed to be fully generic +// in the sense that it can operate on arbitrary hierarchical key/value pairs. The loaded +// configuration will parse the request and find the most specific limit to apply. In addition, +// a RateLimitRequest can contain multiple "descriptors" to limit on. When multiple descriptors +// are provided, the server will limit on *ALL* of them and return an OVER_LIMIT response if any +// of them are over limit. This enables more complex application level rate limiting scenarios +// if desired. +message RateLimitRequest { + // All rate limit requests must specify a domain. This enables the configuration to be per + // application without fear of overlap. E.g., "envoy". + string domain = 1; + + // All rate limit requests must specify at least one RateLimitDescriptor. Each descriptor is + // processed by the service (see below). If any of the descriptors are over limit, the entire + // request is considered to be over limit. + repeated envoy.api.v3alpha.ratelimit.RateLimitDescriptor descriptors = 2; + + // Rate limit requests can optionally specify the number of hits a request adds to the matched + // limit. If the value is not set in the message, a request increases the matched limit by 1. + uint32 hits_addend = 3; +} + +// A response from a ShouldRateLimit call. +message RateLimitResponse { + enum Code { + // The response code is not known. + UNKNOWN = 0; + // The response code to notify that the number of requests are under limit. + OK = 1; + // The response code to notify that the number of requests are over limit. + OVER_LIMIT = 2; + } + + // Defines an actual rate limit in terms of requests per unit of time and the unit itself. + message RateLimit { + enum Unit { + // The time unit is not known. + UNKNOWN = 0; + // The time unit representing a second. + SECOND = 1; + // The time unit representing a minute. + MINUTE = 2; + // The time unit representing an hour. + HOUR = 3; + // The time unit representing a day. + DAY = 4; + } + + // The number of requests per unit of time. + uint32 requests_per_unit = 1; + // The unit of time. + Unit unit = 2; + } + + message DescriptorStatus { + // The response code for an individual descriptor. + Code code = 1; + // The current limit as configured by the server. Useful for debugging, etc. + RateLimit current_limit = 2; + // The limit remaining in the current time unit. + uint32 limit_remaining = 3; + } + + // The overall response code which takes into account all of the descriptors that were passed + // in the RateLimitRequest message. + Code overall_code = 1; + // A list of DescriptorStatus messages which matches the length of the descriptor list passed + // in the RateLimitRequest. This can be used by the caller to determine which individual + // descriptors failed and/or what the currently configured limits are for all of them. + repeated DescriptorStatus statuses = 2; + // A list of headers to add to the response + repeated envoy.api.v3alpha.core.HeaderValue headers = 3; +} diff --git a/api/envoy/service/tap/v3alpha/BUILD b/api/envoy/service/tap/v3alpha/BUILD new file mode 100644 index 000000000000..a90b2d819297 --- /dev/null +++ b/api/envoy/service/tap/v3alpha/BUILD @@ -0,0 +1,36 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "common", + srcs = ["common.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/core:grpc_service", + "//envoy/api/v3alpha/route", + ], +) + +api_proto_library_internal( + name = "tap", + srcs = ["tap.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha:discovery", + "//envoy/api/v3alpha/core:base", + "//envoy/data/tap/v3alpha:wrapper", + ], +) + +api_proto_library_internal( + name = "tapds", + srcs = ["tapds.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v3alpha:discovery", + "//envoy/api/v3alpha/core:base", + "//envoy/service/tap/v3alpha:common", + ], +) diff --git a/api/envoy/service/tap/v3alpha/common.proto b/api/envoy/service/tap/v3alpha/common.proto new file mode 100644 index 000000000000..7c375d913d7a --- /dev/null +++ b/api/envoy/service/tap/v3alpha/common.proto @@ -0,0 +1,200 @@ +syntax = "proto3"; + +import "envoy/api/v3alpha/route/route.proto"; +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/core/grpc_service.proto"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +package envoy.service.tap.v3alpha; + +option java_outer_classname = "CommonProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.tap.v3alpha"; + +// [#protodoc-title: Common tap configuration] + +// Tap configuration. +message TapConfig { + // The match configuration. If the configuration matches the data source being tapped, a tap will + // occur, with the result written to the configured output. + MatchPredicate match_config = 1 [(validate.rules).message.required = true]; + + // The tap output configuration. If a match configuration matches a data source being tapped, + // a tap will occur and the data will be written to the configured output. + OutputConfig output_config = 2 [(validate.rules).message.required = true]; + + // [#not-implemented-hide:] Specify if Tap matching is enabled. The % of requests\connections for + // which the tap matching is enabled. When not enabled, the request\connection will not be + // recorded. + // + // .. note:: + // + // This field defaults to 100/:ref:`HUNDRED + // `. + envoy.api.v3alpha.core.RuntimeFractionalPercent tap_enabled = 3; + + // [#comment:TODO(mattklein123): Rate limiting] +} + +// Tap match configuration. This is a recursive structure which allows complex nested match +// configurations to be built using various logical operators. +message MatchPredicate { + // A set of match configurations used for logical operations. + message MatchSet { + // The list of rules that make up the set. + repeated MatchPredicate rules = 1 [(validate.rules).repeated .min_items = 2]; + } + + oneof rule { + option (validate.required) = true; + + // A set that describes a logical OR. If any member of the set matches, the match configuration + // matches. + MatchSet or_match = 1; + + // A set that describes a logical AND. If all members of the set match, the match configuration + // matches. + MatchSet and_match = 2; + + // A negation match. The match configuration will match if the negated match condition matches. + MatchPredicate not_match = 3; + + // The match configuration will always match. + bool any_match = 4 [(validate.rules).bool.const = true]; + + // HTTP request headers match configuration. + HttpHeadersMatch http_request_headers_match = 5; + + // HTTP request trailers match configuration. + HttpHeadersMatch http_request_trailers_match = 6; + + // HTTP response headers match configuration. + HttpHeadersMatch http_response_headers_match = 7; + + // HTTP response trailers match configuration. + HttpHeadersMatch http_response_trailers_match = 8; + } +} + +// HTTP headers match configuration. +message HttpHeadersMatch { + // HTTP headers to match. + repeated api.v3alpha.route.HeaderMatcher headers = 1; +} + +// Tap output configuration. +message OutputConfig { + // Output sinks for tap data. Currently a single sink is allowed in the list. Once multiple + // sink types are supported this constraint will be relaxed. + repeated OutputSink sinks = 1 [(validate.rules).repeated = {min_items: 1, max_items: 1}]; + + // For buffered tapping, the maximum amount of received body that will be buffered prior to + // truncation. If truncation occurs, the :ref:`truncated + // ` field will be set. If not specified, the + // default is 1KiB. + google.protobuf.UInt32Value max_buffered_rx_bytes = 2; + + // For buffered tapping, the maximum amount of transmitted body that will be buffered prior to + // truncation. If truncation occurs, the :ref:`truncated + // ` field will be set. If not specified, the + // default is 1KiB. + google.protobuf.UInt32Value max_buffered_tx_bytes = 3; + + // Indicates whether taps produce a single buffered message per tap, or multiple streamed + // messages per tap in the emitted :ref:`TraceWrapper + // ` messages. Note that streamed tapping does not + // mean that no buffering takes place. Buffering may be required if data is processed before a + // match can be determined. See the HTTP tap filter :ref:`streaming + // ` documentation for more information. + bool streaming = 4; +} + +// Tap output sink configuration. +message OutputSink { + // Output format. All output is in the form of one or more :ref:`TraceWrapper + // ` messages. This enumeration indicates + // how those messages are written. Note that not all sinks support all output formats. See + // individual sink documentation for more information. + enum Format { + // Each message will be written as JSON. Any :ref:`body ` + // data will be present in the :ref:`as_bytes + // ` field. This means that body data will be + // base64 encoded as per the `proto3 JSON mappings + // `_. + JSON_BODY_AS_BYTES = 0; + + // Each message will be written as JSON. Any :ref:`body ` + // data will be present in the :ref:`as_string + // ` field. This means that body data will be + // string encoded as per the `proto3 JSON mappings + // `_. This format type is + // useful when it is known that that body is human readable (e.g., JSON over HTTP) and the + // user wishes to view it directly without being forced to base64 decode the body. + JSON_BODY_AS_STRING = 1; + + // Binary proto format. Note that binary proto is not self-delimiting. If a sink writes + // multiple binary messages without any length information the data stream will not be + // useful. However, for certain sinks that are self-delimiting (e.g., one message per file) + // this output format makes consumption simpler. + PROTO_BINARY = 2; + + // Messages are written as a sequence tuples, where each tuple is the message length encoded + // as a `protobuf 32-bit varint + // `_ + // followed by the binary message. The messages can be read back using the language specific + // protobuf coded stream implementation to obtain the message length and the message. + PROTO_BINARY_LENGTH_DELIMITED = 3; + + // Text proto format. + PROTO_TEXT = 4; + } + + // Sink output format. + Format format = 1 [(validate.rules).enum.defined_only = true]; + + oneof output_sink_type { + option (validate.required) = true; + + // Tap output will be streamed out the :http:post:`/tap` admin endpoint. + // + // .. attention:: + // + // It is only allowed to specify the streaming admin output sink if the tap is being + // configured from the :http:post:`/tap` admin endpoint. Thus, if an extension has + // been configured to receive tap configuration from some other source (e.g., static + // file, XDS, etc.) configuring the streaming admin output type will fail. + StreamingAdminSink streaming_admin = 2; + + // Tap output will be written to a file per tap sink. + FilePerTapSink file_per_tap = 3; + + // [#not-implemented-hide:] + // GrpcService to stream data to. The format argument must be PROTO_BINARY. + StreamingGrpcSink streaming_grpc = 4; + } +} + +// Streaming admin sink configuration. +message StreamingAdminSink { +} + +// The file per tap sink outputs a discrete file for every tapped stream. +message FilePerTapSink { + // Path prefix. The output file will be of the form _.pb, where is an + // identifier distinguishing the recorded trace for stream instances (the Envoy + // connection ID, HTTP stream ID, etc.). + string path_prefix = 1 [(validate.rules).string.min_bytes = 1]; +} + +// [#not-implemented-hide:] Streaming gRPC sink configuration sends the taps to an external gRPC +// server. +message StreamingGrpcSink { + // Opaque identifier, that will be sent back to the streaming grpc server. + string tap_id = 1; + + // The gRPC server that hosts the Tap Sink Service. + envoy.api.v3alpha.core.GrpcService grpc_service = 2 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/service/tap/v3alpha/tap.proto b/api/envoy/service/tap/v3alpha/tap.proto new file mode 100644 index 000000000000..1e69d421915c --- /dev/null +++ b/api/envoy/service/tap/v3alpha/tap.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/data/tap/v3alpha/wrapper.proto"; + +package envoy.service.tap.v3alpha; + +import "validate/validate.proto"; + +option java_outer_classname = "TapProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.tap.v3alpha"; + +// [#protodoc-title: Tap Sink Service] + +// [#not-implemented-hide:] Stream message for the Tap API. Envoy will open a stream to the server +// and stream taps without ever expecting a response. +message StreamTapsRequest { + message Identifier { + // The node sending taps over the stream. + envoy.api.v3alpha.core.Node node = 1 [(validate.rules).message.required = true]; + // The opaque identifier that was set in the :ref:`output config + // `. + string tap_id = 2; + } + + // Identifier data effectively is a structured metadata. As a performance optimization this will + // only be sent in the first message on the stream. + Identifier identifier = 1; + // The trace id. this can be used to merge together a streaming trace. Note that the trace_id + // is not guaranteed to be spatially or temporally unique. + uint64 trace_id = 2; + // The trace data. + envoy.data.tap.v3alpha.TraceWrapper trace = 3; +} + +// [#not-implemented-hide:] +message StreamTapsResponse { +} + +// [#not-implemented-hide:] A tap service to receive incoming taps. Envoy will call +// StreamTaps to deliver captured taps to the server +service TapSinkService { + + // Envoy will connect and send StreamTapsRequest messages forever. It does not expect any + // response to be sent as nothing would be done in the case of failure. The server should + // disconnect if it expects Envoy to reconnect. + rpc StreamTaps(stream StreamTapsRequest) returns (StreamTapsResponse) { + } +} \ No newline at end of file diff --git a/api/envoy/service/tap/v3alpha/tapds.proto b/api/envoy/service/tap/v3alpha/tapds.proto new file mode 100644 index 000000000000..542d88ed2285 --- /dev/null +++ b/api/envoy/service/tap/v3alpha/tapds.proto @@ -0,0 +1,44 @@ +syntax = "proto3"; + +import "envoy/api/v3alpha/discovery.proto"; +import "envoy/service/tap/v3alpha/common.proto"; +import "validate/validate.proto"; + +package envoy.service.tap.v3alpha; + +import "google/api/annotations.proto"; + +option java_outer_classname = "TapDsProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.tap.v3alpha"; + +// [#protodoc-title: Tap discovery service] + +// [#not-implemented-hide:] Tap discovery service. +service TapDiscoveryService { + rpc StreamTapConfigs(stream envoy.api.v3alpha.DiscoveryRequest) + returns (stream envoy.api.v3alpha.DiscoveryResponse) { + } + + rpc DeltaTapConfigs(stream envoy.api.v3alpha.DeltaDiscoveryRequest) + returns (stream envoy.api.v3alpha.DeltaDiscoveryResponse) { + } + + rpc FetchTapConfigs(envoy.api.v3alpha.DiscoveryRequest) + returns (envoy.api.v3alpha.DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:tap_configs" + body: "*" + }; + } +} + +// [#not-implemented-hide:] A tap resource is essentially a tap configuration with a name +// The filter TapDS config references this name. +message TapResource { + // The name of the tap configuration. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Tap config to apply + TapConfig config = 2; +} \ No newline at end of file diff --git a/api/envoy/service/trace/v3alpha/BUILD b/api/envoy/service/trace/v3alpha/BUILD new file mode 100644 index 000000000000..815d0c4c93cc --- /dev/null +++ b/api/envoy/service/trace/v3alpha/BUILD @@ -0,0 +1,23 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "trace_service", + srcs = ["trace_service.proto"], + has_services = 1, + require_py = 0, + deps = [ + "//envoy/api/v3alpha/core:base", + "@opencensus_proto//opencensus/proto/trace/v1:trace_proto", + ], +) + +api_go_grpc_library( + name = "trace_service", + proto = ":trace_service", + deps = [ + "//envoy/api/v3alpha/core:base_go_proto", + "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_go", + ], +) diff --git a/api/envoy/service/trace/v3alpha/trace_service.proto b/api/envoy/service/trace/v3alpha/trace_service.proto new file mode 100644 index 000000000000..521139a084e5 --- /dev/null +++ b/api/envoy/service/trace/v3alpha/trace_service.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; + +// [#proto-status: draft] + +package envoy.service.trace.v3alpha; + +option java_outer_classname = "TraceServiceProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.service.trace.v3alpha"; +option go_package = "v2"; +option java_generic_services = true; + +import "envoy/api/v3alpha/core/base.proto"; +import "opencensus/proto/trace/v1/trace.proto"; + +import "google/api/annotations.proto"; + +import "validate/validate.proto"; + +// Service for streaming traces to server that consumes the trace data. It +// uses OpenCensus data model as a standard to represent trace information. +service TraceService { + // Envoy will connect and send StreamTracesMessage messages forever. It does + // not expect any response to be sent as nothing would be done in the case + // of failure. + rpc StreamTraces(stream StreamTracesMessage) returns (StreamTracesResponse) { + } +} + +message StreamTracesResponse { +} + +message StreamTracesMessage { + message Identifier { + // The node sending the access log messages over the stream. + envoy.api.v3alpha.core.Node node = 1 [(validate.rules).message.required = true]; + } + + // Identifier data effectively is a structured metadata. + // As a performance optimization this will only be sent in the first message + // on the stream. + Identifier identifier = 1; + + // A list of Span entries + repeated opencensus.proto.trace.v1.Span spans = 2; +} diff --git a/api/migration/v3alpha.sh b/api/migration/v3alpha.sh new file mode 100755 index 000000000000..2b081dabaaaf --- /dev/null +++ b/api/migration/v3alpha.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e + +./tools/api/clone.sh v2 v3alpha +./tools/check_format.py fix diff --git a/tools/api/clone.sh b/tools/api/clone.sh new file mode 100755 index 000000000000..a9cdb63ffd91 --- /dev/null +++ b/tools/api/clone.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Simple script to clone vM to vN API, performing sed-style heuristic fixup of +# build paths and package references. +# +# Usage: +# +# ./tools/api/clone.sh v2 v3 + +set -e + +declare -r OLD_VERSION="$1" +declare -r NEW_VERSION="$2" + +# For vM -> vN, replace //$1*/vMalpha with //$1*/vN in BUILD file $2 +# For vM -> vN, replace //$1*/vM with //$1*/vN in BUILD file $2 +function replace_build() { + sed -i -e "s#\(//$1[^\S]*\)/${OLD_VERSION}alpha#\1/${NEW_VERSION}#g" "$2" + sed -i -e "s#\(//$1[^\S]*\)/${OLD_VERSION}#\1/${NEW_VERSION}#g" "$2" +} + +# For vM -> vN, replace $1*[./]vMalpha with $1*[./]vN in .proto file $2 +# For vM -> vN, replace $1*[./]vM with $1*[./]vN in .proto file $2 +function replace_proto() { + sed -i -e "s#\($1\S*[\./]\)${OLD_VERSION}alpha#\1${NEW_VERSION}#g" "$2" + sed -i -e "s#\($1\S*[\./]\)${OLD_VERSION}#\1${NEW_VERSION}#g" "$2" +} + +# We consider both {vM, vMalpha} to deal with the multiple possible combinations +# of {vM, vMalpha} existence for a given package. +for p in $(find api/ -name "${OLD_VERSION}" -o -name "${OLD_VERSION}alpha") +do + declare PACKAGE_ROOT="$(dirname "$p")" + declare OLD_VERSION_ROOT="${PACKAGE_ROOT}/${OLD_VERSION}" + declare NEW_VERSION_ROOT="${PACKAGE_ROOT}/${NEW_VERSION}" + + # Deal with the situation where there is both vM and vMalpha, we only want vM. + if [[ -a "${OLD_VERSION_ROOT}" && "$p" != "${OLD_VERSION_ROOT}" ]] + then + continue + fi + + # Copy BUILD and .protos across + rsync -a "${p}"/ "${NEW_VERSION_ROOT}/" + + # Update BUILD files with vM -> vN + for b in $(find "${NEW_VERSION_ROOT}" -name BUILD) + do + replace_build envoy "$b" + done + + # Update .proto files with vM -> vN + for f in $(find "${NEW_VERSION_ROOT}" -name "*.proto") + do + replace_proto envoy "$f" + replace_proto api "$f" + replace_proto service "$f" + replace_proto common "$f" + replace_proto config "$f" + replace_proto filter "$f" + done +done From 0ef31372695fc27b5c5dee10edc2563b94e665e0 Mon Sep 17 00:00:00 2001 From: zyfjeff Date: Wed, 4 Sep 2019 04:50:52 +0800 Subject: [PATCH 493/542] dubbo: Fix heartbeat packet parsing error (#8103) Description: The heartbeat packet may carry data, and it is treated as null data when processing the heartbeat packet, causing some data to remain in the buffer. Risk Level: low Testing: Existing unit test Docs Changes: N/A Release Notes: N/A Fixes #7970 Signed-off-by: tianqian.zyf --- source/extensions/filters/network/dubbo_proxy/decoder.cc | 8 ++++++-- .../filters/network/dubbo_proxy/conn_manager_test.cc | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/dubbo_proxy/decoder.cc b/source/extensions/filters/network/dubbo_proxy/decoder.cc index f7e9cfc84a24..3715acf865d5 100644 --- a/source/extensions/filters/network/dubbo_proxy/decoder.cc +++ b/source/extensions/filters/network/dubbo_proxy/decoder.cc @@ -18,12 +18,16 @@ DecoderStateMachine::onDecodeStreamHeader(Buffer::Instance& buffer) { return {ProtocolState::WaitForData}; } - // The heartbeat message has no body. auto context = ret.first; if (metadata->message_type() == MessageType::HeartbeatRequest || metadata->message_type() == MessageType::HeartbeatResponse) { + if (buffer.length() < (context->header_size() + context->body_size())) { + ENVOY_LOG(debug, "dubbo decoder: need more data for {} protocol heartbeat", protocol_.name()); + return {ProtocolState::WaitForData}; + } + ENVOY_LOG(debug, "dubbo decoder: this is the {} heartbeat message", protocol_.name()); - buffer.drain(context->header_size()); + buffer.drain(context->header_size() + context->body_size()); delegate_.onHeartbeat(metadata); return {ProtocolState::Done}; } diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index 536e167b0209..008fb1b9fc5a 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -306,7 +306,8 @@ class ConnectionManagerTest : public testing::Test { buffer.add(static_cast(&msg_type), 1); buffer.add(std::string{0x14}); addInt64(buffer, request_id); // Request Id - buffer.add(std::string{0x00, 0x00, 0x00, 0x00}); // Body Length + buffer.add(std::string{0x00, 0x00, 0x00, 0x01}); // Body Length + buffer.add(std::string{0x01}); // Body } NiceMock factory_context_; @@ -377,6 +378,7 @@ TEST_F(ConnectionManagerTest, OnDataHandlesHeartbeatEvent) { })); EXPECT_EQ(conn_manager_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(0U, buffer_.length()); filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); EXPECT_EQ(0U, store_.counter("test.request").value()); From cf55298b2db5155f2a12e1e71b840f8a300b205e Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 3 Sep 2019 17:08:40 -0400 Subject: [PATCH 494/542] stats: Shared cluster isolated stats (#8118) * shared the main symbol-table with the isolated stats used for cluster info. Signed-off-by: Joshua Marantz --- source/common/upstream/upstream_impl.cc | 2 +- source/common/upstream/upstream_impl.h | 4 +++- test/integration/stats_integration_test.cc | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index c50aa84f85d4..18aac17131d9 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -598,7 +598,7 @@ ClusterInfoImpl::ClusterInfoImpl( per_connection_buffer_limit_bytes_( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, per_connection_buffer_limit_bytes, 1024 * 1024)), transport_socket_factory_(std::move(socket_factory)), stats_scope_(std::move(stats_scope)), - stats_(generateStats(*stats_scope_)), + stats_(generateStats(*stats_scope_)), load_report_stats_store_(stats_scope_->symbolTable()), load_report_stats_(generateLoadReportStats(load_report_stats_store_)), features_(parseFeatures(config)), http2_settings_(Http::Utility::parseHttp2Settings(config.http2_protocol_options())), diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 07469e36d0f3..3f0f34d7545b 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -75,7 +75,9 @@ class HostDescriptionImpl : virtual public HostDescription { .bool_value()), metadata_(std::make_shared(metadata)), locality_(locality), locality_zone_stat_name_(locality.zone(), cluster->statsScope().symbolTable()), - stats_{ALL_HOST_STATS(POOL_COUNTER(stats_store_), POOL_GAUGE(stats_store_))}, + stats_store_(cluster->statsScope().symbolTable()), stats_{ALL_HOST_STATS( + POOL_COUNTER(stats_store_), + POOL_GAUGE(stats_store_))}, priority_(priority) { if (health_check_config.port_value() != 0 && dest_address->type() != Network::Address::Type::Ip) { diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 746baf523926..4cfacefe2d4a 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -225,6 +225,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // 2019/07/15 7555 42806 43000 static link libstdc++ in tests // 2019/07/24 7503 43030 44000 add upstream filters to clusters // 2019/08/13 7877 42838 44000 skip EdfScheduler creation if all host weights equal + // 2019/09/02 8118 42830 43000 Share symbol-tables in cluster/host stats. // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -234,7 +235,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 42838); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 42830); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 44000); } @@ -260,6 +261,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // exact upper-bound // ---------- ----- ----------------- ----- // 2019/08/09 7882 35489 36000 Initial version + // 2019/09/02 8118 34585 34500 Share symbol-tables in cluster/host stats. // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -269,7 +271,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 35489); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 34585); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 36000); } From 39a4423a274278f9b9e8399c901734fc2e87a0aa Mon Sep 17 00:00:00 2001 From: htuch Date: Tue, 3 Sep 2019 17:23:07 -0400 Subject: [PATCH 495/542] protodoc: upgrade to Python 3. (#8129) Risk level: Low Testing: Rebuilt docs, manual inspection of some example generated files. Signed-off-by: Harvey Tuch --- docs/build.sh | 6 +++--- tools/protodoc/BUILD | 2 +- tools/protodoc/protodoc.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/build.sh b/docs/build.sh index 1b8cac8b06fa..b959b159c40b 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -44,11 +44,11 @@ rm -rf "${GENERATED_RST_DIR}" mkdir -p "${GENERATED_RST_DIR}" source_venv "$BUILD_DIR" -pip install -r "${SCRIPT_DIR}"/requirements.txt +pip3 install -r "${SCRIPT_DIR}"/requirements.txt bazel build ${BAZEL_BUILD_OPTIONS} @envoy_api//docs:protos --aspects \ - tools/protodoc/protodoc.bzl%proto_doc_aspect --output_groups=rst --action_env=CPROFILE_ENABLED \ - --action_env=ENVOY_BLOB_SHA --spawn_strategy=standalone --host_force_python=PY2 + tools/protodoc/protodoc.bzl%proto_doc_aspect --output_groups=rst --action_env=CPROFILE_ENABLED=1 \ + --action_env=ENVOY_BLOB_SHA --spawn_strategy=standalone --host_force_python=PY3 # These are the protos we want to put in docs, this list will grow. # TODO(htuch): Factor this out of this script. diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index 8e428b5d24fd..b4b3c3f39acb 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -3,7 +3,7 @@ licenses(["notice"]) # Apache 2 py_binary( name = "protodoc", srcs = ["protodoc.py"], - python_version = "PY2", + python_version = "PY3", visibility = ["//visibility:public"], deps = [ "@com_envoyproxy_protoc_gen_validate//validate:validate_py", diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index bf6a869f0797..562d6de2ac5f 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -6,9 +6,9 @@ from collections import defaultdict import cProfile import functools +import io import os import pstats -import StringIO import re import sys @@ -709,7 +709,7 @@ def GenerateRst(proto_file): def Main(): # http://www.expobrain.net/2015/09/13/create-a-plugin-for-google-protocol-buffer/ request = plugin_pb2.CodeGeneratorRequest() - request.ParseFromString(sys.stdin.read()) + request.ParseFromString(sys.stdin.buffer.read()) response = plugin_pb2.CodeGeneratorResponse() cprofile_enabled = os.getenv('CPROFILE_ENABLED') @@ -724,14 +724,14 @@ def Main(): f.content = GenerateRst(proto_file) if cprofile_enabled: pr.disable() - stats_stream = StringIO.StringIO() + stats_stream = io.StringIO() ps = pstats.Stats(pr, stream=stats_stream).sort_stats(os.getenv('CPROFILE_SORTBY', 'cumulative')) stats_file = response.file.add() stats_file.name = proto_file.name + '.rst.profile' ps.print_stats() stats_file.content = stats_stream.getvalue() - sys.stdout.write(response.SerializeToString()) + sys.stdout.buffer.write(response.SerializeToString()) if __name__ == '__main__': From b06e2b5c473b4b88e23ed66abd501e29f6bddbee Mon Sep 17 00:00:00 2001 From: htuch Date: Tue, 3 Sep 2019 19:25:38 -0400 Subject: [PATCH 496/542] protodoc: single source-of-truth for doc protos. (#8132) This avoids having to list new docs protos in both docs/build.sh and api/docs/BUILD. This technical debt cleanup is helpful in v3 proto work to simplify collecting proto artifacts from a Bazel aspect. Risk level: Low Testing: docs/build.sh, visual inspection of docs. Signed-off-by: Harvey Tuch --- api/CONTRIBUTING.md | 2 +- api/docs/BUILD | 5 +- docs/build.sh | 125 +++---------------------------------- tools/protodoc/protodoc.py | 12 +++- 4 files changed, 22 insertions(+), 122 deletions(-) diff --git a/api/CONTRIBUTING.md b/api/CONTRIBUTING.md index 02f8536906b5..d07e1820a0ab 100644 --- a/api/CONTRIBUTING.md +++ b/api/CONTRIBUTING.md @@ -8,7 +8,7 @@ API changes are regular PRs in https://github.com/envoyproxy/envoy for the API/c changes. They may be as part of a larger implementation PR. Please follow the standard Bazel and CI process for validating build/test sanity of `api/` before submitting a PR. -*Note: New .proto files should be also included to [build.sh](https://github.com/envoyproxy/envoy/blob/master/docs/build.sh) and +*Note: New .proto files should be added to [BUILD](https://github.com/envoyproxy/envoy/blob/master/api/docs/BUILD) in order to get the RSTs generated.* ## Documentation changes diff --git a/api/docs/BUILD b/api/docs/BUILD index 11ef3876b218..5784ecc2d2af 100644 --- a/api/docs/BUILD +++ b/api/docs/BUILD @@ -7,8 +7,7 @@ package_group( ], ) -# TODO(htuch): Grow this to cover everything we want to generate docs for, so we can just invoke -# bazel build //docs:protos --aspects tools/protodoc/protodoc.bzl%proto_doc_aspect --output_groups=rst +# This is where you add protos that will participate in docs RST generation. proto_library( name = "protos", deps = [ @@ -94,8 +93,6 @@ proto_library( "//envoy/service/auth/v2:external_auth", "//envoy/service/discovery/v2:ads", "//envoy/service/discovery/v2:rtds", - "//envoy/service/load_stats/v2:lrs", - "//envoy/service/metrics/v2:metrics_service", "//envoy/service/ratelimit/v2:rls", "//envoy/service/tap/v2alpha:common", "//envoy/type:percent", diff --git a/docs/build.sh b/docs/build.sh index b959b159c40b..b75ebac03e65 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -50,126 +50,19 @@ bazel build ${BAZEL_BUILD_OPTIONS} @envoy_api//docs:protos --aspects \ tools/protodoc/protodoc.bzl%proto_doc_aspect --output_groups=rst --action_env=CPROFILE_ENABLED=1 \ --action_env=ENVOY_BLOB_SHA --spawn_strategy=standalone --host_force_python=PY3 -# These are the protos we want to put in docs, this list will grow. -# TODO(htuch): Factor this out of this script. -PROTO_RST=" - /envoy/admin/v2alpha/certs/envoy/admin/v2alpha/certs.proto.rst - /envoy/admin/v2alpha/clusters/envoy/admin/v2alpha/clusters.proto.rst - /envoy/admin/v2alpha/config_dump/envoy/admin/v2alpha/config_dump.proto.rst - /envoy/admin/v2alpha/listeners/envoy/admin/v2alpha/listeners.proto.rst - /envoy/admin/v2alpha/memory/envoy/admin/v2alpha/memory.proto.rst - /envoy/admin/v2alpha/clusters/envoy/admin/v2alpha/metrics.proto.rst - /envoy/admin/v2alpha/mutex_stats/envoy/admin/v2alpha/mutex_stats.proto.rst - /envoy/admin/v2alpha/server_info/envoy/admin/v2alpha/server_info.proto.rst - /envoy/admin/v2alpha/tap/envoy/admin/v2alpha/tap.proto.rst - /envoy/api/v2/core/address/envoy/api/v2/core/address.proto.rst - /envoy/api/v2/core/base/envoy/api/v2/core/base.proto.rst - /envoy/api/v2/core/http_uri/envoy/api/v2/core/http_uri.proto.rst - /envoy/api/v2/core/config_source/envoy/api/v2/core/config_source.proto.rst - /envoy/api/v2/core/grpc_service/envoy/api/v2/core/grpc_service.proto.rst - /envoy/api/v2/core/health_check/envoy/api/v2/core/health_check.proto.rst - /envoy/api/v2/core/protocol/envoy/api/v2/core/protocol.proto.rst - /envoy/api/v2/discovery/envoy/api/v2/discovery.proto.rst - /envoy/api/v2/auth/cert/envoy/api/v2/auth/cert.proto.rst - /envoy/api/v2/eds/envoy/api/v2/eds.proto.rst - /envoy/api/v2/endpoint/endpoint/envoy/api/v2/endpoint/endpoint.proto.rst - /envoy/api/v2/cds/envoy/api/v2/cds.proto.rst - /envoy/api/v2/cluster/outlier_detection/envoy/api/v2/cluster/outlier_detection.proto.rst - /envoy/api/v2/cluster/circuit_breaker/envoy/api/v2/cluster/circuit_breaker.proto.rst - /envoy/api/v2/cluster/filter/envoy/api/v2/cluster/filter.proto.rst - /envoy/api/v2/rds/envoy/api/v2/rds.proto.rst - /envoy/api/v2/route/route/envoy/api/v2/route/route.proto.rst - /envoy/api/v2/srds/envoy/api/v2/srds.proto.rst - /envoy/api/v2/lds/envoy/api/v2/lds.proto.rst - /envoy/api/v2/listener/listener/envoy/api/v2/listener/listener.proto.rst - /envoy/api/v2/listener/udp_listener_config/envoy/api/v2/listener/udp_listener_config.proto.rst - /envoy/api/v2/ratelimit/ratelimit/envoy/api/v2/ratelimit/ratelimit.proto.rst - /envoy/config/accesslog/v2/als/envoy/config/accesslog/v2/als.proto.rst - /envoy/config/accesslog/v2/file/envoy/config/accesslog/v2/file.proto.rst - /envoy/config/bootstrap/v2/bootstrap/envoy/config/bootstrap/v2/bootstrap.proto.rst - /envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto.rst - /envoy/config/cluster/redis/redis_cluster/envoy/config/cluster/redis/redis_cluster.proto.rst - /envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto.rst - /envoy/config/common/tap/v2alpha/common/envoy/config/common/tap/v2alpha/common.proto.rst - /envoy/config/ratelimit/v2/rls/envoy/config/ratelimit/v2/rls.proto.rst - /envoy/config/metrics/v2/metrics_service/envoy/config/metrics/v2/metrics_service.proto.rst - /envoy/config/metrics/v2/stats/envoy/config/metrics/v2/stats.proto.rst - /envoy/config/trace/v2/trace/envoy/config/trace/v2/trace.proto.rst - /envoy/config/filter/accesslog/v2/accesslog/envoy/config/filter/accesslog/v2/accesslog.proto.rst - /envoy/config/filter/fault/v2/fault/envoy/config/filter/fault/v2/fault.proto.rst - /envoy/config/filter/http/buffer/v2/buffer/envoy/config/filter/http/buffer/v2/buffer.proto.rst - /envoy/config/filter/http/csrf/v2/csrf/envoy/config/filter/http/csrf/v2/csrf.proto.rst - /envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto.rst - /envoy/config/filter/http/ext_authz/v2/ext_authz/envoy/config/filter/http/ext_authz/v2/ext_authz.proto.rst - /envoy/config/filter/http/fault/v2/fault/envoy/config/filter/http/fault/v2/fault.proto.rst - /envoy/config/filter/http/gzip/v2/gzip/envoy/config/filter/http/gzip/v2/gzip.proto.rst - /envoy/config/filter/http/health_check/v2/health_check/envoy/config/filter/http/health_check/v2/health_check.proto.rst - /envoy/config/filter/http/header_to_metadata/v2/header_to_metadata/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto.rst - /envoy/config/filter/http/ip_tagging/v2/ip_tagging/envoy/config/filter/http/ip_tagging/v2/ip_tagging.proto.rst - /envoy/config/filter/http/jwt_authn/v2alpha/jwt_authn/envoy/config/filter/http/jwt_authn/v2alpha/config.proto.rst - /envoy/config/filter/http/lua/v2/lua/envoy/config/filter/http/lua/v2/lua.proto.rst - /envoy/config/filter/http/original_src/v2alpha1/original_src/envoy/config/filter/http/original_src/v2alpha1/original_src.proto.rst - /envoy/config/filter/http/rate_limit/v2/rate_limit/envoy/config/filter/http/rate_limit/v2/rate_limit.proto.rst - /envoy/config/filter/http/rbac/v2/rbac/envoy/config/filter/http/rbac/v2/rbac.proto.rst - /envoy/config/filter/http/router/v2/router/envoy/config/filter/http/router/v2/router.proto.rst - /envoy/config/filter/http/squash/v2/squash/envoy/config/filter/http/squash/v2/squash.proto.rst - /envoy/config/filter/http/tap/v2alpha/tap/envoy/config/filter/http/tap/v2alpha/tap.proto.rst - /envoy/config/filter/http/transcoder/v2/transcoder/envoy/config/filter/http/transcoder/v2/transcoder.proto.rst - /envoy/config/filter/listener/original_src/v2alpha1/original_src/envoy/config/filter/listener/original_src/v2alpha1/original_src.proto.rst - /envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto.rst - /envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto.rst - /envoy/config/filter/dubbo/router/v2alpha1/router/envoy/config/filter/dubbo/router/v2alpha1/router.proto.rst - /envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto.rst - /envoy/config/filter/network/ext_authz/v2/ext_authz/envoy/config/filter/network/ext_authz/v2/ext_authz.proto.rst - /envoy/config/filter/network/http_connection_manager/v2/http_connection_manager/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto.rst - /envoy/config/filter/network/mongo_proxy/v2/mongo_proxy/envoy/config/filter/network/mongo_proxy/v2/mongo_proxy.proto.rst - /envoy/config/filter/network/rate_limit/v2/rate_limit/envoy/config/filter/network/rate_limit/v2/rate_limit.proto.rst - /envoy/config/filter/network/rbac/v2/rbac/envoy/config/filter/network/rbac/v2/rbac.proto.rst - /envoy/config/filter/network/redis_proxy/v2/redis_proxy/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto.rst - /envoy/config/filter/network/tcp_proxy/v2/tcp_proxy/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto.rst - /envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto.rst - /envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto.rst - /envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto.rst - /envoy/config/filter/thrift/router/v2alpha1/router/envoy/config/filter/thrift/router/v2alpha1/router.proto.rst - /envoy/config/grpc_credential/v2alpha/aws_iam/envoy/config/grpc_credential/v2alpha/aws_iam.proto.rst - /envoy/config/health_checker/redis/v2/redis/envoy/config/health_checker/redis/v2/redis.proto.rst - /envoy/config/overload/v2alpha/overload/envoy/config/overload/v2alpha/overload.proto.rst - /envoy/config/rbac/v2/rbac/envoy/config/rbac/v2/rbac.proto.rst - /envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto.rst - /envoy/config/resource_monitor/injected_resource/v2alpha/injected_resource/envoy/config/resource_monitor/injected_resource/v2alpha/injected_resource.proto.rst - /envoy/config/transport_socket/tap/v2alpha/tap/envoy/config/transport_socket/tap/v2alpha/tap.proto.rst - /envoy/data/accesslog/v2/accesslog/envoy/data/accesslog/v2/accesslog.proto.rst - /envoy/data/core/v2alpha/health_check_event/envoy/data/core/v2alpha/health_check_event.proto.rst - /envoy/data/tap/v2alpha/common/envoy/data/tap/v2alpha/common.proto.rst - /envoy/data/tap/v2alpha/transport/envoy/data/tap/v2alpha/transport.proto.rst - /envoy/data/tap/v2alpha/http/envoy/data/tap/v2alpha/http.proto.rst - /envoy/data/tap/v2alpha/wrapper/envoy/data/tap/v2alpha/wrapper.proto.rst - /envoy/data/cluster/v2alpha/outlier_detection_event/envoy/data/cluster/v2alpha/outlier_detection_event.proto.rst - /envoy/service/accesslog/v2/als/envoy/service/accesslog/v2/als.proto.rst - /envoy/service/auth/v2/external_auth/envoy/service/auth/v2/attribute_context.proto.rst - /envoy/service/auth/v2/external_auth/envoy/service/auth/v2/external_auth.proto.rst - /envoy/service/discovery/v2/rtds/envoy/service/discovery/v2/rtds.proto.rst - /envoy/service/ratelimit/v2/rls/envoy/service/ratelimit/v2/rls.proto.rst - /envoy/service/tap/v2alpha/common/envoy/service/tap/v2alpha/common.proto.rst - /envoy/type/http_status/envoy/type/http_status.proto.rst - /envoy/type/percent/envoy/type/percent.proto.rst - /envoy/type/range/envoy/type/range.proto.rst - /envoy/type/matcher/metadata/envoy/type/matcher/metadata.proto.rst - /envoy/type/matcher/value/envoy/type/matcher/value.proto.rst - /envoy/type/matcher/number/envoy/type/matcher/number.proto.rst - /envoy/type/matcher/regex/envoy/type/matcher/regex.proto.rst - /envoy/type/matcher/string/envoy/type/matcher/string.proto.rst -" - -# Dump all the generated RST so they can be added to PROTO_RST easily. -find -L bazel-bin/external/envoy_api -name "*.proto.rst" +declare -r DOC_PROTOS=$(bazel query "deps(@envoy_api//docs:protos)" | grep "^@envoy_api.*proto$") # Only copy in the protos we care about and know how to deal with in protodoc. -for p in $PROTO_RST +for p in ${DOC_PROTOS} do - DEST="${GENERATED_RST_DIR}/api-v2/$(sed -e 's#/envoy\/.*/envoy/##' <<< "$p")" + declare PROTO_TARGET=$(bazel query "kind(proto_library, same_pkg_direct_rdeps($p))") + declare PROTO_TARGET_WITHOUT_PREFIX="${PROTO_TARGET#@envoy_api//}" + declare PROTO_TARGET_CANONICAL="${PROTO_TARGET_WITHOUT_PREFIX/:/\/}" + declare PROTO_FILE_WITHOUT_PREFIX="${p#@envoy_api//}" + declare PROTO_FILE_CANONICAL="${PROTO_FILE_WITHOUT_PREFIX/:/\/}" + declare DEST="${GENERATED_RST_DIR}/api-v2/${PROTO_FILE_CANONICAL#envoy/}".rst mkdir -p "$(dirname "${DEST}")" - cp -f bazel-bin/external/envoy_api/"${p}" "$(dirname "${DEST}")" + cp -f bazel-bin/external/envoy_api/"${PROTO_TARGET_CANONICAL}/${PROTO_FILE_CANONICAL}.rst" "$(dirname "${DEST}")" [ -n "${CPROFILE_ENABLED}" ] && cp -f bazel-bin/"${p}".profile "$(dirname "${DEST}")" done diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 562d6de2ac5f..83bf0946a961 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -713,7 +713,17 @@ def Main(): response = plugin_pb2.CodeGeneratorResponse() cprofile_enabled = os.getenv('CPROFILE_ENABLED') - for proto_file in request.proto_file: + # We use file_to_generate rather than proto_file here since we are invoked + # inside a Bazel aspect, each node in the DAG will be visited once by the + # aspect and we only want to generate docs for the current node. + for file_to_generate in request.file_to_generate: + # Find the FileDescriptorProto for the file we actually are generating. + proto_file = None + for pf in request.proto_file: + if pf.name == file_to_generate: + proto_file = pf + break + assert (proto_file is not None) f = response.file.add() f.name = proto_file.name + '.rst' if cprofile_enabled: From d504fde0ffd97017d1ddff8caa9a3b46bba9ae48 Mon Sep 17 00:00:00 2001 From: Kuat Date: Tue, 3 Sep 2019 17:08:45 -1000 Subject: [PATCH 497/542] api: organize go_proto_libraries (#8003) Fixes #7982 Defines a package level proto library and its associated internal go_proto_library. Deletes all existing api_go_proto_library, api_go_grpc_library, and go_package annotations in protos (they are not required and pollute the sources). I deliberately avoided touching anything under udpa since it's being moved to another repository. Risk Level: low Testing: build completes Signed-off-by: Kuat Yessenov --- api/bazel/BUILD | 12 ++ api/bazel/api_build_system.bzl | 115 ++++++++++-------- api/bazel/repositories.bzl | 7 +- api/envoy/admin/v2alpha/BUILD | 13 +- api/envoy/admin/v3alpha/BUILD | 13 +- api/envoy/api/v2/BUILD | 88 +++----------- api/envoy/api/v2/auth/BUILD | 18 ++- api/envoy/api/v2/auth/cert.proto | 1 - api/envoy/api/v2/cluster/BUILD | 27 ++-- .../api/v2/cluster/circuit_breaker.proto | 1 - api/envoy/api/v2/core/BUILD | 58 ++------- api/envoy/api/v2/core/base.proto | 1 - api/envoy/api/v2/discovery.proto | 1 - api/envoy/api/v2/endpoint/BUILD | 31 ++--- api/envoy/api/v2/endpoint/endpoint.proto | 1 - api/envoy/api/v2/listener/BUILD | 27 ++-- api/envoy/api/v2/listener/listener.proto | 1 - .../api/v2/listener/udp_listener_config.proto | 1 - api/envoy/api/v2/ratelimit/BUILD | 9 +- api/envoy/api/v2/ratelimit/ratelimit.proto | 1 - api/envoy/api/v2/route/BUILD | 22 ++-- api/envoy/api/v2/route/route.proto | 1 - api/envoy/api/v3alpha/BUILD | 88 +++----------- api/envoy/api/v3alpha/auth/BUILD | 18 ++- api/envoy/api/v3alpha/auth/cert.proto | 1 - api/envoy/api/v3alpha/cluster/BUILD | 27 ++-- .../api/v3alpha/cluster/circuit_breaker.proto | 1 - api/envoy/api/v3alpha/core/BUILD | 58 ++------- api/envoy/api/v3alpha/core/base.proto | 1 - api/envoy/api/v3alpha/discovery.proto | 1 - api/envoy/api/v3alpha/endpoint/BUILD | 31 ++--- api/envoy/api/v3alpha/endpoint/endpoint.proto | 1 - api/envoy/api/v3alpha/listener/BUILD | 27 ++-- api/envoy/api/v3alpha/listener/listener.proto | 1 - .../listener/udp_listener_config.proto | 1 - api/envoy/api/v3alpha/ratelimit/BUILD | 9 +- .../api/v3alpha/ratelimit/ratelimit.proto | 1 - api/envoy/api/v3alpha/route/BUILD | 22 ++-- api/envoy/api/v3alpha/route/route.proto | 1 - api/envoy/config/accesslog/v2/BUILD | 12 +- api/envoy/config/accesslog/v2/als.proto | 1 - api/envoy/config/accesslog/v2/file.proto | 1 - api/envoy/config/accesslog/v3alpha/BUILD | 12 +- api/envoy/config/accesslog/v3alpha/als.proto | 1 - api/envoy/config/accesslog/v3alpha/file.proto | 1 - api/envoy/config/bootstrap/v2/BUILD | 32 ++--- api/envoy/config/bootstrap/v2/bootstrap.proto | 1 - api/envoy/config/bootstrap/v3alpha/BUILD | 32 ++--- .../config/bootstrap/v3alpha/bootstrap.proto | 1 - .../dynamic_forward_proxy/v2alpha/BUILD | 6 +- .../v2alpha/cluster.proto | 1 - .../dynamic_forward_proxy/v3alpha/BUILD | 6 +- .../v3alpha/cluster.proto | 1 - api/envoy/config/cluster/redis/BUILD | 4 +- .../config/cluster/redis/redis_cluster.proto | 1 - .../dynamic_forward_proxy/v2alpha/BUILD | 6 +- .../dynamic_forward_proxy/v3alpha/BUILD | 6 +- api/envoy/config/common/tap/v2alpha/BUILD | 9 +- api/envoy/config/common/tap/v3alpha/BUILD | 9 +- api/envoy/config/filter/accesslog/v2/BUILD | 20 ++- .../filter/accesslog/v2/accesslog.proto | 1 - .../config/filter/accesslog/v3alpha/BUILD | 20 ++- .../filter/accesslog/v3alpha/accesslog.proto | 1 - .../config/filter/dubbo/router/v2alpha1/BUILD | 4 +- .../filter/dubbo/router/v2alpha1/router.proto | 1 - api/envoy/config/filter/fault/v2/BUILD | 6 +- api/envoy/config/filter/fault/v2/fault.proto | 1 - api/envoy/config/filter/fault/v3alpha/BUILD | 6 +- .../config/filter/fault/v3alpha/fault.proto | 1 - .../http/adaptive_concurrency/v2alpha/BUILD | 6 +- .../v2alpha/adaptive_concurrency.proto | 1 - .../http/adaptive_concurrency/v3alpha/BUILD | 6 +- .../v3alpha/adaptive_concurrency.proto | 1 - api/envoy/config/filter/http/buffer/v2/BUILD | 4 +- .../config/filter/http/buffer/v2/buffer.proto | 1 - .../config/filter/http/buffer/v3alpha/BUILD | 4 +- .../filter/http/buffer/v3alpha/buffer.proto | 1 - api/envoy/config/filter/http/csrf/v2/BUILD | 9 +- .../config/filter/http/csrf/v2/csrf.proto | 1 - .../config/filter/http/csrf/v3alpha/BUILD | 9 +- .../filter/http/csrf/v3alpha/csrf.proto | 1 - .../http/dynamic_forward_proxy/v2alpha/BUILD | 6 +- .../v2alpha/dynamic_forward_proxy.proto | 1 - .../http/dynamic_forward_proxy/v3alpha/BUILD | 6 +- .../v3alpha/dynamic_forward_proxy.proto | 1 - .../config/filter/http/ext_authz/v2/BUILD | 10 +- .../filter/http/ext_authz/v2/ext_authz.proto | 1 - .../filter/http/ext_authz/v3alpha/BUILD | 10 +- .../http/ext_authz/v3alpha/ext_authz.proto | 1 - api/envoy/config/filter/http/fault/v2/BUILD | 10 +- .../config/filter/http/fault/v2/fault.proto | 1 - .../config/filter/http/fault/v3alpha/BUILD | 10 +- .../filter/http/fault/v3alpha/fault.proto | 1 - .../grpc_http1_reverse_bridge/v2alpha1/BUILD | 4 +- .../v2alpha1/config.proto | 1 - api/envoy/config/filter/http/gzip/v2/BUILD | 4 +- .../config/filter/http/gzip/v2/gzip.proto | 1 - .../config/filter/http/gzip/v3alpha/BUILD | 4 +- .../filter/http/gzip/v3alpha/gzip.proto | 1 - .../filter/http/header_to_metadata/v2/BUILD | 5 +- .../v2/header_to_metadata.proto | 1 - .../http/header_to_metadata/v3alpha/BUILD | 5 +- .../v3alpha/header_to_metadata.proto | 1 - .../config/filter/http/health_check/v2/BUILD | 18 ++- .../http/health_check/v2/health_check.proto | 1 - .../filter/http/health_check/v3alpha/BUILD | 18 ++- .../health_check/v3alpha/health_check.proto | 1 - .../config/filter/http/ip_tagging/v2/BUILD | 6 +- .../http/ip_tagging/v2/ip_tagging.proto | 1 - .../filter/http/ip_tagging/v3alpha/BUILD | 6 +- .../http/ip_tagging/v3alpha/ip_tagging.proto | 1 - .../filter/http/jwt_authn/v2alpha/BUILD | 19 ++- .../filter/http/jwt_authn/v3alpha/BUILD | 19 ++- api/envoy/config/filter/http/lua/v2/BUILD | 4 +- api/envoy/config/filter/http/lua/v2/lua.proto | 1 - .../config/filter/http/lua/v3alpha/BUILD | 4 +- .../config/filter/http/lua/v3alpha/lua.proto | 1 - .../filter/http/original_src/v2alpha1/BUILD | 4 +- .../original_src/v2alpha1/original_src.proto | 2 - .../config/filter/http/rate_limit/v2/BUILD | 6 +- .../http/rate_limit/v2/rate_limit.proto | 1 - .../filter/http/rate_limit/v3alpha/BUILD | 6 +- .../http/rate_limit/v3alpha/rate_limit.proto | 1 - api/envoy/config/filter/http/rbac/v2/BUILD | 6 +- .../config/filter/http/rbac/v2/rbac.proto | 1 - .../config/filter/http/rbac/v3alpha/BUILD | 6 +- .../filter/http/rbac/v3alpha/rbac.proto | 1 - api/envoy/config/filter/http/router/v2/BUILD | 12 +- .../config/filter/http/router/v2/router.proto | 1 - .../config/filter/http/router/v3alpha/BUILD | 12 +- .../filter/http/router/v3alpha/router.proto | 1 - api/envoy/config/filter/http/squash/v2/BUILD | 4 +- .../config/filter/http/squash/v2/squash.proto | 1 - .../config/filter/http/squash/v3alpha/BUILD | 4 +- .../filter/http/squash/v3alpha/squash.proto | 1 - .../config/filter/http/tap/v2alpha/BUILD | 6 +- .../config/filter/http/tap/v3alpha/BUILD | 6 +- .../config/filter/http/transcoder/v2/BUILD | 9 +- .../http/transcoder/v2/transcoder.proto | 1 - .../filter/http/transcoder/v3alpha/BUILD | 9 +- .../http/transcoder/v3alpha/transcoder.proto | 1 - .../listener/original_src/v2alpha1/BUILD | 4 +- .../original_src/v2alpha1/original_src.proto | 2 - .../filter/network/client_ssl_auth/v2/BUILD | 6 +- .../client_ssl_auth/v2/client_ssl_auth.proto | 1 - .../network/client_ssl_auth/v3alpha/BUILD | 6 +- .../v3alpha/client_ssl_auth.proto | 1 - .../filter/network/dubbo_proxy/v2alpha1/BUILD | 11 +- .../dubbo_proxy/v2alpha1/dubbo_proxy.proto | 3 +- .../network/dubbo_proxy/v2alpha1/route.proto | 1 - .../config/filter/network/ext_authz/v2/BUILD | 6 +- .../network/ext_authz/v2/ext_authz.proto | 1 - .../filter/network/ext_authz/v3alpha/BUILD | 6 +- .../network/ext_authz/v3alpha/ext_authz.proto | 1 - .../network/http_connection_manager/v2/BUILD | 25 ++-- .../v2/http_connection_manager.proto | 1 - .../http_connection_manager/v3alpha/BUILD | 25 ++-- .../v3alpha/http_connection_manager.proto | 1 - .../filter/network/mongo_proxy/v2/BUILD | 6 +- .../network/mongo_proxy/v2/mongo_proxy.proto | 1 - .../filter/network/mongo_proxy/v3alpha/BUILD | 6 +- .../mongo_proxy/v3alpha/mongo_proxy.proto | 1 - .../filter/network/mysql_proxy/v1alpha1/BUILD | 4 +- .../mysql_proxy/v1alpha1/mysql_proxy.proto | 1 - .../config/filter/network/rate_limit/v2/BUILD | 9 +- .../network/rate_limit/v2/rate_limit.proto | 1 - .../filter/network/rate_limit/v3alpha/BUILD | 9 +- .../rate_limit/v3alpha/rate_limit.proto | 1 - api/envoy/config/filter/network/rbac/v2/BUILD | 6 +- .../config/filter/network/rbac/v2/rbac.proto | 1 - .../config/filter/network/rbac/v3alpha/BUILD | 6 +- .../filter/network/rbac/v3alpha/rbac.proto | 1 - .../filter/network/redis_proxy/v2/BUILD | 9 +- .../network/redis_proxy/v2/redis_proxy.proto | 1 - .../filter/network/redis_proxy/v3alpha/BUILD | 9 +- .../redis_proxy/v3alpha/redis_proxy.proto | 1 - .../config/filter/network/tcp_proxy/v2/BUILD | 19 ++- .../network/tcp_proxy/v2/tcp_proxy.proto | 1 - .../filter/network/tcp_proxy/v3alpha/BUILD | 19 ++- .../network/tcp_proxy/v3alpha/tcp_proxy.proto | 1 - .../network/thrift_proxy/v2alpha1/BUILD | 9 +- .../network/thrift_proxy/v2alpha1/route.proto | 1 - .../thrift_proxy/v2alpha1/thrift_proxy.proto | 1 - .../network/zookeeper_proxy/v1alpha1/BUILD | 4 +- .../v1alpha1/zookeeper_proxy.proto | 1 - .../filter/thrift/rate_limit/v2alpha1/BUILD | 9 +- .../rate_limit/v2alpha1/rate_limit.proto | 1 - .../filter/thrift/router/v2alpha1/BUILD | 4 +- .../thrift/router/v2alpha1/router.proto | 1 - .../config/grpc_credential/v2alpha/BUILD | 19 +-- .../grpc_credential/v2alpha/aws_iam.proto | 1 - .../v2alpha/file_based_metadata.proto | 1 - .../config/grpc_credential/v3alpha/BUILD | 19 +-- .../grpc_credential/v3alpha/aws_iam.proto | 1 - .../v3alpha/file_based_metadata.proto | 1 - .../config/health_checker/redis/v2/BUILD | 4 +- .../health_checker/redis/v2/redis.proto | 1 - .../config/health_checker/redis/v3alpha/BUILD | 4 +- .../health_checker/redis/v3alpha/redis.proto | 1 - api/envoy/config/metrics/v2/BUILD | 26 ++-- api/envoy/config/metrics/v2/stats.proto | 1 - api/envoy/config/metrics/v3alpha/BUILD | 26 ++-- api/envoy/config/metrics/v3alpha/stats.proto | 1 - api/envoy/config/overload/v2alpha/BUILD | 9 +- .../config/overload/v2alpha/overload.proto | 1 - api/envoy/config/overload/v3alpha/BUILD | 9 +- .../config/overload/v3alpha/overload.proto | 1 - api/envoy/config/ratelimit/v2/BUILD | 14 +-- api/envoy/config/ratelimit/v2/rls.proto | 1 - api/envoy/config/ratelimit/v3alpha/BUILD | 14 +-- api/envoy/config/ratelimit/v3alpha/rls.proto | 1 - api/envoy/config/rbac/v2/BUILD | 23 ++-- api/envoy/config/rbac/v2/rbac.proto | 1 - api/envoy/config/rbac/v3alpha/BUILD | 23 ++-- api/envoy/config/rbac/v3alpha/rbac.proto | 1 - .../resource_monitor/fixed_heap/v2alpha/BUILD | 4 +- .../fixed_heap/v2alpha/fixed_heap.proto | 1 - .../resource_monitor/fixed_heap/v3alpha/BUILD | 4 +- .../fixed_heap/v3alpha/fixed_heap.proto | 1 - .../injected_resource/v2alpha/BUILD | 4 +- .../v2alpha/injected_resource.proto | 1 - .../injected_resource/v3alpha/BUILD | 4 +- .../v3alpha/injected_resource.proto | 1 - .../config/retry/previous_priorities/BUILD | 6 +- api/envoy/config/trace/v2/BUILD | 18 ++- api/envoy/config/trace/v2/trace.proto | 1 - api/envoy/config/trace/v3alpha/BUILD | 18 ++- api/envoy/config/trace/v3alpha/trace.proto | 1 - .../transport_socket/alts/v2alpha/BUILD | 6 +- .../transport_socket/alts/v2alpha/alts.proto | 1 - .../transport_socket/alts/v3alpha/BUILD | 6 +- .../transport_socket/alts/v3alpha/alts.proto | 1 - .../config/transport_socket/tap/v2alpha/BUILD | 9 +- .../transport_socket/tap/v2alpha/tap.proto | 1 - .../config/transport_socket/tap/v3alpha/BUILD | 9 +- .../transport_socket/tap/v3alpha/tap.proto | 1 - api/envoy/data/accesslog/v2/BUILD | 15 +-- api/envoy/data/accesslog/v3alpha/BUILD | 15 +-- api/envoy/data/cluster/v2alpha/BUILD | 4 +- api/envoy/data/cluster/v3alpha/BUILD | 4 +- api/envoy/data/core/v2alpha/BUILD | 6 +- api/envoy/data/core/v3alpha/BUILD | 6 +- api/envoy/data/tap/v2alpha/BUILD | 6 +- api/envoy/data/tap/v2alpha/transport.proto | 1 - api/envoy/data/tap/v3alpha/BUILD | 6 +- api/envoy/data/tap/v3alpha/transport.proto | 1 - api/envoy/service/accesslog/v2/BUILD | 19 ++- api/envoy/service/accesslog/v2/als.proto | 1 - api/envoy/service/accesslog/v3alpha/BUILD | 19 ++- api/envoy/service/accesslog/v3alpha/als.proto | 1 - api/envoy/service/auth/v2/BUILD | 10 +- api/envoy/service/auth/v2/external_auth.proto | 1 - api/envoy/service/auth/v2alpha/BUILD | 9 +- .../service/auth/v2alpha/external_auth.proto | 2 - api/envoy/service/auth/v3alpha/BUILD | 10 +- .../service/auth/v3alpha/external_auth.proto | 1 - api/envoy/service/discovery/v2/BUILD | 45 ++----- api/envoy/service/discovery/v2/ads.proto | 1 - api/envoy/service/discovery/v3alpha/BUILD | 45 ++----- api/envoy/service/discovery/v3alpha/ads.proto | 1 - api/envoy/service/load_stats/v2/BUILD | 19 ++- api/envoy/service/load_stats/v2/lrs.proto | 1 - api/envoy/service/load_stats/v3alpha/BUILD | 19 ++- .../service/load_stats/v3alpha/lrs.proto | 1 - api/envoy/service/metrics/v2/BUILD | 19 ++- .../service/metrics/v2/metrics_service.proto | 1 - api/envoy/service/metrics/v3alpha/BUILD | 19 ++- .../metrics/v3alpha/metrics_service.proto | 1 - api/envoy/service/ratelimit/v2/BUILD | 20 ++- api/envoy/service/ratelimit/v2/rls.proto | 1 - api/envoy/service/ratelimit/v3alpha/BUILD | 20 ++- api/envoy/service/ratelimit/v3alpha/rls.proto | 1 - api/envoy/service/tap/v2alpha/BUILD | 12 +- api/envoy/service/tap/v3alpha/BUILD | 12 +- api/envoy/service/trace/v2/BUILD | 19 ++- .../service/trace/v2/trace_service.proto | 1 - api/envoy/service/trace/v3alpha/BUILD | 19 ++- .../service/trace/v3alpha/trace_service.proto | 1 - api/envoy/type/BUILD | 21 +--- api/envoy/type/matcher/BUILD | 45 +------ api/envoy/type/matcher/metadata.proto | 1 - api/envoy/type/matcher/number.proto | 1 - api/envoy/type/matcher/regex.proto | 1 - api/envoy/type/matcher/string.proto | 1 - api/envoy/type/matcher/value.proto | 1 - api/envoy/type/range.proto | 1 - api/test/build/BUILD | 21 ++-- api/test/build/go_build_test.go | 21 ++-- tools/check_format.py | 15 +++ tools/check_format_test_helper.py | 1 + .../check_format/api/go_package.proto | 5 + 291 files changed, 1057 insertions(+), 1313 deletions(-) create mode 100644 tools/testdata/check_format/api/go_package.proto diff --git a/api/bazel/BUILD b/api/bazel/BUILD index e69de29bb2d1..4b582bb8be3f 100644 --- a/api/bazel/BUILD +++ b/api/bazel/BUILD @@ -0,0 +1,12 @@ +load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler") + +licenses(["notice"]) # Apache 2 + +go_proto_compiler( + name = "pgv_plugin_go", + options = ["lang=go"], + plugin = "@com_envoyproxy_protoc_gen_validate//:protoc-gen-validate", + suffix = ".pb.validate.go", + valid_archive = False, + visibility = ["//visibility:public"], +) diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index 14cc6f89359d..0ad60bf84a32 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -7,9 +7,23 @@ _PY_SUFFIX = "_py" _CC_SUFFIX = "_cc" _CC_EXPORT_SUFFIX = "_export_cc" _GO_PROTO_SUFFIX = "_go_proto" -_GO_GRPC_SUFFIX = "_go_grpc" _GO_IMPORTPATH_PREFIX = "github.com/envoyproxy/data-plane-api/api/" +_COMMON_PROTO_DEPS = [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:descriptor_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + "@com_google_protobuf//:wrappers_proto", + "@com_google_googleapis//google/api:http_proto", + "@com_google_googleapis//google/api:annotations_proto", + "@com_google_googleapis//google/rpc:status_proto", + "@com_github_gogo_protobuf//:gogo_proto", + "@com_envoyproxy_protoc_gen_validate//validate:validate_proto", +] + def _Suffix(d, suffix): return d + suffix @@ -61,41 +75,6 @@ def py_proto_library(name, deps = []): visibility = ["//visibility:public"], ) -def api_go_proto_library(name, proto, deps = []): - go_proto_library( - name = _Suffix(name, _GO_PROTO_SUFFIX), - importpath = _Suffix(_GO_IMPORTPATH_PREFIX, name), - proto = proto, - visibility = ["//visibility:public"], - deps = deps + [ - "@com_github_gogo_protobuf//:gogo_proto_go", - "@io_bazel_rules_go//proto/wkt:any_go_proto", - "@io_bazel_rules_go//proto/wkt:duration_go_proto", - "@io_bazel_rules_go//proto/wkt:struct_go_proto", - "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", - "@io_bazel_rules_go//proto/wkt:wrappers_go_proto", - "@com_envoyproxy_protoc_gen_validate//validate:go_default_library", - "@com_google_googleapis//google/rpc:status_go_proto", - ], - ) - -def api_go_grpc_library(name, proto, deps = []): - go_grpc_library( - name = _Suffix(name, _GO_GRPC_SUFFIX), - importpath = _Suffix(_GO_IMPORTPATH_PREFIX, name), - proto = proto, - visibility = ["//visibility:public"], - deps = deps + [ - "@com_github_gogo_protobuf//:gogo_proto_go", - "@io_bazel_rules_go//proto/wkt:any_go_proto", - "@io_bazel_rules_go//proto/wkt:duration_go_proto", - "@io_bazel_rules_go//proto/wkt:struct_go_proto", - "@io_bazel_rules_go//proto/wkt:wrappers_go_proto", - "@com_envoyproxy_protoc_gen_validate//validate:go_default_library", - "@com_google_googleapis//google/api:annotations_go_proto", - ], - ) - # This is api_proto_library plus some logic internal to //envoy/api. def api_proto_library_internal(visibility = ["//visibility:private"], **kwargs): # //envoy/docs/build.sh needs visibility in order to generate documents. @@ -108,8 +87,6 @@ def api_proto_library_internal(visibility = ["//visibility:private"], **kwargs): # TODO(htuch): has_services is currently ignored but will in future support # gRPC stub generation. -# TODO(htuch): Automatically generate go_proto_library and go_grpc_library -# from api_proto_library. def api_proto_library( name, visibility = ["//visibility:private"], @@ -124,20 +101,7 @@ def api_proto_library( native.proto_library( name = name, srcs = srcs, - deps = deps + external_proto_deps + [ - "@com_google_protobuf//:any_proto", - "@com_google_protobuf//:descriptor_proto", - "@com_google_protobuf//:duration_proto", - "@com_google_protobuf//:empty_proto", - "@com_google_protobuf//:struct_proto", - "@com_google_protobuf//:timestamp_proto", - "@com_google_protobuf//:wrappers_proto", - "@com_google_googleapis//google/api:http_proto", - "@com_google_googleapis//google/api:annotations_proto", - "@com_google_googleapis//google/rpc:status_proto", - "@com_github_gogo_protobuf//:gogo_proto", - "@com_envoyproxy_protoc_gen_validate//validate:validate_proto", - ], + deps = deps + external_proto_deps + _COMMON_PROTO_DEPS, visibility = visibility, ) pgv_cc_proto_library( @@ -181,3 +145,50 @@ def api_go_test(name, size, importpath, srcs = [], deps = []): importpath = importpath, deps = deps, ) + +_GO_BAZEL_RULE_MAPPING = { + "@opencensus_proto//opencensus/proto/trace/v1:trace_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_go", + "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto": "@opencensus_proto//opencensus/proto/trace/v1:trace_and_config_proto_go", + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto": "@com_google_googleapis//google/api/expr/v1alpha1:cel_go_proto", +} + +def go_proto_mapping(dep): + mapped = _GO_BAZEL_RULE_MAPPING.get(dep) + if mapped == None: + return _Suffix("@" + Label(dep).workspace_name + "//" + Label(dep).package + ":" + Label(dep).name, _GO_PROTO_SUFFIX) + return mapped + +def api_proto_package(name = "pkg", srcs = [], deps = [], has_services = False, visibility = ["//visibility:public"]): + if srcs == []: + srcs = native.glob(["*.proto"]) + + native.proto_library( + name = name, + srcs = srcs, + deps = deps + _COMMON_PROTO_DEPS, + visibility = visibility, + ) + + compilers = ["@io_bazel_rules_go//proto:go_proto", "//bazel:pgv_plugin_go"] + if has_services: + compilers = ["@io_bazel_rules_go//proto:go_grpc", "//bazel:pgv_plugin_go"] + + go_proto_library( + name = _Suffix(name, _GO_PROTO_SUFFIX), + compilers = compilers, + importpath = _Suffix(_GO_IMPORTPATH_PREFIX, native.package_name()), + proto = name, + visibility = ["//visibility:public"], + deps = [go_proto_mapping(dep) for dep in deps] + [ + "@com_github_gogo_protobuf//:gogo_proto_go", + "@com_github_golang_protobuf//ptypes:go_default_library", + "@com_github_golang_protobuf//ptypes/any:go_default_library", + "@com_github_golang_protobuf//ptypes/duration:go_default_library", + "@com_github_golang_protobuf//ptypes/struct:go_default_library", + "@com_github_golang_protobuf//ptypes/timestamp:go_default_library", + "@com_github_golang_protobuf//ptypes/wrappers:go_default_library", + "@com_envoyproxy_protoc_gen_validate//validate:go_default_library", + "@com_google_googleapis//google/api:annotations_go_proto", + "@com_google_googleapis//google/rpc:status_go_proto", + ], + ) diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index 7af054b20f4f..b992717d273a 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -161,7 +161,7 @@ filegroup( ZIPKINAPI_BUILD_CONTENT = """ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_go_proto_library") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") api_proto_library( @@ -173,8 +173,9 @@ api_proto_library( visibility = ["//visibility:public"], ) -api_go_proto_library( - name = "zipkin", +go_proto_library( + name = "zipkin_go_proto", proto = ":zipkin", + visibility = ["//visibility:public"], ) """ diff --git a/api/envoy/admin/v2alpha/BUILD b/api/envoy/admin/v2alpha/BUILD index aa35aa7c8d7d..850eb0515865 100644 --- a/api/envoy/admin/v2alpha/BUILD +++ b/api/envoy/admin/v2alpha/BUILD @@ -1,7 +1,18 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2", + "//envoy/api/v2/auth", + "//envoy/api/v2/core", + "//envoy/config/bootstrap/v2:pkg", + "//envoy/service/tap/v2alpha:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "config_dump", srcs = ["config_dump.proto"], diff --git a/api/envoy/admin/v3alpha/BUILD b/api/envoy/admin/v3alpha/BUILD index 71b0790b69a9..9849282084a6 100644 --- a/api/envoy/admin/v3alpha/BUILD +++ b/api/envoy/admin/v3alpha/BUILD @@ -1,7 +1,18 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha", + "//envoy/api/v3alpha/auth", + "//envoy/api/v3alpha/core", + "//envoy/config/bootstrap/v3alpha:pkg", + "//envoy/service/tap/v3alpha:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "config_dump", srcs = ["config_dump.proto"], diff --git a/api/envoy/api/v2/BUILD b/api/envoy/api/v2/BUILD index b86ca2a788bf..72fcb5f56299 100644 --- a/api/envoy/api/v2/BUILD +++ b/api/envoy/api/v2/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 @@ -16,6 +16,21 @@ package_group( ], ) +api_proto_package( + name = "v2", + has_services = True, + deps = [ + "//envoy/api/v2/auth", + "//envoy/api/v2/cluster", + "//envoy/api/v2/core", + "//envoy/api/v2/endpoint:pkg", + "//envoy/api/v2/listener:pkg", + "//envoy/api/v2/ratelimit:pkg", + "//envoy/api/v2/route:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "discovery", srcs = ["discovery.proto"], @@ -23,12 +38,6 @@ api_proto_library_internal( deps = ["//envoy/api/v2/core:base"], ) -api_go_proto_library( - name = "discovery", - proto = ":discovery", - deps = ["//envoy/api/v2/core:base_go_proto"], -) - api_proto_library_internal( name = "eds", srcs = ["eds.proto"], @@ -44,19 +53,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "eds", - proto = ":eds", - deps = [ - ":discovery_go_proto", - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:health_check_go_proto", - "//envoy/api/v2/endpoint:endpoint_go_proto", - "//envoy/type:percent_go_proto", - ], -) - api_proto_library_internal( name = "cds", srcs = ["cds.proto"], @@ -79,26 +75,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "cds", - proto = ":cds", - deps = [ - ":discovery_go_proto", - ":eds_go_grpc", - "//envoy/api/v2/auth:cert_go_proto", - "//envoy/api/v2/cluster:circuit_breaker_go_proto", - "//envoy/api/v2/cluster:filter_go_proto", - "//envoy/api/v2/cluster:outlier_detection_go_proto", - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:config_source_go_proto", - "//envoy/api/v2/core:health_check_go_proto", - "//envoy/api/v2/core:protocol_go_proto", - "//envoy/api/v2/endpoint:endpoint_go_proto", - "//envoy/type:percent_go_proto", - ], -) - api_proto_library_internal( name = "lds", srcs = ["lds.proto"], @@ -113,18 +89,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "lds", - proto = ":lds", - deps = [ - ":discovery_go_proto", - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/listener:listener_go_proto", - "//envoy/api/v2/listener:udp_listener_config_go_proto", - ], -) - api_proto_library_internal( name = "rds", srcs = ["rds.proto"], @@ -138,17 +102,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "rds", - proto = ":rds", - deps = [ - ":discovery_go_proto", - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:config_source_go_proto", - "//envoy/api/v2/route:route_go_proto", - ], -) - api_proto_library_internal( name = "srds", srcs = ["srds.proto"], @@ -160,12 +113,3 @@ api_proto_library_internal( "//envoy/api/v2/route", ], ) - -api_go_grpc_library( - name = "srds", - proto = ":srds", - deps = [ - ":discovery_go_proto", - "//envoy/api/v2/core:base_go_proto", - ], -) diff --git a/api/envoy/api/v2/auth/BUILD b/api/envoy/api/v2/auth/BUILD index acc28aacff05..bb3951fb95aa 100644 --- a/api/envoy/api/v2/auth/BUILD +++ b/api/envoy/api/v2/auth/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 @@ -15,6 +15,13 @@ package_group( ], ) +api_proto_package( + name = "auth", + deps = [ + "//envoy/api/v2/core", + ], +) + api_proto_library_internal( name = "cert", srcs = ["cert.proto"], @@ -24,12 +31,3 @@ api_proto_library_internal( "//envoy/api/v2/core:config_source", ], ) - -api_go_proto_library( - name = "cert", - proto = ":cert", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:config_source_go_proto", - ], -) diff --git a/api/envoy/api/v2/auth/cert.proto b/api/envoy/api/v2/auth/cert.proto index 30db22c6d7a6..2a8267805161 100644 --- a/api/envoy/api/v2/auth/cert.proto +++ b/api/envoy/api/v2/auth/cert.proto @@ -5,7 +5,6 @@ package envoy.api.v2.auth; option java_outer_classname = "CertProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.auth"; -option go_package = "auth"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/core/config_source.proto"; diff --git a/api/envoy/api/v2/cluster/BUILD b/api/envoy/api/v2/cluster/BUILD index 5589905d859b..baf9a4bfdeb7 100644 --- a/api/envoy/api/v2/cluster/BUILD +++ b/api/envoy/api/v2/cluster/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + name = "cluster", + deps = [ + "//envoy/api/v2/core", + ], +) + api_proto_library_internal( name = "circuit_breaker", srcs = ["circuit_breaker.proto"], @@ -13,14 +20,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "circuit_breaker", - proto = ":circuit_breaker", - deps = [ - "//envoy/api/v2/core:base_go_proto", - ], -) - api_proto_library_internal( name = "outlier_detection", srcs = ["outlier_detection.proto"], @@ -29,11 +28,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "outlier_detection", - proto = ":outlier_detection", -) - api_proto_library_internal( name = "filter", srcs = ["filter.proto"], @@ -41,8 +35,3 @@ api_proto_library_internal( "//envoy/api/v2:__pkg__", ], ) - -api_go_proto_library( - name = "filter", - proto = ":filter", -) diff --git a/api/envoy/api/v2/cluster/circuit_breaker.proto b/api/envoy/api/v2/cluster/circuit_breaker.proto index bc2bcf2548d2..5ae8cc3d1a01 100644 --- a/api/envoy/api/v2/cluster/circuit_breaker.proto +++ b/api/envoy/api/v2/cluster/circuit_breaker.proto @@ -5,7 +5,6 @@ package envoy.api.v2.cluster; option java_outer_classname = "CircuitBreakerProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.cluster"; -option go_package = "cluster"; option csharp_namespace = "Envoy.Api.V2.ClusterNS"; option ruby_package = "Envoy.Api.V2.ClusterNS"; diff --git a/api/envoy/api/v2/core/BUILD b/api/envoy/api/v2/core/BUILD index b3d2be2301f1..01234d07b198 100644 --- a/api/envoy/api/v2/core/BUILD +++ b/api/envoy/api/v2/core/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 @@ -16,6 +16,13 @@ package_group( ], ) +api_proto_package( + name = "core", + deps = [ + "//envoy/type", + ], +) + api_proto_library_internal( name = "address", srcs = ["address.proto"], @@ -25,12 +32,6 @@ api_proto_library_internal( deps = [":base"], ) -api_go_proto_library( - name = "address", - proto = ":address", - deps = [":base_go_proto"], -) - api_proto_library_internal( name = "base", srcs = ["base.proto"], @@ -43,15 +44,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "base", - proto = ":base", - deps = [ - ":http_uri_go_proto", - "//envoy/type:percent_go_proto", - ], -) - api_proto_library_internal( name = "health_check", srcs = ["health_check.proto"], @@ -64,15 +56,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "health_check", - proto = ":health_check", - deps = [ - ":base_go_proto", - "//envoy/type:range_go_proto", - ], -) - api_proto_library_internal( name = "config_source", srcs = ["config_source.proto"], @@ -85,20 +68,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "config_source", - proto = ":config_source", - deps = [ - ":base_go_proto", - ":grpc_service_go_proto", - ], -) - -api_go_proto_library( - name = "http_uri", - proto = ":http_uri", -) - api_proto_library_internal( name = "http_uri", srcs = ["http_uri.proto"], @@ -116,12 +85,6 @@ api_proto_library_internal( deps = [":base"], ) -api_go_proto_library( - name = "grpc_service", - proto = ":grpc_service", - deps = [":base_go_proto"], -) - api_proto_library_internal( name = "protocol", srcs = ["protocol.proto"], @@ -129,8 +92,3 @@ api_proto_library_internal( ":friends", ], ) - -api_go_proto_library( - name = "protocol", - proto = ":protocol", -) diff --git a/api/envoy/api/v2/core/base.proto b/api/envoy/api/v2/core/base.proto index 2553fe04de27..0f73b83af03f 100644 --- a/api/envoy/api/v2/core/base.proto +++ b/api/envoy/api/v2/core/base.proto @@ -5,7 +5,6 @@ package envoy.api.v2.core; option java_outer_classname = "BaseProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.core"; -option go_package = "core"; import "envoy/api/v2/core/http_uri.proto"; diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index a3072a817aa4..5328e515bcbb 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -5,7 +5,6 @@ package envoy.api.v2; option java_outer_classname = "DiscoveryProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2"; -option go_package = "v2"; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/api/v2/endpoint/BUILD b/api/envoy/api/v2/endpoint/BUILD index 0dead0f57033..a12db37309ce 100644 --- a/api/envoy/api/v2/endpoint/BUILD +++ b/api/envoy/api/v2/endpoint/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/auth", + "//envoy/api/v2/core", + ], +) + api_proto_library_internal( name = "endpoint", srcs = ["endpoint.proto"], @@ -16,19 +23,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "endpoint", - proto = ":endpoint", - deps = [ - "//envoy/api/v2/auth:cert_go_proto", - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:config_source_go_proto", - "//envoy/api/v2/core:health_check_go_proto", - "//envoy/api/v2/core:protocol_go_proto", - ], -) - api_proto_library_internal( name = "load_report", srcs = ["load_report.proto"], @@ -38,12 +32,3 @@ api_proto_library_internal( "//envoy/api/v2/core:base", ], ) - -api_go_proto_library( - name = "load_report", - proto = ":load_report", - deps = [ - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - ], -) diff --git a/api/envoy/api/v2/endpoint/endpoint.proto b/api/envoy/api/v2/endpoint/endpoint.proto index 7abb7ea5f3c9..6327af00ac99 100644 --- a/api/envoy/api/v2/endpoint/endpoint.proto +++ b/api/envoy/api/v2/endpoint/endpoint.proto @@ -5,7 +5,6 @@ package envoy.api.v2.endpoint; option java_outer_classname = "EndpointProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.endpoint"; -option go_package = "endpoint"; import "envoy/api/v2/core/address.proto"; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/api/v2/listener/BUILD b/api/envoy/api/v2/listener/BUILD index e539c4b8c090..42c79fe45483 100644 --- a/api/envoy/api/v2/listener/BUILD +++ b/api/envoy/api/v2/listener/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/auth", + "//envoy/api/v2/core", + ], +) + api_proto_library_internal( name = "listener", srcs = ["listener.proto"], @@ -13,16 +20,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "listener", - proto = ":listener", - deps = [ - "//envoy/api/v2/auth:cert_go_proto", - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - ], -) - api_proto_library_internal( name = "udp_listener_config", srcs = ["udp_listener_config.proto"], @@ -31,11 +28,3 @@ api_proto_library_internal( "//envoy/api/v2/core:base", ], ) - -api_go_proto_library( - name = "udp_listener_config", - proto = ":udp_listener_config", - deps = [ - "//envoy/api/v2/core:base_go_proto", - ], -) diff --git a/api/envoy/api/v2/listener/listener.proto b/api/envoy/api/v2/listener/listener.proto index 293a4d2da221..3b6cf74ab0a2 100644 --- a/api/envoy/api/v2/listener/listener.proto +++ b/api/envoy/api/v2/listener/listener.proto @@ -5,7 +5,6 @@ package envoy.api.v2.listener; option java_outer_classname = "ListenerProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.listener"; -option go_package = "listener"; option csharp_namespace = "Envoy.Api.V2.ListenerNS"; option ruby_package = "Envoy::Api::V2::ListenerNS"; diff --git a/api/envoy/api/v2/listener/udp_listener_config.proto b/api/envoy/api/v2/listener/udp_listener_config.proto index f75383bab232..28d8233f5ff0 100644 --- a/api/envoy/api/v2/listener/udp_listener_config.proto +++ b/api/envoy/api/v2/listener/udp_listener_config.proto @@ -5,7 +5,6 @@ package envoy.api.v2.listener; option java_outer_classname = "UdpListenerConfigProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.listener"; -option go_package = "listener"; option csharp_namespace = "Envoy.Api.V2.ListenerNS"; option ruby_package = "Envoy::Api::V2::ListenerNS"; diff --git a/api/envoy/api/v2/ratelimit/BUILD b/api/envoy/api/v2/ratelimit/BUILD index 5f2a9201463d..234a3b20f16b 100644 --- a/api/envoy/api/v2/ratelimit/BUILD +++ b/api/envoy/api/v2/ratelimit/BUILD @@ -1,14 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "ratelimit", srcs = ["ratelimit.proto"], visibility = ["//envoy/api/v2:friends"], ) - -api_go_proto_library( - name = "ratelimit", - proto = ":ratelimit", -) diff --git a/api/envoy/api/v2/ratelimit/ratelimit.proto b/api/envoy/api/v2/ratelimit/ratelimit.proto index 8ebec7182257..6f4cd6258283 100644 --- a/api/envoy/api/v2/ratelimit/ratelimit.proto +++ b/api/envoy/api/v2/ratelimit/ratelimit.proto @@ -5,7 +5,6 @@ package envoy.api.v2.ratelimit; option java_outer_classname = "RatelimitProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.ratelimit"; -option go_package = "ratelimit"; import "validate/validate.proto"; diff --git a/api/envoy/api/v2/route/BUILD b/api/envoy/api/v2/route/BUILD index 968ab1c67be2..163281ca35df 100644 --- a/api/envoy/api/v2/route/BUILD +++ b/api/envoy/api/v2/route/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/type", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "route", srcs = ["route.proto"], @@ -14,15 +22,3 @@ api_proto_library_internal( "//envoy/type/matcher:string", ], ) - -api_go_proto_library( - name = "route", - proto = ":route", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "//envoy/type:percent_go_proto", - "//envoy/type:range_go_proto", - "//envoy/type/matcher:regex_go_proto", - "//envoy/type/matcher:string_go_proto", - ], -) diff --git a/api/envoy/api/v2/route/route.proto b/api/envoy/api/v2/route/route.proto index e0fbaf0fd685..d396344a1ff2 100644 --- a/api/envoy/api/v2/route/route.proto +++ b/api/envoy/api/v2/route/route.proto @@ -5,7 +5,6 @@ package envoy.api.v2.route; option java_outer_classname = "RouteProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.route"; -option go_package = "route"; option java_generic_services = true; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/api/v3alpha/BUILD b/api/envoy/api/v3alpha/BUILD index 0e2892e87e69..e61a715ab9de 100644 --- a/api/envoy/api/v3alpha/BUILD +++ b/api/envoy/api/v3alpha/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 @@ -16,6 +16,21 @@ package_group( ], ) +api_proto_package( + name = "v3alpha", + has_services = True, + deps = [ + "//envoy/api/v3alpha/auth", + "//envoy/api/v3alpha/cluster", + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/endpoint:pkg", + "//envoy/api/v3alpha/listener:pkg", + "//envoy/api/v3alpha/ratelimit:pkg", + "//envoy/api/v3alpha/route:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "discovery", srcs = ["discovery.proto"], @@ -23,12 +38,6 @@ api_proto_library_internal( deps = ["//envoy/api/v3alpha/core:base"], ) -api_go_proto_library( - name = "discovery", - proto = ":discovery", - deps = ["//envoy/api/v3alpha/core:base_go_proto"], -) - api_proto_library_internal( name = "eds", srcs = ["eds.proto"], @@ -44,19 +53,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "eds", - proto = ":eds", - deps = [ - ":discovery_go_proto", - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:health_check_go_proto", - "//envoy/api/v3alpha/endpoint:endpoint_go_proto", - "//envoy/type:percent_go_proto", - ], -) - api_proto_library_internal( name = "cds", srcs = ["cds.proto"], @@ -79,26 +75,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "cds", - proto = ":cds", - deps = [ - ":discovery_go_proto", - ":eds_go_grpc", - "//envoy/api/v3alpha/auth:cert_go_proto", - "//envoy/api/v3alpha/cluster:circuit_breaker_go_proto", - "//envoy/api/v3alpha/cluster:filter_go_proto", - "//envoy/api/v3alpha/cluster:outlier_detection_go_proto", - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:config_source_go_proto", - "//envoy/api/v3alpha/core:health_check_go_proto", - "//envoy/api/v3alpha/core:protocol_go_proto", - "//envoy/api/v3alpha/endpoint:endpoint_go_proto", - "//envoy/type:percent_go_proto", - ], -) - api_proto_library_internal( name = "lds", srcs = ["lds.proto"], @@ -113,18 +89,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "lds", - proto = ":lds", - deps = [ - ":discovery_go_proto", - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/listener:listener_go_proto", - "//envoy/api/v3alpha/listener:udp_listener_config_go_proto", - ], -) - api_proto_library_internal( name = "rds", srcs = ["rds.proto"], @@ -138,17 +102,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "rds", - proto = ":rds", - deps = [ - ":discovery_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:config_source_go_proto", - "//envoy/api/v3alpha/route:route_go_proto", - ], -) - api_proto_library_internal( name = "srds", srcs = ["srds.proto"], @@ -160,12 +113,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/route", ], ) - -api_go_grpc_library( - name = "srds", - proto = ":srds", - deps = [ - ":discovery_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - ], -) diff --git a/api/envoy/api/v3alpha/auth/BUILD b/api/envoy/api/v3alpha/auth/BUILD index f206a35f97f2..6c47aff6e2a3 100644 --- a/api/envoy/api/v3alpha/auth/BUILD +++ b/api/envoy/api/v3alpha/auth/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 @@ -15,6 +15,13 @@ package_group( ], ) +api_proto_package( + name = "auth", + deps = [ + "//envoy/api/v3alpha/core", + ], +) + api_proto_library_internal( name = "cert", srcs = ["cert.proto"], @@ -24,12 +31,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/core:config_source", ], ) - -api_go_proto_library( - name = "cert", - proto = ":cert", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:config_source_go_proto", - ], -) diff --git a/api/envoy/api/v3alpha/auth/cert.proto b/api/envoy/api/v3alpha/auth/cert.proto index 925453074ac1..2be82a1c0496 100644 --- a/api/envoy/api/v3alpha/auth/cert.proto +++ b/api/envoy/api/v3alpha/auth/cert.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha.auth; option java_outer_classname = "CertProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.auth"; -option go_package = "auth"; import "envoy/api/v3alpha/core/base.proto"; import "envoy/api/v3alpha/core/config_source.proto"; diff --git a/api/envoy/api/v3alpha/cluster/BUILD b/api/envoy/api/v3alpha/cluster/BUILD index 942701221a37..ef01624057e0 100644 --- a/api/envoy/api/v3alpha/cluster/BUILD +++ b/api/envoy/api/v3alpha/cluster/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + name = "cluster", + deps = [ + "//envoy/api/v3alpha/core", + ], +) + api_proto_library_internal( name = "circuit_breaker", srcs = ["circuit_breaker.proto"], @@ -13,14 +20,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "circuit_breaker", - proto = ":circuit_breaker", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - ], -) - api_proto_library_internal( name = "outlier_detection", srcs = ["outlier_detection.proto"], @@ -29,11 +28,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "outlier_detection", - proto = ":outlier_detection", -) - api_proto_library_internal( name = "filter", srcs = ["filter.proto"], @@ -41,8 +35,3 @@ api_proto_library_internal( "//envoy/api/v3alpha:__pkg__", ], ) - -api_go_proto_library( - name = "filter", - proto = ":filter", -) diff --git a/api/envoy/api/v3alpha/cluster/circuit_breaker.proto b/api/envoy/api/v3alpha/cluster/circuit_breaker.proto index 39f4f77c5ddd..8a70008e49f5 100644 --- a/api/envoy/api/v3alpha/cluster/circuit_breaker.proto +++ b/api/envoy/api/v3alpha/cluster/circuit_breaker.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha.cluster; option java_outer_classname = "CircuitBreakerProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.cluster"; -option go_package = "cluster"; option csharp_namespace = "Envoy.Api.V2.ClusterNS"; option ruby_package = "Envoy.Api.V2.ClusterNS"; diff --git a/api/envoy/api/v3alpha/core/BUILD b/api/envoy/api/v3alpha/core/BUILD index cfc6bd83ca78..871c9fe0e838 100644 --- a/api/envoy/api/v3alpha/core/BUILD +++ b/api/envoy/api/v3alpha/core/BUILD @@ -1,4 +1,4 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 @@ -16,6 +16,13 @@ package_group( ], ) +api_proto_package( + name = "core", + deps = [ + "//envoy/type", + ], +) + api_proto_library_internal( name = "address", srcs = ["address.proto"], @@ -25,12 +32,6 @@ api_proto_library_internal( deps = [":base"], ) -api_go_proto_library( - name = "address", - proto = ":address", - deps = [":base_go_proto"], -) - api_proto_library_internal( name = "base", srcs = ["base.proto"], @@ -43,15 +44,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "base", - proto = ":base", - deps = [ - ":http_uri_go_proto", - "//envoy/type:percent_go_proto", - ], -) - api_proto_library_internal( name = "health_check", srcs = ["health_check.proto"], @@ -64,15 +56,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "health_check", - proto = ":health_check", - deps = [ - ":base_go_proto", - "//envoy/type:range_go_proto", - ], -) - api_proto_library_internal( name = "config_source", srcs = ["config_source.proto"], @@ -85,20 +68,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "config_source", - proto = ":config_source", - deps = [ - ":base_go_proto", - ":grpc_service_go_proto", - ], -) - -api_go_proto_library( - name = "http_uri", - proto = ":http_uri", -) - api_proto_library_internal( name = "http_uri", srcs = ["http_uri.proto"], @@ -116,12 +85,6 @@ api_proto_library_internal( deps = [":base"], ) -api_go_proto_library( - name = "grpc_service", - proto = ":grpc_service", - deps = [":base_go_proto"], -) - api_proto_library_internal( name = "protocol", srcs = ["protocol.proto"], @@ -129,8 +92,3 @@ api_proto_library_internal( ":friends", ], ) - -api_go_proto_library( - name = "protocol", - proto = ":protocol", -) diff --git a/api/envoy/api/v3alpha/core/base.proto b/api/envoy/api/v3alpha/core/base.proto index 0661d99ec546..cf39df887bce 100644 --- a/api/envoy/api/v3alpha/core/base.proto +++ b/api/envoy/api/v3alpha/core/base.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha.core; option java_outer_classname = "BaseProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; -option go_package = "core"; import "envoy/api/v3alpha/core/http_uri.proto"; diff --git a/api/envoy/api/v3alpha/discovery.proto b/api/envoy/api/v3alpha/discovery.proto index 87433f0dca27..105b99888141 100644 --- a/api/envoy/api/v3alpha/discovery.proto +++ b/api/envoy/api/v3alpha/discovery.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha; option java_outer_classname = "DiscoveryProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/api/v3alpha/endpoint/BUILD b/api/envoy/api/v3alpha/endpoint/BUILD index 1630438b13f6..733560514dbd 100644 --- a/api/envoy/api/v3alpha/endpoint/BUILD +++ b/api/envoy/api/v3alpha/endpoint/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/auth", + "//envoy/api/v3alpha/core", + ], +) + api_proto_library_internal( name = "endpoint", srcs = ["endpoint.proto"], @@ -16,19 +23,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "endpoint", - proto = ":endpoint", - deps = [ - "//envoy/api/v3alpha/auth:cert_go_proto", - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:config_source_go_proto", - "//envoy/api/v3alpha/core:health_check_go_proto", - "//envoy/api/v3alpha/core:protocol_go_proto", - ], -) - api_proto_library_internal( name = "load_report", srcs = ["load_report.proto"], @@ -38,12 +32,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/core:base", ], ) - -api_go_proto_library( - name = "load_report", - proto = ":load_report", - deps = [ - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - ], -) diff --git a/api/envoy/api/v3alpha/endpoint/endpoint.proto b/api/envoy/api/v3alpha/endpoint/endpoint.proto index 4bb1b57e8710..15357cdbac8b 100644 --- a/api/envoy/api/v3alpha/endpoint/endpoint.proto +++ b/api/envoy/api/v3alpha/endpoint/endpoint.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha.endpoint; option java_outer_classname = "EndpointProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.endpoint"; -option go_package = "endpoint"; import "envoy/api/v3alpha/core/address.proto"; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/api/v3alpha/listener/BUILD b/api/envoy/api/v3alpha/listener/BUILD index 693ead54dde0..3ee071ca5c03 100644 --- a/api/envoy/api/v3alpha/listener/BUILD +++ b/api/envoy/api/v3alpha/listener/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/auth", + "//envoy/api/v3alpha/core", + ], +) + api_proto_library_internal( name = "listener", srcs = ["listener.proto"], @@ -13,16 +20,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "listener", - proto = ":listener", - deps = [ - "//envoy/api/v3alpha/auth:cert_go_proto", - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - ], -) - api_proto_library_internal( name = "udp_listener_config", srcs = ["udp_listener_config.proto"], @@ -31,11 +28,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/core:base", ], ) - -api_go_proto_library( - name = "udp_listener_config", - proto = ":udp_listener_config", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - ], -) diff --git a/api/envoy/api/v3alpha/listener/listener.proto b/api/envoy/api/v3alpha/listener/listener.proto index 2aa7146a822c..dc44c1454c02 100644 --- a/api/envoy/api/v3alpha/listener/listener.proto +++ b/api/envoy/api/v3alpha/listener/listener.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha.listener; option java_outer_classname = "ListenerProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.listener"; -option go_package = "listener"; option csharp_namespace = "Envoy.Api.V2.ListenerNS"; option ruby_package = "Envoy::Api::V2::ListenerNS"; diff --git a/api/envoy/api/v3alpha/listener/udp_listener_config.proto b/api/envoy/api/v3alpha/listener/udp_listener_config.proto index 763a08a93ad3..532028da9f73 100644 --- a/api/envoy/api/v3alpha/listener/udp_listener_config.proto +++ b/api/envoy/api/v3alpha/listener/udp_listener_config.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha.listener; option java_outer_classname = "UdpListenerConfigProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.listener"; -option go_package = "listener"; option csharp_namespace = "Envoy.Api.V2.ListenerNS"; option ruby_package = "Envoy::Api::V2::ListenerNS"; diff --git a/api/envoy/api/v3alpha/ratelimit/BUILD b/api/envoy/api/v3alpha/ratelimit/BUILD index b08c1fc029a0..a99624b1c421 100644 --- a/api/envoy/api/v3alpha/ratelimit/BUILD +++ b/api/envoy/api/v3alpha/ratelimit/BUILD @@ -1,14 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "ratelimit", srcs = ["ratelimit.proto"], visibility = ["//envoy/api/v3alpha:friends"], ) - -api_go_proto_library( - name = "ratelimit", - proto = ":ratelimit", -) diff --git a/api/envoy/api/v3alpha/ratelimit/ratelimit.proto b/api/envoy/api/v3alpha/ratelimit/ratelimit.proto index c10bfef83b98..9f2b67818a02 100644 --- a/api/envoy/api/v3alpha/ratelimit/ratelimit.proto +++ b/api/envoy/api/v3alpha/ratelimit/ratelimit.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha.ratelimit; option java_outer_classname = "RatelimitProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.ratelimit"; -option go_package = "ratelimit"; import "validate/validate.proto"; diff --git a/api/envoy/api/v3alpha/route/BUILD b/api/envoy/api/v3alpha/route/BUILD index 0b660893c5d4..cbed3ec01f4b 100644 --- a/api/envoy/api/v3alpha/route/BUILD +++ b/api/envoy/api/v3alpha/route/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/type", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "route", srcs = ["route.proto"], @@ -14,15 +22,3 @@ api_proto_library_internal( "//envoy/type/matcher:string", ], ) - -api_go_proto_library( - name = "route", - proto = ":route", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/type:percent_go_proto", - "//envoy/type:range_go_proto", - "//envoy/type/matcher:regex_go_proto", - "//envoy/type/matcher:string_go_proto", - ], -) diff --git a/api/envoy/api/v3alpha/route/route.proto b/api/envoy/api/v3alpha/route/route.proto index 963d94f1b022..e69e116b9b63 100644 --- a/api/envoy/api/v3alpha/route/route.proto +++ b/api/envoy/api/v3alpha/route/route.proto @@ -5,7 +5,6 @@ package envoy.api.v3alpha.route; option java_outer_classname = "RouteProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.route"; -option go_package = "route"; option java_generic_services = true; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/config/accesslog/v2/BUILD b/api/envoy/config/accesslog/v2/BUILD index 85ac228ccf13..22c48f795224 100644 --- a/api/envoy/config/accesslog/v2/BUILD +++ b/api/envoy/config/accesslog/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library_internal( name = "als", srcs = ["als.proto"], @@ -14,9 +18,3 @@ api_proto_library_internal( name = "file", srcs = ["file.proto"], ) - -api_go_proto_library( - name = "als", - proto = ":als", - deps = ["//envoy/api/v2/core:grpc_service_go_proto"], -) diff --git a/api/envoy/config/accesslog/v2/als.proto b/api/envoy/config/accesslog/v2/als.proto index 9d83ebfcfb91..c02835dbbc56 100644 --- a/api/envoy/config/accesslog/v2/als.proto +++ b/api/envoy/config/accesslog/v2/als.proto @@ -5,7 +5,6 @@ package envoy.config.accesslog.v2; option java_outer_classname = "AlsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.accesslog.v2"; -option go_package = "v2"; import "envoy/api/v2/core/grpc_service.proto"; diff --git a/api/envoy/config/accesslog/v2/file.proto b/api/envoy/config/accesslog/v2/file.proto index 48a1841a9614..b88529a3251d 100644 --- a/api/envoy/config/accesslog/v2/file.proto +++ b/api/envoy/config/accesslog/v2/file.proto @@ -5,7 +5,6 @@ package envoy.config.accesslog.v2; option java_outer_classname = "FileProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.accesslog.v2"; -option go_package = "v2"; import "validate/validate.proto"; import "google/protobuf/struct.proto"; diff --git a/api/envoy/config/accesslog/v3alpha/BUILD b/api/envoy/config/accesslog/v3alpha/BUILD index 4f5da73ee424..8409598da650 100644 --- a/api/envoy/config/accesslog/v3alpha/BUILD +++ b/api/envoy/config/accesslog/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library_internal( name = "als", srcs = ["als.proto"], @@ -14,9 +18,3 @@ api_proto_library_internal( name = "file", srcs = ["file.proto"], ) - -api_go_proto_library( - name = "als", - proto = ":als", - deps = ["//envoy/api/v3alpha/core:grpc_service_go_proto"], -) diff --git a/api/envoy/config/accesslog/v3alpha/als.proto b/api/envoy/config/accesslog/v3alpha/als.proto index a194d1449e4b..07ec724d10ef 100644 --- a/api/envoy/config/accesslog/v3alpha/als.proto +++ b/api/envoy/config/accesslog/v3alpha/als.proto @@ -5,7 +5,6 @@ package envoy.config.accesslog.v3alpha; option java_outer_classname = "AlsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.accesslog.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/grpc_service.proto"; diff --git a/api/envoy/config/accesslog/v3alpha/file.proto b/api/envoy/config/accesslog/v3alpha/file.proto index b07658bc9275..2f32da7bb64f 100644 --- a/api/envoy/config/accesslog/v3alpha/file.proto +++ b/api/envoy/config/accesslog/v3alpha/file.proto @@ -5,7 +5,6 @@ package envoy.config.accesslog.v3alpha; option java_outer_classname = "FileProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.accesslog.v3alpha"; -option go_package = "v2"; import "validate/validate.proto"; import "google/protobuf/struct.proto"; diff --git a/api/envoy/config/bootstrap/v2/BUILD b/api/envoy/config/bootstrap/v2/BUILD index 455365ab1e77..1f3a79104b60 100644 --- a/api/envoy/config/bootstrap/v2/BUILD +++ b/api/envoy/config/bootstrap/v2/BUILD @@ -1,7 +1,19 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2", + "//envoy/api/v2/auth", + "//envoy/api/v2/core", + "//envoy/config/metrics/v2:pkg", + "//envoy/config/overload/v2alpha:pkg", + "//envoy/config/ratelimit/v2:pkg", + "//envoy/config/trace/v2:pkg", + ], +) + api_proto_library_internal( name = "bootstrap", srcs = ["bootstrap.proto"], @@ -20,21 +32,3 @@ api_proto_library_internal( "//envoy/config/trace/v2:trace", ], ) - -api_go_proto_library( - name = "bootstrap", - proto = ":bootstrap", - deps = [ - "//envoy/api/v2:cds_go_grpc", - "//envoy/api/v2:lds_go_grpc", - "//envoy/api/v2/auth:cert_go_proto", - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:config_source_go_proto", - "//envoy/config/metrics/v2:metrics_service_go_proto", - "//envoy/config/metrics/v2:stats_go_proto", - "//envoy/config/overload/v2alpha:overload_go_proto", - "//envoy/config/ratelimit/v2:rls_go_grpc", - "//envoy/config/trace/v2:trace_go_proto", - ], -) diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index 9e3d9fe1cdb1..66f05aa78486 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -10,7 +10,6 @@ package envoy.config.bootstrap.v2; option java_outer_classname = "BootstrapProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.bootstrap.v2"; -option go_package = "v2"; import "envoy/api/v2/core/address.proto"; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/config/bootstrap/v3alpha/BUILD b/api/envoy/config/bootstrap/v3alpha/BUILD index d148021c741a..c88b982492ce 100644 --- a/api/envoy/config/bootstrap/v3alpha/BUILD +++ b/api/envoy/config/bootstrap/v3alpha/BUILD @@ -1,7 +1,19 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha", + "//envoy/api/v3alpha/auth", + "//envoy/api/v3alpha/core", + "//envoy/config/metrics/v3alpha:pkg", + "//envoy/config/overload/v3alpha:pkg", + "//envoy/config/ratelimit/v3alpha:pkg", + "//envoy/config/trace/v3alpha:pkg", + ], +) + api_proto_library_internal( name = "bootstrap", srcs = ["bootstrap.proto"], @@ -20,21 +32,3 @@ api_proto_library_internal( "//envoy/config/trace/v3alpha:trace", ], ) - -api_go_proto_library( - name = "bootstrap", - proto = ":bootstrap", - deps = [ - "//envoy/api/v3alpha:cds_go_grpc", - "//envoy/api/v3alpha:lds_go_grpc", - "//envoy/api/v3alpha/auth:cert_go_proto", - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:config_source_go_proto", - "//envoy/config/metrics/v3alpha:metrics_service_go_proto", - "//envoy/config/metrics/v3alpha:stats_go_proto", - "//envoy/config/overload/v3alpha:overload_go_proto", - "//envoy/config/ratelimit/v3alpha:rls_go_grpc", - "//envoy/config/trace/v3alpha:trace_go_proto", - ], -) diff --git a/api/envoy/config/bootstrap/v3alpha/bootstrap.proto b/api/envoy/config/bootstrap/v3alpha/bootstrap.proto index 57157a4ae3f3..0bf18ffa360d 100644 --- a/api/envoy/config/bootstrap/v3alpha/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3alpha/bootstrap.proto @@ -10,7 +10,6 @@ package envoy.config.bootstrap.v3alpha; option java_outer_classname = "BootstrapProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.bootstrap.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/address.proto"; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD index b09f5c858ba9..669b6745ab74 100644 --- a/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/common/dynamic_forward_proxy/v2alpha:pkg"], +) + api_proto_library_internal( name = "cluster", srcs = ["cluster.proto"], diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto index d9ae85903264..c6d47807ce50 100644 --- a/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v2alpha/cluster.proto @@ -5,7 +5,6 @@ package envoy.config.cluster.dynamic_forward_proxy.v2alpha; option java_outer_classname = "DynamicForwardProxyClusterProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.cluster.dynamic_forward_proxy.v2alpha"; -option go_package = "v2alpha"; import "envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto"; diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/BUILD b/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/BUILD index 50d0aa2354eb..3c1d737802cb 100644 --- a/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/BUILD +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/common/dynamic_forward_proxy/v3alpha:pkg"], +) + api_proto_library_internal( name = "cluster", srcs = ["cluster.proto"], diff --git a/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/cluster.proto b/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/cluster.proto index baed68d3b1ac..6bc7bdd4c551 100644 --- a/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/cluster.proto +++ b/api/envoy/config/cluster/dynamic_forward_proxy/v3alpha/cluster.proto @@ -5,7 +5,6 @@ package envoy.config.cluster.dynamic_forward_proxy.v3alpha; option java_outer_classname = "DynamicForwardProxyClusterProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.cluster.dynamic_forward_proxy.v3alpha"; -option go_package = "v2alpha"; import "envoy/config/common/dynamic_forward_proxy/v3alpha/dns_cache.proto"; diff --git a/api/envoy/config/cluster/redis/BUILD b/api/envoy/config/cluster/redis/BUILD index 42e2d408e358..760ae606c05d 100644 --- a/api/envoy/config/cluster/redis/BUILD +++ b/api/envoy/config/cluster/redis/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "redis_cluster", srcs = ["redis_cluster.proto"], diff --git a/api/envoy/config/cluster/redis/redis_cluster.proto b/api/envoy/config/cluster/redis/redis_cluster.proto index 2644288c40d2..abe2c857532d 100644 --- a/api/envoy/config/cluster/redis/redis_cluster.proto +++ b/api/envoy/config/cluster/redis/redis_cluster.proto @@ -5,7 +5,6 @@ package envoy.config.cluster.redis; option java_outer_classname = "RedisClusterProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.cluster.redis"; -option go_package = "v2"; import "google/protobuf/duration.proto"; diff --git a/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD index 53095826454d..312ae36b3762 100644 --- a/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD +++ b/api/envoy/config/common/dynamic_forward_proxy/v2alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2"], +) + api_proto_library_internal( name = "dns_cache", srcs = ["dns_cache.proto"], diff --git a/api/envoy/config/common/dynamic_forward_proxy/v3alpha/BUILD b/api/envoy/config/common/dynamic_forward_proxy/v3alpha/BUILD index bdd23e86de9f..e1853725da14 100644 --- a/api/envoy/config/common/dynamic_forward_proxy/v3alpha/BUILD +++ b/api/envoy/config/common/dynamic_forward_proxy/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha"], +) + api_proto_library_internal( name = "dns_cache", srcs = ["dns_cache.proto"], diff --git a/api/envoy/config/common/tap/v2alpha/BUILD b/api/envoy/config/common/tap/v2alpha/BUILD index 863ba519d128..898773297b51 100644 --- a/api/envoy/config/common/tap/v2alpha/BUILD +++ b/api/envoy/config/common/tap/v2alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/service/tap/v2alpha:pkg", + ], +) + api_proto_library_internal( name = "common", srcs = ["common.proto"], diff --git a/api/envoy/config/common/tap/v3alpha/BUILD b/api/envoy/config/common/tap/v3alpha/BUILD index 673a602800af..55147b12ba3d 100644 --- a/api/envoy/config/common/tap/v3alpha/BUILD +++ b/api/envoy/config/common/tap/v3alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/service/tap/v3alpha:pkg", + ], +) + api_proto_library_internal( name = "common", srcs = ["common.proto"], diff --git a/api/envoy/config/filter/accesslog/v2/BUILD b/api/envoy/config/filter/accesslog/v2/BUILD index fdbf376af177..d9b740921357 100644 --- a/api/envoy/config/filter/accesslog/v2/BUILD +++ b/api/envoy/config/filter/accesslog/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/api/v2/route:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "accesslog", srcs = ["accesslog.proto"], @@ -16,13 +24,3 @@ api_proto_library_internal( "//envoy/type:percent", ], ) - -api_go_proto_library( - name = "accesslog", - proto = ":accesslog", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/route:route_go_proto", - "//envoy/type:percent_go_proto", - ], -) diff --git a/api/envoy/config/filter/accesslog/v2/accesslog.proto b/api/envoy/config/filter/accesslog/v2/accesslog.proto index 76fc4baf80c7..d777708175b5 100644 --- a/api/envoy/config/filter/accesslog/v2/accesslog.proto +++ b/api/envoy/config/filter/accesslog/v2/accesslog.proto @@ -5,7 +5,6 @@ package envoy.config.filter.accesslog.v2; option java_outer_classname = "AccesslogProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.accesslog.v2"; -option go_package = "v2"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/route/route.proto"; diff --git a/api/envoy/config/filter/accesslog/v3alpha/BUILD b/api/envoy/config/filter/accesslog/v3alpha/BUILD index 3f241bc5e10b..454a1ab4a135 100644 --- a/api/envoy/config/filter/accesslog/v3alpha/BUILD +++ b/api/envoy/config/filter/accesslog/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/route:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "accesslog", srcs = ["accesslog.proto"], @@ -16,13 +24,3 @@ api_proto_library_internal( "//envoy/type:percent", ], ) - -api_go_proto_library( - name = "accesslog", - proto = ":accesslog", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/route:route_go_proto", - "//envoy/type:percent_go_proto", - ], -) diff --git a/api/envoy/config/filter/accesslog/v3alpha/accesslog.proto b/api/envoy/config/filter/accesslog/v3alpha/accesslog.proto index 381e7bdf9a87..b7beef0bd974 100644 --- a/api/envoy/config/filter/accesslog/v3alpha/accesslog.proto +++ b/api/envoy/config/filter/accesslog/v3alpha/accesslog.proto @@ -5,7 +5,6 @@ package envoy.config.filter.accesslog.v3alpha; option java_outer_classname = "AccesslogProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.accesslog.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/base.proto"; import "envoy/api/v3alpha/route/route.proto"; diff --git a/api/envoy/config/filter/dubbo/router/v2alpha1/BUILD b/api/envoy/config/filter/dubbo/router/v2alpha1/BUILD index 51c69c0d5b20..68bd8c126b80 100644 --- a/api/envoy/config/filter/dubbo/router/v2alpha1/BUILD +++ b/api/envoy/config/filter/dubbo/router/v2alpha1/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "router", srcs = ["router.proto"], diff --git a/api/envoy/config/filter/dubbo/router/v2alpha1/router.proto b/api/envoy/config/filter/dubbo/router/v2alpha1/router.proto index 37a5542a17bb..4e65f14e0ea9 100644 --- a/api/envoy/config/filter/dubbo/router/v2alpha1/router.proto +++ b/api/envoy/config/filter/dubbo/router/v2alpha1/router.proto @@ -5,7 +5,6 @@ package envoy.config.filter.dubbo.router.v2alpha1; option java_outer_classname = "RouterProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.dubbo.router.v2alpha1"; -option go_package = "v2alpha1"; // [#protodoc-title: Router] // Dubbo router :ref:`configuration overview `. diff --git a/api/envoy/config/filter/fault/v2/BUILD b/api/envoy/config/filter/fault/v2/BUILD index 35419a9902b3..78687f4e4da4 100644 --- a/api/envoy/config/filter/fault/v2/BUILD +++ b/api/envoy/config/filter/fault/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/type"], +) + api_proto_library_internal( name = "fault", srcs = ["fault.proto"], diff --git a/api/envoy/config/filter/fault/v2/fault.proto b/api/envoy/config/filter/fault/v2/fault.proto index f27f9d446267..2298ecf4a1c0 100644 --- a/api/envoy/config/filter/fault/v2/fault.proto +++ b/api/envoy/config/filter/fault/v2/fault.proto @@ -5,7 +5,6 @@ package envoy.config.filter.fault.v2; option java_outer_classname = "FaultProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.fault.v2"; -option go_package = "v2"; import "envoy/type/percent.proto"; diff --git a/api/envoy/config/filter/fault/v3alpha/BUILD b/api/envoy/config/filter/fault/v3alpha/BUILD index 22e3bec56ca3..61bc8dc6bc5e 100644 --- a/api/envoy/config/filter/fault/v3alpha/BUILD +++ b/api/envoy/config/filter/fault/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/type"], +) + api_proto_library_internal( name = "fault", srcs = ["fault.proto"], diff --git a/api/envoy/config/filter/fault/v3alpha/fault.proto b/api/envoy/config/filter/fault/v3alpha/fault.proto index b54a063e7665..054eb8470f3a 100644 --- a/api/envoy/config/filter/fault/v3alpha/fault.proto +++ b/api/envoy/config/filter/fault/v3alpha/fault.proto @@ -5,7 +5,6 @@ package envoy.config.filter.fault.v3alpha; option java_outer_classname = "FaultProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.fault.v3alpha"; -option go_package = "v2"; import "envoy/type/percent.proto"; diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD index 948ceec2223d..b58f88c787ba 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library_internal( name = "adaptive_concurrency", srcs = ["adaptive_concurrency.proto"], diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto index ff19657260e8..303b681471f4 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.adaptive_concurrency.v2alpha; option java_package = "io.envoyproxy.envoy.config.filter.http.adaptive_concurrency.v2alpha"; option java_outer_classname = "AdaptiveConcurrencyProto"; option java_multiple_files = true; -option go_package = "v2alpha"; message AdaptiveConcurrency { } diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD index aa2b0634739c..f9813a6a0829 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD +++ b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library_internal( name = "adaptive_concurrency", srcs = ["adaptive_concurrency.proto"], diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto index 17bac55800ce..3d57196f9db7 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto +++ b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.adaptive_concurrency.v3alpha; option java_package = "io.envoyproxy.envoy.config.filter.http.adaptive_concurrency.v3alpha"; option java_outer_classname = "AdaptiveConcurrencyProto"; option java_multiple_files = true; -option go_package = "v2alpha"; message AdaptiveConcurrency { } diff --git a/api/envoy/config/filter/http/buffer/v2/BUILD b/api/envoy/config/filter/http/buffer/v2/BUILD index e59429af9ace..039ebb63e6d2 100644 --- a/api/envoy/config/filter/http/buffer/v2/BUILD +++ b/api/envoy/config/filter/http/buffer/v2/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "buffer", srcs = ["buffer.proto"], diff --git a/api/envoy/config/filter/http/buffer/v2/buffer.proto b/api/envoy/config/filter/http/buffer/v2/buffer.proto index a203d9d98cc2..92780adad69b 100644 --- a/api/envoy/config/filter/http/buffer/v2/buffer.proto +++ b/api/envoy/config/filter/http/buffer/v2/buffer.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.buffer.v2; option java_outer_classname = "BufferProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.buffer.v2"; -option go_package = "v2"; import "google/protobuf/wrappers.proto"; diff --git a/api/envoy/config/filter/http/buffer/v3alpha/BUILD b/api/envoy/config/filter/http/buffer/v3alpha/BUILD index e59429af9ace..039ebb63e6d2 100644 --- a/api/envoy/config/filter/http/buffer/v3alpha/BUILD +++ b/api/envoy/config/filter/http/buffer/v3alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "buffer", srcs = ["buffer.proto"], diff --git a/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto b/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto index a948493b2450..25530c5a58b4 100644 --- a/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto +++ b/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.buffer.v3alpha; option java_outer_classname = "BufferProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.buffer.v3alpha"; -option go_package = "v2"; import "google/protobuf/wrappers.proto"; diff --git a/api/envoy/config/filter/http/csrf/v2/BUILD b/api/envoy/config/filter/http/csrf/v2/BUILD index 0d58b1ef6d43..af3a87b07c05 100644 --- a/api/envoy/config/filter/http/csrf/v2/BUILD +++ b/api/envoy/config/filter/http/csrf/v2/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "csrf", srcs = ["csrf.proto"], diff --git a/api/envoy/config/filter/http/csrf/v2/csrf.proto b/api/envoy/config/filter/http/csrf/v2/csrf.proto index 525ed118a71b..d4c35291e63d 100644 --- a/api/envoy/config/filter/http/csrf/v2/csrf.proto +++ b/api/envoy/config/filter/http/csrf/v2/csrf.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.csrf.v2; option java_outer_classname = "CsrfPolicyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.csrf.v2"; -option go_package = "v2"; import "envoy/api/v2/core/base.proto"; import "envoy/type/matcher/string.proto"; diff --git a/api/envoy/config/filter/http/csrf/v3alpha/BUILD b/api/envoy/config/filter/http/csrf/v3alpha/BUILD index b5da684c54b4..676559830c1f 100644 --- a/api/envoy/config/filter/http/csrf/v3alpha/BUILD +++ b/api/envoy/config/filter/http/csrf/v3alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "csrf", srcs = ["csrf.proto"], diff --git a/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto b/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto index 5eaa14c567d7..8fe68d5ea2ef 100644 --- a/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto +++ b/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.csrf.v3alpha; option java_outer_classname = "CsrfPolicyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.csrf.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/base.proto"; import "envoy/type/matcher/string.proto"; diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD index 4fd1d84399fa..15d184377ef7 100644 --- a/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/common/dynamic_forward_proxy/v2alpha:pkg"], +) + api_proto_library_internal( name = "dynamic_forward_proxy", srcs = ["dynamic_forward_proxy.proto"], diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto index 631363a6d95e..c315ddb46515 100644 --- a/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v2alpha/dynamic_forward_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.dynamic_forward_proxy.v2alpha; option java_outer_classname = "DynamicForwardProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.dynamic_forward_proxy.v2alpha"; -option go_package = "v2alpha"; import "envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto"; diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/BUILD b/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/BUILD index f09166ba8129..c06227674a08 100644 --- a/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/BUILD +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/common/dynamic_forward_proxy/v3alpha:pkg"], +) + api_proto_library_internal( name = "dynamic_forward_proxy", srcs = ["dynamic_forward_proxy.proto"], diff --git a/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/dynamic_forward_proxy.proto b/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/dynamic_forward_proxy.proto index 0fab44d63db5..f60aaae89e2e 100644 --- a/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/dynamic_forward_proxy.proto +++ b/api/envoy/config/filter/http/dynamic_forward_proxy/v3alpha/dynamic_forward_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.dynamic_forward_proxy.v3alpha; option java_outer_classname = "DynamicForwardProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.dynamic_forward_proxy.v3alpha"; -option go_package = "v2alpha"; import "envoy/config/common/dynamic_forward_proxy/v3alpha/dns_cache.proto"; diff --git a/api/envoy/config/filter/http/ext_authz/v2/BUILD b/api/envoy/config/filter/http/ext_authz/v2/BUILD index b1d02437df04..10187f48bd2c 100644 --- a/api/envoy/config/filter/http/ext_authz/v2/BUILD +++ b/api/envoy/config/filter/http/ext_authz/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/type", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "ext_authz", srcs = ["ext_authz.proto"], diff --git a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto index de105eff3c80..e2922348b6a7 100644 --- a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto +++ b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.ext_authz.v2; option java_outer_classname = "ExtAuthzProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.ext_authz.v2"; -option go_package = "v2"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/core/grpc_service.proto"; diff --git a/api/envoy/config/filter/http/ext_authz/v3alpha/BUILD b/api/envoy/config/filter/http/ext_authz/v3alpha/BUILD index 39f9e44bb382..cb0d25a3eebf 100644 --- a/api/envoy/config/filter/http/ext_authz/v3alpha/BUILD +++ b/api/envoy/config/filter/http/ext_authz/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/type", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "ext_authz", srcs = ["ext_authz.proto"], diff --git a/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto index af6f3c4866e5..8cc48e36ffb0 100644 --- a/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto +++ b/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.ext_authz.v3alpha; option java_outer_classname = "ExtAuthzProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.ext_authz.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/base.proto"; import "envoy/api/v3alpha/core/grpc_service.proto"; diff --git a/api/envoy/config/filter/http/fault/v2/BUILD b/api/envoy/config/filter/http/fault/v2/BUILD index e561e88196b9..b169a0904860 100644 --- a/api/envoy/config/filter/http/fault/v2/BUILD +++ b/api/envoy/config/filter/http/fault/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/route:pkg", + "//envoy/config/filter/fault/v2:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "fault", srcs = ["fault.proto"], diff --git a/api/envoy/config/filter/http/fault/v2/fault.proto b/api/envoy/config/filter/http/fault/v2/fault.proto index 51ee24ac91a8..8256690837fc 100644 --- a/api/envoy/config/filter/http/fault/v2/fault.proto +++ b/api/envoy/config/filter/http/fault/v2/fault.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.fault.v2; option java_outer_classname = "FaultProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.fault.v2"; -option go_package = "v2"; import "envoy/api/v2/route/route.proto"; import "envoy/config/filter/fault/v2/fault.proto"; diff --git a/api/envoy/config/filter/http/fault/v3alpha/BUILD b/api/envoy/config/filter/http/fault/v3alpha/BUILD index 1fd5632c088d..508e2d3c92d2 100644 --- a/api/envoy/config/filter/http/fault/v3alpha/BUILD +++ b/api/envoy/config/filter/http/fault/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/route:pkg", + "//envoy/config/filter/fault/v3alpha:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "fault", srcs = ["fault.proto"], diff --git a/api/envoy/config/filter/http/fault/v3alpha/fault.proto b/api/envoy/config/filter/http/fault/v3alpha/fault.proto index f654ec17f617..2189e4a4c131 100644 --- a/api/envoy/config/filter/http/fault/v3alpha/fault.proto +++ b/api/envoy/config/filter/http/fault/v3alpha/fault.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.fault.v3alpha; option java_outer_classname = "FaultProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.fault.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/route/route.proto"; import "envoy/config/filter/fault/v3alpha/fault.proto"; diff --git a/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/BUILD b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/BUILD index 7c1deb713c34..a88ba2443cad 100644 --- a/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/BUILD +++ b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library( name = "config", srcs = ["config.proto"], diff --git a/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/config.proto b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/config.proto index 0c33b6d077a1..b3b1fde5e1a2 100644 --- a/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/config.proto +++ b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v2alpha1/config.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.grpc_http1_reverse_bridge.v2alpha1; option java_outer_classname = "ConfigProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.grpc_http1_reverse_bridge.v2alpha1"; -option go_package = "v2"; import "validate/validate.proto"; diff --git a/api/envoy/config/filter/http/gzip/v2/BUILD b/api/envoy/config/filter/http/gzip/v2/BUILD index e34d73c51c21..a3f4b0af2a44 100644 --- a/api/envoy/config/filter/http/gzip/v2/BUILD +++ b/api/envoy/config/filter/http/gzip/v2/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "gzip", srcs = ["gzip.proto"], diff --git a/api/envoy/config/filter/http/gzip/v2/gzip.proto b/api/envoy/config/filter/http/gzip/v2/gzip.proto index fb6b8878e652..ec512c495dc8 100644 --- a/api/envoy/config/filter/http/gzip/v2/gzip.proto +++ b/api/envoy/config/filter/http/gzip/v2/gzip.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.gzip.v2; option java_outer_classname = "GzipProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.gzip.v2"; -option go_package = "v2"; import "google/protobuf/wrappers.proto"; diff --git a/api/envoy/config/filter/http/gzip/v3alpha/BUILD b/api/envoy/config/filter/http/gzip/v3alpha/BUILD index e34d73c51c21..a3f4b0af2a44 100644 --- a/api/envoy/config/filter/http/gzip/v3alpha/BUILD +++ b/api/envoy/config/filter/http/gzip/v3alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "gzip", srcs = ["gzip.proto"], diff --git a/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto b/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto index 5b5c6d6d1df7..d7afb89116c2 100644 --- a/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto +++ b/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.gzip.v3alpha; option java_outer_classname = "GzipProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.gzip.v3alpha"; -option go_package = "v2"; import "google/protobuf/wrappers.proto"; diff --git a/api/envoy/config/filter/http/header_to_metadata/v2/BUILD b/api/envoy/config/filter/http/header_to_metadata/v2/BUILD index 3f8503acbe65..cfd34fcf2b08 100644 --- a/api/envoy/config/filter/http/header_to_metadata/v2/BUILD +++ b/api/envoy/config/filter/http/header_to_metadata/v2/BUILD @@ -1,9 +1,10 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "header_to_metadata", srcs = ["header_to_metadata.proto"], - deps = [], ) diff --git a/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto b/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto index 5e70bbfce46f..345c5225edf1 100644 --- a/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto +++ b/api/envoy/config/filter/http/header_to_metadata/v2/header_to_metadata.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.header_to_metadata.v2; option java_outer_classname = "HeaderToMetadataProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.header_to_metadata.v2"; -option go_package = "v2"; import "validate/validate.proto"; diff --git a/api/envoy/config/filter/http/header_to_metadata/v3alpha/BUILD b/api/envoy/config/filter/http/header_to_metadata/v3alpha/BUILD index 3f8503acbe65..cfd34fcf2b08 100644 --- a/api/envoy/config/filter/http/header_to_metadata/v3alpha/BUILD +++ b/api/envoy/config/filter/http/header_to_metadata/v3alpha/BUILD @@ -1,9 +1,10 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "header_to_metadata", srcs = ["header_to_metadata.proto"], - deps = [], ) diff --git a/api/envoy/config/filter/http/header_to_metadata/v3alpha/header_to_metadata.proto b/api/envoy/config/filter/http/header_to_metadata/v3alpha/header_to_metadata.proto index 927574a5a721..c3811a00577a 100644 --- a/api/envoy/config/filter/http/header_to_metadata/v3alpha/header_to_metadata.proto +++ b/api/envoy/config/filter/http/header_to_metadata/v3alpha/header_to_metadata.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.header_to_metadata.v3alpha; option java_outer_classname = "HeaderToMetadataProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.header_to_metadata.v3alpha"; -option go_package = "v2"; import "validate/validate.proto"; diff --git a/api/envoy/config/filter/http/health_check/v2/BUILD b/api/envoy/config/filter/http/health_check/v2/BUILD index 9dc0af2df16f..8a995f1694af 100644 --- a/api/envoy/config/filter/http/health_check/v2/BUILD +++ b/api/envoy/config/filter/http/health_check/v2/BUILD @@ -1,21 +1,19 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 -api_proto_library_internal( - name = "health_check", - srcs = ["health_check.proto"], +api_proto_package( deps = [ - "//envoy/api/v2/route", - "//envoy/type:percent", + "//envoy/api/v2/route:pkg", + "//envoy/type", ], ) -api_go_proto_library( +api_proto_library_internal( name = "health_check", - proto = ":health_check", + srcs = ["health_check.proto"], deps = [ - "//envoy/api/v2/route:route_go_proto", - "//envoy/type:percent_go_proto", + "//envoy/api/v2/route", + "//envoy/type:percent", ], ) diff --git a/api/envoy/config/filter/http/health_check/v2/health_check.proto b/api/envoy/config/filter/http/health_check/v2/health_check.proto index bc8433732d72..2aa6d4191596 100644 --- a/api/envoy/config/filter/http/health_check/v2/health_check.proto +++ b/api/envoy/config/filter/http/health_check/v2/health_check.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.health_check.v2; option java_outer_classname = "HealthCheckProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.health_check.v2"; -option go_package = "v2"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; diff --git a/api/envoy/config/filter/http/health_check/v3alpha/BUILD b/api/envoy/config/filter/http/health_check/v3alpha/BUILD index 89e6eb3af702..b583685750da 100644 --- a/api/envoy/config/filter/http/health_check/v3alpha/BUILD +++ b/api/envoy/config/filter/http/health_check/v3alpha/BUILD @@ -1,21 +1,19 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 -api_proto_library_internal( - name = "health_check", - srcs = ["health_check.proto"], +api_proto_package( deps = [ - "//envoy/api/v3alpha/route", - "//envoy/type:percent", + "//envoy/api/v3alpha/route:pkg", + "//envoy/type", ], ) -api_go_proto_library( +api_proto_library_internal( name = "health_check", - proto = ":health_check", + srcs = ["health_check.proto"], deps = [ - "//envoy/api/v3alpha/route:route_go_proto", - "//envoy/type:percent_go_proto", + "//envoy/api/v3alpha/route", + "//envoy/type:percent", ], ) diff --git a/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto b/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto index 31fcdfffaa80..ecbb8e507851 100644 --- a/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto +++ b/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.health_check.v3alpha; option java_outer_classname = "HealthCheckProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.health_check.v3alpha"; -option go_package = "v2"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; diff --git a/api/envoy/config/filter/http/ip_tagging/v2/BUILD b/api/envoy/config/filter/http/ip_tagging/v2/BUILD index 4c7001972e25..b318ae58f381 100644 --- a/api/envoy/config/filter/http/ip_tagging/v2/BUILD +++ b/api/envoy/config/filter/http/ip_tagging/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library_internal( name = "ip_tagging", srcs = ["ip_tagging.proto"], diff --git a/api/envoy/config/filter/http/ip_tagging/v2/ip_tagging.proto b/api/envoy/config/filter/http/ip_tagging/v2/ip_tagging.proto index 4f5da60150f3..92ec469c62ad 100644 --- a/api/envoy/config/filter/http/ip_tagging/v2/ip_tagging.proto +++ b/api/envoy/config/filter/http/ip_tagging/v2/ip_tagging.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.ip_tagging.v2; option java_outer_classname = "IpTaggingProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.ip_tagging.v2"; -option go_package = "v2"; import "envoy/api/v2/core/address.proto"; diff --git a/api/envoy/config/filter/http/ip_tagging/v3alpha/BUILD b/api/envoy/config/filter/http/ip_tagging/v3alpha/BUILD index 5b34fcd9c458..a05f0fd96bb0 100644 --- a/api/envoy/config/filter/http/ip_tagging/v3alpha/BUILD +++ b/api/envoy/config/filter/http/ip_tagging/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library_internal( name = "ip_tagging", srcs = ["ip_tagging.proto"], diff --git a/api/envoy/config/filter/http/ip_tagging/v3alpha/ip_tagging.proto b/api/envoy/config/filter/http/ip_tagging/v3alpha/ip_tagging.proto index e305800a5fc8..de7871d9e701 100644 --- a/api/envoy/config/filter/http/ip_tagging/v3alpha/ip_tagging.proto +++ b/api/envoy/config/filter/http/ip_tagging/v3alpha/ip_tagging.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.ip_tagging.v3alpha; option java_outer_classname = "IpTaggingProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.ip_tagging.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/address.proto"; diff --git a/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD b/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD index e48aa582676c..80b4345f6151 100644 --- a/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD +++ b/api/envoy/config/filter/http/jwt_authn/v2alpha/BUILD @@ -1,6 +1,13 @@ licenses(["notice"]) # Apache 2 -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/api/v2/route:pkg", + ], +) api_proto_library_internal( name = "jwt_authn", @@ -11,13 +18,3 @@ api_proto_library_internal( "//envoy/api/v2/route", ], ) - -api_go_proto_library( - name = "jwt_authn", - proto = ":jwt_authn", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:http_uri_go_proto", - "//envoy/api/v2/route:route_go_proto", - ], -) diff --git a/api/envoy/config/filter/http/jwt_authn/v3alpha/BUILD b/api/envoy/config/filter/http/jwt_authn/v3alpha/BUILD index 2970da93f467..ea5d0d17b16a 100644 --- a/api/envoy/config/filter/http/jwt_authn/v3alpha/BUILD +++ b/api/envoy/config/filter/http/jwt_authn/v3alpha/BUILD @@ -1,6 +1,13 @@ licenses(["notice"]) # Apache 2 -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/route:pkg", + ], +) api_proto_library_internal( name = "jwt_authn", @@ -11,13 +18,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/route", ], ) - -api_go_proto_library( - name = "jwt_authn", - proto = ":jwt_authn", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:http_uri_go_proto", - "//envoy/api/v3alpha/route:route_go_proto", - ], -) diff --git a/api/envoy/config/filter/http/lua/v2/BUILD b/api/envoy/config/filter/http/lua/v2/BUILD index 6daf0c82f174..7aaf74617c96 100644 --- a/api/envoy/config/filter/http/lua/v2/BUILD +++ b/api/envoy/config/filter/http/lua/v2/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "lua", srcs = ["lua.proto"], diff --git a/api/envoy/config/filter/http/lua/v2/lua.proto b/api/envoy/config/filter/http/lua/v2/lua.proto index f29bcdbe89ef..6fc7fabc6be3 100644 --- a/api/envoy/config/filter/http/lua/v2/lua.proto +++ b/api/envoy/config/filter/http/lua/v2/lua.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.lua.v2; option java_outer_classname = "LuaProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.lua.v2"; -option go_package = "v2"; import "validate/validate.proto"; diff --git a/api/envoy/config/filter/http/lua/v3alpha/BUILD b/api/envoy/config/filter/http/lua/v3alpha/BUILD index 6daf0c82f174..7aaf74617c96 100644 --- a/api/envoy/config/filter/http/lua/v3alpha/BUILD +++ b/api/envoy/config/filter/http/lua/v3alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "lua", srcs = ["lua.proto"], diff --git a/api/envoy/config/filter/http/lua/v3alpha/lua.proto b/api/envoy/config/filter/http/lua/v3alpha/lua.proto index ff586ca2429e..934a592678a4 100644 --- a/api/envoy/config/filter/http/lua/v3alpha/lua.proto +++ b/api/envoy/config/filter/http/lua/v3alpha/lua.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.lua.v3alpha; option java_outer_classname = "LuaProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.lua.v3alpha"; -option go_package = "v2"; import "validate/validate.proto"; diff --git a/api/envoy/config/filter/http/original_src/v2alpha1/BUILD b/api/envoy/config/filter/http/original_src/v2alpha1/BUILD index e064545b21cd..a7435bb55cfc 100644 --- a/api/envoy/config/filter/http/original_src/v2alpha1/BUILD +++ b/api/envoy/config/filter/http/original_src/v2alpha1/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "original_src", srcs = ["original_src.proto"], diff --git a/api/envoy/config/filter/http/original_src/v2alpha1/original_src.proto b/api/envoy/config/filter/http/original_src/v2alpha1/original_src.proto index 32f37a8c48f0..5c09b860fc5c 100644 --- a/api/envoy/config/filter/http/original_src/v2alpha1/original_src.proto +++ b/api/envoy/config/filter/http/original_src/v2alpha1/original_src.proto @@ -6,8 +6,6 @@ option java_outer_classname = "OriginalSrcProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.original_src.v2alpha1"; -option go_package = "v2alpha1"; - import "validate/validate.proto"; // [#protodoc-title: Original Src Filter] diff --git a/api/envoy/config/filter/http/rate_limit/v2/BUILD b/api/envoy/config/filter/http/rate_limit/v2/BUILD index d8fb8e72ffec..4a6d451da981 100644 --- a/api/envoy/config/filter/http/rate_limit/v2/BUILD +++ b/api/envoy/config/filter/http/rate_limit/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/ratelimit/v2:pkg"], +) + api_proto_library_internal( name = "rate_limit", srcs = ["rate_limit.proto"], diff --git a/api/envoy/config/filter/http/rate_limit/v2/rate_limit.proto b/api/envoy/config/filter/http/rate_limit/v2/rate_limit.proto index 9d93e4a255bd..1ad3c4c36d01 100644 --- a/api/envoy/config/filter/http/rate_limit/v2/rate_limit.proto +++ b/api/envoy/config/filter/http/rate_limit/v2/rate_limit.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.rate_limit.v2; option java_outer_classname = "RateLimitProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.rate_limit.v2"; -option go_package = "v2"; import "envoy/config/ratelimit/v2/rls.proto"; diff --git a/api/envoy/config/filter/http/rate_limit/v3alpha/BUILD b/api/envoy/config/filter/http/rate_limit/v3alpha/BUILD index e131d3a92263..7060f7e9ce38 100644 --- a/api/envoy/config/filter/http/rate_limit/v3alpha/BUILD +++ b/api/envoy/config/filter/http/rate_limit/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/ratelimit/v3alpha:pkg"], +) + api_proto_library_internal( name = "rate_limit", srcs = ["rate_limit.proto"], diff --git a/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto b/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto index 69e8d389bc4b..427d22a6c1c6 100644 --- a/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto +++ b/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.rate_limit.v3alpha; option java_outer_classname = "RateLimitProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.rate_limit.v3alpha"; -option go_package = "v2"; import "envoy/config/ratelimit/v3alpha/rls.proto"; diff --git a/api/envoy/config/filter/http/rbac/v2/BUILD b/api/envoy/config/filter/http/rbac/v2/BUILD index 6182fe26748a..ca9aa2ca410c 100644 --- a/api/envoy/config/filter/http/rbac/v2/BUILD +++ b/api/envoy/config/filter/http/rbac/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/rbac/v2:pkg"], +) + api_proto_library_internal( name = "rbac", srcs = ["rbac.proto"], diff --git a/api/envoy/config/filter/http/rbac/v2/rbac.proto b/api/envoy/config/filter/http/rbac/v2/rbac.proto index 0a75d9590fa5..611cdc6ccbed 100644 --- a/api/envoy/config/filter/http/rbac/v2/rbac.proto +++ b/api/envoy/config/filter/http/rbac/v2/rbac.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.rbac.v2; option java_outer_classname = "RbacProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.rbac.v2"; -option go_package = "v2"; import "envoy/config/rbac/v2/rbac.proto"; diff --git a/api/envoy/config/filter/http/rbac/v3alpha/BUILD b/api/envoy/config/filter/http/rbac/v3alpha/BUILD index a6ee42cf7893..1e4d51b50453 100644 --- a/api/envoy/config/filter/http/rbac/v3alpha/BUILD +++ b/api/envoy/config/filter/http/rbac/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/rbac/v3alpha:pkg"], +) + api_proto_library_internal( name = "rbac", srcs = ["rbac.proto"], diff --git a/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto b/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto index 8ec8989652aa..47160ffa9e3c 100644 --- a/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto +++ b/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.rbac.v3alpha; option java_outer_classname = "RbacProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.rbac.v3alpha"; -option go_package = "v2"; import "envoy/config/rbac/v3alpha/rbac.proto"; diff --git a/api/envoy/config/filter/http/router/v2/BUILD b/api/envoy/config/filter/http/router/v2/BUILD index 7a80299a2cf7..9ddaf54b2845 100644 --- a/api/envoy/config/filter/http/router/v2/BUILD +++ b/api/envoy/config/filter/http/router/v2/BUILD @@ -1,15 +1,13 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/filter/accesslog/v2:pkg"], +) + api_proto_library_internal( name = "router", srcs = ["router.proto"], deps = ["//envoy/config/filter/accesslog/v2:accesslog"], ) - -api_go_proto_library( - name = "router", - proto = ":router", - deps = ["//envoy/config/filter/accesslog/v2:accesslog_go_proto"], -) diff --git a/api/envoy/config/filter/http/router/v2/router.proto b/api/envoy/config/filter/http/router/v2/router.proto index e77675673357..fd0cadec9631 100644 --- a/api/envoy/config/filter/http/router/v2/router.proto +++ b/api/envoy/config/filter/http/router/v2/router.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.router.v2; option java_outer_classname = "RouterProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.router.v2"; -option go_package = "v2"; import "envoy/config/filter/accesslog/v2/accesslog.proto"; diff --git a/api/envoy/config/filter/http/router/v3alpha/BUILD b/api/envoy/config/filter/http/router/v3alpha/BUILD index f0b6c100d445..d68a0ac2c2ee 100644 --- a/api/envoy/config/filter/http/router/v3alpha/BUILD +++ b/api/envoy/config/filter/http/router/v3alpha/BUILD @@ -1,15 +1,13 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/filter/accesslog/v3alpha:pkg"], +) + api_proto_library_internal( name = "router", srcs = ["router.proto"], deps = ["//envoy/config/filter/accesslog/v3alpha:accesslog"], ) - -api_go_proto_library( - name = "router", - proto = ":router", - deps = ["//envoy/config/filter/accesslog/v3alpha:accesslog_go_proto"], -) diff --git a/api/envoy/config/filter/http/router/v3alpha/router.proto b/api/envoy/config/filter/http/router/v3alpha/router.proto index 92efe315c6ff..a4ceae7dc1f7 100644 --- a/api/envoy/config/filter/http/router/v3alpha/router.proto +++ b/api/envoy/config/filter/http/router/v3alpha/router.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.router.v3alpha; option java_outer_classname = "RouterProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.router.v3alpha"; -option go_package = "v2"; import "envoy/config/filter/accesslog/v3alpha/accesslog.proto"; diff --git a/api/envoy/config/filter/http/squash/v2/BUILD b/api/envoy/config/filter/http/squash/v2/BUILD index 86bd4e8cfb65..2a0c1c8e30fa 100644 --- a/api/envoy/config/filter/http/squash/v2/BUILD +++ b/api/envoy/config/filter/http/squash/v2/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "squash", srcs = ["squash.proto"], diff --git a/api/envoy/config/filter/http/squash/v2/squash.proto b/api/envoy/config/filter/http/squash/v2/squash.proto index 006af4380d41..2f3a2e21cdd2 100644 --- a/api/envoy/config/filter/http/squash/v2/squash.proto +++ b/api/envoy/config/filter/http/squash/v2/squash.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.squash.v2; option java_outer_classname = "SquashProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.squash.v2"; -option go_package = "v2"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; diff --git a/api/envoy/config/filter/http/squash/v3alpha/BUILD b/api/envoy/config/filter/http/squash/v3alpha/BUILD index 86bd4e8cfb65..2a0c1c8e30fa 100644 --- a/api/envoy/config/filter/http/squash/v3alpha/BUILD +++ b/api/envoy/config/filter/http/squash/v3alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "squash", srcs = ["squash.proto"], diff --git a/api/envoy/config/filter/http/squash/v3alpha/squash.proto b/api/envoy/config/filter/http/squash/v3alpha/squash.proto index 43a62af98c1c..24236def872c 100644 --- a/api/envoy/config/filter/http/squash/v3alpha/squash.proto +++ b/api/envoy/config/filter/http/squash/v3alpha/squash.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.squash.v3alpha; option java_outer_classname = "SquashProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.squash.v3alpha"; -option go_package = "v2"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; diff --git a/api/envoy/config/filter/http/tap/v2alpha/BUILD b/api/envoy/config/filter/http/tap/v2alpha/BUILD index f84625a7da73..0949dad0c6ac 100644 --- a/api/envoy/config/filter/http/tap/v2alpha/BUILD +++ b/api/envoy/config/filter/http/tap/v2alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/common/tap/v2alpha:pkg"], +) + api_proto_library_internal( name = "tap", srcs = ["tap.proto"], diff --git a/api/envoy/config/filter/http/tap/v3alpha/BUILD b/api/envoy/config/filter/http/tap/v3alpha/BUILD index a2af23059be6..0535cfbc21ae 100644 --- a/api/envoy/config/filter/http/tap/v3alpha/BUILD +++ b/api/envoy/config/filter/http/tap/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/common/tap/v3alpha:pkg"], +) + api_proto_library_internal( name = "tap", srcs = ["tap.proto"], diff --git a/api/envoy/config/filter/http/transcoder/v2/BUILD b/api/envoy/config/filter/http/transcoder/v2/BUILD index c1a845bcd96e..33a99a23a061 100644 --- a/api/envoy/config/filter/http/transcoder/v2/BUILD +++ b/api/envoy/config/filter/http/transcoder/v2/BUILD @@ -1,13 +1,10 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "transcoder", srcs = ["transcoder.proto"], ) - -api_go_proto_library( - name = "transcoder", - proto = ":transcoder", -) diff --git a/api/envoy/config/filter/http/transcoder/v2/transcoder.proto b/api/envoy/config/filter/http/transcoder/v2/transcoder.proto index 14f54124508d..85f837fa794f 100644 --- a/api/envoy/config/filter/http/transcoder/v2/transcoder.proto +++ b/api/envoy/config/filter/http/transcoder/v2/transcoder.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.transcoder.v2; option java_outer_classname = "TranscoderProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.transcoder.v2"; -option go_package = "v2"; import "validate/validate.proto"; diff --git a/api/envoy/config/filter/http/transcoder/v3alpha/BUILD b/api/envoy/config/filter/http/transcoder/v3alpha/BUILD index c1a845bcd96e..33a99a23a061 100644 --- a/api/envoy/config/filter/http/transcoder/v3alpha/BUILD +++ b/api/envoy/config/filter/http/transcoder/v3alpha/BUILD @@ -1,13 +1,10 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "transcoder", srcs = ["transcoder.proto"], ) - -api_go_proto_library( - name = "transcoder", - proto = ":transcoder", -) diff --git a/api/envoy/config/filter/http/transcoder/v3alpha/transcoder.proto b/api/envoy/config/filter/http/transcoder/v3alpha/transcoder.proto index 078ac52473ac..630ad245a8a6 100644 --- a/api/envoy/config/filter/http/transcoder/v3alpha/transcoder.proto +++ b/api/envoy/config/filter/http/transcoder/v3alpha/transcoder.proto @@ -5,7 +5,6 @@ package envoy.config.filter.http.transcoder.v3alpha; option java_outer_classname = "TranscoderProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.http.transcoder.v3alpha"; -option go_package = "v2"; import "validate/validate.proto"; diff --git a/api/envoy/config/filter/listener/original_src/v2alpha1/BUILD b/api/envoy/config/filter/listener/original_src/v2alpha1/BUILD index e064545b21cd..a7435bb55cfc 100644 --- a/api/envoy/config/filter/listener/original_src/v2alpha1/BUILD +++ b/api/envoy/config/filter/listener/original_src/v2alpha1/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "original_src", srcs = ["original_src.proto"], diff --git a/api/envoy/config/filter/listener/original_src/v2alpha1/original_src.proto b/api/envoy/config/filter/listener/original_src/v2alpha1/original_src.proto index aa38e1d3df0a..11f55a787fdf 100644 --- a/api/envoy/config/filter/listener/original_src/v2alpha1/original_src.proto +++ b/api/envoy/config/filter/listener/original_src/v2alpha1/original_src.proto @@ -6,8 +6,6 @@ option java_outer_classname = "OriginalSrcProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.listener.original_src.v2alpha1"; -option go_package = "v2alpha1"; - import "validate/validate.proto"; // [#protodoc-title: Original Src Filter] diff --git a/api/envoy/config/filter/network/client_ssl_auth/v2/BUILD b/api/envoy/config/filter/network/client_ssl_auth/v2/BUILD index dad2d7fea262..96b5e9d0d47c 100644 --- a/api/envoy/config/filter/network/client_ssl_auth/v2/BUILD +++ b/api/envoy/config/filter/network/client_ssl_auth/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library_internal( name = "client_ssl_auth", srcs = ["client_ssl_auth.proto"], diff --git a/api/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto b/api/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto index fe0a6a3800b8..6add30a59552 100644 --- a/api/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto +++ b/api/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.client_ssl_auth.v2; option java_outer_classname = "ClientSslAuthProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.client_ssl_auth.v2"; -option go_package = "v2"; import "envoy/api/v2/core/address.proto"; import "google/protobuf/duration.proto"; diff --git a/api/envoy/config/filter/network/client_ssl_auth/v3alpha/BUILD b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/BUILD index bece14103bbe..540d8b4aa1a4 100644 --- a/api/envoy/config/filter/network/client_ssl_auth/v3alpha/BUILD +++ b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library_internal( name = "client_ssl_auth", srcs = ["client_ssl_auth.proto"], diff --git a/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto index a0ea3bf0bfaa..821d63494742 100644 --- a/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto +++ b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.client_ssl_auth.v3alpha; option java_outer_classname = "ClientSslAuthProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.client_ssl_auth.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/address.proto"; import "google/protobuf/duration.proto"; diff --git a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/BUILD b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/BUILD index e3e83a704684..c6cee209c654 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/BUILD +++ b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/BUILD @@ -1,7 +1,16 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/api/v2/route:pkg", + "//envoy/type", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "dubbo_proxy", srcs = [ diff --git a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto index 5b0995ba0022..e9834b704ed3 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto +++ b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.dubbo_proxy.v2alpha1; option java_outer_classname = "DubboProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.dubbo_proxy.v2alpha1"; -option go_package = "v2"; import "envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto"; @@ -58,4 +57,4 @@ message DubboFilter { // Filter specific configuration which depends on the filter being // instantiated. See the supported filters for further documentation. google.protobuf.Any config = 2; -} \ No newline at end of file +} diff --git a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto index 39f16e1a9342..c852a7cf5e53 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto +++ b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.dubbo_proxy.v2alpha1; option java_outer_classname = "RouteProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.dubbo_proxy.v2alpha1"; -option go_package = "v2"; import "envoy/api/v2/route/route.proto"; import "envoy/type/matcher/string.proto"; diff --git a/api/envoy/config/filter/network/ext_authz/v2/BUILD b/api/envoy/config/filter/network/ext_authz/v2/BUILD index 96184437fa54..3bdae60659a1 100644 --- a/api/envoy/config/filter/network/ext_authz/v2/BUILD +++ b/api/envoy/config/filter/network/ext_authz/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library_internal( name = "ext_authz", srcs = ["ext_authz.proto"], diff --git a/api/envoy/config/filter/network/ext_authz/v2/ext_authz.proto b/api/envoy/config/filter/network/ext_authz/v2/ext_authz.proto index f9a2f351f79e..8d0a6c6ca246 100644 --- a/api/envoy/config/filter/network/ext_authz/v2/ext_authz.proto +++ b/api/envoy/config/filter/network/ext_authz/v2/ext_authz.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.ext_authz.v2; option java_outer_classname = "ExtAuthzProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.ext_authz.v2"; -option go_package = "v2"; import "envoy/api/v2/core/grpc_service.proto"; diff --git a/api/envoy/config/filter/network/ext_authz/v3alpha/BUILD b/api/envoy/config/filter/network/ext_authz/v3alpha/BUILD index 839724af13b4..58aa28906331 100644 --- a/api/envoy/config/filter/network/ext_authz/v3alpha/BUILD +++ b/api/envoy/config/filter/network/ext_authz/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library_internal( name = "ext_authz", srcs = ["ext_authz.proto"], diff --git a/api/envoy/config/filter/network/ext_authz/v3alpha/ext_authz.proto b/api/envoy/config/filter/network/ext_authz/v3alpha/ext_authz.proto index 99c0c7239753..c53b509fee79 100644 --- a/api/envoy/config/filter/network/ext_authz/v3alpha/ext_authz.proto +++ b/api/envoy/config/filter/network/ext_authz/v3alpha/ext_authz.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.ext_authz.v3alpha; option java_outer_classname = "ExtAuthzProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.ext_authz.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/grpc_service.proto"; diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/BUILD b/api/envoy/config/filter/network/http_connection_manager/v2/BUILD index 95d3811f426a..6a090f3a115d 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/BUILD +++ b/api/envoy/config/filter/network/http_connection_manager/v2/BUILD @@ -1,7 +1,16 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2", + "//envoy/api/v2/core", + "//envoy/config/filter/accesslog/v2:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "http_connection_manager", srcs = ["http_connection_manager.proto"], @@ -15,17 +24,3 @@ api_proto_library_internal( "//envoy/type:percent", ], ) - -api_go_proto_library( - name = "http_connection_manager", - proto = ":http_connection_manager", - deps = [ - "//envoy/api/v2:rds_go_grpc", - "//envoy/api/v2:srds_go_grpc", - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:config_source_go_proto", - "//envoy/api/v2/core:protocol_go_proto", - "//envoy/config/filter/accesslog/v2:accesslog_go_proto", - "//envoy/type:percent_go_proto", - ], -) diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index bf1195731b86..e20a91542b83 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.http_connection_manager.v2; option java_outer_classname = "HttpConnectionManagerProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.http_connection_manager.v2"; -option go_package = "v2"; import "envoy/api/v2/core/config_source.proto"; import "envoy/api/v2/core/protocol.proto"; diff --git a/api/envoy/config/filter/network/http_connection_manager/v3alpha/BUILD b/api/envoy/config/filter/network/http_connection_manager/v3alpha/BUILD index 300b3c8e671a..57e0528c2ea6 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v3alpha/BUILD +++ b/api/envoy/config/filter/network/http_connection_manager/v3alpha/BUILD @@ -1,7 +1,16 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha", + "//envoy/api/v3alpha/core", + "//envoy/config/filter/accesslog/v3alpha:pkg", + "//envoy/type", + ], +) + api_proto_library_internal( name = "http_connection_manager", srcs = ["http_connection_manager.proto"], @@ -15,17 +24,3 @@ api_proto_library_internal( "//envoy/type:percent", ], ) - -api_go_proto_library( - name = "http_connection_manager", - proto = ":http_connection_manager", - deps = [ - "//envoy/api/v3alpha:rds_go_grpc", - "//envoy/api/v3alpha:srds_go_grpc", - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:config_source_go_proto", - "//envoy/api/v3alpha/core:protocol_go_proto", - "//envoy/config/filter/accesslog/v3alpha:accesslog_go_proto", - "//envoy/type:percent_go_proto", - ], -) diff --git a/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto index 57e529b2164a..4102c7016f2d 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.http_connection_manager.v3alpha; option java_outer_classname = "HttpConnectionManagerProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.http_connection_manager.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/config_source.proto"; import "envoy/api/v3alpha/core/protocol.proto"; diff --git a/api/envoy/config/filter/network/mongo_proxy/v2/BUILD b/api/envoy/config/filter/network/mongo_proxy/v2/BUILD index 5535f010179d..59bad30ed94d 100644 --- a/api/envoy/config/filter/network/mongo_proxy/v2/BUILD +++ b/api/envoy/config/filter/network/mongo_proxy/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/filter/fault/v2:pkg"], +) + api_proto_library_internal( name = "mongo_proxy", srcs = ["mongo_proxy.proto"], diff --git a/api/envoy/config/filter/network/mongo_proxy/v2/mongo_proxy.proto b/api/envoy/config/filter/network/mongo_proxy/v2/mongo_proxy.proto index 0d3d67bf6654..46ef44c96b94 100644 --- a/api/envoy/config/filter/network/mongo_proxy/v2/mongo_proxy.proto +++ b/api/envoy/config/filter/network/mongo_proxy/v2/mongo_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.mongo_proxy.v2; option java_outer_classname = "MongoProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.mongo_proxy.v2"; -option go_package = "v2"; import "envoy/config/filter/fault/v2/fault.proto"; diff --git a/api/envoy/config/filter/network/mongo_proxy/v3alpha/BUILD b/api/envoy/config/filter/network/mongo_proxy/v3alpha/BUILD index a2c09e709030..67dca3bb139a 100644 --- a/api/envoy/config/filter/network/mongo_proxy/v3alpha/BUILD +++ b/api/envoy/config/filter/network/mongo_proxy/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/filter/fault/v3alpha:pkg"], +) + api_proto_library_internal( name = "mongo_proxy", srcs = ["mongo_proxy.proto"], diff --git a/api/envoy/config/filter/network/mongo_proxy/v3alpha/mongo_proxy.proto b/api/envoy/config/filter/network/mongo_proxy/v3alpha/mongo_proxy.proto index 9149b433e372..780483ccb4c8 100644 --- a/api/envoy/config/filter/network/mongo_proxy/v3alpha/mongo_proxy.proto +++ b/api/envoy/config/filter/network/mongo_proxy/v3alpha/mongo_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.mongo_proxy.v3alpha; option java_outer_classname = "MongoProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.mongo_proxy.v3alpha"; -option go_package = "v2"; import "envoy/config/filter/fault/v3alpha/fault.proto"; diff --git a/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD b/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD index fde664838c93..7f7da3af9276 100644 --- a/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD +++ b/api/envoy/config/filter/network/mysql_proxy/v1alpha1/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "mysql_proxy", srcs = ["mysql_proxy.proto"], diff --git a/api/envoy/config/filter/network/mysql_proxy/v1alpha1/mysql_proxy.proto b/api/envoy/config/filter/network/mysql_proxy/v1alpha1/mysql_proxy.proto index e4246c9314aa..dee014556360 100644 --- a/api/envoy/config/filter/network/mysql_proxy/v1alpha1/mysql_proxy.proto +++ b/api/envoy/config/filter/network/mysql_proxy/v1alpha1/mysql_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.mysql_proxy.v1alpha1; option java_outer_classname = "MysqlProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.mysql_proxy.v1alpha1"; -option go_package = "v1alpha1"; import "validate/validate.proto"; diff --git a/api/envoy/config/filter/network/rate_limit/v2/BUILD b/api/envoy/config/filter/network/rate_limit/v2/BUILD index 08d5db95b117..fcdcd0dfa5ef 100644 --- a/api/envoy/config/filter/network/rate_limit/v2/BUILD +++ b/api/envoy/config/filter/network/rate_limit/v2/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/ratelimit:pkg", + "//envoy/config/ratelimit/v2:pkg", + ], +) + api_proto_library_internal( name = "rate_limit", srcs = ["rate_limit.proto"], diff --git a/api/envoy/config/filter/network/rate_limit/v2/rate_limit.proto b/api/envoy/config/filter/network/rate_limit/v2/rate_limit.proto index 6a1b795580c8..9a8f2f02146d 100644 --- a/api/envoy/config/filter/network/rate_limit/v2/rate_limit.proto +++ b/api/envoy/config/filter/network/rate_limit/v2/rate_limit.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.rate_limit.v2; option java_outer_classname = "RateLimitProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.rate_limit.v2"; -option go_package = "v2"; import "envoy/api/v2/ratelimit/ratelimit.proto"; import "envoy/config/ratelimit/v2/rls.proto"; diff --git a/api/envoy/config/filter/network/rate_limit/v3alpha/BUILD b/api/envoy/config/filter/network/rate_limit/v3alpha/BUILD index 9dc17266721c..a13183b9eb75 100644 --- a/api/envoy/config/filter/network/rate_limit/v3alpha/BUILD +++ b/api/envoy/config/filter/network/rate_limit/v3alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/ratelimit:pkg", + "//envoy/config/ratelimit/v3alpha:pkg", + ], +) + api_proto_library_internal( name = "rate_limit", srcs = ["rate_limit.proto"], diff --git a/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto b/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto index a0edc98e561d..60e2d27aff6b 100644 --- a/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto +++ b/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.rate_limit.v3alpha; option java_outer_classname = "RateLimitProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.rate_limit.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/ratelimit/ratelimit.proto"; import "envoy/config/ratelimit/v3alpha/rls.proto"; diff --git a/api/envoy/config/filter/network/rbac/v2/BUILD b/api/envoy/config/filter/network/rbac/v2/BUILD index 6182fe26748a..ca9aa2ca410c 100644 --- a/api/envoy/config/filter/network/rbac/v2/BUILD +++ b/api/envoy/config/filter/network/rbac/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/rbac/v2:pkg"], +) + api_proto_library_internal( name = "rbac", srcs = ["rbac.proto"], diff --git a/api/envoy/config/filter/network/rbac/v2/rbac.proto b/api/envoy/config/filter/network/rbac/v2/rbac.proto index aea17f725ff4..c6c6fac41c57 100644 --- a/api/envoy/config/filter/network/rbac/v2/rbac.proto +++ b/api/envoy/config/filter/network/rbac/v2/rbac.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.rbac.v2; option java_outer_classname = "RbacProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.rbac.v2"; -option go_package = "v2"; import "envoy/config/rbac/v2/rbac.proto"; diff --git a/api/envoy/config/filter/network/rbac/v3alpha/BUILD b/api/envoy/config/filter/network/rbac/v3alpha/BUILD index a6ee42cf7893..1e4d51b50453 100644 --- a/api/envoy/config/filter/network/rbac/v3alpha/BUILD +++ b/api/envoy/config/filter/network/rbac/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/config/rbac/v3alpha:pkg"], +) + api_proto_library_internal( name = "rbac", srcs = ["rbac.proto"], diff --git a/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto b/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto index 5c2114cd6063..5faa5f5c087c 100644 --- a/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto +++ b/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.rbac.v3alpha; option java_outer_classname = "RbacProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.rbac.v3alpha"; -option go_package = "v2"; import "envoy/config/rbac/v3alpha/rbac.proto"; diff --git a/api/envoy/config/filter/network/redis_proxy/v2/BUILD b/api/envoy/config/filter/network/redis_proxy/v2/BUILD index 16cff613b38d..d23450a55d1e 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/BUILD +++ b/api/envoy/config/filter/network/redis_proxy/v2/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/type", + ], +) + api_proto_library_internal( name = "redis_proxy", srcs = ["redis_proxy.proto"], diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index 175e564dec3f..656cedf75025 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.redis_proxy.v2; option java_outer_classname = "RedisProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.redis_proxy.v2"; -option go_package = "v2"; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/config/filter/network/redis_proxy/v3alpha/BUILD b/api/envoy/config/filter/network/redis_proxy/v3alpha/BUILD index ef7cc5683f0c..4db47e3bb664 100644 --- a/api/envoy/config/filter/network/redis_proxy/v3alpha/BUILD +++ b/api/envoy/config/filter/network/redis_proxy/v3alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/type", + ], +) + api_proto_library_internal( name = "redis_proxy", srcs = ["redis_proxy.proto"], diff --git a/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto index 1bda0ab7c466..a690451f7947 100644 --- a/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.redis_proxy.v3alpha; option java_outer_classname = "RedisProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.redis_proxy.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/config/filter/network/tcp_proxy/v2/BUILD b/api/envoy/config/filter/network/tcp_proxy/v2/BUILD index e75ab7036b75..a0cc067086cc 100644 --- a/api/envoy/config/filter/network/tcp_proxy/v2/BUILD +++ b/api/envoy/config/filter/network/tcp_proxy/v2/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/config/filter/accesslog/v2:pkg", + ], +) + api_proto_library_internal( name = "tcp_proxy", srcs = ["tcp_proxy.proto"], @@ -11,13 +18,3 @@ api_proto_library_internal( "//envoy/config/filter/accesslog/v2:accesslog", ], ) - -api_go_proto_library( - name = "tcp_proxy", - proto = ":tcp_proxy", - deps = [ - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - "//envoy/config/filter/accesslog/v2:accesslog_go_proto", - ], -) diff --git a/api/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto b/api/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto index 62874fe1d45d..376c980fb244 100644 --- a/api/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto +++ b/api/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.tcp_proxy.v2; option java_outer_classname = "TcpProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.tcp_proxy.v2"; -option go_package = "v2"; import "envoy/config/filter/accesslog/v2/accesslog.proto"; import "envoy/api/v2/core/address.proto"; diff --git a/api/envoy/config/filter/network/tcp_proxy/v3alpha/BUILD b/api/envoy/config/filter/network/tcp_proxy/v3alpha/BUILD index a9ea8de3bf1e..305e06bc8bfb 100644 --- a/api/envoy/config/filter/network/tcp_proxy/v3alpha/BUILD +++ b/api/envoy/config/filter/network/tcp_proxy/v3alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/config/filter/accesslog/v3alpha:pkg", + ], +) + api_proto_library_internal( name = "tcp_proxy", srcs = ["tcp_proxy.proto"], @@ -11,13 +18,3 @@ api_proto_library_internal( "//envoy/config/filter/accesslog/v3alpha:accesslog", ], ) - -api_go_proto_library( - name = "tcp_proxy", - proto = ":tcp_proxy", - deps = [ - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/config/filter/accesslog/v3alpha:accesslog_go_proto", - ], -) diff --git a/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto b/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto index f2597a3ab361..4e04d7b352d1 100644 --- a/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto +++ b/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.tcp_proxy.v3alpha; option java_outer_classname = "TcpProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.tcp_proxy.v3alpha"; -option go_package = "v2"; import "envoy/config/filter/accesslog/v3alpha/accesslog.proto"; import "envoy/api/v3alpha/core/address.proto"; diff --git a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/BUILD b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/BUILD index f758f7f580f5..28a64a0a329e 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/BUILD +++ b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/api/v2/route:pkg", + ], +) + api_proto_library_internal( name = "thrift_proxy", srcs = [ diff --git a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto index dcd83f2f1a2a..5d230d4474cc 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto +++ b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.thrift_proxy.v2alpha1; option java_outer_classname = "RouteProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.thrift_proxy.v2alpha1"; -option go_package = "v2"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/route/route.proto"; diff --git a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto index 0be6c337037f..4cfe538798a2 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto +++ b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.thrift_proxy.v2alpha1; option java_outer_classname = "ThriftProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.thrift_proxy.v2alpha1"; -option go_package = "v2"; import "envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto"; diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD index 8719f5083f12..02594c24b8ae 100644 --- a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "zookeeper_proxy", srcs = ["zookeeper_proxy.proto"], diff --git a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto index 6a8afdd12ec0..72d09810ff0f 100644 --- a/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto +++ b/api/envoy/config/filter/network/zookeeper_proxy/v1alpha1/zookeeper_proxy.proto @@ -5,7 +5,6 @@ package envoy.config.filter.network.zookeeper_proxy.v1alpha1; option java_outer_classname = "ZookeeperProxyProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.network.zookeeper_proxy.v1alpha1"; -option go_package = "v1alpha1"; import "validate/validate.proto"; import "google/protobuf/wrappers.proto"; diff --git a/api/envoy/config/filter/thrift/rate_limit/v2alpha1/BUILD b/api/envoy/config/filter/thrift/rate_limit/v2alpha1/BUILD index 08d5db95b117..fcdcd0dfa5ef 100644 --- a/api/envoy/config/filter/thrift/rate_limit/v2alpha1/BUILD +++ b/api/envoy/config/filter/thrift/rate_limit/v2alpha1/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/ratelimit:pkg", + "//envoy/config/ratelimit/v2:pkg", + ], +) + api_proto_library_internal( name = "rate_limit", srcs = ["rate_limit.proto"], diff --git a/api/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto b/api/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto index 15a50d553f9b..743bdc7256b7 100644 --- a/api/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto +++ b/api/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto @@ -5,7 +5,6 @@ package envoy.config.filter.thrift.rate_limit.v2alpha1; option java_outer_classname = "RateLimitProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.thrift.rate_limit.v2alpha1"; -option go_package = "v2alpha1"; import "envoy/config/ratelimit/v2/rls.proto"; diff --git a/api/envoy/config/filter/thrift/router/v2alpha1/BUILD b/api/envoy/config/filter/thrift/router/v2alpha1/BUILD index 51c69c0d5b20..68bd8c126b80 100644 --- a/api/envoy/config/filter/thrift/router/v2alpha1/BUILD +++ b/api/envoy/config/filter/thrift/router/v2alpha1/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "router", srcs = ["router.proto"], diff --git a/api/envoy/config/filter/thrift/router/v2alpha1/router.proto b/api/envoy/config/filter/thrift/router/v2alpha1/router.proto index c515752c2a00..9c9383caf33f 100644 --- a/api/envoy/config/filter/thrift/router/v2alpha1/router.proto +++ b/api/envoy/config/filter/thrift/router/v2alpha1/router.proto @@ -5,7 +5,6 @@ package envoy.config.filter.thrift.router.v2alpha1; option java_outer_classname = "RouterProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.filter.thrift.router.v2alpha1"; -option go_package = "v2alpha1"; // [#protodoc-title: Router] // Thrift router :ref:`configuration overview `. diff --git a/api/envoy/config/grpc_credential/v2alpha/BUILD b/api/envoy/config/grpc_credential/v2alpha/BUILD index f299179ecb00..484aa5680d12 100644 --- a/api/envoy/config/grpc_credential/v2alpha/BUILD +++ b/api/envoy/config/grpc_credential/v2alpha/BUILD @@ -1,6 +1,10 @@ licenses(["notice"]) # Apache 2 -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +api_proto_package( + deps = ["//envoy/api/v2/core"], +) api_proto_library_internal( name = "file_based_metadata", @@ -8,20 +12,7 @@ api_proto_library_internal( deps = ["//envoy/api/v2/core:base"], ) -api_go_proto_library( - name = "file_based_metadata", - proto = ":file_based_metadata", - deps = [ - "//envoy/api/v2/core:base_go_proto", - ], -) - api_proto_library_internal( name = "aws_iam", srcs = ["aws_iam.proto"], ) - -api_go_proto_library( - name = "aws_iam", - proto = ":aws_iam", -) diff --git a/api/envoy/config/grpc_credential/v2alpha/aws_iam.proto b/api/envoy/config/grpc_credential/v2alpha/aws_iam.proto index 3689b80611f5..e7a7bf94cce6 100644 --- a/api/envoy/config/grpc_credential/v2alpha/aws_iam.proto +++ b/api/envoy/config/grpc_credential/v2alpha/aws_iam.proto @@ -8,7 +8,6 @@ package envoy.config.grpc_credential.v2alpha; option java_outer_classname = "AwsIamProto"; option java_package = "io.envoyproxy.envoy.config.grpc_credential.v2alpha"; option java_multiple_files = true; -option go_package = "v2alpha"; import "validate/validate.proto"; diff --git a/api/envoy/config/grpc_credential/v2alpha/file_based_metadata.proto b/api/envoy/config/grpc_credential/v2alpha/file_based_metadata.proto index c91c50e39a55..1746492fe261 100644 --- a/api/envoy/config/grpc_credential/v2alpha/file_based_metadata.proto +++ b/api/envoy/config/grpc_credential/v2alpha/file_based_metadata.proto @@ -8,7 +8,6 @@ package envoy.config.grpc_credential.v2alpha; option java_outer_classname = "FileBasedMetadataProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.grpc_credential.v2alpha"; -option go_package = "v2alpha"; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/config/grpc_credential/v3alpha/BUILD b/api/envoy/config/grpc_credential/v3alpha/BUILD index 2f6736732881..7c327f91f031 100644 --- a/api/envoy/config/grpc_credential/v3alpha/BUILD +++ b/api/envoy/config/grpc_credential/v3alpha/BUILD @@ -1,6 +1,10 @@ licenses(["notice"]) # Apache 2 -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) api_proto_library_internal( name = "file_based_metadata", @@ -8,20 +12,7 @@ api_proto_library_internal( deps = ["//envoy/api/v3alpha/core:base"], ) -api_go_proto_library( - name = "file_based_metadata", - proto = ":file_based_metadata", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - ], -) - api_proto_library_internal( name = "aws_iam", srcs = ["aws_iam.proto"], ) - -api_go_proto_library( - name = "aws_iam", - proto = ":aws_iam", -) diff --git a/api/envoy/config/grpc_credential/v3alpha/aws_iam.proto b/api/envoy/config/grpc_credential/v3alpha/aws_iam.proto index 33921db6d69a..29c9cf140a00 100644 --- a/api/envoy/config/grpc_credential/v3alpha/aws_iam.proto +++ b/api/envoy/config/grpc_credential/v3alpha/aws_iam.proto @@ -8,7 +8,6 @@ package envoy.config.grpc_credential.v3alpha; option java_outer_classname = "AwsIamProto"; option java_package = "io.envoyproxy.envoy.config.grpc_credential.v3alpha"; option java_multiple_files = true; -option go_package = "v2alpha"; import "validate/validate.proto"; diff --git a/api/envoy/config/grpc_credential/v3alpha/file_based_metadata.proto b/api/envoy/config/grpc_credential/v3alpha/file_based_metadata.proto index 2886921b3415..9bab390cc833 100644 --- a/api/envoy/config/grpc_credential/v3alpha/file_based_metadata.proto +++ b/api/envoy/config/grpc_credential/v3alpha/file_based_metadata.proto @@ -8,7 +8,6 @@ package envoy.config.grpc_credential.v3alpha; option java_outer_classname = "FileBasedMetadataProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.grpc_credential.v3alpha"; -option go_package = "v2alpha"; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/config/health_checker/redis/v2/BUILD b/api/envoy/config/health_checker/redis/v2/BUILD index 239d1f224fc6..f7b289b08f69 100644 --- a/api/envoy/config/health_checker/redis/v2/BUILD +++ b/api/envoy/config/health_checker/redis/v2/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "redis", srcs = ["redis.proto"], diff --git a/api/envoy/config/health_checker/redis/v2/redis.proto b/api/envoy/config/health_checker/redis/v2/redis.proto index 130454b5d406..8ab2de269a5f 100644 --- a/api/envoy/config/health_checker/redis/v2/redis.proto +++ b/api/envoy/config/health_checker/redis/v2/redis.proto @@ -5,7 +5,6 @@ package envoy.config.health_checker.redis.v2; option java_outer_classname = "RedisProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.health_checker.redis.v2"; -option go_package = "v2"; // [#protodoc-title: Redis] // Redis health checker :ref:`configuration overview `. diff --git a/api/envoy/config/health_checker/redis/v3alpha/BUILD b/api/envoy/config/health_checker/redis/v3alpha/BUILD index 239d1f224fc6..f7b289b08f69 100644 --- a/api/envoy/config/health_checker/redis/v3alpha/BUILD +++ b/api/envoy/config/health_checker/redis/v3alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "redis", srcs = ["redis.proto"], diff --git a/api/envoy/config/health_checker/redis/v3alpha/redis.proto b/api/envoy/config/health_checker/redis/v3alpha/redis.proto index 234da40d56ba..1409e9545f41 100644 --- a/api/envoy/config/health_checker/redis/v3alpha/redis.proto +++ b/api/envoy/config/health_checker/redis/v3alpha/redis.proto @@ -5,7 +5,6 @@ package envoy.config.health_checker.redis.v3alpha; option java_outer_classname = "RedisProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.health_checker.redis.v3alpha"; -option go_package = "v2"; // [#protodoc-title: Redis] // Redis health checker :ref:`configuration overview `. diff --git a/api/envoy/config/metrics/v2/BUILD b/api/envoy/config/metrics/v2/BUILD index 157b09c4d814..13ac8bdd9992 100644 --- a/api/envoy/config/metrics/v2/BUILD +++ b/api/envoy/config/metrics/v2/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "metrics_service", srcs = ["metrics_service.proto"], @@ -13,14 +20,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "metrics_service", - proto = ":metrics_service", - deps = [ - "//envoy/api/v2/core:grpc_service_go_proto", - ], -) - api_proto_library_internal( name = "stats", srcs = ["stats.proto"], @@ -32,12 +31,3 @@ api_proto_library_internal( "//envoy/type/matcher:string", ], ) - -api_go_proto_library( - name = "stats", - proto = ":stats", - deps = [ - "//envoy/api/v2/core:address_go_proto", - "//envoy/type/matcher:string_go_proto", - ], -) diff --git a/api/envoy/config/metrics/v2/stats.proto b/api/envoy/config/metrics/v2/stats.proto index 08172180b545..fea8b9b0f878 100644 --- a/api/envoy/config/metrics/v2/stats.proto +++ b/api/envoy/config/metrics/v2/stats.proto @@ -8,7 +8,6 @@ package envoy.config.metrics.v2; option java_outer_classname = "StatsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.metrics.v2"; -option go_package = "v2"; import "envoy/api/v2/core/address.proto"; import "envoy/type/matcher/string.proto"; diff --git a/api/envoy/config/metrics/v3alpha/BUILD b/api/envoy/config/metrics/v3alpha/BUILD index 39d0b79654d0..399ec444208d 100644 --- a/api/envoy/config/metrics/v3alpha/BUILD +++ b/api/envoy/config/metrics/v3alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/type/matcher", + ], +) + api_proto_library_internal( name = "metrics_service", srcs = ["metrics_service.proto"], @@ -13,14 +20,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "metrics_service", - proto = ":metrics_service", - deps = [ - "//envoy/api/v3alpha/core:grpc_service_go_proto", - ], -) - api_proto_library_internal( name = "stats", srcs = ["stats.proto"], @@ -32,12 +31,3 @@ api_proto_library_internal( "//envoy/type/matcher:string", ], ) - -api_go_proto_library( - name = "stats", - proto = ":stats", - deps = [ - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/type/matcher:string_go_proto", - ], -) diff --git a/api/envoy/config/metrics/v3alpha/stats.proto b/api/envoy/config/metrics/v3alpha/stats.proto index 91324ed0ef61..afa4468b3444 100644 --- a/api/envoy/config/metrics/v3alpha/stats.proto +++ b/api/envoy/config/metrics/v3alpha/stats.proto @@ -8,7 +8,6 @@ package envoy.config.metrics.v3alpha; option java_outer_classname = "StatsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.metrics.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/address.proto"; import "envoy/type/matcher/string.proto"; diff --git a/api/envoy/config/overload/v2alpha/BUILD b/api/envoy/config/overload/v2alpha/BUILD index bfffb5639ca7..e247848d07a9 100644 --- a/api/envoy/config/overload/v2alpha/BUILD +++ b/api/envoy/config/overload/v2alpha/BUILD @@ -1,14 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "overload", srcs = ["overload.proto"], visibility = ["//visibility:public"], ) - -api_go_proto_library( - name = "overload", - proto = ":overload", -) diff --git a/api/envoy/config/overload/v2alpha/overload.proto b/api/envoy/config/overload/v2alpha/overload.proto index efdba5a09a72..e32764675cb5 100644 --- a/api/envoy/config/overload/v2alpha/overload.proto +++ b/api/envoy/config/overload/v2alpha/overload.proto @@ -5,7 +5,6 @@ package envoy.config.overload.v2alpha; option java_outer_classname = "OverloadProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.overload.v2alpha"; -option go_package = "v2alpha"; import "google/protobuf/any.proto"; import "google/protobuf/duration.proto"; diff --git a/api/envoy/config/overload/v3alpha/BUILD b/api/envoy/config/overload/v3alpha/BUILD index bfffb5639ca7..e247848d07a9 100644 --- a/api/envoy/config/overload/v3alpha/BUILD +++ b/api/envoy/config/overload/v3alpha/BUILD @@ -1,14 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "overload", srcs = ["overload.proto"], visibility = ["//visibility:public"], ) - -api_go_proto_library( - name = "overload", - proto = ":overload", -) diff --git a/api/envoy/config/overload/v3alpha/overload.proto b/api/envoy/config/overload/v3alpha/overload.proto index 474c7677002b..857b510e665a 100644 --- a/api/envoy/config/overload/v3alpha/overload.proto +++ b/api/envoy/config/overload/v3alpha/overload.proto @@ -5,7 +5,6 @@ package envoy.config.overload.v3alpha; option java_outer_classname = "OverloadProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.overload.v3alpha"; -option go_package = "v2alpha"; import "google/protobuf/any.proto"; import "google/protobuf/duration.proto"; diff --git a/api/envoy/config/ratelimit/v2/BUILD b/api/envoy/config/ratelimit/v2/BUILD index be3fc1c212bb..432f4b9592d3 100644 --- a/api/envoy/config/ratelimit/v2/BUILD +++ b/api/envoy/config/ratelimit/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library_internal( name = "rls", srcs = ["rls.proto"], @@ -10,11 +14,3 @@ api_proto_library_internal( "//envoy/api/v2/core:grpc_service", ], ) - -api_go_grpc_library( - name = "rls", - proto = ":rls", - deps = [ - "//envoy/api/v2/core:grpc_service_go_proto", - ], -) diff --git a/api/envoy/config/ratelimit/v2/rls.proto b/api/envoy/config/ratelimit/v2/rls.proto index 8f039b44efeb..55577d4ab013 100644 --- a/api/envoy/config/ratelimit/v2/rls.proto +++ b/api/envoy/config/ratelimit/v2/rls.proto @@ -5,7 +5,6 @@ package envoy.config.ratelimit.v2; option java_outer_classname = "RlsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.ratelimit.v2"; -option go_package = "v2"; import "envoy/api/v2/core/grpc_service.proto"; diff --git a/api/envoy/config/ratelimit/v3alpha/BUILD b/api/envoy/config/ratelimit/v3alpha/BUILD index 571a768dde4b..1d009164ba64 100644 --- a/api/envoy/config/ratelimit/v3alpha/BUILD +++ b/api/envoy/config/ratelimit/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library_internal( name = "rls", srcs = ["rls.proto"], @@ -10,11 +14,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/core:grpc_service", ], ) - -api_go_grpc_library( - name = "rls", - proto = ":rls", - deps = [ - "//envoy/api/v3alpha/core:grpc_service_go_proto", - ], -) diff --git a/api/envoy/config/ratelimit/v3alpha/rls.proto b/api/envoy/config/ratelimit/v3alpha/rls.proto index 67ac6479cd23..16d5a4ad7712 100644 --- a/api/envoy/config/ratelimit/v3alpha/rls.proto +++ b/api/envoy/config/ratelimit/v3alpha/rls.proto @@ -5,7 +5,6 @@ package envoy.config.ratelimit.v3alpha; option java_outer_classname = "RlsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.ratelimit.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/grpc_service.proto"; diff --git a/api/envoy/config/rbac/v2/BUILD b/api/envoy/config/rbac/v2/BUILD index fac50eb66f9b..18b1bb24f29d 100644 --- a/api/envoy/config/rbac/v2/BUILD +++ b/api/envoy/config/rbac/v2/BUILD @@ -1,6 +1,15 @@ licenses(["notice"]) # Apache 2 -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/api/v2/route:pkg", + "//envoy/type/matcher", + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", + ], +) api_proto_library_internal( name = "rbac", @@ -22,15 +31,3 @@ api_proto_library_internal( "//envoy/type/matcher:string", ], ) - -api_go_proto_library( - name = "rbac", - proto = ":rbac", - deps = [ - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/route:route_go_proto", - "//envoy/type/matcher:metadata_go_proto", - "//envoy/type/matcher:string_go_proto", - "@com_google_googleapis//google/api/expr/v1alpha1:cel_go_proto", - ], -) diff --git a/api/envoy/config/rbac/v2/rbac.proto b/api/envoy/config/rbac/v2/rbac.proto index 15554e561df4..34a062be535b 100644 --- a/api/envoy/config/rbac/v2/rbac.proto +++ b/api/envoy/config/rbac/v2/rbac.proto @@ -14,7 +14,6 @@ package envoy.config.rbac.v2; option java_outer_classname = "RbacProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.rbac.v2"; -option go_package = "v2"; option (gogoproto.stable_marshaler_all) = true; diff --git a/api/envoy/config/rbac/v3alpha/BUILD b/api/envoy/config/rbac/v3alpha/BUILD index 89f98c97d481..60200f034ea1 100644 --- a/api/envoy/config/rbac/v3alpha/BUILD +++ b/api/envoy/config/rbac/v3alpha/BUILD @@ -1,6 +1,15 @@ licenses(["notice"]) # Apache 2 -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/route:pkg", + "//envoy/type/matcher", + "@com_google_googleapis//google/api/expr/v1alpha1:syntax_proto", + ], +) api_proto_library_internal( name = "rbac", @@ -22,15 +31,3 @@ api_proto_library_internal( "//envoy/type/matcher:string", ], ) - -api_go_proto_library( - name = "rbac", - proto = ":rbac", - deps = [ - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/route:route_go_proto", - "//envoy/type/matcher:metadata_go_proto", - "//envoy/type/matcher:string_go_proto", - "@com_google_googleapis//google/api/expr/v1alpha1:cel_go_proto", - ], -) diff --git a/api/envoy/config/rbac/v3alpha/rbac.proto b/api/envoy/config/rbac/v3alpha/rbac.proto index d299c384da90..9087e745690d 100644 --- a/api/envoy/config/rbac/v3alpha/rbac.proto +++ b/api/envoy/config/rbac/v3alpha/rbac.proto @@ -14,7 +14,6 @@ package envoy.config.rbac.v3alpha; option java_outer_classname = "RbacProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.rbac.v3alpha"; -option go_package = "v2"; option (gogoproto.stable_marshaler_all) = true; diff --git a/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD index 363d90f11808..a5003e219c8a 100644 --- a/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD +++ b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "fixed_heap", srcs = ["fixed_heap.proto"], diff --git a/api/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto index f7efe0b5643a..110123e3c332 100644 --- a/api/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto +++ b/api/envoy/config/resource_monitor/fixed_heap/v2alpha/fixed_heap.proto @@ -5,7 +5,6 @@ package envoy.config.resource_monitor.fixed_heap.v2alpha; option java_outer_classname = "FixedHeapProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.resource_monitor.fixed_heap.v2alpha"; -option go_package = "v2alpha"; import "validate/validate.proto"; diff --git a/api/envoy/config/resource_monitor/fixed_heap/v3alpha/BUILD b/api/envoy/config/resource_monitor/fixed_heap/v3alpha/BUILD index 363d90f11808..a5003e219c8a 100644 --- a/api/envoy/config/resource_monitor/fixed_heap/v3alpha/BUILD +++ b/api/envoy/config/resource_monitor/fixed_heap/v3alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "fixed_heap", srcs = ["fixed_heap.proto"], diff --git a/api/envoy/config/resource_monitor/fixed_heap/v3alpha/fixed_heap.proto b/api/envoy/config/resource_monitor/fixed_heap/v3alpha/fixed_heap.proto index 2bc1baf85243..bc84ee992452 100644 --- a/api/envoy/config/resource_monitor/fixed_heap/v3alpha/fixed_heap.proto +++ b/api/envoy/config/resource_monitor/fixed_heap/v3alpha/fixed_heap.proto @@ -5,7 +5,6 @@ package envoy.config.resource_monitor.fixed_heap.v3alpha; option java_outer_classname = "FixedHeapProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.resource_monitor.fixed_heap.v3alpha"; -option go_package = "v2alpha"; import "validate/validate.proto"; diff --git a/api/envoy/config/resource_monitor/injected_resource/v2alpha/BUILD b/api/envoy/config/resource_monitor/injected_resource/v2alpha/BUILD index 10abf09e9ef8..3a1764216b00 100644 --- a/api/envoy/config/resource_monitor/injected_resource/v2alpha/BUILD +++ b/api/envoy/config/resource_monitor/injected_resource/v2alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "injected_resource", srcs = ["injected_resource.proto"], diff --git a/api/envoy/config/resource_monitor/injected_resource/v2alpha/injected_resource.proto b/api/envoy/config/resource_monitor/injected_resource/v2alpha/injected_resource.proto index cab704a4b64a..64c984fa0cb3 100644 --- a/api/envoy/config/resource_monitor/injected_resource/v2alpha/injected_resource.proto +++ b/api/envoy/config/resource_monitor/injected_resource/v2alpha/injected_resource.proto @@ -5,7 +5,6 @@ package envoy.config.resource_monitor.injected_resource.v2alpha; option java_outer_classname = "InjectedResourceProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.resource_monitor.injected_resource.v2alpha"; -option go_package = "v2alpha"; import "validate/validate.proto"; diff --git a/api/envoy/config/resource_monitor/injected_resource/v3alpha/BUILD b/api/envoy/config/resource_monitor/injected_resource/v3alpha/BUILD index 10abf09e9ef8..3a1764216b00 100644 --- a/api/envoy/config/resource_monitor/injected_resource/v3alpha/BUILD +++ b/api/envoy/config/resource_monitor/injected_resource/v3alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library_internal( name = "injected_resource", srcs = ["injected_resource.proto"], diff --git a/api/envoy/config/resource_monitor/injected_resource/v3alpha/injected_resource.proto b/api/envoy/config/resource_monitor/injected_resource/v3alpha/injected_resource.proto index f5b41ef165c8..555e15323f46 100644 --- a/api/envoy/config/resource_monitor/injected_resource/v3alpha/injected_resource.proto +++ b/api/envoy/config/resource_monitor/injected_resource/v3alpha/injected_resource.proto @@ -5,7 +5,6 @@ package envoy.config.resource_monitor.injected_resource.v3alpha; option java_outer_classname = "InjectedResourceProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.resource_monitor.injected_resource.v3alpha"; -option go_package = "v2alpha"; import "validate/validate.proto"; diff --git a/api/envoy/config/retry/previous_priorities/BUILD b/api/envoy/config/retry/previous_priorities/BUILD index 13a694af37d2..8140346d4747 100644 --- a/api/envoy/config/retry/previous_priorities/BUILD +++ b/api/envoy/config/retry/previous_priorities/BUILD @@ -1,6 +1,10 @@ licenses(["notice"]) # Apache 2 -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +api_proto_package( + deps = ["//envoy/api/v2/core"], +) api_proto_library_internal( name = "previous_priorities", diff --git a/api/envoy/config/trace/v2/BUILD b/api/envoy/config/trace/v2/BUILD index b00f63dafb45..f894a5289fd5 100644 --- a/api/envoy/config/trace/v2/BUILD +++ b/api/envoy/config/trace/v2/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", + ], +) + api_proto_library_internal( name = "trace", srcs = ["trace.proto"], @@ -13,12 +20,3 @@ api_proto_library_internal( "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", ], ) - -api_go_proto_library( - name = "trace", - proto = ":trace", - deps = [ - "//envoy/api/v2/core:grpc_service_go_proto", - "@opencensus_proto//opencensus/proto/trace/v1:trace_and_config_proto_go", - ], -) diff --git a/api/envoy/config/trace/v2/trace.proto b/api/envoy/config/trace/v2/trace.proto index 65c027cd73fe..43f5013b27f1 100644 --- a/api/envoy/config/trace/v2/trace.proto +++ b/api/envoy/config/trace/v2/trace.proto @@ -8,7 +8,6 @@ package envoy.config.trace.v2; option java_outer_classname = "TraceProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.trace.v2"; -option go_package = "v2"; import "envoy/api/v2/core/grpc_service.proto"; import "opencensus/proto/trace/v1/trace_config.proto"; diff --git a/api/envoy/config/trace/v3alpha/BUILD b/api/envoy/config/trace/v3alpha/BUILD index 72056b3ad4b6..97014ca68f6f 100644 --- a/api/envoy/config/trace/v3alpha/BUILD +++ b/api/envoy/config/trace/v3alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", + ], +) + api_proto_library_internal( name = "trace", srcs = ["trace.proto"], @@ -13,12 +20,3 @@ api_proto_library_internal( "@opencensus_proto//opencensus/proto/trace/v1:trace_config_proto", ], ) - -api_go_proto_library( - name = "trace", - proto = ":trace", - deps = [ - "//envoy/api/v3alpha/core:grpc_service_go_proto", - "@opencensus_proto//opencensus/proto/trace/v1:trace_and_config_proto_go", - ], -) diff --git a/api/envoy/config/trace/v3alpha/trace.proto b/api/envoy/config/trace/v3alpha/trace.proto index 2771c1b40f28..f98f1f708962 100644 --- a/api/envoy/config/trace/v3alpha/trace.proto +++ b/api/envoy/config/trace/v3alpha/trace.proto @@ -8,7 +8,6 @@ package envoy.config.trace.v3alpha; option java_outer_classname = "TraceProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.trace.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/grpc_service.proto"; import "opencensus/proto/trace/v1/trace_config.proto"; diff --git a/api/envoy/config/transport_socket/alts/v2alpha/BUILD b/api/envoy/config/transport_socket/alts/v2alpha/BUILD index 6cb181f202d2..eb247ae14b04 100644 --- a/api/envoy/config/transport_socket/alts/v2alpha/BUILD +++ b/api/envoy/config/transport_socket/alts/v2alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library( name = "alts", srcs = ["alts.proto"], diff --git a/api/envoy/config/transport_socket/alts/v2alpha/alts.proto b/api/envoy/config/transport_socket/alts/v2alpha/alts.proto index f5a9db64c0e4..ec294af17426 100644 --- a/api/envoy/config/transport_socket/alts/v2alpha/alts.proto +++ b/api/envoy/config/transport_socket/alts/v2alpha/alts.proto @@ -5,7 +5,6 @@ package envoy.config.transport_socket.alts.v2alpha; option java_outer_classname = "AltsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.transport_socket.alts.v2alpha"; -option go_package = "v2"; // [#protodoc-title: ALTS] diff --git a/api/envoy/config/transport_socket/alts/v3alpha/BUILD b/api/envoy/config/transport_socket/alts/v3alpha/BUILD index 7ffc03097000..4e6642283e3a 100644 --- a/api/envoy/config/transport_socket/alts/v3alpha/BUILD +++ b/api/envoy/config/transport_socket/alts/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library( name = "alts", srcs = ["alts.proto"], diff --git a/api/envoy/config/transport_socket/alts/v3alpha/alts.proto b/api/envoy/config/transport_socket/alts/v3alpha/alts.proto index 22684b862614..adec43c25cb8 100644 --- a/api/envoy/config/transport_socket/alts/v3alpha/alts.proto +++ b/api/envoy/config/transport_socket/alts/v3alpha/alts.proto @@ -5,7 +5,6 @@ package envoy.config.transport_socket.alts.v3alpha; option java_outer_classname = "AltsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.transport_socket.alts.v3alpha"; -option go_package = "v2"; // [#protodoc-title: ALTS] diff --git a/api/envoy/config/transport_socket/tap/v2alpha/BUILD b/api/envoy/config/transport_socket/tap/v2alpha/BUILD index 75810cd0c269..e18d4fc1c128 100644 --- a/api/envoy/config/transport_socket/tap/v2alpha/BUILD +++ b/api/envoy/config/transport_socket/tap/v2alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v2/core", + "//envoy/config/common/tap/v2alpha:pkg", + ], +) + api_proto_library_internal( name = "tap", srcs = ["tap.proto"], diff --git a/api/envoy/config/transport_socket/tap/v2alpha/tap.proto b/api/envoy/config/transport_socket/tap/v2alpha/tap.proto index 84918699ef97..e68b40dae530 100644 --- a/api/envoy/config/transport_socket/tap/v2alpha/tap.proto +++ b/api/envoy/config/transport_socket/tap/v2alpha/tap.proto @@ -5,7 +5,6 @@ package envoy.config.transport_socket.tap.v2alpha; option java_outer_classname = "TapProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.transport_socket.tap.v2alpha"; -option go_package = "v2"; // [#protodoc-title: Tap] diff --git a/api/envoy/config/transport_socket/tap/v3alpha/BUILD b/api/envoy/config/transport_socket/tap/v3alpha/BUILD index 8056ad6f17bb..0f24cca4c1a1 100644 --- a/api/envoy/config/transport_socket/tap/v3alpha/BUILD +++ b/api/envoy/config/transport_socket/tap/v3alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/config/common/tap/v3alpha:pkg", + ], +) + api_proto_library_internal( name = "tap", srcs = ["tap.proto"], diff --git a/api/envoy/config/transport_socket/tap/v3alpha/tap.proto b/api/envoy/config/transport_socket/tap/v3alpha/tap.proto index 1cca6814c803..21625e17ef9c 100644 --- a/api/envoy/config/transport_socket/tap/v3alpha/tap.proto +++ b/api/envoy/config/transport_socket/tap/v3alpha/tap.proto @@ -5,7 +5,6 @@ package envoy.config.transport_socket.tap.v3alpha; option java_outer_classname = "TapProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.transport_socket.tap.v3alpha"; -option go_package = "v2"; // [#protodoc-title: Tap] diff --git a/api/envoy/data/accesslog/v2/BUILD b/api/envoy/data/accesslog/v2/BUILD index d3ade88e922f..22c4c45ee847 100644 --- a/api/envoy/data/accesslog/v2/BUILD +++ b/api/envoy/data/accesslog/v2/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library_internal( name = "accesslog", srcs = ["accesslog.proto"], @@ -13,12 +17,3 @@ api_proto_library_internal( "//envoy/api/v2/core:base", ], ) - -api_go_proto_library( - name = "accesslog", - proto = ":accesslog", - deps = [ - "//envoy/api/v2/core:address_go_proto", - "//envoy/api/v2/core:base_go_proto", - ], -) diff --git a/api/envoy/data/accesslog/v3alpha/BUILD b/api/envoy/data/accesslog/v3alpha/BUILD index 30157958e7fe..e1fafc00343b 100644 --- a/api/envoy/data/accesslog/v3alpha/BUILD +++ b/api/envoy/data/accesslog/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library_internal( name = "accesslog", srcs = ["accesslog.proto"], @@ -13,12 +17,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/core:base", ], ) - -api_go_proto_library( - name = "accesslog", - proto = ":accesslog", - deps = [ - "//envoy/api/v3alpha/core:address_go_proto", - "//envoy/api/v3alpha/core:base_go_proto", - ], -) diff --git a/api/envoy/data/cluster/v2alpha/BUILD b/api/envoy/data/cluster/v2alpha/BUILD index 00edd8294b6f..4d921f4d97ac 100644 --- a/api/envoy/data/cluster/v2alpha/BUILD +++ b/api/envoy/data/cluster/v2alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library( name = "outlier_detection_event", srcs = ["outlier_detection_event.proto"], diff --git a/api/envoy/data/cluster/v3alpha/BUILD b/api/envoy/data/cluster/v3alpha/BUILD index 00edd8294b6f..4d921f4d97ac 100644 --- a/api/envoy/data/cluster/v3alpha/BUILD +++ b/api/envoy/data/cluster/v3alpha/BUILD @@ -1,7 +1,9 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package() + api_proto_library( name = "outlier_detection_event", srcs = ["outlier_detection_event.proto"], diff --git a/api/envoy/data/core/v2alpha/BUILD b/api/envoy/data/core/v2alpha/BUILD index 8320031d8466..331032348388 100644 --- a/api/envoy/data/core/v2alpha/BUILD +++ b/api/envoy/data/core/v2alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library( name = "health_check_event", srcs = ["health_check_event.proto"], diff --git a/api/envoy/data/core/v3alpha/BUILD b/api/envoy/data/core/v3alpha/BUILD index 9e82e3eb1731..6c44f3e4d79e 100644 --- a/api/envoy/data/core/v3alpha/BUILD +++ b/api/envoy/data/core/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library( name = "health_check_event", srcs = ["health_check_event.proto"], diff --git a/api/envoy/data/tap/v2alpha/BUILD b/api/envoy/data/tap/v2alpha/BUILD index 1b373eee86df..bf108c4792a1 100644 --- a/api/envoy/data/tap/v2alpha/BUILD +++ b/api/envoy/data/tap/v2alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v2/core"], +) + api_proto_library_internal( name = "common", srcs = ["common.proto"], diff --git a/api/envoy/data/tap/v2alpha/transport.proto b/api/envoy/data/tap/v2alpha/transport.proto index 3b8c244b9bae..c3a3d8b8eb86 100644 --- a/api/envoy/data/tap/v2alpha/transport.proto +++ b/api/envoy/data/tap/v2alpha/transport.proto @@ -9,7 +9,6 @@ package envoy.data.tap.v2alpha; option java_outer_classname = "TransportProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.data.tap.v2alpha"; -option go_package = "v2"; import "envoy/api/v2/core/address.proto"; import "envoy/data/tap/v2alpha/common.proto"; diff --git a/api/envoy/data/tap/v3alpha/BUILD b/api/envoy/data/tap/v3alpha/BUILD index ab9be74ce98a..33d151e333a9 100644 --- a/api/envoy/data/tap/v3alpha/BUILD +++ b/api/envoy/data/tap/v3alpha/BUILD @@ -1,7 +1,11 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + deps = ["//envoy/api/v3alpha/core"], +) + api_proto_library_internal( name = "common", srcs = ["common.proto"], diff --git a/api/envoy/data/tap/v3alpha/transport.proto b/api/envoy/data/tap/v3alpha/transport.proto index 3dfb0c6478ba..e35f036f5d82 100644 --- a/api/envoy/data/tap/v3alpha/transport.proto +++ b/api/envoy/data/tap/v3alpha/transport.proto @@ -9,7 +9,6 @@ package envoy.data.tap.v3alpha; option java_outer_classname = "TransportProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.data.tap.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/address.proto"; import "envoy/data/tap/v3alpha/common.proto"; diff --git a/api/envoy/service/accesslog/v2/BUILD b/api/envoy/service/accesslog/v2/BUILD index 1dad9447048d..d4f7c300361e 100644 --- a/api/envoy/service/accesslog/v2/BUILD +++ b/api/envoy/service/accesslog/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v2/core", + "//envoy/data/accesslog/v2:pkg", + ], +) + api_proto_library_internal( name = "als", srcs = ["als.proto"], @@ -12,12 +20,3 @@ api_proto_library_internal( "//envoy/data/accesslog/v2:accesslog", ], ) - -api_go_grpc_library( - name = "als", - proto = ":als", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "//envoy/data/accesslog/v2:accesslog_go_proto", - ], -) diff --git a/api/envoy/service/accesslog/v2/als.proto b/api/envoy/service/accesslog/v2/als.proto index 52788e0659c6..c06199a2b208 100644 --- a/api/envoy/service/accesslog/v2/als.proto +++ b/api/envoy/service/accesslog/v2/als.proto @@ -5,7 +5,6 @@ package envoy.service.accesslog.v2; option java_outer_classname = "AlsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.accesslog.v2"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/service/accesslog/v3alpha/BUILD b/api/envoy/service/accesslog/v3alpha/BUILD index 1a8eab975b56..0bb8716b82ab 100644 --- a/api/envoy/service/accesslog/v3alpha/BUILD +++ b/api/envoy/service/accesslog/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/data/accesslog/v3alpha:pkg", + ], +) + api_proto_library_internal( name = "als", srcs = ["als.proto"], @@ -12,12 +20,3 @@ api_proto_library_internal( "//envoy/data/accesslog/v3alpha:accesslog", ], ) - -api_go_grpc_library( - name = "als", - proto = ":als", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/data/accesslog/v3alpha:accesslog_go_proto", - ], -) diff --git a/api/envoy/service/accesslog/v3alpha/als.proto b/api/envoy/service/accesslog/v3alpha/als.proto index 092d4d17696c..ad05b823d1e5 100644 --- a/api/envoy/service/accesslog/v3alpha/als.proto +++ b/api/envoy/service/accesslog/v3alpha/als.proto @@ -5,7 +5,6 @@ package envoy.service.accesslog.v3alpha; option java_outer_classname = "AlsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.accesslog.v3alpha"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/service/auth/v2/BUILD b/api/envoy/service/auth/v2/BUILD index 57041668ddc8..91a4eeebbf13 100644 --- a/api/envoy/service/auth/v2/BUILD +++ b/api/envoy/service/auth/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v2/core", + "//envoy/type", + ], +) + api_proto_library_internal( name = "attribute_context", srcs = [ diff --git a/api/envoy/service/auth/v2/external_auth.proto b/api/envoy/service/auth/v2/external_auth.proto index 0f723c98e46c..8a3d4f1a629e 100644 --- a/api/envoy/service/auth/v2/external_auth.proto +++ b/api/envoy/service/auth/v2/external_auth.proto @@ -5,7 +5,6 @@ package envoy.service.auth.v2; option java_outer_classname = "ExternalAuthProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.auth.v2"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/service/auth/v2alpha/BUILD b/api/envoy/service/auth/v2alpha/BUILD index 1d9873a5ffa4..1940f4f2f885 100644 --- a/api/envoy/service/auth/v2alpha/BUILD +++ b/api/envoy/service/auth/v2alpha/BUILD @@ -1,7 +1,14 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/service/auth/v2:pkg", + ], +) + api_proto_library_internal( name = "external_auth", srcs = [ diff --git a/api/envoy/service/auth/v2alpha/external_auth.proto b/api/envoy/service/auth/v2alpha/external_auth.proto index bdf0d2e4853d..85e9c12c6afb 100644 --- a/api/envoy/service/auth/v2alpha/external_auth.proto +++ b/api/envoy/service/auth/v2alpha/external_auth.proto @@ -2,8 +2,6 @@ syntax = "proto3"; package envoy.service.auth.v2alpha; -option go_package = "v2alpha"; - option java_multiple_files = true; option java_generic_services = true; option java_outer_classname = "CertsProto"; diff --git a/api/envoy/service/auth/v3alpha/BUILD b/api/envoy/service/auth/v3alpha/BUILD index 6a335f88f949..f6a70cb5b9bd 100644 --- a/api/envoy/service/auth/v3alpha/BUILD +++ b/api/envoy/service/auth/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/type", + ], +) + api_proto_library_internal( name = "attribute_context", srcs = [ diff --git a/api/envoy/service/auth/v3alpha/external_auth.proto b/api/envoy/service/auth/v3alpha/external_auth.proto index 4b7e459a4436..0130040c1409 100644 --- a/api/envoy/service/auth/v3alpha/external_auth.proto +++ b/api/envoy/service/auth/v3alpha/external_auth.proto @@ -5,7 +5,6 @@ package envoy.service.auth.v3alpha; option java_outer_classname = "ExternalAuthProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.auth.v3alpha"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/service/discovery/v2/BUILD b/api/envoy/service/discovery/v2/BUILD index a9c2efd02fb9..13db2701c2a5 100644 --- a/api/envoy/service/discovery/v2/BUILD +++ b/api/envoy/service/discovery/v2/BUILD @@ -1,21 +1,22 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 -api_proto_library_internal( - name = "ads", - srcs = ["ads.proto"], - has_services = 1, +api_proto_package( + has_services = True, deps = [ - "//envoy/api/v2:discovery", + "//envoy/api/v2", + "//envoy/api/v2/core", + "//envoy/api/v2/endpoint:pkg", ], ) -api_go_grpc_library( +api_proto_library_internal( name = "ads", - proto = ":ads", + srcs = ["ads.proto"], + has_services = 1, deps = [ - "//envoy/api/v2:discovery_go_proto", + "//envoy/api/v2:discovery", ], ) @@ -30,16 +31,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "hds", - proto = ":hds", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:health_check_go_proto", - "//envoy/api/v2/endpoint:endpoint_go_proto", - ], -) - api_proto_library_internal( name = "sds", srcs = ["sds.proto"], @@ -49,14 +40,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "sds", - proto = ":sds", - deps = [ - "//envoy/api/v2:discovery_go_proto", - ], -) - api_proto_library_internal( name = "rtds", srcs = ["rtds.proto"], @@ -65,11 +48,3 @@ api_proto_library_internal( "//envoy/api/v2:discovery", ], ) - -api_go_grpc_library( - name = "rtds", - proto = ":rtds", - deps = [ - "//envoy/api/v2:discovery_go_proto", - ], -) diff --git a/api/envoy/service/discovery/v2/ads.proto b/api/envoy/service/discovery/v2/ads.proto index 6a9d044ab4bd..45a7407f0c44 100644 --- a/api/envoy/service/discovery/v2/ads.proto +++ b/api/envoy/service/discovery/v2/ads.proto @@ -5,7 +5,6 @@ package envoy.service.discovery.v2; option java_outer_classname = "AdsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.discovery.v2"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v2/discovery.proto"; diff --git a/api/envoy/service/discovery/v3alpha/BUILD b/api/envoy/service/discovery/v3alpha/BUILD index d34955c1cb5a..138186e6ea05 100644 --- a/api/envoy/service/discovery/v3alpha/BUILD +++ b/api/envoy/service/discovery/v3alpha/BUILD @@ -1,21 +1,22 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 -api_proto_library_internal( - name = "ads", - srcs = ["ads.proto"], - has_services = 1, +api_proto_package( + has_services = True, deps = [ - "//envoy/api/v3alpha:discovery", + "//envoy/api/v3alpha", + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/endpoint:pkg", ], ) -api_go_grpc_library( +api_proto_library_internal( name = "ads", - proto = ":ads", + srcs = ["ads.proto"], + has_services = 1, deps = [ - "//envoy/api/v3alpha:discovery_go_proto", + "//envoy/api/v3alpha:discovery", ], ) @@ -30,16 +31,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "hds", - proto = ":hds", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:health_check_go_proto", - "//envoy/api/v3alpha/endpoint:endpoint_go_proto", - ], -) - api_proto_library_internal( name = "sds", srcs = ["sds.proto"], @@ -49,14 +40,6 @@ api_proto_library_internal( ], ) -api_go_grpc_library( - name = "sds", - proto = ":sds", - deps = [ - "//envoy/api/v3alpha:discovery_go_proto", - ], -) - api_proto_library_internal( name = "rtds", srcs = ["rtds.proto"], @@ -65,11 +48,3 @@ api_proto_library_internal( "//envoy/api/v3alpha:discovery", ], ) - -api_go_grpc_library( - name = "rtds", - proto = ":rtds", - deps = [ - "//envoy/api/v3alpha:discovery_go_proto", - ], -) diff --git a/api/envoy/service/discovery/v3alpha/ads.proto b/api/envoy/service/discovery/v3alpha/ads.proto index d6b7897ba7a1..251c51301a16 100644 --- a/api/envoy/service/discovery/v3alpha/ads.proto +++ b/api/envoy/service/discovery/v3alpha/ads.proto @@ -5,7 +5,6 @@ package envoy.service.discovery.v3alpha; option java_outer_classname = "AdsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.discovery.v3alpha"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v3alpha/discovery.proto"; diff --git a/api/envoy/service/load_stats/v2/BUILD b/api/envoy/service/load_stats/v2/BUILD index f126ebcb1d44..af07d8aa101c 100644 --- a/api/envoy/service/load_stats/v2/BUILD +++ b/api/envoy/service/load_stats/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v2/core", + "//envoy/api/v2/endpoint:pkg", + ], +) + api_proto_library_internal( name = "lrs", srcs = ["lrs.proto"], @@ -11,12 +19,3 @@ api_proto_library_internal( "//envoy/api/v2/endpoint:load_report", ], ) - -api_go_grpc_library( - name = "lrs", - proto = ":lrs", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/endpoint:load_report_go_proto", - ], -) diff --git a/api/envoy/service/load_stats/v2/lrs.proto b/api/envoy/service/load_stats/v2/lrs.proto index 2fe95f3b6a90..d7029db0b5ea 100644 --- a/api/envoy/service/load_stats/v2/lrs.proto +++ b/api/envoy/service/load_stats/v2/lrs.proto @@ -5,7 +5,6 @@ package envoy.service.load_stats.v2; option java_outer_classname = "LrsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.load_stats.v2"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/service/load_stats/v3alpha/BUILD b/api/envoy/service/load_stats/v3alpha/BUILD index 42c7ce8438da..bc4ff2642c6d 100644 --- a/api/envoy/service/load_stats/v3alpha/BUILD +++ b/api/envoy/service/load_stats/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/endpoint:pkg", + ], +) + api_proto_library_internal( name = "lrs", srcs = ["lrs.proto"], @@ -11,12 +19,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/endpoint:load_report", ], ) - -api_go_grpc_library( - name = "lrs", - proto = ":lrs", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/endpoint:load_report_go_proto", - ], -) diff --git a/api/envoy/service/load_stats/v3alpha/lrs.proto b/api/envoy/service/load_stats/v3alpha/lrs.proto index 81058ed574a7..ec8adedbf2a8 100644 --- a/api/envoy/service/load_stats/v3alpha/lrs.proto +++ b/api/envoy/service/load_stats/v3alpha/lrs.proto @@ -5,7 +5,6 @@ package envoy.service.load_stats.v3alpha; option java_outer_classname = "LrsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.load_stats.v3alpha"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/service/metrics/v2/BUILD b/api/envoy/service/metrics/v2/BUILD index 7f3921ced629..091d40e7f8e5 100644 --- a/api/envoy/service/metrics/v2/BUILD +++ b/api/envoy/service/metrics/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v2/core", + "@prometheus_metrics_model//:client_model", + ], +) + api_proto_library_internal( name = "metrics_service", srcs = ["metrics_service.proto"], @@ -13,12 +21,3 @@ api_proto_library_internal( "@prometheus_metrics_model//:client_model", ], ) - -api_go_grpc_library( - name = "metrics_service", - proto = ":metrics_service", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "@prometheus_metrics_model//:client_model_go_proto", - ], -) diff --git a/api/envoy/service/metrics/v2/metrics_service.proto b/api/envoy/service/metrics/v2/metrics_service.proto index b70be3bdd9a1..10745ba665c0 100644 --- a/api/envoy/service/metrics/v2/metrics_service.proto +++ b/api/envoy/service/metrics/v2/metrics_service.proto @@ -5,7 +5,6 @@ package envoy.service.metrics.v2; option java_outer_classname = "MetricsServiceProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.metrics.v2"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/service/metrics/v3alpha/BUILD b/api/envoy/service/metrics/v3alpha/BUILD index 1f1bb553cd82..6053aac4f1be 100644 --- a/api/envoy/service/metrics/v3alpha/BUILD +++ b/api/envoy/service/metrics/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v3alpha/core", + "@prometheus_metrics_model//:client_model", + ], +) + api_proto_library_internal( name = "metrics_service", srcs = ["metrics_service.proto"], @@ -13,12 +21,3 @@ api_proto_library_internal( "@prometheus_metrics_model//:client_model", ], ) - -api_go_grpc_library( - name = "metrics_service", - proto = ":metrics_service", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "@prometheus_metrics_model//:client_model_go_proto", - ], -) diff --git a/api/envoy/service/metrics/v3alpha/metrics_service.proto b/api/envoy/service/metrics/v3alpha/metrics_service.proto index 9a5306553b18..bcf1caa28a2d 100644 --- a/api/envoy/service/metrics/v3alpha/metrics_service.proto +++ b/api/envoy/service/metrics/v3alpha/metrics_service.proto @@ -5,7 +5,6 @@ package envoy.service.metrics.v3alpha; option java_outer_classname = "MetricsServiceProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.metrics.v3alpha"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/service/ratelimit/v2/BUILD b/api/envoy/service/ratelimit/v2/BUILD index 24278fbebc1f..7bc5db7113e7 100644 --- a/api/envoy/service/ratelimit/v2/BUILD +++ b/api/envoy/service/ratelimit/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v2/core", + "//envoy/api/v2/ratelimit:pkg", + ], +) + api_proto_library_internal( name = "rls", srcs = ["rls.proto"], @@ -12,13 +20,3 @@ api_proto_library_internal( "//envoy/api/v2/ratelimit", ], ) - -api_go_grpc_library( - name = "rls", - proto = ":rls", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "//envoy/api/v2/core:grpc_service_go_proto", - "//envoy/api/v2/ratelimit:ratelimit_go_proto", - ], -) diff --git a/api/envoy/service/ratelimit/v2/rls.proto b/api/envoy/service/ratelimit/v2/rls.proto index 18b6b678e908..328bb547d630 100644 --- a/api/envoy/service/ratelimit/v2/rls.proto +++ b/api/envoy/service/ratelimit/v2/rls.proto @@ -5,7 +5,6 @@ package envoy.service.ratelimit.v2; option java_outer_classname = "RlsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.ratelimit.v2"; -option go_package = "v2"; import "envoy/api/v2/core/base.proto"; import "envoy/api/v2/ratelimit/ratelimit.proto"; diff --git a/api/envoy/service/ratelimit/v3alpha/BUILD b/api/envoy/service/ratelimit/v3alpha/BUILD index 19954c5bfcc9..965458beaec6 100644 --- a/api/envoy/service/ratelimit/v3alpha/BUILD +++ b/api/envoy/service/ratelimit/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/ratelimit:pkg", + ], +) + api_proto_library_internal( name = "rls", srcs = ["rls.proto"], @@ -12,13 +20,3 @@ api_proto_library_internal( "//envoy/api/v3alpha/ratelimit", ], ) - -api_go_grpc_library( - name = "rls", - proto = ":rls", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "//envoy/api/v3alpha/core:grpc_service_go_proto", - "//envoy/api/v3alpha/ratelimit:ratelimit_go_proto", - ], -) diff --git a/api/envoy/service/ratelimit/v3alpha/rls.proto b/api/envoy/service/ratelimit/v3alpha/rls.proto index 7bbd2e3ec183..57a3ee98de94 100644 --- a/api/envoy/service/ratelimit/v3alpha/rls.proto +++ b/api/envoy/service/ratelimit/v3alpha/rls.proto @@ -5,7 +5,6 @@ package envoy.service.ratelimit.v3alpha; option java_outer_classname = "RlsProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.ratelimit.v3alpha"; -option go_package = "v2"; import "envoy/api/v3alpha/core/base.proto"; import "envoy/api/v3alpha/ratelimit/ratelimit.proto"; diff --git a/api/envoy/service/tap/v2alpha/BUILD b/api/envoy/service/tap/v2alpha/BUILD index 63d11e80a755..621bf208d495 100644 --- a/api/envoy/service/tap/v2alpha/BUILD +++ b/api/envoy/service/tap/v2alpha/BUILD @@ -1,7 +1,17 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v2", + "//envoy/api/v2/core", + "//envoy/api/v2/route:pkg", + "//envoy/data/tap/v2alpha:pkg", + ], +) + api_proto_library_internal( name = "common", srcs = ["common.proto"], diff --git a/api/envoy/service/tap/v3alpha/BUILD b/api/envoy/service/tap/v3alpha/BUILD index a90b2d819297..005ef96b61a0 100644 --- a/api/envoy/service/tap/v3alpha/BUILD +++ b/api/envoy/service/tap/v3alpha/BUILD @@ -1,7 +1,17 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v3alpha", + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/route:pkg", + "//envoy/data/tap/v3alpha:pkg", + ], +) + api_proto_library_internal( name = "common", srcs = ["common.proto"], diff --git a/api/envoy/service/trace/v2/BUILD b/api/envoy/service/trace/v2/BUILD index 2b3367f0af45..cee54d8b34a0 100644 --- a/api/envoy/service/trace/v2/BUILD +++ b/api/envoy/service/trace/v2/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v2/core", + "@opencensus_proto//opencensus/proto/trace/v1:trace_proto", + ], +) + api_proto_library_internal( name = "trace_service", srcs = ["trace_service.proto"], @@ -12,12 +20,3 @@ api_proto_library_internal( "@opencensus_proto//opencensus/proto/trace/v1:trace_proto", ], ) - -api_go_grpc_library( - name = "trace_service", - proto = ":trace_service", - deps = [ - "//envoy/api/v2/core:base_go_proto", - "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_go", - ], -) diff --git a/api/envoy/service/trace/v2/trace_service.proto b/api/envoy/service/trace/v2/trace_service.proto index ec87b3560651..92b8489f2108 100644 --- a/api/envoy/service/trace/v2/trace_service.proto +++ b/api/envoy/service/trace/v2/trace_service.proto @@ -7,7 +7,6 @@ package envoy.service.trace.v2; option java_outer_classname = "TraceServiceProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.trace.v2"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v2/core/base.proto"; diff --git a/api/envoy/service/trace/v3alpha/BUILD b/api/envoy/service/trace/v3alpha/BUILD index 815d0c4c93cc..fbfafec678de 100644 --- a/api/envoy/service/trace/v3alpha/BUILD +++ b/api/envoy/service/trace/v3alpha/BUILD @@ -1,7 +1,15 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_grpc_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + has_services = True, + deps = [ + "//envoy/api/v3alpha/core", + "@opencensus_proto//opencensus/proto/trace/v1:trace_proto", + ], +) + api_proto_library_internal( name = "trace_service", srcs = ["trace_service.proto"], @@ -12,12 +20,3 @@ api_proto_library_internal( "@opencensus_proto//opencensus/proto/trace/v1:trace_proto", ], ) - -api_go_grpc_library( - name = "trace_service", - proto = ":trace_service", - deps = [ - "//envoy/api/v3alpha/core:base_go_proto", - "@opencensus_proto//opencensus/proto/trace/v1:trace_proto_go", - ], -) diff --git a/api/envoy/service/trace/v3alpha/trace_service.proto b/api/envoy/service/trace/v3alpha/trace_service.proto index 521139a084e5..b6559800cd39 100644 --- a/api/envoy/service/trace/v3alpha/trace_service.proto +++ b/api/envoy/service/trace/v3alpha/trace_service.proto @@ -7,7 +7,6 @@ package envoy.service.trace.v3alpha; option java_outer_classname = "TraceServiceProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.service.trace.v3alpha"; -option go_package = "v2"; option java_generic_services = true; import "envoy/api/v3alpha/core/base.proto"; diff --git a/api/envoy/type/BUILD b/api/envoy/type/BUILD index 97f0fd424f36..26dd9730d9ea 100644 --- a/api/envoy/type/BUILD +++ b/api/envoy/type/BUILD @@ -1,36 +1,25 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + name = "type", +) + api_proto_library_internal( name = "http_status", srcs = ["http_status.proto"], visibility = ["//visibility:public"], ) -api_go_proto_library( - name = "http_status", - proto = ":http_status", -) - api_proto_library_internal( name = "percent", srcs = ["percent.proto"], visibility = ["//visibility:public"], ) -api_go_proto_library( - name = "percent", - proto = ":percent", -) - api_proto_library_internal( name = "range", srcs = ["range.proto"], visibility = ["//visibility:public"], ) - -api_go_proto_library( - name = "range", - proto = ":range", -) diff --git a/api/envoy/type/matcher/BUILD b/api/envoy/type/matcher/BUILD index 5fe594db4ca2..c7db01b6cdfe 100644 --- a/api/envoy/type/matcher/BUILD +++ b/api/envoy/type/matcher/BUILD @@ -1,7 +1,12 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_go_proto_library", "api_proto_library_internal") +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") licenses(["notice"]) # Apache 2 +api_proto_package( + name = "matcher", + deps = ["//envoy/type"], +) + api_proto_library_internal( name = "metadata", srcs = ["metadata.proto"], @@ -11,14 +16,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "metadata", - proto = ":metadata", - deps = [ - ":value_go_proto", - ], -) - api_proto_library_internal( name = "number", srcs = ["number.proto"], @@ -28,14 +25,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "number", - proto = ":number", - deps = [ - "//envoy/type:range_go_proto", - ], -) - api_proto_library_internal( name = "string", srcs = ["string.proto"], @@ -45,14 +34,6 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "string", - proto = ":string", - deps = [ - ":regex_go_proto", - ], -) - api_proto_library_internal( name = "value", srcs = ["value.proto"], @@ -63,22 +44,8 @@ api_proto_library_internal( ], ) -api_go_proto_library( - name = "value", - proto = ":value", - deps = [ - ":number_go_proto", - ":string_go_proto", - ], -) - api_proto_library_internal( name = "regex", srcs = ["regex.proto"], visibility = ["//visibility:public"], ) - -api_go_proto_library( - name = "regex", - proto = ":regex", -) diff --git a/api/envoy/type/matcher/metadata.proto b/api/envoy/type/matcher/metadata.proto index 08190a9f5d38..56b69eae5968 100644 --- a/api/envoy/type/matcher/metadata.proto +++ b/api/envoy/type/matcher/metadata.proto @@ -5,7 +5,6 @@ package envoy.type.matcher; option java_outer_classname = "MetadataProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type.matcher"; -option go_package = "matcher"; import "envoy/type/matcher/value.proto"; diff --git a/api/envoy/type/matcher/number.proto b/api/envoy/type/matcher/number.proto index f6c49b3fcdf6..5c8cec7bcbdc 100644 --- a/api/envoy/type/matcher/number.proto +++ b/api/envoy/type/matcher/number.proto @@ -5,7 +5,6 @@ package envoy.type.matcher; option java_outer_classname = "NumberProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type.matcher"; -option go_package = "matcher"; import "envoy/type/range.proto"; diff --git a/api/envoy/type/matcher/regex.proto b/api/envoy/type/matcher/regex.proto index 048a576cc8a6..cf6343c9ac51 100644 --- a/api/envoy/type/matcher/regex.proto +++ b/api/envoy/type/matcher/regex.proto @@ -5,7 +5,6 @@ package envoy.type.matcher; option java_outer_classname = "RegexProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type.matcher"; -option go_package = "matcher"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; diff --git a/api/envoy/type/matcher/string.proto b/api/envoy/type/matcher/string.proto index 35628b7ccb10..986e393be154 100644 --- a/api/envoy/type/matcher/string.proto +++ b/api/envoy/type/matcher/string.proto @@ -5,7 +5,6 @@ package envoy.type.matcher; option java_outer_classname = "StringProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type.matcher"; -option go_package = "matcher"; import "envoy/type/matcher/regex.proto"; diff --git a/api/envoy/type/matcher/value.proto b/api/envoy/type/matcher/value.proto index 52f5e5b100b1..7164504366d9 100644 --- a/api/envoy/type/matcher/value.proto +++ b/api/envoy/type/matcher/value.proto @@ -5,7 +5,6 @@ package envoy.type.matcher; option java_outer_classname = "ValueProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type.matcher"; -option go_package = "matcher"; import "envoy/type/matcher/number.proto"; import "envoy/type/matcher/string.proto"; diff --git a/api/envoy/type/range.proto b/api/envoy/type/range.proto index e64b71e440f3..cc38e8f25f5e 100644 --- a/api/envoy/type/range.proto +++ b/api/envoy/type/range.proto @@ -5,7 +5,6 @@ package envoy.type; option java_outer_classname = "RangeProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type"; -option go_package = "envoy_type"; import "gogoproto/gogo.proto"; diff --git a/api/test/build/BUILD b/api/test/build/BUILD index a271aa5daf82..c8390c1fede4 100644 --- a/api/test/build/BUILD +++ b/api/test/build/BUILD @@ -26,18 +26,13 @@ api_go_test( srcs = ["go_build_test.go"], importpath = "go_build_test", deps = [ - "//envoy/api/v2:cds_go_grpc", - "//envoy/api/v2:eds_go_grpc", - "//envoy/api/v2:lds_go_grpc", - "//envoy/api/v2:rds_go_grpc", - "//envoy/api/v2/auth:cert_go_proto", - "//envoy/config/bootstrap/v2:bootstrap_go_proto", - "//envoy/service/accesslog/v2:als_go_grpc", - "//envoy/service/discovery/v2:ads_go_grpc", - "//envoy/service/discovery/v2:hds_go_grpc", - "//envoy/service/discovery/v2:sds_go_grpc", - "//envoy/service/metrics/v2:metrics_service_go_grpc", - "//envoy/service/ratelimit/v2:rls_go_grpc", - "//envoy/service/trace/v2:trace_service_go_grpc", + "//envoy/api/v2:v2_go_proto", + "//envoy/api/v2/auth:auth_go_proto", + "//envoy/config/bootstrap/v2:pkg_go_proto", + "//envoy/service/accesslog/v2:pkg_go_proto", + "//envoy/service/discovery/v2:pkg_go_proto", + "//envoy/service/metrics/v2:pkg_go_proto", + "//envoy/service/ratelimit/v2:pkg_go_proto", + "//envoy/service/trace/v2:pkg_go_proto", ], ) diff --git a/api/test/build/go_build_test.go b/api/test/build/go_build_test.go index 911d3ef39655..c5c15becff35 100644 --- a/api/test/build/go_build_test.go +++ b/api/test/build/go_build_test.go @@ -3,19 +3,14 @@ package go_build_test import ( "testing" - _ "github.com/envoyproxy/data-plane-api/api/ads" - _ "github.com/envoyproxy/data-plane-api/api/als" - _ "github.com/envoyproxy/data-plane-api/api/bootstrap" - _ "github.com/envoyproxy/data-plane-api/api/cds" - _ "github.com/envoyproxy/data-plane-api/api/cert" - _ "github.com/envoyproxy/data-plane-api/api/eds" - _ "github.com/envoyproxy/data-plane-api/api/hds" - _ "github.com/envoyproxy/data-plane-api/api/lds" - _ "github.com/envoyproxy/data-plane-api/api/metrics_service" - _ "github.com/envoyproxy/data-plane-api/api/rds" - _ "github.com/envoyproxy/data-plane-api/api/rls" - _ "github.com/envoyproxy/data-plane-api/api/sds" - _ "github.com/envoyproxy/data-plane-api/api/trace_service" + _ "github.com/envoyproxy/data-plane-api/api/envoy/api/v2" + _ "github.com/envoyproxy/data-plane-api/api/envoy/api/v2/auth" + _ "github.com/envoyproxy/data-plane-api/api/envoy/config/bootstrap/v2" + _ "github.com/envoyproxy/data-plane-api/api/envoy/service/accesslog/v2" + _ "github.com/envoyproxy/data-plane-api/api/envoy/service/discovery/v2" + _ "github.com/envoyproxy/data-plane-api/api/envoy/service/metrics/v2" + _ "github.com/envoyproxy/data-plane-api/api/envoy/service/ratelimit/v2" + _ "github.com/envoyproxy/data-plane-api/api/envoy/service/trace/v2" ) func TestNoop(t *testing.T) { diff --git a/tools/check_format.py b/tools/check_format.py index 26b3e621848f..ed3b36974963 100755 --- a/tools/check_format.py +++ b/tools/check_format.py @@ -72,6 +72,7 @@ PROTO_OPTION_JAVA_PACKAGE = "option java_package = \"" PROTO_OPTION_JAVA_OUTER_CLASSNAME = "option java_outer_classname = \"" PROTO_OPTION_JAVA_MULTIPLE_FILES = "option java_multiple_files = " +PROTO_OPTION_GO_PACKAGE = "option go_package = \"" # yapf: disable PROTOBUF_TYPE_ERRORS = { @@ -623,6 +624,17 @@ def checkBuildPath(file_path): command = "%s %s | diff %s -" % (ENVOY_BUILD_FIXER_PATH, file_path, file_path) error_messages += executeCommand(command, "envoy_build_fixer check failed", file_path) + if isBuildFile(file_path) and file_path.startswith(args.api_prefix + "envoy"): + found = False + finput = fileinput.input(file_path) + for line in finput: + if "api_proto_package(" in line: + found = True + break + finput.close() + if not found: + error_messages += ["API build file does not provide api_proto_package()"] + command = "%s -mode=diff %s" % (BUILDIFIER_PATH, file_path) error_messages += executeCommand(command, "buildifier check failed", file_path) error_messages += checkFileContents(file_path, checkBuildLine) @@ -672,6 +684,9 @@ def checkSourcePath(file_path): "Java proto option 'java_outer_classname' not set") error_messages += errorIfNoSubstringFound("\n" + PROTO_OPTION_JAVA_MULTIPLE_FILES, file_path, "Java proto option 'java_multiple_files' not set") + with open(file_path) as f: + if PROTO_OPTION_GO_PACKAGE in f.read(): + error_messages += ["go_package option should not be set in %s" % file_path] return error_messages diff --git a/tools/check_format_test_helper.py b/tools/check_format_test_helper.py index c41abbc60890..844488a8aa35 100755 --- a/tools/check_format_test_helper.py +++ b/tools/check_format_test_helper.py @@ -238,6 +238,7 @@ def checkFileExpectingOK(filename): errors += checkAndFixError("bad_envoy_build_sys_ref.BUILD", "Superfluous '@envoy//' prefix") errors += checkAndFixError("proto_format.proto", "clang-format check failed") errors += checkAndFixError("api/java_options.proto", "Java proto option") + errors += checkFileExpectingError("api/go_package.proto", "go_package option should not be set") errors += checkAndFixError( "cpp_std.cc", "term absl::make_unique< should be replaced with standard library term std::make_unique<") diff --git a/tools/testdata/check_format/api/go_package.proto b/tools/testdata/check_format/api/go_package.proto new file mode 100644 index 000000000000..b32347b6e46f --- /dev/null +++ b/tools/testdata/check_format/api/go_package.proto @@ -0,0 +1,5 @@ +option go_package = "foo"; +option java_package = "io.envoyproxy.envoy.foo"; +option java_outer_classname = "JavaOptionsProto"; +option java_multiple_files = true; +package envoy.foo; From 1b3b4ae1180b67bee6395fab5c075896fb1964ec Mon Sep 17 00:00:00 2001 From: htuch Date: Wed, 4 Sep 2019 15:06:39 -0400 Subject: [PATCH 498/542] api: straggler v2alpha1 -> v3alpha clone. (#8133) These were missed in #8125. Signed-off-by: Harvey Tuch --- api/docs/BUILD | 34 ++--- api/envoy/api/v3alpha/cds.proto | 2 +- api/envoy/api/v3alpha/eds.proto | 2 +- api/envoy/api/v3alpha/lds.proto | 2 +- api/envoy/api/v3alpha/rds.proto | 2 +- api/envoy/api/v3alpha/srds.proto | 2 +- .../config/filter/dubbo/router/v3alpha/BUILD | 10 ++ .../filter/dubbo/router/v3alpha/router.proto | 13 ++ .../grpc_http1_reverse_bridge/v3alpha/BUILD | 10 ++ .../v3alpha/config.proto | 26 ++++ .../filter/http/original_src/v3alpha/BUILD | 10 ++ .../original_src/v3alpha/original_src.proto | 24 ++++ .../listener/original_src/v3alpha/BUILD | 10 ++ .../original_src/v3alpha/original_src.proto | 28 ++++ .../filter/network/dubbo_proxy/v3alpha/BUILD | 26 ++++ .../network/dubbo_proxy/v3alpha/README.md | 1 + .../dubbo_proxy/v3alpha/dubbo_proxy.proto | 60 ++++++++ .../network/dubbo_proxy/v3alpha/route.proto | 109 +++++++++++++++ .../filter/network/thrift_proxy/v3alpha/BUILD | 22 +++ .../network/thrift_proxy/v3alpha/README.md | 1 + .../network/thrift_proxy/v3alpha/route.proto | 129 ++++++++++++++++++ .../thrift_proxy/v3alpha/thrift_proxy.proto | 121 ++++++++++++++++ .../filter/thrift/rate_limit/v3alpha/BUILD | 19 +++ .../rate_limit/v3alpha/rate_limit.proto | 50 +++++++ .../config/filter/thrift/router/v3alpha/BUILD | 10 ++ .../filter/thrift/router/v3alpha/router.proto | 13 ++ api/envoy/service/discovery/v3alpha/hds.proto | 2 +- .../service/discovery/v3alpha/rtds.proto | 2 +- api/envoy/service/discovery/v3alpha/sds.proto | 2 +- api/envoy/service/tap/v3alpha/tapds.proto | 2 +- docs/build.sh | 24 ++-- tools/api/clone.sh | 11 +- 32 files changed, 732 insertions(+), 47 deletions(-) create mode 100644 api/envoy/config/filter/dubbo/router/v3alpha/BUILD create mode 100644 api/envoy/config/filter/dubbo/router/v3alpha/router.proto create mode 100644 api/envoy/config/filter/http/grpc_http1_reverse_bridge/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/grpc_http1_reverse_bridge/v3alpha/config.proto create mode 100644 api/envoy/config/filter/http/original_src/v3alpha/BUILD create mode 100644 api/envoy/config/filter/http/original_src/v3alpha/original_src.proto create mode 100644 api/envoy/config/filter/listener/original_src/v3alpha/BUILD create mode 100644 api/envoy/config/filter/listener/original_src/v3alpha/original_src.proto create mode 100644 api/envoy/config/filter/network/dubbo_proxy/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/dubbo_proxy/v3alpha/README.md create mode 100644 api/envoy/config/filter/network/dubbo_proxy/v3alpha/dubbo_proxy.proto create mode 100644 api/envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto create mode 100644 api/envoy/config/filter/network/thrift_proxy/v3alpha/BUILD create mode 100644 api/envoy/config/filter/network/thrift_proxy/v3alpha/README.md create mode 100644 api/envoy/config/filter/network/thrift_proxy/v3alpha/route.proto create mode 100644 api/envoy/config/filter/network/thrift_proxy/v3alpha/thrift_proxy.proto create mode 100644 api/envoy/config/filter/thrift/rate_limit/v3alpha/BUILD create mode 100644 api/envoy/config/filter/thrift/rate_limit/v3alpha/rate_limit.proto create mode 100644 api/envoy/config/filter/thrift/router/v3alpha/BUILD create mode 100644 api/envoy/config/filter/thrift/router/v3alpha/router.proto diff --git a/api/docs/BUILD b/api/docs/BUILD index 5784ecc2d2af..31ed1ee5bae6 100644 --- a/api/docs/BUILD +++ b/api/docs/BUILD @@ -11,23 +11,13 @@ package_group( proto_library( name = "protos", deps = [ - "//envoy/admin/v2alpha:certs", - "//envoy/admin/v2alpha:clusters", - "//envoy/admin/v2alpha:config_dump", - "//envoy/admin/v2alpha:listeners", - "//envoy/admin/v2alpha:memory", - "//envoy/admin/v2alpha:mutex_stats", - "//envoy/admin/v2alpha:server_info", - "//envoy/admin/v2alpha:tap", - "//envoy/api/v2:cds", - "//envoy/api/v2:discovery", - "//envoy/api/v2:eds", - "//envoy/api/v2:lds", - "//envoy/api/v2:rds", - "//envoy/api/v2/cluster:circuit_breaker", - "//envoy/api/v2/cluster:outlier_detection", - "//envoy/api/v2/core:protocol", - "//envoy/api/v2/listener", + "//envoy/admin/v2alpha:pkg", + "//envoy/api/v2", + "//envoy/api/v2/auth", + "//envoy/api/v2/cluster", + "//envoy/api/v2/core", + "//envoy/api/v2/endpoint", + "//envoy/api/v2/listener:pkg", "//envoy/api/v2/ratelimit", "//envoy/api/v2/route", "//envoy/config/accesslog/v2:als", @@ -39,6 +29,7 @@ proto_library( "//envoy/config/common/tap/v2alpha:common", "//envoy/config/filter/accesslog/v2:accesslog", "//envoy/config/filter/dubbo/router/v2alpha1:router", + "//envoy/config/filter/fault/v2:fault", "//envoy/config/filter/http/buffer/v2:buffer", "//envoy/config/filter/http/csrf/v2:csrf", "//envoy/config/filter/http/dynamic_forward_proxy/v2alpha:dynamic_forward_proxy", @@ -75,6 +66,7 @@ proto_library( "//envoy/config/health_checker/redis/v2:redis", "//envoy/config/metrics/v2:metrics_service", "//envoy/config/metrics/v2:stats", + "//envoy/config/overload/v2alpha:overload", "//envoy/config/ratelimit/v2:rls", "//envoy/config/rbac/v2:rbac", "//envoy/config/resource_monitor/fixed_heap/v2alpha:fixed_heap", @@ -95,11 +87,7 @@ proto_library( "//envoy/service/discovery/v2:rtds", "//envoy/service/ratelimit/v2:rls", "//envoy/service/tap/v2alpha:common", - "//envoy/type:percent", - "//envoy/type:range", - "//envoy/type/matcher:metadata", - "//envoy/type/matcher:number", - "//envoy/type/matcher:regex", - "//envoy/type/matcher:string", + "//envoy/type", + "//envoy/type/matcher", ], ) diff --git a/api/envoy/api/v3alpha/cds.proto b/api/envoy/api/v3alpha/cds.proto index 50b7adaf996e..2197ff3efe46 100644 --- a/api/envoy/api/v3alpha/cds.proto +++ b/api/envoy/api/v3alpha/cds.proto @@ -43,7 +43,7 @@ service ClusterDiscoveryService { rpc FetchClusters(DiscoveryRequest) returns (DiscoveryResponse) { option (google.api.http) = { - post: "/v2/discovery:clusters" + post: "/v3alpha/discovery:clusters" body: "*" }; } diff --git a/api/envoy/api/v3alpha/eds.proto b/api/envoy/api/v3alpha/eds.proto index 7ba8592eb793..6d1cf12f3de5 100644 --- a/api/envoy/api/v3alpha/eds.proto +++ b/api/envoy/api/v3alpha/eds.proto @@ -36,7 +36,7 @@ service EndpointDiscoveryService { rpc FetchEndpoints(DiscoveryRequest) returns (DiscoveryResponse) { option (google.api.http) = { - post: "/v2/discovery:endpoints" + post: "/v3alpha/discovery:endpoints" body: "*" }; } diff --git a/api/envoy/api/v3alpha/lds.proto b/api/envoy/api/v3alpha/lds.proto index d9976d6c0e3c..7268d2b15e4b 100644 --- a/api/envoy/api/v3alpha/lds.proto +++ b/api/envoy/api/v3alpha/lds.proto @@ -39,7 +39,7 @@ service ListenerDiscoveryService { rpc FetchListeners(DiscoveryRequest) returns (DiscoveryResponse) { option (google.api.http) = { - post: "/v2/discovery:listeners" + post: "/v3alpha/discovery:listeners" body: "*" }; } diff --git a/api/envoy/api/v3alpha/rds.proto b/api/envoy/api/v3alpha/rds.proto index ed20b34e7cca..b26956dcf712 100644 --- a/api/envoy/api/v3alpha/rds.proto +++ b/api/envoy/api/v3alpha/rds.proto @@ -39,7 +39,7 @@ service RouteDiscoveryService { rpc FetchRoutes(DiscoveryRequest) returns (DiscoveryResponse) { option (google.api.http) = { - post: "/v2/discovery:routes" + post: "/v3alpha/discovery:routes" body: "*" }; } diff --git a/api/envoy/api/v3alpha/srds.proto b/api/envoy/api/v3alpha/srds.proto index 22ad6e675683..62a767ca4440 100644 --- a/api/envoy/api/v3alpha/srds.proto +++ b/api/envoy/api/v3alpha/srds.proto @@ -33,7 +33,7 @@ service ScopedRoutesDiscoveryService { rpc FetchScopedRoutes(DiscoveryRequest) returns (DiscoveryResponse) { option (google.api.http) = { - post: "/v2/discovery:scoped-routes" + post: "/v3alpha/discovery:scoped-routes" body: "*" }; } diff --git a/api/envoy/config/filter/dubbo/router/v3alpha/BUILD b/api/envoy/config/filter/dubbo/router/v3alpha/BUILD new file mode 100644 index 000000000000..68bd8c126b80 --- /dev/null +++ b/api/envoy/config/filter/dubbo/router/v3alpha/BUILD @@ -0,0 +1,10 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package() + +api_proto_library_internal( + name = "router", + srcs = ["router.proto"], +) diff --git a/api/envoy/config/filter/dubbo/router/v3alpha/router.proto b/api/envoy/config/filter/dubbo/router/v3alpha/router.proto new file mode 100644 index 000000000000..46b6609d1c45 --- /dev/null +++ b/api/envoy/config/filter/dubbo/router/v3alpha/router.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package envoy.config.filter.dubbo.router.v3alpha; + +option java_outer_classname = "RouterProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.dubbo.router.v3alpha"; + +// [#protodoc-title: Router] +// Dubbo router :ref:`configuration overview `. + +message Router { +} diff --git a/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v3alpha/BUILD b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v3alpha/BUILD new file mode 100644 index 000000000000..a88ba2443cad --- /dev/null +++ b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v3alpha/BUILD @@ -0,0 +1,10 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package() + +api_proto_library( + name = "config", + srcs = ["config.proto"], +) diff --git a/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v3alpha/config.proto b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v3alpha/config.proto new file mode 100644 index 000000000000..2883701d33d6 --- /dev/null +++ b/api/envoy/config/filter/http/grpc_http1_reverse_bridge/v3alpha/config.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package envoy.config.filter.http.grpc_http1_reverse_bridge.v3alpha; + +option java_outer_classname = "ConfigProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.grpc_http1_reverse_bridge.v3alpha"; + +import "validate/validate.proto"; + +// [#protodoc-title: gRPC HTTP/1.1 Reverse Bridge] +// gRPC HTTP/1.1 Reverse Bridge :ref:`configuration overview +// `. + +// gRPC reverse bridge filter configuration +message FilterConfig { + // The content-type to pass to the upstream when the gRPC bridge filter is applied. + // The filter will also validate that the upstream responds with the same content type. + string content_type = 1 [(validate.rules).string.min_bytes = 1]; + + // If true, Envoy will assume that the upstream doesn't understand gRPC frames and + // strip the gRPC frame from the request, and add it back in to the response. This will + // hide the gRPC semantics from the upstream, allowing it to receive and respond with a + // simple binary encoded protobuf. + bool withhold_grpc_frames = 2; +} diff --git a/api/envoy/config/filter/http/original_src/v3alpha/BUILD b/api/envoy/config/filter/http/original_src/v3alpha/BUILD new file mode 100644 index 000000000000..a7435bb55cfc --- /dev/null +++ b/api/envoy/config/filter/http/original_src/v3alpha/BUILD @@ -0,0 +1,10 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package() + +api_proto_library_internal( + name = "original_src", + srcs = ["original_src.proto"], +) diff --git a/api/envoy/config/filter/http/original_src/v3alpha/original_src.proto b/api/envoy/config/filter/http/original_src/v3alpha/original_src.proto new file mode 100644 index 000000000000..20bd0e920e26 --- /dev/null +++ b/api/envoy/config/filter/http/original_src/v3alpha/original_src.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package envoy.config.filter.http.original_src.v3alpha; + +option java_outer_classname = "OriginalSrcProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.http.original_src.v3alpha"; + +import "validate/validate.proto"; + +// [#protodoc-title: Original Src Filter] +// Use the Original source address on upstream connections. + +// The Original Src filter binds upstream connections to the original source address determined +// for the request. This address could come from something like the Proxy Protocol filter, or it +// could come from trusted http headers. +message OriginalSrc { + + // Sets the SO_MARK option on the upstream connection's socket to the provided value. Used to + // ensure that non-local addresses may be routed back through envoy when binding to the original + // source address. The option will not be applied if the mark is 0. + // [#proto-status: experimental] + uint32 mark = 1; +} diff --git a/api/envoy/config/filter/listener/original_src/v3alpha/BUILD b/api/envoy/config/filter/listener/original_src/v3alpha/BUILD new file mode 100644 index 000000000000..a7435bb55cfc --- /dev/null +++ b/api/envoy/config/filter/listener/original_src/v3alpha/BUILD @@ -0,0 +1,10 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package() + +api_proto_library_internal( + name = "original_src", + srcs = ["original_src.proto"], +) diff --git a/api/envoy/config/filter/listener/original_src/v3alpha/original_src.proto b/api/envoy/config/filter/listener/original_src/v3alpha/original_src.proto new file mode 100644 index 000000000000..3c5fee9505a2 --- /dev/null +++ b/api/envoy/config/filter/listener/original_src/v3alpha/original_src.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +package envoy.config.filter.listener.original_src.v3alpha; + +option java_outer_classname = "OriginalSrcProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.listener.original_src.v3alpha"; + +import "validate/validate.proto"; + +// [#protodoc-title: Original Src Filter] +// Use the Original source address on upstream connections. + +// The Original Src filter binds upstream connections to the original source address determined +// for the connection. This address could come from something like the Proxy Protocol filter, or it +// could come from trusted http headers. +message OriginalSrc { + + // Whether to bind the port to the one used in the original downstream connection. + // [#not-implemented-warn:] + bool bind_port = 1; + + // Sets the SO_MARK option on the upstream connection's socket to the provided value. Used to + // ensure that non-local addresses may be routed back through envoy when binding to the original + // source address. The option will not be applied if the mark is 0. + // [#proto-status: experimental] + uint32 mark = 2; +} diff --git a/api/envoy/config/filter/network/dubbo_proxy/v3alpha/BUILD b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..db73dfbd0848 --- /dev/null +++ b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/BUILD @@ -0,0 +1,26 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/route:pkg", + "//envoy/type", + "//envoy/type/matcher", + ], +) + +api_proto_library_internal( + name = "dubbo_proxy", + srcs = [ + "dubbo_proxy.proto", + "route.proto", + ], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/route", + "//envoy/type:range", + "//envoy/type/matcher:string", + ], +) diff --git a/api/envoy/config/filter/network/dubbo_proxy/v3alpha/README.md b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/README.md new file mode 100644 index 000000000000..c83caca1f8f4 --- /dev/null +++ b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/README.md @@ -0,0 +1 @@ +Protocol buffer definitions for the Dubbo proxy. diff --git a/api/envoy/config/filter/network/dubbo_proxy/v3alpha/dubbo_proxy.proto b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/dubbo_proxy.proto new file mode 100644 index 000000000000..f314e393a85d --- /dev/null +++ b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/dubbo_proxy.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +package envoy.config.filter.network.dubbo_proxy.v3alpha; + +option java_outer_classname = "DubboProxyProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.dubbo_proxy.v3alpha"; + +import "envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto"; + +import "google/protobuf/any.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Dubbo Proxy] +// Dubbo Proxy :ref:`configuration overview `. + +// [#comment:next free field: 6] +message DubboProxy { + // The human readable prefix to use when emitting statistics. + string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + // Configure the protocol used. + ProtocolType protocol_type = 2 [(validate.rules).enum.defined_only = true]; + + // Configure the serialization protocol used. + SerializationType serialization_type = 3 [(validate.rules).enum.defined_only = true]; + + // The route table for the connection manager is static and is specified in this property. + repeated RouteConfiguration route_config = 4; + + // A list of individual Dubbo filters that make up the filter chain for requests made to the + // Dubbo proxy. Order matters as the filters are processed sequentially. For backwards + // compatibility, if no dubbo_filters are specified, a default Dubbo router filter + // (`envoy.filters.dubbo.router`) is used. + repeated DubboFilter dubbo_filters = 5; +} + +// Dubbo Protocol types supported by Envoy. +enum ProtocolType { + Dubbo = 0; // the default protocol. +} + +// Dubbo Serialization types supported by Envoy. +enum SerializationType { + Hessian2 = 0; // the default serialization protocol. +} + +// DubboFilter configures a Dubbo filter. +// [#comment:next free field: 3] +message DubboFilter { + // The name of the filter to instantiate. The name must match a supported + // filter. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Filter specific configuration which depends on the filter being + // instantiated. See the supported filters for further documentation. + google.protobuf.Any config = 2; +} diff --git a/api/envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto new file mode 100644 index 000000000000..180428f8644a --- /dev/null +++ b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto @@ -0,0 +1,109 @@ +syntax = "proto3"; + +package envoy.config.filter.network.dubbo_proxy.v3alpha; + +option java_outer_classname = "RouteProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.dubbo_proxy.v3alpha"; + +import "envoy/api/v3alpha/route/route.proto"; +import "envoy/type/matcher/string.proto"; +import "envoy/type/range.proto"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.stable_marshaler_all) = true; + +// [#protodoc-title: Dubbo Proxy Route Configuration] +// Dubbo Proxy :ref:`configuration overview `. + +// [#comment:next free field: 6] +message RouteConfiguration { + // The name of the route configuration. Reserved for future use in asynchronous route discovery. + string name = 1; + + // The interface name of the service. + string interface = 2; + + // Which group does the interface belong to. + string group = 3; + + // The version number of the interface. + string version = 4; + + // The list of routes that will be matched, in order, against incoming requests. The first route + // that matches will be used. + repeated Route routes = 5; +} + +// [#comment:next free field: 3] +message Route { + // Route matching parameters. + RouteMatch match = 1 [(validate.rules).message.required = true]; + + // Route request to some upstream cluster. + RouteAction route = 2 [(validate.rules).message.required = true]; +} + +// [#comment:next free field: 3] +message RouteMatch { + // Method level routing matching. + MethodMatch method = 1; + + // Specifies a set of headers that the route should match on. The router will check the request’s + // headers against all the specified headers in the route config. A match will happen if all the + // headers in the route are present in the request with the same values (or based on presence if + // the value field is not in the config). + repeated envoy.api.v3alpha.route.HeaderMatcher headers = 2; +} + +// [#comment:next free field: 3] +message RouteAction { + oneof cluster_specifier { + option (validate.required) = true; + + // Indicates the upstream cluster to which the request should be routed. + string cluster = 1; + + // Multiple upstream clusters can be specified for a given route. The + // request is routed to one of the upstream clusters based on weights + // assigned to each cluster. + // Currently ClusterWeight only supports the name and weight fields. + envoy.api.v3alpha.route.WeightedCluster weighted_clusters = 2; + } +} + +// [#comment:next free field: 5] +message MethodMatch { + // The name of the method. + envoy.type.matcher.StringMatcher name = 1; + + // The parameter matching type. + message ParameterMatchSpecifier { + oneof parameter_match_specifier { + // If specified, header match will be performed based on the value of the header. + string exact_match = 3; + + // If specified, header match will be performed based on range. + // The rule will match if the request header value is within this range. + // The entire request header value must represent an integer in base 10 notation: consisting + // of an optional plus or minus sign followed by a sequence of digits. The rule will not match + // if the header value does not represent an integer. Match will fail for empty values, + // floating point numbers or if only a subsequence of the header value is an integer. + // + // Examples: + // + // * For range [-10,0), route will match for header value -1, but not for 0, + // "somestring", 10.9, "-1somestring" + envoy.type.Int64Range range_match = 4; + } + } + + // Method parameter definition. + // The key is the parameter index, starting from 0. + // The value is the parameter matching type. + map params_match = 2; +} diff --git a/api/envoy/config/filter/network/thrift_proxy/v3alpha/BUILD b/api/envoy/config/filter/network/thrift_proxy/v3alpha/BUILD new file mode 100644 index 000000000000..34a2c397ccb8 --- /dev/null +++ b/api/envoy/config/filter/network/thrift_proxy/v3alpha/BUILD @@ -0,0 +1,22 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/api/v3alpha/route:pkg", + ], +) + +api_proto_library_internal( + name = "thrift_proxy", + srcs = [ + "route.proto", + "thrift_proxy.proto", + ], + deps = [ + "//envoy/api/v3alpha/core:base", + "//envoy/api/v3alpha/route", + ], +) diff --git a/api/envoy/config/filter/network/thrift_proxy/v3alpha/README.md b/api/envoy/config/filter/network/thrift_proxy/v3alpha/README.md new file mode 100644 index 000000000000..a7d95c0d4764 --- /dev/null +++ b/api/envoy/config/filter/network/thrift_proxy/v3alpha/README.md @@ -0,0 +1 @@ +Protocol buffer definitions for the Thrift proxy. diff --git a/api/envoy/config/filter/network/thrift_proxy/v3alpha/route.proto b/api/envoy/config/filter/network/thrift_proxy/v3alpha/route.proto new file mode 100644 index 000000000000..1e6777eedef1 --- /dev/null +++ b/api/envoy/config/filter/network/thrift_proxy/v3alpha/route.proto @@ -0,0 +1,129 @@ +syntax = "proto3"; + +package envoy.config.filter.network.thrift_proxy.v3alpha; + +option java_outer_classname = "RouteProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.thrift_proxy.v3alpha"; + +import "envoy/api/v3alpha/core/base.proto"; +import "envoy/api/v3alpha/route/route.proto"; + +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Thrift Proxy Route Configuration] +// Thrift Proxy :ref:`configuration overview `. + +// [#comment:next free field: 3] +message RouteConfiguration { + // The name of the route configuration. Reserved for future use in asynchronous route discovery. + string name = 1; + + // The list of routes that will be matched, in order, against incoming requests. The first route + // that matches will be used. + repeated Route routes = 2; +} + +// [#comment:next free field: 3] +message Route { + // Route matching parameters. + RouteMatch match = 1 [(validate.rules).message.required = true]; + + // Route request to some upstream cluster. + RouteAction route = 2 [(validate.rules).message.required = true]; +} + +// [#comment:next free field: 5] +message RouteMatch { + oneof match_specifier { + option (validate.required) = true; + + // If specified, the route must exactly match the request method name. As a special case, an + // empty string matches any request method name. + string method_name = 1; + + // If specified, the route must have the service name as the request method name prefix. As a + // special case, an empty string matches any service name. Only relevant when service + // multiplexing. + string service_name = 2; + } + + // Inverts whatever matching is done in the :ref:`method_name + // ` or + // :ref:`service_name + // ` fields. + // Cannot be combined with wildcard matching as that would result in routes never being matched. + // + // .. note:: + // + // This does not invert matching done as part of the :ref:`headers field + // ` field. To + // invert header matching, see :ref:`invert_match + // `. + bool invert = 3; + + // Specifies a set of headers that the route should match on. The router will check the request’s + // headers against all the specified headers in the route config. A match will happen if all the + // headers in the route are present in the request with the same values (or based on presence if + // the value field is not in the config). Note that this only applies for Thrift transports and/or + // protocols that support headers. + repeated envoy.api.v3alpha.route.HeaderMatcher headers = 4; +} + +// [#comment:next free field: 5] +message RouteAction { + oneof cluster_specifier { + option (validate.required) = true; + + // Indicates a single upstream cluster to which the request should be routed + // to. + string cluster = 1 [(validate.rules).string.min_bytes = 1]; + + // Multiple upstream clusters can be specified for a given route. The + // request is routed to one of the upstream clusters based on weights + // assigned to each cluster. + WeightedCluster weighted_clusters = 2; + } + + // Optional endpoint metadata match criteria used by the subset load balancer. Only endpoints in + // the upstream cluster with metadata matching what is set in this field will be considered. + // Note that this will be merged with what's provided in :ref: `WeightedCluster.MetadataMatch + // `, + // with values there taking precedence. Keys and values should be provided under the "envoy.lb" + // metadata key. + envoy.api.v3alpha.core.Metadata metadata_match = 3; + + // Specifies a set of rate limit configurations that could be applied to the route. + // N.B. Thrift service or method name matching can be achieved by specifying a RequestHeaders + // action with the header name ":method-name". + repeated envoy.api.v3alpha.route.RateLimit rate_limits = 4; +} + +// Allows for specification of multiple upstream clusters along with weights that indicate the +// percentage of traffic to be forwarded to each cluster. The router selects an upstream cluster +// based on these weights. +message WeightedCluster { + message ClusterWeight { + // Name of the upstream cluster. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // When a request matches the route, the choice of an upstream cluster is determined by its + // weight. The sum of weights across all entries in the clusters array determines the total + // weight. + google.protobuf.UInt32Value weight = 2 [(validate.rules).uint32.gte = 1]; + + // Optional endpoint metadata match criteria used by the subset load balancer. Only endpoints in + // the upstream cluster with metadata matching what is set in this field, combined with what's + // provided in :ref: `RouteAction's metadata_match + // `, + // will be considered. Values here will take precedence. Keys and values should be provided + // under the "envoy.lb" metadata key. + envoy.api.v3alpha.core.Metadata metadata_match = 3; + } + + // Specifies one or more upstream clusters associated with the route. + repeated ClusterWeight clusters = 1 [(validate.rules).repeated .min_items = 1]; +} diff --git a/api/envoy/config/filter/network/thrift_proxy/v3alpha/thrift_proxy.proto b/api/envoy/config/filter/network/thrift_proxy/v3alpha/thrift_proxy.proto new file mode 100644 index 000000000000..3c3c200b2ea3 --- /dev/null +++ b/api/envoy/config/filter/network/thrift_proxy/v3alpha/thrift_proxy.proto @@ -0,0 +1,121 @@ +syntax = "proto3"; + +package envoy.config.filter.network.thrift_proxy.v3alpha; + +option java_outer_classname = "ThriftProxyProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.network.thrift_proxy.v3alpha"; + +import "envoy/config/filter/network/thrift_proxy/v3alpha/route.proto"; + +import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Thrift Proxy] +// Thrift Proxy :ref:`configuration overview `. + +// [#comment:next free field: 6] +message ThriftProxy { + // Supplies the type of transport that the Thrift proxy should use. Defaults to + // :ref:`AUTO_TRANSPORT`. + TransportType transport = 2 [(validate.rules).enum.defined_only = true]; + + // Supplies the type of protocol that the Thrift proxy should use. Defaults to + // :ref:`AUTO_PROTOCOL`. + ProtocolType protocol = 3 [(validate.rules).enum.defined_only = true]; + + // The human readable prefix to use when emitting statistics. + string stat_prefix = 1 [(validate.rules).string.min_bytes = 1]; + + // The route table for the connection manager is static and is specified in this property. + RouteConfiguration route_config = 4; + + // A list of individual Thrift filters that make up the filter chain for requests made to the + // Thrift proxy. Order matters as the filters are processed sequentially. For backwards + // compatibility, if no thrift_filters are specified, a default Thrift router filter + // (`envoy.filters.thrift.router`) is used. + repeated ThriftFilter thrift_filters = 5; +} + +// Thrift transport types supported by Envoy. +enum TransportType { + option (gogoproto.goproto_enum_prefix) = false; + + // For downstream connections, the Thrift proxy will attempt to determine which transport to use. + // For upstream connections, the Thrift proxy will use same transport as the downstream + // connection. + AUTO_TRANSPORT = 0; + + // The Thrift proxy will use the Thrift framed transport. + FRAMED = 1; + + // The Thrift proxy will use the Thrift unframed transport. + UNFRAMED = 2; + + // The Thrift proxy will assume the client is using the Thrift header transport. + HEADER = 3; +} + +// Thrift Protocol types supported by Envoy. +enum ProtocolType { + option (gogoproto.goproto_enum_prefix) = false; + + // For downstream connections, the Thrift proxy will attempt to determine which protocol to use. + // Note that the older, non-strict (or lax) binary protocol is not included in automatic protocol + // detection. For upstream connections, the Thrift proxy will use the same protocol as the + // downstream connection. + AUTO_PROTOCOL = 0; + + // The Thrift proxy will use the Thrift binary protocol. + BINARY = 1; + + // The Thrift proxy will use Thrift non-strict binary protocol. + LAX_BINARY = 2; + + // The Thrift proxy will use the Thrift compact protocol. + COMPACT = 3; + + // The Thrift proxy will use the Thrift "Twitter" protocol implemented by the finagle library. + TWITTER = 4; +} + +// ThriftFilter configures a Thrift filter. +// [#comment:next free field: 3] +message ThriftFilter { + // The name of the filter to instantiate. The name must match a supported + // filter. The built-in filters are: + // + // [#comment:TODO(zuercher): Auto generate the following list] + // * :ref:`envoy.filters.thrift.router ` + // * :ref:`envoy.filters.thrift.rate_limit ` + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Filter specific configuration which depends on the filter being instantiated. See the supported + // filters for further documentation. + oneof config_type { + google.protobuf.Struct config = 2; + + google.protobuf.Any typed_config = 3; + } +} + +// ThriftProtocolOptions specifies Thrift upstream protocol options. This object is used in +// in :ref:`extension_protocol_options`, keyed +// by the name `envoy.filters.network.thrift_proxy`. +// [#comment:next free field: 3] +message ThriftProtocolOptions { + // Supplies the type of transport that the Thrift proxy should use for upstream connections. + // Selecting + // :ref:`AUTO_TRANSPORT`, + // which is the default, causes the proxy to use the same transport as the downstream connection. + TransportType transport = 1 [(validate.rules).enum.defined_only = true]; + + // Supplies the type of protocol that the Thrift proxy should use for upstream connections. + // Selecting + // :ref:`AUTO_PROTOCOL`, + // which is the default, causes the proxy to use the same protocol as the downstream connection. + ProtocolType protocol = 2 [(validate.rules).enum.defined_only = true]; +} diff --git a/api/envoy/config/filter/thrift/rate_limit/v3alpha/BUILD b/api/envoy/config/filter/thrift/rate_limit/v3alpha/BUILD new file mode 100644 index 000000000000..a13183b9eb75 --- /dev/null +++ b/api/envoy/config/filter/thrift/rate_limit/v3alpha/BUILD @@ -0,0 +1,19 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/api/v3alpha/ratelimit:pkg", + "//envoy/config/ratelimit/v3alpha:pkg", + ], +) + +api_proto_library_internal( + name = "rate_limit", + srcs = ["rate_limit.proto"], + deps = [ + "//envoy/api/v3alpha/ratelimit", + "//envoy/config/ratelimit/v3alpha:rls", + ], +) diff --git a/api/envoy/config/filter/thrift/rate_limit/v3alpha/rate_limit.proto b/api/envoy/config/filter/thrift/rate_limit/v3alpha/rate_limit.proto new file mode 100644 index 000000000000..e10197b7103e --- /dev/null +++ b/api/envoy/config/filter/thrift/rate_limit/v3alpha/rate_limit.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +package envoy.config.filter.thrift.rate_limit.v3alpha; + +option java_outer_classname = "RateLimitProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.thrift.rate_limit.v3alpha"; + +import "envoy/config/ratelimit/v3alpha/rls.proto"; + +import "google/protobuf/duration.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +// [#protodoc-title: Rate limit] +// Rate limit :ref:`configuration overview `. + +// [#comment:next free field: 5] +message RateLimit { + // The rate limit domain to use in the rate limit service request. + string domain = 1 [(validate.rules).string.min_bytes = 1]; + + // Specifies the rate limit configuration stage. Each configured rate limit filter performs a + // rate limit check using descriptors configured in the + // :ref:`envoy_api_msg_config.filter.network.thrift_proxy.v3alpha.RouteAction` for the request. + // Only those entries with a matching stage number are used for a given filter. If not set, the + // default stage number is 0. + // + // .. note:: + // + // The filter supports a range of 0 - 10 inclusively for stage numbers. + uint32 stage = 2 [(validate.rules).uint32.lte = 10]; + + // The timeout in milliseconds for the rate limit service RPC. If not + // set, this defaults to 20ms. + google.protobuf.Duration timeout = 3 [(gogoproto.stdduration) = true]; + + // The filter's behaviour in case the rate limiting service does + // not respond back. When it is set to true, Envoy will not allow traffic in case of + // communication failure between rate limiting service and the proxy. + // Defaults to false. + bool failure_mode_deny = 4; + + // Configuration for an external rate limit service provider. If not + // specified, any calls to the rate limit service will immediately return + // success. + envoy.config.ratelimit.v3alpha.RateLimitServiceConfig rate_limit_service = 5 + [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/filter/thrift/router/v3alpha/BUILD b/api/envoy/config/filter/thrift/router/v3alpha/BUILD new file mode 100644 index 000000000000..68bd8c126b80 --- /dev/null +++ b/api/envoy/config/filter/thrift/router/v3alpha/BUILD @@ -0,0 +1,10 @@ +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package() + +api_proto_library_internal( + name = "router", + srcs = ["router.proto"], +) diff --git a/api/envoy/config/filter/thrift/router/v3alpha/router.proto b/api/envoy/config/filter/thrift/router/v3alpha/router.proto new file mode 100644 index 000000000000..9fe86566a488 --- /dev/null +++ b/api/envoy/config/filter/thrift/router/v3alpha/router.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package envoy.config.filter.thrift.router.v3alpha; + +option java_outer_classname = "RouterProto"; +option java_multiple_files = true; +option java_package = "io.envoyproxy.envoy.config.filter.thrift.router.v3alpha"; + +// [#protodoc-title: Router] +// Thrift router :ref:`configuration overview `. + +message Router { +} diff --git a/api/envoy/service/discovery/v3alpha/hds.proto b/api/envoy/service/discovery/v3alpha/hds.proto index 5ec7f491c8f2..14955b15dbe8 100644 --- a/api/envoy/service/discovery/v3alpha/hds.proto +++ b/api/envoy/service/discovery/v3alpha/hds.proto @@ -65,7 +65,7 @@ service HealthDiscoveryService { // to bind with the response? rpc FetchHealthCheck(HealthCheckRequestOrEndpointHealthResponse) returns (HealthCheckSpecifier) { option (google.api.http) = { - post: "/v2/discovery:health_check" + post: "/v3alpha/discovery:health_check" body: "*" }; } diff --git a/api/envoy/service/discovery/v3alpha/rtds.proto b/api/envoy/service/discovery/v3alpha/rtds.proto index 5a59cf13f814..d4184ab6e197 100644 --- a/api/envoy/service/discovery/v3alpha/rtds.proto +++ b/api/envoy/service/discovery/v3alpha/rtds.proto @@ -35,7 +35,7 @@ service RuntimeDiscoveryService { rpc FetchRuntime(envoy.api.v3alpha.DiscoveryRequest) returns (envoy.api.v3alpha.DiscoveryResponse) { option (google.api.http) = { - post: "/v2/discovery:runtime" + post: "/v3alpha/discovery:runtime" body: "*" }; } diff --git a/api/envoy/service/discovery/v3alpha/sds.proto b/api/envoy/service/discovery/v3alpha/sds.proto index 9f8aa92befa2..814edd07196d 100644 --- a/api/envoy/service/discovery/v3alpha/sds.proto +++ b/api/envoy/service/discovery/v3alpha/sds.proto @@ -27,7 +27,7 @@ service SecretDiscoveryService { rpc FetchSecrets(envoy.api.v3alpha.DiscoveryRequest) returns (envoy.api.v3alpha.DiscoveryResponse) { option (google.api.http) = { - post: "/v2/discovery:secrets" + post: "/v3alpha/discovery:secrets" body: "*" }; } diff --git a/api/envoy/service/tap/v3alpha/tapds.proto b/api/envoy/service/tap/v3alpha/tapds.proto index 542d88ed2285..11eea61a1dc8 100644 --- a/api/envoy/service/tap/v3alpha/tapds.proto +++ b/api/envoy/service/tap/v3alpha/tapds.proto @@ -27,7 +27,7 @@ service TapDiscoveryService { rpc FetchTapConfigs(envoy.api.v3alpha.DiscoveryRequest) returns (envoy.api.v3alpha.DiscoveryResponse) { option (google.api.http) = { - post: "/v2/discovery:tap_configs" + post: "/v3alpha/discovery:tap_configs" body: "*" }; } diff --git a/docs/build.sh b/docs/build.sh index b75ebac03e65..add8b95f3037 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -50,20 +50,22 @@ bazel build ${BAZEL_BUILD_OPTIONS} @envoy_api//docs:protos --aspects \ tools/protodoc/protodoc.bzl%proto_doc_aspect --output_groups=rst --action_env=CPROFILE_ENABLED=1 \ --action_env=ENVOY_BLOB_SHA --spawn_strategy=standalone --host_force_python=PY3 -declare -r DOC_PROTOS=$(bazel query "deps(@envoy_api//docs:protos)" | grep "^@envoy_api.*proto$") +declare -r DOCS_DEPS=$(bazel query "labels(deps, @envoy_api//docs:protos)") # Only copy in the protos we care about and know how to deal with in protodoc. -for p in ${DOC_PROTOS} +for PROTO_TARGET in ${DOCS_DEPS} do - declare PROTO_TARGET=$(bazel query "kind(proto_library, same_pkg_direct_rdeps($p))") - declare PROTO_TARGET_WITHOUT_PREFIX="${PROTO_TARGET#@envoy_api//}" - declare PROTO_TARGET_CANONICAL="${PROTO_TARGET_WITHOUT_PREFIX/:/\/}" - declare PROTO_FILE_WITHOUT_PREFIX="${p#@envoy_api//}" - declare PROTO_FILE_CANONICAL="${PROTO_FILE_WITHOUT_PREFIX/:/\/}" - declare DEST="${GENERATED_RST_DIR}/api-v2/${PROTO_FILE_CANONICAL#envoy/}".rst - mkdir -p "$(dirname "${DEST}")" - cp -f bazel-bin/external/envoy_api/"${PROTO_TARGET_CANONICAL}/${PROTO_FILE_CANONICAL}.rst" "$(dirname "${DEST}")" - [ -n "${CPROFILE_ENABLED}" ] && cp -f bazel-bin/"${p}".profile "$(dirname "${DEST}")" + for p in $(bazel query "labels(srcs, ${PROTO_TARGET})" ) + do + declare PROTO_TARGET_WITHOUT_PREFIX="${PROTO_TARGET#@envoy_api//}" + declare PROTO_TARGET_CANONICAL="${PROTO_TARGET_WITHOUT_PREFIX/:/\/}" + declare PROTO_FILE_WITHOUT_PREFIX="${p#@envoy_api//}" + declare PROTO_FILE_CANONICAL="${PROTO_FILE_WITHOUT_PREFIX/:/\/}" + declare DEST="${GENERATED_RST_DIR}/api-v2/${PROTO_FILE_CANONICAL#envoy/}".rst + mkdir -p "$(dirname "${DEST}")" + cp -f bazel-bin/external/envoy_api/"${PROTO_TARGET_CANONICAL}/${PROTO_FILE_CANONICAL}.rst" "$(dirname "${DEST}")" + [ -n "${CPROFILE_ENABLED}" ] && cp -f bazel-bin/"${p}".profile "$(dirname "${DEST}")" + done done mkdir -p ${GENERATED_RST_DIR}/api-docs diff --git a/tools/api/clone.sh b/tools/api/clone.sh index a9cdb63ffd91..a3199793434c 100755 --- a/tools/api/clone.sh +++ b/tools/api/clone.sh @@ -12,23 +12,23 @@ set -e declare -r OLD_VERSION="$1" declare -r NEW_VERSION="$2" -# For vM -> vN, replace //$1*/vMalpha with //$1*/vN in BUILD file $2 +# For vM -> vN, replace //$1*/vMalpha\d* with //$1*/vN in BUILD file $2 # For vM -> vN, replace //$1*/vM with //$1*/vN in BUILD file $2 function replace_build() { - sed -i -e "s#\(//$1[^\S]*\)/${OLD_VERSION}alpha#\1/${NEW_VERSION}#g" "$2" + sed -i -e "s#\(//$1[^\S]*\)/${OLD_VERSION}alpha[[:digit:]]*#\1/${NEW_VERSION}#g" "$2" sed -i -e "s#\(//$1[^\S]*\)/${OLD_VERSION}#\1/${NEW_VERSION}#g" "$2" } # For vM -> vN, replace $1*[./]vMalpha with $1*[./]vN in .proto file $2 # For vM -> vN, replace $1*[./]vM with $1*[./]vN in .proto file $2 function replace_proto() { - sed -i -e "s#\($1\S*[\./]\)${OLD_VERSION}alpha#\1${NEW_VERSION}#g" "$2" + sed -i -e "s#\($1\S*[\./]\)${OLD_VERSION}alpha[[:digit:]]*#\1${NEW_VERSION}#g" "$2" sed -i -e "s#\($1\S*[\./]\)${OLD_VERSION}#\1${NEW_VERSION}#g" "$2" } # We consider both {vM, vMalpha} to deal with the multiple possible combinations # of {vM, vMalpha} existence for a given package. -for p in $(find api/ -name "${OLD_VERSION}" -o -name "${OLD_VERSION}alpha") +for p in $(find api/ -name "${OLD_VERSION}*") do declare PACKAGE_ROOT="$(dirname "$p")" declare OLD_VERSION_ROOT="${PACKAGE_ROOT}/${OLD_VERSION}" @@ -47,6 +47,8 @@ do for b in $(find "${NEW_VERSION_ROOT}" -name BUILD) do replace_build envoy "$b" + # Misc. cleanup for go BUILD rules + sed -i -e "s#\"${OLD_VERSION}\"#\"${NEW_VERSION}\"#g" "$b" done # Update .proto files with vM -> vN @@ -58,5 +60,6 @@ do replace_proto common "$f" replace_proto config "$f" replace_proto filter "$f" + replace_proto "" "$f" done done From 2e6695b91a11ca0943279a57d4bf37b9e39548fc Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Wed, 4 Sep 2019 16:04:01 -0700 Subject: [PATCH 499/542] docs: remove extraneous escape (#8150) Old versions of bash (e.g. on macOS) don't handle ${P/:/\/} the same way as modern versions. In particular, the expanded parameter on macOS includes a backslash, causing subsequent use of the string as a filename to include a slash (/) instead of treating the slash as a directory separator. Both versions of bash accept ${P/://} as a way to substitute : with /. Verified that this change does not alter the generated docs when running under Linux. Risk Level: low Testing: generated docs under linux & macOS Signed-off-by: Stephan Zuercher --- docs/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/build.sh b/docs/build.sh index add8b95f3037..a2c64b123f59 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -58,9 +58,9 @@ do for p in $(bazel query "labels(srcs, ${PROTO_TARGET})" ) do declare PROTO_TARGET_WITHOUT_PREFIX="${PROTO_TARGET#@envoy_api//}" - declare PROTO_TARGET_CANONICAL="${PROTO_TARGET_WITHOUT_PREFIX/:/\/}" + declare PROTO_TARGET_CANONICAL="${PROTO_TARGET_WITHOUT_PREFIX/://}" declare PROTO_FILE_WITHOUT_PREFIX="${p#@envoy_api//}" - declare PROTO_FILE_CANONICAL="${PROTO_FILE_WITHOUT_PREFIX/:/\/}" + declare PROTO_FILE_CANONICAL="${PROTO_FILE_WITHOUT_PREFIX/://}" declare DEST="${GENERATED_RST_DIR}/api-v2/${PROTO_FILE_CANONICAL#envoy/}".rst mkdir -p "$(dirname "${DEST}")" cp -f bazel-bin/external/envoy_api/"${PROTO_TARGET_CANONICAL}/${PROTO_FILE_CANONICAL}.rst" "$(dirname "${DEST}")" From f62a389c2e3752a35077e9d0dfc024844322e84f Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Wed, 4 Sep 2019 16:10:06 -0700 Subject: [PATCH 500/542] Do not 503 on Upgrade: h2c instead remove the header and ignore. (#7981) Description: When a request comes in on http1 with "upgrade: h2c", the current behavior is to 503. Instead we should ignore the upgrade and remove the header and continue with the request as http1. Risk Level: Medium Testing: Unit test. Hand test with ithub.com/rdsubhas/java-istio client server locally. Docs Changes: N/A Release Notes: http1: ignore and remove Upgrade: h2c. Fixes istio/istio#16391 Signed-off-by: John Plevyak --- docs/root/intro/version_history.rst | 1 + source/common/common/utility.cc | 10 ++++ source/common/common/utility.h | 72 ++++++++++++++--------- source/common/http/headers.h | 3 + source/common/http/http1/codec_impl.cc | 33 ++++++++++- test/common/common/utility_speed_test.cc | 22 +++++++ test/common/common/utility_test.cc | 22 +++++++ test/common/http/http1/codec_impl_test.cc | 37 ++++++++++++ 8 files changed, 169 insertions(+), 31 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 653bc4f65d6d..b10f9ce98b29 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -30,6 +30,7 @@ Version history * http: changed Envoy to forward existing x-forwarded-proto from upstream trusted proxies. Guarded by `envoy.reloadable_features.trusted_forwarded_proto` which defaults true. * http: added the ability to configure the behavior of the server response header, via the :ref:`server_header_transformation` field. * http: added the ability to :ref:`merge adjacent slashes` in the path. +* http: remove h2c upgrade headers for HTTP/1 as h2c upgrades are currently not supported. * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. diff --git a/source/common/common/utility.cc b/source/common/common/utility.cc index 3809da40e6d1..4528200b11fc 100644 --- a/source/common/common/utility.cc +++ b/source/common/common/utility.cc @@ -324,6 +324,16 @@ std::vector StringUtil::splitToken(absl::string_view source, return absl::StrSplit(source, absl::ByAnyChar(delimiters), absl::SkipEmpty()); } +std::string StringUtil::removeTokens(absl::string_view source, absl::string_view delimiters, + const CaseUnorderedSet& tokens_to_remove, + absl::string_view joiner) { + auto values = Envoy::StringUtil::splitToken(source, delimiters); + std::for_each(values.begin(), values.end(), [](auto& v) { v = StringUtil::trim(v); }); + auto end = std::remove_if(values.begin(), values.end(), + [&](absl::string_view t) { return tokens_to_remove.count(t) != 0; }); + return absl::StrJoin(values.begin(), end, joiner); +} + uint32_t StringUtil::itoa(char* out, size_t buffer_size, uint64_t i) { // The maximum size required for an unsigned 64-bit integer is 21 chars (including null). if (buffer_size < 21) { diff --git a/source/common/common/utility.h b/source/common/common/utility.h index 48c09be97548..9a74512da743 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -144,6 +144,35 @@ class DateUtil { */ class StringUtil { public: + /** + * Callable struct that returns the result of string comparison ignoring case. + * @param lhs supplies the first string view. + * @param rhs supplies the second string view. + * @return true if strings are semantically the same and false otherwise. + */ + struct CaseInsensitiveCompare { + // Enable heterogeneous lookup (https://abseil.io/tips/144) + using is_transparent = void; + bool operator()(absl::string_view lhs, absl::string_view rhs) const; + }; + + /** + * Callable struct that returns the hash representation of a case-insensitive string_view input. + * @param key supplies the string view. + * @return uint64_t hash representation of the supplied string view. + */ + struct CaseInsensitiveHash { + // Enable heterogeneous lookup (https://abseil.io/tips/144) + using is_transparent = void; + uint64_t operator()(absl::string_view key) const; + }; + + /** + * Definition of unordered set of case-insensitive std::string. + */ + using CaseUnorderedSet = + absl::flat_hash_set; + static const char WhitespaceChars[]; /** @@ -281,6 +310,20 @@ class StringUtil { absl::string_view delimiters, bool keep_empty_string = false); + /** + * Remove tokens from a delimiter-separated string view. The tokens are trimmed before + * they are compared ignoring case with the elements of 'tokens_to_remove'. The output is + * built from the trimmed tokens preserving case. + * @param source supplies the delimiter-separated string view. + * @param multi-delimiters supplies chars used to split the delimiter-separated string view. + * @param tokens_to_remove supplies a set of tokens which should not appear in the result. + * @param joiner contains a string used between tokens in the result. + * @return string of the remaining joined tokens. + */ + static std::string removeTokens(absl::string_view source, absl::string_view delimiters, + const CaseUnorderedSet& tokens_to_remove, + absl::string_view joiner); + /** * Size-bounded string copying and concatenation */ @@ -332,35 +375,6 @@ class StringUtil { */ static std::string toLower(absl::string_view s); - /** - * Callable struct that returns the result of string comparison ignoring case. - * @param lhs supplies the first string view. - * @param rhs supplies the second string view. - * @return true if strings are semantically the same and false otherwise. - */ - struct CaseInsensitiveCompare { - // Enable heterogeneous lookup (https://abseil.io/tips/144) - using is_transparent = void; - bool operator()(absl::string_view lhs, absl::string_view rhs) const; - }; - - /** - * Callable struct that returns the hash representation of a case-insensitive string_view input. - * @param key supplies the string view. - * @return uint64_t hash representation of the supplied string view. - */ - struct CaseInsensitiveHash { - // Enable heterogeneous lookup (https://abseil.io/tips/144) - using is_transparent = void; - uint64_t operator()(absl::string_view key) const; - }; - - /** - * Definition of unordered set of case-insensitive std::string. - */ - using CaseUnorderedSet = - absl::flat_hash_set; - /** * Removes all the character indices from str contained in the interval-set. * @param str the string containing the characters to be removed. diff --git a/source/common/http/headers.h b/source/common/http/headers.h index b0bcb7cc48ce..9d53886d6870 100644 --- a/source/common/http/headers.h +++ b/source/common/http/headers.h @@ -125,6 +125,7 @@ class HeaderValues { const LowerCaseString GrpcAcceptEncoding{"grpc-accept-encoding"}; const LowerCaseString Host{":authority"}; const LowerCaseString HostLegacy{"host"}; + const LowerCaseString Http2Settings{"http2-settings"}; const LowerCaseString KeepAlive{"keep-alive"}; const LowerCaseString LastModified{"last-modified"}; const LowerCaseString Location{"location"}; @@ -153,11 +154,13 @@ class HeaderValues { struct { const std::string Close{"close"}; + const std::string Http2Settings{"http2-settings"}; const std::string KeepAlive{"keep-alive"}; const std::string Upgrade{"upgrade"}; } ConnectionValues; struct { + const std::string H2c{"h2c"}; const std::string WebSocket{"websocket"}; } UpgradeValues; diff --git a/source/common/http/http1/codec_impl.cc b/source/common/http/http1/codec_impl.cc index ba631706f310..afadd309c192 100644 --- a/source/common/http/http1/codec_impl.cc +++ b/source/common/http/http1/codec_impl.cc @@ -21,6 +21,15 @@ namespace Envoy { namespace Http { namespace Http1 { +namespace { + +const StringUtil::CaseUnorderedSet& caseUnorderdSetContainingUpgradeAndHttp2Settings() { + CONSTRUCT_ON_FIRST_USE(StringUtil::CaseUnorderedSet, + Http::Headers::get().ConnectionValues.Upgrade, + Http::Headers::get().ConnectionValues.Http2Settings); +} + +} // namespace const std::string StreamEncoderImpl::CRLF = "\r\n"; const std::string StreamEncoderImpl::LAST_CHUNK = "0\r\n\r\n"; @@ -469,8 +478,28 @@ int ConnectionImpl::onHeadersCompleteBase() { protocol_ = Protocol::Http10; } if (Utility::isUpgrade(*current_header_map_)) { - ENVOY_CONN_LOG(trace, "codec entering upgrade mode.", connection_); - handling_upgrade_ = true; + // Ignore h2c upgrade requests until we support them. + // See https://github.com/envoyproxy/envoy/issues/7161 for details. + if (current_header_map_->Upgrade() && + absl::EqualsIgnoreCase(current_header_map_->Upgrade()->value().getStringView(), + Http::Headers::get().UpgradeValues.H2c)) { + ENVOY_CONN_LOG(trace, "removing unsupported h2c upgrade headers.", connection_); + current_header_map_->removeUpgrade(); + if (current_header_map_->Connection()) { + const auto& tokens_to_remove = caseUnorderdSetContainingUpgradeAndHttp2Settings(); + std::string new_value = StringUtil::removeTokens( + current_header_map_->Connection()->value().getStringView(), ",", tokens_to_remove, ","); + if (new_value.empty()) { + current_header_map_->removeConnection(); + } else { + current_header_map_->Connection()->value(new_value); + } + } + current_header_map_->remove(Headers::get().Http2Settings); + } else { + ENVOY_CONN_LOG(trace, "codec entering upgrade mode.", connection_); + handling_upgrade_ = true; + } } int rc = onHeadersComplete(std::move(current_header_map_)); diff --git a/test/common/common/utility_speed_test.cc b/test/common/common/utility_speed_test.cc index 69ed1b41ff4d..6e262c08bfdb 100644 --- a/test/common/common/utility_speed_test.cc +++ b/test/common/common/utility_speed_test.cc @@ -206,6 +206,28 @@ static void BM_FindTokenValueNoSplit(benchmark::State& state) { } BENCHMARK(BM_FindTokenValueNoSplit); +static void BM_RemoveTokensLong(benchmark::State& state) { + auto size = state.range(0); + std::string input(size, ','); + std::vector to_remove; + StringUtil::CaseUnorderedSet to_remove_set; + for (decltype(size) i = 0; i < size; i++) { + to_remove.push_back(std::to_string(i)); + } + for (int i = 0; i < size; i++) { + if (i & 1) { + to_remove_set.insert(to_remove[i]); + } + input.append(","); + input.append(to_remove[i]); + } + for (auto _ : state) { + Envoy::StringUtil::removeTokens(input, ",", to_remove_set, ","); + state.SetBytesProcessed(static_cast(state.iterations()) * input.size()); + } +} +BENCHMARK(BM_RemoveTokensLong)->Range(8, 8 << 10); + static void BM_IntervalSetInsert17(benchmark::State& state) { for (auto _ : state) { Envoy::IntervalSetImpl interval_set; diff --git a/test/common/common/utility_test.cc b/test/common/common/utility_test.cc index 750973e79d7f..d4ea1e1f4175 100644 --- a/test/common/common/utility_test.cc +++ b/test/common/common/utility_test.cc @@ -368,6 +368,28 @@ TEST(StringUtil, StringViewSplit) { } } +TEST(StringUtil, StringViewRemoveTokens) { + // Basic cases. + EXPECT_EQ(StringUtil::removeTokens("", ",", {"two"}, ","), ""); + EXPECT_EQ(StringUtil::removeTokens("one", ",", {"two"}, ","), "one"); + EXPECT_EQ(StringUtil::removeTokens("one,two ", ",", {"two"}, ","), "one"); + EXPECT_EQ(StringUtil::removeTokens("one,two ", ",", {"two", "one"}, ","), ""); + EXPECT_EQ(StringUtil::removeTokens("one,two ", ",", {"one"}, ","), "two"); + EXPECT_EQ(StringUtil::removeTokens("one,two,three ", ",", {"two"}, ","), "one,three"); + EXPECT_EQ(StringUtil::removeTokens(" one , two , three ", ",", {"two"}, ","), "one,three"); + EXPECT_EQ(StringUtil::removeTokens(" one , two , three ", ",", {"three"}, ","), "one,two"); + EXPECT_EQ(StringUtil::removeTokens(" one , two , three ", ",", {"three"}, ", "), "one, two"); + EXPECT_EQ(StringUtil::removeTokens("one,two,three", ",", {"two", "three"}, ","), "one"); + EXPECT_EQ(StringUtil::removeTokens("one,two,three,four", ",", {"two", "three"}, ","), "one,four"); + // Ignore case. + EXPECT_EQ(StringUtil::removeTokens("One,Two,Three,Four", ",", {"two", "three"}, ","), "One,Four"); + // Longer joiner. + EXPECT_EQ(StringUtil::removeTokens("one,two,three,four", ",", {"two", "three"}, " , "), + "one , four"); + // Delimiters. + EXPECT_EQ(StringUtil::removeTokens("one,two;three ", ",;", {"two"}, ","), "one,three"); +} + TEST(StringUtil, removeCharacters) { IntervalSetImpl removals; removals.insert(3, 5); diff --git a/test/common/http/http1/codec_impl_test.cc b/test/common/http/http1/codec_impl_test.cc index 6329e8bdc1bf..0303cfee5be1 100644 --- a/test/common/http/http1/codec_impl_test.cc +++ b/test/common/http/http1/codec_impl_test.cc @@ -720,6 +720,43 @@ TEST_F(Http1ServerConnectionImplTest, RequestWithTrailers) { EXPECT_EQ(0U, buffer.length()); } +TEST_F(Http1ServerConnectionImplTest, IgnoreUpgradeH2c) { + initialize(); + + TestHeaderMapImpl expected_headers{ + {":authority", "www.somewhere.com"}, {":path", "/"}, {":method", "GET"}}; + Buffer::OwnedImpl buffer( + "GET http://www.somewhere.com/ HTTP/1.1\r\nConnection: " + "Upgrade, HTTP2-Settings\r\nUpgrade: h2c\r\nHTTP2-Settings: token64\r\nHost: bah\r\n\r\n"); + expectHeadersTest(Protocol::Http11, true, buffer, expected_headers); +} + +TEST_F(Http1ServerConnectionImplTest, IgnoreUpgradeH2cClose) { + initialize(); + + TestHeaderMapImpl expected_headers{{":authority", "www.somewhere.com"}, + {":path", "/"}, + {":method", "GET"}, + {"connection", "Close"}}; + Buffer::OwnedImpl buffer("GET http://www.somewhere.com/ HTTP/1.1\r\nConnection: " + "Upgrade, Close, HTTP2-Settings\r\nUpgrade: h2c\r\nHTTP2-Settings: " + "token64\r\nHost: bah\r\n\r\n"); + expectHeadersTest(Protocol::Http11, true, buffer, expected_headers); +} + +TEST_F(Http1ServerConnectionImplTest, IgnoreUpgradeH2cCloseEtc) { + initialize(); + + TestHeaderMapImpl expected_headers{{":authority", "www.somewhere.com"}, + {":path", "/"}, + {":method", "GET"}, + {"connection", "Close,Etc"}}; + Buffer::OwnedImpl buffer("GET http://www.somewhere.com/ HTTP/1.1\r\nConnection: " + "Upgrade, Close, HTTP2-Settings, Etc\r\nUpgrade: h2c\r\nHTTP2-Settings: " + "token64\r\nHost: bah\r\n\r\n"); + expectHeadersTest(Protocol::Http11, true, buffer, expected_headers); +} + TEST_F(Http1ServerConnectionImplTest, UpgradeRequest) { initialize(); From 32590dcb2cfbf88d3af6ea1a4684aac1c077bb97 Mon Sep 17 00:00:00 2001 From: Lisa Lu Date: Wed, 4 Sep 2019 16:27:30 -0700 Subject: [PATCH 501/542] docs: add line on installing xcode for macOS build flow (#8139) Because of rules_foreign_cc in bazelbuild, Envoy will not compile successfully when following the instructions in the build docs due to how the tools are referenced. One fix for this is installing Xcode from the App Store (see bazelbuild/rules_foreign_cc#185). Risk Level: Low Testing: N/A (docs change) Docs Changes: see Description Release Notes: N/A Signed-off-by: Lisa Lu --- bazel/README.md | 1 + ci/README.md | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bazel/README.md b/bazel/README.md index b99b9e50621e..499dd46b80cf 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -65,6 +65,7 @@ for how to update or override dependencies. ``` _notes_: `coreutils` is used for `realpath`, `gmd5sum` and `gsha256sum` + XCode is also required to build Envoy on macOS. Envoy compiles and passes tests with the version of clang installed by XCode 9.3.0: Apple LLVM version 9.1.0 (clang-902.0.30). diff --git a/ci/README.md b/ci/README.md index 04864d581498..72d1600fac52 100644 --- a/ci/README.md +++ b/ci/README.md @@ -133,7 +133,8 @@ The macOS CI build is part of the [CircleCI](https://circleci.com/gh/envoyproxy/ Dependencies are installed by the `ci/mac_ci_setup.sh` script, via [Homebrew](https://brew.sh), which is pre-installed on the CircleCI macOS image. The dependencies are cached are re-installed on every build. The `ci/mac_ci_steps.sh` script executes the specific commands that -build and test Envoy. +build and test Envoy. If Envoy cannot be built (`error: /Library/Developer/CommandLineTools/usr/bin/libtool: no output file specified (specify with -o output)`), +ensure that XCode is installed. # Coverity Scan Build Flow From aeb5d695e6b9eaf775b90e95c21bd9b976752540 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Wed, 4 Sep 2019 16:32:46 -0700 Subject: [PATCH 502/542] docs: note which header expressions cannot be used for request headers (#8138) As discussed in #8127, some custom header expressions evaluate as empty when used in request headers. Risk Level: low, docs only Testing: n/a Docs Changes: updated Release Notes: n/a Signed-off-by: Stephan Zuercher --- docs/root/configuration/http/http_conn_man/headers.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/root/configuration/http/http_conn_man/headers.rst b/docs/root/configuration/http/http_conn_man/headers.rst index bfb3a0682ad7..0a5a1877f7fd 100644 --- a/docs/root/configuration/http/http_conn_man/headers.rst +++ b/docs/root/configuration/http/http_conn_man/headers.rst @@ -602,9 +602,13 @@ Supported variable names are: namespace and key(s) are specified as a JSON array of strings. Finally, percent symbols in the parameters **do not** need to be escaped by doubling them. + Upstream metadata cannot be added to request headers as the upstream host has not been selected + when custom request headers are generated. + %UPSTREAM_REMOTE_ADDRESS% Remote address of the upstream host. If the address is an IP address it includes both address - and port. + and port. The upstream remote address cannot be added to request headers as the upstream host + has not been selected when custom request headers are generated. %PER_REQUEST_STATE(reverse.dns.data.name)% Populates the header with values set on the stream info filterState() object. To be From 4478c1984d17146b1ff78d0babedae2a4752b027 Mon Sep 17 00:00:00 2001 From: Kuat Date: Wed, 4 Sep 2019 13:43:25 -1000 Subject: [PATCH 503/542] api: use traffic_direction over operation_name if specified (#7999) Use the listener-level field for the tracing direction over the per-filter field. Unfortunately, the per filter did not provide an "unspecified" default, so this appears to be the right approach to deprecate the per-filter field with minimal impact. Risk Level: low (uses a newly introduce field traffic_direction) Testing: unit test Docs Changes: proto docs Signed-off-by: Kuat Yessenov --- .../v2/http_connection_manager.proto | 10 +++- docs/root/intro/deprecated.rst | 5 ++ .../network/http_connection_manager/config.cc | 24 ++++++-- .../http_connection_manager/config_test.cc | 56 +++++++++++++++++++ 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index e20a91542b83..735cde619558 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -79,7 +79,6 @@ message HttpConnectionManager { google.protobuf.BoolValue add_user_agent = 6; message Tracing { - // [#comment:TODO(kyessenov): Align this field with listener traffic direction field.] enum OperationName { option (gogoproto.goproto_enum_prefix) = false; @@ -90,8 +89,13 @@ message HttpConnectionManager { EGRESS = 1; } - // The span name will be derived from this field. - OperationName operation_name = 1 [(validate.rules).enum.defined_only = true]; + // The span name will be derived from this field. If + // :ref:`traffic_direction ` is + // specified on the parent listener, then it is used instead of this field. + // + // .. attention:: + // This field has been deprecated in favor of `traffic_direction`. + OperationName operation_name = 1 [(validate.rules).enum.defined_only = true, deprecated = true]; // A list of header names used to create tags for the active span. The header name is used to // populate the tag name, and the header value is used to populate the tag value. The tag is diff --git a/docs/root/intro/deprecated.rst b/docs/root/intro/deprecated.rst index 57fd2dd5dbc3..d353ef72cf15 100644 --- a/docs/root/intro/deprecated.rst +++ b/docs/root/intro/deprecated.rst @@ -34,6 +34,11 @@ Version 1.12.0 (pending) * The use of HTTP_JSON_V1 :ref:`Zipkin collector endpoint version ` or not explicitly specifying it is deprecated, use HTTP_JSON or HTTP_PROTO instead. +* The `operation_name` field in :ref:`HTTP connection manager + ` + has been deprecated in favor of the `traffic_direction` field in + :ref:`Listener `. The latter takes priority if + specified. Version 1.11.0 (July 11, 2019) ============================== diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 81a45e934761..a229ecc08008 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -244,13 +244,27 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( Tracing::OperationName tracing_operation_name; std::vector request_headers_for_tags; - switch (tracing_config.operation_name()) { - case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: - Tracing::INGRESS: + // Listener level traffic direction overrides the operation name + switch (context.direction()) { + case envoy::api::v2::core::TrafficDirection::UNSPECIFIED: { + switch (tracing_config.operation_name()) { + case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: + Tracing::INGRESS: + tracing_operation_name = Tracing::OperationName::Ingress; + break; + case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: + Tracing::EGRESS: + tracing_operation_name = Tracing::OperationName::Egress; + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + break; + } + case envoy::api::v2::core::TrafficDirection::INBOUND: tracing_operation_name = Tracing::OperationName::Ingress; break; - case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: - Tracing::EGRESS: + case envoy::api::v2::core::TrafficDirection::OUTBOUND: tracing_operation_name = Tracing::OperationName::Egress; break; default: diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 0b048d28e3dc..c039d7a89c3f 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -242,6 +242,62 @@ stat_prefix: router EXPECT_EQ(5 * 60 * 1000, config.streamIdleTimeout().count()); } +TEST_F(HttpConnectionManagerConfigTest, ListenerDirectionOutboundOverride) { + const std::string yaml_string = R"EOF( +stat_prefix: router +route_config: + virtual_hosts: + - name: service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: cluster +tracing: + operation_name: ingress +http_filters: +- name: envoy.router + config: {} + )EOF"; + + ON_CALL(context_, direction()) + .WillByDefault(Return(envoy::api::v2::core::TrafficDirection::OUTBOUND)); + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + EXPECT_EQ(Tracing::OperationName::Egress, config.tracingConfig()->operation_name_); +} + +TEST_F(HttpConnectionManagerConfigTest, ListenerDirectionInboundOverride) { + const std::string yaml_string = R"EOF( +stat_prefix: router +route_config: + virtual_hosts: + - name: service + domains: + - "*" + routes: + - match: + prefix: "/" + route: + cluster: cluster +tracing: + operation_name: egress +http_filters: +- name: envoy.router + config: {} + )EOF"; + + ON_CALL(context_, direction()) + .WillByDefault(Return(envoy::api::v2::core::TrafficDirection::INBOUND)); + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + EXPECT_EQ(Tracing::OperationName::Ingress, config.tracingConfig()->operation_name_); +} + TEST_F(HttpConnectionManagerConfigTest, SamplingDefault) { const std::string yaml_string = R"EOF( stat_prefix: ingress_http From d11c7e7a9ee70ec0e5dd7623b3e4d4ff1a4b369c Mon Sep 17 00:00:00 2001 From: Yan Xue <3491507+yxue@users.noreply.github.com> Date: Thu, 5 Sep 2019 07:58:26 -0700 Subject: [PATCH 504/542] add more diagnostic logs (#8153) Istio sets listener filter timeout to 10ms by default but requests fail from time to tome. It's very difficult to debug. Even though downstream_pre_cx_timeout_ is exposed to track the number of timeouts, it would be better to have some debug logs. Description: add more diagnostic logs Risk Level: low Signed-off-by: crazyxy --- source/server/connection_handler_impl.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/server/connection_handler_impl.cc b/source/server/connection_handler_impl.cc index 4362ddb50d0f..29235715b51b 100644 --- a/source/server/connection_handler_impl.cc +++ b/source/server/connection_handler_impl.cc @@ -158,7 +158,11 @@ ConnectionHandlerImpl::findActiveListenerByAddress(const Network::Address::Insta void ConnectionHandlerImpl::ActiveSocket::onTimeout() { listener_.stats_.downstream_pre_cx_timeout_.inc(); ASSERT(inserted()); + ENVOY_LOG_TO_LOGGER(listener_.parent_.logger_, debug, "listener filter times out after {} ms", + listener_.listener_filters_timeout_.count()); + if (listener_.continue_on_listener_filters_timeout_) { + ENVOY_LOG_TO_LOGGER(listener_.parent_.logger_, debug, "fallback to default listener filter"); newConnection(); } unlink(); From 277e7175091cdf9d6c8732bc40381ff4e0fee48c Mon Sep 17 00:00:00 2001 From: Douglas Reid Date: Thu, 5 Sep 2019 11:26:43 -0700 Subject: [PATCH 505/542] http conn man: add tracing config for path length in tag (#8095) This PR adds a configuration option for controlling the length of the request path that is included in the HttpUrl span tag. Currently, this length is hard-coded to 256. With this PR, that length will be configurable (defaulting to the old value). Risk Level: Low Testing: Unit Docs Changes: Inline with the API proto. Documented new field. Release Notes: Added in the PR. Related issue: istio/istio#14563 Signed-off-by: Douglas Reid --- .../v2/http_connection_manager.proto | 5 +++++ docs/root/intro/version_history.rst | 1 + include/envoy/tracing/http_tracer.h | 7 +++++++ source/common/http/conn_manager_config.h | 1 + source/common/http/conn_manager_impl.cc | 4 ++++ source/common/http/conn_manager_impl.h | 1 + source/common/tracing/http_tracer_impl.cc | 8 +++++--- source/common/tracing/http_tracer_impl.h | 1 + .../network/http_connection_manager/config.cc | 6 +++++- test/common/http/conn_manager_impl_test.cc | 12 ++++++++---- test/common/http/conn_manager_utility_test.cc | 3 ++- test/common/tracing/http_tracer_impl_test.cc | 1 + .../network/http_connection_manager/config_test.cc | 3 +++ test/mocks/tracing/mocks.cc | 1 + test/mocks/tracing/mocks.h | 1 + 15 files changed, 46 insertions(+), 9 deletions(-) diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 735cde619558..86076823cc7e 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -130,6 +130,11 @@ message HttpConnectionManager { // Whether to annotate spans with additional data. If true, spans will include logs for stream // events. bool verbose = 6; + + // Maximum length of the request path to extract and include in the HttpUrl tag. Used to + // truncate lengthy request paths to meet the needs of a tracing backend. + // Default: 256 + google.protobuf.UInt32Value max_path_tag_length = 7; } // Presence of the object defines whether the connection manager diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index b10f9ce98b29..eebe4f9dd960 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -52,6 +52,7 @@ Version history * tracing: added support to the Zipkin reporter for sending list of spans as Zipkin JSON v2 and protobuf message over HTTP. certificate validation context. * tracing: added tags for gRPC response status and meesage. +* tracing: added :ref:`max_path_tag_length ` to support customizing the length of the request path included in the extracted `http.url ` tag. * upstream: added :ref:`an option ` that allows draining HTTP, TCP connection pools on cluster membership change. * upstream: added network filter chains to upstream connections, see :ref:`filters`. * upstream: use p2c to select hosts for least-requests load balancers if all host weights are the same, even in cases where weights are not equal to 1. diff --git a/include/envoy/tracing/http_tracer.h b/include/envoy/tracing/http_tracer.h index eb6f4960b62a..ec53d00e4bcc 100644 --- a/include/envoy/tracing/http_tracer.h +++ b/include/envoy/tracing/http_tracer.h @@ -11,6 +11,8 @@ namespace Envoy { namespace Tracing { +constexpr uint32_t DefaultMaxPathTagLength = 256; + enum class OperationName { Ingress, Egress }; /** @@ -58,6 +60,11 @@ class Config { * @return true if spans should be annotated with more detailed information. */ virtual bool verbose() const PURE; + + /** + * @return the maximum length allowed for paths in the extracted HttpUrl tag. + */ + virtual uint32_t maxPathTagLength() const PURE; }; class Span; diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index b32d8f520e40..ce4682de2a15 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -108,6 +108,7 @@ struct TracingConnectionManagerConfig { envoy::type::FractionalPercent random_sampling_; envoy::type::FractionalPercent overall_sampling_; bool verbose_; + uint32_t max_path_tag_length_; }; using TracingConnectionManagerConfigPtr = std::unique_ptr; diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 102373f9cadf..e8bde0714898 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1757,6 +1757,10 @@ bool ConnectionManagerImpl::ActiveStream::verbose() const { return connection_manager_.config_.tracingConfig()->verbose_; } +uint32_t ConnectionManagerImpl::ActiveStream::maxPathTagLength() const { + return connection_manager_.config_.tracingConfig()->max_path_tag_length_; +} + void ConnectionManagerImpl::ActiveStream::callHighWatermarkCallbacks() { ++high_watermark_count_; for (auto watermark_callbacks : watermark_callbacks_) { diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 661871ea991a..f6dd0ceb662c 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -479,6 +479,7 @@ class ConnectionManagerImpl : Logger::Loggable, Tracing::OperationName operationName() const override; const std::vector& requestHeadersForTags() const override; bool verbose() const override; + uint32_t maxPathTagLength() const override; // ScopeTrackedObject void dumpState(std::ostream& os, int indent_level = 0) const override { diff --git a/source/common/tracing/http_tracer_impl.cc b/source/common/tracing/http_tracer_impl.cc index 004e41c84a86..35bee4ff7de1 100644 --- a/source/common/tracing/http_tracer_impl.cc +++ b/source/common/tracing/http_tracer_impl.cc @@ -27,11 +27,12 @@ static std::string valueOrDefault(const Http::HeaderEntry* header, const char* d return header ? std::string(header->value().getStringView()) : default_value; } -static std::string buildUrl(const Http::HeaderMap& request_headers) { +static std::string buildUrl(const Http::HeaderMap& request_headers, + const uint32_t max_path_length) { std::string path(request_headers.EnvoyOriginalPath() ? request_headers.EnvoyOriginalPath()->value().getStringView() : request_headers.Path()->value().getStringView()); - static const size_t max_path_length = 256; + if (path.length() > max_path_length) { path = path.substr(0, max_path_length); } @@ -148,7 +149,8 @@ void HttpTracerUtility::finalizeSpan(Span& span, const Http::HeaderMap* request_ span.setTag(Tracing::Tags::get().GuidXRequestId, std::string(request_headers->RequestId()->value().getStringView())); } - span.setTag(Tracing::Tags::get().HttpUrl, buildUrl(*request_headers)); + span.setTag(Tracing::Tags::get().HttpUrl, + buildUrl(*request_headers, tracing_config.maxPathTagLength())); span.setTag(Tracing::Tags::get().HttpMethod, std::string(request_headers->Method()->value().getStringView())); span.setTag(Tracing::Tags::get().DownstreamCluster, diff --git a/source/common/tracing/http_tracer_impl.h b/source/common/tracing/http_tracer_impl.h index ec421736eab4..e7560acac0ea 100644 --- a/source/common/tracing/http_tracer_impl.h +++ b/source/common/tracing/http_tracer_impl.h @@ -118,6 +118,7 @@ class EgressConfigImpl : public Config { return request_headers_for_tags_; } bool verbose() const override { return false; } + uint32_t maxPathTagLength() const override { return Tracing::DefaultMaxPathTagLength; } private: const std::vector request_headers_for_tags_{}; diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index a229ecc08008..532f0e086e5d 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -8,6 +8,7 @@ #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.validate.h" #include "envoy/filesystem/filesystem.h" #include "envoy/server/admin.h" +#include "envoy/tracing/http_tracer.h" #include "common/access_log/access_log_impl.h" #include "common/common/fmt.h" @@ -288,10 +289,13 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( overall_sampling.set_numerator( tracing_config.has_overall_sampling() ? tracing_config.overall_sampling().value() : 100); + const uint32_t max_path_tag_length = PROTOBUF_GET_WRAPPED_OR_DEFAULT( + tracing_config, max_path_tag_length, Tracing::DefaultMaxPathTagLength); + tracing_config_ = std::make_unique(Http::TracingConnectionManagerConfig{ tracing_operation_name, request_headers_for_tags, client_sampling, random_sampling, - overall_sampling, tracing_config.verbose()}); + overall_sampling, tracing_config.verbose(), max_path_tag_length}); } for (const auto& access_log : config.access_log()) { diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index ebf2c71d59ad..179367e62589 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -128,7 +128,8 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan percent1, percent2, percent1, - false}); + false, + 256}); } } @@ -984,7 +985,8 @@ TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlowEgressDecorato percent1, percent2, percent1, - false}); + false, + 256}); auto* span = new NiceMock(); EXPECT_CALL(tracer_, startSpan_(_, _, _, _)) @@ -1062,7 +1064,8 @@ TEST_F(HttpConnectionManagerImplTest, StartAndFinishSpanNormalFlowEgressDecorato percent1, percent2, percent1, - false}); + false, + 256}); auto* span = new NiceMock(); EXPECT_CALL(tracer_, startSpan_(_, _, _, _)) @@ -1135,7 +1138,8 @@ TEST_F(HttpConnectionManagerImplTest, percent1, percent2, percent1, - false}); + false, + 256}); EXPECT_CALL(runtime_.snapshot_, featureEnabled("tracing.global_enabled", An(), _)) diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index b0c8e005124f..a9694d8ec785 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -102,7 +102,8 @@ class ConnectionManagerUtilityTest : public testing::Test { envoy::type::FractionalPercent percent2; percent2.set_numerator(10000); percent2.set_denominator(envoy::type::FractionalPercent::TEN_THOUSAND); - tracing_config_ = {Tracing::OperationName::Ingress, {}, percent1, percent2, percent1, false}; + tracing_config_ = { + Tracing::OperationName::Ingress, {}, percent1, percent2, percent1, false, 256}; ON_CALL(config_, tracingConfig()).WillByDefault(Return(&tracing_config_)); ON_CALL(config_, via()).WillByDefault(ReturnRef(via_)); diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index 4e1b991507f3..c78fa72e4106 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -336,6 +336,7 @@ TEST(HttpConnManFinalizerImpl, SpanPopulatedFailureResponse) { EXPECT_CALL(span, setTag(Eq("cc"), Eq("c"))); EXPECT_CALL(config, requestHeadersForTags()); EXPECT_CALL(config, verbose).WillOnce(Return(false)); + EXPECT_CALL(config, maxPathTagLength).WillOnce(Return(256)); absl::optional response_code(503); EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(ReturnPointee(&response_code)); diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index c039d7a89c3f..b16fbfc97ebb 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -224,6 +224,7 @@ stat_prefix: router operation_name: ingress request_headers_for_tags: - foo + max_path_tag_length: 128 http_filters: - name: envoy.router config: {} @@ -235,6 +236,7 @@ stat_prefix: router EXPECT_THAT(std::vector({Http::LowerCaseString("foo")}), ContainerEq(config.tracingConfig()->request_headers_for_tags_)); + EXPECT_EQ(128, config.tracingConfig()->max_path_tag_length_); EXPECT_EQ(*context_.local_info_.address_, config.localAddress()); EXPECT_EQ("foo", config.serverName()); EXPECT_EQ(HttpConnectionManagerConfig::HttpConnectionManagerProto::OVERWRITE, @@ -316,6 +318,7 @@ TEST_F(HttpConnectionManagerConfigTest, SamplingDefault) { scoped_routes_config_provider_manager_); EXPECT_EQ(100, config.tracingConfig()->client_sampling_.numerator()); + EXPECT_EQ(Tracing::DefaultMaxPathTagLength, config.tracingConfig()->max_path_tag_length_); EXPECT_EQ(envoy::type::FractionalPercent::HUNDRED, config.tracingConfig()->client_sampling_.denominator()); EXPECT_EQ(10000, config.tracingConfig()->random_sampling_.numerator()); diff --git a/test/mocks/tracing/mocks.cc b/test/mocks/tracing/mocks.cc index db9be3902320..3a688957f0af 100644 --- a/test/mocks/tracing/mocks.cc +++ b/test/mocks/tracing/mocks.cc @@ -16,6 +16,7 @@ MockConfig::MockConfig() { ON_CALL(*this, operationName()).WillByDefault(Return(operation_name_)); ON_CALL(*this, requestHeadersForTags()).WillByDefault(ReturnRef(headers_)); ON_CALL(*this, verbose()).WillByDefault(Return(verbose_)); + ON_CALL(*this, maxPathTagLength()).WillByDefault(Return(uint32_t(256))); } MockConfig::~MockConfig() = default; diff --git a/test/mocks/tracing/mocks.h b/test/mocks/tracing/mocks.h index 22a694edfd9a..1cd5b3a0c9c0 100644 --- a/test/mocks/tracing/mocks.h +++ b/test/mocks/tracing/mocks.h @@ -18,6 +18,7 @@ class MockConfig : public Config { MOCK_CONST_METHOD0(operationName, OperationName()); MOCK_CONST_METHOD0(requestHeadersForTags, const std::vector&()); MOCK_CONST_METHOD0(verbose, bool()); + MOCK_CONST_METHOD0(maxPathTagLength, uint32_t()); OperationName operation_name_{OperationName::Ingress}; std::vector headers_; From d21d92ca21cd827e8ab484244a9eefc31bfd8af4 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Thu, 5 Sep 2019 14:16:09 -0700 Subject: [PATCH 506/542] cds: Add general-purpose LB policy configuration (#7744) This PR adds fields to CDS that allow for general-purpose LB policy configuration. Risk Level: Low Testing: None (but if anything is needed, please let me know) Docs Changes: Inline with API protos Release Notes: N/A Signed-off-by: Mark D. Roth --- api/envoy/api/v2/cds.proto | 50 +++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index dec942853616..e87a992d0f32 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -52,7 +52,7 @@ service ClusterDiscoveryService { // [#protodoc-title: Clusters] // Configuration for a single upstream cluster. -// [#comment:next free field: 41] +// [#comment:next free field: 42] message Cluster { // Supplies the name of the cluster which must be unique across all clusters. // The cluster name is used when emitting @@ -175,6 +175,13 @@ message Cluster { // specific load balancer. Consult the configured cluster's documentation for whether to set // this option or not. CLUSTER_PROVIDED = 6; + + // [#not-implemented-hide:] Use the new :ref:`load_balancing_policy + // ` field to determine the LB policy. + // [#next-major-version: In the v3 API, we should consider deprecating the lb_policy field + // and instead using the new load_balancing_policy field as the one and only mechanism for + // configuring this.] + LOAD_BALANCING_POLICY_CONFIG = 7; } // The :ref:`load balancer type ` to use // when picking a host in the cluster. @@ -645,6 +652,47 @@ message Cluster { // outgoing connections made by the cluster. Order matters as the filters are // processed sequentially as connection events happen. repeated cluster.Filter filters = 40; + + // [#not-implemented-hide:] New mechanism for LB policy configuration. Used only if the + // :ref:`lb_policy` field has the value + // :ref:`LOAD_BALANCING_POLICY_CONFIG`. + LoadBalancingPolicy load_balancing_policy = 41; +} + +// [#not-implemented-hide:] Extensible load balancing policy configuration. +// +// Every LB policy defined via this mechanism will be identified via a unique name using reverse +// DNS notation. If the policy needs configuration parameters, it must define a message for its +// own configuration, which will be stored in the config field. The name of the policy will tell +// clients which type of message they should expect to see in the config field. +// +// Note that there are cases where it is useful to be able to independently select LB policies +// for choosing a locality and for choosing an endpoint within that locality. For example, a +// given deployment may always use the same policy to choose the locality, but for choosing the +// endpoint within the locality, some clusters may use weighted-round-robin, while others may +// use some sort of session-based balancing. +// +// This can be accomplished via hierarchical LB policies, where the parent LB policy creates a +// child LB policy for each locality. For each request, the parent chooses the locality and then +// delegates to the child policy for that locality to choose the endpoint within the locality. +// +// To facilitate this, the config message for the top-level LB policy may include a field of +// type LoadBalancingPolicy that specifies the child policy. +// +// [#proto-status: experimental] +message LoadBalancingPolicy { + message Policy { + // Required. The name of the LB policy. + string name = 1; + // Optional config for the LB policy. + // No more than one of these two fields may be populated. + google.protobuf.Struct config = 2; + google.protobuf.Any typed_config = 3; + } + // Each client will iterate over the list in order and stop at the first policy that it + // supports. This provides a mechanism for starting to use new LB policies that are not yet + // supported by all clients. + repeated Policy policies = 1; } // An extensible structure containing the address Envoy should bind to when From 22e48271092637a40ad73d84a84c1d887bf74bd7 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Thu, 5 Sep 2019 16:27:14 -0700 Subject: [PATCH 507/542] thrift_proxy: fix crash on invalid transport/protocol (#8143) Transport/protocol decoder errors that occur before the connection manager initializes an ActiveRPC to track the request caused a crash. Modifies the connection manager to handle this case, terminating the downstream the connection. Risk Level: low Testing: test case that triggers crash Docs Changes: n/a Release Notes: added Signed-off-by: Stephan Zuercher --- docs/root/intro/version_history.rst | 1 + .../network/thrift_proxy/conn_manager.cc | 10 +++- .../network/thrift_proxy/conn_manager_test.cc | 46 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index eebe4f9dd960..5ce0428bc4d7 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -48,6 +48,7 @@ Version history * router check tool: add coverage reporting & enforcement. * router check tool: add comprehensive coverage reporting. * router check tool: add deprecated field check. +* thrift_proxy: fix crashing bug on invalid transport/protocol framing * tls: added verification of IP address SAN fields in certificates against configured SANs in the * tracing: added support to the Zipkin reporter for sending list of spans as Zipkin JSON v2 and protobuf message over HTTP. certificate validation context. diff --git a/source/extensions/filters/network/thrift_proxy/conn_manager.cc b/source/extensions/filters/network/thrift_proxy/conn_manager.cc index 509d2f42d3e1..8f697c0a606d 100644 --- a/source/extensions/filters/network/thrift_proxy/conn_manager.cc +++ b/source/extensions/filters/network/thrift_proxy/conn_manager.cc @@ -77,8 +77,14 @@ void ConnectionManager::dispatch() { } catch (const EnvoyException& ex) { ENVOY_CONN_LOG(error, "thrift error: {}", read_callbacks_->connection(), ex.what()); - // Use the current rpc to send an error downstream, if possible. - rpcs_.front()->onError(ex.what()); + if (rpcs_.empty()) { + // Transport/protocol mismatch (including errors in automatic detection). Just hang up + // since we don't know how to encode a response. + read_callbacks_->connection().close(Network::ConnectionCloseType::FlushWrite); + } else { + // Use the current rpc's transport/protocol to send an error downstream. + rpcs_.front()->onError(ex.what()); + } } stats_.request_decoding_error_.inc(); diff --git a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc index 0e19ef9e943c..e0ee0f1d5907 100644 --- a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc @@ -509,6 +509,17 @@ TEST_F(ThriftConnectionManagerTest, OnDataHandlesTransportApplicationException) EXPECT_EQ(0U, stats_.request_active_.value()); } +// Tests that OnData handles non-thrift input. Regression test for crash on invalid input. +TEST_F(ThriftConnectionManagerTest, OnDataHandlesGarbageRequest) { + initializeFilter(); + addRepeated(buffer_, 8, 0); + EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::FlushWrite)); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(1U, store_.counter("test.request_decoding_error").value()); + EXPECT_EQ(0U, stats_.request_active_.value()); +} + TEST_F(ThriftConnectionManagerTest, OnEvent) { // No active calls { @@ -895,6 +906,41 @@ TEST_F(ThriftConnectionManagerTest, RequestAndTransportApplicationException) { EXPECT_EQ(1U, store_.counter("test.response_decoding_error").value()); } +// Tests that a request is routed and a non-thrift response is handled. +TEST_F(ThriftConnectionManagerTest, RequestAndGarbageResponse) { + initializeFilter(); + writeFramedBinaryMessage(buffer_, MessageType::Call, 0x0F); + + ThriftFilters::DecoderFilterCallbacks* callbacks{}; + EXPECT_CALL(*decoder_filter_, setDecoderFilterCallbacks(_)) + .WillOnce( + Invoke([&](ThriftFilters::DecoderFilterCallbacks& cb) -> void { callbacks = &cb; })); + + EXPECT_EQ(filter_->onData(buffer_, false), Network::FilterStatus::StopIteration); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + + addRepeated(write_buffer_, 8, 0); + + FramedTransportImpl transport; + BinaryProtocolImpl proto; + callbacks->startUpstreamResponse(transport, proto); + + EXPECT_CALL(filter_callbacks_.connection_.dispatcher_, deferredDelete_(_)).Times(1); + EXPECT_EQ(ThriftFilters::ResponseStatus::Reset, callbacks->upstreamData(write_buffer_)); + + filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList(); + + EXPECT_EQ(1U, store_.counter("test.request").value()); + EXPECT_EQ(1U, store_.counter("test.request_call").value()); + EXPECT_EQ(0U, stats_.request_active_.value()); + EXPECT_EQ(0U, store_.counter("test.response").value()); + EXPECT_EQ(0U, store_.counter("test.response_reply").value()); + EXPECT_EQ(1U, store_.counter("test.response_exception").value()); + EXPECT_EQ(0U, store_.counter("test.response_invalid_type").value()); + EXPECT_EQ(0U, store_.counter("test.response_success").value()); + EXPECT_EQ(0U, store_.counter("test.response_error").value()); +} + TEST_F(ThriftConnectionManagerTest, PipelinedRequestAndResponse) { initializeFilter(); From e7f0b7176efdc65f96eb1697b829d1e6187f4502 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 5 Sep 2019 14:28:05 -1000 Subject: [PATCH 508/542] api: strip gogoproto annotations (#8163) Remove gogoproto annotations. They can be replaced with a custom gogoproto compiler (e.g. something like https://github.com/gogo/googleapis/tree/master/protoc-gen-gogogoogleapis). I have an experimental version of it to validate that it's possible to re-apply important annotations in the compiler. Risk Level: low Testing: builds Signed-off-by: Kuat Yessenov --- api/bazel/api_build_system.bzl | 4 -- api/bazel/repositories.bzl | 61 +------------------ api/bazel/repository_locations.bzl | 8 --- api/envoy/admin/v2alpha/config_dump.proto | 2 - api/envoy/admin/v3alpha/config_dump.proto | 2 - api/envoy/api/v2/auth/cert.proto | 3 - api/envoy/api/v2/cds.proto | 13 +--- .../api/v2/cluster/circuit_breaker.proto | 4 -- api/envoy/api/v2/cluster/filter.proto | 3 - .../api/v2/cluster/outlier_detection.proto | 3 - api/envoy/api/v2/core/address.proto | 4 -- api/envoy/api/v2/core/base.proto | 6 -- api/envoy/api/v2/core/config_source.proto | 8 +-- api/envoy/api/v2/core/grpc_service.proto | 3 - api/envoy/api/v2/core/health_check.proto | 25 +++----- api/envoy/api/v2/core/http_uri.proto | 10 +-- api/envoy/api/v2/core/protocol.proto | 5 +- api/envoy/api/v2/discovery.proto | 4 -- api/envoy/api/v2/eds.proto | 4 -- api/envoy/api/v2/endpoint/endpoint.proto | 3 - api/envoy/api/v2/endpoint/load_report.proto | 1 - api/envoy/api/v2/lds.proto | 5 +- api/envoy/api/v2/listener/listener.proto | 3 - api/envoy/api/v2/rds.proto | 3 - api/envoy/api/v2/route/route.proto | 30 ++++----- api/envoy/api/v2/srds.proto | 2 - api/envoy/api/v3alpha/auth/cert.proto | 3 - api/envoy/api/v3alpha/cds.proto | 13 +--- .../api/v3alpha/cluster/circuit_breaker.proto | 4 -- api/envoy/api/v3alpha/cluster/filter.proto | 3 - .../v3alpha/cluster/outlier_detection.proto | 3 - api/envoy/api/v3alpha/core/address.proto | 4 -- api/envoy/api/v3alpha/core/base.proto | 6 -- .../api/v3alpha/core/config_source.proto | 8 +-- api/envoy/api/v3alpha/core/grpc_service.proto | 3 - api/envoy/api/v3alpha/core/health_check.proto | 25 +++----- api/envoy/api/v3alpha/core/http_uri.proto | 10 +-- api/envoy/api/v3alpha/core/protocol.proto | 5 +- api/envoy/api/v3alpha/discovery.proto | 4 -- api/envoy/api/v3alpha/eds.proto | 4 -- api/envoy/api/v3alpha/endpoint/endpoint.proto | 3 - .../api/v3alpha/endpoint/load_report.proto | 1 - api/envoy/api/v3alpha/lds.proto | 5 +- api/envoy/api/v3alpha/listener/listener.proto | 3 - api/envoy/api/v3alpha/rds.proto | 3 - api/envoy/api/v3alpha/route/route.proto | 30 ++++----- api/envoy/api/v3alpha/srds.proto | 2 - api/envoy/config/bootstrap/v2/bootstrap.proto | 12 ++-- .../config/bootstrap/v3alpha/bootstrap.proto | 12 ++-- .../config/cluster/redis/redis_cluster.proto | 7 +-- .../config/common/tap/v2alpha/common.proto | 1 - .../config/common/tap/v3alpha/common.proto | 1 - api/envoy/config/filter/fault/v2/fault.proto | 4 +- .../config/filter/fault/v3alpha/fault.proto | 4 +- .../config/filter/http/buffer/v2/buffer.proto | 1 - .../filter/http/buffer/v3alpha/buffer.proto | 1 - .../config/filter/http/csrf/v2/csrf.proto | 1 - .../filter/http/csrf/v3alpha/csrf.proto | 1 - .../filter/http/ext_authz/v2/ext_authz.proto | 3 - .../http/ext_authz/v3alpha/ext_authz.proto | 3 - .../config/filter/http/gzip/v2/gzip.proto | 1 - .../filter/http/gzip/v3alpha/gzip.proto | 1 - .../http/health_check/v2/health_check.proto | 5 +- .../health_check/v3alpha/health_check.proto | 5 +- .../http/jwt_authn/v2alpha/config.proto | 3 - .../http/jwt_authn/v3alpha/config.proto | 3 - .../http/rate_limit/v2/rate_limit.proto | 3 +- .../http/rate_limit/v3alpha/rate_limit.proto | 3 +- .../config/filter/http/rbac/v2/rbac.proto | 1 - .../filter/http/rbac/v3alpha/rbac.proto | 1 - .../config/filter/http/squash/v2/squash.proto | 7 +-- .../filter/http/squash/v3alpha/squash.proto | 7 +-- .../client_ssl_auth/v2/client_ssl_auth.proto | 3 +- .../v3alpha/client_ssl_auth.proto | 3 +- .../dubbo_proxy/v2alpha1/dubbo_proxy.proto | 1 - .../network/dubbo_proxy/v2alpha1/route.proto | 3 - .../dubbo_proxy/v3alpha/dubbo_proxy.proto | 1 - .../network/dubbo_proxy/v3alpha/route.proto | 3 - .../v2/http_connection_manager.proto | 15 ++--- .../v3alpha/http_connection_manager.proto | 15 ++--- .../network/rate_limit/v2/rate_limit.proto | 3 +- .../rate_limit/v3alpha/rate_limit.proto | 3 +- .../config/filter/network/rbac/v2/rbac.proto | 1 - .../filter/network/rbac/v3alpha/rbac.proto | 1 - .../network/redis_proxy/v2/redis_proxy.proto | 6 +- .../redis_proxy/v3alpha/redis_proxy.proto | 6 +- .../network/tcp_proxy/v2/tcp_proxy.proto | 4 +- .../network/tcp_proxy/v3alpha/tcp_proxy.proto | 4 +- .../network/thrift_proxy/v2alpha1/route.proto | 1 - .../thrift_proxy/v2alpha1/thrift_proxy.proto | 3 - .../network/thrift_proxy/v3alpha/route.proto | 1 - .../thrift_proxy/v3alpha/thrift_proxy.proto | 3 - .../rate_limit/v2alpha1/rate_limit.proto | 3 +- .../rate_limit/v3alpha/rate_limit.proto | 3 +- api/envoy/config/rbac/v2/rbac.proto | 3 - api/envoy/config/rbac/v3alpha/rbac.proto | 3 - api/envoy/data/accesslog/v2/accesslog.proto | 19 +++--- .../data/accesslog/v3alpha/accesslog.proto | 19 +++--- .../v2alpha/outlier_detection_event.proto | 5 +- .../v3alpha/outlier_detection_event.proto | 5 +- .../core/v2alpha/health_check_event.proto | 5 +- .../core/v3alpha/health_check_event.proto | 5 +- .../service/auth/v2/attribute_context.proto | 3 - .../auth/v3alpha/attribute_context.proto | 3 - api/envoy/type/percent.proto | 3 - api/envoy/type/range.proto | 4 -- 106 files changed, 121 insertions(+), 494 deletions(-) diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index 0ad60bf84a32..9e0d48ee174b 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -20,7 +20,6 @@ _COMMON_PROTO_DEPS = [ "@com_google_googleapis//google/api:http_proto", "@com_google_googleapis//google/api:annotations_proto", "@com_google_googleapis//google/rpc:status_proto", - "@com_github_gogo_protobuf//:gogo_proto", "@com_envoyproxy_protoc_gen_validate//validate:validate_proto", ] @@ -50,7 +49,6 @@ def api_py_proto_library(name, srcs = [], deps = [], external_py_proto_deps = [] "@com_google_googleapis//google/api:annotations_py_proto", "@com_google_googleapis//google/api:http_py_proto", "@com_google_googleapis//google/api:httpbody_py_proto", - "@com_github_gogo_protobuf//:gogo_proto_py", ], visibility = ["//visibility:public"], ) @@ -108,7 +106,6 @@ def api_proto_library( name = _Suffix(name, _CC_SUFFIX), linkstatic = linkstatic, cc_deps = [_LibrarySuffix(d, _CC_SUFFIX) for d in deps] + external_cc_proto_deps + [ - "@com_github_gogo_protobuf//:gogo_proto_cc", "@com_google_googleapis//google/api:http_cc_proto", "@com_google_googleapis//google/api:annotations_cc_proto", "@com_google_googleapis//google/rpc:status_cc_proto", @@ -180,7 +177,6 @@ def api_proto_package(name = "pkg", srcs = [], deps = [], has_services = False, proto = name, visibility = ["//visibility:public"], deps = [go_proto_mapping(dep) for dep in deps] + [ - "@com_github_gogo_protobuf//:gogo_proto_go", "@com_github_golang_protobuf//ptypes:go_default_library", "@com_github_golang_protobuf//ptypes/any:go_default_library", "@com_github_golang_protobuf//ptypes/duration:go_default_library", diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index b992717d273a..1362c5671acb 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -19,11 +19,7 @@ def api_dependencies(): name = "com_github_cncf_udpa", locations = REPOSITORY_LOCATIONS, ) - envoy_http_archive( - name = "com_github_gogo_protobuf", - locations = REPOSITORY_LOCATIONS, - build_file_content = GOGOPROTO_BUILD_CONTENT, - ) + envoy_http_archive( name = "prometheus_metrics_model", locations = REPOSITORY_LOCATIONS, @@ -44,61 +40,6 @@ def api_dependencies(): build_file_content = ZIPKINAPI_BUILD_CONTENT, ) -GOGOPROTO_BUILD_CONTENT = """ -load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") -load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") - -proto_library( - name = "gogo_proto", - srcs = [ - "gogoproto/gogo.proto", - ], - deps = [ - "@com_google_protobuf//:descriptor_proto", - ], - visibility = ["//visibility:public"], -) - -go_proto_library( - name = "descriptor_go_proto", - importpath = "github.com/golang/protobuf/protoc-gen-go/descriptor", - proto = "@com_google_protobuf//:descriptor_proto", - visibility = ["//visibility:public"], -) - -cc_proto_library( - name = "gogo_proto_cc", - srcs = [ - "gogoproto/gogo.proto", - ], - default_runtime = "@com_google_protobuf//:protobuf", - protoc = "@com_google_protobuf//:protoc", - deps = ["@com_google_protobuf//:cc_wkt_protos"], - visibility = ["//visibility:public"], -) - -go_proto_library( - name = "gogo_proto_go", - importpath = "gogoproto", - proto = ":gogo_proto", - visibility = ["//visibility:public"], - deps = [ - ":descriptor_go_proto", - ], -) - -py_proto_library( - name = "gogo_proto_py", - srcs = [ - "gogoproto/gogo.proto", - ], - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - visibility = ["//visibility:public"], - deps = ["@com_google_protobuf//:protobuf_python"], -) -""" - PROMETHEUSMETRICS_BUILD_CONTENT = """ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 8d688fd02e22..7b3702e52b10 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -1,9 +1,6 @@ BAZEL_SKYLIB_RELEASE = "0.8.0" BAZEL_SKYLIB_SHA256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e" -GOGOPROTO_RELEASE = "1.2.1" -GOGOPROTO_SHA256 = "99e423905ba8921e86817607a5294ffeedb66fdd4a85efce5eb2848f715fdb3a" - OPENCENSUS_PROTO_GIT_SHA = "5cec5ea58c3efa81fa808f2bd38ce182da9ee731" # Jul 25, 2019 OPENCENSUS_PROTO_SHA256 = "faeb93f293ff715b0cb530d273901c0e2e99277b9ed1c0a0326bca9ec5774ad2" @@ -45,11 +42,6 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "udpa-" + UDPA_GIT_SHA, urls = ["https://github.com/cncf/udpa/archive/" + UDPA_GIT_SHA + ".tar.gz"], ), - com_github_gogo_protobuf = dict( - sha256 = GOGOPROTO_SHA256, - strip_prefix = "protobuf-" + GOGOPROTO_RELEASE, - urls = ["https://github.com/gogo/protobuf/archive/v" + GOGOPROTO_RELEASE + ".tar.gz"], - ), prometheus_metrics_model = dict( sha256 = PROMETHEUS_SHA, strip_prefix = "client_model-" + PROMETHEUS_GIT_SHA, diff --git a/api/envoy/admin/v2alpha/config_dump.proto b/api/envoy/admin/v2alpha/config_dump.proto index 1025b17fb117..10c57f06f1b7 100644 --- a/api/envoy/admin/v2alpha/config_dump.proto +++ b/api/envoy/admin/v2alpha/config_dump.proto @@ -16,8 +16,6 @@ import "envoy/config/bootstrap/v2/bootstrap.proto"; import "google/protobuf/any.proto"; import "google/protobuf/timestamp.proto"; -import "gogoproto/gogo.proto"; - // [#protodoc-title: ConfigDump] // The :ref:`/config_dump ` admin endpoint uses this wrapper diff --git a/api/envoy/admin/v3alpha/config_dump.proto b/api/envoy/admin/v3alpha/config_dump.proto index e909b1adfa72..cc6afbdbadf9 100644 --- a/api/envoy/admin/v3alpha/config_dump.proto +++ b/api/envoy/admin/v3alpha/config_dump.proto @@ -16,8 +16,6 @@ import "envoy/config/bootstrap/v3alpha/bootstrap.proto"; import "google/protobuf/any.proto"; import "google/protobuf/timestamp.proto"; -import "gogoproto/gogo.proto"; - // [#protodoc-title: ConfigDump] // The :ref:`/config_dump ` admin endpoint uses this wrapper diff --git a/api/envoy/api/v2/auth/cert.proto b/api/envoy/api/v2/auth/cert.proto index 2a8267805161..0f331268205f 100644 --- a/api/envoy/api/v2/auth/cert.proto +++ b/api/envoy/api/v2/auth/cert.proto @@ -14,9 +14,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Common TLS configuration] diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index e87a992d0f32..a0095ac2aa2c 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -28,10 +28,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; // Return list of all clusters this proxy will load balance to. service ClusterDiscoveryService { @@ -127,8 +123,7 @@ message Cluster { EdsClusterConfig eds_cluster_config = 3; // The timeout for new network connections to hosts in the cluster. - google.protobuf.Duration connect_timeout = 4 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration connect_timeout = 4 [(validate.rules).duration.gt = {}]; // Soft limit on size of the cluster’s connections read and write buffers. If // unspecified, an implementation defined default is applied (1MiB). @@ -281,8 +276,7 @@ message Cluster { // :ref:`STRICT_DNS` // and :ref:`LOGICAL_DNS` // this setting is ignored. - google.protobuf.Duration dns_refresh_rate = 16 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration dns_refresh_rate = 16 [(validate.rules).duration.gt = {}]; // Optional configuration for setting cluster's DNS refresh rate. If the value is set to true, // cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS @@ -340,8 +334,7 @@ message Cluster { // value defaults to 5000ms. For cluster types other than // :ref:`ORIGINAL_DST` // this setting is ignored. - google.protobuf.Duration cleanup_interval = 20 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration cleanup_interval = 20 [(validate.rules).duration.gt = {}]; // Optional configuration used to bind newly established upstream connections. // This overrides any bind_config specified in the bootstrap proto. diff --git a/api/envoy/api/v2/cluster/circuit_breaker.proto b/api/envoy/api/v2/cluster/circuit_breaker.proto index 5ae8cc3d1a01..e36677c89b64 100644 --- a/api/envoy/api/v2/cluster/circuit_breaker.proto +++ b/api/envoy/api/v2/cluster/circuit_breaker.proto @@ -12,10 +12,6 @@ import "envoy/api/v2/core/base.proto"; import "google/protobuf/wrappers.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; - // [#protodoc-title: Circuit breakers] // :ref:`Circuit breaking` settings can be diff --git a/api/envoy/api/v2/cluster/filter.proto b/api/envoy/api/v2/cluster/filter.proto index 8a287b399751..94c683913953 100644 --- a/api/envoy/api/v2/cluster/filter.proto +++ b/api/envoy/api/v2/cluster/filter.proto @@ -11,9 +11,6 @@ option ruby_package = "Envoy.Api.V2.ClusterNS"; import "google/protobuf/any.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Upstream filters] // diff --git a/api/envoy/api/v2/cluster/outlier_detection.proto b/api/envoy/api/v2/cluster/outlier_detection.proto index 69feb60e2325..72cf038cb034 100644 --- a/api/envoy/api/v2/cluster/outlier_detection.proto +++ b/api/envoy/api/v2/cluster/outlier_detection.proto @@ -12,9 +12,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Outlier detection] diff --git a/api/envoy/api/v2/core/address.proto b/api/envoy/api/v2/core/address.proto index 88689e2648ce..362395577fc9 100644 --- a/api/envoy/api/v2/core/address.proto +++ b/api/envoy/api/v2/core/address.proto @@ -11,9 +11,6 @@ import "envoy/api/v2/core/base.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Network addresses] @@ -27,7 +24,6 @@ message Pipe { message SocketAddress { enum Protocol { - option (gogoproto.goproto_enum_prefix) = false; TCP = 0; // [#not-implemented-hide:] UDP = 1; diff --git a/api/envoy/api/v2/core/base.proto b/api/envoy/api/v2/core/base.proto index 0f73b83af03f..2a778f19afb1 100644 --- a/api/envoy/api/v2/core/base.proto +++ b/api/envoy/api/v2/core/base.proto @@ -13,13 +13,9 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; import "envoy/type/percent.proto"; -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; - // [#protodoc-title: Common types] // Identifies location of where either Envoy runs or where upstream hosts run. @@ -131,7 +127,6 @@ enum RoutingPriority { // HTTP request method. enum RequestMethod { - option (gogoproto.goproto_enum_prefix) = false; METHOD_UNSPECIFIED = 0; GET = 1; HEAD = 2; @@ -247,7 +242,6 @@ message SocketOption { bytes buf_value = 5; } enum SocketState { - option (gogoproto.goproto_enum_prefix) = false; // Socket options are applied after socket creation but before binding the socket to a port STATE_PREBIND = 0; // Socket options are applied after binding the socket to a port but before calling listen() diff --git a/api/envoy/api/v2/core/config_source.proto b/api/envoy/api/v2/core/config_source.proto index 9b77f12f5bea..d86a2104f7d0 100644 --- a/api/envoy/api/v2/core/config_source.proto +++ b/api/envoy/api/v2/core/config_source.proto @@ -12,9 +12,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Configuration sources] @@ -56,11 +53,10 @@ message ApiConfigSource { repeated GrpcService grpc_services = 4; // For REST APIs, the delay between successive polls. - google.protobuf.Duration refresh_delay = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration refresh_delay = 3; // For REST APIs, the request timeout. If not set, a default value of 1s will be used. - google.protobuf.Duration request_timeout = 5 - [(validate.rules).duration.gt.seconds = 0, (gogoproto.stdduration) = true]; + google.protobuf.Duration request_timeout = 5 [(validate.rules).duration.gt.seconds = 0]; // For GRPC APIs, the rate limit settings. If present, discovery requests made by Envoy will be // rate limited. diff --git a/api/envoy/api/v2/core/grpc_service.proto b/api/envoy/api/v2/core/grpc_service.proto index 404791e1b3a5..705c61f5a133 100644 --- a/api/envoy/api/v2/core/grpc_service.proto +++ b/api/envoy/api/v2/core/grpc_service.proto @@ -14,9 +14,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/empty.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: gRPC services] diff --git a/api/envoy/api/v2/core/health_check.proto b/api/envoy/api/v2/core/health_check.proto index edbcef7b52de..64396f8380a4 100644 --- a/api/envoy/api/v2/core/health_check.proto +++ b/api/envoy/api/v2/core/health_check.proto @@ -15,9 +15,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Health check] // * Health checking :ref:`architecture overview `. @@ -27,22 +24,16 @@ option (gogoproto.equal_all) = true; message HealthCheck { // The time to wait for a health check response. If the timeout is reached the // health check attempt will be considered a failure. - google.protobuf.Duration timeout = 1 [ - (validate.rules).duration = { - required: true, - gt: {seconds: 0} - }, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration timeout = 1 [(validate.rules).duration = { + required: true, + gt: {seconds: 0} + }]; // The interval between health checks. - google.protobuf.Duration interval = 2 [ - (validate.rules).duration = { - required: true, - gt: {seconds: 0} - }, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration interval = 2 [(validate.rules).duration = { + required: true, + gt: {seconds: 0} + }]; // An optional jitter amount in milliseconds. If specified, Envoy will start health // checking after for a random time in ms between 0 and initial_jitter. This only diff --git a/api/envoy/api/v2/core/http_uri.proto b/api/envoy/api/v2/core/http_uri.proto index 4c7a594f0635..debaa4155679 100644 --- a/api/envoy/api/v2/core/http_uri.proto +++ b/api/envoy/api/v2/core/http_uri.proto @@ -7,12 +7,9 @@ option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v2.core"; import "google/protobuf/duration.proto"; -import "gogoproto/gogo.proto"; import "validate/validate.proto"; -option (gogoproto.equal_all) = true; - // [#protodoc-title: HTTP Service URI ] // Envoy external URI descriptor @@ -46,9 +43,6 @@ message HttpUri { } // Sets the maximum duration in milliseconds that a response can take to arrive upon request. - google.protobuf.Duration timeout = 3 [ - (validate.rules).duration.gte = {}, - (validate.rules).duration.required = true, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration timeout = 3 + [(validate.rules).duration.gte = {}, (validate.rules).duration.required = true]; } diff --git a/api/envoy/api/v2/core/protocol.proto b/api/envoy/api/v2/core/protocol.proto index 88a82077a428..a318ba698b59 100644 --- a/api/envoy/api/v2/core/protocol.proto +++ b/api/envoy/api/v2/core/protocol.proto @@ -12,9 +12,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Protocol options] @@ -27,7 +24,7 @@ message HttpProtocolOptions { // period in which there are no active requests. If not set, there is no idle timeout. When the // idle timeout is reached the connection will be closed. Note that request based timeouts mean // that HTTP/2 PINGs will not keep the connection alive. - google.protobuf.Duration idle_timeout = 1 [(gogoproto.stdduration) = true]; + google.protobuf.Duration idle_timeout = 1; } message Http1ProtocolOptions { diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index 5328e515bcbb..2d04a24817f9 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -10,10 +10,6 @@ import "envoy/api/v2/core/base.proto"; import "google/protobuf/any.proto"; import "google/rpc/status.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: Common discovery API components] diff --git a/api/envoy/api/v2/eds.proto b/api/envoy/api/v2/eds.proto index 1719ad6d2c34..01982fbf6f95 100644 --- a/api/envoy/api/v2/eds.proto +++ b/api/envoy/api/v2/eds.proto @@ -15,13 +15,9 @@ import "envoy/type/percent.proto"; import "google/api/annotations.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; import "google/protobuf/wrappers.proto"; import "google/protobuf/duration.proto"; -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; - // [#protodoc-title: EDS] // Endpoint discovery :ref:`architecture overview ` diff --git a/api/envoy/api/v2/endpoint/endpoint.proto b/api/envoy/api/v2/endpoint/endpoint.proto index 6327af00ac99..7d614a26bb76 100644 --- a/api/envoy/api/v2/endpoint/endpoint.proto +++ b/api/envoy/api/v2/endpoint/endpoint.proto @@ -13,9 +13,6 @@ import "envoy/api/v2/core/health_check.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Endpoints] diff --git a/api/envoy/api/v2/endpoint/load_report.proto b/api/envoy/api/v2/endpoint/load_report.proto index df3fd6071e4f..b44313ba4ee3 100644 --- a/api/envoy/api/v2/endpoint/load_report.proto +++ b/api/envoy/api/v2/endpoint/load_report.proto @@ -13,7 +13,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // These are stats Envoy reports to GLB every so often. Report frequency is // defined by diff --git a/api/envoy/api/v2/lds.proto b/api/envoy/api/v2/lds.proto index 643ac146213b..aec4ad85aded 100644 --- a/api/envoy/api/v2/lds.proto +++ b/api/envoy/api/v2/lds.proto @@ -19,9 +19,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Listener] // Listener :ref:`configuration overview ` @@ -136,7 +133,7 @@ message Listener { // the accepted socket is closed without a connection being created unless // `continue_on_listener_filters_timeout` is set to true. Specify 0 to disable the // timeout. If not specified, a default timeout of 15s is used. - google.protobuf.Duration listener_filters_timeout = 15 [(gogoproto.stdduration) = true]; + google.protobuf.Duration listener_filters_timeout = 15; // Whether a connection should be created when listener filters timeout. Default is false. // diff --git a/api/envoy/api/v2/listener/listener.proto b/api/envoy/api/v2/listener/listener.proto index 3b6cf74ab0a2..f6dcecc70805 100644 --- a/api/envoy/api/v2/listener/listener.proto +++ b/api/envoy/api/v2/listener/listener.proto @@ -17,9 +17,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Listener components] // Listener :ref:`configuration overview ` diff --git a/api/envoy/api/v2/rds.proto b/api/envoy/api/v2/rds.proto index 2d82e4ad00cf..9fabaf28af80 100644 --- a/api/envoy/api/v2/rds.proto +++ b/api/envoy/api/v2/rds.proto @@ -17,9 +17,6 @@ import "google/api/annotations.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: HTTP route configuration] // * Routing :ref:`architecture overview ` diff --git a/api/envoy/api/v2/route/route.proto b/api/envoy/api/v2/route/route.proto index d396344a1ff2..a967cdef7bf2 100644 --- a/api/envoy/api/v2/route/route.proto +++ b/api/envoy/api/v2/route/route.proto @@ -19,10 +19,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: HTTP route] // * Routing :ref:`architecture overview ` @@ -602,7 +598,7 @@ message RouteAction { // :ref:`config_http_filters_router_x-envoy-upstream-rq-timeout-ms`, // :ref:`config_http_filters_router_x-envoy-upstream-rq-per-try-timeout-ms`, and the // :ref:`retry overview `. - google.protobuf.Duration timeout = 8 [(gogoproto.stdduration) = true]; + google.protobuf.Duration timeout = 8; // Specifies the idle timeout for the route. If not specified, there is no per-route idle timeout, // although the connection manager wide :ref:`stream_idle_timeout @@ -622,7 +618,7 @@ message RouteAction { // fires, the stream is terminated with a 408 Request Timeout error code if no // upstream response header has been received, otherwise a stream reset // occurs. - google.protobuf.Duration idle_timeout = 24 [(gogoproto.stdduration) = true]; + google.protobuf.Duration idle_timeout = 24; // Indicates that the route has a retry policy. Note that if this is set, // it'll take precedence over the virtual host level retry policy entirely @@ -732,7 +728,7 @@ message RouteAction { // If specified, a cookie with the TTL will be generated if the cookie is // not present. If the TTL is present and zero, the generated cookie will // be a session cookie. - google.protobuf.Duration ttl = 2 [(gogoproto.stdduration) = true]; + google.protobuf.Duration ttl = 2; // The name of the path for the cookie. If no path is specified here, no path // will be set for the cookie. @@ -811,7 +807,7 @@ message RouteAction { // :ref:`timeout ` or its default. // This can be used to prevent unexpected upstream request timeouts due to potentially long // time gaps between gRPC request and response in gRPC streaming mode. - google.protobuf.Duration max_grpc_timeout = 23 [(gogoproto.stdduration) = true]; + google.protobuf.Duration max_grpc_timeout = 23; // If present, Envoy will adjust the timeout provided by the `grpc-timeout` header by subtracting // the provided duration from the header. This is useful in allowing Envoy to set its global @@ -820,7 +816,7 @@ message RouteAction { // The offset will only be applied if the provided grpc_timeout is greater than the offset. This // ensures that the offset will only ever decrease the timeout and never set it to 0 (meaning // infinity). - google.protobuf.Duration grpc_timeout_offset = 28 [(gogoproto.stdduration) = true]; + google.protobuf.Duration grpc_timeout_offset = 28; // Allows enabling and disabling upgrades on a per-route basis. // This overrides any enabled/disabled upgrade filter chain specified in the @@ -875,7 +871,7 @@ message RetryPolicy { // Consequently, when using a :ref:`5xx ` based // retry policy, a request that times out will not be retried as the total timeout budget // would have been exhausted. - google.protobuf.Duration per_try_timeout = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration per_try_timeout = 3; message RetryPriority { string name = 1 [(validate.rules).string.min_bytes = 1]; @@ -919,20 +915,16 @@ message RetryPolicy { // than zero. Values less than 1 ms are rounded up to 1 ms. // See :ref:`config_http_filters_router_x-envoy-max-retries` for a discussion of Envoy's // back-off algorithm. - google.protobuf.Duration base_interval = 1 [ - (validate.rules).duration = { - required: true, - gt: {seconds: 0} - }, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration base_interval = 1 [(validate.rules).duration = { + required: true, + gt: {seconds: 0} + }]; // Specifies the maximum interval between retries. This parameter is optional, but must be // greater than or equal to the `base_interval` if set. The default is 10 times the // `base_interval`. See :ref:`config_http_filters_router_x-envoy-max-retries` for a discussion // of Envoy's back-off algorithm. - google.protobuf.Duration max_interval = 2 - [(validate.rules).duration.gt = {seconds: 0}, (gogoproto.stdduration) = true]; + google.protobuf.Duration max_interval = 2 [(validate.rules).duration.gt = {seconds: 0}]; } // Specifies parameters that control retry back off. This parameter is optional, in which case the diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index 617fdf9ac644..a51426af01b7 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package envoy.api.v2; import "envoy/api/v2/discovery.proto"; -import "gogoproto/gogo.proto"; import "google/api/annotations.proto"; import "validate/validate.proto"; @@ -11,7 +10,6 @@ option java_outer_classname = "SrdsProto"; option java_package = "io.envoyproxy.envoy.api.v2"; option java_multiple_files = true; option java_generic_services = true; -option (gogoproto.equal_all) = true; // [#protodoc-title: HTTP scoped routing configuration] // * Routing :ref:`architecture overview ` diff --git a/api/envoy/api/v3alpha/auth/cert.proto b/api/envoy/api/v3alpha/auth/cert.proto index 2be82a1c0496..83897b268320 100644 --- a/api/envoy/api/v3alpha/auth/cert.proto +++ b/api/envoy/api/v3alpha/auth/cert.proto @@ -14,9 +14,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Common TLS configuration] diff --git a/api/envoy/api/v3alpha/cds.proto b/api/envoy/api/v3alpha/cds.proto index 2197ff3efe46..c1c18b110be0 100644 --- a/api/envoy/api/v3alpha/cds.proto +++ b/api/envoy/api/v3alpha/cds.proto @@ -28,10 +28,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; // Return list of all clusters this proxy will load balance to. service ClusterDiscoveryService { @@ -127,8 +123,7 @@ message Cluster { EdsClusterConfig eds_cluster_config = 3; // The timeout for new network connections to hosts in the cluster. - google.protobuf.Duration connect_timeout = 4 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration connect_timeout = 4 [(validate.rules).duration.gt = {}]; // Soft limit on size of the cluster’s connections read and write buffers. If // unspecified, an implementation defined default is applied (1MiB). @@ -274,8 +269,7 @@ message Cluster { // :ref:`STRICT_DNS` // and :ref:`LOGICAL_DNS` // this setting is ignored. - google.protobuf.Duration dns_refresh_rate = 16 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration dns_refresh_rate = 16 [(validate.rules).duration.gt = {}]; // Optional configuration for setting cluster's DNS refresh rate. If the value is set to true, // cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS @@ -333,8 +327,7 @@ message Cluster { // value defaults to 5000ms. For cluster types other than // :ref:`ORIGINAL_DST` // this setting is ignored. - google.protobuf.Duration cleanup_interval = 20 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration cleanup_interval = 20 [(validate.rules).duration.gt = {}]; // Optional configuration used to bind newly established upstream connections. // This overrides any bind_config specified in the bootstrap proto. diff --git a/api/envoy/api/v3alpha/cluster/circuit_breaker.proto b/api/envoy/api/v3alpha/cluster/circuit_breaker.proto index 8a70008e49f5..0b6d2fe99217 100644 --- a/api/envoy/api/v3alpha/cluster/circuit_breaker.proto +++ b/api/envoy/api/v3alpha/cluster/circuit_breaker.proto @@ -12,10 +12,6 @@ import "envoy/api/v3alpha/core/base.proto"; import "google/protobuf/wrappers.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; - // [#protodoc-title: Circuit breakers] // :ref:`Circuit breaking` settings can be diff --git a/api/envoy/api/v3alpha/cluster/filter.proto b/api/envoy/api/v3alpha/cluster/filter.proto index ab1a3d13747b..1bf3433ad29a 100644 --- a/api/envoy/api/v3alpha/cluster/filter.proto +++ b/api/envoy/api/v3alpha/cluster/filter.proto @@ -11,9 +11,6 @@ option ruby_package = "Envoy.Api.V2.ClusterNS"; import "google/protobuf/any.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Upstream filters] // diff --git a/api/envoy/api/v3alpha/cluster/outlier_detection.proto b/api/envoy/api/v3alpha/cluster/outlier_detection.proto index 6600b566d1bd..0954b85f2cc8 100644 --- a/api/envoy/api/v3alpha/cluster/outlier_detection.proto +++ b/api/envoy/api/v3alpha/cluster/outlier_detection.proto @@ -12,9 +12,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Outlier detection] diff --git a/api/envoy/api/v3alpha/core/address.proto b/api/envoy/api/v3alpha/core/address.proto index 5cec89b11d8e..80ab295b2bf7 100644 --- a/api/envoy/api/v3alpha/core/address.proto +++ b/api/envoy/api/v3alpha/core/address.proto @@ -11,9 +11,6 @@ import "envoy/api/v3alpha/core/base.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Network addresses] @@ -27,7 +24,6 @@ message Pipe { message SocketAddress { enum Protocol { - option (gogoproto.goproto_enum_prefix) = false; TCP = 0; // [#not-implemented-hide:] UDP = 1; diff --git a/api/envoy/api/v3alpha/core/base.proto b/api/envoy/api/v3alpha/core/base.proto index cf39df887bce..a7b2f54d692f 100644 --- a/api/envoy/api/v3alpha/core/base.proto +++ b/api/envoy/api/v3alpha/core/base.proto @@ -13,13 +13,9 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; import "envoy/type/percent.proto"; -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; - // [#protodoc-title: Common types] // Identifies location of where either Envoy runs or where upstream hosts run. @@ -131,7 +127,6 @@ enum RoutingPriority { // HTTP request method. enum RequestMethod { - option (gogoproto.goproto_enum_prefix) = false; METHOD_UNSPECIFIED = 0; GET = 1; HEAD = 2; @@ -247,7 +242,6 @@ message SocketOption { bytes buf_value = 5; } enum SocketState { - option (gogoproto.goproto_enum_prefix) = false; // Socket options are applied after socket creation but before binding the socket to a port STATE_PREBIND = 0; // Socket options are applied after binding the socket to a port but before calling listen() diff --git a/api/envoy/api/v3alpha/core/config_source.proto b/api/envoy/api/v3alpha/core/config_source.proto index 58ef7daeb0a1..1c4510322e15 100644 --- a/api/envoy/api/v3alpha/core/config_source.proto +++ b/api/envoy/api/v3alpha/core/config_source.proto @@ -12,9 +12,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Configuration sources] @@ -56,11 +53,10 @@ message ApiConfigSource { repeated GrpcService grpc_services = 4; // For REST APIs, the delay between successive polls. - google.protobuf.Duration refresh_delay = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration refresh_delay = 3; // For REST APIs, the request timeout. If not set, a default value of 1s will be used. - google.protobuf.Duration request_timeout = 5 - [(validate.rules).duration.gt.seconds = 0, (gogoproto.stdduration) = true]; + google.protobuf.Duration request_timeout = 5 [(validate.rules).duration.gt.seconds = 0]; // For GRPC APIs, the rate limit settings. If present, discovery requests made by Envoy will be // rate limited. diff --git a/api/envoy/api/v3alpha/core/grpc_service.proto b/api/envoy/api/v3alpha/core/grpc_service.proto index 931a4cb8926f..dd8b90d72cad 100644 --- a/api/envoy/api/v3alpha/core/grpc_service.proto +++ b/api/envoy/api/v3alpha/core/grpc_service.proto @@ -14,9 +14,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/empty.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: gRPC services] diff --git a/api/envoy/api/v3alpha/core/health_check.proto b/api/envoy/api/v3alpha/core/health_check.proto index 1a9508fb6490..5000918a40c6 100644 --- a/api/envoy/api/v3alpha/core/health_check.proto +++ b/api/envoy/api/v3alpha/core/health_check.proto @@ -15,9 +15,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Health check] // * Health checking :ref:`architecture overview `. @@ -27,22 +24,16 @@ option (gogoproto.equal_all) = true; message HealthCheck { // The time to wait for a health check response. If the timeout is reached the // health check attempt will be considered a failure. - google.protobuf.Duration timeout = 1 [ - (validate.rules).duration = { - required: true, - gt: {seconds: 0} - }, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration timeout = 1 [(validate.rules).duration = { + required: true, + gt: {seconds: 0} + }]; // The interval between health checks. - google.protobuf.Duration interval = 2 [ - (validate.rules).duration = { - required: true, - gt: {seconds: 0} - }, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration interval = 2 [(validate.rules).duration = { + required: true, + gt: {seconds: 0} + }]; // An optional jitter amount in milliseconds. If specified, Envoy will start health // checking after for a random time in ms between 0 and initial_jitter. This only diff --git a/api/envoy/api/v3alpha/core/http_uri.proto b/api/envoy/api/v3alpha/core/http_uri.proto index b47533f2066a..9c8e85b44150 100644 --- a/api/envoy/api/v3alpha/core/http_uri.proto +++ b/api/envoy/api/v3alpha/core/http_uri.proto @@ -7,12 +7,9 @@ option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.api.v3alpha.core"; import "google/protobuf/duration.proto"; -import "gogoproto/gogo.proto"; import "validate/validate.proto"; -option (gogoproto.equal_all) = true; - // [#protodoc-title: HTTP Service URI ] // Envoy external URI descriptor @@ -46,9 +43,6 @@ message HttpUri { } // Sets the maximum duration in milliseconds that a response can take to arrive upon request. - google.protobuf.Duration timeout = 3 [ - (validate.rules).duration.gte = {}, - (validate.rules).duration.required = true, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration timeout = 3 + [(validate.rules).duration.gte = {}, (validate.rules).duration.required = true]; } diff --git a/api/envoy/api/v3alpha/core/protocol.proto b/api/envoy/api/v3alpha/core/protocol.proto index 63d0f6920d83..b0b29e630eb1 100644 --- a/api/envoy/api/v3alpha/core/protocol.proto +++ b/api/envoy/api/v3alpha/core/protocol.proto @@ -12,9 +12,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Protocol options] @@ -27,7 +24,7 @@ message HttpProtocolOptions { // period in which there are no active requests. If not set, there is no idle timeout. When the // idle timeout is reached the connection will be closed. Note that request based timeouts mean // that HTTP/2 PINGs will not keep the connection alive. - google.protobuf.Duration idle_timeout = 1 [(gogoproto.stdduration) = true]; + google.protobuf.Duration idle_timeout = 1; } message Http1ProtocolOptions { diff --git a/api/envoy/api/v3alpha/discovery.proto b/api/envoy/api/v3alpha/discovery.proto index 105b99888141..1547b030062f 100644 --- a/api/envoy/api/v3alpha/discovery.proto +++ b/api/envoy/api/v3alpha/discovery.proto @@ -10,10 +10,6 @@ import "envoy/api/v3alpha/core/base.proto"; import "google/protobuf/any.proto"; import "google/rpc/status.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: Common discovery API components] diff --git a/api/envoy/api/v3alpha/eds.proto b/api/envoy/api/v3alpha/eds.proto index 6d1cf12f3de5..da149db20bd7 100644 --- a/api/envoy/api/v3alpha/eds.proto +++ b/api/envoy/api/v3alpha/eds.proto @@ -15,13 +15,9 @@ import "envoy/type/percent.proto"; import "google/api/annotations.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; import "google/protobuf/wrappers.proto"; import "google/protobuf/duration.proto"; -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; - // [#protodoc-title: EDS] // Endpoint discovery :ref:`architecture overview ` diff --git a/api/envoy/api/v3alpha/endpoint/endpoint.proto b/api/envoy/api/v3alpha/endpoint/endpoint.proto index 15357cdbac8b..62c3060dee0b 100644 --- a/api/envoy/api/v3alpha/endpoint/endpoint.proto +++ b/api/envoy/api/v3alpha/endpoint/endpoint.proto @@ -13,9 +13,6 @@ import "envoy/api/v3alpha/core/health_check.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Endpoints] diff --git a/api/envoy/api/v3alpha/endpoint/load_report.proto b/api/envoy/api/v3alpha/endpoint/load_report.proto index 7e4aed3ba103..9ab814d30db2 100644 --- a/api/envoy/api/v3alpha/endpoint/load_report.proto +++ b/api/envoy/api/v3alpha/endpoint/load_report.proto @@ -13,7 +13,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // These are stats Envoy reports to GLB every so often. Report frequency is // defined by diff --git a/api/envoy/api/v3alpha/lds.proto b/api/envoy/api/v3alpha/lds.proto index 7268d2b15e4b..794b25a4a395 100644 --- a/api/envoy/api/v3alpha/lds.proto +++ b/api/envoy/api/v3alpha/lds.proto @@ -19,9 +19,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Listener] // Listener :ref:`configuration overview ` @@ -136,7 +133,7 @@ message Listener { // the accepted socket is closed without a connection being created unless // `continue_on_listener_filters_timeout` is set to true. Specify 0 to disable the // timeout. If not specified, a default timeout of 15s is used. - google.protobuf.Duration listener_filters_timeout = 15 [(gogoproto.stdduration) = true]; + google.protobuf.Duration listener_filters_timeout = 15; // Whether a connection should be created when listener filters timeout. Default is false. // diff --git a/api/envoy/api/v3alpha/listener/listener.proto b/api/envoy/api/v3alpha/listener/listener.proto index dc44c1454c02..ef5063969472 100644 --- a/api/envoy/api/v3alpha/listener/listener.proto +++ b/api/envoy/api/v3alpha/listener/listener.proto @@ -17,9 +17,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Listener components] // Listener :ref:`configuration overview ` diff --git a/api/envoy/api/v3alpha/rds.proto b/api/envoy/api/v3alpha/rds.proto index b26956dcf712..36dfbc4a0fee 100644 --- a/api/envoy/api/v3alpha/rds.proto +++ b/api/envoy/api/v3alpha/rds.proto @@ -17,9 +17,6 @@ import "google/api/annotations.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: HTTP route configuration] // * Routing :ref:`architecture overview ` diff --git a/api/envoy/api/v3alpha/route/route.proto b/api/envoy/api/v3alpha/route/route.proto index e69e116b9b63..d4d60e929022 100644 --- a/api/envoy/api/v3alpha/route/route.proto +++ b/api/envoy/api/v3alpha/route/route.proto @@ -19,10 +19,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: HTTP route] // * Routing :ref:`architecture overview ` @@ -602,7 +598,7 @@ message RouteAction { // :ref:`config_http_filters_router_x-envoy-upstream-rq-timeout-ms`, // :ref:`config_http_filters_router_x-envoy-upstream-rq-per-try-timeout-ms`, and the // :ref:`retry overview `. - google.protobuf.Duration timeout = 8 [(gogoproto.stdduration) = true]; + google.protobuf.Duration timeout = 8; // Specifies the idle timeout for the route. If not specified, there is no per-route idle timeout, // although the connection manager wide :ref:`stream_idle_timeout @@ -622,7 +618,7 @@ message RouteAction { // fires, the stream is terminated with a 408 Request Timeout error code if no // upstream response header has been received, otherwise a stream reset // occurs. - google.protobuf.Duration idle_timeout = 24 [(gogoproto.stdduration) = true]; + google.protobuf.Duration idle_timeout = 24; // Indicates that the route has a retry policy. Note that if this is set, // it'll take precedence over the virtual host level retry policy entirely @@ -732,7 +728,7 @@ message RouteAction { // If specified, a cookie with the TTL will be generated if the cookie is // not present. If the TTL is present and zero, the generated cookie will // be a session cookie. - google.protobuf.Duration ttl = 2 [(gogoproto.stdduration) = true]; + google.protobuf.Duration ttl = 2; // The name of the path for the cookie. If no path is specified here, no path // will be set for the cookie. @@ -811,7 +807,7 @@ message RouteAction { // :ref:`timeout ` or its default. // This can be used to prevent unexpected upstream request timeouts due to potentially long // time gaps between gRPC request and response in gRPC streaming mode. - google.protobuf.Duration max_grpc_timeout = 23 [(gogoproto.stdduration) = true]; + google.protobuf.Duration max_grpc_timeout = 23; // If present, Envoy will adjust the timeout provided by the `grpc-timeout` header by subtracting // the provided duration from the header. This is useful in allowing Envoy to set its global @@ -820,7 +816,7 @@ message RouteAction { // The offset will only be applied if the provided grpc_timeout is greater than the offset. This // ensures that the offset will only ever decrease the timeout and never set it to 0 (meaning // infinity). - google.protobuf.Duration grpc_timeout_offset = 28 [(gogoproto.stdduration) = true]; + google.protobuf.Duration grpc_timeout_offset = 28; // Allows enabling and disabling upgrades on a per-route basis. // This overrides any enabled/disabled upgrade filter chain specified in the @@ -875,7 +871,7 @@ message RetryPolicy { // Consequently, when using a :ref:`5xx ` based // retry policy, a request that times out will not be retried as the total timeout budget // would have been exhausted. - google.protobuf.Duration per_try_timeout = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration per_try_timeout = 3; message RetryPriority { string name = 1 [(validate.rules).string.min_bytes = 1]; @@ -919,20 +915,16 @@ message RetryPolicy { // than zero. Values less than 1 ms are rounded up to 1 ms. // See :ref:`config_http_filters_router_x-envoy-max-retries` for a discussion of Envoy's // back-off algorithm. - google.protobuf.Duration base_interval = 1 [ - (validate.rules).duration = { - required: true, - gt: {seconds: 0} - }, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration base_interval = 1 [(validate.rules).duration = { + required: true, + gt: {seconds: 0} + }]; // Specifies the maximum interval between retries. This parameter is optional, but must be // greater than or equal to the `base_interval` if set. The default is 10 times the // `base_interval`. See :ref:`config_http_filters_router_x-envoy-max-retries` for a discussion // of Envoy's back-off algorithm. - google.protobuf.Duration max_interval = 2 - [(validate.rules).duration.gt = {seconds: 0}, (gogoproto.stdduration) = true]; + google.protobuf.Duration max_interval = 2 [(validate.rules).duration.gt = {seconds: 0}]; } // Specifies parameters that control retry back off. This parameter is optional, in which case the diff --git a/api/envoy/api/v3alpha/srds.proto b/api/envoy/api/v3alpha/srds.proto index 62a767ca4440..636ba3917ecc 100644 --- a/api/envoy/api/v3alpha/srds.proto +++ b/api/envoy/api/v3alpha/srds.proto @@ -3,7 +3,6 @@ syntax = "proto3"; package envoy.api.v3alpha; import "envoy/api/v3alpha/discovery.proto"; -import "gogoproto/gogo.proto"; import "google/api/annotations.proto"; import "validate/validate.proto"; @@ -11,7 +10,6 @@ option java_outer_classname = "SrdsProto"; option java_package = "io.envoyproxy.envoy.api.v3alpha"; option java_multiple_files = true; option java_generic_services = true; -option (gogoproto.equal_all) = true; // [#protodoc-title: HTTP scoped routing configuration] // * Routing :ref:`architecture overview ` diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index 66f05aa78486..b8f17de61870 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -25,7 +25,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // Bootstrap :ref:`configuration overview `. message Bootstrap { @@ -99,13 +98,10 @@ message Bootstrap { // gauges at a periodic interval. If not specified the default is 5000ms (5 // seconds). // Duration must be at least 1ms and at most 5 min. - google.protobuf.Duration stats_flush_interval = 7 [ - (validate.rules).duration = { - lt: {seconds: 300}, - gte: {nanos: 1000000} - }, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration stats_flush_interval = 7 [(validate.rules).duration = { + lt: {seconds: 300}, + gte: {nanos: 1000000} + }]; // Optional watchdog configuration. Watchdog watchdog = 8; diff --git a/api/envoy/config/bootstrap/v3alpha/bootstrap.proto b/api/envoy/config/bootstrap/v3alpha/bootstrap.proto index 0bf18ffa360d..f3ac2e8342ea 100644 --- a/api/envoy/config/bootstrap/v3alpha/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3alpha/bootstrap.proto @@ -25,7 +25,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // Bootstrap :ref:`configuration overview `. message Bootstrap { @@ -98,13 +97,10 @@ message Bootstrap { // gauges at a periodic interval. If not specified the default is 5000ms (5 // seconds). // Duration must be at least 1ms and at most 5 min. - google.protobuf.Duration stats_flush_interval = 7 [ - (validate.rules).duration = { - lt: {seconds: 300}, - gte: {nanos: 1000000} - }, - (gogoproto.stdduration) = true - ]; + google.protobuf.Duration stats_flush_interval = 7 [(validate.rules).duration = { + lt: {seconds: 300}, + gte: {nanos: 1000000} + }]; // Optional watchdog configuration. Watchdog watchdog = 8; diff --git a/api/envoy/config/cluster/redis/redis_cluster.proto b/api/envoy/config/cluster/redis/redis_cluster.proto index abe2c857532d..fabaa0274fb7 100644 --- a/api/envoy/config/cluster/redis/redis_cluster.proto +++ b/api/envoy/config/cluster/redis/redis_cluster.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.cluster.redis"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Redis Cluster Configuration] // This cluster adds support for `Redis Cluster `_, as part @@ -44,10 +43,8 @@ import "gogoproto/gogo.proto"; message RedisClusterConfig { // Interval between successive topology refresh requests. If not set, this defaults to 5s. - google.protobuf.Duration cluster_refresh_rate = 1 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration cluster_refresh_rate = 1 [(validate.rules).duration.gt = {}]; // Timeout for topology refresh request. If not set, this defaults to 3s. - google.protobuf.Duration cluster_refresh_timeout = 2 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration cluster_refresh_timeout = 2 [(validate.rules).duration.gt = {}]; } diff --git a/api/envoy/config/common/tap/v2alpha/common.proto b/api/envoy/config/common/tap/v2alpha/common.proto index 6d3c869baffa..ac640b83e4fb 100644 --- a/api/envoy/config/common/tap/v2alpha/common.proto +++ b/api/envoy/config/common/tap/v2alpha/common.proto @@ -4,7 +4,6 @@ import "envoy/service/tap/v2alpha/common.proto"; import "envoy/api/v2/core/config_source.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; package envoy.config.common.tap.v2alpha; diff --git a/api/envoy/config/common/tap/v3alpha/common.proto b/api/envoy/config/common/tap/v3alpha/common.proto index 4e5debddbc51..c260d04afa15 100644 --- a/api/envoy/config/common/tap/v3alpha/common.proto +++ b/api/envoy/config/common/tap/v3alpha/common.proto @@ -4,7 +4,6 @@ import "envoy/service/tap/v3alpha/common.proto"; import "envoy/api/v3alpha/core/config_source.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; package envoy.config.common.tap.v3alpha; diff --git a/api/envoy/config/filter/fault/v2/fault.proto b/api/envoy/config/filter/fault/v2/fault.proto index 2298ecf4a1c0..15164172dcf4 100644 --- a/api/envoy/config/filter/fault/v2/fault.proto +++ b/api/envoy/config/filter/fault/v2/fault.proto @@ -11,7 +11,6 @@ import "envoy/type/percent.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Common fault injection types] @@ -43,8 +42,7 @@ message FaultDelay { // delay will be injected before a new request/operation. For TCP // connections, the proxying of the connection upstream will be delayed // for the specified period. This is required if type is FIXED. - google.protobuf.Duration fixed_delay = 3 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration fixed_delay = 3 [(validate.rules).duration.gt = {}]; // Fault delays are controlled via an HTTP header (if applicable). HeaderDelay header_delay = 5; diff --git a/api/envoy/config/filter/fault/v3alpha/fault.proto b/api/envoy/config/filter/fault/v3alpha/fault.proto index 054eb8470f3a..21e0f9e12e67 100644 --- a/api/envoy/config/filter/fault/v3alpha/fault.proto +++ b/api/envoy/config/filter/fault/v3alpha/fault.proto @@ -11,7 +11,6 @@ import "envoy/type/percent.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Common fault injection types] @@ -43,8 +42,7 @@ message FaultDelay { // delay will be injected before a new request/operation. For TCP // connections, the proxying of the connection upstream will be delayed // for the specified period. This is required if type is FIXED. - google.protobuf.Duration fixed_delay = 3 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration fixed_delay = 3 [(validate.rules).duration.gt = {}]; // Fault delays are controlled via an HTTP header (if applicable). HeaderDelay header_delay = 5; diff --git a/api/envoy/config/filter/http/buffer/v2/buffer.proto b/api/envoy/config/filter/http/buffer/v2/buffer.proto index 92780adad69b..ce6c0d6d1423 100644 --- a/api/envoy/config/filter/http/buffer/v2/buffer.proto +++ b/api/envoy/config/filter/http/buffer/v2/buffer.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.buffer.v2"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Buffer] // Buffer :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto b/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto index 25530c5a58b4..9d44f3503229 100644 --- a/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto +++ b/api/envoy/config/filter/http/buffer/v3alpha/buffer.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.buffer.v3alpha"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Buffer] // Buffer :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/csrf/v2/csrf.proto b/api/envoy/config/filter/http/csrf/v2/csrf.proto index d4c35291e63d..b5c78db544a7 100644 --- a/api/envoy/config/filter/http/csrf/v2/csrf.proto +++ b/api/envoy/config/filter/http/csrf/v2/csrf.proto @@ -10,7 +10,6 @@ import "envoy/api/v2/core/base.proto"; import "envoy/type/matcher/string.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: CSRF] // Cross-Site Request Forgery :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto b/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto index 8fe68d5ea2ef..7c8416878a74 100644 --- a/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto +++ b/api/envoy/config/filter/http/csrf/v3alpha/csrf.proto @@ -10,7 +10,6 @@ import "envoy/api/v3alpha/core/base.proto"; import "envoy/type/matcher/string.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: CSRF] // Cross-Site Request Forgery :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto index e2922348b6a7..84d4ab19495b 100644 --- a/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto +++ b/api/envoy/config/filter/http/ext_authz/v2/ext_authz.proto @@ -14,9 +14,6 @@ import "envoy/type/http_status.proto"; import "envoy/type/matcher/string.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: External Authorization] // External Authorization :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto b/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto index 8cc48e36ffb0..113be0256b1f 100644 --- a/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto +++ b/api/envoy/config/filter/http/ext_authz/v3alpha/ext_authz.proto @@ -14,9 +14,6 @@ import "envoy/type/http_status.proto"; import "envoy/type/matcher/string.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: External Authorization] // External Authorization :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/gzip/v2/gzip.proto b/api/envoy/config/filter/http/gzip/v2/gzip.proto index ec512c495dc8..02041b87fd27 100644 --- a/api/envoy/config/filter/http/gzip/v2/gzip.proto +++ b/api/envoy/config/filter/http/gzip/v2/gzip.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.gzip.v2"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Gzip] // Gzip :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto b/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto index d7afb89116c2..26e437d48c52 100644 --- a/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto +++ b/api/envoy/config/filter/http/gzip/v3alpha/gzip.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.gzip.v3alpha"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Gzip] // Gzip :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/health_check/v2/health_check.proto b/api/envoy/config/filter/http/health_check/v2/health_check.proto index 2aa6d4191596..9cd572b43709 100644 --- a/api/envoy/config/filter/http/health_check/v2/health_check.proto +++ b/api/envoy/config/filter/http/health_check/v2/health_check.proto @@ -13,9 +13,6 @@ import "envoy/api/v2/route/route.proto"; import "envoy/type/percent.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: Health check] // Health check :ref:`configuration overview `. @@ -29,7 +26,7 @@ message HealthCheck { // If operating in pass through mode, the amount of time in milliseconds // that the filter should cache the upstream response. - google.protobuf.Duration cache_time = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration cache_time = 3; // If operating in non-pass-through mode, specifies a set of upstream cluster // names and the minimum percentage of servers in each of those clusters that diff --git a/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto b/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto index ecbb8e507851..c5e91e703d5b 100644 --- a/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto +++ b/api/envoy/config/filter/http/health_check/v3alpha/health_check.proto @@ -13,9 +13,6 @@ import "envoy/api/v3alpha/route/route.proto"; import "envoy/type/percent.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: Health check] // Health check :ref:`configuration overview `. @@ -29,7 +26,7 @@ message HealthCheck { // If operating in pass through mode, the amount of time in milliseconds // that the filter should cache the upstream response. - google.protobuf.Duration cache_time = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration cache_time = 3; // If operating in non-pass-through mode, specifies a set of upstream cluster // names and the minimum percentage of servers in each of those clusters that diff --git a/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto b/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto index 2f8a0ec29c17..c07b780b9649 100644 --- a/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto +++ b/api/envoy/config/filter/http/jwt_authn/v2alpha/config.proto @@ -13,9 +13,6 @@ import "envoy/api/v2/route/route.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/empty.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: JWT Authentication] // JWT Authentication :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/jwt_authn/v3alpha/config.proto b/api/envoy/config/filter/http/jwt_authn/v3alpha/config.proto index 9c7f10c4adbc..bc4785e64e51 100644 --- a/api/envoy/config/filter/http/jwt_authn/v3alpha/config.proto +++ b/api/envoy/config/filter/http/jwt_authn/v3alpha/config.proto @@ -13,9 +13,6 @@ import "envoy/api/v3alpha/route/route.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/empty.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: JWT Authentication] // JWT Authentication :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/rate_limit/v2/rate_limit.proto b/api/envoy/config/filter/http/rate_limit/v2/rate_limit.proto index 1ad3c4c36d01..08189be1df89 100644 --- a/api/envoy/config/filter/http/rate_limit/v2/rate_limit.proto +++ b/api/envoy/config/filter/http/rate_limit/v2/rate_limit.proto @@ -11,7 +11,6 @@ import "envoy/config/ratelimit/v2/rls.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Rate limit] // Rate limit :ref:`configuration overview `. @@ -38,7 +37,7 @@ message RateLimit { // The timeout in milliseconds for the rate limit service RPC. If not // set, this defaults to 20ms. - google.protobuf.Duration timeout = 4 [(gogoproto.stdduration) = true]; + google.protobuf.Duration timeout = 4; // The filter's behaviour in case the rate limiting service does // not respond back. When it is set to true, Envoy will not allow traffic in case of diff --git a/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto b/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto index 427d22a6c1c6..091c5d3d337a 100644 --- a/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto +++ b/api/envoy/config/filter/http/rate_limit/v3alpha/rate_limit.proto @@ -11,7 +11,6 @@ import "envoy/config/ratelimit/v3alpha/rls.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Rate limit] // Rate limit :ref:`configuration overview `. @@ -38,7 +37,7 @@ message RateLimit { // The timeout in milliseconds for the rate limit service RPC. If not // set, this defaults to 20ms. - google.protobuf.Duration timeout = 4 [(gogoproto.stdduration) = true]; + google.protobuf.Duration timeout = 4; // The filter's behaviour in case the rate limiting service does // not respond back. When it is set to true, Envoy will not allow traffic in case of diff --git a/api/envoy/config/filter/http/rbac/v2/rbac.proto b/api/envoy/config/filter/http/rbac/v2/rbac.proto index 611cdc6ccbed..7c9a3c24d017 100644 --- a/api/envoy/config/filter/http/rbac/v2/rbac.proto +++ b/api/envoy/config/filter/http/rbac/v2/rbac.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.rbac.v2"; import "envoy/config/rbac/v2/rbac.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: RBAC] // Role-Based Access Control :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto b/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto index 47160ffa9e3c..3ffe04ec3a31 100644 --- a/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto +++ b/api/envoy/config/filter/http/rbac/v3alpha/rbac.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.rbac.v3alpha"; import "envoy/config/rbac/v3alpha/rbac.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: RBAC] // Role-Based Access Control :ref:`configuration overview `. diff --git a/api/envoy/config/filter/http/squash/v2/squash.proto b/api/envoy/config/filter/http/squash/v2/squash.proto index 2f3a2e21cdd2..54a67ceddf1c 100644 --- a/api/envoy/config/filter/http/squash/v2/squash.proto +++ b/api/envoy/config/filter/http/squash/v2/squash.proto @@ -10,7 +10,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Squash] // Squash :ref:`configuration overview `. @@ -42,13 +41,13 @@ message Squash { google.protobuf.Struct attachment_template = 2; // The timeout for individual requests sent to the Squash cluster. Defaults to 1 second. - google.protobuf.Duration request_timeout = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration request_timeout = 3; // The total timeout Squash will delay a request and wait for it to be attached. Defaults to 60 // seconds. - google.protobuf.Duration attachment_timeout = 4 [(gogoproto.stdduration) = true]; + google.protobuf.Duration attachment_timeout = 4; // Amount of time to poll for the status of the attachment object in the Squash server // (to check if has been attached). Defaults to 1 second. - google.protobuf.Duration attachment_poll_period = 5 [(gogoproto.stdduration) = true]; + google.protobuf.Duration attachment_poll_period = 5; } diff --git a/api/envoy/config/filter/http/squash/v3alpha/squash.proto b/api/envoy/config/filter/http/squash/v3alpha/squash.proto index 24236def872c..a1b355e67cb8 100644 --- a/api/envoy/config/filter/http/squash/v3alpha/squash.proto +++ b/api/envoy/config/filter/http/squash/v3alpha/squash.proto @@ -10,7 +10,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Squash] // Squash :ref:`configuration overview `. @@ -42,13 +41,13 @@ message Squash { google.protobuf.Struct attachment_template = 2; // The timeout for individual requests sent to the Squash cluster. Defaults to 1 second. - google.protobuf.Duration request_timeout = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration request_timeout = 3; // The total timeout Squash will delay a request and wait for it to be attached. Defaults to 60 // seconds. - google.protobuf.Duration attachment_timeout = 4 [(gogoproto.stdduration) = true]; + google.protobuf.Duration attachment_timeout = 4; // Amount of time to poll for the status of the attachment object in the Squash server // (to check if has been attached). Defaults to 1 second. - google.protobuf.Duration attachment_poll_period = 5 [(gogoproto.stdduration) = true]; + google.protobuf.Duration attachment_poll_period = 5; } diff --git a/api/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto b/api/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto index 6add30a59552..bfd59dd5e804 100644 --- a/api/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto +++ b/api/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto @@ -10,7 +10,6 @@ import "envoy/api/v2/core/address.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Client TLS authentication] // Client TLS authentication @@ -31,7 +30,7 @@ message ClientSSLAuth { // authentication service. Default is 60000 (60s). The actual fetch time // will be this value plus a random jittered value between // 0-refresh_delay_ms milliseconds. - google.protobuf.Duration refresh_delay = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration refresh_delay = 3; // An optional list of IP address and subnet masks that should be white // listed for access by the filter. If no list is provided, there is no diff --git a/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto index 821d63494742..e9a27d151ea5 100644 --- a/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto +++ b/api/envoy/config/filter/network/client_ssl_auth/v3alpha/client_ssl_auth.proto @@ -10,7 +10,6 @@ import "envoy/api/v3alpha/core/address.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Client TLS authentication] // Client TLS authentication @@ -31,7 +30,7 @@ message ClientSSLAuth { // authentication service. Default is 60000 (60s). The actual fetch time // will be this value plus a random jittered value between // 0-refresh_delay_ms milliseconds. - google.protobuf.Duration refresh_delay = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration refresh_delay = 3; // An optional list of IP address and subnet masks that should be white // listed for access by the filter. If no list is provided, there is no diff --git a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto index e9834b704ed3..a27d7001cb86 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto +++ b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/dubbo_proxy.proto @@ -11,7 +11,6 @@ import "envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto"; import "google/protobuf/any.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Dubbo Proxy] // Dubbo Proxy :ref:`configuration overview `. diff --git a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto index c852a7cf5e53..02d86443a6f3 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto +++ b/api/envoy/config/filter/network/dubbo_proxy/v2alpha1/route.proto @@ -13,9 +13,6 @@ import "envoy/type/range.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: Dubbo Proxy Route Configuration] // Dubbo Proxy :ref:`configuration overview `. diff --git a/api/envoy/config/filter/network/dubbo_proxy/v3alpha/dubbo_proxy.proto b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/dubbo_proxy.proto index f314e393a85d..211c4cfed1cb 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v3alpha/dubbo_proxy.proto +++ b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/dubbo_proxy.proto @@ -11,7 +11,6 @@ import "envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto"; import "google/protobuf/any.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Dubbo Proxy] // Dubbo Proxy :ref:`configuration overview `. diff --git a/api/envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto index 180428f8644a..6d357b8c138f 100644 --- a/api/envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto +++ b/api/envoy/config/filter/network/dubbo_proxy/v3alpha/route.proto @@ -13,9 +13,6 @@ import "envoy/type/range.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: Dubbo Proxy Route Configuration] // Dubbo Proxy :ref:`configuration overview `. diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 86076823cc7e..05c351eb14d8 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -19,7 +19,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: HTTP connection manager] // HTTP connection manager :ref:`configuration overview `. @@ -27,7 +26,6 @@ import "gogoproto/gogo.proto"; // [#comment:next free field: 35] message HttpConnectionManager { enum CodecType { - option (gogoproto.goproto_enum_prefix) = false; // For every new connection, the connection manager will determine which // codec to use. This mode supports both ALPN for TLS listeners as well as @@ -80,7 +78,6 @@ message HttpConnectionManager { message Tracing { enum OperationName { - option (gogoproto.goproto_enum_prefix) = false; // The HTTP listener is used for ingress/incoming requests. INGRESS = 0; @@ -153,7 +150,6 @@ message HttpConnectionManager { string server_name = 10; enum ServerHeaderTransformation { - option (gogoproto.goproto_enum_prefix) = false; // Overwrite any Server header with the contents of server_name. OVERWRITE = 0; @@ -185,7 +181,7 @@ message HttpConnectionManager { // connection a drain sequence will occur prior to closing the connection. See // :ref:`drain_timeout // `. - google.protobuf.Duration idle_timeout = 11 [(gogoproto.stdduration) = true]; + google.protobuf.Duration idle_timeout = 11; // The stream idle timeout for connections managed by the connection manager. // If not specified, this defaults to 5 minutes. The default value was selected @@ -212,13 +208,13 @@ message HttpConnectionManager { // // A value of 0 will completely disable the connection manager stream idle // timeout, although per-route idle timeout overrides will continue to apply. - google.protobuf.Duration stream_idle_timeout = 24 [(gogoproto.stdduration) = true]; + google.protobuf.Duration stream_idle_timeout = 24; // A timeout for idle requests managed by the connection manager. // The timer is activated when the request is initiated, and is disarmed when the last byte of the // request is sent upstream (i.e. all decoding filters have processed the request), OR when the // response is initiated. If not specified or set to 0, this timeout is disabled. - google.protobuf.Duration request_timeout = 28 [(gogoproto.stdduration) = true]; + google.protobuf.Duration request_timeout = 28; // The time that Envoy will wait between sending an HTTP/2 “shutdown // notification” (GOAWAY frame with max stream ID) and a final GOAWAY frame. @@ -229,7 +225,7 @@ message HttpConnectionManager { // both when a connection hits the idle timeout or during general server // draining. The default grace period is 5000 milliseconds (5 seconds) if this // option is not specified. - google.protobuf.Duration drain_timeout = 12 [(gogoproto.stdduration) = true]; + google.protobuf.Duration drain_timeout = 12; // The delayed close timeout is for downstream connections managed by the HTTP connection manager. // It is defined as a grace period after connection close processing has been locally initiated @@ -261,7 +257,7 @@ message HttpConnectionManager { // A value of 0 will completely disable delayed close processing. When disabled, the downstream // connection's socket will be closed immediately after the write flush is completed or will // never close if the write flush does not complete. - google.protobuf.Duration delayed_close_timeout = 26 [(gogoproto.stdduration) = true]; + google.protobuf.Duration delayed_close_timeout = 26; // Configuration for :ref:`HTTP access logs ` // emitted by the connection manager. @@ -323,7 +319,6 @@ message HttpConnectionManager { // How to handle the :ref:`config_http_conn_man_headers_x-forwarded-client-cert` (XFCC) HTTP // header. enum ForwardClientCertDetails { - option (gogoproto.goproto_enum_prefix) = false; // Do not send the XFCC header to the next hop. This is the default value. SANITIZE = 0; diff --git a/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto index 4102c7016f2d..24e56507cd8d 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v3alpha/http_connection_manager.proto @@ -19,7 +19,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: HTTP connection manager] // HTTP connection manager :ref:`configuration overview `. @@ -27,7 +26,6 @@ import "gogoproto/gogo.proto"; // [#comment:next free field: 35] message HttpConnectionManager { enum CodecType { - option (gogoproto.goproto_enum_prefix) = false; // For every new connection, the connection manager will determine which // codec to use. This mode supports both ALPN for TLS listeners as well as @@ -81,7 +79,6 @@ message HttpConnectionManager { message Tracing { // [#comment:TODO(kyessenov): Align this field with listener traffic direction field.] enum OperationName { - option (gogoproto.goproto_enum_prefix) = false; // The HTTP listener is used for ingress/incoming requests. INGRESS = 0; @@ -144,7 +141,6 @@ message HttpConnectionManager { string server_name = 10; enum ServerHeaderTransformation { - option (gogoproto.goproto_enum_prefix) = false; // Overwrite any Server header with the contents of server_name. OVERWRITE = 0; @@ -176,7 +172,7 @@ message HttpConnectionManager { // connection a drain sequence will occur prior to closing the connection. See // :ref:`drain_timeout // `. - google.protobuf.Duration idle_timeout = 11 [(gogoproto.stdduration) = true]; + google.protobuf.Duration idle_timeout = 11; // The stream idle timeout for connections managed by the connection manager. // If not specified, this defaults to 5 minutes. The default value was selected @@ -203,13 +199,13 @@ message HttpConnectionManager { // // A value of 0 will completely disable the connection manager stream idle // timeout, although per-route idle timeout overrides will continue to apply. - google.protobuf.Duration stream_idle_timeout = 24 [(gogoproto.stdduration) = true]; + google.protobuf.Duration stream_idle_timeout = 24; // A timeout for idle requests managed by the connection manager. // The timer is activated when the request is initiated, and is disarmed when the last byte of the // request is sent upstream (i.e. all decoding filters have processed the request), OR when the // response is initiated. If not specified or set to 0, this timeout is disabled. - google.protobuf.Duration request_timeout = 28 [(gogoproto.stdduration) = true]; + google.protobuf.Duration request_timeout = 28; // The time that Envoy will wait between sending an HTTP/2 “shutdown // notification” (GOAWAY frame with max stream ID) and a final GOAWAY frame. @@ -220,7 +216,7 @@ message HttpConnectionManager { // both when a connection hits the idle timeout or during general server // draining. The default grace period is 5000 milliseconds (5 seconds) if this // option is not specified. - google.protobuf.Duration drain_timeout = 12 [(gogoproto.stdduration) = true]; + google.protobuf.Duration drain_timeout = 12; // The delayed close timeout is for downstream connections managed by the HTTP connection manager. // It is defined as a grace period after connection close processing has been locally initiated @@ -252,7 +248,7 @@ message HttpConnectionManager { // A value of 0 will completely disable delayed close processing. When disabled, the downstream // connection's socket will be closed immediately after the write flush is completed or will // never close if the write flush does not complete. - google.protobuf.Duration delayed_close_timeout = 26 [(gogoproto.stdduration) = true]; + google.protobuf.Duration delayed_close_timeout = 26; // Configuration for :ref:`HTTP access logs ` // emitted by the connection manager. @@ -314,7 +310,6 @@ message HttpConnectionManager { // How to handle the :ref:`config_http_conn_man_headers_x-forwarded-client-cert` (XFCC) HTTP // header. enum ForwardClientCertDetails { - option (gogoproto.goproto_enum_prefix) = false; // Do not send the XFCC header to the next hop. This is the default value. SANITIZE = 0; diff --git a/api/envoy/config/filter/network/rate_limit/v2/rate_limit.proto b/api/envoy/config/filter/network/rate_limit/v2/rate_limit.proto index 9a8f2f02146d..f5d484fac1ba 100644 --- a/api/envoy/config/filter/network/rate_limit/v2/rate_limit.proto +++ b/api/envoy/config/filter/network/rate_limit/v2/rate_limit.proto @@ -12,7 +12,6 @@ import "envoy/config/ratelimit/v2/rls.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Rate limit] // Rate limit :ref:`configuration overview `. @@ -30,7 +29,7 @@ message RateLimit { // The timeout in milliseconds for the rate limit service RPC. If not // set, this defaults to 20ms. - google.protobuf.Duration timeout = 4 [(gogoproto.stdduration) = true]; + google.protobuf.Duration timeout = 4; // The filter's behaviour in case the rate limiting service does // not respond back. When it is set to true, Envoy will not allow traffic in case of diff --git a/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto b/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto index 60e2d27aff6b..522fe145a7f7 100644 --- a/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto +++ b/api/envoy/config/filter/network/rate_limit/v3alpha/rate_limit.proto @@ -12,7 +12,6 @@ import "envoy/config/ratelimit/v3alpha/rls.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Rate limit] // Rate limit :ref:`configuration overview `. @@ -30,7 +29,7 @@ message RateLimit { // The timeout in milliseconds for the rate limit service RPC. If not // set, this defaults to 20ms. - google.protobuf.Duration timeout = 4 [(gogoproto.stdduration) = true]; + google.protobuf.Duration timeout = 4; // The filter's behaviour in case the rate limiting service does // not respond back. When it is set to true, Envoy will not allow traffic in case of diff --git a/api/envoy/config/filter/network/rbac/v2/rbac.proto b/api/envoy/config/filter/network/rbac/v2/rbac.proto index c6c6fac41c57..c192b888e559 100644 --- a/api/envoy/config/filter/network/rbac/v2/rbac.proto +++ b/api/envoy/config/filter/network/rbac/v2/rbac.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.filter.network.rbac.v2"; import "envoy/config/rbac/v2/rbac.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: RBAC] // Role-Based Access Control :ref:`configuration overview `. diff --git a/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto b/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto index 5faa5f5c087c..c1dcd6568262 100644 --- a/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto +++ b/api/envoy/config/filter/network/rbac/v3alpha/rbac.proto @@ -9,7 +9,6 @@ option java_package = "io.envoyproxy.envoy.config.filter.network.rbac.v3alpha"; import "envoy/config/rbac/v3alpha/rbac.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: RBAC] // Role-Based Access Control :ref:`configuration overview `. diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index 656cedf75025..35d1110df12c 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -12,7 +12,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Redis Proxy] // Redis Proxy :ref:`configuration overview `. @@ -40,8 +39,7 @@ message RedisProxy { // The only exception to this behavior is when a connection to a backend is not yet established. // In that case, the connect timeout on the cluster will govern the timeout until the connection // is ready. - google.protobuf.Duration op_timeout = 1 - [(validate.rules).duration.required = true, (gogoproto.stdduration) = true]; + google.protobuf.Duration op_timeout = 1 [(validate.rules).duration.required = true]; // Use hash tagging on every redis key to guarantee that keys with the same hash tag will be // forwarded to the same upstream. The hash key used for determining the upstream in a @@ -81,7 +79,7 @@ message RedisProxy { // before the timer fires. // If `max_buffer_size_before_flush` is set, but `buffer_flush_timeout` is not, the latter // defaults to 3ms. - google.protobuf.Duration buffer_flush_timeout = 5 [(gogoproto.stdduration) = true]; + google.protobuf.Duration buffer_flush_timeout = 5; // `max_upstream_unknown_connections` controls how many upstream connections to unknown hosts // can be created at any given time by any given worker thread (see `enable_redirection` for diff --git a/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto index a690451f7947..bb9307cb327d 100644 --- a/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v3alpha/redis_proxy.proto @@ -12,7 +12,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Redis Proxy] // Redis Proxy :ref:`configuration overview `. @@ -40,8 +39,7 @@ message RedisProxy { // The only exception to this behavior is when a connection to a backend is not yet established. // In that case, the connect timeout on the cluster will govern the timeout until the connection // is ready. - google.protobuf.Duration op_timeout = 1 - [(validate.rules).duration.required = true, (gogoproto.stdduration) = true]; + google.protobuf.Duration op_timeout = 1 [(validate.rules).duration.required = true]; // Use hash tagging on every redis key to guarantee that keys with the same hash tag will be // forwarded to the same upstream. The hash key used for determining the upstream in a @@ -81,7 +79,7 @@ message RedisProxy { // before the timer fires. // If `max_buffer_size_before_flush` is set, but `buffer_flush_timeout` is not, the latter // defaults to 3ms. - google.protobuf.Duration buffer_flush_timeout = 5 [(gogoproto.stdduration) = true]; + google.protobuf.Duration buffer_flush_timeout = 5; // `max_upstream_unknown_connections` controls how many upstream connections to unknown hosts // can be created at any given time by any given worker thread (see `enable_redirection` for diff --git a/api/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto b/api/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto index 376c980fb244..8e4453dd9f7d 100644 --- a/api/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto +++ b/api/envoy/config/filter/network/tcp_proxy/v2/tcp_proxy.proto @@ -14,7 +14,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: TCP Proxy] // TCP Proxy :ref:`configuration overview `. @@ -46,8 +45,7 @@ message TcpProxy { // is defined as the period in which there are no bytes sent or received on either // the upstream or downstream connection. If not set, connections will never be closed // by the TCP proxy due to being idle. - google.protobuf.Duration idle_timeout = 8 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration idle_timeout = 8 [(validate.rules).duration.gt = {}]; // [#not-implemented-hide:] The idle timeout for connections managed by the TCP proxy // filter. The idle timeout is defined as the period in which there is no diff --git a/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto b/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto index 4e04d7b352d1..ff74cc06cd75 100644 --- a/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto +++ b/api/envoy/config/filter/network/tcp_proxy/v3alpha/tcp_proxy.proto @@ -14,7 +14,6 @@ import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: TCP Proxy] // TCP Proxy :ref:`configuration overview `. @@ -46,8 +45,7 @@ message TcpProxy { // is defined as the period in which there are no bytes sent or received on either // the upstream or downstream connection. If not set, connections will never be closed // by the TCP proxy due to being idle. - google.protobuf.Duration idle_timeout = 8 - [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true]; + google.protobuf.Duration idle_timeout = 8 [(validate.rules).duration.gt = {}]; // [#not-implemented-hide:] The idle timeout for connections managed by the TCP proxy // filter. The idle timeout is defined as the period in which there is no diff --git a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto index 5d230d4474cc..33d120047159 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto +++ b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/route.proto @@ -12,7 +12,6 @@ import "envoy/api/v2/route/route.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Thrift Proxy Route Configuration] // Thrift Proxy :ref:`configuration overview `. diff --git a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto index 4cfe538798a2..823a1747527b 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto +++ b/api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto @@ -12,7 +12,6 @@ import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Thrift Proxy] // Thrift Proxy :ref:`configuration overview `. @@ -42,7 +41,6 @@ message ThriftProxy { // Thrift transport types supported by Envoy. enum TransportType { - option (gogoproto.goproto_enum_prefix) = false; // For downstream connections, the Thrift proxy will attempt to determine which transport to use. // For upstream connections, the Thrift proxy will use same transport as the downstream @@ -61,7 +59,6 @@ enum TransportType { // Thrift Protocol types supported by Envoy. enum ProtocolType { - option (gogoproto.goproto_enum_prefix) = false; // For downstream connections, the Thrift proxy will attempt to determine which protocol to use. // Note that the older, non-strict (or lax) binary protocol is not included in automatic protocol diff --git a/api/envoy/config/filter/network/thrift_proxy/v3alpha/route.proto b/api/envoy/config/filter/network/thrift_proxy/v3alpha/route.proto index 1e6777eedef1..3a351b1449d0 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v3alpha/route.proto +++ b/api/envoy/config/filter/network/thrift_proxy/v3alpha/route.proto @@ -12,7 +12,6 @@ import "envoy/api/v3alpha/route/route.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Thrift Proxy Route Configuration] // Thrift Proxy :ref:`configuration overview `. diff --git a/api/envoy/config/filter/network/thrift_proxy/v3alpha/thrift_proxy.proto b/api/envoy/config/filter/network/thrift_proxy/v3alpha/thrift_proxy.proto index 3c3c200b2ea3..83f44bbf720b 100644 --- a/api/envoy/config/filter/network/thrift_proxy/v3alpha/thrift_proxy.proto +++ b/api/envoy/config/filter/network/thrift_proxy/v3alpha/thrift_proxy.proto @@ -12,7 +12,6 @@ import "google/protobuf/any.proto"; import "google/protobuf/struct.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Thrift Proxy] // Thrift Proxy :ref:`configuration overview `. @@ -42,7 +41,6 @@ message ThriftProxy { // Thrift transport types supported by Envoy. enum TransportType { - option (gogoproto.goproto_enum_prefix) = false; // For downstream connections, the Thrift proxy will attempt to determine which transport to use. // For upstream connections, the Thrift proxy will use same transport as the downstream @@ -61,7 +59,6 @@ enum TransportType { // Thrift Protocol types supported by Envoy. enum ProtocolType { - option (gogoproto.goproto_enum_prefix) = false; // For downstream connections, the Thrift proxy will attempt to determine which protocol to use. // Note that the older, non-strict (or lax) binary protocol is not included in automatic protocol diff --git a/api/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto b/api/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto index 743bdc7256b7..ff2463b26c6c 100644 --- a/api/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto +++ b/api/envoy/config/filter/thrift/rate_limit/v2alpha1/rate_limit.proto @@ -11,7 +11,6 @@ import "envoy/config/ratelimit/v2/rls.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Rate limit] // Rate limit :ref:`configuration overview `. @@ -34,7 +33,7 @@ message RateLimit { // The timeout in milliseconds for the rate limit service RPC. If not // set, this defaults to 20ms. - google.protobuf.Duration timeout = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration timeout = 3; // The filter's behaviour in case the rate limiting service does // not respond back. When it is set to true, Envoy will not allow traffic in case of diff --git a/api/envoy/config/filter/thrift/rate_limit/v3alpha/rate_limit.proto b/api/envoy/config/filter/thrift/rate_limit/v3alpha/rate_limit.proto index e10197b7103e..017d9546a9a3 100644 --- a/api/envoy/config/filter/thrift/rate_limit/v3alpha/rate_limit.proto +++ b/api/envoy/config/filter/thrift/rate_limit/v3alpha/rate_limit.proto @@ -11,7 +11,6 @@ import "envoy/config/ratelimit/v3alpha/rls.proto"; import "google/protobuf/duration.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; // [#protodoc-title: Rate limit] // Rate limit :ref:`configuration overview `. @@ -34,7 +33,7 @@ message RateLimit { // The timeout in milliseconds for the rate limit service RPC. If not // set, this defaults to 20ms. - google.protobuf.Duration timeout = 3 [(gogoproto.stdduration) = true]; + google.protobuf.Duration timeout = 3; // The filter's behaviour in case the rate limiting service does // not respond back. When it is set to true, Envoy will not allow traffic in case of diff --git a/api/envoy/config/rbac/v2/rbac.proto b/api/envoy/config/rbac/v2/rbac.proto index 34a062be535b..f3546caa7bff 100644 --- a/api/envoy/config/rbac/v2/rbac.proto +++ b/api/envoy/config/rbac/v2/rbac.proto @@ -1,7 +1,6 @@ syntax = "proto3"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; import "envoy/api/v2/core/address.proto"; import "envoy/api/v2/route/route.proto"; import "envoy/type/matcher/metadata.proto"; @@ -15,8 +14,6 @@ option java_outer_classname = "RbacProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.rbac.v2"; -option (gogoproto.stable_marshaler_all) = true; - // [#protodoc-title: Role Based Access Control (RBAC)] // Role Based Access Control (RBAC) provides service-level and method-level access control for a diff --git a/api/envoy/config/rbac/v3alpha/rbac.proto b/api/envoy/config/rbac/v3alpha/rbac.proto index 9087e745690d..3fe9fe41c9d9 100644 --- a/api/envoy/config/rbac/v3alpha/rbac.proto +++ b/api/envoy/config/rbac/v3alpha/rbac.proto @@ -1,7 +1,6 @@ syntax = "proto3"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; import "envoy/api/v3alpha/core/address.proto"; import "envoy/api/v3alpha/route/route.proto"; import "envoy/type/matcher/metadata.proto"; @@ -15,8 +14,6 @@ option java_outer_classname = "RbacProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.config.rbac.v3alpha"; -option (gogoproto.stable_marshaler_all) = true; - // [#protodoc-title: Role Based Access Control (RBAC)] // Role Based Access Control (RBAC) provides service-level and method-level access control for a diff --git a/api/envoy/data/accesslog/v2/accesslog.proto b/api/envoy/data/accesslog/v2/accesslog.proto index 7309d4a362a6..bc6ff86bbd85 100644 --- a/api/envoy/data/accesslog/v2/accesslog.proto +++ b/api/envoy/data/accesslog/v2/accesslog.proto @@ -12,11 +12,8 @@ import "envoy/api/v2/core/base.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; -import "gogoproto/gogo.proto"; import "validate/validate.proto"; -option (gogoproto.stable_marshaler_all) = true; - // [#protodoc-title: gRPC access logs] // Envoy access logs describe incoming interaction with Envoy over a fixed // period of time, and typically cover a single request/response exchange, @@ -85,37 +82,37 @@ message AccessLogCommon { // The time that Envoy started servicing this request. This is effectively the time that the first // downstream byte is received. - google.protobuf.Timestamp start_time = 5 [(gogoproto.stdtime) = true]; + google.protobuf.Timestamp start_time = 5; // Interval between the first downstream byte received and the last // downstream byte received (i.e. time it takes to receive a request). - google.protobuf.Duration time_to_last_rx_byte = 6 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_last_rx_byte = 6; // Interval between the first downstream byte received and the first upstream byte sent. There may // by considerable delta between *time_to_last_rx_byte* and this value due to filters. // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about // not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_first_upstream_tx_byte = 7 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_first_upstream_tx_byte = 7; // Interval between the first downstream byte received and the last upstream byte sent. There may // by considerable delta between *time_to_last_rx_byte* and this value due to filters. // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about // not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_last_upstream_tx_byte = 8 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_last_upstream_tx_byte = 8; // Interval between the first downstream byte received and the first upstream // byte received (i.e. time it takes to start receiving a response). - google.protobuf.Duration time_to_first_upstream_rx_byte = 9 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_first_upstream_rx_byte = 9; // Interval between the first downstream byte received and the last upstream // byte received (i.e. time it takes to receive a complete response). - google.protobuf.Duration time_to_last_upstream_rx_byte = 10 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_last_upstream_rx_byte = 10; // Interval between the first downstream byte received and the first downstream byte sent. // There may be a considerable delta between the *time_to_first_upstream_rx_byte* and this field // due to filters. Additionally, the same caveats apply as documented in // *time_to_last_downstream_tx_byte* about not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_first_downstream_tx_byte = 11 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_first_downstream_tx_byte = 11; // Interval between the first downstream byte received and the last downstream byte sent. // Depending on protocol, buffering, windowing, filters, etc. there may be a considerable delta @@ -123,7 +120,7 @@ message AccessLogCommon { // time. In the current implementation it does not include kernel socket buffer time. In the // current implementation it also does not include send window buffering inside the HTTP/2 codec. // In the future it is likely that work will be done to make this duration more accurate. - google.protobuf.Duration time_to_last_downstream_tx_byte = 12 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_last_downstream_tx_byte = 12; // The upstream remote/destination address that handles this exchange. This does not include // retries. diff --git a/api/envoy/data/accesslog/v3alpha/accesslog.proto b/api/envoy/data/accesslog/v3alpha/accesslog.proto index b4588ecd31ff..2cdd44bbd10f 100644 --- a/api/envoy/data/accesslog/v3alpha/accesslog.proto +++ b/api/envoy/data/accesslog/v3alpha/accesslog.proto @@ -12,11 +12,8 @@ import "envoy/api/v3alpha/core/base.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; -import "gogoproto/gogo.proto"; import "validate/validate.proto"; -option (gogoproto.stable_marshaler_all) = true; - // [#protodoc-title: gRPC access logs] // Envoy access logs describe incoming interaction with Envoy over a fixed // period of time, and typically cover a single request/response exchange, @@ -85,37 +82,37 @@ message AccessLogCommon { // The time that Envoy started servicing this request. This is effectively the time that the first // downstream byte is received. - google.protobuf.Timestamp start_time = 5 [(gogoproto.stdtime) = true]; + google.protobuf.Timestamp start_time = 5; // Interval between the first downstream byte received and the last // downstream byte received (i.e. time it takes to receive a request). - google.protobuf.Duration time_to_last_rx_byte = 6 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_last_rx_byte = 6; // Interval between the first downstream byte received and the first upstream byte sent. There may // by considerable delta between *time_to_last_rx_byte* and this value due to filters. // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about // not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_first_upstream_tx_byte = 7 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_first_upstream_tx_byte = 7; // Interval between the first downstream byte received and the last upstream byte sent. There may // by considerable delta between *time_to_last_rx_byte* and this value due to filters. // Additionally, the same caveats apply as documented in *time_to_last_downstream_tx_byte* about // not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_last_upstream_tx_byte = 8 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_last_upstream_tx_byte = 8; // Interval between the first downstream byte received and the first upstream // byte received (i.e. time it takes to start receiving a response). - google.protobuf.Duration time_to_first_upstream_rx_byte = 9 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_first_upstream_rx_byte = 9; // Interval between the first downstream byte received and the last upstream // byte received (i.e. time it takes to receive a complete response). - google.protobuf.Duration time_to_last_upstream_rx_byte = 10 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_last_upstream_rx_byte = 10; // Interval between the first downstream byte received and the first downstream byte sent. // There may be a considerable delta between the *time_to_first_upstream_rx_byte* and this field // due to filters. Additionally, the same caveats apply as documented in // *time_to_last_downstream_tx_byte* about not accounting for kernel socket buffer time, etc. - google.protobuf.Duration time_to_first_downstream_tx_byte = 11 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_first_downstream_tx_byte = 11; // Interval between the first downstream byte received and the last downstream byte sent. // Depending on protocol, buffering, windowing, filters, etc. there may be a considerable delta @@ -123,7 +120,7 @@ message AccessLogCommon { // time. In the current implementation it does not include kernel socket buffer time. In the // current implementation it also does not include send window buffering inside the HTTP/2 codec. // In the future it is likely that work will be done to make this duration more accurate. - google.protobuf.Duration time_to_last_downstream_tx_byte = 12 [(gogoproto.stdduration) = true]; + google.protobuf.Duration time_to_last_downstream_tx_byte = 12; // The upstream remote/destination address that handles this exchange. This does not include // retries. diff --git a/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto b/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto index 65559abc5485..836eeec42837 100644 --- a/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto +++ b/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto @@ -10,9 +10,6 @@ import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Outlier detection logging events] // :ref:`Outlier detection logging `. @@ -21,7 +18,7 @@ message OutlierDetectionEvent { // In case of eject represents type of ejection that took place. OutlierEjectionType type = 1 [(validate.rules).enum.defined_only = true]; // Timestamp for event. - google.protobuf.Timestamp timestamp = 2 [(gogoproto.stdtime) = true]; + google.protobuf.Timestamp timestamp = 2; // The time in seconds since the last action (either an ejection or unejection) took place. google.protobuf.UInt64Value secs_since_last_action = 3; // The :ref:`cluster ` that owns the ejected host. diff --git a/api/envoy/data/cluster/v3alpha/outlier_detection_event.proto b/api/envoy/data/cluster/v3alpha/outlier_detection_event.proto index 48f0e27b86bf..305163659728 100644 --- a/api/envoy/data/cluster/v3alpha/outlier_detection_event.proto +++ b/api/envoy/data/cluster/v3alpha/outlier_detection_event.proto @@ -10,9 +10,6 @@ import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Outlier detection logging events] // :ref:`Outlier detection logging `. @@ -21,7 +18,7 @@ message OutlierDetectionEvent { // In case of eject represents type of ejection that took place. OutlierEjectionType type = 1 [(validate.rules).enum.defined_only = true]; // Timestamp for event. - google.protobuf.Timestamp timestamp = 2 [(gogoproto.stdtime) = true]; + google.protobuf.Timestamp timestamp = 2; // The time in seconds since the last action (either an ejection or unejection) took place. google.protobuf.UInt64Value secs_since_last_action = 3; // The :ref:`cluster ` that owns the ejected host. diff --git a/api/envoy/data/core/v2alpha/health_check_event.proto b/api/envoy/data/core/v2alpha/health_check_event.proto index adfb6c67e5c7..c5b2f70a5e24 100644 --- a/api/envoy/data/core/v2alpha/health_check_event.proto +++ b/api/envoy/data/core/v2alpha/health_check_event.proto @@ -11,9 +11,6 @@ import "envoy/api/v2/core/address.proto"; import "google/protobuf/timestamp.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Health check logging events] // :ref:`Health check logging `. @@ -43,7 +40,7 @@ message HealthCheckEvent { } // Timestamp for event. - google.protobuf.Timestamp timestamp = 6 [(gogoproto.stdtime) = true]; + google.protobuf.Timestamp timestamp = 6; } enum HealthCheckFailureType { diff --git a/api/envoy/data/core/v3alpha/health_check_event.proto b/api/envoy/data/core/v3alpha/health_check_event.proto index 628b6870b64d..c714743f7070 100644 --- a/api/envoy/data/core/v3alpha/health_check_event.proto +++ b/api/envoy/data/core/v3alpha/health_check_event.proto @@ -11,9 +11,6 @@ import "envoy/api/v3alpha/core/address.proto"; import "google/protobuf/timestamp.proto"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Health check logging events] // :ref:`Health check logging `. @@ -43,7 +40,7 @@ message HealthCheckEvent { } // Timestamp for event. - google.protobuf.Timestamp timestamp = 6 [(gogoproto.stdtime) = true]; + google.protobuf.Timestamp timestamp = 6; } enum HealthCheckFailureType { diff --git a/api/envoy/service/auth/v2/attribute_context.proto b/api/envoy/service/auth/v2/attribute_context.proto index fe5bee033909..cfb71ec0ce5f 100644 --- a/api/envoy/service/auth/v2/attribute_context.proto +++ b/api/envoy/service/auth/v2/attribute_context.proto @@ -10,9 +10,6 @@ import "envoy/api/v2/core/address.proto"; import "envoy/api/v2/core/base.proto"; import "google/protobuf/timestamp.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: Attribute Context ] diff --git a/api/envoy/service/auth/v3alpha/attribute_context.proto b/api/envoy/service/auth/v3alpha/attribute_context.proto index de3164583167..95ac3428fc49 100644 --- a/api/envoy/service/auth/v3alpha/attribute_context.proto +++ b/api/envoy/service/auth/v3alpha/attribute_context.proto @@ -10,9 +10,6 @@ import "envoy/api/v3alpha/core/address.proto"; import "envoy/api/v3alpha/core/base.proto"; import "google/protobuf/timestamp.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.stable_marshaler_all) = true; // [#protodoc-title: Attribute Context ] diff --git a/api/envoy/type/percent.proto b/api/envoy/type/percent.proto index 551e93bfdd1e..c577093eea0a 100644 --- a/api/envoy/type/percent.proto +++ b/api/envoy/type/percent.proto @@ -7,9 +7,6 @@ option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type"; import "validate/validate.proto"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; // [#protodoc-title: Percent] diff --git a/api/envoy/type/range.proto b/api/envoy/type/range.proto index cc38e8f25f5e..f31cf32f07c4 100644 --- a/api/envoy/type/range.proto +++ b/api/envoy/type/range.proto @@ -6,10 +6,6 @@ option java_outer_classname = "RangeProto"; option java_multiple_files = true; option java_package = "io.envoyproxy.envoy.type"; -import "gogoproto/gogo.proto"; - -option (gogoproto.equal_all) = true; - // [#protodoc-title: Range] // Specifies the int64 start and end of the range using half-open interval semantics [start, From c6f0ee2d69525664e831198b2b2bff7c50e7efa4 Mon Sep 17 00:00:00 2001 From: l8huang Date: Thu, 5 Sep 2019 17:28:48 -0700 Subject: [PATCH 509/542] hotrestart: remove dynamic_resources from server config used by hotrestart_test (#8162) In the server config file `test/config/integration/server.yaml` used by //test/integration:hotrestart_test, `dynamic_resources` includes `lds_config` and `cds_config` definitions, which use HTTP API to fetch config, but CDS and LDS service do not exist, so the initial fetch will be failed with a connection failure, then Envoy server continue startup. Envoy server shouldn't continue startup because connection failure, see issue #8046. For this test, `dynamic_resources` is not needed, this change clean it up. Signed-off-by: lhuang8 --- test/config/integration/server.yaml | 35 +---------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/test/config/integration/server.yaml b/test/config/integration/server.yaml index 95a9a439efa4..e26cc5cf5e98 100644 --- a/test/config/integration/server.yaml +++ b/test/config/integration/server.yaml @@ -79,27 +79,6 @@ static_resources: catch_all_route: cluster: redis clusters: - - name: cds - connect_timeout: 5s - hosts: - - socket_address: - address: {{ ip_loopback_address }} - port_value: 4 - dns_lookup_family: "{{ dns_lookup_family }}" - - name: rds - connect_timeout: 5s - hosts: - - socket_address: - address: {{ ip_loopback_address }} - port_value: 4 - dns_lookup_family: "{{ dns_lookup_family }}" - - name: lds - connect_timeout: 5s - hosts: - - socket_address: - address: {{ ip_loopback_address }} - port_value: 4 - dns_lookup_family: "{{ dns_lookup_family }}" - name: cluster_1 connect_timeout: 5s hosts: @@ -141,19 +120,7 @@ static_resources: port_value: 4 dns_lookup_family: "{{ dns_lookup_family }}" outlier_detection: {} -dynamic_resources: - lds_config: - api_config_source: - api_type: REST - cluster_names: - - lds - refresh_delay: 30s - cds_config: - api_config_source: - api_type: REST - cluster_names: - - cds - refresh_delay: 30s +dynamic_resources: {} cluster_manager: {} flags_path: "/invalid_flags" stats_sinks: From 84ceadc16ae763c6c7340acea8915509aa2f5eab Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Thu, 5 Sep 2019 22:04:44 -0700 Subject: [PATCH 510/542] clang-tidy: misc-unused-using-decls (#8159) Description: clang-tidy check to flag unused using statements. There's a lot in test code that's just copy pasta, and it's hard to manually review whether it's being used, especially for things like using testing::_; Risk Level: low Testing: existing Docs Changes: N/A Release Notes: N/A Signed-off-by: Derek Argueta --- .clang-tidy | 2 ++ .../http/grpc_json_transcoder/json_transcoder_filter.cc | 2 -- test/common/access_log/access_log_formatter_test.cc | 1 - test/common/config/filesystem_subscription_impl_test.cc | 3 ++- test/common/config/filesystem_subscription_test_harness.h | 1 - test/common/config/rds_json_test.cc | 2 -- test/common/config/utility_test.cc | 2 -- test/common/event/dispatcher_impl_test.cc | 2 -- test/common/grpc/async_client_impl_test.cc | 1 - test/common/http/async_client_impl_test.cc | 1 - test/common/http/codec_client_test.cc | 2 -- test/common/http/codes_test.cc | 1 - test/common/http/conn_manager_impl_test.cc | 2 -- test/common/http/conn_manager_utility_test.cc | 1 - test/common/http/http1/conn_pool_test.cc | 1 - test/common/http/http2/conn_pool_test.cc | 1 - test/common/init/manager_impl_test.cc | 1 - test/common/json/config_schemas_test.cc | 2 -- test/common/network/dns_impl_test.cc | 1 - test/common/network/socket_option_test.h | 1 - test/common/router/config_impl_test.cc | 2 -- test/common/router/rds_impl_test.cc | 3 --- test/common/router/router_ratelimit_test.cc | 2 -- test/common/router/router_test.cc | 2 -- test/common/router/scoped_config_impl_test.cc | 1 - test/common/router/scoped_rds_test.cc | 3 --- test/common/router/vhds_test.cc | 7 ------- test/common/runtime/runtime_impl_test.cc | 2 -- test/common/secret/sds_api_test.cc | 1 - test/common/stats/stats_matcher_impl_test.cc | 3 --- test/common/stats/thread_local_store_test.cc | 1 - test/common/tcp/conn_pool_test.cc | 3 --- test/common/tracing/http_tracer_impl_test.cc | 2 -- test/common/upstream/cds_api_impl_test.cc | 3 --- test/common/upstream/cluster_factory_impl_test.cc | 4 ---- test/common/upstream/cluster_manager_impl_test.cc | 2 -- test/common/upstream/conn_pool_map_impl_test.cc | 1 - test/common/upstream/eds_test.cc | 3 --- test/common/upstream/health_checker_impl_test.cc | 2 -- test/common/upstream/original_dst_cluster_test.cc | 2 -- test/common/upstream/ring_hash_lb_test.cc | 1 - test/common/upstream/subset_lb_test.cc | 2 -- test/common/upstream/upstream_impl_test.cc | 2 -- .../access_loggers/grpc/grpc_access_log_impl_test.cc | 1 - test/extensions/access_loggers/grpc/http_config_test.cc | 1 - test/extensions/clusters/redis/mocks.cc | 3 --- test/extensions/clusters/redis/redis_cluster_test.cc | 5 ----- test/extensions/filters/common/expr/context_test.cc | 2 -- .../filters/common/ext_authz/ext_authz_http_impl_test.cc | 3 --- .../common/original_src/original_src_socket_option_test.cc | 1 - .../filters/common/ratelimit/ratelimit_impl_test.cc | 2 -- test/extensions/filters/common/rbac/engine_impl_test.cc | 2 -- test/extensions/filters/common/rbac/matchers_test.cc | 1 - test/extensions/filters/http/buffer/buffer_filter_test.cc | 3 --- test/extensions/filters/http/common/jwks_fetcher_test.cc | 1 - test/extensions/filters/http/cors/cors_filter_test.cc | 5 ----- test/extensions/filters/http/csrf/csrf_filter_test.cc | 5 ----- test/extensions/filters/http/dynamo/dynamo_filter_test.cc | 1 - test/extensions/filters/http/dynamo/dynamo_stats_test.cc | 4 ---- test/extensions/filters/http/fault/config_test.cc | 1 - test/extensions/filters/http/fault/fault_filter_test.cc | 3 --- .../http/grpc_http1_bridge/http1_bridge_filter_test.cc | 2 -- .../http/grpc_http1_reverse_bridge/reverse_bridge_test.cc | 1 - .../grpc_json_transcoder_integration_test.cc | 1 - .../grpc_json_transcoder/json_transcoder_filter_test.cc | 4 ---- .../filters/http/health_check/health_check_test.cc | 3 --- .../filters/http/ip_tagging/ip_tagging_filter_test.cc | 1 - test/extensions/filters/http/lua/lua_filter_test.cc | 1 - test/extensions/filters/http/lua/wrappers_test.cc | 1 - .../filters/http/original_src/original_src_test.cc | 1 - test/extensions/filters/http/ratelimit/config_test.cc | 1 - test/extensions/filters/http/ratelimit/ratelimit_test.cc | 1 - .../filters/listener/http_inspector/http_inspector_test.cc | 2 -- .../filters/listener/original_src/original_src_test.cc | 1 - .../filters/listener/proxy_protocol/proxy_protocol_test.cc | 1 - .../listener/tls_inspector/tls_inspector_benchmark.cc | 7 ------- .../filters/listener/tls_inspector/tls_inspector_test.cc | 2 -- .../network/client_ssl_auth/client_ssl_auth_test.cc | 2 -- .../filters/network/common/redis/client_impl_test.cc | 4 ---- .../filters/network/dubbo_proxy/conn_manager_test.cc | 3 --- .../extensions/filters/network/dubbo_proxy/decoder_test.cc | 3 --- .../dubbo_proxy/dubbo_hessian2_serializer_impl_test.cc | 2 -- .../network/dubbo_proxy/dubbo_protocol_impl_test.cc | 2 -- test/extensions/filters/network/dubbo_proxy/mocks.h | 2 -- test/extensions/filters/network/dubbo_proxy/router_test.cc | 2 -- .../extensions/filters/network/ext_authz/ext_authz_test.cc | 1 - .../filters/network/kafka/request_codec_unit_test.cc | 2 -- .../filters/network/kafka/response_codec_unit_test.cc | 2 -- test/extensions/filters/network/mongo_proxy/proxy_test.cc | 1 - test/extensions/filters/network/ratelimit/config_test.cc | 1 - test/extensions/filters/network/redis_proxy/mocks.cc | 2 -- .../network/redis_proxy/redis_proxy_integration_test.cc | 1 - .../filters/network/redis_proxy/router_impl_test.cc | 4 ---- .../filters/network/sni_cluster/sni_cluster_test.cc | 1 - .../network/thrift_proxy/auto_protocol_impl_test.cc | 1 - .../filters/network/thrift_proxy/conn_manager_test.cc | 1 - .../filters/network/thrift_proxy/decoder_test.cc | 1 - .../network/thrift_proxy/filters/ratelimit/config_test.cc | 1 - .../thrift_proxy/filters/ratelimit/ratelimit_test.cc | 1 - .../network/thrift_proxy/header_transport_impl_test.cc | 1 - .../filters/network/thrift_proxy/route_matcher_test.cc | 2 -- .../extensions/filters/network/thrift_proxy/router_test.cc | 1 - .../network/thrift_proxy/thrift_object_impl_test.cc | 2 -- test/extensions/filters/network/thrift_proxy/utility.h | 4 ++-- test/extensions/health_checkers/redis/redis_test.cc | 2 -- test/extensions/stats_sinks/dog_statsd/config_test.cc | 3 --- test/extensions/stats_sinks/hystrix/config_test.cc | 3 --- test/extensions/stats_sinks/hystrix/hystrix_test.cc | 1 - .../metrics_service/grpc_metrics_service_impl_test.cc | 1 - test/extensions/stats_sinks/statsd/config_test.cc | 3 --- test/extensions/tracers/datadog/config_test.cc | 1 - .../extensions/tracers/datadog/datadog_tracer_impl_test.cc | 1 - test/extensions/tracers/dynamic_ot/config_test.cc | 1 - test/extensions/tracers/lightstep/config_test.cc | 1 - test/extensions/tracers/opencensus/tracer_test.cc | 1 - test/extensions/transport_sockets/alts/config_test.cc | 3 --- .../transport_sockets/alts/tsi_frame_protector_test.cc | 5 ----- test/extensions/transport_sockets/alts/tsi_socket_test.cc | 1 - .../tls/integration/ssl_integration_test.cc | 2 -- .../tls/integration/ssl_integration_test.h | 2 -- test/integration/ads_integration.cc | 3 --- test/integration/ads_integration_test.cc | 3 --- test/integration/cds_integration_test.cc | 3 --- test/integration/http2_upstream_integration_test.cc | 2 -- test/integration/http_integration.cc | 6 ------ test/integration/integration_test.cc | 1 - test/integration/protocol_integration_test.cc | 4 ---- test/integration/sds_dynamic_integration_test.cc | 3 --- test/integration/sds_static_integration_test.cc | 3 --- test/integration/vhds_integration_test.cc | 3 --- test/integration/websocket_integration_test.cc | 2 -- test/mocks/access_log/mocks.cc | 1 - test/mocks/api/mocks.cc | 1 - test/mocks/http/mocks.cc | 5 ----- test/mocks/local_info/mocks.cc | 1 - test/mocks/server/mocks.cc | 1 - test/mocks/stats/mocks.cc | 1 - test/mocks/stream_info/mocks.cc | 1 - test/mocks/upstream/mocks.cc | 1 - test/server/configuration_impl_test.cc | 2 -- test/server/drain_manager_impl_test.cc | 1 - test/server/filter_chain_manager_impl_test.cc | 4 ---- test/server/hot_restart_impl_test.cc | 2 -- test/server/hot_restarting_parent_test.cc | 1 - test/server/http/config_tracker_impl_test.cc | 2 -- test/server/lds_api_test.cc | 1 - test/server/options_impl_test.cc | 2 -- test/server/server_test.cc | 2 -- test/test_common/utility.h | 4 ++-- test/test_common/utility_test.cc | 2 -- tools/spelling_dictionary.txt | 1 + 151 files changed, 9 insertions(+), 299 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 248d7e642762..c5d817a7f047 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -2,6 +2,7 @@ Checks: 'abseil-*, bugprone-*, clang-analyzer-*, clang-diagnostic-*, + misc-unused-using-decls, modernize-*, performance-*, readability-braces-around-statements, @@ -20,6 +21,7 @@ WarningsAsErrors: 'abseil-duration-*, bugprone-unused-raii, bugprone-use-after-move, clang-analyzer-core.DivideZero, + misc-unused-using-decls, modernize-deprecated-headers, modernize-loop-convert, modernize-make-shared, diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc index fb26f1f39c0d..0e7f8caba9a8 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc @@ -23,8 +23,6 @@ #include "grpc_transcoding/path_matcher_utility.h" #include "grpc_transcoding/response_to_json_translator.h" -using Envoy::Protobuf::DescriptorPool; -using Envoy::Protobuf::FileDescriptor; using Envoy::Protobuf::FileDescriptorSet; using Envoy::Protobuf::io::ZeroCopyInputStream; using Envoy::ProtobufUtil::Status; diff --git a/test/common/access_log/access_log_formatter_test.cc b/test/common/access_log/access_log_formatter_test.cc index d10665a034f4..ece0a648a3d2 100644 --- a/test/common/access_log/access_log_formatter_test.cc +++ b/test/common/access_log/access_log_formatter_test.cc @@ -17,7 +17,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::Const; using testing::NiceMock; using testing::Return; diff --git a/test/common/config/filesystem_subscription_impl_test.cc b/test/common/config/filesystem_subscription_impl_test.cc index 524914346484..8c0f19775c22 100644 --- a/test/common/config/filesystem_subscription_impl_test.cc +++ b/test/common/config/filesystem_subscription_impl_test.cc @@ -6,7 +6,8 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using ::testing::Throw; +using testing::Return; +using testing::Throw; namespace Envoy { namespace Config { diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 5bed6e9b1942..901b5cf29b31 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -20,7 +20,6 @@ using testing::_; using testing::NiceMock; -using testing::Return; namespace Envoy { namespace Config { diff --git a/test/common/config/rds_json_test.cc b/test/common/config/rds_json_test.cc index 190220b84fae..6a0ed6717e0e 100644 --- a/test/common/config/rds_json_test.cc +++ b/test/common/config/rds_json_test.cc @@ -6,8 +6,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; - namespace Envoy { namespace Config { namespace { diff --git a/test/common/config/utility_test.cc b/test/common/config/utility_test.cc index dea04ddb1fd6..a84513f03633 100644 --- a/test/common/config/utility_test.cc +++ b/test/common/config/utility_test.cc @@ -18,10 +18,8 @@ #include "gtest/gtest.h" using testing::_; -using testing::AtLeast; using testing::Ref; using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Config { diff --git a/test/common/event/dispatcher_impl_test.cc b/test/common/event/dispatcher_impl_test.cc index 8df3bf5c55f4..58e750b29375 100644 --- a/test/common/event/dispatcher_impl_test.cc +++ b/test/common/event/dispatcher_impl_test.cc @@ -18,8 +18,6 @@ using testing::_; using testing::InSequence; using testing::NiceMock; -using testing::Return; -using testing::StartsWith; namespace Envoy { namespace Event { diff --git a/test/common/grpc/async_client_impl_test.cc b/test/common/grpc/async_client_impl_test.cc index 1a33686075c4..469b23bfe601 100644 --- a/test/common/grpc/async_client_impl_test.cc +++ b/test/common/grpc/async_client_impl_test.cc @@ -14,7 +14,6 @@ using testing::Eq; using testing::Invoke; using testing::Return; using testing::ReturnRef; -using testing::Throw; namespace Envoy { namespace Grpc { diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index 29985046fa7f..f9b90ff2ab8b 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -26,7 +26,6 @@ using testing::_; using testing::Invoke; using testing::NiceMock; -using testing::Ref; using testing::Return; using testing::ReturnRef; diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 7fc083eba090..75693aebbd13 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -30,8 +30,6 @@ using testing::InvokeWithoutArgs; using testing::NiceMock; using testing::Pointee; using testing::Ref; -using testing::Return; -using testing::SaveArg; using testing::Throw; namespace Envoy { diff --git a/test/common/http/codes_test.cc b/test/common/http/codes_test.cc index de05c4178e94..4f7766fc2d49 100644 --- a/test/common/http/codes_test.cc +++ b/test/common/http/codes_test.cc @@ -17,7 +17,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::Property; namespace Envoy { diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 179367e62589..846c674d1ec6 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -50,7 +50,6 @@ using testing::_; using testing::An; using testing::AnyNumber; using testing::AtLeast; -using testing::DoAll; using testing::Eq; using testing::HasSubstr; using testing::InSequence; @@ -61,7 +60,6 @@ using testing::NiceMock; using testing::Ref; using testing::Return; using testing::ReturnRef; -using testing::Sequence; namespace Envoy { namespace Http { diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index a9694d8ec785..67b3e412583d 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -21,7 +21,6 @@ using testing::_; using testing::An; -using testing::InSequence; using testing::Matcher; using testing::NiceMock; using testing::Return; diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc index 73b79b35205f..a667ea493898 100644 --- a/test/common/http/http1/conn_pool_test.cc +++ b/test/common/http/http1/conn_pool_test.cc @@ -30,7 +30,6 @@ using testing::NiceMock; using testing::Property; using testing::Return; using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Http { diff --git a/test/common/http/http2/conn_pool_test.cc b/test/common/http/http2/conn_pool_test.cc index 5e2cac038136..8ac3f66a31b8 100644 --- a/test/common/http/http2/conn_pool_test.cc +++ b/test/common/http/http2/conn_pool_test.cc @@ -27,7 +27,6 @@ using testing::NiceMock; using testing::Property; using testing::Return; using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Http { diff --git a/test/common/init/manager_impl_test.cc b/test/common/init/manager_impl_test.cc index 8a479b0c1977..28465b1d2a16 100644 --- a/test/common/init/manager_impl_test.cc +++ b/test/common/init/manager_impl_test.cc @@ -5,7 +5,6 @@ #include "gtest/gtest.h" using ::testing::InSequence; -using ::testing::InvokeWithoutArgs; namespace Envoy { namespace Init { diff --git a/test/common/json/config_schemas_test.cc b/test/common/json/config_schemas_test.cc index 29585f2e65fa..1eefa9064847 100644 --- a/test/common/json/config_schemas_test.cc +++ b/test/common/json/config_schemas_test.cc @@ -12,8 +12,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; - namespace Envoy { namespace Json { namespace { diff --git a/test/common/network/dns_impl_test.cc b/test/common/network/dns_impl_test.cc index 49d01a430d0c..a5b1cee0bcd5 100644 --- a/test/common/network/dns_impl_test.cc +++ b/test/common/network/dns_impl_test.cc @@ -34,7 +34,6 @@ using testing::_; using testing::InSequence; -using testing::Mock; using testing::NiceMock; using testing::Return; diff --git a/test/common/network/socket_option_test.h b/test/common/network/socket_option_test.h index 2fe7a59d2c7e..c4d57591209a 100644 --- a/test/common/network/socket_option_test.h +++ b/test/common/network/socket_option_test.h @@ -13,7 +13,6 @@ using testing::_; using testing::Invoke; using testing::NiceMock; -using testing::Return; namespace Envoy { namespace Network { diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 717bf35651a7..bc5004098057 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -31,14 +31,12 @@ using testing::_; using testing::ContainerEq; -using testing::ElementsAreArray; using testing::Eq; using testing::Matcher; using testing::MockFunction; using testing::NiceMock; using testing::Return; using testing::ReturnRef; -using testing::StrNe; namespace Envoy { namespace Router { diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 8bd32f455302..b581017f3f92 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -29,9 +29,6 @@ using testing::_; using testing::Eq; using testing::InSequence; using testing::Invoke; -using testing::Return; -using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Router { diff --git a/test/common/router/router_ratelimit_test.cc b/test/common/router/router_ratelimit_test.cc index 207a82a2829f..c5cd117dda26 100644 --- a/test/common/router/router_ratelimit_test.cc +++ b/test/common/router/router_ratelimit_test.cc @@ -20,9 +20,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::NiceMock; -using testing::ReturnRef; namespace Envoy { namespace Router { diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 342c82c1150c..a6e67958042f 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -44,11 +44,9 @@ using testing::Invoke; using testing::Matcher; using testing::MockFunction; using testing::NiceMock; -using testing::Ref; using testing::Return; using testing::ReturnPointee; using testing::ReturnRef; -using testing::SaveArg; using testing::StartsWith; namespace Envoy { diff --git a/test/common/router/scoped_config_impl_test.cc b/test/common/router/scoped_config_impl_test.cc index cf9bfd83055c..0decf3b2e7a3 100644 --- a/test/common/router/scoped_config_impl_test.cc +++ b/test/common/router/scoped_config_impl_test.cc @@ -13,7 +13,6 @@ namespace { using ::Envoy::Http::TestHeaderMapImpl; using ::testing::NiceMock; -using ::testing::Return; class FooFragment : public ScopeKeyFragmentBase { public: diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 8ee6cfe8aecf..3967703c4125 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -20,15 +20,12 @@ #include "gtest/gtest.h" using testing::AnyNumber; -using testing::ByMove; -using testing::DoAll; using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::IsNull; using testing::NiceMock; using testing::Return; -using testing::ReturnRefOfCopy; namespace Envoy { namespace Router { diff --git a/test/common/router/vhds_test.cc b/test/common/router/vhds_test.cc index 01d3287421e0..4bac9088b22e 100644 --- a/test/common/router/vhds_test.cc +++ b/test/common/router/vhds_test.cc @@ -24,13 +24,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; -using testing::InSequence; -using testing::Invoke; -using testing::Return; -using testing::ReturnRef; -using testing::SaveArg; - namespace Envoy { namespace Router { namespace { diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 55d9bf662744..11d029309411 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -23,8 +23,6 @@ using testing::Invoke; using testing::InvokeWithoutArgs; using testing::NiceMock; using testing::Return; -using testing::ReturnNew; -using testing::ReturnRef; namespace Envoy { namespace Runtime { diff --git a/test/common/secret/sds_api_test.cc b/test/common/secret/sds_api_test.cc index ea65ba4f5f10..bcedd9d363a3 100644 --- a/test/common/secret/sds_api_test.cc +++ b/test/common/secret/sds_api_test.cc @@ -21,7 +21,6 @@ using ::testing::_; using ::testing::Invoke; -using ::testing::Return; namespace Envoy { namespace Secret { diff --git a/test/common/stats/stats_matcher_impl_test.cc b/test/common/stats/stats_matcher_impl_test.cc index 485560f2b702..9277a545e16a 100644 --- a/test/common/stats/stats_matcher_impl_test.cc +++ b/test/common/stats/stats_matcher_impl_test.cc @@ -6,9 +6,6 @@ #include "gtest/gtest.h" -using testing::IsFalse; -using testing::IsTrue; - namespace Envoy { namespace Stats { diff --git a/test/common/stats/thread_local_store_test.cc b/test/common/stats/thread_local_store_test.cc index fb4b2d594ee4..baba626c31a6 100644 --- a/test/common/stats/thread_local_store_test.cc +++ b/test/common/stats/thread_local_store_test.cc @@ -26,7 +26,6 @@ using testing::_; using testing::InSequence; -using testing::Invoke; using testing::NiceMock; using testing::Ref; using testing::Return; diff --git a/test/common/tcp/conn_pool_test.cc b/test/common/tcp/conn_pool_test.cc index d1521e28226a..9877d4437709 100644 --- a/test/common/tcp/conn_pool_test.cc +++ b/test/common/tcp/conn_pool_test.cc @@ -18,14 +18,11 @@ #include "gtest/gtest.h" using testing::_; -using testing::DoAll; using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Property; using testing::Return; -using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Tcp { diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index c78fa72e4106..496da50b5873 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -25,11 +25,9 @@ using testing::_; using testing::Eq; -using testing::Invoke; using testing::NiceMock; using testing::Return; using testing::ReturnPointee; -using testing::ReturnRef; namespace Envoy { namespace Tracing { diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index 72c0aeb6fed2..48101c09578b 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -19,11 +19,8 @@ #include "gtest/gtest.h" using testing::_; -using testing::AnyNumber; using testing::InSequence; -using testing::Invoke; using testing::Return; -using testing::ReturnRef; using testing::StrEq; using testing::Throw; diff --git a/test/common/upstream/cluster_factory_impl_test.cc b/test/common/upstream/cluster_factory_impl_test.cc index 85ed32e327c6..6b4091db67ae 100644 --- a/test/common/upstream/cluster_factory_impl_test.cc +++ b/test/common/upstream/cluster_factory_impl_test.cc @@ -24,11 +24,7 @@ #include "test/mocks/server/mocks.h" #include "test/mocks/ssl/mocks.h" -using testing::_; -using testing::ContainerEq; -using testing::Invoke; using testing::NiceMock; -using testing::ReturnRef; namespace Envoy { namespace Upstream { diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 6fbc2e5afea2..a5ef0c27c2fc 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -48,10 +48,8 @@ using testing::InSequence; using testing::Invoke; using testing::Mock; using testing::NiceMock; -using testing::Pointee; using testing::Return; using testing::ReturnNew; -using testing::ReturnRef; using testing::SaveArg; namespace Envoy { diff --git a/test/common/upstream/conn_pool_map_impl_test.cc b/test/common/upstream/conn_pool_map_impl_test.cc index 83839f4e76b6..059fec2ef0da 100644 --- a/test/common/upstream/conn_pool_map_impl_test.cc +++ b/test/common/upstream/conn_pool_map_impl_test.cc @@ -16,7 +16,6 @@ using testing::AtLeast; using testing::Invoke; -using testing::InvokeArgument; using testing::NiceMock; using testing::Return; using testing::SaveArg; diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index b71150712939..688c162807ff 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -22,9 +22,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::AtLeast; -using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Upstream { diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index 4753cf9da7be..5087f44d37de 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -35,11 +35,9 @@ using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; using testing::NiceMock; -using testing::Ref; using testing::Return; using testing::ReturnRef; using testing::SaveArg; -using testing::WithArg; namespace Envoy { namespace Upstream { diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index 3c6546f472f7..660d76cc8f93 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -29,10 +29,8 @@ #include "gtest/gtest.h" using testing::_; -using testing::Invoke; using testing::NiceMock; using testing::Return; -using testing::ReturnRef; using testing::SaveArg; namespace Envoy { diff --git a/test/common/upstream/ring_hash_lb_test.cc b/test/common/upstream/ring_hash_lb_test.cc index 243f297bf178..c2cb3c1c8079 100644 --- a/test/common/upstream/ring_hash_lb_test.cc +++ b/test/common/upstream/ring_hash_lb_test.cc @@ -17,7 +17,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::NiceMock; using testing::Return; diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index c18dc9a4f8f0..ed96bf70f6c3 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -22,8 +22,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; -using testing::EndsWith; using testing::NiceMock; using testing::Return; using testing::ReturnRef; diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index c03093e01b33..9934f68f7ceb 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -39,8 +39,6 @@ using testing::_; using testing::ContainerEq; using testing::Invoke; using testing::NiceMock; -using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Upstream { diff --git a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc index 07875b008ddf..77c9f42554b3 100644 --- a/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/grpc_access_log_impl_test.cc @@ -16,7 +16,6 @@ using testing::_; using testing::InSequence; using testing::Invoke; using testing::NiceMock; -using testing::Return; namespace Envoy { namespace Extensions { diff --git a/test/extensions/access_loggers/grpc/http_config_test.cc b/test/extensions/access_loggers/grpc/http_config_test.cc index 7cd5bbc00f97..2a23658a91b7 100644 --- a/test/extensions/access_loggers/grpc/http_config_test.cc +++ b/test/extensions/access_loggers/grpc/http_config_test.cc @@ -12,7 +12,6 @@ using testing::_; using testing::Invoke; -using testing::Return; namespace Envoy { namespace Extensions { diff --git a/test/extensions/clusters/redis/mocks.cc b/test/extensions/clusters/redis/mocks.cc index b66ddd7c0505..f0ae690f29dc 100644 --- a/test/extensions/clusters/redis/mocks.cc +++ b/test/extensions/clusters/redis/mocks.cc @@ -1,10 +1,7 @@ #include "test/extensions/clusters/redis/mocks.h" using testing::_; -using testing::Invoke; using testing::Return; -using testing::ReturnPointee; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 39f23ef1be78..c8b941760555 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -21,15 +21,10 @@ using testing::_; using testing::ContainerEq; -using testing::DoAll; using testing::Eq; -using testing::InvokeWithoutArgs; using testing::NiceMock; using testing::Ref; using testing::Return; -using testing::ReturnRef; -using testing::SaveArg; -using testing::WithArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/common/expr/context_test.cc b/test/extensions/filters/common/expr/context_test.cc index 0e79abe362ec..396a8557533c 100644 --- a/test/extensions/filters/common/expr/context_test.cc +++ b/test/extensions/filters/common/expr/context_test.cc @@ -10,8 +10,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; -using testing::Const; using testing::Return; using testing::ReturnRef; diff --git a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc index 62e89216af2b..e8d40a655a3f 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc @@ -17,12 +17,9 @@ using testing::_; using testing::AllOf; using testing::Invoke; -using testing::Ref; using testing::Return; -using testing::ReturnPointee; using testing::ReturnRef; using testing::WhenDynamicCastTo; -using testing::WithArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/common/original_src/original_src_socket_option_test.cc b/test/extensions/filters/common/original_src/original_src_socket_option_test.cc index 9fc29b0aa526..c2e108cb666a 100644 --- a/test/extensions/filters/common/original_src/original_src_socket_option_test.cc +++ b/test/extensions/filters/common/original_src/original_src_socket_option_test.cc @@ -12,7 +12,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Eq; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc b/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc index 97a5863c0bb6..37024b6790c6 100644 --- a/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc +++ b/test/extensions/filters/common/ratelimit/ratelimit_impl_test.cc @@ -20,12 +20,10 @@ #include "gtest/gtest.h" using testing::_; -using testing::AtLeast; using testing::Eq; using testing::Invoke; using testing::Ref; using testing::Return; -using testing::WithArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/common/rbac/engine_impl_test.cc b/test/extensions/filters/common/rbac/engine_impl_test.cc index 6d346dca62cf..7324099d2071 100644 --- a/test/extensions/filters/common/rbac/engine_impl_test.cc +++ b/test/extensions/filters/common/rbac/engine_impl_test.cc @@ -10,9 +10,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::Const; -using testing::Return; using testing::ReturnRef; namespace Envoy { diff --git a/test/extensions/filters/common/rbac/matchers_test.cc b/test/extensions/filters/common/rbac/matchers_test.cc index 43012da47eda..649e0776a136 100644 --- a/test/extensions/filters/common/rbac/matchers_test.cc +++ b/test/extensions/filters/common/rbac/matchers_test.cc @@ -8,7 +8,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::Const; using testing::Return; using testing::ReturnRef; diff --git a/test/extensions/filters/http/buffer/buffer_filter_test.cc b/test/extensions/filters/http/buffer/buffer_filter_test.cc index 7f8a46899cd6..393ee4723518 100644 --- a/test/extensions/filters/http/buffer/buffer_filter_test.cc +++ b/test/extensions/filters/http/buffer/buffer_filter_test.cc @@ -18,12 +18,9 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; -using testing::DoAll; using testing::InSequence; using testing::NiceMock; using testing::Return; -using testing::SaveArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/common/jwks_fetcher_test.cc b/test/extensions/filters/http/common/jwks_fetcher_test.cc index 6932eee59fcf..feaea0e29bab 100644 --- a/test/extensions/filters/http/common/jwks_fetcher_test.cc +++ b/test/extensions/filters/http/common/jwks_fetcher_test.cc @@ -11,7 +11,6 @@ #include "test/test_common/utility.h" using ::envoy::api::v2::core::HttpUri; -using ::google::jwt_verify::Status; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/cors/cors_filter_test.cc b/test/extensions/filters/http/cors/cors_filter_test.cc index 863bef99a8ef..503f624dd741 100644 --- a/test/extensions/filters/http/cors/cors_filter_test.cc +++ b/test/extensions/filters/http/cors/cors_filter_test.cc @@ -12,13 +12,8 @@ #include "gtest/gtest.h" using testing::_; -using testing::DoAll; -using testing::InSequence; -using testing::Invoke; using testing::NiceMock; using testing::Return; -using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/csrf/csrf_filter_test.cc b/test/extensions/filters/http/csrf/csrf_filter_test.cc index 99835d935c1a..22ab5972488f 100644 --- a/test/extensions/filters/http/csrf/csrf_filter_test.cc +++ b/test/extensions/filters/http/csrf/csrf_filter_test.cc @@ -11,13 +11,8 @@ #include "gtest/gtest.h" using testing::_; -using testing::DoAll; -using testing::InSequence; -using testing::Invoke; using testing::NiceMock; using testing::Return; -using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/dynamo/dynamo_filter_test.cc b/test/extensions/filters/http/dynamo/dynamo_filter_test.cc index eb93be0c670c..ce8638ccbe14 100644 --- a/test/extensions/filters/http/dynamo/dynamo_filter_test.cc +++ b/test/extensions/filters/http/dynamo/dynamo_filter_test.cc @@ -19,7 +19,6 @@ using testing::_; using testing::NiceMock; using testing::Property; using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/dynamo/dynamo_stats_test.cc b/test/extensions/filters/http/dynamo/dynamo_stats_test.cc index 0458c5dc3879..d479f4a1092f 100644 --- a/test/extensions/filters/http/dynamo/dynamo_stats_test.cc +++ b/test/extensions/filters/http/dynamo/dynamo_stats_test.cc @@ -7,10 +7,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; -using testing::NiceMock; -using testing::Return; - namespace Envoy { namespace Extensions { namespace HttpFilters { diff --git a/test/extensions/filters/http/fault/config_test.cc b/test/extensions/filters/http/fault/config_test.cc index 0417b007c050..af4e626a58df 100644 --- a/test/extensions/filters/http/fault/config_test.cc +++ b/test/extensions/filters/http/fault/config_test.cc @@ -8,7 +8,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Invoke; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/fault/fault_filter_test.cc b/test/extensions/filters/http/fault/fault_filter_test.cc index f4e0bc1fcb58..24442c47747b 100644 --- a/test/extensions/filters/http/fault/fault_filter_test.cc +++ b/test/extensions/filters/http/fault/fault_filter_test.cc @@ -26,13 +26,10 @@ using testing::_; using testing::AnyNumber; -using testing::DoAll; -using testing::Invoke; using testing::Matcher; using testing::NiceMock; using testing::Return; using testing::ReturnRef; -using testing::WithArgs; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc index aafdb5bdeaa2..644157a15f2f 100644 --- a/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc +++ b/test/extensions/filters/http/grpc_http1_bridge/http1_bridge_filter_test.cc @@ -14,11 +14,9 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::NiceMock; using testing::Return; using testing::ReturnPointee; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc index 137970bb6bb6..503c2c1d4400 100644 --- a/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc +++ b/test/extensions/filters/http/grpc_http1_reverse_bridge/reverse_bridge_test.cc @@ -20,7 +20,6 @@ using Envoy::Http::HeaderValueOf; using testing::_; -using testing::Return; using testing::ReturnRef; namespace Envoy { diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index 14f491375a8a..76fed14e164f 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -11,7 +11,6 @@ #include "absl/strings/match.h" #include "gtest/gtest.h" -using Envoy::Protobuf::Message; using Envoy::Protobuf::TextFormat; using Envoy::Protobuf::util::MessageDifferencer; using Envoy::ProtobufUtil::Status; diff --git a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc index 673a13e0a80c..35d3bc2cbe60 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc @@ -23,16 +23,12 @@ using testing::_; using testing::Invoke; using testing::NiceMock; -using testing::Return; -using testing::ReturnPointee; -using testing::ReturnRef; using Envoy::Protobuf::MethodDescriptor; using Envoy::Protobuf::FileDescriptorProto; using Envoy::Protobuf::FileDescriptorSet; using Envoy::Protobuf::util::MessageDifferencer; -using Envoy::ProtobufUtil::Status; using Envoy::ProtobufUtil::error::Code; using google::api::HttpRule; using google::grpc::transcoding::Transcoder; diff --git a/test/extensions/filters/http/health_check/health_check_test.cc b/test/extensions/filters/http/health_check/health_check_test.cc index a6cfb78104ea..5c4472e669f5 100644 --- a/test/extensions/filters/http/health_check/health_check_test.cc +++ b/test/extensions/filters/http/health_check/health_check_test.cc @@ -16,13 +16,10 @@ #include "gtest/gtest.h" using testing::_; -using testing::DoAll; using testing::Eq; using testing::Invoke; using testing::NiceMock; using testing::Return; -using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc index 774f4e54c216..c5901986db2e 100644 --- a/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc +++ b/test/extensions/filters/http/ip_tagging/ip_tagging_filter_test.cc @@ -16,7 +16,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::Return; using testing::ReturnRef; diff --git a/test/extensions/filters/http/lua/lua_filter_test.cc b/test/extensions/filters/http/lua/lua_filter_test.cc index ea09b1c38e41..95883ce51629 100644 --- a/test/extensions/filters/http/lua/lua_filter_test.cc +++ b/test/extensions/filters/http/lua/lua_filter_test.cc @@ -22,7 +22,6 @@ using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::Return; -using testing::ReturnPointee; using testing::ReturnRef; using testing::StrEq; diff --git a/test/extensions/filters/http/lua/wrappers_test.cc b/test/extensions/filters/http/lua/wrappers_test.cc index a1e151593132..3b934e4981fe 100644 --- a/test/extensions/filters/http/lua/wrappers_test.cc +++ b/test/extensions/filters/http/lua/wrappers_test.cc @@ -8,7 +8,6 @@ #include "test/test_common/utility.h" using testing::InSequence; -using testing::Return; using testing::ReturnPointee; namespace Envoy { diff --git a/test/extensions/filters/http/original_src/original_src_test.cc b/test/extensions/filters/http/original_src/original_src_test.cc index c166824bfab3..9eba38759156 100644 --- a/test/extensions/filters/http/original_src/original_src_test.cc +++ b/test/extensions/filters/http/original_src/original_src_test.cc @@ -16,7 +16,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Exactly; using testing::SaveArg; using testing::StrictMock; diff --git a/test/extensions/filters/http/ratelimit/config_test.cc b/test/extensions/filters/http/ratelimit/config_test.cc index 7c68bd808eff..5b239b974097 100644 --- a/test/extensions/filters/http/ratelimit/config_test.cc +++ b/test/extensions/filters/http/ratelimit/config_test.cc @@ -10,7 +10,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/http/ratelimit/ratelimit_test.cc b/test/extensions/filters/http/ratelimit/ratelimit_test.cc index 4e030d64b8e7..95a95a16a2a2 100644 --- a/test/extensions/filters/http/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/http/ratelimit/ratelimit_test.cc @@ -27,7 +27,6 @@ using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Return; -using testing::ReturnRef; using testing::SetArgReferee; using testing::WithArgs; diff --git a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc index afca8e66e9f7..718c03de16cb 100644 --- a/test/extensions/filters/listener/http_inspector/http_inspector_test.cc +++ b/test/extensions/filters/listener/http_inspector/http_inspector_test.cc @@ -11,8 +11,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::AtLeast; -using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; diff --git a/test/extensions/filters/listener/original_src/original_src_test.cc b/test/extensions/filters/listener/original_src/original_src_test.cc index ffed37e4855c..9684e6e59b92 100644 --- a/test/extensions/filters/listener/original_src/original_src_test.cc +++ b/test/extensions/filters/listener/original_src/original_src_test.cc @@ -14,7 +14,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Exactly; using testing::SaveArg; namespace Envoy { diff --git a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc index 1f4d8a750c85..683bdfc1c5fd 100644 --- a/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc +++ b/test/extensions/filters/listener/proxy_protocol/proxy_protocol_test.cc @@ -31,7 +31,6 @@ using testing::_; using testing::AnyNumber; using testing::AtLeast; -using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Return; diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_benchmark.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_benchmark.cc index 12dbdd442c6a..5f8c1431ce6f 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_benchmark.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_benchmark.cc @@ -15,14 +15,7 @@ #include "gtest/gtest.h" #include "openssl/ssl.h" -using testing::_; -using testing::AtLeast; -using testing::Invoke; using testing::NiceMock; -using testing::Return; -using testing::ReturnNew; -using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc index d9793c98a8e6..c0416e52823e 100644 --- a/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc +++ b/test/extensions/filters/listener/tls_inspector/tls_inspector_test.cc @@ -12,13 +12,11 @@ #include "openssl/ssl.h" using testing::_; -using testing::AtLeast; using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; using testing::NiceMock; -using testing::Return; using testing::ReturnNew; using testing::ReturnRef; using testing::SaveArg; diff --git a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc index 5e50b3769158..48a5d727b714 100644 --- a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc +++ b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc @@ -25,9 +25,7 @@ using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::Return; -using testing::ReturnNew; using testing::ReturnRef; -using testing::WithArg; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index a6646c86741a..ca944879f516 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -15,14 +15,10 @@ #include "gtest/gtest.h" using testing::_; -using testing::DoAll; -using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::Ref; using testing::Return; -using testing::ReturnNew; -using testing::ReturnRef; using testing::SaveArg; namespace Envoy { diff --git a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc index 008fb1b9fc5a..f37ced3f9d0b 100644 --- a/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/conn_manager_test.cc @@ -20,13 +20,10 @@ #include "gtest/gtest.h" using testing::_; -using testing::AnyNumber; using testing::InSequence; using testing::Invoke; using testing::NiceMock; -using testing::Ref; using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc index bbada6519a39..c904a3f40113 100644 --- a/test/extensions/filters/network/dubbo_proxy/decoder_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/decoder_test.cc @@ -12,9 +12,6 @@ using testing::_; using testing::Return; using testing::ReturnRef; -using testing::TestParamInfo; -using testing::TestWithParam; -using testing::Values; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl_test.cc b/test/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl_test.cc index 3a74b52b3f89..39272bf9ddfa 100644 --- a/test/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/dubbo_hessian2_serializer_impl_test.cc @@ -10,8 +10,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::NotNull; - namespace Envoy { namespace Extensions { namespace NetworkFilters { diff --git a/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc b/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc index 38e3d97ad935..6aa87a5f993b 100644 --- a/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/dubbo_protocol_impl_test.cc @@ -14,8 +14,6 @@ namespace Extensions { namespace NetworkFilters { namespace DubboProxy { -using testing::StrictMock; - TEST(DubboProtocolImplTest, NotEnoughData) { Buffer::OwnedImpl buffer; DubboProtocolImpl dubbo_protocol; diff --git a/test/extensions/filters/network/dubbo_proxy/mocks.h b/test/extensions/filters/network/dubbo_proxy/mocks.h index 4e55ca6b950a..dbe59849c523 100644 --- a/test/extensions/filters/network/dubbo_proxy/mocks.h +++ b/test/extensions/filters/network/dubbo_proxy/mocks.h @@ -16,8 +16,6 @@ #include "gmock/gmock.h" -using testing::_; - namespace Envoy { namespace Extensions { namespace NetworkFilters { diff --git a/test/extensions/filters/network/dubbo_proxy/router_test.cc b/test/extensions/filters/network/dubbo_proxy/router_test.cc index 8caf2cfd4097..77f8df258809 100644 --- a/test/extensions/filters/network/dubbo_proxy/router_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/router_test.cc @@ -16,13 +16,11 @@ using testing::_; using testing::ContainsRegex; using testing::Eq; -using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Ref; using testing::Return; using testing::ReturnRef; -using testing::Values; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/ext_authz/ext_authz_test.cc b/test/extensions/filters/network/ext_authz/ext_authz_test.cc index a9d86928e3ed..0fe0ef85315a 100644 --- a/test/extensions/filters/network/ext_authz/ext_authz_test.cc +++ b/test/extensions/filters/network/ext_authz/ext_authz_test.cc @@ -26,7 +26,6 @@ using testing::_; using testing::InSequence; using testing::Invoke; using testing::NiceMock; -using testing::Return; using testing::ReturnRef; using testing::WithArgs; diff --git a/test/extensions/filters/network/kafka/request_codec_unit_test.cc b/test/extensions/filters/network/kafka/request_codec_unit_test.cc index 9c7ff5218738..f10af9f953d3 100644 --- a/test/extensions/filters/network/kafka/request_codec_unit_test.cc +++ b/test/extensions/filters/network/kafka/request_codec_unit_test.cc @@ -8,9 +8,7 @@ using testing::_; using testing::AnyNumber; -using testing::Eq; using testing::Invoke; -using testing::ResultOf; using testing::Return; namespace Envoy { diff --git a/test/extensions/filters/network/kafka/response_codec_unit_test.cc b/test/extensions/filters/network/kafka/response_codec_unit_test.cc index 6d20d6ef6dc5..274da6525f82 100644 --- a/test/extensions/filters/network/kafka/response_codec_unit_test.cc +++ b/test/extensions/filters/network/kafka/response_codec_unit_test.cc @@ -8,9 +8,7 @@ using testing::_; using testing::AnyNumber; -using testing::Eq; using testing::Invoke; -using testing::ResultOf; using testing::Return; namespace Envoy { diff --git a/test/extensions/filters/network/mongo_proxy/proxy_test.cc b/test/extensions/filters/network/mongo_proxy/proxy_test.cc index a70c1a6f279c..a0e080dca2be 100644 --- a/test/extensions/filters/network/mongo_proxy/proxy_test.cc +++ b/test/extensions/filters/network/mongo_proxy/proxy_test.cc @@ -23,7 +23,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::AnyNumber; using testing::AtLeast; using testing::Invoke; using testing::Matcher; diff --git a/test/extensions/filters/network/ratelimit/config_test.cc b/test/extensions/filters/network/ratelimit/config_test.cc index 387a03b216d4..24aeb7e74ee4 100644 --- a/test/extensions/filters/network/ratelimit/config_test.cc +++ b/test/extensions/filters/network/ratelimit/config_test.cc @@ -10,7 +10,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/redis_proxy/mocks.cc b/test/extensions/filters/network/redis_proxy/mocks.cc index f06a6bd9b0b9..9c1457258377 100644 --- a/test/extensions/filters/network/redis_proxy/mocks.cc +++ b/test/extensions/filters/network/redis_proxy/mocks.cc @@ -1,7 +1,5 @@ #include "mocks.h" -using testing::_; -using testing::Invoke; using testing::Return; using testing::ReturnRef; diff --git a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc index c8c2ecc5e74c..228135bef51f 100644 --- a/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc +++ b/test/extensions/filters/network/redis_proxy/redis_proxy_integration_test.cc @@ -7,7 +7,6 @@ #include "gtest/gtest.h" -using testing::Matcher; using testing::Return; namespace RedisCmdSplitter = Envoy::Extensions::NetworkFilters::RedisProxy::CommandSplitter; diff --git a/test/extensions/filters/network/redis_proxy/router_impl_test.cc b/test/extensions/filters/network/redis_proxy/router_impl_test.cc index 9ff4bc5ade82..28f66bbd871b 100644 --- a/test/extensions/filters/network/redis_proxy/router_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/router_impl_test.cc @@ -8,14 +8,10 @@ #include "test/mocks/runtime/mocks.h" #include "test/test_common/utility.h" -using testing::_; using testing::Eq; -using testing::InSequence; using testing::Matcher; using testing::NiceMock; -using testing::Ref; using testing::Return; -using testing::StrEq; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/sni_cluster/sni_cluster_test.cc b/test/extensions/filters/network/sni_cluster/sni_cluster_test.cc index 9cc39bd6d2e6..2625e70db449 100644 --- a/test/extensions/filters/network/sni_cluster/sni_cluster_test.cc +++ b/test/extensions/filters/network/sni_cluster/sni_cluster_test.cc @@ -11,7 +11,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::Matcher; using testing::NiceMock; using testing::Return; using testing::ReturnRef; diff --git a/test/extensions/filters/network/thrift_proxy/auto_protocol_impl_test.cc b/test/extensions/filters/network/thrift_proxy/auto_protocol_impl_test.cc index 63f33d6f103d..cec16c3a5948 100644 --- a/test/extensions/filters/network/thrift_proxy/auto_protocol_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/auto_protocol_impl_test.cc @@ -18,7 +18,6 @@ using testing::NiceMock; using testing::Ref; using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc index e0ee0f1d5907..45e763cb009f 100644 --- a/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc +++ b/test/extensions/filters/network/thrift_proxy/conn_manager_test.cc @@ -28,7 +28,6 @@ using testing::Invoke; using testing::NiceMock; using testing::Ref; using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/thrift_proxy/decoder_test.cc b/test/extensions/filters/network/thrift_proxy/decoder_test.cc index 74d706a5416e..1dc42a1a116b 100644 --- a/test/extensions/filters/network/thrift_proxy/decoder_test.cc +++ b/test/extensions/filters/network/thrift_proxy/decoder_test.cc @@ -16,7 +16,6 @@ using testing::_; using testing::AnyNumber; using testing::Combine; using testing::DoAll; -using testing::Expectation; using testing::ExpectationSet; using testing::InSequence; using testing::Invoke; diff --git a/test/extensions/filters/network/thrift_proxy/filters/ratelimit/config_test.cc b/test/extensions/filters/network/thrift_proxy/filters/ratelimit/config_test.cc index e801e8784202..63a5f256a913 100644 --- a/test/extensions/filters/network/thrift_proxy/filters/ratelimit/config_test.cc +++ b/test/extensions/filters/network/thrift_proxy/filters/ratelimit/config_test.cc @@ -9,7 +9,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit_test.cc b/test/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit_test.cc index 01877c365c4e..83f0b7edaf9c 100644 --- a/test/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit_test.cc +++ b/test/extensions/filters/network/thrift_proxy/filters/ratelimit/ratelimit_test.cc @@ -28,7 +28,6 @@ using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Return; -using testing::ReturnRef; using testing::SetArgReferee; using testing::WithArgs; diff --git a/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc b/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc index 791a2f9f4d59..4ed3f51fda97 100644 --- a/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/header_transport_impl_test.cc @@ -13,7 +13,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::NiceMock; using testing::Return; namespace Envoy { diff --git a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc index 75b096d15bf0..1966560ca38c 100644 --- a/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc +++ b/test/extensions/filters/network/thrift_proxy/route_matcher_test.cc @@ -9,8 +9,6 @@ #include "gtest/gtest.h" -using testing::_; - namespace Envoy { namespace Extensions { namespace NetworkFilters { diff --git a/test/extensions/filters/network/thrift_proxy/router_test.cc b/test/extensions/filters/network/thrift_proxy/router_test.cc index ad4686746b4b..6e6743016fe2 100644 --- a/test/extensions/filters/network/thrift_proxy/router_test.cc +++ b/test/extensions/filters/network/thrift_proxy/router_test.cc @@ -24,7 +24,6 @@ using testing::_; using testing::ContainsRegex; using testing::Eq; -using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Ref; diff --git a/test/extensions/filters/network/thrift_proxy/thrift_object_impl_test.cc b/test/extensions/filters/network/thrift_proxy/thrift_object_impl_test.cc index 7c4bad9614e1..a79d30505644 100644 --- a/test/extensions/filters/network/thrift_proxy/thrift_object_impl_test.cc +++ b/test/extensions/filters/network/thrift_proxy/thrift_object_impl_test.cc @@ -10,14 +10,12 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::AnyNumber; using testing::Expectation; using testing::ExpectationSet; using testing::InSequence; using testing::NiceMock; using testing::Ref; using testing::Return; -using testing::ReturnRef; using testing::Values; namespace Envoy { diff --git a/test/extensions/filters/network/thrift_proxy/utility.h b/test/extensions/filters/network/thrift_proxy/utility.h index a3ab975a645d..bd91e3b36b95 100644 --- a/test/extensions/filters/network/thrift_proxy/utility.h +++ b/test/extensions/filters/network/thrift_proxy/utility.h @@ -23,8 +23,8 @@ namespace NetworkFilters { namespace ThriftProxy { namespace { -using Envoy::Buffer::addRepeated; -using Envoy::Buffer::addSeq; +using Envoy::Buffer::addRepeated; // NOLINT(misc-unused-using-decls) +using Envoy::Buffer::addSeq; // NOLINT(misc-unused-using-decls) inline std::string fieldTypeToString(const FieldType& field_type) { switch (field_type) { diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index 2d721cacf56d..aa18cd8d8e63 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -16,8 +16,6 @@ using testing::InSequence; using testing::NiceMock; using testing::Ref; using testing::Return; -using testing::ReturnRef; -using testing::SaveArg; using testing::WithArg; namespace Envoy { diff --git a/test/extensions/stats_sinks/dog_statsd/config_test.cc b/test/extensions/stats_sinks/dog_statsd/config_test.cc index 3299f42aa8c2..993fc0da13c8 100644 --- a/test/extensions/stats_sinks/dog_statsd/config_test.cc +++ b/test/extensions/stats_sinks/dog_statsd/config_test.cc @@ -16,10 +16,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::NiceMock; -using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/stats_sinks/hystrix/config_test.cc b/test/extensions/stats_sinks/hystrix/config_test.cc index 510e4ed2a8c6..360b98275273 100644 --- a/test/extensions/stats_sinks/hystrix/config_test.cc +++ b/test/extensions/stats_sinks/hystrix/config_test.cc @@ -15,10 +15,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::NiceMock; -using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/stats_sinks/hystrix/hystrix_test.cc b/test/extensions/stats_sinks/hystrix/hystrix_test.cc index 677f37de8351..64aec79b5735 100644 --- a/test/extensions/stats_sinks/hystrix/hystrix_test.cc +++ b/test/extensions/stats_sinks/hystrix/hystrix_test.cc @@ -21,7 +21,6 @@ using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Return; -using testing::ReturnPointee; using testing::ReturnRef; namespace Envoy { diff --git a/test/extensions/stats_sinks/metrics_service/grpc_metrics_service_impl_test.cc b/test/extensions/stats_sinks/metrics_service/grpc_metrics_service_impl_test.cc index 4ed180c0a05f..9988ef4d8993 100644 --- a/test/extensions/stats_sinks/metrics_service/grpc_metrics_service_impl_test.cc +++ b/test/extensions/stats_sinks/metrics_service/grpc_metrics_service_impl_test.cc @@ -12,7 +12,6 @@ using testing::_; using testing::InSequence; using testing::Invoke; using testing::NiceMock; -using testing::Return; namespace Envoy { namespace Extensions { diff --git a/test/extensions/stats_sinks/statsd/config_test.cc b/test/extensions/stats_sinks/statsd/config_test.cc index b71d29c94a6f..3ea7353f77e9 100644 --- a/test/extensions/stats_sinks/statsd/config_test.cc +++ b/test/extensions/stats_sinks/statsd/config_test.cc @@ -17,10 +17,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::NiceMock; -using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Extensions { diff --git a/test/extensions/tracers/datadog/config_test.cc b/test/extensions/tracers/datadog/config_test.cc index 363be39bba76..9ffe6a003561 100644 --- a/test/extensions/tracers/datadog/config_test.cc +++ b/test/extensions/tracers/datadog/config_test.cc @@ -5,7 +5,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::Eq; using testing::NiceMock; using testing::Return; diff --git a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc index cc80f5672c0e..770d43655b29 100644 --- a/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc +++ b/test/extensions/tracers/datadog/datadog_tracer_impl_test.cc @@ -27,7 +27,6 @@ #include "gtest/gtest.h" using testing::_; -using testing::AtLeast; using testing::Eq; using testing::Invoke; using testing::NiceMock; diff --git a/test/extensions/tracers/dynamic_ot/config_test.cc b/test/extensions/tracers/dynamic_ot/config_test.cc index 825f8b0aae70..418609f0be21 100644 --- a/test/extensions/tracers/dynamic_ot/config_test.cc +++ b/test/extensions/tracers/dynamic_ot/config_test.cc @@ -7,7 +7,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::Eq; using testing::NiceMock; using testing::Return; diff --git a/test/extensions/tracers/lightstep/config_test.cc b/test/extensions/tracers/lightstep/config_test.cc index c0a681183cd1..7cdd4595ef3e 100644 --- a/test/extensions/tracers/lightstep/config_test.cc +++ b/test/extensions/tracers/lightstep/config_test.cc @@ -5,7 +5,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; using testing::Eq; using testing::NiceMock; using testing::Return; diff --git a/test/extensions/tracers/opencensus/tracer_test.cc b/test/extensions/tracers/opencensus/tracer_test.cc index 241f1d0a9a89..aef4d925b129 100644 --- a/test/extensions/tracers/opencensus/tracer_test.cc +++ b/test/extensions/tracers/opencensus/tracer_test.cc @@ -27,7 +27,6 @@ #include "opencensus/trace/span_id.h" using testing::NiceMock; -using testing::Return; namespace opencensus { namespace trace { diff --git a/test/extensions/transport_sockets/alts/config_test.cc b/test/extensions/transport_sockets/alts/config_test.cc index 68c602ea50fd..a17405dc7e08 100644 --- a/test/extensions/transport_sockets/alts/config_test.cc +++ b/test/extensions/transport_sockets/alts/config_test.cc @@ -11,10 +11,7 @@ #include "gtest/gtest.h" using Envoy::Server::Configuration::MockTransportSocketFactoryContext; -using testing::_; -using testing::Invoke; using testing::ReturnRef; -using testing::StrictMock; namespace Envoy { namespace Extensions { diff --git a/test/extensions/transport_sockets/alts/tsi_frame_protector_test.cc b/test/extensions/transport_sockets/alts/tsi_frame_protector_test.cc index 09d4191c37cd..c0db008cb216 100644 --- a/test/extensions/transport_sockets/alts/tsi_frame_protector_test.cc +++ b/test/extensions/transport_sockets/alts/tsi_frame_protector_test.cc @@ -12,11 +12,6 @@ namespace TransportSockets { namespace Alts { namespace { -using testing::_; -using testing::InSequence; -using testing::Invoke; -using testing::NiceMock; -using testing::SaveArg; using namespace std::string_literals; /** diff --git a/test/extensions/transport_sockets/alts/tsi_socket_test.cc b/test/extensions/transport_sockets/alts/tsi_socket_test.cc index f95db040fbb2..3fd39b520020 100644 --- a/test/extensions/transport_sockets/alts/tsi_socket_test.cc +++ b/test/extensions/transport_sockets/alts/tsi_socket_test.cc @@ -17,7 +17,6 @@ namespace { using testing::NiceMock; using testing::Return; using testing::ReturnRef; -using testing::StrictMock; class TsiSocketTest : public testing::Test { protected: diff --git a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc index 785e9956ee9d..ae3205fa75c3 100644 --- a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc +++ b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.cc @@ -22,8 +22,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::Return; - namespace Envoy { namespace Ssl { diff --git a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.h b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.h index 18b276d6a396..133e73bd433e 100644 --- a/test/extensions/transport_sockets/tls/integration/ssl_integration_test.h +++ b/test/extensions/transport_sockets/tls/integration/ssl_integration_test.h @@ -11,8 +11,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::NiceMock; - namespace Envoy { namespace Ssl { diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index 48a8af72bd5e..4e2590e4b41d 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -14,10 +14,7 @@ #include "test/test_common/network_utility.h" #include "test/test_common/utility.h" -using testing::AssertionFailure; using testing::AssertionResult; -using testing::AssertionSuccess; -using testing::IsSubstring; namespace Envoy { diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 92357a7a24bb..e2cb9250c0e6 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -22,10 +22,7 @@ #include "gtest/gtest.h" -using testing::AssertionFailure; using testing::AssertionResult; -using testing::AssertionSuccess; -using testing::IsSubstring; namespace Envoy { diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index a8efa18cfae1..1ea5f5fe67e8 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -19,10 +19,7 @@ #include "absl/synchronization/notification.h" #include "gtest/gtest.h" -using testing::AssertionFailure; using testing::AssertionResult; -using testing::AssertionSuccess; -using testing::IsSubstring; namespace Envoy { namespace { diff --git a/test/integration/http2_upstream_integration_test.cc b/test/integration/http2_upstream_integration_test.cc index 15303a0a6f94..5854164ed301 100644 --- a/test/integration/http2_upstream_integration_test.cc +++ b/test/integration/http2_upstream_integration_test.cc @@ -10,8 +10,6 @@ #include "gtest/gtest.h" -using testing::HasSubstr; - namespace Envoy { INSTANTIATE_TEST_SUITE_P(IpVersions, Http2UpstreamIntegrationTest, diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 62090c09d7c9..be622d93f3b8 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -34,12 +34,6 @@ #include "gtest/gtest.h" -using testing::_; -using testing::AnyNumber; -using testing::HasSubstr; -using testing::Invoke; -using testing::Not; - namespace Envoy { namespace { diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index 7ad5e339703b..74a487a25e82 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -23,7 +23,6 @@ using Envoy::Http::HeaderValueOf; using Envoy::Http::HttpStatusIs; using testing::EndsWith; using testing::HasSubstr; -using testing::MatchesRegex; using testing::Not; namespace Envoy { diff --git a/test/integration/protocol_integration_test.cc b/test/integration/protocol_integration_test.cc index 5d6bd41de406..545e17686395 100644 --- a/test/integration/protocol_integration_test.cc +++ b/test/integration/protocol_integration_test.cc @@ -33,11 +33,7 @@ #include "gtest/gtest.h" -using testing::_; -using testing::AnyNumber; using testing::HasSubstr; -using testing::Invoke; -using testing::Not; namespace Envoy { diff --git a/test/integration/sds_dynamic_integration_test.cc b/test/integration/sds_dynamic_integration_test.cc index d260bdf6f0d0..dc5867b0faed 100644 --- a/test/integration/sds_dynamic_integration_test.cc +++ b/test/integration/sds_dynamic_integration_test.cc @@ -28,9 +28,6 @@ #include "integration.h" #include "utility.h" -using testing::NiceMock; -using testing::Return; - namespace Envoy { namespace Ssl { diff --git a/test/integration/sds_static_integration_test.cc b/test/integration/sds_static_integration_test.cc index 453942faf0c0..cbbf3085e987 100644 --- a/test/integration/sds_static_integration_test.cc +++ b/test/integration/sds_static_integration_test.cc @@ -26,9 +26,6 @@ #include "integration.h" #include "utility.h" -using testing::NiceMock; -using testing::Return; - namespace Envoy { namespace Ssl { diff --git a/test/integration/vhds_integration_test.cc b/test/integration/vhds_integration_test.cc index 83c018b2badf..1276d5e8171d 100644 --- a/test/integration/vhds_integration_test.cc +++ b/test/integration/vhds_integration_test.cc @@ -19,10 +19,7 @@ #include "absl/synchronization/notification.h" #include "gtest/gtest.h" -using testing::AssertionFailure; using testing::AssertionResult; -using testing::AssertionSuccess; -using testing::IsSubstring; namespace Envoy { namespace { diff --git a/test/integration/websocket_integration_test.cc b/test/integration/websocket_integration_test.cc index 0a650a87312c..9e0b490ef441 100644 --- a/test/integration/websocket_integration_test.cc +++ b/test/integration/websocket_integration_test.cc @@ -15,8 +15,6 @@ #include "absl/strings/str_cat.h" #include "gtest/gtest.h" -using testing::MatchesRegex; - namespace Envoy { namespace { diff --git a/test/mocks/access_log/mocks.cc b/test/mocks/access_log/mocks.cc index 33c7b022fd31..9a3ff3f7cfe2 100644 --- a/test/mocks/access_log/mocks.cc +++ b/test/mocks/access_log/mocks.cc @@ -5,7 +5,6 @@ using testing::_; using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace AccessLog { diff --git a/test/mocks/api/mocks.cc b/test/mocks/api/mocks.cc index 0132f89b272e..40204115f2ba 100644 --- a/test/mocks/api/mocks.cc +++ b/test/mocks/api/mocks.cc @@ -8,7 +8,6 @@ using testing::_; using testing::Invoke; -using testing::Return; namespace Envoy { namespace Api { diff --git a/test/mocks/http/mocks.cc b/test/mocks/http/mocks.cc index 0f1161c5725e..2ab8a561b6bf 100644 --- a/test/mocks/http/mocks.cc +++ b/test/mocks/http/mocks.cc @@ -8,13 +8,8 @@ using testing::_; using testing::Invoke; -using testing::MakeMatcher; -using testing::Matcher; -using testing::MatcherInterface; -using testing::MatchResultListener; using testing::Return; using testing::ReturnRef; -using testing::SaveArg; namespace Envoy { namespace Http { diff --git a/test/mocks/local_info/mocks.cc b/test/mocks/local_info/mocks.cc index 7faff3e6b8b8..bcf3251c6906 100644 --- a/test/mocks/local_info/mocks.cc +++ b/test/mocks/local_info/mocks.cc @@ -5,7 +5,6 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::Invoke; using testing::Return; using testing::ReturnRef; diff --git a/test/mocks/server/mocks.cc b/test/mocks/server/mocks.cc index 42bcd95947a8..e97b45e7e8a7 100644 --- a/test/mocks/server/mocks.cc +++ b/test/mocks/server/mocks.cc @@ -10,7 +10,6 @@ using testing::_; using testing::Invoke; using testing::Return; -using testing::ReturnNew; using testing::ReturnPointee; using testing::ReturnRef; using testing::SaveArg; diff --git a/test/mocks/stats/mocks.cc b/test/mocks/stats/mocks.cc index ef5eeebeb488..8196b0a43b03 100644 --- a/test/mocks/stats/mocks.cc +++ b/test/mocks/stats/mocks.cc @@ -10,7 +10,6 @@ using testing::_; using testing::Invoke; using testing::NiceMock; -using testing::Return; using testing::ReturnPointee; using testing::ReturnRef; diff --git a/test/mocks/stream_info/mocks.cc b/test/mocks/stream_info/mocks.cc index 6c842c9ecb3c..b554b0ac30e0 100644 --- a/test/mocks/stream_info/mocks.cc +++ b/test/mocks/stream_info/mocks.cc @@ -8,7 +8,6 @@ using testing::_; using testing::Const; using testing::Invoke; -using testing::Return; using testing::ReturnPointee; using testing::ReturnRef; diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 05785eead37b..7d629b106108 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -12,7 +12,6 @@ using testing::_; using testing::Eq; using testing::Invoke; using testing::Return; -using testing::ReturnPointee; using testing::ReturnRef; using testing::SaveArg; diff --git a/test/server/configuration_impl_test.cc b/test/server/configuration_impl_test.cc index dbc13a45d189..823290aa4fa2 100644 --- a/test/server/configuration_impl_test.cc +++ b/test/server/configuration_impl_test.cc @@ -22,9 +22,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::InSequence; using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Server { diff --git a/test/server/drain_manager_impl_test.cc b/test/server/drain_manager_impl_test.cc index 0a3076c265c2..8197d23356df 100644 --- a/test/server/drain_manager_impl_test.cc +++ b/test/server/drain_manager_impl_test.cc @@ -10,7 +10,6 @@ using testing::_; using testing::InSequence; using testing::Return; -using testing::SaveArg; namespace Envoy { namespace Server { diff --git a/test/server/filter_chain_manager_impl_test.cc b/test/server/filter_chain_manager_impl_test.cc index 3a6f3caee1a8..ee5bc1b15bc6 100644 --- a/test/server/filter_chain_manager_impl_test.cc +++ b/test/server/filter_chain_manager_impl_test.cc @@ -37,13 +37,9 @@ #include "absl/strings/match.h" #include "gtest/gtest.h" -using testing::_; -using testing::InSequence; -using testing::Invoke; using testing::NiceMock; using testing::Return; using testing::ReturnRef; -using testing::Throw; namespace Envoy { namespace Server { diff --git a/test/server/hot_restart_impl_test.cc b/test/server/hot_restart_impl_test.cc index 94ab208fae70..817aae5e3aef 100644 --- a/test/server/hot_restart_impl_test.cc +++ b/test/server/hot_restart_impl_test.cc @@ -20,8 +20,6 @@ using testing::_; using testing::AnyNumber; using testing::Invoke; using testing::InvokeWithoutArgs; -using testing::Return; -using testing::ReturnRef; using testing::WithArg; namespace Envoy { diff --git a/test/server/hot_restarting_parent_test.cc b/test/server/hot_restarting_parent_test.cc index 3b2fc9617937..7b65c3316e61 100644 --- a/test/server/hot_restarting_parent_test.cc +++ b/test/server/hot_restarting_parent_test.cc @@ -6,7 +6,6 @@ #include "gtest/gtest.h" -using testing::_; using testing::Return; using testing::ReturnRef; diff --git a/test/server/http/config_tracker_impl_test.cc b/test/server/http/config_tracker_impl_test.cc index 60d4633f08a7..2fcd777fca55 100644 --- a/test/server/http/config_tracker_impl_test.cc +++ b/test/server/http/config_tracker_impl_test.cc @@ -4,8 +4,6 @@ #include "gmock/gmock.h" -using testing::_; - namespace Envoy { namespace Server { diff --git a/test/server/lds_api_test.cc b/test/server/lds_api_test.cc index 3f54aed53551..1cedee3abb09 100644 --- a/test/server/lds_api_test.cc +++ b/test/server/lds_api_test.cc @@ -18,7 +18,6 @@ using testing::_; using testing::InSequence; using testing::Invoke; using testing::Return; -using testing::ReturnRef; using testing::Throw; namespace Envoy { diff --git a/test/server/options_impl_test.cc b/test/server/options_impl_test.cc index 4372585c4b48..0a365db18544 100644 --- a/test/server/options_impl_test.cc +++ b/test/server/options_impl_test.cc @@ -26,8 +26,6 @@ #include "gtest/gtest.h" #include "spdlog/spdlog.h" -using testing::HasSubstr; - namespace Envoy { namespace { diff --git a/test/server/server_test.cc b/test/server/server_test.cc index ab0d89ebad28..4f0cf9ecbc60 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -26,8 +26,6 @@ using testing::HasSubstr; using testing::InSequence; using testing::Invoke; using testing::InvokeWithoutArgs; -using testing::Property; -using testing::Ref; using testing::Return; using testing::SaveArg; using testing::StrictMock; diff --git a/test/test_common/utility.h b/test/test_common/utility.h index b4f048396ac6..a669f8cb9686 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -31,11 +31,11 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -using testing::_; +using testing::_; // NOLINT(misc-unused-using-decls) using testing::AssertionFailure; using testing::AssertionResult; using testing::AssertionSuccess; -using testing::Invoke; +using testing::Invoke; // NOLINT(misc-unused-using-decls) namespace Envoy { diff --git a/test/test_common/utility_test.cc b/test/test_common/utility_test.cc index 3725ba564e9b..cb911489d179 100644 --- a/test/test_common/utility_test.cc +++ b/test/test_common/utility_test.cc @@ -4,8 +4,6 @@ #include "gtest/gtest.h" -using Envoy::Http::HeaderMap; - namespace Envoy { TEST(HeaderMapEqualIgnoreOrder, ActuallyEqual) { diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index e0ccece9cfc5..f568a88130d2 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -414,6 +414,7 @@ decompressor decrement decrypt decrypting +decls dedup dedupe deduplicate From 5dc6f09f53c781b113d99a5c73aa8fe5e8ddb19e Mon Sep 17 00:00:00 2001 From: troshko111 <931973+troshko111@users.noreply.github.com> Date: Thu, 5 Sep 2019 22:29:14 -0700 Subject: [PATCH 511/542] build: curl with c-ares, nghttp2 and zlib (#8154) Build curl dependency with async DNS resolver c-ares avoiding potential crashes due to longjmp on modern kernels. Add zlib and nghttp2. Use Envoy's version of all of the above libraries. Signed-off-by: Taras Roshko --- bazel/foreign_cc/BUILD | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 1a96306c001b..588673187b08 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -64,24 +64,47 @@ envoy_cmake_external( envoy_cmake_external( name = "curl", + # Options from https://github.com/curl/curl/blob/master/CMakeLists.txt. cache_entries = { "BUILD_CURL_EXE": "off", - "BUILD_SHARED_LIBS": "off", - "CMAKE_BUILD_TYPE": "RelWithDebInfo", - "HTTP_ONLY": "on", "BUILD_TESTING": "off", - "CMAKE_USE_OPENSSL": "off", - "CURL_CA_PATH": "none", + "BUILD_SHARED_LIBS": "off", "CURL_HIDDEN_SYMBOLS": "off", "CMAKE_USE_LIBSSH2": "off", + "CURL_BROTLI": "off", + "CMAKE_USE_GSSAPI": "off", + "HTTP_ONLY": "on", + "CMAKE_BUILD_TYPE": "RelWithDebInfo", "CMAKE_INSTALL_LIBDIR": "lib", - "INCLUDE_DIRECTORIES": "include/curl", + # C-Ares. + "ENABLE_ARES": "on", + "CARES_LIBRARY": "$EXT_BUILD_DEPS/ares", + "CARES_INCLUDE_DIR": "$EXT_BUILD_DEPS/ares/include", + # SSL (via Envoy's SSL dependency) is disabled, curl's CMake uses FindOpenSSL.cmake which fails at what looks like + # version parsing (the libraries are found ok). + "CURL_CA_PATH": "none", + "CMAKE_USE_OPENSSL": "off", + "OPENSSL_ROOT_DIR": "$EXT_BUILD_DEPS", + # NGHTTP2. + "USE_NGHTTP2": "on", + "NGHTTP2_LIBRARY": "$EXT_BUILD_DEPS/nghttp2", + "NGHTTP2_INCLUDE_DIR": "$EXT_BUILD_DEPS/nghttp2/include", + # ZLIB. + "CURL_ZLIB": "on", + "ZLIB_LIBRARY": "$EXT_BUILD_DEPS/zlib", + "ZLIB_INCLUDE_DIR": "$EXT_BUILD_DEPS/zlib/include", }, lib_source = "@com_github_curl//:all", static_libraries = select({ "//bazel:windows_x86_64": ["curl.lib"], "//conditions:default": ["libcurl.a"], }), + deps = [ + ":ares", + ":nghttp2", + ":zlib", + "//external:ssl", + ], ) envoy_cmake_external( From 838eb20c67d594425fb98f203857b292f37b5778 Mon Sep 17 00:00:00 2001 From: Kuat Date: Fri, 6 Sep 2019 13:41:10 -0700 Subject: [PATCH 512/542] log: add upstream TLS info (#7911) Description: add upstream TLS info for logging purposes Refactor SSL connection info to be a shared pointer. Use read-only interface. Cache computed values in the SSL info object (this allows transition to remove the underlying SSL object if necessary). Risk Level: medium due to use of bssl::SSL to back ConnectionInfo Testing: unit Docs Changes: none Release Notes: add upstream TLS info Signed-off-by: Kuat Yessenov --- include/envoy/http/conn_pool.h | 4 +- include/envoy/network/connection.h | 2 +- include/envoy/network/transport_socket.h | 2 +- include/envoy/ssl/connection.h | 2 + include/envoy/stream_info/stream_info.h | 17 +- source/common/http/codec_client.cc | 1 + source/common/http/codec_client.h | 2 + source/common/http/http1/conn_pool.cc | 3 +- source/common/http/http2/conn_pool.cc | 3 +- source/common/network/connection_impl.h | 2 +- source/common/network/raw_buffer_socket.h | 2 +- source/common/router/router.cc | 6 +- source/common/router/router.h | 3 +- source/common/stream_info/stream_info_impl.h | 16 +- source/common/tcp/conn_pool.cc | 1 + source/common/tcp_proxy/tcp_proxy.cc | 1 + source/common/tcp_proxy/tcp_proxy.h | 4 +- .../grpc/grpc_access_log_utils.cc | 3 +- .../common/ext_authz/check_request_utils.cc | 2 +- .../extensions/filters/common/lua/wrappers.h | 2 +- .../filters/common/rbac/matchers.cc | 2 +- .../transport_sockets/alts/tsi_socket.h | 2 +- .../extensions/transport_sockets/tap/tap.cc | 2 +- source/extensions/transport_sockets/tap/tap.h | 2 +- .../transport_sockets/tls/ssl_socket.cc | 163 +++++++++----- .../transport_sockets/tls/ssl_socket.h | 46 ++-- .../access_log_formatter_fuzz_test.cc | 8 +- .../access_log/access_log_formatter_test.cc | 200 +++++++++--------- test/common/http/BUILD | 1 + test/common/http/async_client_impl_test.cc | 59 +++--- test/common/http/codec_client_test.cc | 14 ++ test/common/http/common.h | 4 +- .../http/conn_manager_impl_fuzz_test.cc | 6 +- test/common/http/conn_manager_impl_test.cc | 23 +- test/common/http/conn_manager_utility_test.cc | 104 ++++----- test/common/router/header_formatter_test.cc | 169 +++++++-------- test/common/router/header_parser_fuzz_test.cc | 3 +- test/common/router/router_test.cc | 195 ++++++++++------- .../common/router/router_upstream_log_test.cc | 10 +- test/common/stream_info/test_util.h | 16 +- test/common/tcp_proxy/tcp_proxy_test.cc | 27 ++- .../grpc/http_grpc_access_log_impl_test.cc | 50 ++--- .../filters/common/expr/context_test.cc | 6 +- .../ext_authz/check_request_utils_test.cc | 52 +++-- .../filters/common/lua/wrappers_test.cc | 7 +- .../filters/common/rbac/matchers_test.cc | 38 ++-- .../proxy_filter_integration_test.cc | 12 +- .../filters/http/lua/lua_filter_test.cc | 5 +- .../client_ssl_auth/client_ssl_auth_test.cc | 11 +- .../transport_sockets/tls/ssl_socket_test.cc | 24 ++- .../transport_sockets/tls/utility_test.cc | 9 +- test/fuzz/utility.h | 6 +- test/mocks/network/connection.h | 6 +- test/mocks/network/mocks.h | 2 +- test/mocks/stream_info/mocks.cc | 8 +- test/mocks/stream_info/mocks.h | 9 +- test/server/listener_manager_impl_test.cc | 54 ++--- 57 files changed, 833 insertions(+), 600 deletions(-) diff --git a/include/envoy/http/conn_pool.h b/include/envoy/http/conn_pool.h index cb2d6a8607bd..8ee9270e6dd5 100644 --- a/include/envoy/http/conn_pool.h +++ b/include/envoy/http/conn_pool.h @@ -58,9 +58,11 @@ class Callbacks { * @param encoder supplies the request encoder to use. * @param host supplies the description of the host that will carry the request. For logical * connection pools the description may be different each time this is called. + * @param info supplies the stream info object associated with the upstream connection. */ virtual void onPoolReady(Http::StreamEncoder& encoder, - Upstream::HostDescriptionConstSharedPtr host) PURE; + Upstream::HostDescriptionConstSharedPtr host, + const StreamInfo::StreamInfo& info) PURE; }; /** diff --git a/include/envoy/network/connection.h b/include/envoy/network/connection.h index 0eeaa2855d04..f97391bf5ecd 100644 --- a/include/envoy/network/connection.h +++ b/include/envoy/network/connection.h @@ -215,7 +215,7 @@ class Connection : public Event::DeferredDeletable, public FilterManager { * @return the const SSL connection data if this is an SSL connection, or nullptr if it is not. */ // TODO(snowp): Remove this in favor of StreamInfo::downstreamSslConnection. - virtual const Ssl::ConnectionInfo* ssl() const PURE; + virtual Ssl::ConnectionInfoConstSharedPtr ssl() const PURE; /** * @return requested server name (e.g. SNI in TLS), if any. diff --git a/include/envoy/network/transport_socket.h b/include/envoy/network/transport_socket.h index 1167f46e19d3..a04b711c0102 100644 --- a/include/envoy/network/transport_socket.h +++ b/include/envoy/network/transport_socket.h @@ -144,7 +144,7 @@ class TransportSocket { /** * @return the const SSL connection data if this is an SSL connection, or nullptr if it is not. */ - virtual const Ssl::ConnectionInfo* ssl() const PURE; + virtual Ssl::ConnectionInfoConstSharedPtr ssl() const PURE; }; using TransportSocketPtr = std::unique_ptr; diff --git a/include/envoy/ssl/connection.h b/include/envoy/ssl/connection.h index 203bd9a4c812..d4e54ba92e41 100644 --- a/include/envoy/ssl/connection.h +++ b/include/envoy/ssl/connection.h @@ -126,5 +126,7 @@ class ConnectionInfo { virtual std::string tlsVersion() const PURE; }; +using ConnectionInfoConstSharedPtr = std::shared_ptr; + } // namespace Ssl } // namespace Envoy diff --git a/include/envoy/stream_info/stream_info.h b/include/envoy/stream_info/stream_info.h index 442d8a1699a1..786c1aa8d42c 100644 --- a/include/envoy/stream_info/stream_info.h +++ b/include/envoy/stream_info/stream_info.h @@ -424,13 +424,26 @@ class StreamInfo { /** * @param connection_info sets the downstream ssl connection. */ - virtual void setDownstreamSslConnection(const Ssl::ConnectionInfo* ssl_connection_info) PURE; + virtual void + setDownstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& ssl_connection_info) PURE; /** * @return the downstream SSL connection. This will be nullptr if the downstream * connection does not use SSL. */ - virtual const Ssl::ConnectionInfo* downstreamSslConnection() const PURE; + virtual Ssl::ConnectionInfoConstSharedPtr downstreamSslConnection() const PURE; + + /** + * @param connection_info sets the upstream ssl connection. + */ + virtual void + setUpstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& ssl_connection_info) PURE; + + /** + * @return the upstream SSL connection. This will be nullptr if the upstream + * connection does not use SSL. + */ + virtual Ssl::ConnectionInfoConstSharedPtr upstreamSslConnection() const PURE; /** * @return const Router::RouteEntry* Get the route entry selected for this request. Note: this diff --git a/source/common/http/codec_client.cc b/source/common/http/codec_client.cc index 7d9ad41f3af0..2523d5970b01 100644 --- a/source/common/http/codec_client.cc +++ b/source/common/http/codec_client.cc @@ -62,6 +62,7 @@ StreamEncoder& CodecClient::newStream(StreamDecoder& response_decoder) { void CodecClient::onEvent(Network::ConnectionEvent event) { if (event == Network::ConnectionEvent::Connected) { ENVOY_CONN_LOG(debug, "connected", *connection_); + connection_->streamInfo().setDownstreamSslConnection(connection_->ssl()); connected_ = true; } diff --git a/source/common/http/codec_client.h b/source/common/http/codec_client.h index b9cf3482dce5..66c6c6d6bbc6 100644 --- a/source/common/http/codec_client.h +++ b/source/common/http/codec_client.h @@ -118,6 +118,8 @@ class CodecClient : Logger::Loggable, Type type() const { return type_; } + const StreamInfo::StreamInfo& streamInfo() { return connection_->streamInfo(); } + protected: /** * Create a codec client and connect to a remote host/port. diff --git a/source/common/http/http1/conn_pool.cc b/source/common/http/http1/conn_pool.cc index f785298cf80a..07f96ce46e63 100644 --- a/source/common/http/http1/conn_pool.cc +++ b/source/common/http/http1/conn_pool.cc @@ -70,7 +70,8 @@ void ConnPoolImpl::attachRequestToClient(ActiveClient& client, StreamDecoder& re host_->cluster().stats().upstream_rq_total_.inc(); host_->stats().rq_total_.inc(); client.stream_wrapper_ = std::make_unique(response_decoder, client); - callbacks.onPoolReady(*client.stream_wrapper_, client.real_host_description_); + callbacks.onPoolReady(*client.stream_wrapper_, client.real_host_description_, + client.codec_client_->streamInfo()); } void ConnPoolImpl::checkForDrained() { diff --git a/source/common/http/http2/conn_pool.cc b/source/common/http/http2/conn_pool.cc index a205a9e8b704..62c952e48d9a 100644 --- a/source/common/http/http2/conn_pool.cc +++ b/source/common/http/http2/conn_pool.cc @@ -101,7 +101,8 @@ void ConnPoolImpl::newClientStream(Http::StreamDecoder& response_decoder, host_->cluster().stats().upstream_rq_active_.inc(); host_->cluster().resourceManager(priority_).requests().inc(); callbacks.onPoolReady(primary_client_->client_->newStream(response_decoder), - primary_client_->real_host_description_); + primary_client_->real_host_description_, + primary_client_->client_->streamInfo()); } } diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index 96bb32693691..5923951a34e5 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -81,7 +81,7 @@ class ConnectionImpl : public FilterManagerConnection, } absl::optional unixSocketPeerCredentials() const override; void setConnectionStats(const ConnectionStats& stats) override; - const Ssl::ConnectionInfo* ssl() const override { return transport_socket_->ssl(); } + Ssl::ConnectionInfoConstSharedPtr ssl() const override { return transport_socket_->ssl(); } State state() const override; void write(Buffer::Instance& data, bool end_stream) override; void setBufferLimits(uint32_t limit) override; diff --git a/source/common/network/raw_buffer_socket.h b/source/common/network/raw_buffer_socket.h index 5183ce18fbc9..fe87bbeda605 100644 --- a/source/common/network/raw_buffer_socket.h +++ b/source/common/network/raw_buffer_socket.h @@ -20,7 +20,7 @@ class RawBufferSocket : public TransportSocket, protected Logger::Loggablescope(), parent_.callbacks_->dispatcher()); ENVOY_STREAM_LOG(debug, "pool ready", *parent_.callbacks_); @@ -1549,6 +1550,9 @@ void Filter::UpstreamRequest::onPoolReady(Http::StreamEncoder& request_encoder, onUpstreamHostSelected(host); request_encoder.getStream().addCallbacks(*this); + stream_info_.setUpstreamSslConnection(info.downstreamSslConnection()); + parent_.callbacks_->streamInfo().setUpstreamSslConnection(info.downstreamSslConnection()); + if (parent_.downstream_end_stream_) { setupPerTryTimeout(); } else { diff --git a/source/common/router/router.h b/source/common/router/router.h index fac9bbb07ee0..3585ad888f73 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -427,7 +427,8 @@ class Filter : Logger::Loggable, absl::string_view transport_failure_reason, Upstream::HostDescriptionConstSharedPtr host) override; void onPoolReady(Http::StreamEncoder& request_encoder, - Upstream::HostDescriptionConstSharedPtr host) override; + Upstream::HostDescriptionConstSharedPtr host, + const StreamInfo::StreamInfo& info) override; void setRequestEncoder(Http::StreamEncoder& request_encoder); void clearRequestEncoder(); diff --git a/source/common/stream_info/stream_info_impl.h b/source/common/stream_info/stream_info_impl.h index 7f7dd70ad3ef..cd9f0db4a381 100644 --- a/source/common/stream_info/stream_info_impl.h +++ b/source/common/stream_info/stream_info_impl.h @@ -175,14 +175,23 @@ struct StreamInfoImpl : public StreamInfo { return downstream_remote_address_; } - void setDownstreamSslConnection(const Ssl::ConnectionInfo* connection_info) override { + void + setDownstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& connection_info) override { downstream_ssl_info_ = connection_info; } - const Ssl::ConnectionInfo* downstreamSslConnection() const override { + Ssl::ConnectionInfoConstSharedPtr downstreamSslConnection() const override { return downstream_ssl_info_; } + void setUpstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& connection_info) override { + upstream_ssl_info_ = connection_info; + } + + Ssl::ConnectionInfoConstSharedPtr upstreamSslConnection() const override { + return upstream_ssl_info_; + } + const Router::RouteEntry* routeEntry() const override { return route_entry_; } envoy::api::v2::core::Metadata& dynamicMetadata() override { return metadata_; }; @@ -243,7 +252,8 @@ struct StreamInfoImpl : public StreamInfo { Network::Address::InstanceConstSharedPtr downstream_local_address_; Network::Address::InstanceConstSharedPtr downstream_direct_remote_address_; Network::Address::InstanceConstSharedPtr downstream_remote_address_; - const Ssl::ConnectionInfo* downstream_ssl_info_{}; + Ssl::ConnectionInfoConstSharedPtr downstream_ssl_info_; + Ssl::ConnectionInfoConstSharedPtr upstream_ssl_info_; std::string requested_server_name_; UpstreamTiming upstream_timing_; std::string upstream_transport_failure_reason_; diff --git a/source/common/tcp/conn_pool.cc b/source/common/tcp/conn_pool.cc index 66cd20da46a2..f7b7467f8982 100644 --- a/source/common/tcp/conn_pool.cc +++ b/source/common/tcp/conn_pool.cc @@ -194,6 +194,7 @@ void ConnPoolImpl::onConnectionEvent(ActiveConn& conn, Network::ConnectionEvent // whether the connection is in the ready list (connected) or the pending list (failed to // connect). if (event == Network::ConnectionEvent::Connected) { + conn.conn_->streamInfo().setDownstreamSslConnection(conn.conn_->ssl()); conn_connect_ms_->complete(); processIdleConnection(conn, true, false); } diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index ce466c2512c6..aee0dbb70fb5 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -439,6 +439,7 @@ void Filter::onPoolReady(Tcp::ConnectionPool::ConnectionDataPtr&& conn_data, getStreamInfo().onUpstreamHostSelected(host); getStreamInfo().setUpstreamLocalAddress(connection.localAddress()); + getStreamInfo().setUpstreamSslConnection(connection.streamInfo().downstreamSslConnection()); // Simulate the event that onPoolReady represents. upstream_callbacks_->onEvent(Network::ConnectionEvent::Connected); diff --git a/source/common/tcp_proxy/tcp_proxy.h b/source/common/tcp_proxy/tcp_proxy.h index 5a8534a56287..928accb654ef 100644 --- a/source/common/tcp_proxy/tcp_proxy.h +++ b/source/common/tcp_proxy/tcp_proxy.h @@ -231,6 +231,8 @@ class Filter : public Network::ReadFilter, bool on_high_watermark_called_{false}; }; + virtual StreamInfo::StreamInfo& getStreamInfo() { return stream_info_; } + protected: struct DownstreamCallbacks : public Network::ConnectionCallbacks { DownstreamCallbacks(Filter& parent) : parent_(parent) {} @@ -260,8 +262,6 @@ class Filter : public Network::ReadFilter, read_callbacks_->connection().close(Network::ConnectionCloseType::NoFlush); } - virtual StreamInfo::StreamInfo& getStreamInfo() { return stream_info_; } - void initialize(Network::ReadFilterCallbacks& callbacks, bool set_connection_stats); Network::FilterStatus initializeUpstreamConnection(); void onConnectTimeout(); diff --git a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc index 66cf3c0a7489..a8610a68f050 100644 --- a/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc +++ b/source/extensions/access_loggers/grpc/grpc_access_log_utils.cc @@ -129,7 +129,8 @@ void Utility::extractCommonAccessLogProperties( } if (stream_info.downstreamSslConnection() != nullptr) { auto* tls_properties = common_access_log.mutable_tls_properties(); - const auto* downstream_ssl_connection = stream_info.downstreamSslConnection(); + const Ssl::ConnectionInfoConstSharedPtr downstream_ssl_connection = + stream_info.downstreamSslConnection(); tls_properties->set_tls_sni_hostname(stream_info.requestedServerName()); diff --git a/source/extensions/filters/common/ext_authz/check_request_utils.cc b/source/extensions/filters/common/ext_authz/check_request_utils.cc index 707053f46bf0..b29c39708476 100644 --- a/source/extensions/filters/common/ext_authz/check_request_utils.cc +++ b/source/extensions/filters/common/ext_authz/check_request_utils.cc @@ -39,7 +39,7 @@ void CheckRequestUtils::setAttrContextPeer(envoy::service::auth::v2::AttributeCo // Set the principal. Preferably the URI SAN, DNS SAN or Subject in that order from the peer's // cert. - Ssl::ConnectionInfo* ssl = const_cast(connection.ssl()); + auto ssl = connection.ssl(); if (ssl != nullptr) { if (local) { const auto uri_sans = ssl->uriSanLocalCertificate(); diff --git a/source/extensions/filters/common/lua/wrappers.h b/source/extensions/filters/common/lua/wrappers.h index 5ac8ff217c3f..3cdff2298d02 100644 --- a/source/extensions/filters/common/lua/wrappers.h +++ b/source/extensions/filters/common/lua/wrappers.h @@ -106,7 +106,7 @@ class MetadataMapWrapper : public BaseLuaObject { */ class SslConnectionWrapper : public BaseLuaObject { public: - SslConnectionWrapper(const Ssl::ConnectionInfo*) {} + SslConnectionWrapper(const Ssl::ConnectionInfoConstSharedPtr) {} static ExportedFunctions exportedFunctions() { return {}; } // TODO(dio): Add more Lua APIs around Ssl::Connection. diff --git a/source/extensions/filters/common/rbac/matchers.cc b/source/extensions/filters/common/rbac/matchers.cc index d2742e2e9ea1..271d54c29523 100644 --- a/source/extensions/filters/common/rbac/matchers.cc +++ b/source/extensions/filters/common/rbac/matchers.cc @@ -133,7 +133,7 @@ bool PortMatcher::matches(const Network::Connection& connection, const Envoy::Ht bool AuthenticatedMatcher::matches(const Network::Connection& connection, const Envoy::Http::HeaderMap&, const StreamInfo::StreamInfo&) const { - const auto* ssl = connection.ssl(); + const auto& ssl = connection.ssl(); if (!ssl) { // connection was not authenticated return false; } else if (!matcher_.has_value()) { // matcher allows any subject diff --git a/source/extensions/transport_sockets/alts/tsi_socket.h b/source/extensions/transport_sockets/alts/tsi_socket.h index bbd7b19fadcb..0acba405022d 100644 --- a/source/extensions/transport_sockets/alts/tsi_socket.h +++ b/source/extensions/transport_sockets/alts/tsi_socket.h @@ -58,7 +58,7 @@ class TsiSocket : public Network::TransportSocket, std::string protocol() const override; absl::string_view failureReason() const override; bool canFlushClose() override { return handshake_complete_; } - const Envoy::Ssl::ConnectionInfo* ssl() const override { return nullptr; } + Envoy::Ssl::ConnectionInfoConstSharedPtr ssl() const override { return nullptr; } Network::IoResult doWrite(Buffer::Instance& buffer, bool end_stream) override; void closeSocket(Network::ConnectionEvent event) override; Network::IoResult doRead(Buffer::Instance& buffer) override; diff --git a/source/extensions/transport_sockets/tap/tap.cc b/source/extensions/transport_sockets/tap/tap.cc index fc34b5ee2c6d..634bfb6759f9 100644 --- a/source/extensions/transport_sockets/tap/tap.cc +++ b/source/extensions/transport_sockets/tap/tap.cc @@ -50,7 +50,7 @@ Network::IoResult TapSocket::doWrite(Buffer::Instance& buffer, bool end_stream) void TapSocket::onConnected() { transport_socket_->onConnected(); } -const Ssl::ConnectionInfo* TapSocket::ssl() const { return transport_socket_->ssl(); } +Ssl::ConnectionInfoConstSharedPtr TapSocket::ssl() const { return transport_socket_->ssl(); } TapSocketFactory::TapSocketFactory( const envoy::config::transport_socket::tap::v2alpha::Tap& proto_config, diff --git a/source/extensions/transport_sockets/tap/tap.h b/source/extensions/transport_sockets/tap/tap.h index eb5f76959d14..6e1e6a98cd12 100644 --- a/source/extensions/transport_sockets/tap/tap.h +++ b/source/extensions/transport_sockets/tap/tap.h @@ -26,7 +26,7 @@ class TapSocket : public Network::TransportSocket { Network::IoResult doRead(Buffer::Instance& buffer) override; Network::IoResult doWrite(Buffer::Instance& buffer, bool end_stream) override; void onConnected() override; - const Ssl::ConnectionInfo* ssl() const override; + Ssl::ConnectionInfoConstSharedPtr ssl() const override; private: SocketTapConfigSharedPtr config_; diff --git a/source/extensions/transport_sockets/tls/ssl_socket.cc b/source/extensions/transport_sockets/tls/ssl_socket.cc index c079bb989fdc..f0ec870245a9 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.cc +++ b/source/extensions/transport_sockets/tls/ssl_socket.cc @@ -38,20 +38,22 @@ class NotReadySslSocket : public Network::TransportSocket { return {PostIoAction::Close, 0, false}; } void onConnected() override {} - const Ssl::ConnectionInfo* ssl() const override { return nullptr; } + Ssl::ConnectionInfoConstSharedPtr ssl() const override { return nullptr; } }; } // namespace SslSocket::SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, const Network::TransportSocketOptionsSharedPtr& transport_socket_options) : transport_socket_options_(transport_socket_options), - ctx_(std::dynamic_pointer_cast(ctx)), - ssl_(ctx_->newSsl(transport_socket_options_.get())), state_(SocketState::PreHandshake) { + ctx_(std::dynamic_pointer_cast(ctx)), state_(SocketState::PreHandshake) { + bssl::UniquePtr ssl = ctx_->newSsl(transport_socket_options_.get()); + ssl_ = ssl.get(); + info_ = std::make_shared(std::move(ssl)); if (state == InitialState::Client) { - SSL_set_connect_state(ssl_.get()); + SSL_set_connect_state(ssl_); } else { ASSERT(state == InitialState::Server); - SSL_set_accept_state(ssl_.get()); + SSL_set_accept_state(ssl_); } } @@ -62,11 +64,11 @@ void SslSocket::setTransportSocketCallbacks(Network::TransportSocketCallbacks& c // Associate this SSL connection with all the certificates (with their potentially different // private key methods). for (auto const& provider : ctx_->getPrivateKeyMethodProviders()) { - provider->registerPrivateKeyMethod(ssl_.get(), *this, callbacks_->connection().dispatcher()); + provider->registerPrivateKeyMethod(ssl_, *this, callbacks_->connection().dispatcher()); } BIO* bio = BIO_new_socket(callbacks_->ioHandle().fd(), 0); - SSL_set_bio(ssl_.get(), bio, bio); + SSL_set_bio(ssl_, bio, bio); } SslSocket::ReadResult SslSocket::sslReadIntoSlice(Buffer::RawSlice& slice) { @@ -74,7 +76,7 @@ SslSocket::ReadResult SslSocket::sslReadIntoSlice(Buffer::RawSlice& slice) { uint8_t* mem = static_cast(slice.mem_); size_t remaining = slice.len_; while (remaining > 0) { - int rc = SSL_read(ssl_.get(), mem, remaining); + int rc = SSL_read(ssl_, mem, remaining); ENVOY_CONN_LOG(trace, "ssl read returns: {}", callbacks_->connection(), rc); if (rc > 0) { ASSERT(static_cast(rc) <= remaining); @@ -121,7 +123,7 @@ Network::IoResult SslSocket::doRead(Buffer::Instance& read_buffer) { } if (result.error_.has_value()) { keep_reading = false; - int err = SSL_get_error(ssl_.get(), result.error_.value()); + int err = SSL_get_error(ssl_, result.error_.value()); switch (err) { case SSL_ERROR_WANT_READ: break; @@ -169,11 +171,11 @@ void SslSocket::onPrivateKeyMethodComplete() { PostIoAction SslSocket::doHandshake() { ASSERT(state_ != SocketState::HandshakeComplete && state_ != SocketState::ShutdownSent); - int rc = SSL_do_handshake(ssl_.get()); + int rc = SSL_do_handshake(ssl_); if (rc == 1) { ENVOY_CONN_LOG(debug, "handshake complete", callbacks_->connection()); state_ = SocketState::HandshakeComplete; - ctx_->logHandshake(ssl_.get()); + ctx_->logHandshake(ssl_); callbacks_->raiseEvent(Network::ConnectionEvent::Connected); // It's possible that we closed during the handshake callback. @@ -181,7 +183,7 @@ PostIoAction SslSocket::doHandshake() { ? PostIoAction::KeepOpen : PostIoAction::Close; } else { - int err = SSL_get_error(ssl_.get(), rc); + int err = SSL_get_error(ssl_, rc); switch (err) { case SSL_ERROR_WANT_READ: case SSL_ERROR_WANT_WRITE: @@ -253,7 +255,7 @@ Network::IoResult SslSocket::doWrite(Buffer::Instance& write_buffer, bool end_st // it again with the same parameters. This is done by tracking last write size, but not write // data, since linearize() will return the same undrained data anyway. ASSERT(bytes_to_write <= write_buffer.length()); - int rc = SSL_write(ssl_.get(), write_buffer.linearize(bytes_to_write), bytes_to_write); + int rc = SSL_write(ssl_, write_buffer.linearize(bytes_to_write), bytes_to_write); ENVOY_CONN_LOG(trace, "ssl write returns: {}", callbacks_->connection(), rc); if (rc > 0) { ASSERT(rc == static_cast(bytes_to_write)); @@ -261,7 +263,7 @@ Network::IoResult SslSocket::doWrite(Buffer::Instance& write_buffer, bool end_st write_buffer.drain(rc); bytes_to_write = std::min(write_buffer.length(), static_cast(16384)); } else { - int err = SSL_get_error(ssl_.get(), rc); + int err = SSL_get_error(ssl_, rc); switch (err) { case SSL_ERROR_WANT_WRITE: bytes_to_retry_ = bytes_to_write; @@ -286,40 +288,54 @@ Network::IoResult SslSocket::doWrite(Buffer::Instance& write_buffer, bool end_st void SslSocket::onConnected() { ASSERT(state_ == SocketState::PreHandshake); } +Ssl::ConnectionInfoConstSharedPtr SslSocket::ssl() const { return info_; } + void SslSocket::shutdownSsl() { ASSERT(state_ != SocketState::PreHandshake); if (state_ != SocketState::ShutdownSent && callbacks_->connection().state() != Network::Connection::State::Closed) { - int rc = SSL_shutdown(ssl_.get()); + int rc = SSL_shutdown(ssl_); ENVOY_CONN_LOG(debug, "SSL shutdown: rc={}", callbacks_->connection(), rc); drainErrorQueue(); state_ = SocketState::ShutdownSent; } } -bool SslSocket::peerCertificatePresented() const { +bool SslSocketInfo::peerCertificatePresented() const { bssl::UniquePtr cert(SSL_get_peer_certificate(ssl_.get())); return cert != nullptr; } -std::vector SslSocket::uriSanLocalCertificate() const { +std::vector SslSocketInfo::uriSanLocalCertificate() const { + if (!cached_uri_san_local_certificate_.empty()) { + return cached_uri_san_local_certificate_; + } + // The cert object is not owned. X509* cert = SSL_get_certificate(ssl_.get()); if (!cert) { - return {}; + ASSERT(cached_uri_san_local_certificate_.empty()); + return cached_uri_san_local_certificate_; } - return Utility::getSubjectAltNames(*cert, GEN_URI); + cached_uri_san_local_certificate_ = Utility::getSubjectAltNames(*cert, GEN_URI); + return cached_uri_san_local_certificate_; } -std::vector SslSocket::dnsSansLocalCertificate() const { +std::vector SslSocketInfo::dnsSansLocalCertificate() const { + if (!cached_dns_san_local_certificate_.empty()) { + return cached_dns_san_local_certificate_; + } + X509* cert = SSL_get_certificate(ssl_.get()); if (!cert) { - return {}; + ASSERT(cached_dns_san_local_certificate_.empty()); + return cached_dns_san_local_certificate_; } - return Utility::getSubjectAltNames(*cert, GEN_DNS); + cached_dns_san_local_certificate_ = Utility::getSubjectAltNames(*cert, GEN_DNS); + return cached_dns_san_local_certificate_; } -const std::string& SslSocket::sha256PeerCertificateDigest() const { +const std::string& SslSocketInfo::sha256PeerCertificateDigest() const { if (!cached_sha_256_peer_certificate_digest_.empty()) { return cached_sha_256_peer_certificate_digest_; } @@ -337,7 +353,7 @@ const std::string& SslSocket::sha256PeerCertificateDigest() const { return cached_sha_256_peer_certificate_digest_; } -const std::string& SslSocket::urlEncodedPemEncodedPeerCertificate() const { +const std::string& SslSocketInfo::urlEncodedPemEncodedPeerCertificate() const { if (!cached_url_encoded_pem_encoded_peer_certificate_.empty()) { return cached_url_encoded_pem_encoded_peer_certificate_; } @@ -359,7 +375,7 @@ const std::string& SslSocket::urlEncodedPemEncodedPeerCertificate() const { return cached_url_encoded_pem_encoded_peer_certificate_; } -const std::string& SslSocket::urlEncodedPemEncodedPeerCertificateChain() const { +const std::string& SslSocketInfo::urlEncodedPemEncodedPeerCertificateChain() const { if (!cached_url_encoded_pem_encoded_peer_cert_chain_.empty()) { return cached_url_encoded_pem_encoded_peer_cert_chain_; } @@ -389,26 +405,38 @@ const std::string& SslSocket::urlEncodedPemEncodedPeerCertificateChain() const { return cached_url_encoded_pem_encoded_peer_cert_chain_; } -std::vector SslSocket::uriSanPeerCertificate() const { +std::vector SslSocketInfo::uriSanPeerCertificate() const { + if (!cached_uri_san_peer_certificate_.empty()) { + return cached_uri_san_peer_certificate_; + } + bssl::UniquePtr cert(SSL_get_peer_certificate(ssl_.get())); if (!cert) { - return {}; + ASSERT(cached_uri_san_peer_certificate_.empty()); + return cached_uri_san_peer_certificate_; } - return Utility::getSubjectAltNames(*cert, GEN_URI); + cached_uri_san_peer_certificate_ = Utility::getSubjectAltNames(*cert, GEN_URI); + return cached_uri_san_peer_certificate_; } -std::vector SslSocket::dnsSansPeerCertificate() const { +std::vector SslSocketInfo::dnsSansPeerCertificate() const { + if (!cached_dns_san_peer_certificate_.empty()) { + return cached_dns_san_peer_certificate_; + } + bssl::UniquePtr cert(SSL_get_peer_certificate(ssl_.get())); if (!cert) { - return {}; + ASSERT(cached_dns_san_peer_certificate_.empty()); + return cached_dns_san_peer_certificate_; } - return Utility::getSubjectAltNames(*cert, GEN_DNS); + cached_dns_san_peer_certificate_ = Utility::getSubjectAltNames(*cert, GEN_DNS); + return cached_dns_san_peer_certificate_; } void SslSocket::closeSocket(Network::ConnectionEvent) { // Unregister the SSL connection object from private key method providers. for (auto const& provider : ctx_->getPrivateKeyMethodProviders()) { - provider->unregisterPrivateKeyMethod(ssl_.get()); + provider->unregisterPrivateKeyMethod(ssl_); } // Attempt to send a shutdown before closing the socket. It's possible this won't go out if @@ -422,11 +450,11 @@ void SslSocket::closeSocket(Network::ConnectionEvent) { std::string SslSocket::protocol() const { const unsigned char* proto; unsigned int proto_len; - SSL_get0_alpn_selected(ssl_.get(), &proto, &proto_len); + SSL_get0_alpn_selected(ssl_, &proto, &proto_len); return std::string(reinterpret_cast(proto), proto_len); } -uint16_t SslSocket::ciphersuiteId() const { +uint16_t SslSocketInfo::ciphersuiteId() const { const SSL_CIPHER* cipher = SSL_get_current_cipher(ssl_.get()); if (cipher == nullptr) { return 0xffff; @@ -438,52 +466,72 @@ uint16_t SslSocket::ciphersuiteId() const { return static_cast(SSL_CIPHER_get_id(cipher)); } -std::string SslSocket::ciphersuiteString() const { +std::string SslSocketInfo::ciphersuiteString() const { const SSL_CIPHER* cipher = SSL_get_current_cipher(ssl_.get()); if (cipher == nullptr) { - return std::string(); + return {}; } - return std::string(SSL_CIPHER_get_name(cipher)); + return SSL_CIPHER_get_name(cipher); } -std::string SslSocket::tlsVersion() const { return std::string(SSL_get_version(ssl_.get())); } +std::string SslSocketInfo::tlsVersion() const { return SSL_get_version(ssl_.get()); } absl::string_view SslSocket::failureReason() const { return failure_reason_; } -std::string SslSocket::serialNumberPeerCertificate() const { +std::string SslSocketInfo::serialNumberPeerCertificate() const { + if (!cached_serial_number_peer_certificate_.empty()) { + return cached_serial_number_peer_certificate_; + } bssl::UniquePtr cert(SSL_get_peer_certificate(ssl_.get())); if (!cert) { - return ""; + ASSERT(cached_serial_number_peer_certificate_.empty()); + return cached_serial_number_peer_certificate_; } - return Utility::getSerialNumberFromCertificate(*cert.get()); + cached_serial_number_peer_certificate_ = Utility::getSerialNumberFromCertificate(*cert.get()); + return cached_serial_number_peer_certificate_; } -std::string SslSocket::issuerPeerCertificate() const { +std::string SslSocketInfo::issuerPeerCertificate() const { + if (!cached_issuer_peer_certificate_.empty()) { + return cached_issuer_peer_certificate_; + } bssl::UniquePtr cert(SSL_get_peer_certificate(ssl_.get())); if (!cert) { - return ""; + ASSERT(cached_issuer_peer_certificate_.empty()); + return cached_issuer_peer_certificate_; } - return Utility::getIssuerFromCertificate(*cert); + cached_issuer_peer_certificate_ = Utility::getIssuerFromCertificate(*cert); + return cached_issuer_peer_certificate_; } -std::string SslSocket::subjectPeerCertificate() const { +std::string SslSocketInfo::subjectPeerCertificate() const { + if (!cached_subject_peer_certificate_.empty()) { + return cached_subject_peer_certificate_; + } bssl::UniquePtr cert(SSL_get_peer_certificate(ssl_.get())); if (!cert) { - return ""; + ASSERT(cached_subject_peer_certificate_.empty()); + return cached_subject_peer_certificate_; } - return Utility::getSubjectFromCertificate(*cert); + cached_subject_peer_certificate_ = Utility::getSubjectFromCertificate(*cert); + return cached_subject_peer_certificate_; } -std::string SslSocket::subjectLocalCertificate() const { +std::string SslSocketInfo::subjectLocalCertificate() const { + if (!cached_subject_local_certificate_.empty()) { + return cached_subject_local_certificate_; + } X509* cert = SSL_get_certificate(ssl_.get()); if (!cert) { - return ""; + ASSERT(cached_subject_local_certificate_.empty()); + return cached_subject_local_certificate_; } - return Utility::getSubjectFromCertificate(*cert); + cached_subject_local_certificate_ = Utility::getSubjectFromCertificate(*cert); + return cached_subject_local_certificate_; } -absl::optional SslSocket::validFromPeerCertificate() const { +absl::optional SslSocketInfo::validFromPeerCertificate() const { bssl::UniquePtr cert(SSL_get_peer_certificate(ssl_.get())); if (!cert) { return absl::nullopt; @@ -491,7 +539,7 @@ absl::optional SslSocket::validFromPeerCertificate() const { return Utility::getValidFrom(*cert); } -absl::optional SslSocket::expirationPeerCertificate() const { +absl::optional SslSocketInfo::expirationPeerCertificate() const { bssl::UniquePtr cert(SSL_get_peer_certificate(ssl_.get())); if (!cert) { return absl::nullopt; @@ -499,15 +547,20 @@ absl::optional SslSocket::expirationPeerCertificate() const { return Utility::getExpirationTime(*cert); } -std::string SslSocket::sessionId() const { +std::string SslSocketInfo::sessionId() const { + if (!cached_session_id_.empty()) { + return cached_session_id_; + } SSL_SESSION* session = SSL_get_session(ssl_.get()); if (session == nullptr) { - return ""; + ASSERT(cached_session_id_.empty()); + return cached_session_id_; } unsigned int session_id_length = 0; const uint8_t* session_id = SSL_SESSION_get_id(session, &session_id_length); - return Hex::encode(session_id, session_id_length); + cached_session_id_ = Hex::encode(session_id, session_id_length); + return cached_session_id_; } namespace { diff --git a/source/extensions/transport_sockets/tls/ssl_socket.h b/source/extensions/transport_sockets/tls/ssl_socket.h index 8e0eb6601058..67ab1b28582a 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.h +++ b/source/extensions/transport_sockets/tls/ssl_socket.h @@ -41,13 +41,9 @@ struct SslSocketFactoryStats { enum class InitialState { Client, Server }; enum class SocketState { PreHandshake, HandshakeInProgress, HandshakeComplete, ShutdownSent }; -class SslSocket : public Network::TransportSocket, - public Envoy::Ssl::ConnectionInfo, - public Envoy::Ssl::PrivateKeyConnectionCallbacks, - protected Logger::Loggable { +class SslSocketInfo : public Envoy::Ssl::ConnectionInfo { public: - SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, - const Network::TransportSocketOptionsSharedPtr& transport_socket_options); + SslSocketInfo(bssl::UniquePtr ssl) : ssl_(std::move(ssl)) {} // Ssl::ConnectionInfo bool peerCertificatePresented() const override; @@ -69,6 +65,32 @@ class SslSocket : public Network::TransportSocket, std::string ciphersuiteString() const override; std::string tlsVersion() const override; + SSL* rawSslForTest() const { return ssl_.get(); } + + bssl::UniquePtr ssl_; + +private: + mutable std::vector cached_uri_san_local_certificate_; + mutable std::string cached_sha_256_peer_certificate_digest_; + mutable std::string cached_serial_number_peer_certificate_; + mutable std::string cached_issuer_peer_certificate_; + mutable std::string cached_subject_peer_certificate_; + mutable std::string cached_subject_local_certificate_; + mutable std::vector cached_uri_san_peer_certificate_; + mutable std::string cached_url_encoded_pem_encoded_peer_certificate_; + mutable std::string cached_url_encoded_pem_encoded_peer_cert_chain_; + mutable std::vector cached_dns_san_peer_certificate_; + mutable std::vector cached_dns_san_local_certificate_; + mutable std::string cached_session_id_; +}; + +class SslSocket : public Network::TransportSocket, + public Envoy::Ssl::PrivateKeyConnectionCallbacks, + protected Logger::Loggable { +public: + SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state, + const Network::TransportSocketOptionsSharedPtr& transport_socket_options); + // Network::TransportSocket void setTransportSocketCallbacks(Network::TransportSocketCallbacks& callbacks) override; std::string protocol() const override; @@ -78,12 +100,11 @@ class SslSocket : public Network::TransportSocket, Network::IoResult doRead(Buffer::Instance& read_buffer) override; Network::IoResult doWrite(Buffer::Instance& write_buffer, bool end_stream) override; void onConnected() override; - const Ssl::ConnectionInfo* ssl() const override { return this; } - + Ssl::ConnectionInfoConstSharedPtr ssl() const override; // Ssl::PrivateKeyConnectionCallbacks void onPrivateKeyMethodComplete() override; - SSL* rawSslForTest() const { return ssl_.get(); } + SSL* rawSslForTest() const { return ssl_; } private: struct ReadResult { @@ -102,13 +123,12 @@ class SslSocket : public Network::TransportSocket, const Network::TransportSocketOptionsSharedPtr transport_socket_options_; Network::TransportSocketCallbacks* callbacks_{}; ContextImplSharedPtr ctx_; - bssl::UniquePtr ssl_; uint64_t bytes_to_retry_{}; std::string failure_reason_; - mutable std::string cached_sha_256_peer_certificate_digest_; - mutable std::string cached_url_encoded_pem_encoded_peer_certificate_; - mutable std::string cached_url_encoded_pem_encoded_peer_cert_chain_; SocketState state_; + + SSL* ssl_; + Ssl::ConnectionInfoConstSharedPtr info_; }; class ClientSslSocketFactory : public Network::TransportSocketFactory, diff --git a/test/common/access_log/access_log_formatter_fuzz_test.cc b/test/common/access_log/access_log_formatter_fuzz_test.cc index 09d6b4322510..1df65dd1736e 100644 --- a/test/common/access_log/access_log_formatter_fuzz_test.cc +++ b/test/common/access_log/access_log_formatter_fuzz_test.cc @@ -10,14 +10,12 @@ namespace { DEFINE_PROTO_FUZZER(const test::common::access_log::TestCase& input) { try { - NiceMock connection_info; std::vector formatters = AccessLog::AccessLogFormatParser::parse(input.format()); for (const auto& it : formatters) { - it->format(Fuzz::fromHeaders(input.request_headers()), - Fuzz::fromHeaders(input.response_headers()), - Fuzz::fromHeaders(input.response_trailers()), - Fuzz::fromStreamInfo(input.stream_info(), &connection_info)); + it->format( + Fuzz::fromHeaders(input.request_headers()), Fuzz::fromHeaders(input.response_headers()), + Fuzz::fromHeaders(input.response_trailers()), Fuzz::fromStreamInfo(input.stream_info())); } ENVOY_LOG_MISC(trace, "Success"); } catch (const EnvoyException& e) { diff --git a/test/common/access_log/access_log_formatter_test.cc b/test/common/access_log/access_log_formatter_test.cc index ece0a648a3d2..db682288410b 100644 --- a/test/common/access_log/access_log_formatter_test.cc +++ b/test/common/access_log/access_log_formatter_test.cc @@ -219,26 +219,27 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_URI_SAN"); - NiceMock connection_info; + auto connection_info = std::make_shared(); const std::vector sans{"san"}; - ON_CALL(connection_info, uriSanPeerCertificate()).WillByDefault(Return(sans)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("san", upstream_format.format(header, header, header, stream_info)); } + { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_URI_SAN"); - NiceMock connection_info; + auto connection_info = std::make_shared(); const std::vector sans{"san1", "san2"}; - ON_CALL(connection_info, uriSanPeerCertificate()).WillByDefault(Return(sans)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("san1,san2", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_URI_SAN"); - NiceMock connection_info; - ON_CALL(connection_info, uriSanPeerCertificate()) - .WillByDefault(Return(std::vector())); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, uriSanPeerCertificate()) + .WillRepeatedly(Return(std::vector())); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -248,26 +249,26 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_URI_SAN"); - NiceMock connection_info; + auto connection_info = std::make_shared(); const std::vector sans{"san"}; - ON_CALL(connection_info, uriSanLocalCertificate()).WillByDefault(Return(sans)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, uriSanLocalCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("san", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_URI_SAN"); - NiceMock connection_info; + auto connection_info = std::make_shared(); const std::vector sans{"san1", "san2"}; - ON_CALL(connection_info, uriSanLocalCertificate()).WillByDefault(Return(sans)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, uriSanLocalCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("san1,san2", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_URI_SAN"); - NiceMock connection_info; - ON_CALL(connection_info, uriSanLocalCertificate()) - .WillByDefault(Return(std::vector())); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, uriSanLocalCertificate()) + .WillRepeatedly(Return(std::vector())); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -277,16 +278,16 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_SUBJECT"); - NiceMock connection_info; - ON_CALL(connection_info, subjectLocalCertificate()).WillByDefault(Return("subject")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, subjectLocalCertificate()).WillRepeatedly(Return("subject")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("subject", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_SUBJECT"); - NiceMock connection_info; - ON_CALL(connection_info, subjectLocalCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, subjectLocalCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -296,16 +297,16 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); - NiceMock connection_info; - ON_CALL(connection_info, subjectPeerCertificate()).WillByDefault(Return("subject")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(Return("subject")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("subject", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); - NiceMock connection_info; - ON_CALL(connection_info, subjectPeerCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -315,16 +316,16 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_SESSION_ID"); - NiceMock connection_info; - ON_CALL(connection_info, sessionId()).WillByDefault(Return("deadbeef")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(Return("deadbeef")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("deadbeef", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_SESSION_ID"); - NiceMock connection_info; - ON_CALL(connection_info, sessionId()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -334,18 +335,18 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_CIPHER"); - NiceMock connection_info; - ON_CALL(connection_info, ciphersuiteString()) - .WillByDefault(Return("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, ciphersuiteString()) + .WillRepeatedly(Return("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_CIPHER"); - NiceMock connection_info; - ON_CALL(connection_info, ciphersuiteString()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, ciphersuiteString()).WillRepeatedly(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -355,16 +356,16 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_VERSION"); - NiceMock connection_info; - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("TLSv1.2")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(Return("TLSv1.2")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("TLSv1.2", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_VERSION"); - NiceMock connection_info; - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -374,18 +375,20 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_FINGERPRINT_256"); - NiceMock connection_info; + auto connection_info = std::make_shared(); std::string expected_sha = "685a2db593d5f86d346cb1a297009c3b467ad77f1944aa799039a2fb3d531f3f"; - ON_CALL(connection_info, sha256PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, sha256PeerCertificateDigest()) + .WillRepeatedly(ReturnRef(expected_sha)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ(expected_sha, upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_FINGERPRINT_256"); - NiceMock connection_info; + auto connection_info = std::make_shared(); std::string expected_sha; - ON_CALL(connection_info, sha256PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, sha256PeerCertificateDigest()) + .WillRepeatedly(ReturnRef(expected_sha)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -395,17 +398,17 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SERIAL"); - NiceMock connection_info; - ON_CALL(connection_info, serialNumberPeerCertificate()) - .WillByDefault(Return("b8b5ecc898f2124a")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, serialNumberPeerCertificate()) + .WillRepeatedly(Return("b8b5ecc898f2124a")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("b8b5ecc898f2124a", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SERIAL"); - NiceMock connection_info; - ON_CALL(connection_info, serialNumberPeerCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, serialNumberPeerCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -415,19 +418,19 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_ISSUER"); - NiceMock connection_info; - ON_CALL(connection_info, issuerPeerCertificate()) - .WillByDefault( + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, issuerPeerCertificate()) + .WillRepeatedly( Return("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_ISSUER"); - NiceMock connection_info; - ON_CALL(connection_info, issuerPeerCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, issuerPeerCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -437,19 +440,19 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); - NiceMock connection_info; - ON_CALL(connection_info, subjectPeerCertificate()) - .WillByDefault( + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, subjectPeerCertificate()) + .WillRepeatedly( Return("CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); - NiceMock connection_info; - ON_CALL(connection_info, subjectPeerCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -459,20 +462,20 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT"); - NiceMock connection_info; + auto connection_info = std::make_shared(); std::string expected_cert = ""; - ON_CALL(connection_info, urlEncodedPemEncodedPeerCertificate()) - .WillByDefault(ReturnRef(expected_cert)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) + .WillRepeatedly(ReturnRef(expected_cert)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ(expected_cert, upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT"); - NiceMock connection_info; + auto connection_info = std::make_shared(); std::string expected_cert = ""; - ON_CALL(connection_info, urlEncodedPemEncodedPeerCertificate()) - .WillByDefault(ReturnRef(expected_cert)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) + .WillRepeatedly(ReturnRef(expected_cert)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -482,20 +485,20 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_START"); - NiceMock connection_info; + auto connection_info = std::make_shared(); absl::Time abslStartTime = TestUtility::parseTime("Dec 18 01:50:34 2018 GMT", "%b %e %H:%M:%S %Y GMT"); SystemTime startTime = absl::ToChronoTime(abslStartTime); - ON_CALL(connection_info, validFromPeerCertificate()).WillByDefault(Return(startTime)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(startTime)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("2018-12-18T01:50:34.000Z", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_START"); - NiceMock connection_info; - ON_CALL(connection_info, validFromPeerCertificate()).WillByDefault(Return(absl::nullopt)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, validFromPeerCertificate()).WillRepeatedly(Return(absl::nullopt)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { @@ -505,20 +508,21 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_END"); - NiceMock connection_info; + auto connection_info = std::make_shared(); absl::Time abslEndTime = TestUtility::parseTime("Dec 17 01:50:34 2020 GMT", "%b %e %H:%M:%S %Y GMT"); SystemTime endTime = absl::ToChronoTime(abslEndTime); - ON_CALL(connection_info, expirationPeerCertificate()).WillByDefault(Return(endTime)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(*connection_info, expirationPeerCertificate()).WillRepeatedly(Return(endTime)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("2020-12-17T01:50:34.000Z", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_CERT_V_END"); - NiceMock connection_info; - ON_CALL(connection_info, expirationPeerCertificate()).WillByDefault(Return(absl::nullopt)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared(); + EXPECT_CALL(*connection_info, expirationPeerCertificate()) + .WillRepeatedly(Return(absl::nullopt)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } { diff --git a/test/common/http/BUILD b/test/common/http/BUILD index 1766f89aa0d7..5c6a91624f31 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -51,6 +51,7 @@ envoy_cc_test( "//test/mocks/event:event_mocks", "//test/mocks/http:http_mocks", "//test/mocks/network:network_mocks", + "//test/mocks/ssl:ssl_mocks", "//test/mocks/upstream:upstream_mocks", "//test/test_common:environment_lib", "//test/test_common:network_utility_lib", diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index f9b90ff2ab8b..bac7b1c2f1ab 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -75,6 +75,7 @@ class AsyncClientImplTest : public testing::Test { NiceMock local_info_; Http::ContextImpl http_context_; AsyncClientImpl client_; + NiceMock stream_info_; }; TEST_F(AsyncClientImplTest, BasicStream) { @@ -83,7 +84,7 @@ TEST_F(AsyncClientImplTest, BasicStream) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -125,7 +126,7 @@ TEST_F(AsyncClientImplTest, Basic) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -164,7 +165,7 @@ TEST_F(AsyncClientImplTest, Retry) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -184,7 +185,7 @@ TEST_F(AsyncClientImplTest, Retry) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -207,7 +208,7 @@ TEST_F(AsyncClientImplTest, RetryWithStream) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -232,7 +233,7 @@ TEST_F(AsyncClientImplTest, RetryWithStream) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -255,7 +256,7 @@ TEST_F(AsyncClientImplTest, MultipleStreams) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -281,7 +282,7 @@ TEST_F(AsyncClientImplTest, MultipleStreams) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder2, cm_.conn_pool_.host_, stream_info_); response_decoder2 = &decoder; return nullptr; })); @@ -315,7 +316,7 @@ TEST_F(AsyncClientImplTest, MultipleRequests) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -334,7 +335,7 @@ TEST_F(AsyncClientImplTest, MultipleRequests) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder2, cm_.conn_pool_.host_, stream_info_); response_decoder2 = &decoder; return nullptr; })); @@ -361,7 +362,7 @@ TEST_F(AsyncClientImplTest, StreamAndRequest) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -379,7 +380,7 @@ TEST_F(AsyncClientImplTest, StreamAndRequest) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder2, cm_.conn_pool_.host_, stream_info_); response_decoder2 = &decoder; return nullptr; })); @@ -418,7 +419,7 @@ TEST_F(AsyncClientImplTest, StreamWithTrailers) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -451,7 +452,7 @@ TEST_F(AsyncClientImplTest, Trailers) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -471,7 +472,7 @@ TEST_F(AsyncClientImplTest, ImmediateReset) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -492,7 +493,7 @@ TEST_F(AsyncClientImplTest, LocalResetAfterStreamStart) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -529,7 +530,7 @@ TEST_F(AsyncClientImplTest, ResetInOnHeaders) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -564,7 +565,7 @@ TEST_F(AsyncClientImplTest, RemoteResetAfterStreamStart) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -597,7 +598,7 @@ TEST_F(AsyncClientImplTest, ResetAfterResponseStart) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); @@ -615,7 +616,7 @@ TEST_F(AsyncClientImplTest, ResetStream) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -632,7 +633,7 @@ TEST_F(AsyncClientImplTest, CancelRequest) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -648,7 +649,7 @@ TEST_F(AsyncClientImplTest, DestroyWithActiveStream) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -663,7 +664,7 @@ TEST_F(AsyncClientImplTest, DestroyWithActiveRequest) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -712,7 +713,7 @@ TEST_F(AsyncClientImplTest, StreamTimeout) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -745,7 +746,7 @@ TEST_F(AsyncClientImplTest, StreamTimeoutHeadReply) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -771,7 +772,7 @@ TEST_F(AsyncClientImplTest, RequestTimeout) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -797,7 +798,7 @@ TEST_F(AsyncClientImplTest, DisableTimer) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -816,7 +817,7 @@ TEST_F(AsyncClientImplTest, DisableTimerWithStream) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder&, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); return nullptr; })); @@ -840,7 +841,7 @@ TEST_F(AsyncClientImplTest, MultipleDataStream) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](StreamDecoder& decoder, ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { - callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); response_decoder_ = &decoder; return nullptr; })); diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 75693aebbd13..8c7d63a7f6bb 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -14,6 +14,7 @@ #include "test/mocks/event/mocks.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/mocks/ssl/mocks.h" #include "test/mocks/upstream/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/network_utility.h" @@ -30,6 +31,7 @@ using testing::InvokeWithoutArgs; using testing::NiceMock; using testing::Pointee; using testing::Ref; +using testing::Return; using testing::Throw; namespace Envoy { @@ -54,6 +56,7 @@ class CodecClientTest : public testing::Test { EXPECT_CALL(dispatcher_, createTimer_(_)); client_ = std::make_unique(std::move(connection), codec_, nullptr, host_, dispatcher_); + ON_CALL(*connection_, streamInfo()).WillByDefault(ReturnRef(stream_info_)); } ~CodecClientTest() override { EXPECT_EQ(0U, client_->numActiveRequests()); } @@ -68,6 +71,7 @@ class CodecClientTest : public testing::Test { new NiceMock()}; Upstream::HostDescriptionConstSharedPtr host_{ Upstream::makeTestHostDescription(cluster_, "tcp://127.0.0.1:80")}; + NiceMock stream_info_; }; TEST_F(CodecClientTest, BasicHeaderOnlyResponse) { @@ -255,6 +259,16 @@ TEST_F(CodecClientTest, WatermarkPassthrough) { connection_cb_->onBelowWriteBufferLowWatermark(); } +TEST_F(CodecClientTest, SSLConnectionInfo) { + const auto session_id = "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, sessionId()).WillByDefault(Return(session_id)); + EXPECT_CALL(*connection_, ssl()).WillRepeatedly(Return(connection_info)); + connection_cb_->onEvent(Network::ConnectionEvent::Connected); + EXPECT_NE(nullptr, stream_info_.downstreamSslConnection()); + EXPECT_EQ(session_id, stream_info_.downstreamSslConnection()->sessionId()); +} + // Test the codec getting input from a real TCP connection. class CodecNetworkTest : public testing::TestWithParam { public: diff --git a/test/common/http/common.h b/test/common/http/common.h index 9aa57ef87c8e..d2c6f60747ab 100644 --- a/test/common/http/common.h +++ b/test/common/http/common.h @@ -39,8 +39,8 @@ class CodecClientForTest : public Http::CodecClient { * Mock callbacks used for conn pool testing. */ struct ConnPoolCallbacks : public Http::ConnectionPool::Callbacks { - void onPoolReady(Http::StreamEncoder& encoder, - Upstream::HostDescriptionConstSharedPtr host) override { + void onPoolReady(Http::StreamEncoder& encoder, Upstream::HostDescriptionConstSharedPtr host, + const StreamInfo::StreamInfo&) override { outer_encoder_ = &encoder; host_ = host; pool_ready_.ready(); diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 60a097dec870..3ddb49f3dbd7 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -405,11 +405,11 @@ DEFINE_PROTO_FUZZER(const test::common::http::ConnManagerImplTestCase& input) { NiceMock local_info; NiceMock cluster_manager; NiceMock filter_callbacks; - std::unique_ptr ssl_connection; + auto ssl_connection = std::make_shared(); bool connection_alive = true; - ON_CALL(filter_callbacks.connection_, ssl()).WillByDefault(Return(ssl_connection.get())); - ON_CALL(Const(filter_callbacks.connection_), ssl()).WillByDefault(Return(ssl_connection.get())); + ON_CALL(filter_callbacks.connection_, ssl()).WillByDefault(Return(ssl_connection)); + ON_CALL(Const(filter_callbacks.connection_), ssl()).WillByDefault(Return(ssl_connection)); ON_CALL(filter_callbacks.connection_, close(_)) .WillByDefault(InvokeWithoutArgs([&connection_alive] { connection_alive = false; })); filter_callbacks.connection_.local_address_ = diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 846c674d1ec6..2b8cc61b2a1f 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -98,13 +98,12 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan void setup(bool ssl, const std::string& server_name, bool tracing = true, bool use_srds = false) { use_srds_ = use_srds; if (ssl) { - ssl_connection_ = std::make_unique(); + ssl_connection_ = std::make_shared(); } server_name_ = server_name; - ON_CALL(filter_callbacks_.connection_, ssl()).WillByDefault(Return(ssl_connection_.get())); - ON_CALL(Const(filter_callbacks_.connection_), ssl()) - .WillByDefault(Return(ssl_connection_.get())); + ON_CALL(filter_callbacks_.connection_, ssl()).WillByDefault(Return(ssl_connection_)); + ON_CALL(Const(filter_callbacks_.connection_), ssl()).WillByDefault(Return(ssl_connection_)); filter_callbacks_.connection_.local_address_ = std::make_shared("127.0.0.1"); filter_callbacks_.connection_.remote_address_ = @@ -347,7 +346,7 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan NiceMock random_; NiceMock local_info_; NiceMock factory_context_; - std::unique_ptr ssl_connection_; + std::shared_ptr ssl_connection_; TracingConnectionManagerConfigPtr tracing_config_; SlowDateProviderImpl date_provider_{test_time_.timeSystem()}; MockStream stream_; @@ -2237,7 +2236,7 @@ TEST_F(HttpConnectionManagerImplTest, DrainClose) { EXPECT_CALL(drain_close_, drainClose()).WillOnce(Return(true)); EXPECT_CALL(*codec_, shutdownNotice()); filter->callbacks_->encodeHeaders(std::move(response_headers), true); - EXPECT_EQ(ssl_connection_.get(), filter->callbacks_->connection()->ssl()); + EXPECT_EQ(ssl_connection_.get(), filter->callbacks_->connection()->ssl().get()); EXPECT_CALL(*codec_, goAway()); EXPECT_CALL(filter_callbacks_.connection_, @@ -4139,7 +4138,8 @@ TEST_F(HttpConnectionManagerImplTest, MultipleFilters) { .WillOnce(InvokeWithoutArgs([&]() -> FilterHeadersStatus { EXPECT_EQ(route_config_provider_.route_config_->route_, decoder_filters_[0]->callbacks_->route()); - EXPECT_EQ(ssl_connection_.get(), decoder_filters_[0]->callbacks_->connection()->ssl()); + EXPECT_EQ(ssl_connection_.get(), + decoder_filters_[0]->callbacks_->connection()->ssl().get()); return FilterHeadersStatus::StopIteration; })); @@ -4159,7 +4159,8 @@ TEST_F(HttpConnectionManagerImplTest, MultipleFilters) { .WillOnce(InvokeWithoutArgs([&]() -> FilterHeadersStatus { EXPECT_EQ(route_config_provider_.route_config_->route_, decoder_filters_[1]->callbacks_->route()); - EXPECT_EQ(ssl_connection_.get(), decoder_filters_[1]->callbacks_->connection()->ssl()); + EXPECT_EQ(ssl_connection_.get(), + decoder_filters_[1]->callbacks_->connection()->ssl().get()); return FilterHeadersStatus::StopIteration; })); EXPECT_CALL(*decoder_filters_[1], decodeData(_, true)) @@ -4180,14 +4181,14 @@ TEST_F(HttpConnectionManagerImplTest, MultipleFilters) { EXPECT_CALL(*encoder_filters_[1], encodeTrailers(_)) .WillOnce(Return(FilterTrailersStatus::StopIteration)); EXPECT_CALL(*encoder_filters_[1], encodeComplete()); - EXPECT_EQ(ssl_connection_.get(), encoder_filters_[1]->callbacks_->connection()->ssl()); + EXPECT_EQ(ssl_connection_.get(), encoder_filters_[1]->callbacks_->connection()->ssl().get()); decoder_filters_[2]->callbacks_->encodeHeaders( HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}}}, false); Buffer::OwnedImpl response_body("response"); decoder_filters_[2]->callbacks_->encodeData(response_body, false); decoder_filters_[2]->callbacks_->encodeTrailers( HeaderMapPtr{new TestHeaderMapImpl{{"some", "trailer"}}}); - EXPECT_EQ(ssl_connection_.get(), decoder_filters_[2]->callbacks_->connection()->ssl()); + EXPECT_EQ(ssl_connection_.get(), decoder_filters_[2]->callbacks_->connection()->ssl().get()); // Now finish the encode. EXPECT_CALL(*encoder_filters_[0], encodeHeaders(_, false)) @@ -4203,7 +4204,7 @@ TEST_F(HttpConnectionManagerImplTest, MultipleFilters) { expectOnDestroy(); encoder_filters_[1]->callbacks_->continueEncoding(); - EXPECT_EQ(ssl_connection_.get(), encoder_filters_[0]->callbacks_->connection()->ssl()); + EXPECT_EQ(ssl_connection_.get(), encoder_filters_[0]->callbacks_->connection()->ssl().get()); } TEST(HttpConnectionManagerTracingStatsTest, verifyTracingStats) { diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 67b3e412583d..1eb5ab52e5fc 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -776,9 +776,9 @@ TEST_F(ConnectionManagerUtilityTest, MutateResponseHeadersReturnXRequestId) { // Test full sanitization of x-forwarded-client-cert. TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeClientCert) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(true)); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(true)); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::Sanitize)); std::vector details; @@ -792,9 +792,9 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeClientCert) { // Test that we sanitize and set x-forwarded-client-cert. TEST_F(ConnectionManagerUtilityTest, MtlsForwardOnlyClientCert) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(true)); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(true)); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::ForwardOnly)); std::vector details; @@ -811,22 +811,22 @@ TEST_F(ConnectionManagerUtilityTest, MtlsForwardOnlyClientCert) { // The server (local) identity is foo.com/be. The client does not set XFCC. TEST_F(ConnectionManagerUtilityTest, MtlsSetForwardClientCert) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(true)); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(true)); const std::vector local_uri_sans{"test://foo.com/be"}; - EXPECT_CALL(ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); + EXPECT_CALL(*ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); std::string expected_sha("abcdefg"); - EXPECT_CALL(ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); + EXPECT_CALL(*ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); const std::vector peer_uri_sans{"test://foo.com/fe"}; - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); std::string expected_pem("%3D%3Dabc%0Ade%3D"); - EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); + EXPECT_CALL(*ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); std::string expected_chain_pem(expected_pem + "%3D%3Dlmn%0Aop%3D"); - EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificateChain()) + EXPECT_CALL(*ssl, urlEncodedPemEncodedPeerCertificateChain()) .WillOnce(ReturnRef(expected_chain_pem)); std::vector expected_dns = {"www.example.com"}; - EXPECT_CALL(ssl, dnsSansPeerCertificate()).WillOnce(Return(expected_dns)); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + EXPECT_CALL(*ssl, dnsSansPeerCertificate()).WillOnce(Return(expected_dns)); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::AppendForward)); std::vector details = std::vector(); @@ -854,22 +854,22 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSetForwardClientCert) { // also sends the XFCC header with the authentication result of the previous hop, (bar.com/be // calling foo.com/fe). TEST_F(ConnectionManagerUtilityTest, MtlsAppendForwardClientCert) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(true)); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(true)); const std::vector local_uri_sans{"test://foo.com/be"}; - EXPECT_CALL(ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); + EXPECT_CALL(*ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); std::string expected_sha("abcdefg"); - EXPECT_CALL(ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); + EXPECT_CALL(*ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); const std::vector peer_uri_sans{"test://foo.com/fe"}; - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); std::string expected_pem("%3D%3Dabc%0Ade%3D"); - EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); + EXPECT_CALL(*ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); std::string expected_chain_pem(expected_pem + "%3D%3Dlmn%0Aop%3D"); - EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificateChain()) + EXPECT_CALL(*ssl, urlEncodedPemEncodedPeerCertificateChain()) .WillOnce(ReturnRef(expected_chain_pem)); std::vector expected_dns = {"www.example.com"}; - EXPECT_CALL(ssl, dnsSansPeerCertificate()).WillOnce(Return(expected_dns)); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + EXPECT_CALL(*ssl, dnsSansPeerCertificate()).WillOnce(Return(expected_dns)); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::AppendForward)); std::vector details = std::vector(); @@ -897,14 +897,14 @@ TEST_F(ConnectionManagerUtilityTest, MtlsAppendForwardClientCert) { // also sends the XFCC header with the authentication result of the previous hop, (bar.com/be // calling foo.com/fe). TEST_F(ConnectionManagerUtilityTest, MtlsAppendForwardClientCertLocalSanEmpty) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(true)); - EXPECT_CALL(ssl, uriSanLocalCertificate()).WillOnce(Return(std::vector())); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(true)); + EXPECT_CALL(*ssl, uriSanLocalCertificate()).WillOnce(Return(std::vector())); std::string expected_sha("abcdefg"); - EXPECT_CALL(ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); + EXPECT_CALL(*ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); const std::vector peer_uri_sans{"test://foo.com/fe"}; - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::AppendForward)); std::vector details = std::vector(); @@ -926,22 +926,22 @@ TEST_F(ConnectionManagerUtilityTest, MtlsAppendForwardClientCertLocalSanEmpty) { // also sends the XFCC header with the authentication result of the previous hop, (bar.com/be // calling foo.com/fe). TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCert) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(true)); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(true)); const std::vector local_uri_sans{"test://foo.com/be"}; - EXPECT_CALL(ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); + EXPECT_CALL(*ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); std::string expected_sha("abcdefg"); - EXPECT_CALL(ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); - EXPECT_CALL(ssl, subjectPeerCertificate()) + EXPECT_CALL(*ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); + EXPECT_CALL(*ssl, subjectPeerCertificate()) .WillOnce(Return("/C=US/ST=CA/L=San Francisco/OU=Lyft/CN=test.lyft.com")); const std::vector peer_uri_sans{"test://foo.com/fe"}; - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); std::string expected_pem("abcde="); - EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); + EXPECT_CALL(*ssl, urlEncodedPemEncodedPeerCertificate()).WillOnce(ReturnRef(expected_pem)); std::string expected_chain_pem(expected_pem + "lmnop="); - EXPECT_CALL(ssl, urlEncodedPemEncodedPeerCertificateChain()) + EXPECT_CALL(*ssl, urlEncodedPemEncodedPeerCertificateChain()) .WillOnce(ReturnRef(expected_chain_pem)); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::SanitizeSet)); std::vector details = std::vector(); @@ -967,16 +967,16 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCert) { // also sends the XFCC header with the authentication result of the previous hop, (bar.com/be // calling foo.com/fe). TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCertPeerSanEmpty) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(true)); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(true)); const std::vector local_uri_sans{"test://foo.com/be"}; - EXPECT_CALL(ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); + EXPECT_CALL(*ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); std::string expected_sha("abcdefg"); - EXPECT_CALL(ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); - EXPECT_CALL(ssl, subjectPeerCertificate()) + EXPECT_CALL(*ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); + EXPECT_CALL(*ssl, subjectPeerCertificate()) .WillOnce(Return("/C=US/ST=CA/L=San Francisco/OU=Lyft/CN=test.lyft.com")); - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(std::vector())); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(std::vector())); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::SanitizeSet)); std::vector details = std::vector(); @@ -996,9 +996,9 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCertPeerSanEmpty) { // forward_only, append_forward and sanitize_set are only effective in mTLS connection. TEST_F(ConnectionManagerUtilityTest, TlsSanitizeClientCertWhenForward) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(false)); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(false)); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::ForwardOnly)); std::vector details; @@ -1012,9 +1012,9 @@ TEST_F(ConnectionManagerUtilityTest, TlsSanitizeClientCertWhenForward) { // always_forward_only works regardless whether the connection is TLS/mTLS. TEST_F(ConnectionManagerUtilityTest, TlsAlwaysForwardOnlyClientCert) { - NiceMock ssl; - ON_CALL(ssl, peerCertificatePresented()).WillByDefault(Return(false)); - ON_CALL(connection_, ssl()).WillByDefault(Return(&ssl)); + auto ssl = std::make_shared>(); + ON_CALL(*ssl, peerCertificatePresented()).WillByDefault(Return(false)); + ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) .WillByDefault(Return(Http::ForwardClientCertType::AlwaysForwardOnly)); std::vector details; diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index e6c7c6aa7f45..3cbe129caea2 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -89,28 +89,28 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithProtocolVariable) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerUriSanVariableSingleSan) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); const std::vector sans{"san"}; - ON_CALL(connection_info, uriSanPeerCertificate()).WillByDefault(Return(sans)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + ON_CALL(*connection_info, uriSanPeerCertificate()).WillByDefault(Return(sans)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_URI_SAN", "san"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerUriSanVariableMultipleSans) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); const std::vector sans{"san1", "san2"}; - ON_CALL(connection_info, uriSanPeerCertificate()).WillByDefault(Return(sans)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + ON_CALL(*connection_info, uriSanPeerCertificate()).WillByDefault(Return(sans)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_URI_SAN", "san1,san2"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerUriSanEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, uriSanPeerCertificate()) + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, uriSanPeerCertificate()) .WillByDefault(Return(std::vector())); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_URI_SAN", EMPTY_STRING); } @@ -122,28 +122,28 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanVariableSingleSan) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); const std::vector sans{"san"}; - ON_CALL(connection_info, uriSanLocalCertificate()).WillByDefault(Return(sans)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + ON_CALL(*connection_info, uriSanLocalCertificate()).WillByDefault(Return(sans)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_LOCAL_URI_SAN", "san"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanVariableMultipleSans) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); const std::vector sans{"san1", "san2"}; - ON_CALL(connection_info, uriSanLocalCertificate()).WillByDefault(Return(sans)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + ON_CALL(*connection_info, uriSanLocalCertificate()).WillByDefault(Return(sans)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_LOCAL_URI_SAN", "san1,san2"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanVariableNoSans) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, uriSanLocalCertificate()) + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, uriSanLocalCertificate()) .WillByDefault(Return(std::vector())); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_LOCAL_URI_SAN", EMPTY_STRING); } @@ -155,17 +155,17 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanNoTls) TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubject) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, subjectLocalCertificate()).WillByDefault(Return("subject")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(Return("subject")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_LOCAL_SUBJECT", "subject"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubjectEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, subjectLocalCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_LOCAL_SUBJECT", EMPTY_STRING); } @@ -177,17 +177,17 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubjectNoTls) TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionId) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, sessionId()).WillByDefault(Return("deadbeef")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, sessionId()).WillByDefault(Return("deadbeef")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_SESSION_ID", "deadbeef"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionIdEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, sessionId()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, sessionId()).WillByDefault(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_SESSION_ID", EMPTY_STRING); } @@ -199,18 +199,18 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionIdNoTls) TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsCipher) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, ciphersuiteString()) + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, ciphersuiteString()) .WillByDefault(Return("TLS_DHE_RSA_WITH_AES_256_GCM_SHA384")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_CIPHER", "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsCipherEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, ciphersuiteString()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, ciphersuiteString()).WillByDefault(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_CIPHER", EMPTY_STRING); } @@ -222,17 +222,17 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsCipherNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersion) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("TLSv1.2")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.2")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_VERSION", "TLSv1.2"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersionEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_VERSION", EMPTY_STRING); } @@ -244,20 +244,20 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersionNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerFingerprint) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); std::string expected_sha = "685a2db593d5f86d346cb1a297009c3b467ad77f1944aa799039a2fb3d531f3f"; - ON_CALL(connection_info, sha256PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + ON_CALL(*connection_info, sha256PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_FINGERPRINT_256", "685a2db593d5f86d346cb1a297009c3b467ad77f1944aa799039a2fb3d531f3f"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerFingerprintEmpty) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); std::string expected_sha; - ON_CALL(connection_info, sha256PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + ON_CALL(*connection_info, sha256PeerCertificateDigest()).WillByDefault(ReturnRef(expected_sha)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_FINGERPRINT_256", EMPTY_STRING); } @@ -269,17 +269,18 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerFingerprintNoT TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerial) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, serialNumberPeerCertificate()).WillByDefault(Return("b8b5ecc898f2124a")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, serialNumberPeerCertificate()) + .WillByDefault(Return("b8b5ecc898f2124a")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_SERIAL", "b8b5ecc898f2124a"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerialEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, serialNumberPeerCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, serialNumberPeerCertificate()).WillByDefault(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_SERIAL", EMPTY_STRING); } @@ -291,20 +292,20 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerialNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuer) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, issuerPeerCertificate()) + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, issuerPeerCertificate()) .WillByDefault( Return("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_ISSUER", "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuerEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, issuerPeerCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, issuerPeerCertificate()).WillByDefault(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_ISSUER", EMPTY_STRING); } @@ -316,20 +317,20 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuerNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubject) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, subjectPeerCertificate()) + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, subjectPeerCertificate()) .WillByDefault( Return("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_SUBJECT", "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubjectEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, subjectPeerCertificate()).WillByDefault(Return("")); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(Return("")); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_SUBJECT", EMPTY_STRING); } @@ -341,21 +342,21 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubjectNoTls) TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCert) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); std::string expected_cert = ""; - ON_CALL(connection_info, urlEncodedPemEncodedPeerCertificate()) + ON_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) .WillByDefault(ReturnRef(expected_cert)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_CERT", expected_cert); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertEmpty) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); std::string expected_cert; - ON_CALL(connection_info, urlEncodedPemEncodedPeerCertificate()) + ON_CALL(*connection_info, urlEncodedPemEncodedPeerCertificate()) .WillByDefault(ReturnRef(expected_cert)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_CERT", EMPTY_STRING); } @@ -367,20 +368,20 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVStart) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); absl::Time abslStartTime = TestUtility::parseTime("Dec 18 01:50:34 2018 GMT", "%b %e %H:%M:%S %Y GMT"); SystemTime startTime = absl::ToChronoTime(abslStartTime); - ON_CALL(connection_info, validFromPeerCertificate()).WillByDefault(Return(startTime)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + ON_CALL(*connection_info, validFromPeerCertificate()).WillByDefault(Return(startTime)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_START", "2018-12-18T01:50:34.000Z"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVStartEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, validFromPeerCertificate()).WillByDefault(Return(absl::nullopt)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, validFromPeerCertificate()).WillByDefault(Return(absl::nullopt)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_START", EMPTY_STRING); } @@ -392,20 +393,20 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVStartNoTl TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVEnd) { NiceMock stream_info; - NiceMock connection_info; + auto connection_info = std::make_shared>(); absl::Time abslStartTime = TestUtility::parseTime("Dec 17 01:50:34 2020 GMT", "%b %e %H:%M:%S %Y GMT"); SystemTime startTime = absl::ToChronoTime(abslStartTime); - ON_CALL(connection_info, expirationPeerCertificate()).WillByDefault(Return(startTime)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + ON_CALL(*connection_info, expirationPeerCertificate()).WillByDefault(Return(startTime)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_END", "2020-12-17T01:50:34.000Z"); } TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerCertVEndEmpty) { NiceMock stream_info; - NiceMock connection_info; - ON_CALL(connection_info, expirationPeerCertificate()).WillByDefault(Return(absl::nullopt)); - EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, expirationPeerCertificate()).WillByDefault(Return(absl::nullopt)); + EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_CERT_V_END", EMPTY_STRING); } diff --git a/test/common/router/header_parser_fuzz_test.cc b/test/common/router/header_parser_fuzz_test.cc index cb62744f52d1..f2ca239baee6 100644 --- a/test/common/router/header_parser_fuzz_test.cc +++ b/test/common/router/header_parser_fuzz_test.cc @@ -20,8 +20,7 @@ DEFINE_PROTO_FUZZER(const test::common::router::TestCase& input) { Router::HeaderParserPtr parser = Router::HeaderParser::configure(headers_to_add, headers_to_remove); Http::HeaderMapImpl header_map; - NiceMock connection_info; - TestStreamInfo test_stream_info = fromStreamInfo(input.stream_info(), &connection_info); + TestStreamInfo test_stream_info = fromStreamInfo(input.stream_info()); parser->evaluateHeaders(header_map, test_stream_info); ENVOY_LOG_MISC(trace, "Success"); } catch (const EnvoyException& e) { diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index a6e67958042f..bc8d462d3bbc 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -204,7 +204,7 @@ class RouterTestBase : public testing::Test { Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder_ = &decoder; EXPECT_CALL(callbacks_.dispatcher_, setTrackedObject(_)).Times(testing::AtLeast(2)); - callbacks.onPoolReady(original_encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(original_encoder_, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); HttpTestUtility::addDefaultHeaders(default_request_headers_); @@ -257,6 +257,7 @@ class RouterTestBase : public testing::Test { Http::HeaderMapPtr redirect_headers_{ new Http::TestHeaderMapImpl{{":status", "302"}, {"location", "http://www.foo.com"}}}; NiceMock span_; + NiceMock upstream_stream_info_; }; class RouterTest : public RouterTestBase { @@ -495,7 +496,7 @@ TEST_F(RouterTest, AddCookie) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return &cancellable_; })); @@ -544,7 +545,7 @@ TEST_F(RouterTest, AddCookieNoDuplicate) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return &cancellable_; })); @@ -591,7 +592,7 @@ TEST_F(RouterTest, AddMultipleCookies) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return &cancellable_; })); @@ -775,7 +776,7 @@ TEST_F(RouterTest, ResponseCodeDetailsSetByUpstream) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -800,7 +801,7 @@ TEST_F(RouterTest, EnvoyUpstreamServiceTime) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -838,7 +839,7 @@ void RouterTestBase::testAppendCluster(absl::optional clu .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -890,7 +891,7 @@ void RouterTestBase::testAppendUpstreamHost( .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1011,7 +1012,7 @@ TEST_F(RouterTestSuppressEnvoyHeaders, EnvoyUpstreamServiceTime) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1039,7 +1040,7 @@ TEST_F(RouterTest, NoRetriesOverflow) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1062,7 +1063,7 @@ TEST_F(RouterTest, NoRetriesOverflow) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); router_.retry_state_->callback_(); @@ -1085,7 +1086,7 @@ TEST_F(RouterTest, ResetDuringEncodeHeaders) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -1115,7 +1116,7 @@ TEST_F(RouterTest, UpstreamTimeout) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(callbacks_.stream_info_, onUpstreamHostSelected(_)) @@ -1158,7 +1159,7 @@ TEST_F(RouterTest, GrpcOkTrailersOnly) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1182,7 +1183,7 @@ TEST_F(RouterTest, GrpcAlreadyExistsTrailersOnly) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1206,7 +1207,7 @@ TEST_F(RouterTest, GrpcInternalTrailersOnly) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1231,7 +1232,7 @@ TEST_F(RouterTest, GrpcDataEndStream) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1258,7 +1259,7 @@ TEST_F(RouterTest, GrpcReset) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1286,7 +1287,7 @@ TEST_F(RouterTest, GrpcOk) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1315,7 +1316,7 @@ TEST_F(RouterTest, GrpcInternal) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1340,7 +1341,7 @@ TEST_F(RouterTest, UpstreamTimeoutWithAltResponse) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(callbacks_.stream_info_, onUpstreamHostSelected(_)) @@ -1383,7 +1384,7 @@ TEST_F(RouterTest, UpstreamPerTryTimeout) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(callbacks_.stream_info_, onUpstreamHostSelected(_)) @@ -1454,7 +1455,7 @@ TEST_F(RouterTest, UpstreamPerTryTimeoutDelayedPoolReady) { EXPECT_EQ(host_address_, host->address()); })); - pool_callbacks->onPoolReady(encoder, cm_.conn_pool_.host_); + pool_callbacks->onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); EXPECT_CALL(callbacks_.stream_info_, setResponseFlag(StreamInfo::ResponseFlag::UpstreamRequestTimeout)); @@ -1507,7 +1508,7 @@ TEST_F(RouterTest, UpstreamPerTryTimeoutExcludesNewStream) { per_try_timeout_ = new Event::MockTimer(&callbacks_.dispatcher_); EXPECT_CALL(*per_try_timeout_, enableTimer(_, _)); // The per try timeout timer should not be started yet. - pool_callbacks->onPoolReady(encoder, cm_.conn_pool_.host_); + pool_callbacks->onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); EXPECT_CALL(encoder.stream_, resetStream(Http::StreamResetReason::LocalReset)); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -1542,7 +1543,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { -> Http::ConnectionPool::Cancellable* { response_decoder1 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -1570,7 +1571,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutFirstRequestSucceeds) { -> Http::ConnectionPool::Cancellable* { response_decoder2 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -1615,7 +1616,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutResetsOnBadHeaders) { -> Http::ConnectionPool::Cancellable* { response_decoder1 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -1643,7 +1644,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutResetsOnBadHeaders) { -> Http::ConnectionPool::Cancellable* { response_decoder2 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -1693,7 +1694,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutThirdRequestSucceeds) { -> Http::ConnectionPool::Cancellable* { response_decoder1 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1724,7 +1725,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutThirdRequestSucceeds) { -> Http::ConnectionPool::Cancellable* { response_decoder2 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -1744,7 +1745,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutThirdRequestSucceeds) { -> Http::ConnectionPool::Cancellable* { response_decoder3 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder3, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder3, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -1786,7 +1787,7 @@ TEST_F(RouterTest, RetryOnlyOnceForSameUpstreamRequest) { -> Http::ConnectionPool::Cancellable* { response_decoder1 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -1815,7 +1816,7 @@ TEST_F(RouterTest, RetryOnlyOnceForSameUpstreamRequest) { -> Http::ConnectionPool::Cancellable* { response_decoder2 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -1850,7 +1851,7 @@ TEST_F(RouterTest, BadHeadersDroppedIfPreviousRetryScheduled) { -> Http::ConnectionPool::Cancellable* { response_decoder1 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -1891,7 +1892,7 @@ TEST_F(RouterTest, BadHeadersDroppedIfPreviousRetryScheduled) { -> Http::ConnectionPool::Cancellable* { response_decoder2 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); router_.retry_state_->callback_(); @@ -1914,7 +1915,7 @@ TEST_F(RouterTest, RetryRequestNotComplete) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(callbacks_.stream_info_, @@ -1948,7 +1949,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutGlobalTimeout) { -> Http::ConnectionPool::Cancellable* { response_decoder1 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -1977,7 +1978,7 @@ TEST_F(RouterTest, HedgedPerTryTimeoutGlobalTimeout) { -> Http::ConnectionPool::Cancellable* { response_decoder2 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -2014,7 +2015,7 @@ TEST_F(RouterTest, HedgingRetriesExhaustedBadResponse) { -> Http::ConnectionPool::Cancellable* { response_decoder1 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -2043,7 +2044,7 @@ TEST_F(RouterTest, HedgingRetriesExhaustedBadResponse) { -> Http::ConnectionPool::Cancellable* { response_decoder2 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -2094,7 +2095,7 @@ TEST_F(RouterTest, HedgingRetriesProceedAfterReset) { -> Http::ConnectionPool::Cancellable* { response_decoder1 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); // First is reset @@ -2127,7 +2128,7 @@ TEST_F(RouterTest, HedgingRetriesProceedAfterReset) { -> Http::ConnectionPool::Cancellable* { response_decoder2 = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -2173,7 +2174,7 @@ TEST_F(RouterTest, HedgingRetryImmediatelyReset) { -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, @@ -2241,7 +2242,7 @@ TEST_F(RouterTest, RetryNoneHealthy) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -2278,7 +2279,7 @@ TEST_F(RouterTest, RetryUpstreamReset) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2301,7 +2302,7 @@ TEST_F(RouterTest, RetryUpstreamReset) { EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, absl::optional(absl::nullopt))); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); router_.retry_state_->callback_(); @@ -2326,7 +2327,7 @@ TEST_F(RouterTest, RetryUpstreamPerTryTimeout) { -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -2354,7 +2355,7 @@ TEST_F(RouterTest, RetryUpstreamPerTryTimeout) { EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, absl::optional(absl::nullopt))); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -2399,7 +2400,7 @@ TEST_F(RouterTest, RetryUpstreamConnectionFailure) { -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); router_.retry_state_->callback_(); @@ -2419,7 +2420,7 @@ TEST_F(RouterTest, DontResetStartedResponseOnUpstreamPerTryTimeout) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -2453,7 +2454,7 @@ TEST_F(RouterTest, RetryUpstreamResetResponseStarted) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2485,7 +2486,7 @@ TEST_F(RouterTest, RetryUpstreamReset100ContinueResponseStarted) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2516,7 +2517,7 @@ TEST_F(RouterTest, RetryUpstream5xx) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2539,7 +2540,7 @@ TEST_F(RouterTest, RetryUpstream5xx) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); router_.retry_state_->callback_(); @@ -2560,7 +2561,7 @@ TEST_F(RouterTest, RetryTimeoutDuringRetryDelay) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2596,7 +2597,7 @@ TEST_F(RouterTest, RetryTimeoutDuringRetryDelayWithUpstreamRequestNoHost) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2643,7 +2644,7 @@ TEST_F(RouterTest, RetryTimeoutDuringRetryDelayWithUpstreamRequestNoHostAltRespo .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2689,7 +2690,7 @@ TEST_F(RouterTest, RetryUpstream5xxNotComplete) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2720,7 +2721,7 @@ TEST_F(RouterTest, RetryUpstream5xxNotComplete) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); ON_CALL(callbacks_, decodingBuffer()).WillByDefault(Return(body_data.get())); @@ -2760,7 +2761,7 @@ TEST_F(RouterTest, RetryUpstreamGrpcCancelled) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2787,7 +2788,7 @@ TEST_F(RouterTest, RetryUpstreamGrpcCancelled) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); router_.retry_state_->callback_(); @@ -2812,7 +2813,7 @@ TEST_F(RouterTest, RetryRespsectsMaxHostSelectionCount) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2847,7 +2848,7 @@ TEST_F(RouterTest, RetryRespsectsMaxHostSelectionCount) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); ON_CALL(callbacks_, decodingBuffer()).WillByDefault(Return(body_data.get())); @@ -2879,7 +2880,7 @@ TEST_F(RouterTest, RetryRespectsRetryHostPredicate) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2914,7 +2915,7 @@ TEST_F(RouterTest, RetryRespectsRetryHostPredicate) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); ON_CALL(callbacks_, decodingBuffer()).WillByDefault(Return(body_data.get())); @@ -3048,13 +3049,13 @@ TEST_F(RouterTest, HttpInternalRedirectSucceeded) { } TEST_F(RouterTest, HttpsInternalRedirectSucceeded) { - Ssl::MockConnectionInfo ssl_connection; + auto ssl_connection = std::make_shared(); enableRedirects(); sendRequest(); redirect_headers_->insertLocation().value(std::string("https://www.foo.com")); - EXPECT_CALL(connection_, ssl()).Times(1).WillOnce(Return(&ssl_connection)); + EXPECT_CALL(connection_, ssl()).Times(1).WillOnce(Return(ssl_connection)); EXPECT_CALL(callbacks_, decodingBuffer()).Times(1); EXPECT_CALL(callbacks_, recreateStream()).Times(1).WillOnce(Return(true)); response_decoder_->decodeHeaders(std::move(redirect_headers_), false); @@ -3077,7 +3078,7 @@ TEST_F(RouterTest, Shadow) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -3121,7 +3122,7 @@ TEST_F(RouterTest, AltStatName) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -3241,6 +3242,36 @@ TEST_F(RouterTest, DirectResponseWithBody) { EXPECT_EQ(1UL, config_.stats_.rq_direct_response_.value()); } +TEST_F(RouterTest, UpstreamSSLConnection) { + NiceMock encoder; + Http::StreamDecoder* response_decoder = nullptr; + + const auto session_id = "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, sessionId()).WillByDefault(Return(session_id)); + upstream_stream_info_.setDownstreamSslConnection(connection_info); + + expectResponseTimerCreate(); + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + Http::TestHeaderMapImpl headers{}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + Http::HeaderMapPtr response_headers(new Http::TestHeaderMapImpl{{":status", "200"}}); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 0)); + + ASSERT_NE(nullptr, callbacks_.streamInfo().upstreamSslConnection()); + EXPECT_EQ(session_id, callbacks_.streamInfo().upstreamSslConnection()->sessionId()); +} + // Verify that upstream timing information is set into the StreamInfo after the upstream // request completes. TEST_F(RouterTest, UpstreamTimingSingleRequest) { @@ -3250,7 +3281,7 @@ TEST_F(RouterTest, UpstreamTimingSingleRequest) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -3307,7 +3338,7 @@ TEST_F(RouterTest, UpstreamTimingRetry) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -3332,7 +3363,7 @@ TEST_F(RouterTest, UpstreamTimingRetry) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -3387,7 +3418,7 @@ TEST_F(RouterTest, UpstreamTimingTimeout) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -3888,7 +3919,7 @@ TEST_F(RouterTest, CanaryStatusTrue) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -3921,7 +3952,7 @@ TEST_F(RouterTest, CanaryStatusFalse) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -3961,7 +3992,7 @@ TEST_F(RouterTest, AutoHostRewriteEnabled) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](Http::StreamDecoder&, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -3996,7 +4027,7 @@ TEST_F(RouterTest, AutoHostRewriteDisabled) { EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) .WillOnce(Invoke([&](Http::StreamDecoder&, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -4052,7 +4083,7 @@ class WatermarkTest : public RouterTest { response_decoder_ = &decoder; pool_callbacks_ = &callbacks; if (pool_ready) { - callbacks.onPoolReady(encoder_, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder_, cm_.conn_pool_.host_, upstream_stream_info_); } return nullptr; })); @@ -4143,7 +4174,7 @@ TEST_F(WatermarkTest, FilterWatermarks) { .value()); EXPECT_CALL(encoder_, encodeData(_, true)) .WillOnce(Invoke([&](Buffer::Instance& data, bool) -> void { data.drain(data.length()); })); - pool_callbacks_->onPoolReady(encoder_, cm_.conn_pool_.host_); + pool_callbacks_->onPoolReady(encoder_, cm_.conn_pool_.host_, upstream_stream_info_); EXPECT_EQ(1U, cm_.thread_local_cluster_.cluster_.info_->stats_store_ .counter("upstream_flow_control_drained_total") .value()); @@ -4163,7 +4194,7 @@ TEST_F(WatermarkTest, RetryRequestNotComplete) { [&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); EXPECT_CALL(callbacks_.stream_info_, @@ -4209,7 +4240,7 @@ TEST_F(RouterTestChildSpan, BasicFlow) { -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; EXPECT_CALL(*child_span, injectContext(_)); - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); @@ -4241,7 +4272,7 @@ TEST_F(RouterTestChildSpan, ResetFlow) { -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; EXPECT_CALL(*child_span, injectContext(_)); - callbacks.onPoolReady(encoder, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); diff --git a/test/common/router/router_upstream_log_test.cc b/test/common/router/router_upstream_log_test.cc index 554aa3c198f5..0b74b3e068d3 100644 --- a/test/common/router/router_upstream_log_test.cc +++ b/test/common/router/router_upstream_log_test.cc @@ -129,7 +129,8 @@ class RouterUpstreamLogTest : public testing::Test { [&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder, context_.cluster_manager_.conn_pool_.host_); + callbacks.onPoolReady(encoder, context_.cluster_manager_.conn_pool_.host_, + stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -161,7 +162,8 @@ class RouterUpstreamLogTest : public testing::Test { [&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, context_.cluster_manager_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, context_.cluster_manager_.conn_pool_.host_, + stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -187,7 +189,8 @@ class RouterUpstreamLogTest : public testing::Test { response_decoder = &decoder; EXPECT_CALL(context_.cluster_manager_.conn_pool_.host_->outlier_detector_, putResult(Upstream::Outlier::Result::LOCAL_ORIGIN_CONNECT_SUCCESS, _)); - callbacks.onPoolReady(encoder2, context_.cluster_manager_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, context_.cluster_manager_.conn_pool_.host_, + stream_info_); return nullptr; })); expectPerTryTimerCreate(); @@ -214,6 +217,7 @@ class RouterUpstreamLogTest : public testing::Test { NiceMock callbacks_; std::shared_ptr config_; std::shared_ptr router_; + NiceMock stream_info_; }; TEST_F(RouterUpstreamLogTest, NoLogConfigured) { diff --git a/test/common/stream_info/test_util.h b/test/common/stream_info/test_util.h index a532ba1a8dc8..83627789c3dc 100644 --- a/test/common/stream_info/test_util.h +++ b/test/common/stream_info/test_util.h @@ -85,13 +85,22 @@ class TestStreamInfo : public StreamInfo::StreamInfo { return downstream_remote_address_; } - void setDownstreamSslConnection(const Ssl::ConnectionInfo* connection_info) override { + void + setDownstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& connection_info) override { downstream_connection_info_ = connection_info; } - const Ssl::ConnectionInfo* downstreamSslConnection() const override { + Ssl::ConnectionInfoConstSharedPtr downstreamSslConnection() const override { return downstream_connection_info_; } + + void setUpstreamSslConnection(const Ssl::ConnectionInfoConstSharedPtr& connection_info) override { + upstream_connection_info_ = connection_info; + } + + Ssl::ConnectionInfoConstSharedPtr upstreamSslConnection() const override { + return upstream_connection_info_; + } void setRouteName(absl::string_view route_name) override { route_name_ = std::string(route_name); } @@ -207,7 +216,8 @@ class TestStreamInfo : public StreamInfo::StreamInfo { Network::Address::InstanceConstSharedPtr downstream_local_address_; Network::Address::InstanceConstSharedPtr downstream_direct_remote_address_; Network::Address::InstanceConstSharedPtr downstream_remote_address_; - const Ssl::ConnectionInfo* downstream_connection_info_{}; + Ssl::ConnectionInfoConstSharedPtr downstream_connection_info_; + Ssl::ConnectionInfoConstSharedPtr upstream_connection_info_; const Router::RouteEntry* route_entry_{}; envoy::api::v2::core::Metadata metadata_{}; Envoy::StreamInfo::FilterStateImpl filter_state_{}; diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index 88d21bfdc4b6..a7a5b415d867 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -947,9 +947,9 @@ TEST_F(TcpProxyTest, AccessLogPeerUriSan) { Network::Utility::resolveUrl("tcp://1.1.1.1:40000"); const std::vector uriSan{"someSan"}; - Ssl::MockConnectionInfo mockConnectionInfo; - EXPECT_CALL(mockConnectionInfo, uriSanPeerCertificate()).WillOnce(Return(uriSan)); - EXPECT_CALL(filter_callbacks_.connection_, ssl()).WillRepeatedly(Return(&mockConnectionInfo)); + auto mockConnectionInfo = std::make_shared(); + EXPECT_CALL(*mockConnectionInfo, uriSanPeerCertificate()).WillOnce(Return(uriSan)); + EXPECT_CALL(filter_callbacks_.connection_, ssl()).WillRepeatedly(Return(mockConnectionInfo)); setup(1, accessLogConfig("%DOWNSTREAM_PEER_URI_SAN%")); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); @@ -966,9 +966,9 @@ TEST_F(TcpProxyTest, AccessLogTlsSessionId) { const std::string tlsSessionId{ "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"}; - Ssl::MockConnectionInfo mockConnectionInfo; - EXPECT_CALL(mockConnectionInfo, sessionId()).WillOnce(Return(tlsSessionId)); - EXPECT_CALL(filter_callbacks_.connection_, ssl()).WillRepeatedly(Return(&mockConnectionInfo)); + auto mockConnectionInfo = std::make_shared(); + EXPECT_CALL(*mockConnectionInfo, sessionId()).WillOnce(Return(tlsSessionId)); + EXPECT_CALL(filter_callbacks_.connection_, ssl()).WillRepeatedly(Return(mockConnectionInfo)); setup(1, accessLogConfig("%DOWNSTREAM_TLS_SESSION_ID%")); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose); @@ -1010,6 +1010,21 @@ TEST_F(TcpProxyTest, AccessLogBytesRxTxDuration) { "bytesreceived=1 bytessent=2 datetime=[0-9-]+T[0-9:.]+Z nonzeronum=[1-9][0-9]*")); } +TEST_F(TcpProxyTest, AccessLogUpstreamSSLConnection) { + setup(1); + + NiceMock stream_info; + const std::string session_id = "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; + auto ssl_info = std::make_shared(); + EXPECT_CALL(*ssl_info, sessionId()).WillRepeatedly(Return(session_id)); + stream_info.setDownstreamSslConnection(ssl_info); + EXPECT_CALL(*upstream_connections_.at(0), streamInfo()).WillRepeatedly(ReturnRef(stream_info)); + + raiseEventUpstreamConnected(0); + ASSERT_NE(nullptr, filter_->getStreamInfo().upstreamSslConnection()); + EXPECT_EQ(session_id, filter_->getStreamInfo().upstreamSslConnection()->sessionId()); +} + // Tests that upstream flush works properly with no idle timeout configured. TEST_F(TcpProxyTest, UpstreamFlushNoTimeout) { setup(1); diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 893d23d61243..1c5575186ab3 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -305,18 +305,18 @@ response: {} stream_info.host_ = nullptr; stream_info.start_time_ = SystemTime(1h); - NiceMock connection_info; + auto connection_info = std::make_shared>(); const std::vector peerSans{"peerSan1", "peerSan2"}; - ON_CALL(connection_info, uriSanPeerCertificate()).WillByDefault(Return(peerSans)); + ON_CALL(*connection_info, uriSanPeerCertificate()).WillByDefault(Return(peerSans)); const std::vector localSans{"localSan1", "localSan2"}; - ON_CALL(connection_info, uriSanLocalCertificate()).WillByDefault(Return(localSans)); - ON_CALL(connection_info, subjectPeerCertificate()).WillByDefault(Return("peerSubject")); - ON_CALL(connection_info, subjectLocalCertificate()).WillByDefault(Return("localSubject")); - ON_CALL(connection_info, sessionId()) + ON_CALL(*connection_info, uriSanLocalCertificate()).WillByDefault(Return(localSans)); + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(Return("peerSubject")); + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(Return("localSubject")); + ON_CALL(*connection_info, sessionId()) .WillByDefault(Return("D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B")); - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("TLSv1.3")); - ON_CALL(connection_info, ciphersuiteId()).WillByDefault(Return(0x2CC0)); - stream_info.setDownstreamSslConnection(&connection_info); + ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.3")); + ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2CC0)); + stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; Http::TestHeaderMapImpl request_headers{ @@ -364,10 +364,10 @@ response: {} stream_info.host_ = nullptr; stream_info.start_time_ = SystemTime(1h); - NiceMock connection_info; - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("TLSv1.2")); - ON_CALL(connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); - stream_info.setDownstreamSslConnection(&connection_info); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.2")); + ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); + stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; Http::TestHeaderMapImpl request_headers{ @@ -405,10 +405,10 @@ response: {} stream_info.host_ = nullptr; stream_info.start_time_ = SystemTime(1h); - NiceMock connection_info; - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("TLSv1.1")); - ON_CALL(connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); - stream_info.setDownstreamSslConnection(&connection_info); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.1")); + ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); + stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; Http::TestHeaderMapImpl request_headers{ @@ -446,10 +446,10 @@ response: {} stream_info.host_ = nullptr; stream_info.start_time_ = SystemTime(1h); - NiceMock connection_info; - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("TLSv1")); - ON_CALL(connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); - stream_info.setDownstreamSslConnection(&connection_info); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1")); + ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); + stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; Http::TestHeaderMapImpl request_headers{ @@ -487,10 +487,10 @@ response: {} stream_info.host_ = nullptr; stream_info.start_time_ = SystemTime(1h); - NiceMock connection_info; - ON_CALL(connection_info, tlsVersion()).WillByDefault(Return("TLSv1.4")); - ON_CALL(connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); - stream_info.setDownstreamSslConnection(&connection_info); + auto connection_info = std::make_shared>(); + ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.4")); + ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); + stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; Http::TestHeaderMapImpl request_headers{ diff --git a/test/extensions/filters/common/expr/context_test.cc b/test/extensions/filters/common/expr/context_test.cc index 396a8557533c..d7ac41229d3d 100644 --- a/test/extensions/filters/common/expr/context_test.cc +++ b/test/extensions/filters/common/expr/context_test.cc @@ -256,7 +256,7 @@ TEST(Context, ConnectionAttributes) { NiceMock info; std::shared_ptr> host( new NiceMock()); - NiceMock connection_info; + auto connection_info = std::make_shared>(); ConnectionWrapper connection(info); PeerWrapper source(info, false); PeerWrapper destination(info, true); @@ -270,10 +270,10 @@ TEST(Context, ConnectionAttributes) { const std::string sni_name = "kittens.com"; EXPECT_CALL(info, downstreamLocalAddress()).WillRepeatedly(ReturnRef(local)); EXPECT_CALL(info, downstreamRemoteAddress()).WillRepeatedly(ReturnRef(remote)); - EXPECT_CALL(info, downstreamSslConnection()).WillRepeatedly(Return(&connection_info)); + EXPECT_CALL(info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_CALL(info, upstreamHost()).WillRepeatedly(Return(host)); EXPECT_CALL(info, requestedServerName()).WillRepeatedly(ReturnRef(sni_name)); - EXPECT_CALL(connection_info, peerCertificatePresented()).WillRepeatedly(Return(true)); + EXPECT_CALL(*connection_info, peerCertificatePresented()).WillRepeatedly(Return(true)); EXPECT_CALL(*host, address()).WillRepeatedly(Return(upstream)); { diff --git a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc index bc5df7ec323d..d04b54ad8ebe 100644 --- a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc +++ b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc @@ -28,13 +28,14 @@ class CheckRequestUtilsTest : public testing::Test { addr_ = std::make_shared("1.2.3.4", 1111); protocol_ = Envoy::Http::Protocol::Http10; buffer_ = CheckRequestUtilsTest::newTestBuffer(8192); + ssl_ = std::make_shared>(); }; void expectBasicHttp() { EXPECT_CALL(callbacks_, connection()).Times(2).WillRepeatedly(Return(&connection_)); EXPECT_CALL(connection_, remoteAddress()).WillOnce(ReturnRef(addr_)); EXPECT_CALL(connection_, localAddress()).WillOnce(ReturnRef(addr_)); - EXPECT_CALL(Const(connection_), ssl()).Times(2).WillRepeatedly(Return(&ssl_)); + EXPECT_CALL(Const(connection_), ssl()).Times(2).WillRepeatedly(Return(ssl_)); EXPECT_CALL(callbacks_, streamId()).Times(1).WillOnce(Return(0)); EXPECT_CALL(callbacks_, decodingBuffer()).WillOnce(Return(buffer_.get())); EXPECT_CALL(callbacks_, streamInfo()).Times(3).WillRepeatedly(ReturnRef(req_info_)); @@ -84,7 +85,7 @@ class CheckRequestUtilsTest : public testing::Test { NiceMock callbacks_; NiceMock net_callbacks_; NiceMock connection_; - NiceMock ssl_; + std::shared_ptr> ssl_; NiceMock req_info_; Buffer::InstancePtr buffer_; }; @@ -95,7 +96,7 @@ TEST_F(CheckRequestUtilsTest, BasicTcp) { EXPECT_CALL(net_callbacks_, connection()).Times(2).WillRepeatedly(ReturnRef(connection_)); EXPECT_CALL(connection_, remoteAddress()).WillOnce(ReturnRef(addr_)); EXPECT_CALL(connection_, localAddress()).WillOnce(ReturnRef(addr_)); - EXPECT_CALL(Const(connection_), ssl()).Times(2).WillRepeatedly(Return(&ssl_)); + EXPECT_CALL(Const(connection_), ssl()).Times(2).WillRepeatedly(Return(ssl_)); CheckRequestUtils::createTcpCheck(&net_callbacks_, request); } @@ -154,13 +155,34 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithFullBody) { Http::Headers::get().EnvoyAuthPartialBody.get())); } +// Verify that createHttpCheck extract the proper attributes from the http request into CheckRequest +// proto object. +TEST_F(CheckRequestUtilsTest, CheckAttrContextPeer) { + Http::TestHeaderMapImpl request_headers{{"x-envoy-downstream-service-cluster", "foo"}, + {":path", "/bar"}}; + envoy::service::auth::v2::CheckRequest request; + EXPECT_CALL(callbacks_, connection()).WillRepeatedly(Return(&connection_)); + EXPECT_CALL(connection_, remoteAddress()).WillRepeatedly(ReturnRef(addr_)); + EXPECT_CALL(connection_, localAddress()).WillRepeatedly(ReturnRef(addr_)); + EXPECT_CALL(Const(connection_), ssl()).WillRepeatedly(Return(ssl_)); + EXPECT_CALL(callbacks_, streamId()).WillRepeatedly(Return(0)); + EXPECT_CALL(callbacks_, streamInfo()).WillRepeatedly(ReturnRef(req_info_)); + EXPECT_CALL(callbacks_, decodingBuffer()).Times(1); + EXPECT_CALL(req_info_, protocol()).WillRepeatedly(ReturnPointee(&protocol_)); + EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{"source"})); + EXPECT_CALL(*ssl_, uriSanLocalCertificate()) + .WillOnce(Return(std::vector{"destination"})); + + callHttpCheckAndValidateRequestAttributes(); +} + // Verify that createHttpCheck extract the attributes from the HTTP request into CheckRequest // proto object and URI SAN is used as principal if present. TEST_F(CheckRequestUtilsTest, CheckAttrContextPeerUriSans) { expectBasicHttp(); - EXPECT_CALL(ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{"source"})); - EXPECT_CALL(ssl_, uriSanLocalCertificate()) + EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{"source"})); + EXPECT_CALL(*ssl_, uriSanLocalCertificate()) .WillOnce(Return(std::vector{"destination"})); callHttpCheckAndValidateRequestAttributes(); @@ -171,11 +193,11 @@ TEST_F(CheckRequestUtilsTest, CheckAttrContextPeerUriSans) { TEST_F(CheckRequestUtilsTest, CheckAttrContextPeerDnsSans) { expectBasicHttp(); - EXPECT_CALL(ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{})); - EXPECT_CALL(ssl_, dnsSansPeerCertificate()).WillOnce(Return(std::vector{"source"})); + EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(*ssl_, dnsSansPeerCertificate()).WillOnce(Return(std::vector{"source"})); - EXPECT_CALL(ssl_, uriSanLocalCertificate()).WillOnce(Return(std::vector{})); - EXPECT_CALL(ssl_, dnsSansLocalCertificate()) + EXPECT_CALL(*ssl_, uriSanLocalCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(*ssl_, dnsSansLocalCertificate()) .WillOnce(Return(std::vector{"destination"})); Protobuf::Map context_extensions; @@ -189,13 +211,13 @@ TEST_F(CheckRequestUtilsTest, CheckAttrContextPeerDnsSans) { TEST_F(CheckRequestUtilsTest, CheckAttrContextSubject) { expectBasicHttp(); - EXPECT_CALL(ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{})); - EXPECT_CALL(ssl_, dnsSansPeerCertificate()).WillOnce(Return(std::vector{})); - EXPECT_CALL(ssl_, subjectPeerCertificate()).WillOnce(Return("source")); + EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(*ssl_, dnsSansPeerCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(*ssl_, subjectPeerCertificate()).WillOnce(Return("source")); - EXPECT_CALL(ssl_, uriSanLocalCertificate()).WillOnce(Return(std::vector{})); - EXPECT_CALL(ssl_, dnsSansLocalCertificate()).WillOnce(Return(std::vector{})); - EXPECT_CALL(ssl_, subjectLocalCertificate()).WillOnce(Return("destination")); + EXPECT_CALL(*ssl_, uriSanLocalCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(*ssl_, dnsSansLocalCertificate()).WillOnce(Return(std::vector{})); + EXPECT_CALL(*ssl_, subjectLocalCertificate()).WillOnce(Return("destination")); callHttpCheckAndValidateRequestAttributes(); } diff --git a/test/extensions/filters/common/lua/wrappers_test.cc b/test/extensions/filters/common/lua/wrappers_test.cc index 926db9ce3d5e..8d4a9af87ec2 100644 --- a/test/extensions/filters/common/lua/wrappers_test.cc +++ b/test/extensions/filters/common/lua/wrappers_test.cc @@ -35,6 +35,7 @@ class LuaConnectionWrapperTest : public LuaWrappersTestBase { void setup(const std::string& script) override { LuaWrappersTestBase::setup(script); state_->registerType(); + ssl_ = std::make_shared>(); } protected: @@ -53,17 +54,17 @@ class LuaConnectionWrapperTest : public LuaWrappersTestBase { setup(SCRIPT); // Setup secure connection if required. - EXPECT_CALL(Const(connection_), ssl()).WillOnce(Return(secure ? &ssl_ : nullptr)); + EXPECT_CALL(Const(connection_), ssl()).WillOnce(Return(secure ? ssl_ : nullptr)); ConnectionWrapper::create(coroutine_->luaState(), &connection_); EXPECT_CALL(*this, testPrint(secure ? "secure" : "plain")); - EXPECT_CALL(Const(connection_), ssl()).WillOnce(Return(secure ? &ssl_ : nullptr)); + EXPECT_CALL(Const(connection_), ssl()).WillOnce(Return(secure ? ssl_ : nullptr)); EXPECT_CALL(*this, testPrint(secure ? "userdata" : "nil")); start("callMe"); } NiceMock connection_; - NiceMock ssl_; + std::shared_ptr> ssl_; }; // Basic buffer wrapper methods test. diff --git a/test/extensions/filters/common/rbac/matchers_test.cc b/test/extensions/filters/common/rbac/matchers_test.cc index 649e0776a136..7139ad1d1847 100644 --- a/test/extensions/filters/common/rbac/matchers_test.cc +++ b/test/extensions/filters/common/rbac/matchers_test.cc @@ -191,11 +191,11 @@ TEST(PortMatcher, PortMatcher) { TEST(AuthenticatedMatcher, uriSanPeerCertificate) { Envoy::Network::MockConnection conn; - Envoy::Ssl::MockConnectionInfo ssl; + auto ssl = std::make_shared(); const std::vector sans{"foo", "baz"}; - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); - EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(&ssl)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(ssl)); // We should get the first URI SAN. envoy::config::rbac::v2::Principal_Authenticated auth; @@ -208,16 +208,16 @@ TEST(AuthenticatedMatcher, uriSanPeerCertificate) { TEST(AuthenticatedMatcher, dnsSanPeerCertificate) { Envoy::Network::MockConnection conn; - Envoy::Ssl::MockConnectionInfo ssl; + auto ssl = std::make_shared(); const std::vector uri_sans; const std::vector dns_sans{"foo", "baz"}; - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(uri_sans)); - EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(&ssl)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(uri_sans)); + EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(ssl)); - EXPECT_CALL(ssl, dnsSansPeerCertificate()).WillRepeatedly(Return(dns_sans)); - EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(&ssl)); + EXPECT_CALL(*ssl, dnsSansPeerCertificate()).WillRepeatedly(Return(dns_sans)); + EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(ssl)); // We should get the first DNS SAN as URI SAN is not available. envoy::config::rbac::v2::Principal_Authenticated auth; @@ -230,13 +230,13 @@ TEST(AuthenticatedMatcher, dnsSanPeerCertificate) { TEST(AuthenticatedMatcher, subjectPeerCertificate) { Envoy::Network::MockConnection conn; - Envoy::Ssl::MockConnectionInfo ssl; + auto ssl = std::make_shared(); const std::vector sans; - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); - EXPECT_CALL(ssl, dnsSansPeerCertificate()).WillRepeatedly(Return(sans)); - EXPECT_CALL(ssl, subjectPeerCertificate()).WillRepeatedly(Return("bar")); - EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(&ssl)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(*ssl, dnsSansPeerCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(*ssl, subjectPeerCertificate()).WillRepeatedly(Return("bar")); + EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(ssl)); envoy::config::rbac::v2::Principal_Authenticated auth; auth.mutable_principal_name()->set_exact("bar"); @@ -248,10 +248,10 @@ TEST(AuthenticatedMatcher, subjectPeerCertificate) { TEST(AuthenticatedMatcher, AnySSLSubject) { Envoy::Network::MockConnection conn; - Envoy::Ssl::MockConnectionInfo ssl; + auto ssl = std::make_shared(); const std::vector sans{"foo", "baz"}; - EXPECT_CALL(ssl, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); - EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(&ssl)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); + EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(ssl)); envoy::config::rbac::v2::Principal_Authenticated auth; checkMatcher(AuthenticatedMatcher(auth), true, conn); @@ -297,13 +297,13 @@ TEST(PolicyMatcher, PolicyMatcher) { RBAC::PolicyMatcher matcher(policy, nullptr); Envoy::Network::MockConnection conn; - Envoy::Ssl::MockConnectionInfo ssl; + auto ssl = std::make_shared(); Envoy::Network::Address::InstanceConstSharedPtr addr = Envoy::Network::Utility::parseInternetAddress("1.2.3.4", 456, false); const std::vector sans{"bar", "baz"}; - EXPECT_CALL(ssl, uriSanPeerCertificate()).Times(2).WillRepeatedly(Return(sans)); - EXPECT_CALL(Const(conn), ssl()).Times(2).WillRepeatedly(Return(&ssl)); + EXPECT_CALL(*ssl, uriSanPeerCertificate()).Times(2).WillRepeatedly(Return(sans)); + EXPECT_CALL(Const(conn), ssl()).Times(2).WillRepeatedly(Return(ssl)); EXPECT_CALL(conn, localAddress()).Times(2).WillRepeatedly(ReturnRef(addr)); checkMatcher(matcher, true, conn); diff --git a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc index 076fca7e3511..2c2afbf12177 100644 --- a/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc +++ b/test/extensions/filters/http/dynamic_forward_proxy/proxy_filter_integration_test.cc @@ -198,9 +198,9 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTls) { auto response = codec_client_->makeHeaderOnlyRequest(request_headers); waitForNextUpstreamRequest(); - const Extensions::TransportSockets::Tls::SslSocket* ssl_socket = - dynamic_cast( - fake_upstream_connection_->connection().ssl()); + const Extensions::TransportSockets::Tls::SslSocketInfo* ssl_socket = + dynamic_cast( + fake_upstream_connection_->connection().ssl().get()); EXPECT_STREQ("localhost", SSL_get_servername(ssl_socket->rawSslForTest(), TLSEXT_NAMETYPE_host_name)); @@ -224,9 +224,9 @@ TEST_P(ProxyFilterIntegrationTest, UpstreamTlsWithIpHost) { waitForNextUpstreamRequest(); // No SNI for IP hosts. - const Extensions::TransportSockets::Tls::SslSocket* ssl_socket = - dynamic_cast( - fake_upstream_connection_->connection().ssl()); + const Extensions::TransportSockets::Tls::SslSocketInfo* ssl_socket = + dynamic_cast( + fake_upstream_connection_->connection().ssl().get()); EXPECT_STREQ(nullptr, SSL_get_servername(ssl_socket->rawSslForTest(), TLSEXT_NAMETYPE_host_name)); upstream_request_->encodeHeaders(default_response_headers_, true); diff --git a/test/extensions/filters/http/lua/lua_filter_test.cc b/test/extensions/filters/http/lua/lua_filter_test.cc index 95883ce51629..a6bdafe76e14 100644 --- a/test/extensions/filters/http/lua/lua_filter_test.cc +++ b/test/extensions/filters/http/lua/lua_filter_test.cc @@ -79,8 +79,9 @@ class LuaHttpFilterTest : public testing::Test { } void setupSecureConnection(const bool secure) { + ssl_ = std::make_shared>(); EXPECT_CALL(decoder_callbacks_, connection()).WillOnce(Return(&connection_)); - EXPECT_CALL(Const(connection_), ssl()).Times(1).WillOnce(Return(secure ? &ssl_ : nullptr)); + EXPECT_CALL(Const(connection_), ssl()).Times(1).WillOnce(Return(secure ? ssl_ : nullptr)); } void setupMetadata(const std::string& yaml) { @@ -96,7 +97,7 @@ class LuaHttpFilterTest : public testing::Test { Http::MockStreamDecoderFilterCallbacks decoder_callbacks_; Http::MockStreamEncoderFilterCallbacks encoder_callbacks_; envoy::api::v2::core::Metadata metadata_; - NiceMock ssl_; + std::shared_ptr> ssl_; NiceMock connection_; NiceMock stream_info_; diff --git a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc index 48a5d727b714..4c98ceff6df7 100644 --- a/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc +++ b/test/extensions/filters/network/client_ssl_auth/client_ssl_auth_test.cc @@ -56,7 +56,8 @@ class ClientSslAuthFilterTest : public testing::Test { protected: ClientSslAuthFilterTest() : request_(&cm_.async_client_), interval_timer_(new Event::MockTimer(&dispatcher_)), - api_(Api::createApiForTest(stats_store_)) {} + api_(Api::createApiForTest(stats_store_)), + ssl_(std::make_shared()) {} ~ClientSslAuthFilterTest() override { tls_.shutdownThread(); } void setup() { @@ -110,10 +111,10 @@ stat_prefix: vpn std::unique_ptr instance_; Event::MockTimer* interval_timer_; Http::AsyncClient::Callbacks* callbacks_; - Ssl::MockConnectionInfo ssl_; Stats::IsolatedStoreImpl stats_store_; NiceMock random_; Api::ApiPtr api_; + std::shared_ptr ssl_; }; TEST_F(ClientSslAuthFilterTest, NoCluster) { @@ -154,11 +155,11 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { // Create a new filter for an SSL connection, with no backing auth data yet. createAuthFilter(); - ON_CALL(filter_callbacks_.connection_, ssl()).WillByDefault(Return(&ssl_)); + ON_CALL(filter_callbacks_.connection_, ssl()).WillByDefault(Return(ssl_)); filter_callbacks_.connection_.remote_address_ = std::make_shared("192.168.1.1"); std::string expected_sha_1("digest"); - EXPECT_CALL(ssl_, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha_1)); + EXPECT_CALL(*ssl_, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha_1)); EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); EXPECT_EQ(Network::FilterStatus::StopIteration, instance_->onNewConnection()); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::Connected); @@ -182,7 +183,7 @@ TEST_F(ClientSslAuthFilterTest, Ssl) { filter_callbacks_.connection_.remote_address_ = std::make_shared("192.168.1.1"); std::string expected_sha_2("1b7d42ef0025ad89c1c911d6c10d7e86a4cb7c5863b2980abcbad1895f8b5314"); - EXPECT_CALL(ssl_, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha_2)); + EXPECT_CALL(*ssl_, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha_2)); EXPECT_EQ(Network::FilterStatus::StopIteration, instance_->onNewConnection()); EXPECT_CALL(filter_callbacks_, continueReading()); filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::Connected); diff --git a/test/extensions/transport_sockets/tls/ssl_socket_test.cc b/test/extensions/transport_sockets/tls/ssl_socket_test.cc index 4fc6090dde14..aff65278f087 100644 --- a/test/extensions/transport_sockets/tls/ssl_socket_test.cc +++ b/test/extensions/transport_sockets/tls/ssl_socket_test.cc @@ -584,7 +584,8 @@ const std::string testUtilV2(const TestUtilOptionsV2& options) { client_ssl_socket_factory.createTransportSocket(options.transportSocketOptions()), nullptr); if (!options.clientSession().empty()) { - const SslSocket* ssl_socket = dynamic_cast(client_connection->ssl()); + const SslSocketInfo* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL* client_ssl_socket = ssl_socket->rawSslForTest(); SSL_CTX* client_ssl_context = SSL_get_SSL_CTX(client_ssl_socket); SSL_SESSION* client_ssl_session = @@ -629,7 +630,8 @@ const std::string testUtilV2(const TestUtilOptionsV2& options) { EXPECT_EQ(options.expectedALPNProtocol(), client_connection->nextProtocol()); } EXPECT_EQ(options.expectedClientCertUri(), server_connection->ssl()->uriSanPeerCertificate()); - const SslSocket* ssl_socket = dynamic_cast(client_connection->ssl()); + const SslSocketInfo* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL* client_ssl_socket = ssl_socket->rawSslForTest(); if (!options.expectedProtocolVersion().empty()) { EXPECT_EQ(options.expectedProtocolVersion(), client_connection->ssl()->tlsVersion()); @@ -643,7 +645,8 @@ const std::string testUtilV2(const TestUtilOptionsV2& options) { } absl::optional server_ssl_requested_server_name; - const SslSocket* server_ssl_socket = dynamic_cast(server_connection->ssl()); + const SslSocketInfo* server_ssl_socket = + dynamic_cast(server_connection->ssl().get()); SSL* server_ssl = server_ssl_socket->rawSslForTest(); auto requested_server_name = SSL_get_servername(server_ssl, TLSEXT_NAMETYPE_host_name); if (requested_server_name != nullptr) { @@ -2341,7 +2344,8 @@ TEST_P(SslSocketTest, ClientAuthMultipleCAs) { ssl_socket_factory.createTransportSocket(nullptr), nullptr); // Verify that server sent list with 2 acceptable client certificate CA names. - const SslSocket* ssl_socket = dynamic_cast(client_connection->ssl()); + const SslSocketInfo* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL_set_cert_cb( ssl_socket->rawSslForTest(), [](SSL* ssl, void*) -> int { @@ -2460,7 +2464,8 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::Connected)) .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { - const SslSocket* ssl_socket = dynamic_cast(client_connection->ssl()); + const SslSocketInfo* ssl_socket = + dynamic_cast(client_connection->ssl().get()); ssl_session = SSL_get1_session(ssl_socket->rawSslForTest()); EXPECT_TRUE(SSL_SESSION_is_resumable(ssl_session)); client_connection->close(Network::ConnectionCloseType::NoFlush); @@ -2478,7 +2483,8 @@ void testTicketSessionResumption(const std::string& server_ctx_yaml1, socket2.localAddress(), Network::Address::InstanceConstSharedPtr(), ssl_socket_factory.createTransportSocket(nullptr), nullptr); client_connection->addConnectionCallbacks(client_connection_callbacks); - const SslSocket* ssl_socket = dynamic_cast(client_connection->ssl()); + const SslSocketInfo* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL_set_session(ssl_socket->rawSslForTest(), ssl_session); SSL_SESSION_free(ssl_session); @@ -2883,7 +2889,8 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) { EXPECT_CALL(server_connection_callbacks, onEvent(Network::ConnectionEvent::LocalClose)); EXPECT_CALL(client_connection_callbacks, onEvent(Network::ConnectionEvent::Connected)) .WillOnce(Invoke([&](Network::ConnectionEvent) -> void { - const SslSocket* ssl_socket = dynamic_cast(client_connection->ssl()); + const SslSocketInfo* ssl_socket = + dynamic_cast(client_connection->ssl().get()); ssl_session = SSL_get1_session(ssl_socket->rawSslForTest()); EXPECT_TRUE(SSL_SESSION_is_resumable(ssl_session)); server_connection->close(Network::ConnectionCloseType::NoFlush); @@ -2901,7 +2908,8 @@ TEST_P(SslSocketTest, ClientAuthCrossListenerSessionResumption) { socket2.localAddress(), Network::Address::InstanceConstSharedPtr(), ssl_socket_factory.createTransportSocket(nullptr), nullptr); client_connection->addConnectionCallbacks(client_connection_callbacks); - const SslSocket* ssl_socket = dynamic_cast(client_connection->ssl()); + const SslSocketInfo* ssl_socket = + dynamic_cast(client_connection->ssl().get()); SSL_set_session(ssl_socket->rawSslForTest(), ssl_session); SSL_SESSION_free(ssl_session); diff --git a/test/extensions/transport_sockets/tls/utility_test.cc b/test/extensions/transport_sockets/tls/utility_test.cc index 502058490ca2..ccf803faa0d0 100644 --- a/test/extensions/transport_sockets/tls/utility_test.cc +++ b/test/extensions/transport_sockets/tls/utility_test.cc @@ -22,7 +22,7 @@ namespace { TEST(UtilityTest, TestGetSubjectAlternateNamesWithDNS) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_dns_cert.pem")); - const std::vector& subject_alt_names = Utility::getSubjectAltNames(*cert, GEN_DNS); + const auto& subject_alt_names = Utility::getSubjectAltNames(*cert, GEN_DNS); EXPECT_EQ(1, subject_alt_names.size()); } @@ -30,22 +30,21 @@ TEST(UtilityTest, TestMultipleGetSubjectAlternateNamesWithDNS) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir " "}}/test/extensions/transport_sockets/tls/test_data/san_multiple_dns_cert.pem")); - const std::vector& subject_alt_names = Utility::getSubjectAltNames(*cert, GEN_DNS); + const auto& subject_alt_names = Utility::getSubjectAltNames(*cert, GEN_DNS); EXPECT_EQ(2, subject_alt_names.size()); } TEST(UtilityTest, TestGetSubjectAlternateNamesWithUri) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/san_uri_cert.pem")); - const std::vector& subject_alt_names = Utility::getSubjectAltNames(*cert, GEN_URI); + const auto& subject_alt_names = Utility::getSubjectAltNames(*cert, GEN_URI); EXPECT_EQ(1, subject_alt_names.size()); } TEST(UtilityTest, TestGetSubjectAlternateNamesWithNoSAN) { bssl::UniquePtr cert = readCertFromFile(TestEnvironment::substitute( "{{ test_rundir }}/test/extensions/transport_sockets/tls/test_data/no_san_cert.pem")); - const std::vector& uri_subject_alt_names = - Utility::getSubjectAltNames(*cert, GEN_URI); + const auto& uri_subject_alt_names = Utility::getSubjectAltNames(*cert, GEN_URI); EXPECT_EQ(0, uri_subject_alt_names.size()); } diff --git a/test/fuzz/utility.h b/test/fuzz/utility.h index 155226aa026c..2fda4efb8de7 100644 --- a/test/fuzz/utility.h +++ b/test/fuzz/utility.h @@ -103,8 +103,7 @@ inline test::fuzz::Headers toHeaders(const Http::HeaderMap& headers) { return fuzz_headers; } -inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info, - const Ssl::MockConnectionInfo* connection_info) { +inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info) { // Set mocks' default string return value to be an empty string. testing::DefaultValue::Set(EMPTY_STRING); TestStreamInfo test_stream_info; @@ -129,10 +128,11 @@ inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info, test_stream_info.downstream_local_address_ = address; test_stream_info.downstream_direct_remote_address_ = address; test_stream_info.downstream_remote_address_ = address; - test_stream_info.setDownstreamSslConnection(connection_info); + auto connection_info = std::make_shared>(); ON_CALL(*connection_info, subjectPeerCertificate()) .WillByDefault(testing::Return( "CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); + test_stream_info.setDownstreamSslConnection(connection_info); return test_stream_info; } diff --git a/test/mocks/network/connection.h b/test/mocks/network/connection.h index f660269de13a..501fa8a2dfb0 100644 --- a/test/mocks/network/connection.h +++ b/test/mocks/network/connection.h @@ -71,7 +71,7 @@ class MockConnection : public Connection, public MockConnectionBase { absl::optional()); MOCK_CONST_METHOD0(localAddress, const Address::InstanceConstSharedPtr&()); MOCK_METHOD1(setConnectionStats, void(const ConnectionStats& stats)); - MOCK_CONST_METHOD0(ssl, const Ssl::ConnectionInfo*()); + MOCK_CONST_METHOD0(ssl, Ssl::ConnectionInfoConstSharedPtr()); MOCK_CONST_METHOD0(requestedServerName, absl::string_view()); MOCK_CONST_METHOD0(state, State()); MOCK_METHOD2(write, void(Buffer::Instance& data, bool end_stream)); @@ -117,7 +117,7 @@ class MockClientConnection : public ClientConnection, public MockConnectionBase absl::optional()); MOCK_CONST_METHOD0(localAddress, const Address::InstanceConstSharedPtr&()); MOCK_METHOD1(setConnectionStats, void(const ConnectionStats& stats)); - MOCK_CONST_METHOD0(ssl, const Ssl::ConnectionInfo*()); + MOCK_CONST_METHOD0(ssl, Ssl::ConnectionInfoConstSharedPtr()); MOCK_CONST_METHOD0(requestedServerName, absl::string_view()); MOCK_CONST_METHOD0(state, State()); MOCK_METHOD2(write, void(Buffer::Instance& data, bool end_stream)); @@ -166,7 +166,7 @@ class MockFilterManagerConnection : public FilterManagerConnection, public MockC absl::optional()); MOCK_CONST_METHOD0(localAddress, const Address::InstanceConstSharedPtr&()); MOCK_METHOD1(setConnectionStats, void(const ConnectionStats& stats)); - MOCK_CONST_METHOD0(ssl, const Ssl::ConnectionInfo*()); + MOCK_CONST_METHOD0(ssl, Ssl::ConnectionInfoConstSharedPtr()); MOCK_CONST_METHOD0(requestedServerName, absl::string_view()); MOCK_CONST_METHOD0(state, State()); MOCK_METHOD2(write, void(Buffer::Instance& data, bool end_stream)); diff --git a/test/mocks/network/mocks.h b/test/mocks/network/mocks.h index 33931209fffe..ee7551960bf5 100644 --- a/test/mocks/network/mocks.h +++ b/test/mocks/network/mocks.h @@ -398,7 +398,7 @@ class MockTransportSocket : public TransportSocket { MOCK_METHOD1(doRead, IoResult(Buffer::Instance& buffer)); MOCK_METHOD2(doWrite, IoResult(Buffer::Instance& buffer, bool end_stream)); MOCK_METHOD0(onConnected, void()); - MOCK_CONST_METHOD0(ssl, const Ssl::ConnectionInfo*()); + MOCK_CONST_METHOD0(ssl, Ssl::ConnectionInfoConstSharedPtr()); TransportSocketCallbacks* callbacks_{}; }; diff --git a/test/mocks/stream_info/mocks.cc b/test/mocks/stream_info/mocks.cc index b554b0ac30e0..1f4c4a8f7228 100644 --- a/test/mocks/stream_info/mocks.cc +++ b/test/mocks/stream_info/mocks.cc @@ -63,10 +63,16 @@ MockStreamInfo::MockStreamInfo() ON_CALL(*this, downstreamRemoteAddress()).WillByDefault(ReturnRef(downstream_remote_address_)); ON_CALL(*this, setDownstreamSslConnection(_)) .WillByDefault(Invoke( - [this](const auto* connection_info) { downstream_connection_info_ = connection_info; })); + [this](const auto& connection_info) { downstream_connection_info_ = connection_info; })); + ON_CALL(*this, setUpstreamSslConnection(_)) + .WillByDefault(Invoke( + [this](const auto& connection_info) { upstream_connection_info_ = connection_info; })); ON_CALL(*this, downstreamSslConnection()).WillByDefault(Invoke([this]() { return downstream_connection_info_; })); + ON_CALL(*this, upstreamSslConnection()).WillByDefault(Invoke([this]() { + return upstream_connection_info_; + })); ON_CALL(*this, protocol()).WillByDefault(ReturnPointee(&protocol_)); ON_CALL(*this, responseCode()).WillByDefault(ReturnPointee(&response_code_)); ON_CALL(*this, responseCodeDetails()).WillByDefault(ReturnPointee(&response_code_details_)); diff --git a/test/mocks/stream_info/mocks.h b/test/mocks/stream_info/mocks.h index 0fbd34222291..648cafe20a5c 100644 --- a/test/mocks/stream_info/mocks.h +++ b/test/mocks/stream_info/mocks.h @@ -65,8 +65,10 @@ class MockStreamInfo : public StreamInfo { const Network::Address::InstanceConstSharedPtr&()); MOCK_METHOD1(setDownstreamRemoteAddress, void(const Network::Address::InstanceConstSharedPtr&)); MOCK_CONST_METHOD0(downstreamRemoteAddress, const Network::Address::InstanceConstSharedPtr&()); - MOCK_METHOD1(setDownstreamSslConnection, void(const Ssl::ConnectionInfo*)); - MOCK_CONST_METHOD0(downstreamSslConnection, const Ssl::ConnectionInfo*()); + MOCK_METHOD1(setDownstreamSslConnection, void(const Ssl::ConnectionInfoConstSharedPtr&)); + MOCK_CONST_METHOD0(downstreamSslConnection, Ssl::ConnectionInfoConstSharedPtr()); + MOCK_METHOD1(setUpstreamSslConnection, void(const Ssl::ConnectionInfoConstSharedPtr&)); + MOCK_CONST_METHOD0(upstreamSslConnection, Ssl::ConnectionInfoConstSharedPtr()); MOCK_CONST_METHOD0(routeEntry, const Router::RouteEntry*()); MOCK_METHOD0(dynamicMetadata, envoy::api::v2::core::Metadata&()); MOCK_CONST_METHOD0(dynamicMetadata, const envoy::api::v2::core::Metadata&()); @@ -103,7 +105,8 @@ class MockStreamInfo : public StreamInfo { Network::Address::InstanceConstSharedPtr downstream_local_address_; Network::Address::InstanceConstSharedPtr downstream_direct_remote_address_; Network::Address::InstanceConstSharedPtr downstream_remote_address_; - const Ssl::ConnectionInfo* downstream_connection_info_{}; + Ssl::ConnectionInfoConstSharedPtr downstream_connection_info_; + Ssl::ConnectionInfoConstSharedPtr upstream_connection_info_; std::string requested_server_name_; std::string route_name_; std::string upstream_transport_failure_reason_; diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index e5b570919e19..08953751f7cd 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -1437,7 +1437,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithDestinationP auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -1480,7 +1480,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithDestinationI auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -1528,7 +1528,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithServerNamesM auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); } @@ -1567,7 +1567,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithTransportPro auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); } @@ -1607,7 +1607,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithApplicationP auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); } @@ -1648,7 +1648,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithSourceTypeMa auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -1659,7 +1659,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithSourceTypeMa EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - server_names = ssl_socket->dnsSansLocalCertificate(); + server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); } @@ -1702,7 +1702,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithSourceIpMatc auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -1786,7 +1786,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, SingleFilterChainWithSourcePortMa auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -1846,7 +1846,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainWithSourceType auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -1856,7 +1856,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainWithSourceType EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - auto uri = ssl_socket->uriSanLocalCertificate(); + auto uri = ssl_socket->ssl()->uriSanLocalCertificate(); EXPECT_EQ(uri[0], "spiffe://lyft.com/test-team"); // EXTERNAL TLS client without "http/1.1" ALPN - using 3nd filter chain. @@ -1865,7 +1865,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainWithSourceType EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - server_names = ssl_socket->dnsSansLocalCertificate(); + server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 2); EXPECT_EQ(server_names.front(), "*.example.com"); } @@ -1914,7 +1914,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithDestinati auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto uri = ssl_socket->uriSanLocalCertificate(); + auto uri = ssl_socket->ssl()->uriSanLocalCertificate(); EXPECT_EQ(uri[0], "spiffe://lyft.com/test-team"); // IPv4 client connects to port 8080 - using 2nd filter chain. @@ -1923,7 +1923,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithDestinati EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -1933,7 +1933,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithDestinati EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - server_names = ssl_socket->dnsSansLocalCertificate(); + server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 2); EXPECT_EQ(server_names.front(), "*.example.com"); @@ -1943,7 +1943,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithDestinati EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - uri = ssl_socket->uriSanLocalCertificate(); + uri = ssl_socket->ssl()->uriSanLocalCertificate(); EXPECT_EQ(uri[0], "spiffe://lyft.com/test-team"); } @@ -1991,7 +1991,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithDestinati auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto uri = ssl_socket->uriSanLocalCertificate(); + auto uri = ssl_socket->ssl()->uriSanLocalCertificate(); EXPECT_EQ(uri[0], "spiffe://lyft.com/test-team"); // IPv4 client connects to exact IP match - using 2nd filter chain. @@ -2000,7 +2000,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithDestinati EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -2010,7 +2010,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithDestinati EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - server_names = ssl_socket->dnsSansLocalCertificate(); + server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 2); EXPECT_EQ(server_names.front(), "*.example.com"); @@ -2020,7 +2020,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithDestinati EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - uri = ssl_socket->uriSanLocalCertificate(); + uri = ssl_socket->ssl()->uriSanLocalCertificate(); EXPECT_EQ(uri[0], "spiffe://lyft.com/test-team"); } @@ -2077,7 +2077,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithServerNam auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto uri = ssl_socket->uriSanLocalCertificate(); + auto uri = ssl_socket->ssl()->uriSanLocalCertificate(); EXPECT_EQ(uri[0], "spiffe://lyft.com/test-team"); // TLS client with exact SNI match - using 2nd filter chain. @@ -2087,7 +2087,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithServerNam EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); @@ -2098,7 +2098,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithServerNam EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - server_names = ssl_socket->dnsSansLocalCertificate(); + server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 2); EXPECT_EQ(server_names.front(), "*.example.com"); @@ -2109,7 +2109,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithServerNam EXPECT_TRUE(filter_chain->transportSocketFactory().implementsSecureTransport()); transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); ssl_socket = dynamic_cast(transport_socket.get()); - server_names = ssl_socket->dnsSansLocalCertificate(); + server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 2); EXPECT_EQ(server_names.front(), "*.example.com"); } @@ -2151,7 +2151,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithTransport auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); } @@ -2194,7 +2194,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithApplicati auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); } @@ -2250,7 +2250,7 @@ TEST_F(ListenerManagerImplWithRealFiltersTest, MultipleFilterChainsWithMultipleR auto transport_socket = filter_chain->transportSocketFactory().createTransportSocket(nullptr); auto ssl_socket = dynamic_cast(transport_socket.get()); - auto server_names = ssl_socket->dnsSansLocalCertificate(); + auto server_names = ssl_socket->ssl()->dnsSansLocalCertificate(); EXPECT_EQ(server_names.size(), 1); EXPECT_EQ(server_names.front(), "server1.example.com"); } From cc03f7972ae9b5ceed87a917bebdc20c97cfca58 Mon Sep 17 00:00:00 2001 From: Yechiel Kalmenson Date: Fri, 6 Sep 2019 18:28:39 -0400 Subject: [PATCH 513/542] fix windows implementation of PlatformImpl (#8169) Add missing destructor to class declaration. Fix copy/paste errors. These errors were apparently introduced in e1cd4cc. Risk Level: Low Testing: Passed Windows testing locally Docs Changes: n/a Release Notes: n/a Signed-off-by: William Rowe wrowe@pivotal.io Signed-off-by: Yechiel Kalmenson --- source/exe/platform_impl.h | 1 + source/exe/posix/platform_impl.cc | 2 ++ source/exe/win32/platform_impl.cc | 4 +--- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/exe/platform_impl.h b/source/exe/platform_impl.h index c52152c8fe6d..4c05dff22584 100644 --- a/source/exe/platform_impl.h +++ b/source/exe/platform_impl.h @@ -8,6 +8,7 @@ namespace Envoy { class PlatformImpl { public: PlatformImpl(); + ~PlatformImpl(); Thread::ThreadFactory& threadFactory() { return *thread_factory_; } Filesystem::Instance& fileSystem() { return *file_system_; } diff --git a/source/exe/posix/platform_impl.cc b/source/exe/posix/platform_impl.cc index f664071815ee..8fc227724bee 100644 --- a/source/exe/posix/platform_impl.cc +++ b/source/exe/posix/platform_impl.cc @@ -9,4 +9,6 @@ PlatformImpl::PlatformImpl() : thread_factory_(std::make_unique()), file_system_(std::make_unique()) {} +PlatformImpl::~PlatformImpl() = default; + } // namespace Envoy diff --git a/source/exe/win32/platform_impl.cc b/source/exe/win32/platform_impl.cc index 860a209a8e05..674ad0db0b1f 100644 --- a/source/exe/win32/platform_impl.cc +++ b/source/exe/win32/platform_impl.cc @@ -19,8 +19,6 @@ PlatformImpl::PlatformImpl() RELEASE_ASSERT(rc == 0, "WSAStartup failed with error"); } -~PlatformImpl() { ::WSACleanup(); } - -}; // namespace Envoy +PlatformImpl::~PlatformImpl() { ::WSACleanup(); } } // namespace Envoy From 451bfe802772465088b3eb6820936afdee710048 Mon Sep 17 00:00:00 2001 From: Pengyuan Bian Date: Fri, 6 Sep 2019 19:59:01 -0700 Subject: [PATCH 514/542] Update Opencensus SHA (#8173) Signed-off-by: Pengyuan Bian --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 6a28c7026cca..9712e2c093be 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -232,10 +232,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://files.pythonhosted.org/packages/b3/b2/238e2590826bfdd113244a40d9d3eb26918bd798fc187e2360a8367068db/six-1.10.0.tar.gz"], ), io_opencensus_cpp = dict( - sha256 = "b5fcc36a994a4ecb6e53c901e33579ed1ac238cff9975807db760918a15f43ff", - strip_prefix = "opencensus-cpp-8058a1b8efe6a63bd18673abc51223917d12d45d", - # 2019-08-22 - urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/8058a1b8efe6a63bd18673abc51223917d12d45d.tar.gz"], + sha256 = "145e42594db358905737dc07400657be62a2961f4e93ab7f4c9765dd2441033c", + strip_prefix = "opencensus-cpp-cc198ff64569bc47beed5384777a4bb563d268e7", + # 2019-09-04 + urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/cc198ff64569bc47beed5384777a4bb563d268e7.tar.gz"], ), com_github_curl = dict( sha256 = "4376ac72b95572fb6c4fbffefb97c7ea0dd083e1974c0e44cd7e49396f454839", From 6f273c2163a4d2e1c9c94e1d60aa9bec048ea007 Mon Sep 17 00:00:00 2001 From: Zhouyihai Ding Date: Fri, 6 Sep 2019 20:00:09 -0700 Subject: [PATCH 515/542] Outlier Detection: use gRPC status code for detecting failures (#7942) Signed-off-by: ZhouyihaiDing --- .../intro/arch_overview/upstream/outlier.rst | 7 + docs/root/intro/version_history.rst | 1 + source/common/router/router.cc | 31 ++- source/common/router/router.h | 5 +- source/common/runtime/runtime_features.cc | 1 + test/common/router/BUILD | 1 + test/common/router/router_test.cc | 179 +++++++++++++++++- 7 files changed, 213 insertions(+), 12 deletions(-) diff --git a/docs/root/intro/arch_overview/upstream/outlier.rst b/docs/root/intro/arch_overview/upstream/outlier.rst index b8b4fce4b7a1..5b3c392b6a70 100644 --- a/docs/root/intro/arch_overview/upstream/outlier.rst +++ b/docs/root/intro/arch_overview/upstream/outlier.rst @@ -145,6 +145,13 @@ Most configuration items, namely types of errors, but :ref:`outlier_detection.enforcing_success_rate` applies to externally originated errors only and :ref:`outlier_detection.enforcing_local_origin_success_rate` applies to locally originated errors only. +.. _arch_overview_outlier_detection_grpc: + +gRPC +---------------------- + +For gRPC requests, the outlier detection will use the HTTP status mapped from the `grpc-status `_ response header. This behavior is guarded by the runtime feature `envoy.reloadable_features.outlier_detection_support_for_grpc_status` which defaults to true. + .. _arch_overview_outlier_detection_logging: diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 5ce0428bc4d7..d641e1d098db 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -34,6 +34,7 @@ Version history * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. +* outlier_detector: added :ref:`support for the grpc-status response header ` by mapping it to HTTP status. Guarded by envoy.reloadable_features.outlier_detection_support_for_grpc_status which defaults to true. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). * performance: stats symbol table implementation (disabled by default; to test it, add "--use-fake-symbol-table 0" to the command-line arguments when starting Envoy). * rbac: added support for DNS SAN as :ref:`principal_name `. diff --git a/source/common/router/router.cc b/source/common/router/router.cc index e9341fb7b1e8..412bed0392b7 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -27,6 +27,7 @@ #include "common/router/config_impl.h" #include "common/router/debug_config.h" #include "common/router/retry_state_impl.h" +#include "common/runtime/runtime_impl.h" #include "common/tracing/http_tracer_impl.h" #include "extensions/filters/http/well_known_names.h" @@ -931,15 +932,14 @@ Filter::streamResetReasonToResponseFlag(Http::StreamResetReason reset_reason) { NOT_REACHED_GCOVR_EXCL_LINE; } -void Filter::handleNon5xxResponseHeaders(const Http::HeaderMap& headers, - UpstreamRequest& upstream_request, bool end_stream) { +void Filter::handleNon5xxResponseHeaders(absl::optional grpc_status, + UpstreamRequest& upstream_request, bool end_stream, + uint64_t grpc_to_http_status) { // We need to defer gRPC success until after we have processed grpc-status in // the trailers. if (grpc_request_) { if (end_stream) { - absl::optional grpc_status = Grpc::Common::getGrpcStatus(headers); - if (grpc_status && - !Http::CodeUtility::is5xx(Grpc::Utility::grpcToHttpStatus(grpc_status.value()))) { + if (grpc_status && !Http::CodeUtility::is5xx(grpc_to_http_status)) { upstream_request.upstream_host_->stats().rq_success_.inc(); } else { upstream_request.upstream_host_->stats().rq_error_.inc(); @@ -1002,8 +1002,25 @@ void Filter::onUpstreamHeaders(uint64_t response_code, Http::HeaderMapPtr&& head ENVOY_STREAM_LOG(debug, "upstream headers complete: end_stream={}", *callbacks_, end_stream); modify_headers_(*headers); + // When grpc-status appears in response headers, convert grpc-status to HTTP status code + // for outlier detection. This does not currently change any stats or logging and does not + // handle the case when an error grpc-status is sent as a trailer. + absl::optional grpc_status; + uint64_t grpc_to_http_status = 0; + if (grpc_request_) { + grpc_status = Grpc::Common::getGrpcStatus(*headers); + if (grpc_status.has_value()) { + grpc_to_http_status = Grpc::Utility::grpcToHttpStatus(grpc_status.value()); + } + } - upstream_request.upstream_host_->outlierDetector().putHttpResponseCode(response_code); + if (grpc_status.has_value() && + Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.outlier_detection_support_for_grpc_status")) { + upstream_request.upstream_host_->outlierDetector().putHttpResponseCode(grpc_to_http_status); + } else { + upstream_request.upstream_host_->outlierDetector().putHttpResponseCode(response_code); + } if (headers->EnvoyImmediateHealthCheckFail() != nullptr) { upstream_request.upstream_host_->healthChecker().setUnhealthy(); @@ -1090,7 +1107,7 @@ void Filter::onUpstreamHeaders(uint64_t response_code, Http::HeaderMapPtr&& head upstream_request.upstream_host_->canary(); chargeUpstreamCode(response_code, *headers, upstream_request.upstream_host_, false); if (!Http::CodeUtility::is5xx(response_code)) { - handleNon5xxResponseHeaders(*headers, upstream_request, end_stream); + handleNon5xxResponseHeaders(grpc_status, upstream_request, end_stream, grpc_to_http_status); } // Append routing cookies diff --git a/source/common/router/router.h b/source/common/router/router.h index 3585ad888f73..89de83d0cf2a 100644 --- a/source/common/router/router.h +++ b/source/common/router/router.h @@ -536,8 +536,9 @@ class Filter : Logger::Loggable, void doRetry(); // Called immediately after a non-5xx header is received from upstream, performs stats accounting // and handle difference between gRPC and non-gRPC requests. - void handleNon5xxResponseHeaders(const Http::HeaderMap& headers, - UpstreamRequest& upstream_request, bool end_stream); + void handleNon5xxResponseHeaders(absl::optional grpc_status, + UpstreamRequest& upstream_request, bool end_stream, + uint64_t grpc_to_http_status); TimeSource& timeSource() { return config_.timeSource(); } Http::Context& httpContext() { return config_.http_context_; } diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index db42217c8233..6c3076d5466b 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -27,6 +27,7 @@ constexpr const char* runtime_features[] = { "envoy.reloadable_features.test_feature_true", "envoy.reloadable_features.buffer_filter_populate_content_length", "envoy.reloadable_features.trusted_forwarded_proto", + "envoy.reloadable_features.outlier_detection_support_for_grpc_status", }; // This is a list of configuration fields which are disallowed by default in Envoy diff --git a/test/common/router/BUILD b/test/common/router/BUILD index 2ac71059b8bc..48304edb719a 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -247,6 +247,7 @@ envoy_cc_test( "//test/mocks/upstream:upstream_mocks", "//test/test_common:environment_lib", "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", "//test/test_common:utility_lib", ], ) diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index bc8d462d3bbc..de3ae6d3f17f 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -27,6 +27,7 @@ #include "test/test_common/environment.h" #include "test/test_common/printers.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" #include "test/test_common/utility.h" #include "gmock/gmock.h" @@ -1176,7 +1177,10 @@ TEST_F(RouterTest, GrpcOkTrailersOnly) { } // Validate gRPC AlreadyExists response stats are sane when response is trailers only. -TEST_F(RouterTest, GrpcAlreadyExistsTrailersOnly) { +TEST_F(RouterTest, GrpcAlreadyExistsTrailersOnlyRuntimeGuard) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.outlier_detection_support_for_grpc_status", "false"}}); NiceMock encoder1; Http::StreamDecoder* response_decoder = nullptr; EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) @@ -1199,8 +1203,94 @@ TEST_F(RouterTest, GrpcAlreadyExistsTrailersOnly) { EXPECT_TRUE(verifyHostUpstreamStats(1, 0)); } +// Validate gRPC AlreadyExists response stats are sane when response is trailers only. +TEST_F(RouterTest, GrpcAlreadyExistsTrailersOnly) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.outlier_detection_support_for_grpc_status", "true"}}); + NiceMock encoder1; + Http::StreamDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + return nullptr; + })); + expectResponseTimerCreate(); + + Http::TestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "20S"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + Http::HeaderMapPtr response_headers( + new Http::TestHeaderMapImpl{{":status", "200"}, {"grpc-status", "6"}}); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(409)); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 0)); +} + +// Validate gRPC Unavailable response stats are sane when response is trailers only. +TEST_F(RouterTest, GrpcOutlierDetectionUnavailableStatusCodeRuntimeGuard) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.outlier_detection_support_for_grpc_status", "false"}}); + NiceMock encoder1; + Http::StreamDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + return nullptr; + })); + expectResponseTimerCreate(); + + Http::TestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "20S"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + Http::HeaderMapPtr response_headers( + new Http::TestHeaderMapImpl{{":status", "200"}, {"grpc-status", "14"}}); + // Outlier detector will use the gRPC response status code. + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(200)); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); +} + +// Validate gRPC Unavailable response stats are sane when response is trailers only. +TEST_F(RouterTest, GrpcOutlierDetectionUnavailableStatusCode) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.outlier_detection_support_for_grpc_status", "true"}}); + NiceMock encoder1; + Http::StreamDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + return nullptr; + })); + expectResponseTimerCreate(); + + Http::TestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "20S"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + Http::HeaderMapPtr response_headers( + new Http::TestHeaderMapImpl{{":status", "200"}, {"grpc-status", "14"}}); + // Outlier detector will use the gRPC response status code. + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(503)); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); +} + // Validate gRPC Internal response stats are sane when response is trailers only. -TEST_F(RouterTest, GrpcInternalTrailersOnly) { +TEST_F(RouterTest, GrpcInternalTrailersOnlyRuntimeGuard) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.outlier_detection_support_for_grpc_status", "false"}}); NiceMock encoder1; Http::StreamDecoder* response_decoder = nullptr; EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) @@ -1223,6 +1313,33 @@ TEST_F(RouterTest, GrpcInternalTrailersOnly) { EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); } +// Validate gRPC Internal response stats are sane when response is trailers only. +TEST_F(RouterTest, GrpcInternalTrailersOnly) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.outlier_detection_support_for_grpc_status", "true"}}); + NiceMock encoder1; + Http::StreamDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + return nullptr; + })); + expectResponseTimerCreate(); + + Http::TestHeaderMapImpl headers{{"content-type", "application/grpc"}, {"grpc-timeout", "20S"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + Http::HeaderMapPtr response_headers( + new Http::TestHeaderMapImpl{{":status", "200"}, {"grpc-status", "13"}}); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(500)); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); +} + // Validate gRPC response stats are sane when response is ended in a DATA // frame. TEST_F(RouterTest, GrpcDataEndStream) { @@ -2754,7 +2871,11 @@ TEST_F(RouterTest, RetryUpstream5xxNotComplete) { .value()); } -TEST_F(RouterTest, RetryUpstreamGrpcCancelled) { +// Validate gRPC Cancelled response stats are sane when retry is taking effect. +TEST_F(RouterTest, RetryUpstreamGrpcCancelledRuntimeGuard) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.outlier_detection_support_for_grpc_status", "false"}}); NiceMock encoder1; Http::StreamDecoder* response_decoder = nullptr; EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) @@ -2802,6 +2923,58 @@ TEST_F(RouterTest, RetryUpstreamGrpcCancelled) { EXPECT_TRUE(verifyHostUpstreamStats(1, 1)); } +// Validate gRPC Cancelled response stats are sane when retry is taking effect. +TEST_F(RouterTest, RetryUpstreamGrpcCancelled) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.outlier_detection_support_for_grpc_status", "true"}}); + NiceMock encoder1; + Http::StreamDecoder* response_decoder = nullptr; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + return nullptr; + })); + expectResponseTimerCreate(); + + Http::TestHeaderMapImpl headers{{"x-envoy-retry-grpc-on", "cancelled"}, + {"x-envoy-internal", "true"}, + {"content-type", "application/grpc"}, + {"grpc-timeout", "20S"}}; + HttpTestUtility::addDefaultHeaders(headers); + router_.decodeHeaders(headers, true); + + // gRPC with status "cancelled" (1) + router_.retry_state_->expectHeadersRetry(); + Http::HeaderMapPtr response_headers1( + new Http::TestHeaderMapImpl{{":status", "200"}, {"grpc-status", "1"}}); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(499)); + response_decoder->decodeHeaders(std::move(response_headers1), true); + EXPECT_TRUE(verifyHostUpstreamStats(0, 1)); + + // We expect the grpc-status to result in a retried request. + EXPECT_CALL(encoder1.stream_, resetStream(_)).Times(0); + NiceMock encoder2; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + return nullptr; + })); + router_.retry_state_->callback_(); + + // Normal response. + EXPECT_CALL(*router_.retry_state_, shouldRetryHeaders(_, _)).WillOnce(Return(RetryStatus::No)); + Http::HeaderMapPtr response_headers( + new Http::TestHeaderMapImpl{{":status", "200"}, {"grpc-status", "0"}}); + EXPECT_CALL(cm_.conn_pool_.host_->outlier_detector_, putHttpResponseCode(200)); + response_decoder->decodeHeaders(std::move(response_headers), true); + EXPECT_TRUE(verifyHostUpstreamStats(1, 1)); +} + // Verifies that the initial host is select with max host count of one, but during retries // RetryPolicy will be consulted. TEST_F(RouterTest, RetryRespsectsMaxHostSelectionCount) { From 99e3c658ed8fa6283fd972523cbefe2df2eace4e Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Sun, 8 Sep 2019 07:35:29 -0700 Subject: [PATCH 516/542] fix build (#8177) Signed-off-by: Derek Argueta --- test/common/router/router_test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index de3ae6d3f17f..2e8cfe69871f 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -1214,7 +1214,7 @@ TEST_F(RouterTest, GrpcAlreadyExistsTrailersOnly) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1241,7 +1241,7 @@ TEST_F(RouterTest, GrpcOutlierDetectionUnavailableStatusCodeRuntimeGuard) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1269,7 +1269,7 @@ TEST_F(RouterTest, GrpcOutlierDetectionUnavailableStatusCode) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -1324,7 +1324,7 @@ TEST_F(RouterTest, GrpcInternalTrailersOnly) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2934,7 +2934,7 @@ TEST_F(RouterTest, RetryUpstreamGrpcCancelled) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); expectResponseTimerCreate(); @@ -2961,7 +2961,7 @@ TEST_F(RouterTest, RetryUpstreamGrpcCancelled) { .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) -> Http::ConnectionPool::Cancellable* { response_decoder = &decoder; - callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); return nullptr; })); router_.retry_state_->callback_(); From ced130ad92d863b1ffb2988cb07ea62decc9cf6a Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Mon, 9 Sep 2019 11:39:34 -0400 Subject: [PATCH 517/542] docs: improving websocket docs (#8156) Making it clear H2 websockets don't work by default Risk Level: n/a Testing: n/a Docs Changes: yes Release Notes: no #8147 Signed-off-by: Alyssa Wilk --- docs/root/intro/arch_overview/http/websocket.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/root/intro/arch_overview/http/websocket.rst b/docs/root/intro/arch_overview/http/websocket.rst index e854eb53bb27..fa4e0b1f055d 100644 --- a/docs/root/intro/arch_overview/http/websocket.rst +++ b/docs/root/intro/arch_overview/http/websocket.rst @@ -36,19 +36,21 @@ Note that the statistics for upgrades are all bundled together so WebSocket :ref:`statistics ` are tracked by stats such as downstream_cx_upgrades_total and downstream_cx_upgrades_active -Handling H2 hops -^^^^^^^^^^^^^^^^ +Handling HTTP/2 hops +^^^^^^^^^^^^^^^^^^^^ -Envoy supports tunneling WebSockets over H2 streams for deployments that prefer a uniform -H2 mesh throughout; this enables, for example, a deployment of the form: +While HTTP/2 support for WebSockets is off by default, Envoy does support tunneling WebSockets over +HTTP/2 streams for deployments that prefer a uniform HTTP/2 mesh throughout; this enables, for example, +a deployment of the form: [Client] ---- HTTP/1.1 ---- [Front Envoy] ---- HTTP/2 ---- [Sidecar Envoy ---- H1 ---- App] In this case, if a client is for example using WebSocket, we want the Websocket to arrive at the upstream server functionally intact, which means it needs to traverse the HTTP/2 hop. -This is accomplished via -`extended CONNECT `_ support. The +This is accomplished via `extended CONNECT `_ support, +turned on by setting :ref:`allow_connect ` +true at the second layer Envoy. The WebSocket request will be transformed into an HTTP/2 CONNECT stream, with :protocol header indicating the original upgrade, traverse the HTTP/2 hop, and be downgraded back into an HTTP/1 WebSocket Upgrade. This same Upgrade-CONNECT-Upgrade transformation will be performed on any @@ -57,5 +59,5 @@ Non-WebSocket upgrades are allowed to use any valid HTTP method (i.e. POST) and upgrade/downgrade mechanism will drop the original method and transform the Upgrade request to a GET method on the final Envoy-Upstream hop. -Note that the H2 upgrade path has very strict HTTP/1.1 compliance, so will not proxy WebSocket +Note that the HTTP/2 upgrade path has very strict HTTP/1.1 compliance, so will not proxy WebSocket upgrade requests or responses with bodies. From 3f63838d054994d8c23bf909502b1fb381a8d0b1 Mon Sep 17 00:00:00 2001 From: John Plevyak Date: Mon, 9 Sep 2019 12:09:22 -0700 Subject: [PATCH 518/542] Upstream WebAssembly VM and Null VM from envoyproxy/envoy-wasm. (#8020) Description: Upstream from envoyproxy/envoy-wasm the WebAssembly VM support along with the Null VM support and tests. This is the first PR dealing with WebAssembly filter support in envoy. See https://github.com/envoyproxy/envoy-wasm/blob/master/WASM.md and https://github.com/envoyproxy/envoy-wasm/blob/master/docs/root/api-v2/config/wasm/wasm.rst for details. Risk Level: Medium Testing: Unit tests. Docs Changes: N/A Release Notes: N/A Part of #4272 Signed-off-by: John Plevyak --- CODEOWNERS | 2 + source/common/common/logger.h | 3 +- source/extensions/common/wasm/BUILD | 36 ++ source/extensions/common/wasm/null/BUILD | 47 +++ source/extensions/common/wasm/null/null.cc | 27 ++ source/extensions/common/wasm/null/null.h | 20 ++ source/extensions/common/wasm/null/null_vm.cc | 104 ++++++ source/extensions/common/wasm/null/null_vm.h | 75 ++++ .../common/wasm/null/null_vm_plugin.h | 50 +++ source/extensions/common/wasm/wasm_vm.cc | 31 ++ source/extensions/common/wasm/wasm_vm.h | 326 ++++++++++++++++++ .../extensions/common/wasm/well_known_names.h | 28 ++ test/extensions/common/wasm/BUILD | 19 + test/extensions/common/wasm/wasm_vm_test.cc | 103 ++++++ tools/spelling_dictionary.txt | 5 + 15 files changed, 875 insertions(+), 1 deletion(-) create mode 100644 source/extensions/common/wasm/BUILD create mode 100644 source/extensions/common/wasm/null/BUILD create mode 100644 source/extensions/common/wasm/null/null.cc create mode 100644 source/extensions/common/wasm/null/null.h create mode 100644 source/extensions/common/wasm/null/null_vm.cc create mode 100644 source/extensions/common/wasm/null/null_vm.h create mode 100644 source/extensions/common/wasm/null/null_vm_plugin.h create mode 100644 source/extensions/common/wasm/wasm_vm.cc create mode 100644 source/extensions/common/wasm/wasm_vm.h create mode 100644 source/extensions/common/wasm/well_known_names.h create mode 100644 test/extensions/common/wasm/BUILD create mode 100644 test/extensions/common/wasm/wasm_vm_test.cc diff --git a/CODEOWNERS b/CODEOWNERS index 7c3617ef6292..59b85b44b615 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -55,3 +55,5 @@ extensions/filters/common/original_src @snowp @klarose /*/extensions/filters/listener/http_inspector @crazyxy @PiotrSikora @lizan # attribute context /*/extensions/filters/common/expr @kyessenov @yangminzhu +# webassembly common extension +/*/extensions/common/wasm @jplevyak @PiotrSikora diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 42360c572b39..a443ce11ab5e 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -60,7 +60,8 @@ namespace Logger { FUNCTION(thrift) \ FUNCTION(tracing) \ FUNCTION(upstream) \ - FUNCTION(udp) + FUNCTION(udp) \ + FUNCTION(wasm) enum class Id { ALL_LOGGER_IDS(GENERATE_ENUM) diff --git a/source/extensions/common/wasm/BUILD b/source/extensions/common/wasm/BUILD new file mode 100644 index 000000000000..7eda2486185d --- /dev/null +++ b/source/extensions/common/wasm/BUILD @@ -0,0 +1,36 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "well_known_names", + hdrs = ["well_known_names.h"], + deps = [ + "//source/common/singleton:const_singleton", + ], +) + +envoy_cc_library( + name = "wasm_vm_interface", + hdrs = ["wasm_vm.h"], + deps = [ + ":well_known_names", + "//source/common/common:minimal_logger_lib", + ], +) + +envoy_cc_library( + name = "wasm_vm_lib", + srcs = ["wasm_vm.cc"], + deps = [ + ":wasm_vm_interface", + "//source/common/common:assert_lib", + "//source/extensions/common/wasm/null:null_lib", + ], +) diff --git a/source/extensions/common/wasm/null/BUILD b/source/extensions/common/wasm/null/BUILD new file mode 100644 index 000000000000..eed8e62d2e49 --- /dev/null +++ b/source/extensions/common/wasm/null/BUILD @@ -0,0 +1,47 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "null_vm_plugin_interface", + hdrs = ["null_vm_plugin.h"], + deps = [ + "//source/extensions/common/wasm:wasm_vm_interface", + "//source/extensions/common/wasm:well_known_names", + ], +) + +envoy_cc_library( + name = "null_vm_lib", + srcs = ["null_vm.cc"], + hdrs = ["null_vm.h"], + deps = [ + ":null_vm_plugin_interface", + "//external:abseil_node_hash_map", + "//include/envoy/registry", + "//source/common/common:assert_lib", + "//source/extensions/common/wasm:wasm_vm_interface", + "//source/extensions/common/wasm:well_known_names", + ], +) + +envoy_cc_library( + name = "null_lib", + srcs = ["null.cc"], + hdrs = ["null.h"], + deps = [ + ":null_vm_lib", + ":null_vm_plugin_interface", + "//external:abseil_node_hash_map", + "//include/envoy/registry", + "//source/common/common:assert_lib", + "//source/extensions/common/wasm:wasm_vm_interface", + "//source/extensions/common/wasm:well_known_names", + ], +) diff --git a/source/extensions/common/wasm/null/null.cc b/source/extensions/common/wasm/null/null.cc new file mode 100644 index 000000000000..185dde60780a --- /dev/null +++ b/source/extensions/common/wasm/null/null.cc @@ -0,0 +1,27 @@ +#include "extensions/common/wasm/null/null.h" + +#include +#include +#include + +#include "envoy/registry/registry.h" + +#include "common/common/assert.h" + +#include "extensions/common/wasm/null/null_vm.h" +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +WasmVmPtr createVm() { return std::make_unique(); } + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/null/null.h b/source/extensions/common/wasm/null/null.h new file mode 100644 index 000000000000..7d88fb356923 --- /dev/null +++ b/source/extensions/common/wasm/null/null.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/wasm_vm.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +WasmVmPtr createVm(); + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/null/null_vm.cc b/source/extensions/common/wasm/null/null_vm.cc new file mode 100644 index 000000000000..b9957c6a67f2 --- /dev/null +++ b/source/extensions/common/wasm/null/null_vm.cc @@ -0,0 +1,104 @@ +#include "extensions/common/wasm/null/null_vm.h" + +#include +#include +#include + +#include "envoy/registry/registry.h" + +#include "common/common/assert.h" + +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +WasmVmPtr NullVm::clone() { + auto cloned_null_vm = std::make_unique(*this); + cloned_null_vm->load(plugin_name_, false /* unused */); + return cloned_null_vm; +} + +// "Load" the plugin by obtaining a pointer to it from the factory. +bool NullVm::load(const std::string& name, bool /* allow_precompiled */) { + auto factory = Registry::FactoryRegistry::getFactory(name); + if (!factory) { + return false; + } + plugin_name_ = name; + plugin_ = factory->create(); + return true; +} + +void NullVm::link(absl::string_view /* name */, bool /* needs_emscripten */) {} + +void NullVm::makeModule(absl::string_view /* name */) { + // NullVm does not advertise code as emscripten so this will not get called. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void NullVm::start(Common::Wasm::Context* context) { + SaveRestoreContext saved_context(context); + plugin_->start(); +} + +uint64_t NullVm::getMemorySize() { return std::numeric_limits::max(); } + +// NulVm pointers are just native pointers. +absl::optional NullVm::getMemory(uint64_t pointer, uint64_t size) { + if (pointer == 0 && size != 0) { + return absl::nullopt; + } + return absl::string_view(reinterpret_cast(pointer), static_cast(size)); +} + +bool NullVm::getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) { + *vm_pointer = reinterpret_cast(host_pointer); + return true; +} + +bool NullVm::setMemory(uint64_t pointer, uint64_t size, const void* data) { + if ((pointer == 0 || data == nullptr)) { + if (size != 0) { + return false; + } else { + return true; + } + } + auto p = reinterpret_cast(pointer); + memcpy(p, data, size); + return true; +} + +bool NullVm::setWord(uint64_t pointer, Word data) { + if (pointer == 0) { + return false; + } + auto p = reinterpret_cast(pointer); + memcpy(p, &data.u64_, sizeof(data.u64_)); + return true; +} + +bool NullVm::getWord(uint64_t pointer, Word* data) { + if (pointer == 0) { + return false; + } + auto p = reinterpret_cast(pointer); + memcpy(&data->u64_, p, sizeof(data->u64_)); + return true; +} + +absl::string_view NullVm::getUserSection(absl::string_view /* name */) { + // Return nothing: there is no WASM file. + return {}; +} + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/null/null_vm.h b/source/extensions/common/wasm/null/null_vm.h new file mode 100644 index 000000000000..f3b90fabf1c8 --- /dev/null +++ b/source/extensions/common/wasm/null/null_vm.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include + +#include "envoy/registry/registry.h" + +#include "common/common/assert.h" + +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +// The NullVm wraps a C++ WASM plugin which has been compiled with the WASM API +// and linked directly into the Envoy process. This is useful for development +// in that it permits the debugger to set breakpoints in both Envoy and the plugin. +struct NullVm : public WasmVm { + NullVm() = default; + NullVm(const NullVm& other) : plugin_name_(other.plugin_name_) {} + + // WasmVm + absl::string_view vm() override { return WasmVmNames::get().Null; } + bool cloneable() override { return true; }; + WasmVmPtr clone() override; + bool load(const std::string& code, bool allow_precompiled) override; + void link(absl::string_view debug_name, bool needs_emscripten) override; + void setMemoryLayout(uint64_t, uint64_t, uint64_t) override {} + void start(Common::Wasm::Context* context) override; + uint64_t getMemorySize() override; + absl::optional getMemory(uint64_t pointer, uint64_t size) override; + bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) override; + bool setMemory(uint64_t pointer, uint64_t size, const void* data) override; + bool setWord(uint64_t pointer, Word data) override; + bool getWord(uint64_t pointer, Word* data) override; + void makeModule(absl::string_view name) override; + absl::string_view getUserSection(absl::string_view name) override; + +#define _FORWARD_GET_FUNCTION(_T) \ + void getFunction(absl::string_view function_name, _T* f) override { \ + plugin_->getFunction(function_name, f); \ + } + FOR_ALL_WASM_VM_EXPORTS(_FORWARD_GET_FUNCTION) +#undef _FORWARD_GET_FUNCTION + + // These are not needed for NullVm which invokes the handlers directly. +#define _REGISTER_CALLBACK(_T) \ + void registerCallback(absl::string_view, absl::string_view, _T, \ + typename ConvertFunctionTypeWordToUint32<_T>::type) override{}; + FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) +#undef _REGISTER_CALLBACK + + // NullVm does not advertise code as emscripten so this will not get called. + std::unique_ptr> makeGlobal(absl::string_view, absl::string_view, + double) override { + NOT_REACHED_GCOVR_EXCL_LINE; + }; + std::unique_ptr> makeGlobal(absl::string_view, absl::string_view, Word) override { + NOT_REACHED_GCOVR_EXCL_LINE; + }; + + std::string plugin_name_; + std::unique_ptr plugin_; +}; + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/null/null_vm_plugin.h b/source/extensions/common/wasm/null/null_vm_plugin.h new file mode 100644 index 000000000000..4dce2c617238 --- /dev/null +++ b/source/extensions/common/wasm/null/null_vm_plugin.h @@ -0,0 +1,50 @@ +#pragma once + +#include "extensions/common/wasm/wasm_vm.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace Null { + +// A wrapper for the natively compiled NullVm plugin which implements the WASM ABI. +class NullVmPlugin { +public: + NullVmPlugin() = default; + virtual ~NullVmPlugin() = default; + + // NB: These are defined rather than declared PURE because gmock uses __LINE__ internally for + // uniqueness, making it impossible to use FOR_ALL_WASM_VM_EXPORTS with MOCK_METHOD2. +#define _DEFINE_GET_FUNCTION(_T) \ + virtual void getFunction(absl::string_view, _T* f) { *f = nullptr; } + FOR_ALL_WASM_VM_EXPORTS(_DEFINE_GET_FUNCTION) +#undef _DEFIN_GET_FUNCTIONE + + virtual void start() PURE; +}; + +/** + * Pseudo-WASM plugins using the NullVM should implement this factory and register via + * Registry::registerFactory or the convenience class RegisterFactory. + */ +class NullVmPluginFactory { +public: + virtual ~NullVmPluginFactory() = default; + + /** + * Name of the plugin. + */ + virtual const std::string name() const PURE; + + /** + * Create an instance of the plugin. + */ + virtual std::unique_ptr create() const PURE; +}; + +} // namespace Null +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/wasm_vm.cc b/source/extensions/common/wasm/wasm_vm.cc new file mode 100644 index 000000000000..9a8dc2f98778 --- /dev/null +++ b/source/extensions/common/wasm/wasm_vm.cc @@ -0,0 +1,31 @@ +#include "extensions/common/wasm/wasm_vm.h" + +#include + +#include "extensions/common/wasm/null/null.h" +#include "extensions/common/wasm/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { + +thread_local Envoy::Extensions::Common::Wasm::Context* current_context_ = nullptr; +thread_local uint32_t effective_context_id_ = 0; + +WasmVmPtr createWasmVm(absl::string_view wasm_vm) { + if (wasm_vm.empty()) { + throw WasmException("Failed to create WASM VM with unspecified runtime."); + } else if (wasm_vm == WasmVmNames::get().Null) { + return Null::createVm(); + } else { + throw WasmException(fmt::format( + "Failed to create WASM VM using {} runtime. Envoy was compiled without support for it.", + wasm_vm)); + } +} + +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h new file mode 100644 index 000000000000..42e5a08b0d33 --- /dev/null +++ b/source/extensions/common/wasm/wasm_vm.h @@ -0,0 +1,326 @@ +#pragma once + +#include + +#include "envoy/common/exception.h" + +#include "common/common/logger.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { + +class Context; + +// Represents a WASM-native word-sized datum. On 32-bit VMs, the high bits are always zero. +// The WASM/VM API treats all bits as significant. +struct Word { + Word(uint64_t w) : u64_(w) {} // Implicit conversion into Word. + uint32_t u32() const { return static_cast(u64_); } + uint64_t u64_; +}; + +// Convert Word type for use by 32-bit VMs. +template struct ConvertWordTypeToUint32 { using type = T; }; +template <> struct ConvertWordTypeToUint32 { using type = uint32_t; }; + +// Convert Word-based function types for 32-bit VMs. +template struct ConvertFunctionTypeWordToUint32 {}; +template struct ConvertFunctionTypeWordToUint32 { + using type = typename ConvertWordTypeToUint32::type (*)( + typename ConvertWordTypeToUint32::type...); +}; + +// A wrapper for a global variable within the VM. +template struct Global { + virtual ~Global() = default; + virtual T get() PURE; + virtual void set(const T& t) PURE; +}; + +// These are templates and its helper for constructing signatures of functions calling into and out +// of WASM VMs. +// - WasmFuncTypeHelper is a helper for WasmFuncType and shouldn't be used anywhere else than +// WasmFuncType definition. +// - WasmFuncType takes 4 template parameter which are number of argument, return type, context type +// and param type respectively, resolve to a function type. +// For example `WasmFuncType<3, void, Context*, Word>` resolves to `void(Context*, Word, Word, +// Word)` +template +struct WasmFuncTypeHelper {}; + +template +struct WasmFuncTypeHelper { + using type = typename WasmFuncTypeHelper::type; +}; + +template +struct WasmFuncTypeHelper<0, ReturnType, ContextType, ParamType, ReturnType(ContextType, Args...)> { + using type = ReturnType(ContextType, Args...); +}; + +template +using WasmFuncType = typename WasmFuncTypeHelper::type; + +// Calls into the WASM VM. +// 1st arg is always a pointer to Context (Context*). +template using WasmCallVoid = std::function>; +template using WasmCallWord = std::function>; + +#define FOR_ALL_WASM_VM_EXPORTS(_f) \ + _f(WasmCallVoid<0>) _f(WasmCallVoid<1>) _f(WasmCallVoid<2>) _f(WasmCallVoid<3>) \ + _f(WasmCallVoid<4>) _f(WasmCallVoid<5>) _f(WasmCallVoid<8>) _f(WasmCallWord<0>) \ + _f(WasmCallWord<1>) _f(WasmCallWord<3>) + +// Calls out of the WASM VM. +// 1st arg is always a pointer to raw_context (void*). +template using WasmCallbackVoid = WasmFuncType*; +template using WasmCallbackWord = WasmFuncType*; + +// Using the standard g++/clang mangling algorithm: +// https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling-builtin +// Extended with W = Word +// Z = void, j = uint32_t, l = int64_t, m = uint64_t +using WasmCallback_WWl = Word (*)(void*, Word, int64_t); +using WasmCallback_WWm = Word (*)(void*, Word, uint64_t); + +#define FOR_ALL_WASM_VM_IMPORTS(_f) \ + _f(WasmCallbackVoid<0>) _f(WasmCallbackVoid<1>) _f(WasmCallbackVoid<2>) _f(WasmCallbackVoid<3>) \ + _f(WasmCallbackVoid<4>) _f(WasmCallbackWord<0>) _f(WasmCallbackWord<1>) \ + _f(WasmCallbackWord<2>) _f(WasmCallbackWord<3>) _f(WasmCallbackWord<4>) \ + _f(WasmCallbackWord<5>) _f(WasmCallbackWord<6>) _f(WasmCallbackWord<7>) \ + _f(WasmCallbackWord<8>) _f(WasmCallbackWord<9>) _f(WasmCallback_WWl) \ + _f(WasmCallback_WWm) + +// Wasm VM instance. Provides the low level WASM interface. +class WasmVm : public Logger::Loggable { +public: + using WasmVmPtr = std::unique_ptr; + + virtual ~WasmVm() = default; + /** + * Return the VM identifier. + * @return one of WasmVmValues from well_known_names.h e.g. "envoy.wasm.vm.null". + */ + virtual absl::string_view vm() PURE; + + /** + * Whether or not the VM implementation supports cloning. Cloning is VM system dependent. + * When a VM is configured a single VM is instantiated to check that the .wasm file is valid and + * to do VM system specific initialization. In the case of WAVM this is potentially ahead-of-time + * compilation. Then, if cloning is supported, we clone that VM for each worker, potentially + * copying and sharing the initialized data structures for efficiency. Otherwise we create an new + * VM from scratch for each worker. + * @return true if the VM is cloneable. + */ + virtual bool cloneable() PURE; + + /** + * Make a worker/thread-specific copy if supported by the underlying VM system (see cloneable() + * above). If not supported, the caller will need to create a new VM from scratch. If supported, + * the clone may share compiled code and other read-only data with the source VM. + * @return a clone of 'this' (e.g. for a different worker/thread). + */ + virtual WasmVmPtr clone() PURE; + + /** + * Load the WASM code from a file. Return true on success. Once the module is loaded it can be + * queried, e.g. to see which version of emscripten support is required. After loading, the + * appropriate ABI callbacks can be registered and then the module can be link()ed (see below). + * @param code the WASM binary code (or registered NullVm plugin name). + * @param allow_precompiled if true, allows supporting VMs (e.g. WAVM) to load the binary + * machine code from a user-defined section of the WASM file. Because that code is not verified by + * the envoy process it is up to the user to ensure that the code is both safe and is built for + * the linked in version of WAVM. + * @return whether or not the load was successful. + */ + virtual bool load(const std::string& code, bool allow_precompiled) PURE; + + /** + * Link the WASM code to the host-provided functions and globals, e.g. the ABI. Prior to linking, + * the module should be loaded and the ABI callbacks registered (see above). Linking should be + * done once between load() and start(). + * @param debug_name user-provided name for use in log and error messages. + * @param needs_emscripten whether emscripten support should be provided (e.g. + * _emscripten_memcpy_bigHandler). Emscripten (http://https://emscripten.org/) is + * a C++ WebAssembly tool chain. + */ + virtual void link(absl::string_view debug_name, bool needs_emscripten) PURE; + + /** + * Set memory layout (start of dynamic heap base, etc.) in the VM. + * @param stack_base the location in VM memory of the stack. + * @param heap_base the location in VM memory of the heap. + * @param heap_base_ptr the location in VM memory of a location to store the heap pointer. + */ + virtual void setMemoryLayout(uint64_t stack_base, uint64_t heap_base, + uint64_t heap_base_pointer) PURE; + + /** + * Initialize globals (including calling global constructors) and call the 'start' function. + * Prior to calling start() the module should be load()ed, ABI callbacks should be registered + * (registerCallback), the module link()ed, and any exported functions should be gotten + * (getFunction). + * @param vm_context a context which represents the caller: in this case Envoy itself. + */ + virtual void start(Context* vm_context) PURE; + + /** + * Get size of the currently allocated memory in the VM. + * @return the size of memory in bytes. + */ + virtual uint64_t getMemorySize() PURE; + + /** + * Convert a block of memory in the VM to a string_view. + * @param pointer the offset into VM memory of the requested VM memory block. + * @param size the size of the requested VM memory block. + * @return if std::nullopt then the pointer/size pair were invalid, otherwise returns + * a host string_view pointing to the pointer/size pair in VM memory. + */ + virtual absl::optional getMemory(uint64_t pointer, uint64_t size) PURE; + + /** + * Convert a host pointer to memory in the VM into a VM "pointer" (an offset into the Memory). + * @param host_pointer a pointer to host memory to be converted into a VM offset (pointer). + * @param vm_pointer a pointer to an uint64_t to be filled with the offset in VM memory + * corresponding to 'host_pointer'. + * @return whether or not the host_pointer was a valid VM memory offset. + */ + virtual bool getMemoryOffset(void* host_pointer, uint64_t* vm_pointer) PURE; + + /** + * Set a block of memory in the VM, returns true on success, false if the pointer/size is invalid. + * @param pointer the offset into VM memory describing the start of a region of VM memory. + * @param size the size of the region of VM memory. + * @return whether or not the pointer/size pair was a valid VM memory block. + */ + virtual bool setMemory(uint64_t pointer, uint64_t size, const void* data) PURE; + + /** + * Get a VM native Word (e.g. sizeof(void*) or sizeof(size_t)) from VM memory, returns true on + * success, false if the pointer is invalid. WASM-32 VMs have 32-bit native words and WASM-64 VMs + * (not yet supported) will have 64-bit words as does the Null VM (compiled into 64-bit Envoy). + * This function can be used to chase pointers in VM memory. + * @param pointer the offset into VM memory describing the start of VM native word size block. + * @param data a pointer to a Word whose contents will be filled from the VM native word at + * 'pointer'. + * @return whether or not the pointer was to a valid VM memory block of VM native word size. + */ + virtual bool getWord(uint64_t pointer, Word* data) PURE; + + /** + * Set a Word in the VM, returns true on success, false if the pointer is invalid. + * See getWord above for details. This function can be used (for example) to set indirect pointer + * return values (e.g. proxy_getHeaderHapValue(... const char** value_ptr, size_t* value_size). + * @param pointer the offset into VM memory describing the start of VM native word size block. + * @param data a Word whose contents will be written in VM native word size at 'pointer'. + * @return whether or not the pointer was to a valid VM memory block of VM native word size. + */ + virtual bool setWord(uint64_t pointer, Word data) PURE; + + /** + * Make a new intrinsic module (e.g. for Emscripten support). + * @param name the name of the module to make. + */ + virtual void makeModule(absl::string_view name) PURE; + + /** + * Get the contents of the user section with the given name or "" if it does not exist. + * @param name the name of the user section to get. + * @return the contents of the user section (if any). The result will be empty() if there + * is no such section. + */ + virtual absl::string_view getUserSection(absl::string_view name) PURE; + + /** + * Get typed function exported by the WASM module. + */ +#define _GET_FUNCTION(_T) virtual void getFunction(absl::string_view function_name, _T* f) PURE; + FOR_ALL_WASM_VM_EXPORTS(_GET_FUNCTION) +#undef _GET_FUNCTION + + /** + * Register typed callbacks exported by the host environment. + */ +#define _REGISTER_CALLBACK(_T) \ + virtual void registerCallback(absl::string_view moduleName, absl::string_view function_name, \ + _T f, typename ConvertFunctionTypeWordToUint32<_T>::type) PURE; + FOR_ALL_WASM_VM_IMPORTS(_REGISTER_CALLBACK) +#undef _REGISTER_CALLBACK + + /** + * Register typed value exported by the host environment. + * @param module_name the name of the module to which to export the global. + * @param name the name of the global variable to export. + * @param initial_value the initial value of the global. + * @return a Global object which can be used to access the exported global. + */ + virtual std::unique_ptr> makeGlobal(absl::string_view module_name, + absl::string_view name, Word initial_value) PURE; + + /** + * Register typed value exported by the host environment. + * @param module_name the name of the module to which to export the global. + * @param name the name of the global variable to export. + * @param initial_value the initial value of the global. + * @return a Global object which can be used to access the exported global. + */ + virtual std::unique_ptr> + makeGlobal(absl::string_view module_name, absl::string_view name, double initial_value) PURE; +}; +using WasmVmPtr = std::unique_ptr; + +// Exceptions for issues with the WasmVm. +class WasmVmException : public EnvoyException { +public: + using EnvoyException::EnvoyException; +}; + +// Exceptions for issues with the WebAssembly code. +class WasmException : public EnvoyException { +public: + using EnvoyException::EnvoyException; +}; + +// Thread local state set during a call into a WASM VM so that calls coming out of the +// VM can be attributed correctly to calling Filter. We use thread_local instead of ThreadLocal +// because this state is live only during the calls and does not need to be initialized consistently +// over all workers as with ThreadLocal data. +extern thread_local Envoy::Extensions::Common::Wasm::Context* current_context_; + +// Requested effective context set by code within the VM to request that the calls coming out of the +// VM be attributed to another filter, for example if a control plane gRPC comes back to the +// RootContext which effects some set of waiting filters. +extern thread_local uint32_t effective_context_id_; + +// Helper to save and restore thread local VM call context information to support reentrant calls. +// NB: this happens for example when a call from the VM invokes a handler which needs to _malloc +// memory in the VM. +struct SaveRestoreContext { + explicit SaveRestoreContext(Context* context) { + saved_context = current_context_; + saved_effective_context_id_ = effective_context_id_; + current_context_ = context; + effective_context_id_ = 0; // No effective context id. + } + ~SaveRestoreContext() { + current_context_ = saved_context; + effective_context_id_ = saved_effective_context_id_; + } + Context* saved_context; + uint32_t saved_effective_context_id_; +}; + +// Create a new low-level WASM VM of the give type (e.g. "envoy.wasm.vm.wavm"). +WasmVmPtr createWasmVm(absl::string_view vm); + +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/wasm/well_known_names.h b/source/extensions/common/wasm/well_known_names.h new file mode 100644 index 000000000000..3674ed3ee8b5 --- /dev/null +++ b/source/extensions/common/wasm/well_known_names.h @@ -0,0 +1,28 @@ +#pragma once + +#include + +#include "common/singleton/const_singleton.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { + +/** + * Well-known wasm VM names. + * NOTE: New wasm VMs should use the well known name: envoy.wasm.vm.name. + */ +class WasmVmValues { +public: + // Null sandbox: modules must be compiled into envoy and registered name is given in the + // DataSource.inline_string. + const std::string Null = "envoy.wasm.vm.null"; +}; + +using WasmVmNames = ConstSingleton; + +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/common/wasm/BUILD b/test/extensions/common/wasm/BUILD new file mode 100644 index 000000000000..490402dc1784 --- /dev/null +++ b/test/extensions/common/wasm/BUILD @@ -0,0 +1,19 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_cc_test_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_test( + name = "wasm_vm_test", + srcs = ["wasm_vm_test.cc"], + deps = [ + "//source/extensions/common/wasm:wasm_vm_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/common/wasm/wasm_vm_test.cc b/test/extensions/common/wasm/wasm_vm_test.cc new file mode 100644 index 000000000000..e1ecd2590f6f --- /dev/null +++ b/test/extensions/common/wasm/wasm_vm_test.cc @@ -0,0 +1,103 @@ +#include "envoy/registry/registry.h" + +#include "extensions/common/wasm/null/null_vm_plugin.h" +#include "extensions/common/wasm/wasm_vm.h" + +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Wasm { +namespace { + +class TestNullVmPlugin : public Null::NullVmPlugin { +public: + TestNullVmPlugin() = default; + ~TestNullVmPlugin() override = default; + + MOCK_METHOD0(start, void()); +}; + +class PluginFactory : public Null::NullVmPluginFactory { +public: + PluginFactory() = default; + + const std::string name() const override { return "test_null_vm_plugin"; } + std::unique_ptr create() const override; +}; + +TestNullVmPlugin* test_null_vm_plugin_ = nullptr; +Envoy::Registry::RegisterFactory register_; + +std::unique_ptr PluginFactory::create() const { + auto result = std::make_unique(); + test_null_vm_plugin_ = result.get(); + return result; +} + +TEST(WasmVmTest, BadVmType) { EXPECT_THROW(createWasmVm("bad.vm"), WasmException); } + +TEST(WasmVmTest, NullVmStartup) { + auto wasm_vm = createWasmVm("envoy.wasm.vm.null"); + EXPECT_TRUE(wasm_vm != nullptr); + EXPECT_TRUE(wasm_vm->cloneable()); + auto wasm_vm_clone = wasm_vm->clone(); + EXPECT_TRUE(wasm_vm_clone != nullptr); + EXPECT_TRUE(wasm_vm->getUserSection("user").empty()); +} + +TEST(WasmVmTest, NullVmMemory) { + auto wasm_vm = createWasmVm("envoy.wasm.vm.null"); + EXPECT_EQ(wasm_vm->getMemorySize(), std::numeric_limits::max()); + std::string d = "data"; + auto m = wasm_vm->getMemory(reinterpret_cast(d.data()), d.size()).value(); + EXPECT_EQ(m.data(), d.data()); + EXPECT_EQ(m.size(), d.size()); + uint64_t offset; + char l; + EXPECT_TRUE(wasm_vm->getMemoryOffset(&l, &offset)); + EXPECT_EQ(offset, reinterpret_cast(&l)); + char c; + char z = 'z'; + EXPECT_TRUE(wasm_vm->setMemory(reinterpret_cast(&c), 1, &z)); + EXPECT_EQ(c, z); + + Word w(13); + EXPECT_TRUE( + wasm_vm->setWord(reinterpret_cast(&w), std::numeric_limits::max())); + EXPECT_EQ(w.u64_, std::numeric_limits::max()); + + Word w2(0); + w.u64_ = 7; + EXPECT_TRUE(wasm_vm->getWord(reinterpret_cast(&w), &w2)); + EXPECT_EQ(w2.u64_, 7); +} + +TEST(WasmVmTest, NullVmStart) { + auto wasm_vm = createWasmVm("envoy.wasm.vm.null"); + EXPECT_TRUE(wasm_vm->load("test_null_vm_plugin", true)); + wasm_vm->link("test", false); + // Test that context argument to start is pushed and that the effective_context_id_ is reset. + // Test that the original values are restored. + Context* context1 = reinterpret_cast(1); + Context* context2 = reinterpret_cast(2); + current_context_ = context1; + effective_context_id_ = 1; + EXPECT_CALL(*test_null_vm_plugin_, start()).WillOnce(Invoke([context2]() { + EXPECT_EQ(current_context_, context2); + EXPECT_EQ(effective_context_id_, 0); + })); + wasm_vm->start(context2); + EXPECT_EQ(current_context_, context1); + EXPECT_EQ(effective_context_id_, 1); +} + +} // namespace +} // namespace Wasm +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index f568a88130d2..e7100af807df 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -9,6 +9,7 @@ ALPN ALS AMZ API +ABI ASAN ASCII ASSERTs @@ -294,6 +295,9 @@ VC VH VHDS VLOG +VM +WASM +WAVM WKT WRR WS @@ -452,6 +456,7 @@ dynamodb eg emplace emplaced +emscripten enablement encodings endian From 44a8588219209c4b96fca2e7e9f009138e1f3ca6 Mon Sep 17 00:00:00 2001 From: danzh Date: Mon, 9 Sep 2019 16:29:07 -0400 Subject: [PATCH 519/542] quiche: implement Envoy Quic stream and connection (#7721) Implement QuicStream|Session|Disptacher in Envoy. Weir up QUIC stream and connection with HCM callbacks. Risk Level: low, not in use Testing: Added unit tests for all new classes Part of #2557 Signed-off-by: Dan Zhang --- bazel/external/quiche.BUILD | 3 + include/envoy/api/io_error.h | 2 + source/extensions/quic_listeners/quiche/BUILD | 118 +++++- .../quic_listeners/quiche/codec_impl.cc | 23 + .../quic_listeners/quiche/codec_impl.h | 58 +++ .../quiche/envoy_quic_connection.cc | 59 +++ .../quiche/envoy_quic_connection.h | 71 ++++ .../quiche/envoy_quic_connection_helper.h | 42 ++ .../quiche/envoy_quic_dispatcher.cc | 58 +++ .../quiche/envoy_quic_dispatcher.h | 78 ++++ .../quiche/envoy_quic_fake_proof_source.h | 4 +- .../quiche/envoy_quic_fake_proof_verifier.h | 5 +- .../quiche/envoy_quic_server_session.cc | 190 +++++++++ .../quiche/envoy_quic_server_session.h | 163 +++++++ .../quiche/envoy_quic_server_stream.cc | 132 ++++++ .../quiche/envoy_quic_server_stream.h | 52 +++ .../quic_listeners/quiche/envoy_quic_stream.h | 42 ++ .../quic_listeners/quiche/envoy_quic_utils.cc | 66 +++ .../quic_listeners/quiche/envoy_quic_utils.h | 27 +- .../quiche/platform/quic_pcc_sender_impl.h | 10 +- .../quiche/quic_io_handle_wrapper.h | 68 +++ .../transport_sockets/well_known_names.h | 1 + source/server/connection_handler_impl.h | 11 + test/extensions/quic_listeners/quiche/BUILD | 89 ++++ .../quiche/envoy_quic_dispatcher_test.cc | 262 ++++++++++++ .../quiche/envoy_quic_proof_source_test.cc | 2 +- .../quiche/envoy_quic_server_session_test.cc | 401 ++++++++++++++++++ .../quiche/envoy_quic_server_stream_test.cc | 253 +++++++++++ .../quiche/envoy_quic_utils_test.cc | 65 +++ .../quiche/quic_io_handle_wrapper_test.cc | 88 ++++ test/mocks/api/mocks.h | 2 + test/run_envoy_bazel_coverage.sh | 2 +- tools/spelling_dictionary.txt | 3 + 33 files changed, 2424 insertions(+), 26 deletions(-) create mode 100644 source/extensions/quic_listeners/quiche/codec_impl.cc create mode 100644 source/extensions/quic_listeners/quiche/codec_impl.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_connection.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_connection.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_connection_helper.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_server_session.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_server_session.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_server_stream.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_stream.h create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_utils.cc create mode 100644 source/extensions/quic_listeners/quiche/quic_io_handle_wrapper.h create mode 100644 test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc create mode 100644 test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc create mode 100644 test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc create mode 100644 test/extensions/quic_listeners/quiche/envoy_quic_utils_test.cc create mode 100644 test/extensions/quic_listeners/quiche/quic_io_handle_wrapper_test.cc diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD index 7f59318c97d5..67d7d4207ce9 100644 --- a/bazel/external/quiche.BUILD +++ b/bazel/external/quiche.BUILD @@ -59,6 +59,8 @@ quiche_copt = [ "-Wno-type-limits", # quic_inlined_frame.h uses offsetof() to optimize memory usage in frames. "-Wno-invalid-offsetof", + "-Wno-type-limits", + "-Wno-return-type", ] envoy_cc_test_library( @@ -2014,6 +2016,7 @@ envoy_cc_library( copts = quiche_copt, repository = "@envoy", tags = ["nofips"], + visibility = ["//visibility:public"], deps = [ ":quic_core_packets_lib", ":quic_platform_base", diff --git a/include/envoy/api/io_error.h b/include/envoy/api/io_error.h index 70699e7443dc..247e102a49d5 100644 --- a/include/envoy/api/io_error.h +++ b/include/envoy/api/io_error.h @@ -31,6 +31,8 @@ class IoError { Interrupt, // Requested a nonexistent interface or a non-local source address. AddressNotAvailable, + // Bad file descriptor. + BadFd, // Other error codes cannot be mapped to any one above in getErrorCode(). UnknownError }; diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index c71bb49c6f35..c7ed7ae4b08b 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -35,6 +35,18 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "envoy_quic_connection_helper_lib", + hdrs = ["envoy_quic_connection_helper.h"], + tags = ["nofips"], + deps = [ + "//source/extensions/quic_listeners/quiche/platform:envoy_quic_clock_lib", + "@com_googlesource_quiche//:quic_core_buffer_allocator_lib", + "@com_googlesource_quiche//:quic_core_connection_lib", + "@com_googlesource_quiche//:quic_core_crypto_random_lib", + ], +) + envoy_cc_library( name = "envoy_quic_packet_writer_lib", srcs = ["envoy_quic_packet_writer.cc"], @@ -80,10 +92,114 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "envoy_quic_stream_lib", + hdrs = ["envoy_quic_stream.h"], + tags = ["nofips"], + deps = [ + "//include/envoy/http:codec_interface", + "//source/common/http:codec_helper_lib", + ], +) + +envoy_cc_library( + name = "envoy_quic_server_stream_lib", + srcs = ["envoy_quic_server_stream.cc"], + hdrs = ["envoy_quic_server_stream.h"], + tags = ["nofips"], + deps = [ + ":envoy_quic_stream_lib", + ":envoy_quic_utils_lib", + "//source/common/buffer:buffer_lib", + "//source/common/common:assert_lib", + "//source/common/http:header_map_lib", + "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", + ], +) + +envoy_cc_library( + name = "codec_impl_lib", + srcs = ["codec_impl.cc"], + hdrs = ["codec_impl.h"], + tags = ["nofips"], + deps = [ + ":envoy_quic_server_session_lib", + "//include/envoy/http:codec_interface", + "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", + ], +) + +envoy_cc_library( + name = "envoy_quic_server_session_lib", + srcs = ["envoy_quic_server_session.cc"], + hdrs = ["envoy_quic_server_session.h"], + tags = ["nofips"], + deps = [ + ":envoy_quic_connection_lib", + ":envoy_quic_server_stream_lib", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/network:connection_interface", + "//source/common/common:empty_string", + "//source/common/network:filter_manager_lib", + "//source/common/stream_info:stream_info_lib", + "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", + ], +) + +envoy_cc_library( + name = "quic_io_handle_wrapper_lib", + hdrs = ["quic_io_handle_wrapper.h"], + deps = [ + "//include/envoy/network:io_handle_interface", + "//source/common/network:io_socket_error_lib", + ], +) + +envoy_cc_library( + name = "envoy_quic_connection_lib", + srcs = ["envoy_quic_connection.cc"], + hdrs = ["envoy_quic_connection.h"], + tags = ["nofips"], + deps = [ + ":quic_io_handle_wrapper_lib", + "//include/envoy/network:connection_interface", + "//source/common/network:listen_socket_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_utils_lib", + "//source/server:connection_handler_lib", + "@com_googlesource_quiche//:quic_core_connection_lib", + ], +) + +envoy_cc_library( + name = "envoy_quic_dispatcher_lib", + srcs = [ + "envoy_quic_dispatcher.cc", + ], + hdrs = [ + "envoy_quic_dispatcher.h", + ], + tags = ["nofips"], + deps = [ + ":envoy_quic_connection_lib", + ":envoy_quic_proof_source_lib", + ":envoy_quic_server_session_lib", + "//include/envoy/network:listener_interface", + "//source/server:connection_handler_lib", + "@com_googlesource_quiche//:quic_core_server_lib", + "@com_googlesource_quiche//:quic_core_utils_lib", + ], +) + envoy_cc_library( name = "envoy_quic_utils_lib", + srcs = ["envoy_quic_utils.cc"], hdrs = ["envoy_quic_utils.h"], external_deps = ["quiche_quic_platform"], tags = ["nofips"], - deps = ["//source/common/network:address_lib"], + deps = [ + "//include/envoy/http:codec_interface", + "//source/common/http:header_map_lib", + "//source/common/network:address_lib", + "@com_googlesource_quiche//:quic_core_http_header_list_lib", + ], ) diff --git a/source/extensions/quic_listeners/quiche/codec_impl.cc b/source/extensions/quic_listeners/quiche/codec_impl.cc new file mode 100644 index 000000000000..4d9846ac6b26 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/codec_impl.cc @@ -0,0 +1,23 @@ +#include "extensions/quic_listeners/quiche/codec_impl.h" + +namespace Envoy { +namespace Quic { + +void QuicHttpConnectionImplBase::goAway() { + quic_session_.SendGoAway(quic::QUIC_PEER_GOING_AWAY, "server shutdown imminent"); +} + +bool QuicHttpConnectionImplBase::wantsToWrite() { return quic_session_.HasDataToWrite(); } + +// TODO(danzh): modify QUIC stack to react based on aggregated bytes across all +// the streams. And call StreamCallbackHelper::runHighWatermarkCallbacks() for each stream. +void QuicHttpConnectionImplBase::onUnderlyingConnectionAboveWriteBufferHighWatermark() { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +void QuicHttpConnectionImplBase::onUnderlyingConnectionBelowWriteBufferLowWatermark() { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/codec_impl.h b/source/extensions/quic_listeners/quiche/codec_impl.h new file mode 100644 index 000000000000..4920b50a0247 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/codec_impl.h @@ -0,0 +1,58 @@ +#include "envoy/http/codec.h" + +#include "common/common/assert.h" +#include "common/common/logger.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" + +namespace Envoy { +namespace Quic { + +// QuicHttpConnectionImplBase instance is a thin QUIC codec just providing quic interface to HCM. +// Owned by HCM and created during onNewConnection() if the network connection +// is a QUIC connection. +class QuicHttpConnectionImplBase : public virtual Http::Connection, + protected Logger::Loggable { +public: + QuicHttpConnectionImplBase(EnvoyQuicServerSession& quic_session) : quic_session_(quic_session) {} + + // Http::Connection + void dispatch(Buffer::Instance& /*data*/) override { + // Bypassed. QUIC connection already hands all data to streams. + NOT_REACHED_GCOVR_EXCL_LINE; + } + Http::Protocol protocol() override { + // From HCM's view, QUIC should behave the same as Http2, only the stats + // should be different. + // TODO(danzh) add Http3 enum value for QUIC. + return Http::Protocol::Http2; + } + void goAway() override; + // Returns true if the session has data to send but queued in connection or + // stream send buffer. + bool wantsToWrite() override; + void onUnderlyingConnectionAboveWriteBufferHighWatermark() override; + void onUnderlyingConnectionBelowWriteBufferLowWatermark() override; + +protected: + EnvoyQuicServerSession& quic_session_; +}; + +class QuicHttpServerConnectionImpl : public QuicHttpConnectionImplBase, + public Http::ServerConnection { +public: + QuicHttpServerConnectionImpl(EnvoyQuicServerSession& quic_session, + Http::ServerConnectionCallbacks& callbacks) + : QuicHttpConnectionImplBase(quic_session) { + quic_session.setHttpConnectionCallbacks(callbacks); + } + + // Http::Connection + void shutdownNotice() override { + // TODO(danzh): Add double-GOAWAY support in QUIC. + ENVOY_CONN_LOG(error, "Shutdown notice is not propagated to QUIC.", quic_session_); + } +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_connection.cc b/source/extensions/quic_listeners/quiche/envoy_quic_connection.cc new file mode 100644 index 000000000000..95e97e5f6966 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_connection.cc @@ -0,0 +1,59 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_connection.h" + +#include "common/network/listen_socket_impl.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/quic_listeners/quiche/quic_io_handle_wrapper.h" +#include "extensions/transport_sockets/well_known_names.h" + +namespace Envoy { +namespace Quic { + +EnvoyQuicConnection::EnvoyQuicConnection( + const quic::QuicConnectionId& server_connection_id, + quic::QuicSocketAddress initial_peer_address, quic::QuicConnectionHelperInterface& helper, + quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter& writer, bool owns_writer, + quic::Perspective perspective, const quic::ParsedQuicVersionVector& supported_versions, + Network::ListenerConfig& listener_config, Server::ListenerStats& listener_stats) + : quic::QuicConnection(server_connection_id, initial_peer_address, &helper, &alarm_factory, + &writer, owns_writer, perspective, supported_versions), + listener_config_(listener_config), listener_stats_(listener_stats) {} + +bool EnvoyQuicConnection::OnPacketHeader(const quic::QuicPacketHeader& header) { + if (quic::QuicConnection::OnPacketHeader(header) && connection_socket_ == nullptr) { + ASSERT(self_address().IsInitialized()); + // Self address should be initialized by now. It's time to install filters. + Network::Address::InstanceConstSharedPtr local_addr = + quicAddressToEnvoyAddressInstance(self_address()); + + Network::Address::InstanceConstSharedPtr remote_addr = + quicAddressToEnvoyAddressInstance(peer_address()); + connection_socket_ = std::make_unique( + // Wraps the real IoHandle instance so that if this socket gets closed, + // the real IoHandle won't be affected. + std::make_unique(listener_config_.socket().ioHandle()), + std::move(local_addr), std::move(remote_addr)); + connection_socket_->setDetectedTransportProtocol( + Extensions::TransportSockets::TransportSocketNames::get().Quic); + + filter_chain_ = listener_config_.filterChainManager().findFilterChain(*connection_socket_); + if (filter_chain_ == nullptr) { + listener_stats_.no_filter_chain_match_.inc(); + CloseConnection(quic::QUIC_CRYPTO_INTERNAL_ERROR, + "closing connection: no matching filter chain found for handshake", + quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); + return false; + } + const bool empty_filter_chain = !listener_config_.filterChainFactory().createNetworkFilterChain( + *envoy_connection_, filter_chain_->networkFilterFactories()); + if (empty_filter_chain) { + CloseConnection(quic::QUIC_CRYPTO_INTERNAL_ERROR, "closing connection: filter chain is empty", + quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); + return false; + } + } + return true; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_connection.h b/source/extensions/quic_listeners/quiche/envoy_quic_connection.h new file mode 100644 index 000000000000..04381bf383d1 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_connection.h @@ -0,0 +1,71 @@ +#pragma once + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/core/quic_connection.h" + +#pragma GCC diagnostic pop + +#include + +#include "common/common/logger.h" +#include "envoy/network/connection.h" +#include "envoy/network/listener.h" +#include "server/connection_handler_impl.h" + +namespace Envoy { +namespace Quic { + +// Derived for network filter chain, stats and QoS. This is used on both client +// and server side. +class EnvoyQuicConnection : public quic::QuicConnection, + protected Logger::Loggable { +public: + EnvoyQuicConnection(const quic::QuicConnectionId& server_connection_id, + quic::QuicSocketAddress initial_peer_address, + quic::QuicConnectionHelperInterface& helper, + quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter& writer, + bool owns_writer, quic::Perspective perspective, + const quic::ParsedQuicVersionVector& supported_versions, + Network::ListenerConfig& listener_config, + Server::ListenerStats& listener_stats); + + ~EnvoyQuicConnection() override = default; + + // Called by EnvoyQuicSession::setConnectionStats(). + void setConnectionStats(const Network::Connection::ConnectionStats& stats) { + connection_stats_ = std::make_unique(stats); + } + + // quic::QuicConnection + // Overridden to retrieve filter chain with initialized self address. + bool OnPacketHeader(const quic::QuicPacketHeader& header) override; + + // Called in session Initialize(). + void setEnvoyConnection(Network::Connection& connection) { envoy_connection_ = &connection; } + + const Network::ConnectionSocketPtr& connectionSocket() const { return connection_socket_; } + +protected: + Network::Connection::ConnectionStats& connectionStats() const { return *connection_stats_; } + +private: + // TODO(danzh): populate stats. + std::unique_ptr connection_stats_; + // Only initialized after self address is known. Must not own the underlying + // socket because UDP socket is shared among all connections. + Network::ConnectionSocketPtr connection_socket_; + Network::Connection* envoy_connection_{nullptr}; + Network::ListenerConfig& listener_config_; + Server::ListenerStats& listener_stats_; + // Latched to the corresponding quic FilterChain after connection_socket_ is + // initialized. + const Network::FilterChain* filter_chain_{nullptr}; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_connection_helper.h b/source/extensions/quic_listeners/quiche/envoy_quic_connection_helper.h new file mode 100644 index 000000000000..6af08fdece7a --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_connection_helper.h @@ -0,0 +1,42 @@ +#pragma once + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wtype-limits" + +#include "quiche/quic/core/crypto/quic_random.h" +#include "quiche/quic/core/quic_connection.h" +#include "quiche/quic/core/quic_simple_buffer_allocator.h" + +#pragma GCC diagnostic pop + +#include "extensions/quic_listeners/quiche/platform/envoy_quic_clock.h" + +namespace Envoy { +namespace Quic { + +// Derived to provide EnvoyQuicClock and default random generator and buffer +// allocator. +class EnvoyQuicConnectionHelper : public quic::QuicConnectionHelperInterface { +public: + EnvoyQuicConnectionHelper(Event::Dispatcher& dispatcher) + : clock_(dispatcher), random_generator_(quic::QuicRandom::GetInstance()) {} + + ~EnvoyQuicConnectionHelper() override = default; + + // QuicConnectionHelperInterface + const quic::QuicClock* GetClock() const override { return &clock_; } + quic::QuicRandom* GetRandomGenerator() override { return random_generator_; } + quic::QuicBufferAllocator* GetStreamSendBufferAllocator() override { return &buffer_allocator_; } + +private: + EnvoyQuicClock clock_; + quic::QuicRandom* random_generator_ = nullptr; + quic::SimpleBufferAllocator buffer_allocator_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc new file mode 100644 index 000000000000..4ceace4ce48f --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc @@ -0,0 +1,58 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_dispatcher.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" + +namespace Envoy { +namespace Quic { + +EnvoyQuicDispatcher::EnvoyQuicDispatcher( + const quic::QuicCryptoServerConfig* crypto_config, quic::QuicVersionManager* version_manager, + std::unique_ptr helper, + std::unique_ptr alarm_factory, + uint8_t expected_server_connection_id_length, Server::ConnectionHandlerImpl& connection_handler, + Network::ListenerConfig& listener_config, Server::ListenerStats& listener_stats) + : quic::QuicDispatcher(&quic_config_, crypto_config, version_manager, std::move(helper), + std::make_unique(), + std::move(alarm_factory), expected_server_connection_id_length), + connection_handler_(connection_handler), listener_config_(listener_config), + listener_stats_(listener_stats) { + // Turn off chlo buffering in QuicDispatcher because per event loop clean + // up is not implemented. + // TODO(danzh): Add a per event loop callback to + // Network::UdpListenerCallbacks which should be called at the beginning + // of HandleReadEvent(). And this callback should call quic::Dispatcher::ProcessBufferedChlos(). + SetQuicFlag(FLAGS_quic_allow_chlo_buffering, false); +} + +void EnvoyQuicDispatcher::OnConnectionClosed(quic::QuicConnectionId connection_id, + quic::QuicErrorCode error, + const std::string& error_details, + quic::ConnectionCloseSource source) { + quic::QuicDispatcher::OnConnectionClosed(connection_id, error, error_details, source); + ASSERT(connection_handler_.num_connections_ > 0); + --connection_handler_.num_connections_; +} + +quic::QuicSession* EnvoyQuicDispatcher::CreateQuicSession( + quic::QuicConnectionId server_connection_id, const quic::QuicSocketAddress& peer_address, + quic::QuicStringPiece /*alpn*/, const quic::ParsedQuicVersion& version) { + auto quic_connection = std::make_unique( + server_connection_id, peer_address, *helper(), *alarm_factory(), *writer(), + /*owns_writer=*/false, quic::Perspective::IS_SERVER, quic::ParsedQuicVersionVector{version}, + listener_config_, listener_stats_); + auto quic_session = new EnvoyQuicServerSession( + config(), quic::ParsedQuicVersionVector{version}, std::move(quic_connection), this, + session_helper(), crypto_config(), compressed_certs_cache(), connection_handler_.dispatcher_); + quic_session->Initialize(); + // Filter chain can't be retrieved here as self address is unknown at this + // point. + // TODO(danzh): change QUIC interface to pass in self address as it is already + // known. In this way, filter chain can be retrieved at this point. But one + // thing to pay attention is that if the retrival fails, connection needs to + // be closed, and it should be added to time wait list instead of session map. + ++connection_handler_.num_connections_; + return quic_session; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h new file mode 100644 index 000000000000..5797ca544755 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.h @@ -0,0 +1,78 @@ +#pragma once + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wtype-limits" + +#include "quiche/quic/core/quic_dispatcher.h" +#include "quiche/quic/core/quic_utils.h" + +#pragma GCC diagnostic pop + +#include + +#include "envoy/network/listener.h" +#include "server/connection_handler_impl.h" + +namespace Envoy { +namespace Quic { + +// Envoy specific provider of server connection id and decision maker of +// accepting new connection or not. +class EnvoyQuicCryptoServerStreamHelper : public quic::QuicCryptoServerStream::Helper { +public: + ~EnvoyQuicCryptoServerStreamHelper() override = default; + + // quic::QuicCryptoServerStream::Helper + quic::QuicConnectionId + GenerateConnectionIdForReject(quic::QuicTransportVersion /*version*/, + quic::QuicConnectionId /*connection_id*/) const override { + // TODO(danzh): create reject connection id based on given connection_id. + return quic::QuicUtils::CreateRandomConnectionId(); + } + + bool CanAcceptClientHello(const quic::CryptoHandshakeMessage& /*message*/, + const quic::QuicSocketAddress& /*client_address*/, + const quic::QuicSocketAddress& /*peer_address*/, + const quic::QuicSocketAddress& /*self_address*/, + std::string* /*error_details*/) const override { + // TODO(danzh): decide to accept or not based on information from given handshake message, i.e. + // user agent and SNI. + return true; + } +}; + +class EnvoyQuicDispatcher : public quic::QuicDispatcher { +public: + EnvoyQuicDispatcher(const quic::QuicCryptoServerConfig* crypto_config, + quic::QuicVersionManager* version_manager, + std::unique_ptr helper, + std::unique_ptr alarm_factory, + uint8_t expected_server_connection_id_length, + Server::ConnectionHandlerImpl& connection_handler, + Network::ListenerConfig& listener_config, + Server::ListenerStats& listener_stats); + + void OnConnectionClosed(quic::QuicConnectionId connection_id, quic::QuicErrorCode error, + const std::string& error_details, + quic::ConnectionCloseSource source) override; + +protected: + quic::QuicSession* CreateQuicSession(quic::QuicConnectionId server_connection_id, + const quic::QuicSocketAddress& peer_address, + quic::QuicStringPiece alpn, + const quic::ParsedQuicVersion& version) override; + +private: + // TODO(danzh): initialize from Envoy config. + quic::QuicConfig quic_config_; + Server::ConnectionHandlerImpl& connection_handler_; + Network::ListenerConfig& listener_config_; + Server::ListenerStats& listener_stats_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h index 455c76aebc41..f55ae9656221 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h @@ -49,9 +49,9 @@ class EnvoyQuicFakeProofSource : public quic::ProofSource { // Returns a certs chain with a fake certificate "Fake cert from [host_name]". quic::QuicReferenceCountedPointer GetCertChain(const quic::QuicSocketAddress& /*server_address*/, - const std::string& hostname) override { + const std::string& /*hostname*/) override { std::vector certs; - certs.push_back(absl::StrCat("Fake cert from ", hostname)); + certs.push_back(absl::StrCat("Fake cert")); return quic::QuicReferenceCountedPointer( new quic::ProofSource::Chain(certs)); } diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h index 09cbf73b4a38..0861e09fb4d9 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_fake_proof_verifier.h @@ -43,13 +43,12 @@ class EnvoyQuicFakeProofVerifier : public quic::ProofVerifier { // Return success if the certs chain has only one fake certificate "Fake cert from [host_name]" // and its SCT is "Fake timestamp". Otherwise failure. quic::QuicAsyncStatus - VerifyCertChain(const std::string& hostname, const std::vector& certs, + VerifyCertChain(const std::string& /*hostname*/, const std::vector& certs, const std::string& /*ocsp_response*/, const std::string& cert_sct, const quic::ProofVerifyContext* /*context*/, std::string* /*error_details*/, std::unique_ptr* /*details*/, std::unique_ptr /*callback*/) override { - std::string cert = absl::StrCat("Fake cert from ", hostname); - if (cert_sct == "Fake timestamp" && certs.size() == 1 && certs[0] == cert) { + if (cert_sct == "Fake timestamp" && certs.size() == 1 && certs[0] == "Fake cert") { return quic::QUIC_SUCCESS; } return quic::QUIC_FAILURE; diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_session.cc b/source/extensions/quic_listeners/quiche/envoy_quic_server_session.cc new file mode 100644 index 000000000000..a6452fca1b12 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_session.cc @@ -0,0 +1,190 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/core/quic_crypto_server_stream.h" +#pragma GCC diagnostic pop + +#include "extensions/quic_listeners/quiche/envoy_quic_server_stream.h" + +namespace Envoy { +namespace Quic { + +EnvoyQuicServerSession::EnvoyQuicServerSession( + const quic::QuicConfig& config, const quic::ParsedQuicVersionVector& supported_versions, + std::unique_ptr connection, quic::QuicSession::Visitor* visitor, + quic::QuicCryptoServerStream::Helper* helper, const quic::QuicCryptoServerConfig* crypto_config, + quic::QuicCompressedCertsCache* compressed_certs_cache, Event::Dispatcher& dispatcher) + : quic::QuicServerSessionBase(config, supported_versions, connection.get(), visitor, helper, + crypto_config, compressed_certs_cache), + quic_connection_(std::move(connection)), filter_manager_(*this), dispatcher_(dispatcher), + stream_info_(dispatcher.timeSource()) { + // TODO(danzh): Use QUIC specific enum value. + stream_info_.protocol(Http::Protocol::Http2); +} + +quic::QuicCryptoServerStreamBase* EnvoyQuicServerSession::CreateQuicCryptoServerStream( + const quic::QuicCryptoServerConfig* crypto_config, + quic::QuicCompressedCertsCache* compressed_certs_cache) { + return new quic::QuicCryptoServerStream(crypto_config, compressed_certs_cache, this, + stream_helper()); +} + +quic::QuicSpdyStream* EnvoyQuicServerSession::CreateIncomingStream(quic::QuicStreamId id) { + if (!ShouldCreateIncomingStream(id)) { + return nullptr; + } + auto stream = new EnvoyQuicServerStream(id, this, quic::BIDIRECTIONAL); + ActivateStream(absl::WrapUnique(stream)); + setUpRequestDecoder(*stream); + return stream; +} + +quic::QuicSpdyStream* +EnvoyQuicServerSession::CreateIncomingStream(quic::PendingStream* /*pending*/) { + // Only client side server push stream should trigger this call. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +quic::QuicSpdyStream* EnvoyQuicServerSession::CreateOutgoingBidirectionalStream() { + // Disallow server initiated stream. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +quic::QuicSpdyStream* EnvoyQuicServerSession::CreateOutgoingUnidirectionalStream() { + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void EnvoyQuicServerSession::setUpRequestDecoder(EnvoyQuicStream& stream) { + ASSERT(http_connection_callbacks_ != nullptr); + Http::StreamDecoder& decoder = http_connection_callbacks_->newStream(stream); + stream.setDecoder(decoder); +} + +void EnvoyQuicServerSession::OnConnectionClosed(const quic::QuicConnectionCloseFrame& frame, + quic::ConnectionCloseSource source) { + quic::QuicServerSessionBase::OnConnectionClosed(frame, source); + for (auto callback : network_connection_callbacks_) { + // Tell filters about connection close. + callback->onEvent(source == quic::ConnectionCloseSource::FROM_PEER + ? Network::ConnectionEvent::RemoteClose + : Network::ConnectionEvent::LocalClose); + } + transport_failure_reason_ = absl::StrCat(quic::QuicErrorCodeToString(frame.quic_error_code), + " with details: ", frame.error_details); +} + +void EnvoyQuicServerSession::Initialize() { + quic::QuicServerSessionBase::Initialize(); + quic_connection_->setEnvoyConnection(*this); +} + +void EnvoyQuicServerSession::SendGoAway(quic::QuicErrorCode error_code, const std::string& reason) { + if (transport_version() < quic::QUIC_VERSION_99) { + quic::QuicServerSessionBase::SendGoAway(error_code, reason); + } +} + +void EnvoyQuicServerSession::addWriteFilter(Network::WriteFilterSharedPtr filter) { + filter_manager_.addWriteFilter(filter); +} + +void EnvoyQuicServerSession::addFilter(Network::FilterSharedPtr filter) { + filter_manager_.addFilter(filter); +} + +void EnvoyQuicServerSession::addReadFilter(Network::ReadFilterSharedPtr filter) { + filter_manager_.addReadFilter(filter); +} + +bool EnvoyQuicServerSession::initializeReadFilters() { + return filter_manager_.initializeReadFilters(); +} + +void EnvoyQuicServerSession::addConnectionCallbacks(Network::ConnectionCallbacks& cb) { + network_connection_callbacks_.push_back(&cb); +} + +void EnvoyQuicServerSession::addBytesSentCallback(Network::Connection::BytesSentCb /*cb*/) { + // TODO(danzh): implement to support proxy. This interface is only called from + // TCP proxy code. + ASSERT(false, "addBytesSentCallback is not implemented for QUIC"); +} + +void EnvoyQuicServerSession::enableHalfClose(bool enabled) { + ASSERT(!enabled, "Quic connection doesn't support half close."); +} + +void EnvoyQuicServerSession::setBufferLimits(uint32_t /*limit*/) { + // TODO(danzh): add interface to quic for connection level buffer throttling. + // Currently read buffer is capped by connection level flow control. And + // write buffer is not capped. + ENVOY_CONN_LOG(error, "Quic manages its own buffer currently.", *this); +} + +uint32_t EnvoyQuicServerSession::bufferLimit() const { + // As quic connection is not HTTP1.1, this method shouldn't be called by HCM. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void EnvoyQuicServerSession::close(Network::ConnectionCloseType type) { + if (type != Network::ConnectionCloseType::NoFlush) { + // TODO(danzh): Implement FlushWrite and FlushWriteAndDelay mode. + ENVOY_CONN_LOG(error, "Flush write is not implemented for QUIC.", *this); + } + connection()->CloseConnection(quic::QUIC_NO_ERROR, "Closed by application", + quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); +} + +void EnvoyQuicServerSession::setDelayedCloseTimeout(std::chrono::milliseconds timeout) { + ASSERT(timeout == std::chrono::milliseconds::zero(), + "Delayed close of connection is not supported"); +} + +std::chrono::milliseconds EnvoyQuicServerSession::delayedCloseTimeout() const { + // Not called outside of Network::ConnectionImpl. Maybe remove this interface + // from Network::Connection. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +const Network::ConnectionSocket::OptionsSharedPtr& EnvoyQuicServerSession::socketOptions() const { + ENVOY_CONN_LOG( + error, + "QUIC connection socket is merely a wrapper, and doesn't have any specific socket options.", + *this); + return quic_connection_->connectionSocket()->options(); +} + +absl::string_view EnvoyQuicServerSession::requestedServerName() const { + return {GetCryptoStream()->crypto_negotiated_params().sni}; +} + +const Network::Address::InstanceConstSharedPtr& EnvoyQuicServerSession::remoteAddress() const { + ASSERT(quic_connection_->connectionSocket() != nullptr, + "remoteAddress() should only be called after OnPacketHeader"); + return quic_connection_->connectionSocket()->remoteAddress(); +} + +const Network::Address::InstanceConstSharedPtr& EnvoyQuicServerSession::localAddress() const { + ASSERT(quic_connection_->connectionSocket() != nullptr, + "localAddress() should only be called after OnPacketHeader"); + return quic_connection_->connectionSocket()->localAddress(); +} + +Ssl::ConnectionInfoConstSharedPtr EnvoyQuicServerSession::ssl() const { + // TODO(danzh): construct Ssl::ConnectionInfo from crypto stream + ENVOY_CONN_LOG(error, "Ssl::ConnectionInfo instance is not populated.", *this); + return nullptr; +} + +void EnvoyQuicServerSession::rawWrite(Buffer::Instance& /*data*/, bool /*end_stream*/) { + // Network filter should stop iteration. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_session.h b/source/extensions/quic_listeners/quiche/envoy_quic_server_session.h new file mode 100644 index 000000000000..0cb2d0e84d4e --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_session.h @@ -0,0 +1,163 @@ +#pragma once + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#pragma GCC diagnostic ignored "-Wtype-limits" + +#include "quiche/quic/core/http/quic_server_session_base.h" + +#pragma GCC diagnostic pop + +#include + +#include "common/common/logger.h" +#include "common/common/empty_string.h" +#include "common/network/filter_manager_impl.h" +#include "common/stream_info/stream_info_impl.h" +#include "envoy/network/connection.h" +#include "envoy/event/dispatcher.h" +#include "extensions/quic_listeners/quiche/envoy_quic_stream.h" +#include "extensions/quic_listeners/quiche/envoy_quic_connection.h" + +namespace Envoy { +namespace Quic { + +// Act as a Network::Connection to HCM and a FilterManager to FilterFactoryCb. +class EnvoyQuicServerSession : public quic::QuicServerSessionBase, + public Network::FilterManagerConnection, + protected Logger::Loggable { +public: + EnvoyQuicServerSession(const quic::QuicConfig& config, + const quic::ParsedQuicVersionVector& supported_versions, + std::unique_ptr connection, + quic::QuicSession::Visitor* visitor, + quic::QuicCryptoServerStream::Helper* helper, + const quic::QuicCryptoServerConfig* crypto_config, + quic::QuicCompressedCertsCache* compressed_certs_cache, + Event::Dispatcher& dispatcher); + + // Network::FilterManager + // Overridden to delegate calls to filter_manager_. + void addWriteFilter(Network::WriteFilterSharedPtr filter) override; + void addFilter(Network::FilterSharedPtr filter) override; + void addReadFilter(Network::ReadFilterSharedPtr filter) override; + bool initializeReadFilters() override; + + // Network::Connection + void addConnectionCallbacks(Network::ConnectionCallbacks& cb) override; + void addBytesSentCallback(Network::Connection::BytesSentCb /*cb*/) override; + void enableHalfClose(bool enabled) override; + void close(Network::ConnectionCloseType type) override; + Event::Dispatcher& dispatcher() override { return dispatcher_; } + uint64_t id() const override { + // QUIC connection id can be 18 types. It's easier to use hash value instead + // of trying to map it into a 64-bit space. + return connection_id().Hash(); + } + std::string nextProtocol() const override { return EMPTY_STRING; } + void noDelay(bool /*enable*/) override { + // No-op. TCP_NODELAY doesn't apply to UDP. + } + void setDelayedCloseTimeout(std::chrono::milliseconds timeout) override; + std::chrono::milliseconds delayedCloseTimeout() const override; + void readDisable(bool disable) override { + ASSERT(!disable, "Quic connection should be able to read through out its life time."); + } + void detectEarlyCloseWhenReadDisabled(bool /*value*/) override { NOT_REACHED_GCOVR_EXCL_LINE; } + bool readEnabled() const override { return true; } + const Network::Address::InstanceConstSharedPtr& remoteAddress() const override; + const Network::Address::InstanceConstSharedPtr& localAddress() const override; + absl::optional + unixSocketPeerCredentials() const override { + ASSERT(false, "Unix domain socket is not supported."); + return absl::nullopt; + } + void setConnectionStats(const Network::Connection::ConnectionStats& stats) override { + stats_ = std::make_unique(stats); + quic_connection_->setConnectionStats(stats); + } + Ssl::ConnectionInfoConstSharedPtr ssl() const override; + Network::Connection::State state() const override { + return connection()->connected() ? Network::Connection::State::Open + : Network::Connection::State::Closed; + } + void write(Buffer::Instance& /*data*/, bool /*end_stream*/) override { + // All writes should be handled by Quic internally. + NOT_REACHED_GCOVR_EXCL_LINE; + } + void setBufferLimits(uint32_t limit) override; + uint32_t bufferLimit() const override; + bool localAddressRestored() const override { + // SO_ORIGINAL_DST not supported by QUIC. + return false; + } + bool aboveHighWatermark() const override { + ENVOY_CONN_LOG(error, "QUIC doesn't have connection level write buffer limit.", *this); + return false; + } + const Network::ConnectionSocket::OptionsSharedPtr& socketOptions() const override; + absl::string_view requestedServerName() const override; + StreamInfo::StreamInfo& streamInfo() override { return stream_info_; } + const StreamInfo::StreamInfo& streamInfo() const override { return stream_info_; } + absl::string_view transportFailureReason() const override { return transport_failure_reason_; } + + // Network::FilterManagerConnection + void rawWrite(Buffer::Instance& data, bool end_stream) override; + + // Network::ReadBufferSource + Network::StreamBuffer getReadBuffer() override { + // Network filter has to stop iteration to prevent hitting this line. + NOT_REACHED_GCOVR_EXCL_LINE; + } + // Network::WriteBufferSource + Network::StreamBuffer getWriteBuffer() override { NOT_REACHED_GCOVR_EXCL_LINE; } + + // Called by QuicHttpServerConnectionImpl before creating data streams. + void setHttpConnectionCallbacks(Http::ServerConnectionCallbacks& callbacks) { + http_connection_callbacks_ = &callbacks; + } + + // quic::QuicSession + void OnConnectionClosed(const quic::QuicConnectionCloseFrame& frame, + quic::ConnectionCloseSource source) override; + void Initialize() override; + void SendGoAway(quic::QuicErrorCode error_code, const std::string& reason) override; + +protected: + // quic::QuicServerSessionBase + quic::QuicCryptoServerStreamBase* + CreateQuicCryptoServerStream(const quic::QuicCryptoServerConfig* crypto_config, + quic::QuicCompressedCertsCache* compressed_certs_cache) override; + + // quic::QuicSession + // Overridden to create stream as encoder and associate it with an decoder. + quic::QuicSpdyStream* CreateIncomingStream(quic::QuicStreamId id) override; + quic::QuicSpdyStream* CreateIncomingStream(quic::PendingStream* pending) override; + quic::QuicSpdyStream* CreateOutgoingBidirectionalStream() override; + quic::QuicSpdyStream* CreateOutgoingUnidirectionalStream() override; + +private: + void setUpRequestDecoder(EnvoyQuicStream& stream); + + std::unique_ptr quic_connection_; + // Currently ConnectionManagerImpl is the one and only filter. If more network + // filters are added, ConnectionManagerImpl should always be the last one. + // Its onRead() is only called once to trigger ReadFilter::onNewConnection() + // and the rest incoming data bypasses these filters. + Network::FilterManagerImpl filter_manager_; + Event::Dispatcher& dispatcher_; + StreamInfo::StreamInfoImpl stream_info_; + std::string transport_failure_reason_; + // TODO(danzh): populate stats. + std::unique_ptr stats_; + // These callbacks are owned by network filters and quic session should out live + // them. + Http::ServerConnectionCallbacks* http_connection_callbacks_{nullptr}; + std::list network_connection_callbacks_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc b/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc new file mode 100644 index 000000000000..35bd63f81135 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.cc @@ -0,0 +1,132 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_server_stream.h" + +#include +#include + +#include + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/core/http/quic_header_list.h" +#include "quiche/quic/core/quic_session.h" +#include "quiche/spdy/core/spdy_header_block.h" + +#pragma GCC diagnostic pop + +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "common/buffer/buffer_impl.h" +#include "common/http/header_map_impl.h" +#include "common/common/assert.h" + +namespace Envoy { +namespace Quic { + +void EnvoyQuicServerStream::encode100ContinueHeaders(const Http::HeaderMap& headers) { + ASSERT(headers.Status()->value() == "100"); + encodeHeaders(headers, false); +} +void EnvoyQuicServerStream::encodeHeaders(const Http::HeaderMap& /*headers*/, bool /*end_stream*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} +void EnvoyQuicServerStream::encodeData(Buffer::Instance& /*data*/, bool /*end_stream*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} +void EnvoyQuicServerStream::encodeTrailers(const Http::HeaderMap& /*trailers*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} +void EnvoyQuicServerStream::encodeMetadata(const Http::MetadataMapVector& /*metadata_map_vector*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +void EnvoyQuicServerStream::resetStream(Http::StreamResetReason /*reason*/) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +void EnvoyQuicServerStream::readDisable(bool /*disable*/) { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + +void EnvoyQuicServerStream::OnInitialHeadersComplete(bool fin, size_t frame_len, + const quic::QuicHeaderList& header_list) { + quic::QuicSpdyServerStreamBase::OnInitialHeadersComplete(fin, frame_len, header_list); + ASSERT(decoder() != nullptr); + ASSERT(headers_decompressed()); + decoder()->decodeHeaders(quicHeadersToEnvoyHeaders(header_list), /*end_stream=*/fin); + ConsumeHeaderList(); +} + +void EnvoyQuicServerStream::OnBodyAvailable() { + Buffer::InstancePtr buffer = std::make_unique(); + // TODO(danzh): check Envoy per stream buffer limit. + // Currently read out all the data. + while (HasBytesToRead()) { + struct iovec iov; + int num_regions = GetReadableRegions(&iov, 1); + ASSERT(num_regions > 0); + size_t bytes_read = iov.iov_len; + Buffer::RawSlice slice; + buffer->reserve(bytes_read, &slice, 1); + ASSERT(slice.len_ >= bytes_read); + slice.len_ = bytes_read; + memcpy(slice.mem_, iov.iov_base, iov.iov_len); + buffer->commit(&slice, 1); + MarkConsumed(bytes_read); + } + + // True if no trailer and FIN read. + bool finished_reading = IsDoneReading(); + // If this is the last stream data, set end_stream if there is no + // trailers. + ASSERT(decoder() != nullptr); + decoder()->decodeData(*buffer, finished_reading); + if (!quic::VersionUsesQpack(transport_version()) && sequencer()->IsClosed() && + !FinishedReadingTrailers()) { + // For Google QUIC implementation, trailers may arrived earlier and wait to + // be consumed after reading all the body. Consume it here. + // IETF QUIC shouldn't reach here because trailers are sent on same stream. + decoder()->decodeTrailers(spdyHeaderBlockToEnvoyHeaders(received_trailers())); + MarkTrailersConsumed(); + } +} + +void EnvoyQuicServerStream::OnTrailingHeadersComplete(bool fin, size_t frame_len, + const quic::QuicHeaderList& header_list) { + quic::QuicSpdyServerStreamBase::OnTrailingHeadersComplete(fin, frame_len, header_list); + if (session()->connection()->connected() && + (quic::VersionUsesQpack(transport_version()) || sequencer()->IsClosed()) && + !FinishedReadingTrailers()) { + // Before QPack trailers can arrive before body. Only decode trailers after finishing decoding + // body. + ASSERT(decoder() != nullptr); + decoder()->decodeTrailers(spdyHeaderBlockToEnvoyHeaders(received_trailers())); + MarkTrailersConsumed(); + } +} + +void EnvoyQuicServerStream::OnStreamReset(const quic::QuicRstStreamFrame& frame) { + quic::QuicSpdyServerStreamBase::OnStreamReset(frame); + Http::StreamResetReason reason; + if (frame.error_code == quic::QUIC_REFUSED_STREAM) { + reason = Http::StreamResetReason::RemoteRefusedStreamReset; + } else { + reason = Http::StreamResetReason::RemoteReset; + } + runResetCallbacks(reason); +} + +void EnvoyQuicServerStream::OnConnectionClosed(quic::QuicErrorCode error, + quic::ConnectionCloseSource source) { + quic::QuicSpdyServerStreamBase::OnConnectionClosed(error, source); + Http::StreamResetReason reason; + if (error == quic::QUIC_NO_ERROR) { + reason = Http::StreamResetReason::ConnectionTermination; + } else { + reason = Http::StreamResetReason::ConnectionFailure; + } + runResetCallbacks(reason); +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.h b/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.h new file mode 100644 index 000000000000..047970f4fdbe --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_stream.h @@ -0,0 +1,52 @@ +#pragma once + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" +#include "quiche/quic/core/http/quic_spdy_server_stream_base.h" + +#pragma GCC diagnostic pop + +#include "extensions/quic_listeners/quiche/envoy_quic_stream.h" + +namespace Envoy { +namespace Quic { + +// This class is a quic stream and also a response encoder. +class EnvoyQuicServerStream : public quic::QuicSpdyServerStreamBase, public EnvoyQuicStream { +public: + EnvoyQuicServerStream(quic::QuicStreamId id, quic::QuicSpdySession* session, + quic::StreamType type) + : quic::QuicSpdyServerStreamBase(id, session, type) {} + EnvoyQuicServerStream(quic::PendingStream* pending, quic::QuicSpdySession* session, + quic::StreamType type) + : quic::QuicSpdyServerStreamBase(pending, session, type) {} + + // Http::StreamEncoder + void encode100ContinueHeaders(const Http::HeaderMap& headers) override; + void encodeHeaders(const Http::HeaderMap& headers, bool end_stream) override; + void encodeData(Buffer::Instance& data, bool end_stream) override; + void encodeTrailers(const Http::HeaderMap& trailers) override; + void encodeMetadata(const Http::MetadataMapVector& metadata_map_vector) override; + + // Http::Stream + void resetStream(Http::StreamResetReason reason) override; + void readDisable(bool disable) override; + // quic::QuicSpdyStream + void OnBodyAvailable() override; + void OnStreamReset(const quic::QuicRstStreamFrame& frame) override; + // quic::QuicServerSessionBase + void OnConnectionClosed(quic::QuicErrorCode error, quic::ConnectionCloseSource source) override; + +protected: + // quic::QuicSpdyStream + void OnInitialHeadersComplete(bool fin, size_t frame_len, + const quic::QuicHeaderList& header_list) override; + void OnTrailingHeadersComplete(bool fin, size_t frame_len, + const quic::QuicHeaderList& header_list) override; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_stream.h b/source/extensions/quic_listeners/quiche/envoy_quic_stream.h new file mode 100644 index 000000000000..a6126224cc68 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_stream.h @@ -0,0 +1,42 @@ +#pragma once + +#include "envoy/http/codec.h" + +#include "common/http/codec_helper.h" + +namespace Envoy { +namespace Quic { + +// Base class for EnvoyQuicServer|ClientStream. +class EnvoyQuicStream : public Http::StreamEncoder, + public Http::Stream, + public Http::StreamCallbackHelper { +public: + // Http::StreamEncoder + Stream& getStream() override { return *this; } + + // Http::Stream + void addCallbacks(Http::StreamCallbacks& callbacks) override { + ASSERT(!local_end_stream_); + addCallbacks_(callbacks); + } + void removeCallbacks(Http::StreamCallbacks& callbacks) override { removeCallbacks_(callbacks); } + uint32_t bufferLimit() override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + // Needs to be called during quic stream creation before the stream receives + // any headers and data. + void setDecoder(Http::StreamDecoder& decoder) { decoder_ = &decoder; } + +protected: + Http::StreamDecoder* decoder() { + ASSERT(decoder_ != nullptr); + return decoder_; + } + +private: + // Not owned. + Http::StreamDecoder* decoder_{nullptr}; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc b/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc new file mode 100644 index 000000000000..33e0c43fc035 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_utils.cc @@ -0,0 +1,66 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" + +#include + +namespace Envoy { +namespace Quic { + +// TODO(danzh): this is called on each write. Consider to return an address instance on the stack if +// the heap allocation is too expensive. +Network::Address::InstanceConstSharedPtr +quicAddressToEnvoyAddressInstance(const quic::QuicSocketAddress& quic_address) { + return quic_address.IsInitialized() + ? Network::Address::addressFromSockAddr(quic_address.generic_address(), + quic_address.host().address_family() == + quic::IpAddressFamily::IP_V4 + ? sizeof(sockaddr_in) + : sizeof(sockaddr_in6), + false) + : nullptr; +} + +quic::QuicSocketAddress envoyAddressInstanceToQuicSocketAddress( + const Network::Address::InstanceConstSharedPtr& envoy_address) { + ASSERT(envoy_address != nullptr && envoy_address->type() == Network::Address::Type::Ip); + uint32_t port = envoy_address->ip()->port(); + sockaddr_storage ss; + if (envoy_address->ip()->version() == Network::Address::IpVersion::v4) { + auto ipv4_addr = reinterpret_cast(&ss); + memset(ipv4_addr, 0, sizeof(sockaddr_in)); + ipv4_addr->sin_family = AF_INET; + ipv4_addr->sin_port = htons(port); + ipv4_addr->sin_addr.s_addr = envoy_address->ip()->ipv4()->address(); + } else { + auto ipv6_addr = reinterpret_cast(&ss); + memset(ipv6_addr, 0, sizeof(sockaddr_in6)); + ipv6_addr->sin6_family = AF_INET6; + ipv6_addr->sin6_port = htons(port); + ASSERT(sizeof(ipv6_addr->sin6_addr.s6_addr) == 16u); + *reinterpret_cast(ipv6_addr->sin6_addr.s6_addr) = + envoy_address->ip()->ipv6()->address(); + } + return quic::QuicSocketAddress(ss); +} + +Http::HeaderMapImplPtr quicHeadersToEnvoyHeaders(const quic::QuicHeaderList& header_list) { + Http::HeaderMapImplPtr headers = std::make_unique(); + for (const auto& entry : header_list) { + // TODO(danzh): Avoid copy by referencing entry as header_list is already validated by QUIC. + headers->addCopy(Http::LowerCaseString(entry.first), entry.second); + } + return headers; +} + +Http::HeaderMapImplPtr spdyHeaderBlockToEnvoyHeaders(const spdy::SpdyHeaderBlock& header_block) { + Http::HeaderMapImplPtr headers = std::make_unique(); + for (auto entry : header_block) { + // TODO(danzh): Avoid temporary strings and addCopy() with std::string_view. + std::string key(entry.first); + std::string value(entry.second); + headers->addCopy(Http::LowerCaseString(key), value); + } + return headers; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_utils.h b/source/extensions/quic_listeners/quiche/envoy_quic_utils.h index 2d411aa8dc98..54b1bf07f603 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_utils.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_utils.h @@ -1,8 +1,11 @@ -#include +#include "envoy/http/codec.h" #include "common/common/assert.h" +#include "common/http/header_map_impl.h" #include "common/network/address_impl.h" +#include "quiche/quic/core/http/quic_header_list.h" +#include "quiche/quic/core/quic_error_codes.h" #include "quiche/quic/platform/api/quic_ip_address.h" #include "quiche/quic/platform/api/quic_socket_address.h" @@ -11,18 +14,16 @@ namespace Quic { // TODO(danzh): this is called on each write. Consider to return an address instance on the stack if // the heap allocation is too expensive. -inline Network::Address::InstanceConstSharedPtr -quicAddressToEnvoyAddressInstance(const quic::QuicSocketAddress& quic_address) { - ASSERT(quic_address.host().address_family() != quic::IpAddressFamily::IP_UNSPEC); - return quic_address.IsInitialized() - ? Network::Address::addressFromSockAddr(quic_address.generic_address(), - quic_address.host().address_family() == - quic::IpAddressFamily::IP_V4 - ? sizeof(sockaddr_in) - : sizeof(sockaddr_in6), - false) - : nullptr; -} +Network::Address::InstanceConstSharedPtr +quicAddressToEnvoyAddressInstance(const quic::QuicSocketAddress& quic_address); + +quic::QuicSocketAddress envoyAddressInstanceToQuicSocketAddress( + const Network::Address::InstanceConstSharedPtr& envoy_address); + +// The returned header map has all keys in lower case. +Http::HeaderMapImplPtr quicHeadersToEnvoyHeaders(const quic::QuicHeaderList& header_list); + +Http::HeaderMapImplPtr spdyHeaderBlockToEnvoyHeaders(const spdy::SpdyHeaderBlock& header_block); } // namespace Quic } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/platform/quic_pcc_sender_impl.h b/source/extensions/quic_listeners/quiche/platform/quic_pcc_sender_impl.h index 011120b1c298..1e0f2bcbaf8b 100644 --- a/source/extensions/quic_listeners/quiche/platform/quic_pcc_sender_impl.h +++ b/source/extensions/quic_listeners/quiche/platform/quic_pcc_sender_impl.h @@ -20,11 +20,11 @@ class RttStats; class SendAlgorithmInterface; // Interface for creating a PCC SendAlgorithmInterface. -SendAlgorithmInterface* CreatePccSenderImpl(const QuicClock* clock, const RttStats* rtt_stats, - const QuicUnackedPacketMap* unacked_packets, - QuicRandom* random, QuicConnectionStats* stats, - QuicPacketCount initial_congestion_window, - QuicPacketCount max_congestion_window) { +inline SendAlgorithmInterface* +CreatePccSenderImpl(const QuicClock* /*clock*/, const RttStats* /*rtt_stats*/, + const QuicUnackedPacketMap* /*unacked_packets*/, QuicRandom* /*random*/, + QuicConnectionStats* /*stats*/, QuicPacketCount /*initial_congestion_window*/, + QuicPacketCount /*max_congestion_window*/) { PANIC("PccSender is not supported."); return nullptr; } diff --git a/source/extensions/quic_listeners/quiche/quic_io_handle_wrapper.h b/source/extensions/quic_listeners/quiche/quic_io_handle_wrapper.h new file mode 100644 index 000000000000..231f1bf08ba8 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/quic_io_handle_wrapper.h @@ -0,0 +1,68 @@ +#include "envoy/network/io_handle.h" + +#include "common/network/io_socket_error_impl.h" + +namespace Envoy { +namespace Quic { + +// A wrapper class around IoHandle object which doesn't close() upon destruction. It is used to +// create ConnectionSocket as the actual IoHandle instance should out live connection socket. +class QuicIoHandleWrapper : public Network::IoHandle { +public: + QuicIoHandleWrapper(Network::IoHandle& io_handle) : io_handle_(io_handle) {} + + // Network::IoHandle + int fd() const override { return io_handle_.fd(); } + Api::IoCallUint64Result close() override { + closed_ = true; + return Api::ioCallUint64ResultNoError(); + } + bool isOpen() const override { return !closed_; } + Api::IoCallUint64Result readv(uint64_t max_length, Buffer::RawSlice* slices, + uint64_t num_slice) override { + if (closed_) { + return Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), + Network::IoSocketError::deleteIoError)); + } + return io_handle_.readv(max_length, slices, num_slice); + } + Api::IoCallUint64Result writev(const Buffer::RawSlice* slices, uint64_t num_slice) override { + if (closed_) { + return Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), + Network::IoSocketError::deleteIoError)); + } + return io_handle_.writev(slices, num_slice); + } + Api::IoCallUint64Result sendto(const Buffer::RawSlice& slice, int flags, + const Network::Address::Instance& address) override { + if (closed_) { + return Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), + Network::IoSocketError::deleteIoError)); + } + return io_handle_.sendto(slice, flags, address); + } + Api::IoCallUint64Result sendmsg(const Buffer::RawSlice* slices, uint64_t num_slice, int flags, + const Envoy::Network::Address::Ip* self_ip, + const Network::Address::Instance& peer_address) override { + if (closed_) { + return Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), + Network::IoSocketError::deleteIoError)); + } + return io_handle_.sendmsg(slices, num_slice, flags, self_ip, peer_address); + } + Api::IoCallUint64Result recvmsg(Buffer::RawSlice* slices, const uint64_t num_slice, + uint32_t self_port, RecvMsgOutput& output) override { + if (closed_) { + return Api::IoCallUint64Result(0, Api::IoErrorPtr(new Network::IoSocketError(EBADF), + Network::IoSocketError::deleteIoError)); + } + return io_handle_.recvmsg(slices, num_slice, self_port, output); + } + +private: + Network::IoHandle& io_handle_; + bool closed_{false}; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/transport_sockets/well_known_names.h b/source/extensions/transport_sockets/well_known_names.h index 3d2270a63230..078337c2566d 100644 --- a/source/extensions/transport_sockets/well_known_names.h +++ b/source/extensions/transport_sockets/well_known_names.h @@ -18,6 +18,7 @@ class TransportSocketNameValues { const std::string Tap = "envoy.transport_sockets.tap"; const std::string RawBuffer = "raw_buffer"; const std::string Tls = "tls"; + const std::string Quic = "quic"; }; using TransportSocketNames = ConstSingleton; diff --git a/source/server/connection_handler_impl.h b/source/server/connection_handler_impl.h index 4c1354e29870..7e09608098e6 100644 --- a/source/server/connection_handler_impl.h +++ b/source/server/connection_handler_impl.h @@ -23,6 +23,12 @@ #include "spdlog/spdlog.h" namespace Envoy { + +namespace Quic { +class ActiveQuicListener; +class EnvoyQuicDispatcher; +} // namespace Quic + namespace Server { #define ALL_LISTENER_STATS(COUNTER, GAUGE, HISTOGRAM) \ @@ -84,6 +90,8 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { }; private: + class ActiveUdpListener; + using ActiveUdpListenerPtr = std::unique_ptr; class ActiveTcpListener; using ActiveTcpListenerPtr = std::unique_ptr; struct ActiveConnection; @@ -91,6 +99,9 @@ class ConnectionHandlerImpl : public Network::ConnectionHandler, NonCopyable { struct ActiveSocket; using ActiveSocketPtr = std::unique_ptr; + friend class Quic::ActiveQuicListener; + friend class Quic::EnvoyQuicDispatcher; + /** * Wrapper for an active tcp listener owned by this handler. */ diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index b588a5c069fa..a9c7c7c54608 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -50,6 +50,71 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "envoy_quic_server_stream_test", + srcs = ["envoy_quic_server_stream_test.cc"], + tags = ["nofips"], + deps = [ + ":quic_test_utils_for_envoy_lib", + "//source/common/http:headers_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_factory_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_connection_helper_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_connection_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_server_stream_lib", + "//test/mocks/http:stream_decoder_mock", + "//test/mocks/network:network_mocks", + "//test/test_common:utility_lib", + "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", + ], +) + +envoy_cc_test( + name = "envoy_quic_server_session_test", + srcs = ["envoy_quic_server_session_test.cc"], + tags = ["nofips"], + deps = [ + ":quic_test_utils_for_envoy_lib", + "//include/envoy/stats:stats_macros", + "//source/extensions/quic_listeners/quiche:codec_impl_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_factory_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_connection_helper_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_source_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_server_session_lib", + "//source/server:configuration_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/http:http_mocks", + "//test/mocks/http:stream_decoder_mock", + "//test/mocks/network:network_mocks", + "//test/mocks/stats:stats_mocks", + "//test/test_common:global_lib", + "//test/test_common:logging_lib", + "//test/test_common:simulated_time_system_lib", + ], +) + +envoy_cc_test( + name = "envoy_quic_dispatcher_test", + srcs = ["envoy_quic_dispatcher_test.cc"], + tags = ["nofips"], + deps = [ + ":quic_test_utils_for_envoy_lib", + "//include/envoy/stats:stats_macros", + "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_factory_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_connection_helper_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_dispatcher_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_proof_source_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_server_session_lib", + "//source/server:configuration_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/http:http_mocks", + "//test/mocks/network:network_mocks", + "//test/mocks/stats:stats_mocks", + "//test/test_common:environment_lib", + "//test/test_common:global_lib", + "//test/test_common:simulated_time_system_lib", + ], +) + envoy_cc_test_library( name = "quic_test_utils_for_envoy_lib", srcs = ["crypto_test_utils_for_envoy.cc"], @@ -60,3 +125,27 @@ envoy_cc_test_library( "@com_googlesource_quiche//:quic_test_tools_test_utils_interface_lib", ], ) + +envoy_cc_test( + name = "quic_io_handle_wrapper_test", + srcs = ["quic_io_handle_wrapper_test.cc"], + tags = ["nofips"], + deps = [ + "//source/extensions/quic_listeners/quiche:quic_io_handle_wrapper_lib", + "//test/mocks/api:api_mocks", + "//test/mocks/network:network_mocks", + "//test/test_common:threadsafe_singleton_injector_lib", + ], +) + +envoy_cc_test( + name = "envoy_quic_utils_test", + srcs = ["envoy_quic_utils_test.cc"], + tags = ["nofips"], + deps = [ + ":quic_test_utils_for_envoy_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_utils_lib", + "//test/mocks/api:api_mocks", + "//test/test_common:threadsafe_singleton_injector_lib", + ], +) diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc new file mode 100644 index 000000000000..295f743a12a0 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc @@ -0,0 +1,262 @@ +#include + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/core/quic_dispatcher.h" +#include "quiche/quic/test_tools/crypto_test_utils.h" +#include "quiche/quic/test_tools/quic_test_utils.h" +#include "quiche/quic/platform/api/quic_text_utils.h" +#pragma GCC diagnostic pop + +#include + +#include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" +#include "common/network/listen_socket_impl.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/environment.h" +#include "test/mocks/network/mocks.h" +#include "test/test_common/utility.h" +#include "test/test_common/network_utility.h" +#include "extensions/quic_listeners/quiche/platform/envoy_quic_clock.h" +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/quic_listeners/quiche/envoy_quic_dispatcher.h" +#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h" +#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/transport_sockets/well_known_names.h" +#include "server/configuration_impl.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::Invoke; +using testing::Return; +using testing::ReturnRef; + +namespace quic { +namespace test { +class QuicDispatcherPeer { +public: + static quic::QuicTimeWaitListManager* time_wait_list_manager(QuicDispatcher* dispatcher) { + return dispatcher->time_wait_list_manager_.get(); + } +}; + +} // namespace test +} // namespace quic + +namespace Envoy { +namespace Quic { + +class EnvoyQuicDispatcherTest : public testing::TestWithParam, + protected Logger::Loggable { +public: + EnvoyQuicDispatcherTest() + : version_(GetParam()), api_(Api::createApiForTest(time_system_)), + dispatcher_(api_->allocateDispatcher()), connection_helper_(*dispatcher_), + crypto_config_(quic::QuicCryptoServerConfig::TESTING, quic::QuicRandom::GetInstance(), + std::make_unique(), + quic::KeyExchangeSource::Default()), + version_manager_(quic::CurrentSupportedVersions()), + listener_stats_({ALL_LISTENER_STATS(POOL_COUNTER(listener_config_.listenerScope()), + POOL_GAUGE(listener_config_.listenerScope()), + POOL_HISTOGRAM(listener_config_.listenerScope()))}), + connection_handler_(ENVOY_LOGGER(), *dispatcher_), + envoy_quic_dispatcher_( + &crypto_config_, &version_manager_, + std::make_unique(*dispatcher_), + std::make_unique(*dispatcher_, *connection_helper_.GetClock()), + quic::kQuicDefaultConnectionIdLength, connection_handler_, listener_config_, + listener_stats_) { + auto writer = new testing::NiceMock(); + envoy_quic_dispatcher_.InitializeWithWriter(writer); + EXPECT_CALL(*writer, WritePacket(_, _, _, _, _)) + .WillRepeatedly(Return(quic::WriteResult(quic::WRITE_STATUS_OK, 0))); + } + + void SetUp() override { + listen_socket_ = std::make_unique>>( + Network::Test::getCanonicalLoopbackAddress(version_), nullptr, /*bind*/ true); + // Advance time a bit because QuicTime regards 0 as uninitialized timestamp. + time_system_.sleep(std::chrono::milliseconds(100)); + EXPECT_CALL(listener_config_, socket()).WillRepeatedly(ReturnRef(*listen_socket_)); + } + + void TearDown() override { + envoy_quic_dispatcher_.Shutdown(); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + std::unique_ptr + createFullChloPacket(const quic::QuicConnectionId& connection_id, + quic::QuicSocketAddress client_address) { + EnvoyQuicClock clock(*dispatcher_); + quic::CryptoHandshakeMessage chlo = quic::test::crypto_test_utils::GenerateDefaultInchoateCHLO( + &clock, quic::AllSupportedVersions()[0].transport_version, &crypto_config_); + chlo.SetVector(quic::kCOPT, quic::QuicTagVector{quic::kREJ}); + chlo.SetStringPiece(quic::kSNI, "www.abc.com"); + quic::CryptoHandshakeMessage full_chlo; + quic::QuicReferenceCountedPointer signed_config( + new quic::QuicSignedServerConfig); + quic::QuicCompressedCertsCache cache( + quic::QuicCompressedCertsCache::kQuicCompressedCertsCacheSize); + quic::test::crypto_test_utils::GenerateFullCHLO( + chlo, &crypto_config_, + envoyAddressInstanceToQuicSocketAddress(listen_socket_->localAddress()), client_address, + quic::AllSupportedVersions()[0].transport_version, &clock, signed_config, &cache, + &full_chlo); + // Overwrite version label to highest current supported version. + full_chlo.SetVersion(quic::kVER, quic::CurrentSupportedVersions()[0]); + quic::QuicConfig quic_config; + quic_config.ToHandshakeMessage(&full_chlo, + quic::CurrentSupportedVersions()[0].transport_version); + + std::string packet_content(full_chlo.GetSerialized().AsStringPiece()); + std::unique_ptr encrypted_packet( + quic::test::ConstructEncryptedPacket(connection_id, quic::EmptyQuicConnectionId(), + /*version_flag=*/true, /*reset_flag*/ false, + /*packet_number=*/1, packet_content)); + return std::unique_ptr( + quic::test::ConstructReceivedPacket(*encrypted_packet, clock.Now())); + } + +protected: + Network::Address::IpVersion version_; + Event::SimulatedTimeSystemHelper time_system_; + Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; + Network::SocketPtr listen_socket_; + EnvoyQuicConnectionHelper connection_helper_; + quic::QuicCryptoServerConfig crypto_config_; + quic::QuicVersionManager version_manager_; + + testing::NiceMock listener_config_; + Server::ListenerStats listener_stats_; + Server::ConnectionHandlerImpl connection_handler_; + EnvoyQuicDispatcher envoy_quic_dispatcher_; +}; + +INSTANTIATE_TEST_SUITE_P(IpVersions, EnvoyQuicDispatcherTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +TEST_P(EnvoyQuicDispatcherTest, CreateNewConnectionUponCHLO) { + quic::SetVerbosityLogThreshold(2); + quic::QuicSocketAddress peer_addr(version_ == Network::Address::IpVersion::v4 + ? quic::QuicIpAddress::Loopback4() + : quic::QuicIpAddress::Loopback6(), + 54321); + Network::MockFilterChain filter_chain; + Network::MockFilterChainManager filter_chain_manager; + EXPECT_CALL(listener_config_, filterChainManager()).WillOnce(ReturnRef(filter_chain_manager)); + EXPECT_CALL(filter_chain_manager, findFilterChain(_)) + .WillOnce(Invoke([&](const Network::ConnectionSocket& socket) { + EXPECT_EQ(*listen_socket_->localAddress(), *socket.localAddress()); + EXPECT_EQ(Extensions::TransportSockets::TransportSocketNames::get().Quic, + socket.detectedTransportProtocol()); + EXPECT_EQ(peer_addr, envoyAddressInstanceToQuicSocketAddress(socket.remoteAddress())); + return &filter_chain; + })); + std::shared_ptr read_filter(new Network::MockReadFilter()); + Network::MockConnectionCallbacks network_connection_callbacks; + std::vector filter_factory( + {[&](Network::FilterManager& filter_manager) { + filter_manager.addReadFilter(read_filter); + read_filter->callbacks_->connection().addConnectionCallbacks(network_connection_callbacks); + }}); + EXPECT_CALL(filter_chain, networkFilterFactories()).WillOnce(ReturnRef(filter_factory)); + EXPECT_CALL(listener_config_, filterChainFactory()); + EXPECT_CALL(listener_config_.filter_chain_factory_, createNetworkFilterChain(_, _)) + .WillOnce(Invoke([](Network::Connection& connection, + const std::vector& filter_factories) { + EXPECT_EQ(1u, filter_factories.size()); + Server::Configuration::FilterChainUtility::buildFilterChain(connection, filter_factories); + return true; + })); + EXPECT_CALL(*read_filter, onNewConnection()) + // Stop iteration to avoid calling getRead/WriteBuffer(). + .WillOnce(Invoke([]() { return Network::FilterStatus::StopIteration; })); + + quic::QuicConnectionId connection_id = quic::test::TestConnectionId(1); + // Upon receiving a full CHLO. A new quic connection should be created and have its filter + // installed based on self and peer address. + std::unique_ptr received_packet = + createFullChloPacket(connection_id, peer_addr); + envoy_quic_dispatcher_.ProcessPacket( + envoyAddressInstanceToQuicSocketAddress(listen_socket_->localAddress()), peer_addr, + *received_packet); + EXPECT_EQ(1u, envoy_quic_dispatcher_.session_map().size()); + EXPECT_TRUE( + envoy_quic_dispatcher_.session_map().find(connection_id)->second->IsEncryptionEstablished()); + EXPECT_EQ(1u, connection_handler_.numConnections()); + EXPECT_EQ("www.abc.com", read_filter->callbacks_->connection().requestedServerName()); + EXPECT_EQ(peer_addr, envoyAddressInstanceToQuicSocketAddress( + read_filter->callbacks_->connection().remoteAddress())); + EXPECT_EQ(*listen_socket_->localAddress(), *read_filter->callbacks_->connection().localAddress()); + EXPECT_CALL(network_connection_callbacks, onEvent(Network::ConnectionEvent::LocalClose)); + // Shutdown() to close the connection. + envoy_quic_dispatcher_.Shutdown(); +} + +TEST_P(EnvoyQuicDispatcherTest, CloseConnectionDueToMissingFilterChain) { + quic::QuicSocketAddress peer_addr(version_ == Network::Address::IpVersion::v4 + ? quic::QuicIpAddress::Loopback4() + : quic::QuicIpAddress::Loopback6(), + 54321); + Network::MockFilterChainManager filter_chain_manager; + EXPECT_CALL(listener_config_, filterChainManager()).WillOnce(ReturnRef(filter_chain_manager)); + EXPECT_CALL(filter_chain_manager, findFilterChain(_)) + .WillOnce(Invoke([&](const Network::ConnectionSocket& socket) { + EXPECT_EQ(*listen_socket_->localAddress(), *socket.localAddress()); + EXPECT_EQ(peer_addr, envoyAddressInstanceToQuicSocketAddress(socket.remoteAddress())); + return nullptr; + })); + quic::QuicConnectionId connection_id = quic::test::TestConnectionId(1); + std::unique_ptr received_packet = + createFullChloPacket(connection_id, peer_addr); + envoy_quic_dispatcher_.ProcessPacket( + envoyAddressInstanceToQuicSocketAddress(listen_socket_->localAddress()), peer_addr, + *received_packet); + EXPECT_EQ(0u, envoy_quic_dispatcher_.session_map().size()); + EXPECT_EQ(0u, connection_handler_.numConnections()); + EXPECT_TRUE(quic::test::QuicDispatcherPeer::time_wait_list_manager(&envoy_quic_dispatcher_) + ->IsConnectionIdInTimeWait(connection_id)); + EXPECT_EQ(1u, listener_stats_.no_filter_chain_match_.value()); +} + +TEST_P(EnvoyQuicDispatcherTest, CloseConnectionDueToEmptyFilterChain) { + quic::QuicSocketAddress peer_addr(version_ == Network::Address::IpVersion::v4 + ? quic::QuicIpAddress::Loopback4() + : quic::QuicIpAddress::Loopback6(), + 54321); + Network::MockFilterChain filter_chain; + Network::MockFilterChainManager filter_chain_manager; + EXPECT_CALL(listener_config_, filterChainManager()).WillOnce(ReturnRef(filter_chain_manager)); + EXPECT_CALL(filter_chain_manager, findFilterChain(_)) + .WillOnce(Invoke([&](const Network::ConnectionSocket& socket) { + EXPECT_EQ(*listen_socket_->localAddress(), *socket.localAddress()); + EXPECT_EQ(peer_addr, envoyAddressInstanceToQuicSocketAddress(socket.remoteAddress())); + return &filter_chain; + })); + // Empty filter_factory should cause connection close. + std::vector filter_factory; + EXPECT_CALL(filter_chain, networkFilterFactories()).WillOnce(ReturnRef(filter_factory)); + + quic::QuicConnectionId connection_id = quic::test::TestConnectionId(1); + std::unique_ptr received_packet = + createFullChloPacket(connection_id, peer_addr); + envoy_quic_dispatcher_.ProcessPacket( + envoyAddressInstanceToQuicSocketAddress(listen_socket_->localAddress()), peer_addr, + *received_packet); + EXPECT_EQ(0u, envoy_quic_dispatcher_.session_map().size()); + EXPECT_EQ(0u, connection_handler_.numConnections()); + EXPECT_TRUE(quic::test::QuicDispatcherPeer::time_wait_list_manager(&envoy_quic_dispatcher_) + ->IsConnectionIdInTimeWait(connection_id)); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc index e548fe72ce72..4737a532f558 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_proof_source_test.cc @@ -44,7 +44,7 @@ class EnvoyQuicFakeProofSourceTest : public ::testing::Test { quic::QuicTransportVersion version_{quic::QUIC_VERSION_UNSUPPORTED}; quic::QuicStringPiece chlo_hash_{""}; std::string server_config_{"Server Config"}; - std::vector expected_certs_{absl::StrCat("Fake cert from ", hostname_)}; + std::vector expected_certs_{"Fake cert"}; std::string expected_signature_{absl::StrCat("Fake signature for { ", server_config_, " }")}; EnvoyQuicFakeProofSource proof_source_; EnvoyQuicFakeProofVerifier proof_verifier_; diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc new file mode 100644 index 000000000000..8b5f127cbcf1 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc @@ -0,0 +1,401 @@ +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/core/quic_versions.h" +#include "quiche/quic/test_tools/crypto_test_utils.h" +#include "quiche/quic/test_tools/quic_test_utils.h" + +#pragma GCC diagnostic pop + +#include + +#include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" +#include "extensions/quic_listeners/quiche/codec_impl.h" +#include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" +#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/quic_listeners/quiche/envoy_quic_fake_proof_source.h" +#include "extensions/transport_sockets/well_known_names.h" + +#include "envoy/stats/stats_macros.h" +#include "common/event/libevent_scheduler.h" +#include "server/configuration_impl.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/http/stream_decoder.h" +#include "test/mocks/http/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/mocks/stats/mocks.h" +#include "test/test_common/global.h" +#include "test/test_common/logging.h" +#include "test/test_common/simulated_time_system.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; +using testing::Return; +using testing::ReturnRef; + +#include + +namespace Envoy { +namespace Quic { + +class TestEnvoyQuicConnection : public EnvoyQuicConnection { +public: + TestEnvoyQuicConnection(quic::QuicConnectionHelperInterface& helper, + quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter& writer, + const quic::ParsedQuicVersionVector& supported_versions, + Network::ListenerConfig& listener_config, Server::ListenerStats& stats) + : EnvoyQuicConnection(quic::test::TestConnectionId(), + quic::QuicSocketAddress(quic::QuicIpAddress::Loopback4(), 12345), + helper, alarm_factory, writer, /*owns_writer=*/false, + quic::Perspective::IS_SERVER, supported_versions, listener_config, + stats) {} + + Network::Connection::ConnectionStats& connectionStats() const { + return EnvoyQuicConnection::connectionStats(); + } + + MOCK_METHOD2(SendConnectionClosePacket, void(quic::QuicErrorCode, const std::string&)); + MOCK_METHOD1(SendControlFrame, bool(const quic::QuicFrame& frame)); +}; + +class EnvoyQuicServerSessionTest : public testing::TestWithParam { +public: + EnvoyQuicServerSessionTest() + : api_(Api::createApiForTest(time_system_)), dispatcher_(api_->allocateDispatcher()), + connection_helper_(*dispatcher_), + alarm_factory_(*dispatcher_, *connection_helper_.GetClock()), quic_version_([]() { + SetQuicReloadableFlag(quic_enable_version_99, GetParam()); + return quic::ParsedVersionOfIndex(quic::CurrentSupportedVersions(), 0); + }()), + listener_stats_({ALL_LISTENER_STATS(POOL_COUNTER(listener_config_.listenerScope()), + POOL_GAUGE(listener_config_.listenerScope()), + POOL_HISTOGRAM(listener_config_.listenerScope()))}), + quic_connection_(new TestEnvoyQuicConnection(connection_helper_, alarm_factory_, writer_, + quic_version_, listener_config_, + listener_stats_)), + crypto_config_(quic::QuicCryptoServerConfig::TESTING, quic::QuicRandom::GetInstance(), + std::make_unique(), + quic::KeyExchangeSource::Default()), + envoy_quic_session_(quic_config_, quic_version_, + std::unique_ptr(quic_connection_), + /*visitor=*/nullptr, &crypto_stream_helper_, &crypto_config_, + &compressed_certs_cache_, *dispatcher_), + read_filter_(new Network::MockReadFilter()) { + EXPECT_EQ(time_system_.systemTime(), envoy_quic_session_.streamInfo().startTime()); + EXPECT_EQ(EMPTY_STRING, envoy_quic_session_.nextProtocol()); + time_system_.sleep(std::chrono::milliseconds(1)); + ON_CALL(writer_, WritePacket(_, _, _, _, _)) + .WillByDefault(testing::Return(quic::WriteResult(quic::WRITE_STATUS_OK, 1))); + ON_CALL(crypto_stream_helper_, CanAcceptClientHello(_, _, _, _, _)).WillByDefault(Return(true)); + } + + void SetUp() override { envoy_quic_session_.Initialize(); } + + bool installReadFilter() { + // Setup read filter. + envoy_quic_session_.addReadFilter(read_filter_); + EXPECT_EQ(Http::Protocol::Http2, + read_filter_->callbacks_->connection().streamInfo().protocol().value()); + EXPECT_EQ(envoy_quic_session_.id(), read_filter_->callbacks_->connection().id()); + EXPECT_EQ(&envoy_quic_session_, &read_filter_->callbacks_->connection()); + read_filter_->callbacks_->connection().addConnectionCallbacks(network_connection_callbacks_); + read_filter_->callbacks_->connection().setConnectionStats( + {read_total_, read_current_, write_total_, write_current_, nullptr, nullptr}); + EXPECT_EQ(&read_total_, &quic_connection_->connectionStats().read_total_); + EXPECT_CALL(*read_filter_, onNewConnection()).WillOnce(Invoke([this]() { + // Create ServerConnection instance and setup callbacks for it. + http_connection_ = std::make_unique(envoy_quic_session_, + http_connection_callbacks_); + EXPECT_EQ(Http::Protocol::Http2, http_connection_->protocol()); + // Stop iteration to avoid calling getRead/WriteBuffer(). + return Network::FilterStatus::StopIteration; + })); + return envoy_quic_session_.initializeReadFilters(); + } + + void TearDown() override { + if (quic_connection_->connected()) { + EXPECT_CALL(*quic_connection_, + SendConnectionClosePacket(quic::QUIC_NO_ERROR, "Closed by application")); + EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::LocalClose)); + envoy_quic_session_.close(Network::ConnectionCloseType::NoFlush); + } + } + +protected: + Event::SimulatedTimeSystemHelper time_system_; + Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; + EnvoyQuicConnectionHelper connection_helper_; + EnvoyQuicAlarmFactory alarm_factory_; + quic::ParsedQuicVersionVector quic_version_; + testing::NiceMock writer_; + testing::NiceMock listener_config_; + Server::ListenerStats listener_stats_; + TestEnvoyQuicConnection* quic_connection_; + quic::QuicConfig quic_config_; + quic::QuicCryptoServerConfig crypto_config_; + testing::NiceMock crypto_stream_helper_; + EnvoyQuicServerSession envoy_quic_session_; + quic::QuicCompressedCertsCache compressed_certs_cache_{100}; + std::shared_ptr read_filter_; + Network::MockConnectionCallbacks network_connection_callbacks_; + Http::MockServerConnectionCallbacks http_connection_callbacks_; + testing::StrictMock read_total_; + testing::StrictMock read_current_; + testing::StrictMock write_total_; + testing::StrictMock write_current_; + Http::ServerConnectionPtr http_connection_; +}; + +INSTANTIATE_TEST_SUITE_P(EnvoyQuicServerSessionTests, EnvoyQuicServerSessionTest, + testing::ValuesIn({true, false})); + +TEST_P(EnvoyQuicServerSessionTest, NewStream) { + installReadFilter(); + + Http::MockStreamDecoder request_decoder; + EXPECT_CALL(http_connection_callbacks_, newStream(_, false)) + .WillOnce(testing::ReturnRef(request_decoder)); + quic::QuicStreamId stream_id = + quic_version_[0].transport_version == quic::QUIC_VERSION_99 ? 4u : 5u; + auto stream = + reinterpret_cast(envoy_quic_session_.GetOrCreateStream(stream_id)); + // Receive a GET request on created stream. + quic::QuicHeaderList headers; + headers.OnHeaderBlockStart(); + std::string host("www.abc.com"); + headers.OnHeader(":authority", host); + headers.OnHeader(":method", "GET"); + headers.OnHeader(":path", "/"); + headers.OnHeaderBlockEnd(/*uncompressed_header_bytes=*/0, /*compressed_header_bytes=*/0); + // Request headers should be propagated to decoder. + EXPECT_CALL(request_decoder, decodeHeaders_(_, /*end_stream=*/true)) + .WillOnce(Invoke([&host](const Http::HeaderMapPtr& decoded_headers, bool) { + EXPECT_EQ(host, decoded_headers->Host()->value().getStringView()); + EXPECT_EQ("/", decoded_headers->Path()->value().getStringView()); + EXPECT_EQ(Http::Headers::get().MethodValues.Get, + decoded_headers->Method()->value().getStringView()); + })); + EXPECT_CALL(request_decoder, decodeData(_, true)) + .Times(testing::AtMost(1)) + .WillOnce(Invoke([](Buffer::Instance& buffer, bool) { EXPECT_EQ(0, buffer.length()); })); + stream->OnStreamHeaderList(/*fin=*/true, headers.uncompressed_header_bytes(), headers); +} + +TEST_P(EnvoyQuicServerSessionTest, InvalidIncomingStreamId) { + quic::SetVerbosityLogThreshold(1); + installReadFilter(); + Http::MockStreamDecoder request_decoder; + Http::MockStreamCallbacks stream_callbacks; + // IETF stream 5 and G-Quic stream 2 are server initiated. + quic::QuicStreamId stream_id = + quic_version_[0].transport_version == quic::QUIC_VERSION_99 ? 5u : 2u; + std::string data("aaaa"); + quic::QuicStreamFrame stream_frame(stream_id, false, 0, data); + EXPECT_CALL(http_connection_callbacks_, newStream(_, false)).Times(0); + EXPECT_CALL(*quic_connection_, SendConnectionClosePacket(quic::QUIC_INVALID_STREAM_ID, + "Data for nonexistent stream")); + EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::LocalClose)); + + envoy_quic_session_.OnStreamFrame(stream_frame); +} + +TEST_P(EnvoyQuicServerSessionTest, NoNewStreamForInvalidIncomingStream) { + quic::SetVerbosityLogThreshold(1); + installReadFilter(); + Http::MockStreamDecoder request_decoder; + Http::MockStreamCallbacks stream_callbacks; + // IETF stream 5 and G-Quic stream 2 are server initiated. + quic::QuicStreamId stream_id = + quic_version_[0].transport_version == quic::QUIC_VERSION_99 ? 5u : 2u; + EXPECT_CALL(http_connection_callbacks_, newStream(_, false)).Times(0); + EXPECT_CALL(*quic_connection_, SendConnectionClosePacket(quic::QUIC_INVALID_STREAM_ID, + "Data for nonexistent stream")); + EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::LocalClose)); + + // Stream creation on closed connection should fail. + EXPECT_EQ(nullptr, envoy_quic_session_.GetOrCreateStream(stream_id)); +} + +TEST_P(EnvoyQuicServerSessionTest, OnResetFrame) { + installReadFilter(); + Http::MockStreamDecoder request_decoder; + Http::MockStreamCallbacks stream_callbacks; + EXPECT_CALL(http_connection_callbacks_, newStream(_, false)) + .WillRepeatedly(Invoke([&request_decoder, &stream_callbacks](Http::StreamEncoder& encoder, + bool) -> Http::StreamDecoder& { + encoder.getStream().addCallbacks(stream_callbacks); + return request_decoder; + })); + + // G-QUIC or IETF bi-directional stream. + quic::QuicStreamId stream_id = + quic_version_[0].transport_version == quic::QUIC_VERSION_99 ? 4u : 5u; + + quic::QuicStream* stream1 = envoy_quic_session_.GetOrCreateStream(stream_id); + quic::QuicRstStreamFrame rst1(/*control_frame_id=*/1u, stream1->id(), + quic::QUIC_ERROR_PROCESSING_STREAM, /*bytes_written=*/0u); + EXPECT_CALL(stream_callbacks, onResetStream(Http::StreamResetReason::RemoteReset, _)); + if (quic_version_[0].transport_version < quic::QUIC_VERSION_99) { + EXPECT_CALL(*quic_connection_, SendControlFrame(_)) + .WillOnce(Invoke([stream_id](const quic::QuicFrame& frame) { + EXPECT_EQ(stream_id, frame.rst_stream_frame->stream_id); + EXPECT_EQ(quic::QUIC_RST_ACKNOWLEDGEMENT, frame.rst_stream_frame->error_code); + return false; + })); + } + stream1->OnStreamReset(rst1); + + // G-QUIC bi-directional stream or IETF read uni-directional stream. + quic::QuicStream* stream2 = envoy_quic_session_.GetOrCreateStream(stream_id + 4u); + quic::QuicRstStreamFrame rst2(/*control_frame_id=*/1u, stream2->id(), quic::QUIC_REFUSED_STREAM, + /*bytes_written=*/0u); + EXPECT_CALL(stream_callbacks, + onResetStream(Http::StreamResetReason::RemoteRefusedStreamReset, _)); + stream2->OnStreamReset(rst2); +} + +TEST_P(EnvoyQuicServerSessionTest, ConnectionClose) { + installReadFilter(); + + std::string error_details("dummy details"); + quic::QuicErrorCode error(quic::QUIC_INVALID_FRAME_DATA); + quic::QuicConnectionCloseFrame frame(error, error_details); + EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::RemoteClose)); + quic_connection_->OnConnectionCloseFrame(frame); + EXPECT_EQ(absl::StrCat(quic::QuicErrorCodeToString(error), " with details: ", error_details), + envoy_quic_session_.transportFailureReason()); + EXPECT_EQ(Network::Connection::State::Closed, envoy_quic_session_.state()); +} + +TEST_P(EnvoyQuicServerSessionTest, ConnectionCloseWithActiveStream) { + installReadFilter(); + + Http::MockStreamDecoder request_decoder; + Http::MockStreamCallbacks stream_callbacks; + EXPECT_CALL(http_connection_callbacks_, newStream(_, false)) + .WillOnce(Invoke([&request_decoder, &stream_callbacks](Http::StreamEncoder& encoder, + bool) -> Http::StreamDecoder& { + encoder.getStream().addCallbacks(stream_callbacks); + return request_decoder; + })); + quic::QuicStreamId stream_id = + quic_version_[0].transport_version == quic::QUIC_VERSION_99 ? 4u : 5u; + quic::QuicStream* stream = envoy_quic_session_.GetOrCreateStream(stream_id); + EXPECT_CALL(*quic_connection_, + SendConnectionClosePacket(quic::QUIC_NO_ERROR, "Closed by application")); + EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::LocalClose)); + EXPECT_CALL(stream_callbacks, onResetStream(Http::StreamResetReason::ConnectionTermination, _)); + envoy_quic_session_.close(Network::ConnectionCloseType::NoFlush); + EXPECT_EQ(Network::Connection::State::Closed, envoy_quic_session_.state()); + EXPECT_TRUE(stream->write_side_closed() && stream->reading_stopped()); +} + +TEST_P(EnvoyQuicServerSessionTest, FlushCloseNotSupported) { + installReadFilter(); + + EXPECT_CALL(*quic_connection_, + SendConnectionClosePacket(quic::QUIC_NO_ERROR, "Closed by application")); + EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::LocalClose)); + EXPECT_LOG_CONTAINS("error", "Flush write is not implemented for QUIC.", + envoy_quic_session_.close(Network::ConnectionCloseType::FlushWrite)); +} + +TEST_P(EnvoyQuicServerSessionTest, ShutdownNotice) { + installReadFilter(); + // Not verifying dummy implementation, just to have coverage. + EXPECT_DEBUG_DEATH(envoy_quic_session_.enableHalfClose(true), ""); + envoy_quic_session_.setBufferLimits(1024 * 1024); + EXPECT_EQ(nullptr, envoy_quic_session_.ssl()); + EXPECT_FALSE(envoy_quic_session_.aboveHighWatermark()); + EXPECT_DEBUG_DEATH(envoy_quic_session_.setDelayedCloseTimeout(std::chrono::milliseconds(1)), ""); + http_connection_->shutdownNotice(); +} + +TEST_P(EnvoyQuicServerSessionTest, GoAway) { + installReadFilter(); + if (quic_version_[0].transport_version < quic::QUIC_VERSION_99) { + EXPECT_CALL(*quic_connection_, SendControlFrame(_)); + } + http_connection_->goAway(); +} + +TEST_P(EnvoyQuicServerSessionTest, InitializeFilterChain) { + // Generate a CHLO packet. + quic::CryptoHandshakeMessage chlo = quic::test::crypto_test_utils::GenerateDefaultInchoateCHLO( + connection_helper_.GetClock(), quic::CurrentSupportedVersions()[0].transport_version, + &crypto_config_); + chlo.SetVector(quic::kCOPT, quic::QuicTagVector{quic::kREJ}); + std::string packet_content(chlo.GetSerialized().AsStringPiece()); + auto encrypted_packet = + std::unique_ptr(quic::test::ConstructEncryptedPacket( + quic_connection_->connection_id(), quic::EmptyQuicConnectionId(), /*version_flag=*/true, + /*reset_flag*/ false, /*packet_number=*/1, packet_content)); + + quic::QuicSocketAddress self_address( + envoyAddressInstanceToQuicSocketAddress(listener_config_.socket().localAddress())); + auto packet = std::unique_ptr( + quic::test::ConstructReceivedPacket(*encrypted_packet, connection_helper_.GetClock()->Now())); + + // Receiving above packet should trigger filter chain retrival. + Network::MockFilterChainManager filter_chain_manager; + EXPECT_CALL(listener_config_, filterChainManager()).WillOnce(ReturnRef(filter_chain_manager)); + Network::MockFilterChain filter_chain; + EXPECT_CALL(filter_chain_manager, findFilterChain(_)) + .WillOnce(Invoke([&](const Network::ConnectionSocket& socket) { + EXPECT_EQ(*quicAddressToEnvoyAddressInstance(quic_connection_->peer_address()), + *socket.remoteAddress()); + EXPECT_EQ(*quicAddressToEnvoyAddressInstance(self_address), *socket.localAddress()); + EXPECT_EQ(listener_config_.socket().ioHandle().fd(), socket.ioHandle().fd()); + EXPECT_EQ(Extensions::TransportSockets::TransportSocketNames::get().Quic, + socket.detectedTransportProtocol()); + return &filter_chain; + })); + std::vector filter_factory{[this]( + Network::FilterManager& filter_manager) { + filter_manager.addReadFilter(read_filter_); + read_filter_->callbacks_->connection().addConnectionCallbacks(network_connection_callbacks_); + }}; + EXPECT_CALL(filter_chain, networkFilterFactories()).WillOnce(ReturnRef(filter_factory)); + EXPECT_CALL(*read_filter_, onNewConnection()) + // Stop iteration to avoid calling getRead/WriteBuffer(). + .WillOnce(Return(Network::FilterStatus::StopIteration)); + EXPECT_CALL(listener_config_.filter_chain_factory_, createNetworkFilterChain(_, _)) + .WillOnce(Invoke([](Network::Connection& connection, + const std::vector& filter_factories) { + EXPECT_EQ(1u, filter_factories.size()); + Server::Configuration::FilterChainUtility::buildFilterChain(connection, filter_factories); + return true; + })); + // A reject should be sent because of the inchoate CHLO. + EXPECT_CALL(writer_, WritePacket(_, _, _, _, _)) + .WillOnce(testing::Return(quic::WriteResult(quic::WRITE_STATUS_OK, 1))); + quic_connection_->ProcessUdpPacket(self_address, quic_connection_->peer_address(), *packet); + EXPECT_TRUE(quic_connection_->connected()); + EXPECT_EQ(nullptr, envoy_quic_session_.socketOptions()); + EXPECT_FALSE(envoy_quic_session_.IsEncryptionEstablished()); + EXPECT_TRUE(quic_connection_->connectionSocket()->ioHandle().isOpen()); + EXPECT_TRUE(quic_connection_->connectionSocket()->ioHandle().close().ok()); + EXPECT_FALSE(quic_connection_->connectionSocket()->ioHandle().isOpen()); +} + +TEST_P(EnvoyQuicServerSessionTest, NetworkConnectionInterface) { + installReadFilter(); + EXPECT_EQ(dispatcher_.get(), &envoy_quic_session_.dispatcher()); + EXPECT_TRUE(envoy_quic_session_.readEnabled()); + EXPECT_FALSE(envoy_quic_session_.localAddressRestored()); + EXPECT_DEBUG_DEATH(envoy_quic_session_.unixSocketPeerCredentials(), + "Unix domain socket is not supported."); + EXPECT_DEBUG_DEATH(envoy_quic_session_.readDisable(true), + "Quic connection should be able to read through out its life time."); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc new file mode 100644 index 000000000000..31df75c0a2b0 --- /dev/null +++ b/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc @@ -0,0 +1,253 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_server_stream.h" + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/core/quic_versions.h" +#include "quiche/quic/core/http/quic_server_session_base.h" +#include "quiche/quic/test_tools/quic_test_utils.h" + +#pragma GCC diagnostic pop + +#include + +#include "common/event/libevent_scheduler.h" +#include "common/http/headers.h" +#include "test/test_common/utility.h" +#include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" +#include "extensions/quic_listeners/quiche/envoy_quic_connection.h" +#include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" +#include "test/mocks/http/stream_decoder.h" +#include "test/mocks/network/mocks.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; + +namespace Envoy { +namespace Quic { + +class MockQuicServerSession : public quic::QuicServerSessionBase { +public: + MockQuicServerSession(const quic::QuicConfig& config, + const quic::ParsedQuicVersionVector& supported_versions, + quic::QuicConnection* connection, quic::QuicSession::Visitor* visitor, + quic::QuicCryptoServerStream::Helper* helper, + const quic::QuicCryptoServerConfig* crypto_config, + quic::QuicCompressedCertsCache* compressed_certs_cache) + : quic::QuicServerSessionBase(config, supported_versions, connection, visitor, helper, + crypto_config, compressed_certs_cache) {} + + MOCK_METHOD1(CreateIncomingStream, quic::QuicSpdyStream*(quic::QuicStreamId id)); + MOCK_METHOD1(CreateIncomingStream, quic::QuicSpdyStream*(quic::PendingStream* pending)); + MOCK_METHOD0(CreateOutgoingBidirectionalStream, quic::QuicSpdyStream*()); + MOCK_METHOD0(CreateOutgoingUnidirectionalStream, quic::QuicSpdyStream*()); + MOCK_METHOD1(ShouldCreateIncomingStream, bool(quic::QuicStreamId id)); + MOCK_METHOD0(ShouldCreateOutgoingBidirectionalStream, bool()); + MOCK_METHOD0(ShouldCreateOutgoingUnidirectionalStream, bool()); + MOCK_METHOD2(CreateQuicCryptoServerStream, + quic::QuicCryptoServerStream*(const quic::QuicCryptoServerConfig*, + quic::QuicCompressedCertsCache*)); +}; + +class EnvoyQuicServerStreamTest : public testing::TestWithParam { +public: + EnvoyQuicServerStreamTest() + : api_(Api::createApiForTest()), dispatcher_(api_->allocateDispatcher()), + connection_helper_(*dispatcher_), + alarm_factory_(*dispatcher_, *connection_helper_.GetClock()), quic_version_([]() { + SetQuicReloadableFlag(quic_enable_version_99, GetParam()); + return quic::CurrentSupportedVersions()[0]; + }()), + listener_stats_({ALL_LISTENER_STATS(POOL_COUNTER(listener_config_.listenerScope()), + POOL_GAUGE(listener_config_.listenerScope()), + POOL_HISTOGRAM(listener_config_.listenerScope()))}), + quic_connection_(quic::test::TestConnectionId(), + quic::QuicSocketAddress(quic::QuicIpAddress::Any6(), 12345), + connection_helper_, alarm_factory_, writer_, + /*owns_writer=*/false, quic::Perspective::IS_SERVER, {quic_version_}, + listener_config_, listener_stats_), + quic_session_(quic_config_, {quic_version_}, &quic_connection_, /*visitor=*/nullptr, + /*helper=*/nullptr, /*crypto_config=*/nullptr, + /*compressed_certs_cache=*/nullptr), + stream_id_(quic_version_.transport_version == quic::QUIC_VERSION_99 ? 4u : 5u), + quic_stream_(stream_id_, &quic_session_, quic::BIDIRECTIONAL) { + quic::SetVerbosityLogThreshold(3); + + quic_stream_.setDecoder(stream_decoder_); + } + + void SetUp() override { + headers_.OnHeaderBlockStart(); + headers_.OnHeader(":authority", host_); + headers_.OnHeader(":method", "GET"); + headers_.OnHeader(":path", "/"); + headers_.OnHeaderBlockEnd(/*uncompressed_header_bytes=*/0, /*compressed_header_bytes=*/0); + + trailers_.OnHeaderBlockStart(); + trailers_.OnHeader("key1", "value1"); + if (quic_version_.transport_version != quic::QUIC_VERSION_99) { + // ":final-offset" is required and stripped off by quic. + trailers_.OnHeader(":final-offset", absl::StrCat("", request_body_.length())); + } + trailers_.OnHeaderBlockEnd(/*uncompressed_header_bytes=*/0, /*compressed_header_bytes=*/0); + } + +protected: + Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; + EnvoyQuicConnectionHelper connection_helper_; + EnvoyQuicAlarmFactory alarm_factory_; + testing::NiceMock writer_; + quic::ParsedQuicVersion quic_version_; + quic::QuicConfig quic_config_; + testing::NiceMock listener_config_; + Server::ListenerStats listener_stats_; + EnvoyQuicConnection quic_connection_; + MockQuicServerSession quic_session_; + quic::QuicStreamId stream_id_; + EnvoyQuicServerStream quic_stream_; + Http::MockStreamDecoder stream_decoder_; + quic::QuicHeaderList headers_; + quic::QuicHeaderList trailers_; + std::string host_{"www.abc.com"}; + std::string request_body_{"Hello world"}; +}; + +INSTANTIATE_TEST_SUITE_P(EnvoyQuicServerStreamTests, EnvoyQuicServerStreamTest, + testing::ValuesIn({true, false})); + +TEST_P(EnvoyQuicServerStreamTest, DecodeHeadersAndBody) { + EXPECT_CALL(stream_decoder_, decodeHeaders_(_, /*end_stream=*/false)) + .WillOnce(Invoke([this](const Http::HeaderMapPtr& headers, bool) { + EXPECT_EQ(host_, headers->Host()->value().getStringView()); + EXPECT_EQ("/", headers->Path()->value().getStringView()); + EXPECT_EQ(Http::Headers::get().MethodValues.Get, + headers->Method()->value().getStringView()); + })); + if (quic_version_.transport_version == quic::QUIC_VERSION_99) { + quic_stream_.OnHeadersDecoded(headers_); + } else { + quic_stream_.OnStreamHeaderList(/*fin=*/false, headers_.uncompressed_header_bytes(), headers_); + } + EXPECT_TRUE(quic_stream_.FinishedReadingHeaders()); + + EXPECT_CALL(stream_decoder_, decodeData(_, _)) + .WillOnce(Invoke([this](Buffer::Instance& buffer, bool finished_reading) { + EXPECT_EQ(request_body_, buffer.toString()); + EXPECT_TRUE(finished_reading); + })); + std::string data = request_body_; + if (quic_version_.transport_version == quic::QUIC_VERSION_99) { + std::unique_ptr data_buffer; + quic::HttpEncoder encoder; + quic::QuicByteCount data_frame_header_length = + encoder.SerializeDataFrameHeader(request_body_.length(), &data_buffer); + quic::QuicStringPiece data_frame_header(data_buffer.get(), data_frame_header_length); + data = absl::StrCat(data_frame_header, request_body_); + } + quic::QuicStreamFrame frame(stream_id_, true, 0, data); + quic_stream_.OnStreamFrame(frame); +} + +TEST_P(EnvoyQuicServerStreamTest, DecodeHeadersBodyAndTrailers) { + EXPECT_CALL(stream_decoder_, decodeHeaders_(_, /*end_stream=*/false)) + .WillOnce(Invoke([this](const Http::HeaderMapPtr& headers, bool) { + EXPECT_EQ(host_, headers->Host()->value().getStringView()); + EXPECT_EQ("/", headers->Path()->value().getStringView()); + EXPECT_EQ(Http::Headers::get().MethodValues.Get, + headers->Method()->value().getStringView()); + })); + quic_stream_.OnStreamHeaderList(/*fin=*/false, headers_.uncompressed_header_bytes(), headers_); + EXPECT_TRUE(quic_stream_.FinishedReadingHeaders()); + + std::string data = request_body_; + if (quic_version_.transport_version == quic::QUIC_VERSION_99) { + std::unique_ptr data_buffer; + quic::HttpEncoder encoder; + quic::QuicByteCount data_frame_header_length = + encoder.SerializeDataFrameHeader(request_body_.length(), &data_buffer); + quic::QuicStringPiece data_frame_header(data_buffer.get(), data_frame_header_length); + data = absl::StrCat(data_frame_header, request_body_); + } + quic::QuicStreamFrame frame(stream_id_, false, 0, data); + EXPECT_CALL(stream_decoder_, decodeData(_, _)) + .Times(testing::AtMost(2)) + .WillOnce(Invoke([this](Buffer::Instance& buffer, bool finished_reading) { + EXPECT_EQ(request_body_, buffer.toString()); + EXPECT_FALSE(finished_reading); + })) + // Depends on QUIC version, there may be an empty STREAM_FRAME with FIN. But + // since there is trailers, finished_reading should always be false. + .WillOnce(Invoke([](Buffer::Instance& buffer, bool finished_reading) { + EXPECT_FALSE(finished_reading); + EXPECT_EQ(0, buffer.length()); + })); + quic_stream_.OnStreamFrame(frame); + + EXPECT_CALL(stream_decoder_, decodeTrailers_(_)) + .WillOnce(Invoke([](const Http::HeaderMapPtr& headers) { + Http::LowerCaseString key1("key1"); + Http::LowerCaseString key2(":final-offset"); + EXPECT_EQ("value1", headers->get(key1)->value().getStringView()); + EXPECT_EQ(nullptr, headers->get(key2)); + })); + quic_stream_.OnStreamHeaderList(/*fin=*/true, trailers_.uncompressed_header_bytes(), trailers_); +} + +TEST_P(EnvoyQuicServerStreamTest, OutOfOrderTrailers) { + if (quic::VersionUsesQpack(quic_version_.transport_version)) { + return; + } + EXPECT_CALL(stream_decoder_, decodeHeaders_(_, /*end_stream=*/false)) + .WillOnce(Invoke([this](const Http::HeaderMapPtr& headers, bool) { + EXPECT_EQ(host_, headers->Host()->value().getStringView()); + EXPECT_EQ("/", headers->Path()->value().getStringView()); + EXPECT_EQ(Http::Headers::get().MethodValues.Get, + headers->Method()->value().getStringView()); + })); + quic_stream_.OnStreamHeaderList(/*fin=*/false, headers_.uncompressed_header_bytes(), headers_); + EXPECT_TRUE(quic_stream_.FinishedReadingHeaders()); + + // Trailer should be delivered to HCM later after body arrives. + quic_stream_.OnStreamHeaderList(/*fin=*/true, trailers_.uncompressed_header_bytes(), trailers_); + + std::string data = request_body_; + if (quic_version_.transport_version == quic::QUIC_VERSION_99) { + std::unique_ptr data_buffer; + quic::HttpEncoder encoder; + quic::QuicByteCount data_frame_header_length = + encoder.SerializeDataFrameHeader(request_body_.length(), &data_buffer); + quic::QuicStringPiece data_frame_header(data_buffer.get(), data_frame_header_length); + data = absl::StrCat(data_frame_header, request_body_); + } + quic::QuicStreamFrame frame(stream_id_, false, 0, data); + EXPECT_CALL(stream_decoder_, decodeData(_, _)) + .Times(testing::AtMost(2)) + .WillOnce(Invoke([this](Buffer::Instance& buffer, bool finished_reading) { + EXPECT_EQ(request_body_, buffer.toString()); + EXPECT_FALSE(finished_reading); + })) + // Depends on QUIC version, there may be an empty STREAM_FRAME with FIN. But + // since there is trailers, finished_reading should always be false. + .WillOnce(Invoke([](Buffer::Instance& buffer, bool finished_reading) { + EXPECT_FALSE(finished_reading); + EXPECT_EQ(0, buffer.length()); + })); + + EXPECT_CALL(stream_decoder_, decodeTrailers_(_)) + .WillOnce(Invoke([](const Http::HeaderMapPtr& headers) { + Http::LowerCaseString key1("key1"); + Http::LowerCaseString key2(":final-offset"); + EXPECT_EQ("value1", headers->get(key1)->value().getStringView()); + EXPECT_EQ(nullptr, headers->get(key2)); + })); + quic_stream_.OnStreamFrame(frame); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_utils_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_utils_test.cc new file mode 100644 index 000000000000..7fff3cc792ef --- /dev/null +++ b/test/extensions/quic_listeners/quiche/envoy_quic_utils_test.cc @@ -0,0 +1,65 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" + +#pragma GCC diagnostic push +// QUICHE allows unused parameters. +#pragma GCC diagnostic ignored "-Wunused-parameter" +// QUICHE uses offsetof(). +#pragma GCC diagnostic ignored "-Winvalid-offsetof" + +#include "quiche/quic/test_tools/quic_test_utils.h" + +#pragma GCC diagnostic pop + +#include "test/mocks/api/mocks.h" +#include "test/test_common/threadsafe_singleton_injector.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Return; + +namespace Envoy { +namespace Quic { + +TEST(EnvoyQuicUtilsTest, ConversionBetweenQuicAddressAndEnvoyAddress) { + // Mock out socket() system call to test both V4 and V6 address conversion. + testing::NiceMock os_sys_calls; + TestThreadsafeSingletonInjector os_calls{&os_sys_calls}; + ON_CALL(os_sys_calls, socket(_, _, _)).WillByDefault(Return(Api::SysCallIntResult{1, 0})); + ON_CALL(os_sys_calls, close(_)).WillByDefault(Return(Api::SysCallIntResult{0, 0})); + + quic::QuicSocketAddress quic_uninitialized_addr; + EXPECT_EQ(nullptr, quicAddressToEnvoyAddressInstance(quic_uninitialized_addr)); + + for (const std::string& ip_str : {"fd00:0:0:1::1", "1.2.3.4"}) { + quic::QuicIpAddress quic_ip; + quic_ip.FromString(ip_str); + quic::QuicSocketAddress quic_addr(quic_ip, 12345); + Network::Address::InstanceConstSharedPtr envoy_addr = + quicAddressToEnvoyAddressInstance(quic_addr); + EXPECT_EQ(quic_addr.ToString(), envoy_addr->asStringView()); + EXPECT_EQ(quic_addr, envoyAddressInstanceToQuicSocketAddress(envoy_addr)); + } +} + +TEST(EnvoyQuicUtilsTest, HeadersConversion) { + spdy::SpdyHeaderBlock headers_block; + headers_block[":host"] = "www.google.com"; + headers_block[":path"] = "/index.hml"; + headers_block[":scheme"] = "https"; + Http::HeaderMapImplPtr envoy_headers = spdyHeaderBlockToEnvoyHeaders(headers_block); + EXPECT_EQ(headers_block.size(), envoy_headers->size()); + EXPECT_EQ("www.google.com", + envoy_headers->get(Http::LowerCaseString(":host"))->value().getStringView()); + EXPECT_EQ("/index.hml", + envoy_headers->get(Http::LowerCaseString(":path"))->value().getStringView()); + EXPECT_EQ("https", envoy_headers->get(Http::LowerCaseString(":scheme"))->value().getStringView()); + + quic::QuicHeaderList quic_headers = quic::test::AsHeaderList(headers_block); + Http::HeaderMapImplPtr envoy_headers2 = quicHeadersToEnvoyHeaders(quic_headers); + EXPECT_EQ(*envoy_headers, *envoy_headers2); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/quic_io_handle_wrapper_test.cc b/test/extensions/quic_listeners/quiche/quic_io_handle_wrapper_test.cc new file mode 100644 index 000000000000..e72edcd45ebd --- /dev/null +++ b/test/extensions/quic_listeners/quiche/quic_io_handle_wrapper_test.cc @@ -0,0 +1,88 @@ +#include + +#include + +#include "common/network/address_impl.h" + +#include "extensions/quic_listeners/quiche/quic_io_handle_wrapper.h" + +#include "test/mocks/api/mocks.h" +#include "test/mocks/network/mocks.h" +#include "test/test_common/threadsafe_singleton_injector.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::Return; + +namespace Envoy { +namespace Quic { + +class QuicIoHandleWrapperTest : public testing::Test { +public: + QuicIoHandleWrapperTest() : wrapper_(std::make_unique(socket_.ioHandle())) { + EXPECT_TRUE(wrapper_->isOpen()); + EXPECT_FALSE(socket_.ioHandle().isOpen()); + } + ~QuicIoHandleWrapperTest() override = default; + +protected: + testing::NiceMock socket_; + std::unique_ptr wrapper_; + testing::StrictMock os_sys_calls_; + TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; +}; + +TEST_F(QuicIoHandleWrapperTest, Close) { + EXPECT_TRUE(wrapper_->close().ok()); + EXPECT_FALSE(wrapper_->isOpen()); +} + +TEST_F(QuicIoHandleWrapperTest, DelegateIoHandleCalls) { + int fd = socket_.ioHandle().fd(); + char data[5]; + Buffer::RawSlice slice{data, 5}; + EXPECT_CALL(os_sys_calls_, readv(fd, _, 1)).WillOnce(Return(Api::SysCallSizeResult{5u, 0})); + wrapper_->readv(5, &slice, 1); + + EXPECT_CALL(os_sys_calls_, writev(fd, _, 1)).WillOnce(Return(Api::SysCallSizeResult{5u, 0})); + wrapper_->writev(&slice, 1); + + EXPECT_CALL(os_sys_calls_, socket(AF_INET6, SOCK_STREAM, 0)) + .WillRepeatedly(Return(Api::SysCallIntResult{1, 0})); + EXPECT_CALL(os_sys_calls_, close(1)).WillRepeatedly(Return(Api::SysCallIntResult{0, 0})); + + Network::Address::InstanceConstSharedPtr addr(new Network::Address::Ipv4Instance(12345)); + EXPECT_CALL(os_sys_calls_, sendto(fd, data, 5u, 0, _, _)) + .WillOnce(Return(Api::SysCallSizeResult{5u, 0})); + wrapper_->sendto(slice, 0, *addr); + + EXPECT_CALL(os_sys_calls_, sendmsg(fd, _, 0)).WillOnce(Return(Api::SysCallSizeResult{5u, 0})); + wrapper_->sendmsg(&slice, 1, 0, /*self_ip=*/nullptr, *addr); + + Network::IoHandle::RecvMsgOutput output(nullptr); + EXPECT_CALL(os_sys_calls_, recvmsg(fd, _, 0)).WillOnce(Invoke([](int, struct msghdr* msg, int) { + sockaddr_storage ss; + auto ipv6_addr = reinterpret_cast(&ss); + memset(ipv6_addr, 0, sizeof(sockaddr_in6)); + ipv6_addr->sin6_family = AF_INET6; + ipv6_addr->sin6_addr = in6addr_loopback; + ipv6_addr->sin6_port = htons(54321); + *reinterpret_cast(msg->msg_name) = *ipv6_addr; + msg->msg_namelen = sizeof(sockaddr_in6); + return Api::SysCallSizeResult{5u, 0}; + })); + wrapper_->recvmsg(&slice, 1, /*self_port=*/12345, output); + + EXPECT_TRUE(wrapper_->close().ok()); + + // Following calls shouldn't be delegated. + wrapper_->readv(5, &slice, 1); + wrapper_->writev(&slice, 1); + wrapper_->sendto(slice, 0, *addr); + wrapper_->sendmsg(&slice, 1, 0, /*self_ip=*/nullptr, *addr); + wrapper_->recvmsg(&slice, 1, /*self_port=*/12345, output); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/mocks/api/mocks.h b/test/mocks/api/mocks.h index a85587a62683..f415023ba75b 100644 --- a/test/mocks/api/mocks.h +++ b/test/mocks/api/mocks.h @@ -66,6 +66,8 @@ class MockOsSysCalls : public OsSysCallsImpl { MOCK_METHOD4(recv, SysCallSizeResult(int socket, void* buffer, size_t length, int flags)); MOCK_METHOD6(recvfrom, SysCallSizeResult(int sockfd, void* buffer, size_t length, int flags, struct sockaddr* addr, socklen_t* addrlen)); + MOCK_METHOD6(sendto, SysCallSizeResult(int sockfd, const void* buffer, size_t length, int flags, + const struct sockaddr* addr, socklen_t addrlen)); MOCK_METHOD3(recvmsg, SysCallSizeResult(int socket, struct msghdr* msg, int flags)); MOCK_METHOD2(ftruncate, SysCallIntResult(int fd, off_t length)); MOCK_METHOD6(mmap, SysCallPtrResult(void* addr, size_t length, int prot, int flags, int fd, diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index da55f2095fef..4c4af655bbf0 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -34,7 +34,7 @@ BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" -COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp)" +COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp|/source/extensions/quic_listeners/quiche/)" COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests" COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat" diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index e7100af807df..7bb603b68a4c 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -122,6 +122,7 @@ HV IAM IANA IDL +IETF INADDR INET IO @@ -201,8 +202,10 @@ POSTs PREBIND PRNG PROT +ProcessBufferedChlos PSS QPS +QoS QUIC RAII RANLUX From 809f0e35cd3762c7addaa916be0f85925c078d28 Mon Sep 17 00:00:00 2001 From: htuch Date: Mon, 9 Sep 2019 23:48:32 +0300 Subject: [PATCH 520/542] protodoc/api_proto_plugin: generic API protoc plugin framework. (#8157) Split out the generic plugin and FileDescriptorProto traversal bits from protodoc. This is in aid of the work in #8082 ad #8083, where additional protoc plugins will be responsible for v2 -> v3alpha API migrations and translation code generation. This is only the start really of the api_proto_plugin framework. I anticipate additional bits of protodoc will move here later, including field type analysis and oneof handling. In some respects, this is a re-implementation of some of https://github.com/lyft/protoc-gen-star in Python. The advantage is that this is super lightweight, has few dependencies and can be easily hacked. We also embed various bits of API business logic, e.g. annotations, in the framework (for now). Risk level: Low Testing: diff -ru against previous protodoc.py RST output, identical modulo some trivial whitespace that doesn't appear in generated HTML. There are no real tests yet, I anticipate adding some golden proto style tests. Signed-off-by: Harvey Tuch --- tools/api_proto_plugin/BUILD | 17 + tools/api_proto_plugin/__init__.py | 0 tools/api_proto_plugin/annotations.py | 79 ++++ tools/api_proto_plugin/plugin.py | 57 +++ tools/api_proto_plugin/traverse.py | 72 +++ tools/api_proto_plugin/type_context.py | 195 +++++++++ tools/api_proto_plugin/visitor.py | 45 ++ tools/protodoc/BUILD | 1 + tools/protodoc/protodoc.py | 579 ++++++++----------------- 9 files changed, 636 insertions(+), 409 deletions(-) create mode 100644 tools/api_proto_plugin/BUILD create mode 100644 tools/api_proto_plugin/__init__.py create mode 100644 tools/api_proto_plugin/annotations.py create mode 100644 tools/api_proto_plugin/plugin.py create mode 100644 tools/api_proto_plugin/traverse.py create mode 100644 tools/api_proto_plugin/type_context.py create mode 100644 tools/api_proto_plugin/visitor.py diff --git a/tools/api_proto_plugin/BUILD b/tools/api_proto_plugin/BUILD new file mode 100644 index 000000000000..646490276955 --- /dev/null +++ b/tools/api_proto_plugin/BUILD @@ -0,0 +1,17 @@ +licenses(["notice"]) # Apache 2 + +py_library( + name = "api_proto_plugin", + srcs = [ + "annotations.py", + "plugin.py", + "traverse.py", + "type_context.py", + "visitor.py", + ], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + "@com_google_protobuf//:protobuf_python", + ], +) diff --git a/tools/api_proto_plugin/__init__.py b/tools/api_proto_plugin/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/api_proto_plugin/annotations.py b/tools/api_proto_plugin/annotations.py new file mode 100644 index 000000000000..eadd080b03aa --- /dev/null +++ b/tools/api_proto_plugin/annotations.py @@ -0,0 +1,79 @@ +"""Envoy API annotations.""" + +from collections import namedtuple + +import re + +# Key-value annotation regex. +ANNOTATION_REGEX = re.compile('\[#([\w-]+?):(.*?)\]\s?', re.DOTALL) + +# Page/section titles with special prefixes in the proto comments +DOC_TITLE_ANNOTATION = 'protodoc-title' + +# Not implemented yet annotation on leading comments, leading to insertion of +# warning on field. +NOT_IMPLEMENTED_WARN_ANNOTATION = 'not-implemented-warn' + +# Not implemented yet annotation on leading comments, leading to hiding of +# field. +NOT_IMPLEMENTED_HIDE_ANNOTATION = 'not-implemented-hide' + +# Comment that allows for easy searching for things that need cleaning up in the next major +# API version. +NEXT_MAJOR_VERSION_ANNOTATION = 'next-major-version' + +# Comment. Just used for adding text that will not go into the docs at all. +COMMENT_ANNOTATION = 'comment' + +# proto compatibility status. +PROTO_STATUS_ANNOTATION = 'proto-status' + +# Where v2 differs from v1.. +V2_API_DIFF_ANNOTATION = 'v2-api-diff' + +VALID_ANNOTATIONS = set([ + DOC_TITLE_ANNOTATION, + NOT_IMPLEMENTED_WARN_ANNOTATION, + NOT_IMPLEMENTED_HIDE_ANNOTATION, + V2_API_DIFF_ANNOTATION, + NEXT_MAJOR_VERSION_ANNOTATION, + COMMENT_ANNOTATION, + PROTO_STATUS_ANNOTATION, +]) + +# These can propagate from file scope to message/enum scope (and be overridden). +INHERITED_ANNOTATIONS = set([ + PROTO_STATUS_ANNOTATION, +]) + + +class AnnotationError(Exception): + """Base error class for the annotations module.""" + + +def ExtractAnnotations(s, inherited_annotations=None): + """Extract annotations map from a given comment string. + + Args: + s: string that may contains annotations. + inherited_annotations: annotation map from file-level inherited annotations + (or None) if this is a file-level comment. + + Returns: + Annotation map. + """ + annotations = { + k: v for k, v in (inherited_annotations or {}).items() if k in INHERITED_ANNOTATIONS + } + # Extract annotations. + groups = re.findall(ANNOTATION_REGEX, s) + for group in groups: + annotation = group[0] + if annotation not in VALID_ANNOTATIONS: + raise AnnotationError('Unknown annotation: %s' % annotation) + annotations[group[0]] = group[1].lstrip() + return annotations + + +def WithoutAnnotations(s): + return re.sub(ANNOTATION_REGEX, '', s) diff --git a/tools/api_proto_plugin/plugin.py b/tools/api_proto_plugin/plugin.py new file mode 100644 index 000000000000..0a56ea8e3f8b --- /dev/null +++ b/tools/api_proto_plugin/plugin.py @@ -0,0 +1,57 @@ +"""Python protoc plugin for Envoy APIs.""" + +import cProfile +import io +import os +import pstats +import sys + +from tools.api_proto_plugin import traverse + +from google.protobuf.compiler import plugin_pb2 + + +def Plugin(output_suffix, visitor): + """Protoc plugin entry point. + + This defines protoc plugin and manages the stdin -> stdout flow. An + api_proto_plugin is defined by the provided visitor. + + See + http://www.expobrain.net/2015/09/13/create-a-plugin-for-google-protocol-buffer/ + for further details on protoc plugin basics. + + Args: + output_suffix: output files are generated alongside their corresponding + input .proto, with this filename suffix. + visitor: visitor.Visitor defining the business logic of the plugin. + """ + request = plugin_pb2.CodeGeneratorRequest() + request.ParseFromString(sys.stdin.buffer.read()) + response = plugin_pb2.CodeGeneratorResponse() + cprofile_enabled = os.getenv('CPROFILE_ENABLED') + + # We use request.file_to_generate rather than request.file_proto here since we + # are invoked inside a Bazel aspect, each node in the DAG will be visited once + # by the aspect and we only want to generate docs for the current node. + for file_to_generate in request.file_to_generate: + # Find the FileDescriptorProto for the file we actually are generating. + file_proto = [pf for pf in request.proto_file if pf.name == file_to_generate][0] + f = response.file.add() + f.name = file_proto.name + output_suffix + if cprofile_enabled: + pr = cProfile.Profile() + pr.enable() + # We don't actually generate any RST right now, we just string dump the + # input proto file descriptor into the output file. + f.content = traverse.TraverseFile(file_proto, visitor) + if cprofile_enabled: + pr.disable() + stats_stream = io.StringIO() + ps = pstats.Stats(pr, + stream=stats_stream).sort_stats(os.getenv('CPROFILE_SORTBY', 'cumulative')) + stats_file = response.file.add() + stats_file.name = file_proto.name + output_suffix + '.profile' + ps.print_stats() + stats_file.content = stats_stream.getvalue() + sys.stdout.buffer.write(response.SerializeToString()) diff --git a/tools/api_proto_plugin/traverse.py b/tools/api_proto_plugin/traverse.py new file mode 100644 index 000000000000..6ad97b8699aa --- /dev/null +++ b/tools/api_proto_plugin/traverse.py @@ -0,0 +1,72 @@ +"""FileDescriptorProto traversal for api_proto_plugin framework.""" + +from tools.api_proto_plugin import type_context + + +def TraverseEnum(type_context, enum_proto, visitor): + """Traverse an enum definition. + + Args: + type_context: type_context.TypeContext for enum type. + enum_proto: EnumDescriptorProto for enum. + visitor: visitor.Visitor defining the business logic of the plugin. + + Returns: + Plugin specific output. + """ + return visitor.VisitEnum(enum_proto, type_context) + + +def TraverseMessage(type_context, msg_proto, visitor): + """Traverse a message definition. + + Args: + type_context: type_context.TypeContext for message type. + msg_proto: DescriptorProto for message. + visitor: visitor.Visitor defining the business logic of the plugin. + + Returns: + Plugin specific output. + """ + # Skip messages synthesized to represent map types. + if msg_proto.options.map_entry: + return '' + # We need to do some extra work to recover the map type annotation from the + # synthesized messages. + type_context.map_typenames = { + '%s.%s' % (type_context.name, nested_msg.name): (nested_msg.field[0], nested_msg.field[1]) + for nested_msg in msg_proto.nested_type + if nested_msg.options.map_entry + } + nested_msgs = [ + TraverseMessage(type_context.ExtendNestedMessage(index, nested_msg.name), nested_msg, visitor) + for index, nested_msg in enumerate(msg_proto.nested_type) + ] + nested_enums = [ + TraverseEnum(type_context.ExtendNestedEnum(index, nested_enum.name), nested_enum, visitor) + for index, nested_enum in enumerate(msg_proto.enum_type) + ] + return visitor.VisitMessage(msg_proto, type_context, nested_msgs, nested_enums) + + +def TraverseFile(file_proto, visitor): + """Traverse a proto file definition. + + Args: + file_proto: FileDescriptorProto for file. + visitor: visitor.Visitor defining the business logic of the plugin. + + Returns: + Plugin specific output. + """ + source_code_info = type_context.SourceCodeInfo(file_proto.name, file_proto.source_code_info) + package_type_context = type_context.TypeContext(source_code_info, file_proto.package) + msgs = [ + TraverseMessage(package_type_context.ExtendMessage(index, msg.name), msg, visitor) + for index, msg in enumerate(file_proto.message_type) + ] + enums = [ + TraverseEnum(package_type_context.ExtendEnum(index, enum.name), enum, visitor) + for index, enum in enumerate(file_proto.enum_type) + ] + return visitor.VisitFile(file_proto, package_type_context, msgs, enums) diff --git a/tools/api_proto_plugin/type_context.py b/tools/api_proto_plugin/type_context.py new file mode 100644 index 000000000000..d69c120a1a63 --- /dev/null +++ b/tools/api_proto_plugin/type_context.py @@ -0,0 +1,195 @@ +"""Type context for FileDescriptorProto traversal.""" + +from collections import namedtuple + +from tools.api_proto_plugin import annotations + +# A comment is a (raw comment, annotation map) pair. +Comment = namedtuple('Comment', ['raw', 'annotations']) + + +class SourceCodeInfo(object): + """Wrapper for SourceCodeInfo proto.""" + + def __init__(self, name, source_code_info): + self.name = name + self.proto = source_code_info + # Map from path to SourceCodeInfo.Location + self._locations = {str(location.path): location for location in self.proto.location} + self._file_level_comments = None + self._file_level_annotations = None + + @property + def file_level_comments(self): + """Obtain inferred file level comment.""" + if self._file_level_comments: + return self._file_level_comments + comments = [] + # We find the earliest detached comment by first finding the maximum start + # line for any location and then scanning for any earlier locations with + # detached comments. + earliest_detached_comment = max(location.span[0] for location in self.proto.location) + 1 + for location in self.proto.location: + if location.leading_detached_comments and location.span[0] < earliest_detached_comment: + comments = location.leading_detached_comments + earliest_detached_comment = location.span[0] + self._file_level_comments = comments + return comments + + @property + def file_level_annotations(self): + """Obtain inferred file level annotations.""" + if self._file_level_annotations: + return self._file_level_annotations + self._file_level_annotations = dict( + sum([list(annotations.ExtractAnnotations(c).items()) for c in self.file_level_comments], + [])) + return self._file_level_annotations + + def LocationPathLookup(self, path): + """Lookup SourceCodeInfo.Location by path in SourceCodeInfo. + + Args: + path: a list of path indexes as per + https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717. + + Returns: + SourceCodeInfo.Location object if found, otherwise None. + """ + return self._locations.get(str(path), None) + + # TODO(htuch): consider integrating comment lookup with overall + # FileDescriptorProto, perhaps via two passes. + def LeadingCommentPathLookup(self, path): + """Lookup leading comment by path in SourceCodeInfo. + + Args: + path: a list of path indexes as per + https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717. + + Returns: + Comment object. + """ + location = self.LocationPathLookup(path) + if location is not None: + return Comment( + location.leading_comments, + annotations.ExtractAnnotations(location.leading_comments, self.file_level_annotations)) + return Comment('', {}) + + +class TypeContext(object): + """Contextual information for a message/field. + + Provides information around namespaces and enclosing types for fields and + nested messages/enums. + """ + + def __init__(self, source_code_info, name): + # SourceCodeInfo as per + # https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto. + self.source_code_info = source_code_info + # path: a list of path indexes as per + # https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717. + # Extended as nested objects are traversed. + self.path = [] + # Message/enum/field name. Extended as nested objects are traversed. + self.name = name + # Map from type name to the correct type annotation string, e.g. from + # ".envoy.api.v2.Foo.Bar" to "map". This is lost during + # proto synthesis and is dynamically recovered in TraverseMessage. + self.map_typenames = {} + # Map from a message's oneof index to the fields sharing a oneof. + self.oneof_fields = {} + # Map from a message's oneof index to the name of oneof. + self.oneof_names = {} + # Map from a message's oneof index to the "required" bool property. + self.oneof_required = {} + self.type_name = 'file' + + def _Extend(self, path, type_name, name): + if not self.name: + extended_name = name + else: + extended_name = '%s.%s' % (self.name, name) + extended = TypeContext(self.source_code_info, extended_name) + extended.path = self.path + path + extended.type_name = type_name + extended.map_typenames = self.map_typenames.copy() + extended.oneof_fields = self.oneof_fields.copy() + extended.oneof_names = self.oneof_names.copy() + extended.oneof_required = self.oneof_required.copy() + return extended + + def ExtendMessage(self, index, name): + """Extend type context with a message. + + Args: + index: message index in file. + name: message name. + """ + return self._Extend([4, index], 'message', name) + + def ExtendNestedMessage(self, index, name): + """Extend type context with a nested message. + + Args: + index: nested message index in message. + name: message name. + """ + return self._Extend([3, index], 'message', name) + + def ExtendField(self, index, name): + """Extend type context with a field. + + Args: + index: field index in message. + name: field name. + """ + return self._Extend([2, index], 'field', name) + + def ExtendEnum(self, index, name): + """Extend type context with an enum. + + Args: + index: enum index in file. + name: enum name. + """ + return self._Extend([5, index], 'enum', name) + + def ExtendNestedEnum(self, index, name): + """Extend type context with a nested enum. + + Args: + index: enum index in message. + name: enum name. + """ + return self._Extend([4, index], 'enum', name) + + def ExtendEnumValue(self, index, name): + """Extend type context with an enum enum. + + Args: + index: enum value index in enum. + name: value name. + """ + return self._Extend([2, index], 'enum_value', name) + + def ExtendOneof(self, index, name): + """Extend type context with an oneof declaration. + + Args: + index: oneof index in oneof_decl. + name: oneof name. + """ + return self._Extend([8, index], 'oneof', name) + + @property + def location(self): + """SourceCodeInfo.Location for type context.""" + return self.source_code_info.LocationPathLookup(self.path) + + @property + def leading_comment(self): + """Leading comment for type context.""" + return self.source_code_info.LeadingCommentPathLookup(self.path) diff --git a/tools/api_proto_plugin/visitor.py b/tools/api_proto_plugin/visitor.py new file mode 100644 index 000000000000..0065537f0a6e --- /dev/null +++ b/tools/api_proto_plugin/visitor.py @@ -0,0 +1,45 @@ +"""FileDescriptorProto visitor interface for api_proto_plugin implementations.""" + + +class Visitor(object): + """Abstract visitor interface for api_proto_plugin implementation.""" + + def VisitEnum(self, enum_proto, type_context): + """Visit an enum definition. + + Args: + enum_proto: EnumDescriptorProto for enum. + type_context: type_context.TypeContext for enum type. + + Returns: + Plugin specific output. + """ + pass + + def VisitMessage(self, msg_proto, type_context, nested_msgs, nested_enums): + """Visit a message definition. + + Args: + msg_proto: DescriptorProto for message. + type_context: type_context.TypeContext for message type. + nested_msgs: a list of results from visiting nested messages. + nested_enums: a list of results from visiting nested enums. + + Returns: + Plugin specific output. + """ + pass + + def VisitFile(self, file_proto, type_context, msgs, enums): + """Visit a proto file definition. + + Args: + file_proto: FileDescriptorProto for file. + type_context: type_context.TypeContext for file. + msgs: a list of results from visiting messages. + enums: a list of results from visiting enums. + + Returns: + Plugin specific output. + """ + pass diff --git a/tools/protodoc/BUILD b/tools/protodoc/BUILD index b4b3c3f39acb..d2c9b12a6727 100644 --- a/tools/protodoc/BUILD +++ b/tools/protodoc/BUILD @@ -6,6 +6,7 @@ py_binary( python_version = "PY3", visibility = ["//visibility:public"], deps = [ + "//tools/api_proto_plugin", "@com_envoyproxy_protoc_gen_validate//validate:validate_py", "@com_google_protobuf//:protobuf_python", ], diff --git a/tools/protodoc/protodoc.py b/tools/protodoc/protodoc.py index 83bf0946a961..e386757a5228 100755 --- a/tools/protodoc/protodoc.py +++ b/tools/protodoc/protodoc.py @@ -4,15 +4,14 @@ # https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html for Sphinx RST syntax. from collections import defaultdict -import cProfile import functools -import io import os -import pstats import re -import sys -from google.protobuf.compiler import plugin_pb2 +from tools.api_proto_plugin import annotations +from tools.api_proto_plugin import plugin +from tools.api_proto_plugin import visitor + from validate import validate_pb2 # Namespace prefix for Envoy core APIs. @@ -30,65 +29,55 @@ # http://www.fileformat.info/info/unicode/char/2063/index.htm UNICODE_INVISIBLE_SEPARATOR = u'\u2063' -# Key-value annotation regex. -ANNOTATION_REGEX = re.compile('\[#([\w-]+?):(.*?)\]\s?', re.DOTALL) - -# Page/section titles with special prefixes in the proto comments -DOC_TITLE_ANNOTATION = 'protodoc-title' - -# Not implemented yet annotation on leading comments, leading to insertion of -# warning on field. -NOT_IMPLEMENTED_WARN_ANNOTATION = 'not-implemented-warn' +# Template for data plane API URLs. +DATA_PLANE_API_URL_FMT = 'https://github.com/envoyproxy/envoy/blob/{}/api/%s#L%d'.format( + os.environ['ENVOY_BLOB_SHA']) -# Not implemented yet annotation on leading comments, leading to hiding of -# field. -NOT_IMPLEMENTED_HIDE_ANNOTATION = 'not-implemented-hide' -# Comment that allows for easy searching for things that need cleaning up in the next major -# API version. -NEXT_MAJOR_VERSION_ANNOTATION = 'next-major-version' +class ProtodocError(Exception): + """Base error class for the protodoc module.""" -# Comment. Just used for adding text that will not go into the docs at all. -COMMENT_ANNOTATION = 'comment' -# proto compatibility status. -PROTO_STATUS_ANNOTATION = 'proto-status' +def HideNotImplemented(comment): + """Should a given type_context.Comment be hidden because it is tagged as [#not-implemented-hide:]?""" + return annotations.NOT_IMPLEMENTED_HIDE_ANNOTATION in comment.annotations -# Where v2 differs from v1.. -V2_API_DIFF_ANNOTATION = 'v2-api-diff' -VALID_ANNOTATIONS = set([ - DOC_TITLE_ANNOTATION, - NOT_IMPLEMENTED_WARN_ANNOTATION, - NOT_IMPLEMENTED_HIDE_ANNOTATION, - V2_API_DIFF_ANNOTATION, - NEXT_MAJOR_VERSION_ANNOTATION, - COMMENT_ANNOTATION, - PROTO_STATUS_ANNOTATION, -]) +def GithubUrl(type_context): + """Obtain data plane API Github URL by path from a TypeContext. -# These can propagate from file scope to message/enum scope (and be overridden). -INHERITED_ANNOTATIONS = set([ - PROTO_STATUS_ANNOTATION, -]) + Args: + type_context: type_context.TypeContext for node. -# Template for data plane API URLs. -DATA_PLANE_API_URL_FMT = 'https://github.com/envoyproxy/envoy/blob/{}/api/%s#L%d'.format( - os.environ['ENVOY_BLOB_SHA']) + Returns: + A string with a corresponding data plane API GitHub Url. + """ + if type_context.location is not None: + return DATA_PLANE_API_URL_FMT % (type_context.source_code_info.name, + type_context.location.span[0]) + return '' -class ProtodocError(Exception): - """Base error class for the protodoc module.""" +def FormatCommentWithAnnotations(comment, type_name=''): + """Format a comment string with additional RST for annotations. + Args: + comment: comment string. + type_name: optional, 'message' or 'enum' may be specified for additional + message/enum specific annotations. -def FormatCommentWithAnnotations(s, annotations, type_name): - if NOT_IMPLEMENTED_WARN_ANNOTATION in annotations: + Returns: + A string with additional RST from annotations. + """ + s = annotations.WithoutAnnotations(StripLeadingSpace(comment.raw) + '\n') + if annotations.NOT_IMPLEMENTED_WARN_ANNOTATION in comment.annotations: s += '\n.. WARNING::\n Not implemented yet\n' - if V2_API_DIFF_ANNOTATION in annotations: - s += '\n.. NOTE::\n **v2 API difference**: ' + annotations[V2_API_DIFF_ANNOTATION] + '\n' + if annotations.V2_API_DIFF_ANNOTATION in comment.annotations: + s += '\n.. NOTE::\n **v2 API difference**: ' + comment.annotations[ + annotations.V2_API_DIFF_ANNOTATION] + '\n' if type_name == 'message' or type_name == 'enum': - if PROTO_STATUS_ANNOTATION in annotations: - status = annotations[PROTO_STATUS_ANNOTATION] + if annotations.PROTO_STATUS_ANNOTATION in comment.annotations: + status = comment.annotations[annotations.PROTO_STATUS_ANNOTATION] if status not in ['frozen', 'draft', 'experimental']: raise ProtodocError('Unknown proto status: %s' % status) if status == 'draft' or status == 'experimental': @@ -97,209 +86,13 @@ def FormatCommentWithAnnotations(s, annotations, type_name): return s -def ExtractAnnotations(s, inherited_annotations=None, type_name='file'): - """Extract annotations from a given comment string. - - Args: - s: string that may contains annotations. - inherited_annotations: annotation map from file-level inherited annotations - (or None) if this is a file-level comment. - Returns: - Pair of string with with annotations stripped and annotation map. - """ - annotations = { - k: v for k, v in (inherited_annotations or {}).items() if k in INHERITED_ANNOTATIONS - } - # Extract annotations. - groups = re.findall(ANNOTATION_REGEX, s) - # Remove annotations. - without_annotations = re.sub(ANNOTATION_REGEX, '', s) - for group in groups: - annotation = group[0] - if annotation not in VALID_ANNOTATIONS: - raise ProtodocError('Unknown annotation: %s' % annotation) - annotations[group[0]] = group[1].lstrip() - return FormatCommentWithAnnotations(without_annotations, annotations, type_name), annotations - - -class SourceCodeInfo(object): - """Wrapper for SourceCodeInfo proto.""" - - def __init__(self, name, source_code_info): - self._name = name - self._proto = source_code_info - self._leading_comments = { - str(location.path): location.leading_comments for location in self._proto.location - } - self._file_level_comment = None - - @property - def file_level_comment(self): - """Obtain inferred file level comment.""" - if self._file_level_comment: - return self._file_level_comment - comment = '' - earliest_detached_comment = max(max(location.span) for location in self._proto.location) - for location in self._proto.location: - if location.leading_detached_comments and location.span[0] < earliest_detached_comment: - comment = StripLeadingSpace(''.join(location.leading_detached_comments)) + '\n' - earliest_detached_comment = location.span[0] - self._file_level_comment = comment - return comment - - def LeadingCommentPathLookup(self, path, type_name): - """Lookup leading comment by path in SourceCodeInfo. - - Args: - path: a list of path indexes as per - https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717. - type_name: name of type the comment belongs to. - Returns: - Pair of attached leading comment and Annotation objects, where there is a - leading comment - otherwise ('', []). - """ - leading_comment = self._leading_comments.get(str(path), None) - if leading_comment is not None: - _, file_annotations = ExtractAnnotations(self.file_level_comment) - return ExtractAnnotations( - StripLeadingSpace(leading_comment) + '\n', file_annotations, type_name) - return '', [] - - def GithubUrl(self, path): - """Obtain data plane API Github URL by path from SourceCodeInfo. - - Args: - path: a list of path indexes as per - https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717. - Returns: - A string with a corresponding data plane API GitHub Url. - """ - for location in self._proto.location: - if location.path == path: - return DATA_PLANE_API_URL_FMT % (self._name, location.span[0]) - return '' - - -class TypeContext(object): - """Contextual information for a message/field. - - Provides information around namespaces and enclosing types for fields and - nested messages/enums. - """ - - def __init__(self, source_code_info, name): - # SourceCodeInfo as per - # https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto. - self.source_code_info = source_code_info - # path: a list of path indexes as per - # https://github.com/google/protobuf/blob/a08b03d4c00a5793b88b494f672513f6ad46a681/src/google/protobuf/descriptor.proto#L717. - # Extended as nested objects are traversed. - self.path = [] - # Message/enum/field name. Extended as nested objects are traversed. - self.name = name - # Map from type name to the correct type annotation string, e.g. from - # ".envoy.api.v2.Foo.Bar" to "map". This is lost during - # proto synthesis and is dynamically recovered in FormatMessage. - self.map_typenames = {} - # Map from a message's oneof index to the fields sharing a oneof. - self.oneof_fields = {} - # Map from a message's oneof index to the name of oneof. - self.oneof_names = {} - # Map from a message's oneof index to the "required" bool property. - self.oneof_required = {} - self.type_name = 'file' - - def _Extend(self, path, type_name, name): - if not self.name: - extended_name = name - else: - extended_name = '%s.%s' % (self.name, name) - extended = TypeContext(self.source_code_info, extended_name) - extended.path = self.path + path - extended.type_name = type_name - extended.map_typenames = self.map_typenames.copy() - extended.oneof_fields = self.oneof_fields.copy() - extended.oneof_names = self.oneof_names.copy() - extended.oneof_required = self.oneof_required.copy() - return extended - - def ExtendMessage(self, index, name): - """Extend type context with a message. - - Args: - index: message index in file. - name: message name. - """ - return self._Extend([4, index], 'message', name) - - def ExtendNestedMessage(self, index, name): - """Extend type context with a nested message. - - Args: - index: nested message index in message. - name: message name. - """ - return self._Extend([3, index], 'message', name) - - def ExtendField(self, index, name): - """Extend type context with a field. - - Args: - index: field index in message. - name: field name. - """ - return self._Extend([2, index], 'field', name) - - def ExtendEnum(self, index, name): - """Extend type context with an enum. - - Args: - index: enum index in file. - name: enum name. - """ - return self._Extend([5, index], 'enum', name) - - def ExtendNestedEnum(self, index, name): - """Extend type context with a nested enum. - - Args: - index: enum index in message. - name: enum name. - """ - return self._Extend([4, index], 'enum', name) - - def ExtendEnumValue(self, index, name): - """Extend type context with an enum enum. - - Args: - index: enum value index in enum. - name: value name. - """ - return self._Extend([2, index], 'enum_value', name) - - def ExtendOneof(self, index, name): - """Extend type context with an oneof declaration. - - Args: - index: oneof index in oneof_decl. - name: oneof name. - """ - return self._Extend([8, index], "oneof", name) - - def LeadingCommentPathLookup(self): - return self.source_code_info.LeadingCommentPathLookup(self.path, self.type_name) - - def GithubUrl(self): - return self.source_code_info.GithubUrl(self.path) - - def MapLines(f, s): """Apply a function across each line in a flat string. Args: f: A string transform function for a line. s: A string consisting of potentially multiple lines. + Returns: A flat string with f applied to each line. """ @@ -330,28 +123,33 @@ def FormatHeader(style, text): Args: style: underline style, e.g. '=', '-'. text: header text + Returns: RST formatted header. """ return '%s\n%s\n\n' % (text, style * len(text)) -def FormatHeaderFromFile(style, file_level_comment, alt): +def FormatHeaderFromFile(style, source_code_info, proto_name): """Format RST header based on special file level title Args: style: underline style, e.g. '=', '-'. - file_level_comment: detached comment at top of file. - alt: If the file_level_comment does not contain a user - specified title, use the alt text as page title. + source_code_info: SourceCodeInfo object. + proto_name: If the file_level_comment does not contain a user specified + title, use this as page title. + Returns: RST formatted header, and file level comment without page title strings. """ - anchor = FormatAnchor(FileCrossRefLabel(alt)) - stripped_comment, annotations = ExtractAnnotations(file_level_comment) - if DOC_TITLE_ANNOTATION in annotations: - return anchor + FormatHeader(style, annotations[DOC_TITLE_ANNOTATION]), stripped_comment - return anchor + FormatHeader(style, alt), stripped_comment + anchor = FormatAnchor(FileCrossRefLabel(proto_name)) + stripped_comment = annotations.WithoutAnnotations( + StripLeadingSpace('\n'.join(c + '\n' for c in source_code_info.file_level_comments))) + if annotations.DOC_TITLE_ANNOTATION in source_code_info.file_level_annotations: + return anchor + FormatHeader( + style, + source_code_info.file_level_annotations[annotations.DOC_TITLE_ANNOTATION]), stripped_comment + return anchor + FormatHeader(style, proto_name), stripped_comment def FormatFieldTypeAsJson(type_context, field): @@ -360,10 +158,9 @@ def FormatFieldTypeAsJson(type_context, field): Args: type_context: contextual information for message/enum/field. field: FieldDescriptor proto. - Return: - RST formatted pseudo-JSON string representation of field type. + Return: RST formatted pseudo-JSON string representation of field type. """ - if NormalizeFQN(field.type_name) in type_context.map_typenames: + if TypeNameFromFQN(field.type_name) in type_context.map_typenames: return '"{...}"' if field.label == field.LABEL_REPEATED: return '[]' @@ -378,14 +175,13 @@ def FormatMessageAsJson(type_context, msg): Args: type_context: contextual information for message/enum/field. msg: message definition DescriptorProto. - Return: - RST formatted pseudo-JSON string representation of message definition. + Return: RST formatted pseudo-JSON string representation of message definition. """ lines = [] for index, field in enumerate(msg.field): field_type_context = type_context.ExtendField(index, field.name) - leading_comment, comment_annotations = field_type_context.LeadingCommentPathLookup() - if NOT_IMPLEMENTED_HIDE_ANNOTATION in comment_annotations: + leading_comment = field_type_context.leading_comment + if HideNotImplemented(leading_comment): continue lines.append('"%s": %s' % (field.name, FormatFieldTypeAsJson(type_context, field))) @@ -395,21 +191,44 @@ def FormatMessageAsJson(type_context, msg): return '.. code-block:: json\n\n {}\n\n' -def NormalizeFQN(fqn): - """Normalize a fully qualified field type name. +def NormalizeFieldTypeName(field_fqn): + """Normalize a fully qualified field type name, e.g. + + .envoy.foo.bar. + + Strips leading ENVOY_API_NAMESPACE_PREFIX and ENVOY_PREFIX. - Strips leading ENVOY_API_NAMESPACE_PREFIX and ENVOY_PREFIX and makes pretty wrapped type names. + Args: + field_fqn: a fully qualified type name from FieldDescriptorProto.type_name. + Return: Normalized type name. + """ + if field_fqn.startswith(ENVOY_API_NAMESPACE_PREFIX): + return field_fqn[len(ENVOY_API_NAMESPACE_PREFIX):] + if field_fqn.startswith(ENVOY_PREFIX): + return field_fqn[len(ENVOY_PREFIX):] + return field_fqn + + +def NormalizeTypeContextName(type_name): + """Normalize a type name, e.g. + + envoy.foo.bar. + + Strips leading ENVOY_API_NAMESPACE_PREFIX and ENVOY_PREFIX. Args: - fqn: a fully qualified type name from FieldDescriptorProto.type_name. - Return: - Normalized type name. + type_name: a name from a TypeContext. + Return: Normalized type name. """ - if fqn.startswith(ENVOY_API_NAMESPACE_PREFIX): - return fqn[len(ENVOY_API_NAMESPACE_PREFIX):] - if fqn.startswith(ENVOY_PREFIX): - return fqn[len(ENVOY_PREFIX):] - return fqn + return NormalizeFieldTypeName(QualifyTypeName(type_name)) + + +def QualifyTypeName(type_name): + return '.' + type_name + + +def TypeNameFromFQN(fqn): + return fqn[1:] def FormatEmph(s): @@ -426,15 +245,17 @@ def FormatFieldType(type_context, field): Args: type_context: contextual information for message/enum/field. field: FieldDescriptor proto. - Return: - RST formatted field type. + Return: RST formatted field type. """ if field.type_name.startswith(ENVOY_API_NAMESPACE_PREFIX) or field.type_name.startswith( ENVOY_PREFIX): - type_name = NormalizeFQN(field.type_name) + type_name = NormalizeFieldTypeName(field.type_name) if field.type == field.TYPE_MESSAGE: - if type_context.map_typenames and type_name in type_context.map_typenames: - return type_context.map_typenames[type_name] + if type_context.map_typenames and TypeNameFromFQN( + field.type_name) in type_context.map_typenames: + return 'map<%s, %s>' % tuple( + map(functools.partial(FormatFieldType, type_context), + type_context.map_typenames[TypeNameFromFQN(field.type_name)])) return FormatInternalLink(type_name, MessageCrossRefLabel(type_name)) if field.type == field.TYPE_ENUM: return FormatInternalLink(type_name, EnumCrossRefLabel(type_name)) @@ -516,49 +337,55 @@ def FormatFieldAsDefinitionListItem(outer_type_context, type_context, field): outer_type_context: contextual information for enclosing message. type_context: contextual information for message/enum/field. field: FieldDescriptorProto. + Returns: RST formatted definition list item. """ - annotations = [] + field_annotations = [] - anchor = FormatAnchor(FieldCrossRefLabel(type_context.name)) + anchor = FormatAnchor(FieldCrossRefLabel(NormalizeTypeContextName(type_context.name))) if field.options.HasExtension(validate_pb2.rules): rule = field.options.Extensions[validate_pb2.rules] if ((rule.HasField('message') and rule.message.required) or (rule.HasField('string') and rule.string.min_bytes > 0) or (rule.HasField('repeated') and rule.repeated.min_items > 0)): - annotations = ['*REQUIRED*'] - leading_comment, comment_annotations = type_context.LeadingCommentPathLookup() - if NOT_IMPLEMENTED_HIDE_ANNOTATION in comment_annotations: + field_annotations = ['*REQUIRED*'] + leading_comment = type_context.leading_comment + formatted_leading_comment = FormatCommentWithAnnotations(leading_comment) + if HideNotImplemented(leading_comment): return '' if field.HasField('oneof_index'): oneof_context = outer_type_context.ExtendOneof(field.oneof_index, type_context.oneof_names[field.oneof_index]) - oneof_comment, oneof_comment_annotations = oneof_context.LeadingCommentPathLookup() - if NOT_IMPLEMENTED_HIDE_ANNOTATION in oneof_comment_annotations: + oneof_comment = oneof_context.leading_comment + formatted_oneof_comment = FormatCommentWithAnnotations(oneof_comment) + if HideNotImplemented(oneof_comment): return '' # If the oneof only has one field and marked required, mark the field as required. if len(type_context.oneof_fields[field.oneof_index]) == 1 and type_context.oneof_required[ field.oneof_index]: - annotations = ['*REQUIRED*'] + field_annotations = ['*REQUIRED*'] if len(type_context.oneof_fields[field.oneof_index]) > 1: # Fields in oneof shouldn't be marked as required when we have oneof comment below it. - annotations = [] + field_annotations = [] oneof_template = '\nPrecisely one of %s must be set.\n' if type_context.oneof_required[ field.oneof_index] else '\nOnly one of %s may be set.\n' - oneof_comment += oneof_template % ', '.join( - FormatInternalLink(f, FieldCrossRefLabel(outer_type_context.ExtendField(i, f).name)) + formatted_oneof_comment += oneof_template % ', '.join( + FormatInternalLink( + f, + FieldCrossRefLabel(NormalizeTypeContextName( + outer_type_context.ExtendField(i, f).name))) for i, f in type_context.oneof_fields[field.oneof_index]) else: - oneof_comment = '' + formatted_oneof_comment = '' comment = '(%s) ' % ', '.join([FormatFieldType(type_context, field)] + - annotations) + leading_comment + field_annotations) + formatted_leading_comment return anchor + field.name + '\n' + MapLines(functools.partial(Indent, 2), - comment + oneof_comment) + comment + formatted_oneof_comment) def FormatMessageAsDefinitionList(type_context, msg): @@ -567,6 +394,7 @@ def FormatMessageAsDefinitionList(type_context, msg): Args: type_context: contextual information for message/enum/field. msg: DescriptorProto. + Returns: RST formatted definition list item. """ @@ -575,9 +403,8 @@ def FormatMessageAsDefinitionList(type_context, msg): type_context.oneof_names = defaultdict(list) for index, field in enumerate(msg.field): if field.HasField('oneof_index'): - _, comment_annotations = type_context.ExtendField(index, - field.name).LeadingCommentPathLookup() - if NOT_IMPLEMENTED_HIDE_ANNOTATION in comment_annotations: + leading_comment = type_context.ExtendField(index, field.name).leading_comment + if HideNotImplemented(leading_comment): continue type_context.oneof_fields[field.oneof_index].append((index, field.name)) for index, oneof_decl in enumerate(msg.oneof_decl): @@ -589,59 +416,23 @@ def FormatMessageAsDefinitionList(type_context, msg): field) for index, field in enumerate(msg.field)) + '\n' -def FormatMessage(type_context, msg): - """Format a DescriptorProto as RST section. - - Args: - type_context: contextual information for message/enum/field. - msg: DescriptorProto. - Returns: - RST formatted section. - """ - # Skip messages synthesized to represent map types. - if msg.options.map_entry: - return '' - # We need to do some extra work to recover the map type annotation from the - # synthesized messages. - type_context.map_typenames = { - '%s.%s' % (type_context.name, nested_msg.name): - 'map<%s, %s>' % tuple(map(functools.partial(FormatFieldType, type_context), nested_msg.field)) - for nested_msg in msg.nested_type - if nested_msg.options.map_entry - } - nested_msgs = '\n'.join( - FormatMessage(type_context.ExtendNestedMessage(index, nested_msg.name), nested_msg) - for index, nested_msg in enumerate(msg.nested_type)) - nested_enums = '\n'.join( - FormatEnum(type_context.ExtendNestedEnum(index, nested_enum.name), nested_enum) - for index, nested_enum in enumerate(msg.enum_type)) - anchor = FormatAnchor(MessageCrossRefLabel(type_context.name)) - header = FormatHeader('-', type_context.name) - proto_link = FormatExternalLink('[%s proto]' % type_context.name, - type_context.GithubUrl()) + '\n\n' - leading_comment, annotations = type_context.LeadingCommentPathLookup() - if NOT_IMPLEMENTED_HIDE_ANNOTATION in annotations: - return '' - return anchor + header + proto_link + leading_comment + FormatMessageAsJson( - type_context, msg) + FormatMessageAsDefinitionList(type_context, - msg) + nested_msgs + '\n' + nested_enums - - def FormatEnumValueAsDefinitionListItem(type_context, enum_value): """Format a EnumValueDescriptorProto as RST definition list item. Args: type_context: contextual information for message/enum/field. enum_value: EnumValueDescriptorProto. + Returns: RST formatted definition list item. """ - anchor = FormatAnchor(EnumValueCrossRefLabel(type_context.name)) + anchor = FormatAnchor(EnumValueCrossRefLabel(NormalizeTypeContextName(type_context.name))) default_comment = '*(DEFAULT)* ' if enum_value.number == 0 else '' - leading_comment, annotations = type_context.LeadingCommentPathLookup() - if NOT_IMPLEMENTED_HIDE_ANNOTATION in annotations: + leading_comment = type_context.leading_comment + formatted_leading_comment = FormatCommentWithAnnotations(leading_comment) + if HideNotImplemented(leading_comment): return '' - comment = default_comment + UNICODE_INVISIBLE_SEPARATOR + leading_comment + comment = default_comment + UNICODE_INVISIBLE_SEPARATOR + formatted_leading_comment return anchor + enum_value.name + '\n' + MapLines(functools.partial(Indent, 2), comment) @@ -651,6 +442,7 @@ def FormatEnumAsDefinitionList(type_context, enum): Args: type_context: contextual information for message/enum/field. enum: DescriptorProto. + Returns: RST formatted definition list item. """ @@ -660,88 +452,57 @@ def FormatEnumAsDefinitionList(type_context, enum): for index, enum_value in enumerate(enum.value)) + '\n' -def FormatEnum(type_context, enum): - """Format an EnumDescriptorProto as RST section. +def FormatProtoAsBlockComment(proto): + """Format a proto as a RST block comment. - Args: - type_context: contextual information for message/enum/field. - enum: EnumDescriptorProto. - Returns: - RST formatted section. + Useful in debugging, not usually referenced. """ - anchor = FormatAnchor(EnumCrossRefLabel(type_context.name)) - header = FormatHeader('-', 'Enum %s' % type_context.name) - proto_link = FormatExternalLink('[%s proto]' % type_context.name, - type_context.GithubUrl()) + '\n\n' - leading_comment, annotations = type_context.LeadingCommentPathLookup() - if NOT_IMPLEMENTED_HIDE_ANNOTATION in annotations: - return '' - return anchor + header + proto_link + leading_comment + FormatEnumAsDefinitionList( - type_context, enum) + return '\n\nproto::\n\n' + MapLines(functools.partial(Indent, 2), str(proto)) + '\n' -def FormatProtoAsBlockComment(proto): - """Format as RST a proto as a block comment. +class RstFormatVisitor(visitor.Visitor): + """Visitor to generate a RST representation from a FileDescriptor proto. - Useful in debugging, not usually referenced. + See visitor.Visitor for visitor method docs comments. """ - return '\n\nproto::\n\n' + MapLines(functools.partial(Indent, 2), str(proto)) + '\n' + def VisitEnum(self, enum_proto, type_context): + normal_enum_type = NormalizeTypeContextName(type_context.name) + anchor = FormatAnchor(EnumCrossRefLabel(normal_enum_type)) + header = FormatHeader('-', 'Enum %s' % normal_enum_type) + github_url = GithubUrl(type_context) + proto_link = FormatExternalLink('[%s proto]' % normal_enum_type, github_url) + '\n\n' + leading_comment = type_context.leading_comment + formatted_leading_comment = FormatCommentWithAnnotations(leading_comment, 'enum') + if HideNotImplemented(leading_comment): + return '' + return anchor + header + proto_link + formatted_leading_comment + FormatEnumAsDefinitionList( + type_context, enum_proto) + + def VisitMessage(self, msg_proto, type_context, nested_msgs, nested_enums): + normal_msg_type = NormalizeTypeContextName(type_context.name) + anchor = FormatAnchor(MessageCrossRefLabel(normal_msg_type)) + header = FormatHeader('-', normal_msg_type) + github_url = GithubUrl(type_context) + proto_link = FormatExternalLink('[%s proto]' % normal_msg_type, github_url) + '\n\n' + leading_comment = type_context.leading_comment + formatted_leading_comment = FormatCommentWithAnnotations(leading_comment, 'message') + if HideNotImplemented(leading_comment): + return '' + return anchor + header + proto_link + formatted_leading_comment + FormatMessageAsJson( + type_context, msg_proto) + FormatMessageAsDefinitionList( + type_context, msg_proto) + '\n'.join(nested_msgs) + '\n' + '\n'.join(nested_enums) -def GenerateRst(proto_file): - """Generate a RST representation from a FileDescriptor proto.""" - source_code_info = SourceCodeInfo(proto_file.name, proto_file.source_code_info) - # Find the earliest detached comment, attribute it to file level. - # Also extract file level titles if any. - header, comment = FormatHeaderFromFile('=', source_code_info.file_level_comment, proto_file.name) - package_prefix = NormalizeFQN('.' + proto_file.package + '.')[:-1] - package_type_context = TypeContext(source_code_info, package_prefix) - msgs = '\n'.join( - FormatMessage(package_type_context.ExtendMessage(index, msg.name), msg) - for index, msg in enumerate(proto_file.message_type)) - enums = '\n'.join( - FormatEnum(package_type_context.ExtendEnum(index, enum.name), enum) - for index, enum in enumerate(proto_file.enum_type)) - debug_proto = FormatProtoAsBlockComment(proto_file) - return header + comment + msgs + enums # + debug_proto + def VisitFile(self, file_proto, type_context, msgs, enums): + # Find the earliest detached comment, attribute it to file level. + # Also extract file level titles if any. + header, comment = FormatHeaderFromFile('=', type_context.source_code_info, file_proto.name) + debug_proto = FormatProtoAsBlockComment(file_proto) + return header + comment + '\n'.join(msgs) + '\n'.join(enums) # + debug_proto def Main(): - # http://www.expobrain.net/2015/09/13/create-a-plugin-for-google-protocol-buffer/ - request = plugin_pb2.CodeGeneratorRequest() - request.ParseFromString(sys.stdin.buffer.read()) - response = plugin_pb2.CodeGeneratorResponse() - cprofile_enabled = os.getenv('CPROFILE_ENABLED') - - # We use file_to_generate rather than proto_file here since we are invoked - # inside a Bazel aspect, each node in the DAG will be visited once by the - # aspect and we only want to generate docs for the current node. - for file_to_generate in request.file_to_generate: - # Find the FileDescriptorProto for the file we actually are generating. - proto_file = None - for pf in request.proto_file: - if pf.name == file_to_generate: - proto_file = pf - break - assert (proto_file is not None) - f = response.file.add() - f.name = proto_file.name + '.rst' - if cprofile_enabled: - pr = cProfile.Profile() - pr.enable() - # We don't actually generate any RST right now, we just string dump the - # input proto file descriptor into the output file. - f.content = GenerateRst(proto_file) - if cprofile_enabled: - pr.disable() - stats_stream = io.StringIO() - ps = pstats.Stats(pr, - stream=stats_stream).sort_stats(os.getenv('CPROFILE_SORTBY', 'cumulative')) - stats_file = response.file.add() - stats_file.name = proto_file.name + '.rst.profile' - ps.print_stats() - stats_file.content = stats_stream.getvalue() - sys.stdout.buffer.write(response.SerializeToString()) + plugin.Plugin('.rst', RstFormatVisitor()) if __name__ == '__main__': From 874d7d35ddfbe7798585e89a7990081d0a12913c Mon Sep 17 00:00:00 2001 From: Tony Allen Date: Mon, 9 Sep 2019 14:21:53 -0700 Subject: [PATCH 521/542] adaptive concurrency: Gradient algorithm implementation (#7908) Signed-off-by: Tony Allen --- .../http/adaptive_concurrency/v2alpha/BUILD | 8 +- .../v2alpha/adaptive_concurrency.proto | 54 ++ .../http/adaptive_concurrency/v3alpha/BUILD | 15 - .../v3alpha/adaptive_concurrency.proto | 10 - source/common/common/cleanup.h | 10 +- source/common/protobuf/utility.h | 9 + .../adaptive_concurrency_filter.cc | 28 +- .../adaptive_concurrency_filter.h | 6 +- .../concurrency_controller/BUILD | 12 +- .../concurrency_controller.h | 13 +- .../gradient_controller.cc | 186 +++++++ .../gradient_controller.h | 205 ++++++++ test/common/common/cleanup_test.cc | 12 + test/common/protobuf/utility_test.cc | 2 + .../adaptive_concurrency_filter_test.cc | 72 ++- .../concurrency_controller/BUILD | 28 + .../gradient_controller_test.cc | 497 ++++++++++++++++++ tools/spelling_dictionary.txt | 1 + 18 files changed, 1127 insertions(+), 41 deletions(-) delete mode 100644 api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD delete mode 100644 api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto create mode 100644 source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc create mode 100644 source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h create mode 100644 test/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD create mode 100644 test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD index b58f88c787ba..a02fc542756c 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/BUILD @@ -3,13 +3,17 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "ap licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["//envoy/api/v2/core"], + deps = [ + "//envoy/api/v3alpha/core", + "//envoy/type", + ], ) api_proto_library_internal( name = "adaptive_concurrency", srcs = ["adaptive_concurrency.proto"], deps = [ - "//envoy/api/v2/core:base", + "//envoy/api/v3alpha/core:base", + "//envoy/type:percent", ], ) diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto index 303b681471f4..9b03169f7dd0 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto @@ -6,5 +6,59 @@ option java_package = "io.envoyproxy.envoy.config.filter.http.adaptive_concurren option java_outer_classname = "AdaptiveConcurrencyProto"; option java_multiple_files = true; +import "envoy/type/percent.proto"; + +import "google/protobuf/duration.proto"; +import "google/api/annotations.proto"; +import "google/protobuf/wrappers.proto"; + +import "validate/validate.proto"; + +// Configuration parameters for the gradient controller. +message GradientControllerConfig { + // The percentile to use when summarizing aggregated samples. Defaults to p50. + envoy.type.Percent sample_aggregate_percentile = 1; + + // Parameters controlling the periodic recalculation of the concurrency limit from sampled request + // latencies. + message ConcurrencyLimitCalculationParams { + // The maximum value the gradient is allowed to take. This influences how aggressively the + // concurrency limit can increase. Defaults to 2.0. + google.protobuf.DoubleValue max_gradient = 1 [(validate.rules).double.gt = 1.0]; + + // The allowed upper-bound on the calculated concurrency limit. Defaults to 1000. + google.protobuf.UInt32Value max_concurrency_limit = 2 [(validate.rules).uint32.gt = 0]; + + // The period of time samples are taken to recalculate the concurrency limit. + google.protobuf.Duration concurrency_update_interval = 3 [(validate.rules).duration = { + required: true, + gt: {seconds: 0} + }]; + } + ConcurrencyLimitCalculationParams concurrency_limit_params = 2 + [(validate.rules).message.required = true]; + + // Parameters controlling the periodic minRTT recalculation. + message MinimumRTTCalculationParams { + // The time interval between recalculating the minimum request round-trip time. + google.protobuf.Duration interval = 1 [(validate.rules).duration = { + required: true, + gt: {seconds: 0} + }]; + + // The number of requests to aggregate/sample during the minRTT recalculation window before + // updating. Defaults to 50. + google.protobuf.UInt32Value request_count = 2 [(validate.rules).uint32.gt = 0]; + }; + MinimumRTTCalculationParams min_rtt_calc_params = 3 [(validate.rules).message.required = true]; +} + message AdaptiveConcurrency { + oneof concurrency_controller_config { + option (validate.required) = true; + + // Gradient concurrency control will be used. + GradientControllerConfig gradient_controller_config = 1 + [(validate.rules).message.required = true]; + } } diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD deleted file mode 100644 index f9813a6a0829..000000000000 --- a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/BUILD +++ /dev/null @@ -1,15 +0,0 @@ -load("@envoy_api//bazel:api_build_system.bzl", "api_proto_library_internal", "api_proto_package") - -licenses(["notice"]) # Apache 2 - -api_proto_package( - deps = ["//envoy/api/v3alpha/core"], -) - -api_proto_library_internal( - name = "adaptive_concurrency", - srcs = ["adaptive_concurrency.proto"], - deps = [ - "//envoy/api/v3alpha/core:base", - ], -) diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto deleted file mode 100644 index 3d57196f9db7..000000000000 --- a/api/envoy/config/filter/http/adaptive_concurrency/v3alpha/adaptive_concurrency.proto +++ /dev/null @@ -1,10 +0,0 @@ -syntax = "proto3"; - -package envoy.config.filter.http.adaptive_concurrency.v3alpha; - -option java_package = "io.envoyproxy.envoy.config.filter.http.adaptive_concurrency.v3alpha"; -option java_outer_classname = "AdaptiveConcurrencyProto"; -option java_multiple_files = true; - -message AdaptiveConcurrency { -} diff --git a/source/common/common/cleanup.h b/source/common/common/cleanup.h index e7039ef069ce..1eafa29d44d6 100644 --- a/source/common/common/cleanup.h +++ b/source/common/common/cleanup.h @@ -10,11 +10,19 @@ namespace Envoy { // RAII cleanup via functor. class Cleanup { public: - Cleanup(std::function f) : f_(std::move(f)) {} + Cleanup(std::function f) : f_(std::move(f)), cancelled_(false) {} ~Cleanup() { f_(); } + void cancel() { + cancelled_ = true; + f_ = []() {}; + } + + bool cancelled() { return cancelled_; } + private: std::function f_; + bool cancelled_; }; // RAII helper class to add an element to an std::list on construction and erase diff --git a/source/common/protobuf/utility.h b/source/common/protobuf/utility.h index 00ae4bceb66c..1f29ea1d7921 100644 --- a/source/common/protobuf/utility.h +++ b/source/common/protobuf/utility.h @@ -84,6 +84,15 @@ uint64_t fractionalPercentDenominatorToInt( } // namespace ProtobufPercentHelper } // namespace Envoy +// Convert an envoy::api::v2::core::Percent to a double or a default. +// @param message supplies the proto message containing the field. +// @param field_name supplies the field name in the message. +// @param default_value supplies the default if the field is not present. +#define PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(message, field_name, default_value) \ + (!std::isnan((message).field_name().value()) \ + ? (message).has_##field_name() ? (message).field_name().value() : default_value \ + : throw EnvoyException(fmt::format("Value not in the range of 0..100 range."))) + // Convert an envoy::api::v2::core::Percent to a rounded integer or a default. // @param message supplies the proto message containing the field. // @param field_name supplies the field name in the message. diff --git a/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc index 1ec4dd8247e2..076ff9c57b60 100644 --- a/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc +++ b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc @@ -32,13 +32,35 @@ Http::FilterHeadersStatus AdaptiveConcurrencyFilter::decodeHeaders(Http::HeaderM return Http::FilterHeadersStatus::StopIteration; } - rq_start_time_ = config_->timeSource().monotonicTime(); + // When the deferred_sample_task_ object is destroyed, the time difference between its destruction + // and the request start time is measured as the request latency. This value is sampled by the + // concurrency controller either when encoding is complete or during destruction of this filter + // object. + deferred_sample_task_ = + std::make_unique([this, rq_start_time = config_->timeSource().monotonicTime()]() { + const auto now = config_->timeSource().monotonicTime(); + const std::chrono::nanoseconds rq_latency = now - rq_start_time; + controller_->recordLatencySample(rq_latency); + }); + return Http::FilterHeadersStatus::Continue; } void AdaptiveConcurrencyFilter::encodeComplete() { - const auto rq_latency = config_->timeSource().monotonicTime() - rq_start_time_; - controller_->recordLatencySample(rq_latency); + ASSERT(deferred_sample_task_); + deferred_sample_task_.reset(); +} + +void AdaptiveConcurrencyFilter::onDestroy() { + if (deferred_sample_task_) { + // The sampling task hasn't been destroyed yet, so this implies we did not complete encoding. + // Let's stop the sampling from happening and perform request cleanup inside the controller. + // + // TODO (tonya11en): Return some RAII handle from the concurrency controller that performs this + // logic as part of its lifecycle. + deferred_sample_task_->cancel(); + controller_->cancelLatencySample(); + } } } // namespace AdaptiveConcurrency diff --git a/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h index 88070180272b..0ebf7479b008 100644 --- a/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h +++ b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h @@ -11,6 +11,8 @@ #include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" +#include "common/common/cleanup.h" + #include "extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h" #include "extensions/filters/http/common/pass_through_filter.h" @@ -57,12 +59,12 @@ class AdaptiveConcurrencyFilter : public Http::PassThroughFilter, // Http::StreamEncoderFilter void encodeComplete() override; + void onDestroy() override; private: AdaptiveConcurrencyFilterConfigSharedPtr config_; const ConcurrencyControllerSharedPtr controller_; - MonotonicTime rq_start_time_; - std::unique_ptr forwarding_action_; + std::unique_ptr deferred_sample_task_; }; } // namespace AdaptiveConcurrency diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD index d213690d63c6..604221865c11 100644 --- a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD @@ -14,10 +14,20 @@ envoy_package() envoy_cc_library( name = "concurrency_controller_lib", - srcs = [], + srcs = ["gradient_controller.cc"], hdrs = [ "concurrency_controller.h", + "gradient_controller.h", + ], + external_deps = [ + "libcircllhist", ], deps = [ + "//source/common/event:dispatcher_lib", + "//source/common/protobuf", + "//source/common/runtime:runtime_lib", + "//source/common/stats:isolated_store_lib", + "//source/common/stats:stats_lib", + "@envoy_api//envoy/config/filter/http/adaptive_concurrency/v2alpha:adaptive_concurrency_cc", ], ) diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h index 0c0dbe456c7d..20342c0bd6cf 100644 --- a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h @@ -43,7 +43,18 @@ class ConcurrencyController { * * @param rq_latency is the clocked round-trip time for the request. */ - virtual void recordLatencySample(const std::chrono::nanoseconds& rq_latency) PURE; + virtual void recordLatencySample(std::chrono::nanoseconds rq_latency) PURE; + + /** + * Omit sampling an outstanding request and update the internal state of the controller to reflect + * request completion. + */ + virtual void cancelLatencySample() PURE; + + /** + * Returns the current concurrency limit. + */ + virtual uint32_t concurrencyLimit() const PURE; }; } // namespace ConcurrencyController diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc new file mode 100644 index 000000000000..3391c55fb6c3 --- /dev/null +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc @@ -0,0 +1,186 @@ +#include "extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h" + +#include +#include + +#include "envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.pb.h" +#include "envoy/event/dispatcher.h" +#include "envoy/runtime/runtime.h" +#include "envoy/stats/stats.h" + +#include "common/common/cleanup.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +#include "extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h" + +#include "absl/synchronization/mutex.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace AdaptiveConcurrency { +namespace ConcurrencyController { + +GradientControllerConfig::GradientControllerConfig( + const envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig& + proto_config) + : min_rtt_calc_interval_(std::chrono::milliseconds( + DurationUtil::durationToMilliseconds(proto_config.min_rtt_calc_params().interval()))), + sample_rtt_calc_interval_(std::chrono::milliseconds(DurationUtil::durationToMilliseconds( + proto_config.concurrency_limit_params().concurrency_update_interval()))), + max_concurrency_limit_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( + proto_config.concurrency_limit_params(), max_concurrency_limit, 1000)), + min_rtt_aggregate_request_count_( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config.min_rtt_calc_params(), request_count, 50)), + max_gradient_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config.concurrency_limit_params(), + max_gradient, 2.0)), + sample_aggregate_percentile_( + PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(proto_config, sample_aggregate_percentile, 50) / + 100.0) {} + +GradientController::GradientController(GradientControllerConfigSharedPtr config, + Event::Dispatcher& dispatcher, Runtime::Loader&, + const std::string& stats_prefix, Stats::Scope& scope) + : config_(std::move(config)), dispatcher_(dispatcher), scope_(scope), + stats_(generateStats(scope_, stats_prefix)), deferred_limit_value_(1), num_rq_outstanding_(0), + concurrency_limit_(1), latency_sample_hist_(hist_fast_alloc(), hist_free) { + min_rtt_calc_timer_ = dispatcher_.createTimer([this]() -> void { enterMinRTTSamplingWindow(); }); + + sample_reset_timer_ = dispatcher_.createTimer([this]() -> void { + if (inMinRTTSamplingWindow()) { + // The minRTT sampling window started since the sample reset timer was enabled last. Since the + // minRTT value is being calculated, let's give up on this timer to avoid blocking the + // dispatcher thread and rely on it being enabled again as part of the minRTT calculation. + return; + } + + { + absl::MutexLock ml(&sample_mutation_mtx_); + resetSampleWindow(); + } + + sample_reset_timer_->enableTimer(config_->sampleRTTCalcInterval()); + }); + + sample_reset_timer_->enableTimer(config_->sampleRTTCalcInterval()); + stats_.concurrency_limit_.set(concurrency_limit_.load()); +} + +GradientControllerStats GradientController::generateStats(Stats::Scope& scope, + const std::string& stats_prefix) { + return {ALL_GRADIENT_CONTROLLER_STATS(POOL_GAUGE_PREFIX(scope, stats_prefix))}; +} + +void GradientController::enterMinRTTSamplingWindow() { + absl::MutexLock ml(&sample_mutation_mtx_); + + // Set the minRTT flag to indicate we're gathering samples to update the value. This will + // prevent the sample window from resetting until enough requests are gathered to complete the + // recalculation. + deferred_limit_value_.store(concurrencyLimit()); + updateConcurrencyLimit(1); + + // Throw away any latency samples from before the recalculation window as it may not represent + // the minRTT. + hist_clear(latency_sample_hist_.get()); +} + +void GradientController::updateMinRTT() { + ASSERT(inMinRTTSamplingWindow()); + + { + absl::MutexLock ml(&sample_mutation_mtx_); + min_rtt_ = processLatencySamplesAndClear(); + stats_.min_rtt_msecs_.set( + std::chrono::duration_cast(min_rtt_).count()); + updateConcurrencyLimit(deferred_limit_value_.load()); + deferred_limit_value_.store(0); + } + + min_rtt_calc_timer_->enableTimer(config_->minRTTCalcInterval()); +} + +void GradientController::resetSampleWindow() { + // The sampling window must not be reset while sampling for the new minRTT value. + ASSERT(!inMinRTTSamplingWindow()); + + if (hist_sample_count(latency_sample_hist_.get()) == 0) { + return; + } + + sample_rtt_ = processLatencySamplesAndClear(); + updateConcurrencyLimit(calculateNewLimit()); +} + +std::chrono::microseconds GradientController::processLatencySamplesAndClear() { + const std::array quantile{config_->sampleAggregatePercentile()}; + std::array calculated_quantile; + hist_approx_quantile(latency_sample_hist_.get(), quantile.data(), 1, calculated_quantile.data()); + hist_clear(latency_sample_hist_.get()); + return std::chrono::microseconds(static_cast(calculated_quantile[0])); +} + +uint32_t GradientController::calculateNewLimit() { + // Calculate the gradient value, ensuring it remains below the configured maximum. + ASSERT(sample_rtt_.count() > 0); + const double raw_gradient = static_cast(min_rtt_.count()) / sample_rtt_.count(); + const double gradient = std::min(config_->maxGradient(), raw_gradient); + stats_.gradient_.set(gradient); + + const double limit = concurrencyLimit() * gradient; + const double burst_headroom = sqrt(limit); + stats_.burst_queue_size_.set(burst_headroom); + + // The final concurrency value factors in the burst headroom and must be clamped to keep the value + // in the range [1, configured_max]. + const auto clamp = [](int min, int max, int val) { return std::max(min, std::min(max, val)); }; + const uint32_t new_limit = limit + burst_headroom; + return clamp(1, config_->maxConcurrencyLimit(), new_limit); +} + +RequestForwardingAction GradientController::forwardingDecision() { + // Note that a race condition exists here which would allow more outstanding requests than the + // concurrency limit bounded by the number of worker threads. After loading num_rq_outstanding_ + // and before loading concurrency_limit_, another thread could potentially swoop in and modify + // num_rq_outstanding_, causing us to move forward with stale values and increment + // num_rq_outstanding_. + // + // TODO (tonya11en): Reconsider using a CAS loop here. + if (num_rq_outstanding_.load() < concurrencyLimit()) { + ++num_rq_outstanding_; + return RequestForwardingAction::Forward; + } + return RequestForwardingAction::Block; +} + +void GradientController::recordLatencySample(std::chrono::nanoseconds rq_latency) { + const uint32_t latency_usec = + std::chrono::duration_cast(rq_latency).count(); + ASSERT(num_rq_outstanding_.load() > 0); + --num_rq_outstanding_; + + uint32_t sample_count; + { + absl::MutexLock ml(&sample_mutation_mtx_); + hist_insert(latency_sample_hist_.get(), latency_usec, 1); + sample_count = hist_sample_count(latency_sample_hist_.get()); + } + + if (inMinRTTSamplingWindow() && sample_count >= config_->minRTTAggregateRequestCount()) { + // This sample has pushed the request count over the request count requirement for the minRTT + // recalculation. It must now be finished. + updateMinRTT(); + } +} + +void GradientController::cancelLatencySample() { + ASSERT(num_rq_outstanding_.load() > 0); + --num_rq_outstanding_; +} + +} // namespace ConcurrencyController +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h new file mode 100644 index 000000000000..a7e27f311467 --- /dev/null +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h @@ -0,0 +1,205 @@ +#pragma once + +#include +#include + +#include "envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.pb.h" +#include "envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.pb.validate.h" +#include "envoy/event/dispatcher.h" +#include "envoy/runtime/runtime.h" +#include "envoy/stats/stats_macros.h" + +#include "extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h" + +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" +#include "circllhist.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace AdaptiveConcurrency { +namespace ConcurrencyController { + +/** + * All stats for the gradient controller. + */ +#define ALL_GRADIENT_CONTROLLER_STATS(GAUGE) \ + GAUGE(concurrency_limit, NeverImport) \ + GAUGE(gradient, NeverImport) \ + GAUGE(burst_queue_size, NeverImport) \ + GAUGE(min_rtt_msecs, NeverImport) + +/** + * Wrapper struct for gradient controller stats. @see stats_macros.h + */ +struct GradientControllerStats { + ALL_GRADIENT_CONTROLLER_STATS(GENERATE_GAUGE_STRUCT) +}; + +class GradientControllerConfig { +public: + GradientControllerConfig( + const envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig& + proto_config); + + std::chrono::milliseconds minRTTCalcInterval() const { return min_rtt_calc_interval_; } + std::chrono::milliseconds sampleRTTCalcInterval() const { return sample_rtt_calc_interval_; } + uint32_t maxConcurrencyLimit() const { return max_concurrency_limit_; } + uint32_t minRTTAggregateRequestCount() const { return min_rtt_aggregate_request_count_; } + double maxGradient() const { return max_gradient_; } + double sampleAggregatePercentile() const { return sample_aggregate_percentile_; } + +private: + // The measured request round-trip time under ideal conditions. + const std::chrono::milliseconds min_rtt_calc_interval_; + + // The measured sample round-trip time from the previous time window. + const std::chrono::milliseconds sample_rtt_calc_interval_; + + // The maximum allowed concurrency value. + const uint32_t max_concurrency_limit_; + + // The number of requests to aggregate/sample during the minRTT recalculation. + const uint32_t min_rtt_aggregate_request_count_; + + // The maximum value the gradient may take. + const double max_gradient_; + + // The percentile value considered when processing samples. + const double sample_aggregate_percentile_; +}; +using GradientControllerConfigSharedPtr = std::shared_ptr; + +/** + * A concurrency controller that implements a variation of the Gradient algorithm described in: + * + * https://medium.com/@NetflixTechBlog/performance-under-load-3e6fa9a60581 + * + * This is used to control the allowed request concurrency limit in the adaptive concurrency control + * filter. + * + * The algorithm: + * ============== + * An ideal round-trip time (minRTT) is measured periodically by only allowing a single outstanding + * request at a time and measuring the round-trip time to the upstream. This information is then + * used in the calculation of a number called the gradient, using time-sampled latencies + * (sampleRTT): + * + * gradient = minRTT / sampleRTT + * + * This gradient value has a useful property, such that it decreases as the sampled latencies + * increase. The value is then used to periodically update the concurrency limit via: + * + * limit = old_limit * gradient + * new_limit = limit + headroom + * + * The headroom value allows for request bursts and is also the driving factor behind increasing the + * concurrency limit when the sampleRTT is in the same ballpark as the minRTT. This value must be + * present in the calculation, since it forces the concurrency limit to increase until there is a + * deviation from the minRTT latency. In its absence, the concurrency limit could remain stagnant at + * an unnecessarily small value if sampleRTT ~= minRTT. Therefore, the headroom value is + * unconfigurable and is set to the square-root of the new limit. + * + * Sampling: + * ========= + * The controller makes use of latency samples to either determine the minRTT or the sampleRTT which + * is used to periodically update the concurrency limit. Each calculation occurs at separate + * configurable frequencies and they may not occur at the same time. To prevent this, there exists a + * concept of mutually exclusive sampling windows. + * + * When the gradient controller is instantiated, it starts inside of a minRTT calculation window + * (indicated by inMinRTTSamplingWindow() returning true) and the concurrency limit is pinned to 1. + * This window lasts until the configured number of requests is received, the minRTT value is + * updated, and the minRTT value is set by a single worker thread. To prevent sampleRTT calculations + * from triggering during this window, the update window mutex is held. Since it's necessary for a + * worker thread to know which update window update window mutex is held for, they check the state + * of inMinRTTSamplingWindow() after each sample. When the minRTT calculation is complete, a timer + * is set to trigger the next minRTT sampling window by the worker thread who updates the minRTT + * value. + * + * If the controller is not in a minRTT sampling window, it's possible that the controller is in a + * sampleRTT calculation window. In this, all of the latency samples are consolidated into a + * configurable quantile value to represent the measured latencies. This quantile value sets + * sampleRTT and the concurrency limit is updated as described in the algorithm section above. + * + * When not in a sampling window, the controller is simply servicing the adaptive concurrency filter + * via the public functions. + * + * Locking: + * ======== + * There are 2 mutually exclusive calculation windows, so the sample mutation mutex is held to + * prevent the overlap of these windows. It is necessary for a worker thread to know specifically if + * the controller is inside of a minRTT recalculation window during the recording of a latency + * sample, so this extra bit of information is stored in inMinRTTSamplingWindow(). + */ +class GradientController : public ConcurrencyController { +public: + GradientController(GradientControllerConfigSharedPtr config, Event::Dispatcher& dispatcher, + Runtime::Loader& runtime, const std::string& stats_prefix, + Stats::Scope& scope); + + // ConcurrencyController. + RequestForwardingAction forwardingDecision() override; + void recordLatencySample(std::chrono::nanoseconds rq_latency) override; + void cancelLatencySample() override; + uint32_t concurrencyLimit() const override { return concurrency_limit_.load(); } + +private: + static GradientControllerStats generateStats(Stats::Scope& scope, + const std::string& stats_prefix); + void updateMinRTT(); + std::chrono::microseconds processLatencySamplesAndClear() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(sample_mutation_mtx_); + uint32_t calculateNewLimit() ABSL_EXCLUSIVE_LOCKS_REQUIRED(sample_mutation_mtx_); + void enterMinRTTSamplingWindow(); + bool inMinRTTSamplingWindow() const { return deferred_limit_value_.load() > 0; } + void resetSampleWindow() ABSL_EXCLUSIVE_LOCKS_REQUIRED(sample_mutation_mtx_); + void updateConcurrencyLimit(const uint32_t new_limit) { + concurrency_limit_.store(new_limit); + stats_.concurrency_limit_.set(concurrency_limit_.load()); + } + + const GradientControllerConfigSharedPtr config_; + Event::Dispatcher& dispatcher_; + Stats::Scope& scope_; + GradientControllerStats stats_; + + // Protects data related to latency sampling and RTT values. In addition to protecting the latency + // sample histogram, the mutex ensures that the minRTT calculation window and the sample window + // (where the new concurrency limit is determined) do not overlap. + absl::Mutex sample_mutation_mtx_; + + // Stores the value of the concurrency limit prior to entering the minRTT update window. If this + // is non-zero, then we are actively in the minRTT sampling window. + std::atomic deferred_limit_value_; + + // Stores the expected upstream latency value under ideal conditions. This is the numerator in the + // gradient value explained above. + std::chrono::nanoseconds min_rtt_; + std::chrono::nanoseconds sample_rtt_ ABSL_GUARDED_BY(sample_mutation_mtx_); + + // Tracks the count of requests that have been forwarded whose replies have + // not been sampled yet. Atomicity is required because this variable is used to make the + // forwarding decision without locking. + std::atomic num_rq_outstanding_; + + // Stores the current concurrency limit. Atomicity is required because this variable is used to + // make the forwarding decision without locking. + std::atomic concurrency_limit_; + + // Stores all sampled latencies and provides percentile estimations when using the sampled data to + // calculate a new concurrency limit. + std::unique_ptr + latency_sample_hist_ ABSL_GUARDED_BY(sample_mutation_mtx_); + + Event::TimerPtr min_rtt_calc_timer_; + Event::TimerPtr sample_reset_timer_; +}; +using GradientControllerSharedPtr = std::shared_ptr; + +} // namespace ConcurrencyController +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/common/common/cleanup_test.cc b/test/common/common/cleanup_test.cc index 7b666c163caf..98a590308727 100644 --- a/test/common/common/cleanup_test.cc +++ b/test/common/common/cleanup_test.cc @@ -13,6 +13,18 @@ TEST(CleanupTest, ScopeExitCallback) { EXPECT_TRUE(callback_fired); } +TEST(CleanupTest, Cancel) { + bool callback_fired = false; + { + Cleanup cleanup([&callback_fired] { callback_fired = true; }); + EXPECT_FALSE(cleanup.cancelled()); + cleanup.cancel(); + EXPECT_FALSE(callback_fired); + EXPECT_TRUE(cleanup.cancelled()); + } + EXPECT_FALSE(callback_fired); +} + TEST(RaiiListElementTest, DeleteOnDestruction) { std::list l; diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 539dfb6a28f6..945bec99d5f1 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -37,6 +37,8 @@ TEST_F(ProtobufUtilityTest, convertPercentNaN) { EXPECT_THROW(PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(common_config_, healthy_panic_threshold, 100, 50), EnvoyException); + EXPECT_THROW(PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(common_config_, healthy_panic_threshold, 0.5), + EnvoyException); } namespace ProtobufPercentHelper { diff --git a/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_test.cc b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_test.cc index 60ad871dc3e5..f859c031e660 100644 --- a/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_test.cc +++ b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_test.cc @@ -24,28 +24,35 @@ using ConcurrencyController::RequestForwardingAction; class MockConcurrencyController : public ConcurrencyController::ConcurrencyController { public: MOCK_METHOD0(forwardingDecision, RequestForwardingAction()); - MOCK_METHOD1(recordLatencySample, void(const std::chrono::nanoseconds&)); + MOCK_METHOD0(cancelLatencySample, void()); + MOCK_METHOD1(recordLatencySample, void(std::chrono::nanoseconds)); + + uint32_t concurrencyLimit() const override { return 0; } }; class AdaptiveConcurrencyFilterTest : public testing::Test { public: - AdaptiveConcurrencyFilterTest() { - filter_.reset(); + AdaptiveConcurrencyFilterTest() = default; + void SetUp() override { const envoy::config::filter::http::adaptive_concurrency::v2alpha::AdaptiveConcurrency config; auto config_ptr = std::make_shared( config, runtime_, "testprefix.", stats_, time_system_); filter_ = std::make_unique(config_ptr, controller_); filter_->setDecoderFilterCallbacks(decoder_callbacks_); + filter_->setEncoderFilterCallbacks(encoder_callbacks_); } - std::unique_ptr filter_; + void TearDown() override { filter_.reset(); } + Event::SimulatedTimeSystem time_system_; Stats::IsolatedStoreImpl stats_; NiceMock runtime_; std::shared_ptr controller_{new MockConcurrencyController()}; NiceMock decoder_callbacks_; + NiceMock encoder_callbacks_; + std::unique_ptr filter_; }; TEST_F(AdaptiveConcurrencyFilterTest, DecodeHeadersTestForwarding) { @@ -53,6 +60,8 @@ TEST_F(AdaptiveConcurrencyFilterTest, DecodeHeadersTestForwarding) { EXPECT_CALL(*controller_, forwardingDecision()) .WillOnce(Return(RequestForwardingAction::Forward)); + EXPECT_CALL(*controller_, recordLatencySample(_)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); Buffer::OwnedImpl request_body; @@ -71,9 +80,60 @@ TEST_F(AdaptiveConcurrencyFilterTest, DecodeHeadersTestBlock) { filter_->decodeHeaders(request_headers, true)); } +TEST_F(AdaptiveConcurrencyFilterTest, RecordSampleInDestructor) { + // Verify that the request latency is always sampled even if encodeComplete() is never called. + EXPECT_CALL(*controller_, forwardingDecision()) + .WillOnce(Return(RequestForwardingAction::Forward)); + Http::TestHeaderMapImpl request_headers; + filter_->decodeHeaders(request_headers, true); + + EXPECT_CALL(*controller_, recordLatencySample(_)); + filter_.reset(); +} + +TEST_F(AdaptiveConcurrencyFilterTest, RecordSampleOmission) { + // Verify that the request latency is not sampled if forwardingDecision blocks the request. + EXPECT_CALL(*controller_, forwardingDecision()).WillOnce(Return(RequestForwardingAction::Block)); + Http::TestHeaderMapImpl request_headers; + filter_->decodeHeaders(request_headers, true); + + filter_.reset(); +} + +TEST_F(AdaptiveConcurrencyFilterTest, OnDestroyCleanupResetTest) { + // Get the filter to record the request start time via decode. + Http::TestHeaderMapImpl request_headers; + EXPECT_CALL(*controller_, forwardingDecision()) + .WillOnce(Return(RequestForwardingAction::Forward)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); + + EXPECT_CALL(*controller_, cancelLatencySample()); + + // Encode step is not performed prior to destruction. + filter_->onDestroy(); +} + +TEST_F(AdaptiveConcurrencyFilterTest, OnDestroyCleanupTest) { + // Get the filter to record the request start time via decode. + Http::TestHeaderMapImpl request_headers; + EXPECT_CALL(*controller_, forwardingDecision()) + .WillOnce(Return(RequestForwardingAction::Forward)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); + + const auto advance_time = std::chrono::nanoseconds(42); + time_system_.sleep(advance_time); + + Http::TestHeaderMapImpl response_headers; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, false)); + EXPECT_CALL(*controller_, recordLatencySample(advance_time)); + filter_->encodeComplete(); + + filter_->onDestroy(); +} + TEST_F(AdaptiveConcurrencyFilterTest, EncodeHeadersValidTest) { auto mt = time_system_.monotonicTime(); - time_system_.setMonotonicTime(mt + std::chrono::milliseconds(123)); + time_system_.setMonotonicTime(mt + std::chrono::nanoseconds(123)); // Get the filter to record the request start time via decode. Http::TestHeaderMapImpl request_headers; @@ -81,7 +141,7 @@ TEST_F(AdaptiveConcurrencyFilterTest, EncodeHeadersValidTest) { .WillOnce(Return(RequestForwardingAction::Forward)); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); - const std::chrono::nanoseconds advance_time = std::chrono::milliseconds(42); + const auto advance_time = std::chrono::nanoseconds(42); mt = time_system_.monotonicTime(); time_system_.setMonotonicTime(mt + advance_time); diff --git a/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD b/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD new file mode 100644 index 000000000000..eda772937cd7 --- /dev/null +++ b/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/BUILD @@ -0,0 +1,28 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test_library", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "gradient_controller_test", + srcs = ["gradient_controller_test.cc"], + extension_name = "envoy.filters.http.adaptive_concurrency", + deps = [ + "//source/common/stats:isolated_store_lib", + "//source/extensions/filters/http/adaptive_concurrency:adaptive_concurrency_filter_lib", + "//source/extensions/filters/http/adaptive_concurrency/concurrency_controller:concurrency_controller_lib", + "//test/mocks/event:event_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc b/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc new file mode 100644 index 000000000000..1a523df9730a --- /dev/null +++ b/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc @@ -0,0 +1,497 @@ +#include +#include + +#include "envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.pb.h" +#include "envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.pb.validate.h" + +#include "common/stats/isolated_store_impl.h" + +#include "extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.h" +#include "extensions/filters/http/adaptive_concurrency/concurrency_controller/concurrency_controller.h" +#include "extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h" + +#include "test/mocks/event/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::AllOf; +using testing::Ge; +using testing::Le; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace AdaptiveConcurrency { +namespace ConcurrencyController { +namespace { + +class GradientControllerConfigTest : public testing::Test { +public: + GradientControllerConfigTest() = default; +}; + +class GradientControllerTest : public testing::Test { +public: + GradientControllerTest() + : api_(Api::createApiForTest(time_system_)), dispatcher_(api_->allocateDispatcher()) {} + + GradientControllerSharedPtr makeController(const std::string& yaml_config) { + return std::make_shared(makeConfig(yaml_config), *dispatcher_, runtime_, + "test_prefix.", stats_); + } + +protected: + GradientControllerConfigSharedPtr makeConfig(const std::string& yaml_config) { + envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig proto = + TestUtility::parseYaml< + envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig>( + yaml_config); + return std::make_shared(proto); + } + + // Helper function that will attempt to pull forwarding decisions. + void tryForward(const GradientControllerSharedPtr& controller, + const bool expect_forward_response) { + const auto expected_resp = + expect_forward_response ? RequestForwardingAction::Forward : RequestForwardingAction::Block; + EXPECT_EQ(expected_resp, controller->forwardingDecision()); + } + + // Gets the controller past the initial minRTT stage. + void advancePastMinRTTStage(const GradientControllerSharedPtr& controller, + const std::string& yaml_config, + std::chrono::milliseconds latency = std::chrono::milliseconds(5)) { + const auto config = makeConfig(yaml_config); + for (uint32_t ii = 0; ii <= config->minRTTAggregateRequestCount(); ++ii) { + tryForward(controller, true); + controller->recordLatencySample(latency); + } + } + + Event::SimulatedTimeSystem time_system_; + Stats::IsolatedStoreImpl stats_; + NiceMock runtime_; + Api::ApiPtr api_; + Event::DispatcherPtr dispatcher_; +}; + +TEST_F(GradientControllerConfigTest, BasicTest) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 42 +concurrency_limit_params: + max_gradient: 2.1 + max_concurrency_limit: 1337 + concurrency_update_interval: + nanos: 123000000 +min_rtt_calc_params: + interval: + seconds: 31 + request_count: 52 +)EOF"; + + envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig proto = + TestUtility::parseYaml< + envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig>( + yaml); + GradientControllerConfig config(proto); + + EXPECT_EQ(config.minRTTCalcInterval(), std::chrono::seconds(31)); + EXPECT_EQ(config.sampleRTTCalcInterval(), std::chrono::milliseconds(123)); + EXPECT_EQ(config.maxConcurrencyLimit(), 1337); + EXPECT_EQ(config.minRTTAggregateRequestCount(), 52); + EXPECT_EQ(config.maxGradient(), 2.1); + EXPECT_EQ(config.sampleAggregatePercentile(), 0.42); +} + +TEST_F(GradientControllerConfigTest, DefaultValuesTest) { + const std::string yaml = R"EOF( +concurrency_limit_params: + concurrency_update_interval: + nanos: 123000000 +min_rtt_calc_params: + interval: + seconds: 31 +)EOF"; + + envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig proto = + TestUtility::parseYaml< + envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig>( + yaml); + GradientControllerConfig config(proto); + + EXPECT_EQ(config.minRTTCalcInterval(), std::chrono::seconds(31)); + EXPECT_EQ(config.sampleRTTCalcInterval(), std::chrono::milliseconds(123)); + EXPECT_EQ(config.maxConcurrencyLimit(), 1000); + EXPECT_EQ(config.minRTTAggregateRequestCount(), 50); + EXPECT_EQ(config.maxGradient(), 2.0); + EXPECT_EQ(config.sampleAggregatePercentile(), 0.5); +} + +TEST_F(GradientControllerTest, MinRTTLogicTest) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 2.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 100000000 # 100ms +min_rtt_calc_params: + interval: + seconds: 30 + request_count: 50 +)EOF"; + + auto controller = makeController(yaml); + const auto min_rtt = std::chrono::milliseconds(13); + + // The controller should be measuring minRTT upon creation, so the concurrency window is 1. + EXPECT_EQ(controller->concurrencyLimit(), 1); + tryForward(controller, true); + tryForward(controller, false); + tryForward(controller, false); + controller->recordLatencySample(min_rtt); + + // 49 more requests should cause the minRTT to be done calculating. + for (int ii = 0; ii < 49; ++ii) { + EXPECT_EQ(controller->concurrencyLimit(), 1); + tryForward(controller, true); + tryForward(controller, false); + controller->recordLatencySample(min_rtt); + } + + // Verify the minRTT value measured is accurate. + EXPECT_EQ( + 13, stats_.gauge("test_prefix.min_rtt_msecs", Stats::Gauge::ImportMode::NeverImport).value()); +} + +TEST_F(GradientControllerTest, CancelLatencySample) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 2.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 100000000 # 100ms +min_rtt_calc_params: + interval: + seconds: 30 + request_count: 5 +)EOF"; + + auto controller = makeController(yaml); + + for (int ii = 1; ii <= 5; ++ii) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(ii)); + } + EXPECT_EQ( + 3, stats_.gauge("test_prefix.min_rtt_msecs", Stats::Gauge::ImportMode::NeverImport).value()); +} + +TEST_F(GradientControllerTest, SamplePercentileProcessTest) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 2.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 100000000 # 100ms +min_rtt_calc_params: + interval: + seconds: 30 + request_count: 5 +)EOF"; + + auto controller = makeController(yaml); + + tryForward(controller, true); + tryForward(controller, false); + controller->cancelLatencySample(); + tryForward(controller, true); + tryForward(controller, false); +} + +TEST_F(GradientControllerTest, ConcurrencyLimitBehaviorTestBasic) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 2.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 100000000 # 100ms +min_rtt_calc_params: + interval: + seconds: 30 + request_count: 5 +)EOF"; + + auto controller = makeController(yaml); + EXPECT_EQ(controller->concurrencyLimit(), 1); + + // Force a minRTT of 5ms. + advancePastMinRTTStage(controller, yaml, std::chrono::milliseconds(5)); + EXPECT_EQ( + 5, stats_.gauge("test_prefix.min_rtt_msecs", Stats::Gauge::ImportMode::NeverImport).value()); + + // Ensure that the concurrency window increases on its own due to the headroom calculation. + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_GT(controller->concurrencyLimit(), 1); + + // Make it seem as if the recorded latencies are consistently lower than the measured minRTT. + // Ensure that it grows. + for (int recalcs = 0; recalcs < 10; ++recalcs) { + const auto last_concurrency = controller->concurrencyLimit(); + for (int ii = 1; ii <= 5; ++ii) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(4)); + } + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_GT(controller->concurrencyLimit(), last_concurrency); + } + + // Verify that the concurrency limit can now shrink as necessary. + for (int recalcs = 0; recalcs < 10; ++recalcs) { + const auto last_concurrency = controller->concurrencyLimit(); + for (int ii = 1; ii <= 5; ++ii) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(6)); + } + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_LT(controller->concurrencyLimit(), last_concurrency); + } +} + +TEST_F(GradientControllerTest, MaxGradientTest) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 3.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 100000000 # 100ms +min_rtt_calc_params: + interval: + seconds: 30 + request_count: 5 +)EOF"; + + auto controller = makeController(yaml); + EXPECT_EQ(controller->concurrencyLimit(), 1); + + // Force a minRTT of 5 seconds. + advancePastMinRTTStage(controller, yaml, std::chrono::seconds(5)); + + // circllhist approximates the percentiles, so we can expect it to be within a certain range. + EXPECT_THAT( + stats_.gauge("test_prefix.min_rtt_msecs", Stats::Gauge::ImportMode::NeverImport).value(), + AllOf(Ge(4950), Le(5050))); + + // Now verify max gradient value by forcing dramatically faster latency measurements.. + for (int ii = 1; ii <= 5; ++ii) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(4)); + } + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_EQ(3.0, + stats_.gauge("test_prefix.gradient", Stats::Gauge::ImportMode::NeverImport).value()); +} + +TEST_F(GradientControllerTest, MinRTTReturnToPreviousLimit) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 3.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 100000000 # 100ms +min_rtt_calc_params: + interval: + seconds: 30 + request_count: 5 +)EOF"; + + auto controller = makeController(yaml); + EXPECT_EQ(controller->concurrencyLimit(), 1); + + // Get initial minRTT measurement out of the way. + advancePastMinRTTStage(controller, yaml, std::chrono::milliseconds(5)); + + // Force the limit calculation to run a few times from some measurements. + for (int sample_iters = 0; sample_iters < 5; ++sample_iters) { + const auto last_concurrency = controller->concurrencyLimit(); + for (int ii = 1; ii <= 5; ++ii) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(4)); + } + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + // Verify the value is growing. + EXPECT_GT(controller->concurrencyLimit(), last_concurrency); + } + + const auto limit_val = controller->concurrencyLimit(); + + // Wait until the minRTT recalculation is triggered again and verify the limit drops. + time_system_.sleep(std::chrono::seconds(31)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_EQ(controller->concurrencyLimit(), 1); + + // 49 more requests should cause the minRTT to be done calculating. + for (int ii = 0; ii < 5; ++ii) { + EXPECT_EQ(controller->concurrencyLimit(), 1); + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(13)); + } + + // Check that we restored the old concurrency limit value. + EXPECT_EQ(limit_val, controller->concurrencyLimit()); +} + +TEST_F(GradientControllerTest, MinRTTRescheduleTest) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 3.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 100000000 # 100ms +min_rtt_calc_params: + interval: + seconds: 30 + request_count: 5 +)EOF"; + + auto controller = makeController(yaml); + EXPECT_EQ(controller->concurrencyLimit(), 1); + + // Get initial minRTT measurement out of the way. + advancePastMinRTTStage(controller, yaml, std::chrono::milliseconds(5)); + + // Force the limit calculation to run a few times from some measurements. + for (int sample_iters = 0; sample_iters < 5; ++sample_iters) { + const auto last_concurrency = controller->concurrencyLimit(); + for (int ii = 1; ii <= 5; ++ii) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(4)); + } + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + // Verify the value is growing. + EXPECT_GT(controller->concurrencyLimit(), last_concurrency); + } + + // Wait until the minRTT recalculation is triggered again and verify the limit drops. + time_system_.sleep(std::chrono::seconds(31)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_EQ(controller->concurrencyLimit(), 1); + + // Verify sample recalculation doesn't occur during the minRTT window. + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_EQ(controller->concurrencyLimit(), 1); +} + +TEST_F(GradientControllerTest, NoSamplesTest) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 3.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 100000000 # 100ms +min_rtt_calc_params: + interval: + seconds: 30 + request_count: 5 +)EOF"; + + auto controller = makeController(yaml); + EXPECT_EQ(controller->concurrencyLimit(), 1); + + // Get minRTT measurement out of the way. + advancePastMinRTTStage(controller, yaml, std::chrono::milliseconds(5)); + + // Force the limit calculation to run a few times from some measurements. + for (int sample_iters = 0; sample_iters < 5; ++sample_iters) { + const auto last_concurrency = controller->concurrencyLimit(); + for (int ii = 1; ii <= 5; ++ii) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(4)); + } + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + // Verify the value is growing. + EXPECT_GT(controller->concurrencyLimit(), last_concurrency); + } + + // Now we make sure that the limit value doesn't change in the absence of samples. + for (int sample_iters = 0; sample_iters < 5; ++sample_iters) { + const auto old_limit = controller->concurrencyLimit(); + time_system_.sleep(std::chrono::milliseconds(101)); + dispatcher_->run(Event::Dispatcher::RunType::Block); + EXPECT_EQ(old_limit, controller->concurrencyLimit()); + } +} + +TEST_F(GradientControllerTest, TimerAccuracyTest) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 3.0 + max_concurrency_limit: + concurrency_update_interval: + nanos: 123000000 # 123ms +min_rtt_calc_params: + interval: + seconds: 45 + request_count: 5 +)EOF"; + + // Verify the configuration affects the timers that are kicked off. + NiceMock fake_dispatcher; + auto sample_timer = new NiceMock; + auto rtt_timer = new NiceMock; + + // Expect the sample timer to trigger start immediately upon controller creation. + EXPECT_CALL(fake_dispatcher, createTimer_(_)) + .Times(2) + .WillOnce(Return(rtt_timer)) + .WillOnce(Return(sample_timer)); + EXPECT_CALL(*sample_timer, enableTimer(std::chrono::milliseconds(123), _)); + auto controller = std::make_shared(makeConfig(yaml), fake_dispatcher, + runtime_, "test_prefix.", stats_); + + // Set the minRTT- this will trigger the timer for the next minRTT calculation. + EXPECT_CALL(*rtt_timer, enableTimer(std::chrono::milliseconds(45000), _)); + for (int ii = 1; ii <= 6; ++ii) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(5)); + } +} + +} // namespace +} // namespace ConcurrencyController +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 7bb603b68a4c..530f79bb325a 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -15,6 +15,7 @@ ASCII ASSERTs AWS BSON +CAS CB CBs CDS From 5aea70c46dae5c78bd09702801fca3620437057a Mon Sep 17 00:00:00 2001 From: Dhi Aurrahman Date: Tue, 10 Sep 2019 07:15:28 +0700 Subject: [PATCH 522/542] ext_authz: Check for cluster before sending HTTP request (#8144) Signed-off-by: Dhi Aurrahman --- .../common/ext_authz/ext_authz_http_impl.cc | 17 ++++++++++++++--- .../ext_authz/ext_authz_http_impl_test.cc | 14 ++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc index b40a927fdbfb..b49960bf511a 100644 --- a/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc +++ b/source/extensions/filters/common/ext_authz/ext_authz_http_impl.cc @@ -202,9 +202,20 @@ void RawHttpClientImpl::check(RequestCallbacks& callbacks, std::make_unique(request.attributes().request().http().body()); } - request_ = cm_.httpAsyncClientForCluster(config_->cluster()) - .send(std::move(message), *this, - Http::AsyncClient::RequestOptions().setTimeout(config_->timeout())); + const std::string& cluster = config_->cluster(); + + // It's possible that the cluster specified in the filter configuration no longer exists due to a + // CDS removal. + if (cm_.get(cluster) == nullptr) { + // TODO(dio): Add stats and tracing related to this. + ENVOY_LOG(debug, "ext_authz cluster '{}' does not exist", cluster); + callbacks_->onComplete(std::make_unique(errorResponse())); + callbacks_ = nullptr; + } else { + request_ = cm_.httpAsyncClientForCluster(cluster).send( + std::move(message), *this, + Http::AsyncClient::RequestOptions().setTimeout(config_->timeout())); + } } void RawHttpClientImpl::onSuccess(Http::MessagePtr&& message) { diff --git a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc index e8d40a655a3f..6941c1b3624d 100644 --- a/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc +++ b/test/extensions/filters/common/ext_authz/ext_authz_http_impl_test.cc @@ -16,6 +16,8 @@ using testing::_; using testing::AllOf; +using testing::Eq; +using testing::InSequence; using testing::Invoke; using testing::Return; using testing::ReturnRef; @@ -374,6 +376,18 @@ TEST_F(ExtAuthzHttpClientTest, CancelledAuthorizationRequest) { client_.cancel(); } +// Test the client when the configured cluster is missing/removed. +TEST_F(ExtAuthzHttpClientTest, NoCluster) { + InSequence s; + + EXPECT_CALL(cm_, get(Eq("ext_authz"))).WillOnce(Return(nullptr)); + EXPECT_CALL(cm_, httpAsyncClientForCluster("ext_authz")).Times(0); + EXPECT_CALL(request_callbacks_, + onComplete_(WhenDynamicCastTo(AuthzErrorResponse(CheckStatus::Error)))); + client_.check(request_callbacks_, envoy::service::auth::v2::CheckRequest{}, + Tracing::NullSpan::instance()); +} + } // namespace } // namespace ExtAuthz } // namespace Common From 66cc26a824d77696ad900c988bb6986a9da0a163 Mon Sep 17 00:00:00 2001 From: Kuat Date: Tue, 10 Sep 2019 11:33:54 -0700 Subject: [PATCH 523/542] make getters const-ref (#8192) Description: Follow-up to #7911 to make cached values be exposed as const-references, saving on a copy of a string during retrieval. Risk Level: low Testing: updated mocks to return references Docs Changes: none Release Notes: none Signed-off-by: Kuat Yessenov --- include/envoy/ssl/connection.h | 12 ++--- .../transport_sockets/tls/ssl_socket.cc | 18 +++++--- .../transport_sockets/tls/ssl_socket.h | 13 +++--- .../access_log/access_log_formatter_test.cc | 44 +++++++++++-------- test/common/http/codec_client_test.cc | 5 ++- test/common/http/conn_manager_utility_test.cc | 8 ++-- test/common/router/header_formatter_test.cc | 42 +++++++++++------- test/common/router/router_test.cc | 4 +- test/common/tcp_proxy/tcp_proxy_test.cc | 4 +- .../grpc/http_grpc_access_log_impl_test.cc | 43 ++++++++++++++---- .../ext_authz/check_request_utils_test.cc | 18 +++++++- .../filters/common/rbac/matchers_test.cc | 3 +- test/fuzz/utility.h | 6 ++- test/mocks/ssl/mocks.h | 12 ++--- 14 files changed, 149 insertions(+), 83 deletions(-) diff --git a/include/envoy/ssl/connection.h b/include/envoy/ssl/connection.h index d4e54ba92e41..d586d9fe09a5 100644 --- a/include/envoy/ssl/connection.h +++ b/include/envoy/ssl/connection.h @@ -33,7 +33,7 @@ class ConnectionInfo { * @return std::string the subject field of the local certificate in RFC 2253 format. Returns "" * if there is no local certificate, or no subject. **/ - virtual std::string subjectLocalCertificate() const PURE; + virtual const std::string& subjectLocalCertificate() const PURE; /** * @return std::string the SHA256 digest of the peer certificate. Returns "" if there is no peer @@ -45,19 +45,19 @@ class ConnectionInfo { * @return std::string the serial number field of the peer certificate. Returns "" if * there is no peer certificate, or no serial number. **/ - virtual std::string serialNumberPeerCertificate() const PURE; + virtual const std::string& serialNumberPeerCertificate() const PURE; /** * @return std::string the issuer field of the peer certificate in RFC 2253 format. Returns "" if * there is no peer certificate, or no issuer. **/ - virtual std::string issuerPeerCertificate() const PURE; + virtual const std::string& issuerPeerCertificate() const PURE; /** * @return std::string the subject field of the peer certificate in RFC 2253 format. Returns "" if * there is no peer certificate, or no subject. **/ - virtual std::string subjectPeerCertificate() const PURE; + virtual const std::string& subjectPeerCertificate() const PURE; /** * @return std::string the URIs in the SAN field of the peer certificate. Returns {} if there is @@ -105,7 +105,7 @@ class ConnectionInfo { /** * @return std::string the hex-encoded TLS session ID as defined in rfc5246. **/ - virtual std::string sessionId() const PURE; + virtual const std::string& sessionId() const PURE; /** * @return uint16_t the standard ID for the ciphers used in the established TLS connection. @@ -123,7 +123,7 @@ class ConnectionInfo { * @return std::string the TLS version (e.g., TLSv1.2, TLSv1.3) used in the established TLS * connection. **/ - virtual std::string tlsVersion() const PURE; + virtual const std::string& tlsVersion() const PURE; }; using ConnectionInfoConstSharedPtr = std::shared_ptr; diff --git a/source/extensions/transport_sockets/tls/ssl_socket.cc b/source/extensions/transport_sockets/tls/ssl_socket.cc index f0ec870245a9..7737d36b160f 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.cc +++ b/source/extensions/transport_sockets/tls/ssl_socket.cc @@ -475,11 +475,17 @@ std::string SslSocketInfo::ciphersuiteString() const { return SSL_CIPHER_get_name(cipher); } -std::string SslSocketInfo::tlsVersion() const { return SSL_get_version(ssl_.get()); } +const std::string& SslSocketInfo::tlsVersion() const { + if (!cached_tls_version_.empty()) { + return cached_tls_version_; + } + cached_tls_version_ = SSL_get_version(ssl_.get()); + return cached_tls_version_; +} absl::string_view SslSocket::failureReason() const { return failure_reason_; } -std::string SslSocketInfo::serialNumberPeerCertificate() const { +const std::string& SslSocketInfo::serialNumberPeerCertificate() const { if (!cached_serial_number_peer_certificate_.empty()) { return cached_serial_number_peer_certificate_; } @@ -492,7 +498,7 @@ std::string SslSocketInfo::serialNumberPeerCertificate() const { return cached_serial_number_peer_certificate_; } -std::string SslSocketInfo::issuerPeerCertificate() const { +const std::string& SslSocketInfo::issuerPeerCertificate() const { if (!cached_issuer_peer_certificate_.empty()) { return cached_issuer_peer_certificate_; } @@ -505,7 +511,7 @@ std::string SslSocketInfo::issuerPeerCertificate() const { return cached_issuer_peer_certificate_; } -std::string SslSocketInfo::subjectPeerCertificate() const { +const std::string& SslSocketInfo::subjectPeerCertificate() const { if (!cached_subject_peer_certificate_.empty()) { return cached_subject_peer_certificate_; } @@ -518,7 +524,7 @@ std::string SslSocketInfo::subjectPeerCertificate() const { return cached_subject_peer_certificate_; } -std::string SslSocketInfo::subjectLocalCertificate() const { +const std::string& SslSocketInfo::subjectLocalCertificate() const { if (!cached_subject_local_certificate_.empty()) { return cached_subject_local_certificate_; } @@ -547,7 +553,7 @@ absl::optional SslSocketInfo::expirationPeerCertificate() const { return Utility::getExpirationTime(*cert); } -std::string SslSocketInfo::sessionId() const { +const std::string& SslSocketInfo::sessionId() const { if (!cached_session_id_.empty()) { return cached_session_id_; } diff --git a/source/extensions/transport_sockets/tls/ssl_socket.h b/source/extensions/transport_sockets/tls/ssl_socket.h index 67ab1b28582a..c7b183451b4f 100644 --- a/source/extensions/transport_sockets/tls/ssl_socket.h +++ b/source/extensions/transport_sockets/tls/ssl_socket.h @@ -49,10 +49,10 @@ class SslSocketInfo : public Envoy::Ssl::ConnectionInfo { bool peerCertificatePresented() const override; std::vector uriSanLocalCertificate() const override; const std::string& sha256PeerCertificateDigest() const override; - std::string serialNumberPeerCertificate() const override; - std::string issuerPeerCertificate() const override; - std::string subjectPeerCertificate() const override; - std::string subjectLocalCertificate() const override; + const std::string& serialNumberPeerCertificate() const override; + const std::string& issuerPeerCertificate() const override; + const std::string& subjectPeerCertificate() const override; + const std::string& subjectLocalCertificate() const override; std::vector uriSanPeerCertificate() const override; const std::string& urlEncodedPemEncodedPeerCertificate() const override; const std::string& urlEncodedPemEncodedPeerCertificateChain() const override; @@ -60,10 +60,10 @@ class SslSocketInfo : public Envoy::Ssl::ConnectionInfo { std::vector dnsSansLocalCertificate() const override; absl::optional validFromPeerCertificate() const override; absl::optional expirationPeerCertificate() const override; - std::string sessionId() const override; + const std::string& sessionId() const override; uint16_t ciphersuiteId() const override; std::string ciphersuiteString() const override; - std::string tlsVersion() const override; + const std::string& tlsVersion() const override; SSL* rawSslForTest() const { return ssl_.get(); } @@ -82,6 +82,7 @@ class SslSocketInfo : public Envoy::Ssl::ConnectionInfo { mutable std::vector cached_dns_san_peer_certificate_; mutable std::vector cached_dns_san_local_certificate_; mutable std::string cached_session_id_; + mutable std::string cached_tls_version_; }; class SslSocket : public Network::TransportSocket, diff --git a/test/common/access_log/access_log_formatter_test.cc b/test/common/access_log/access_log_formatter_test.cc index db682288410b..e05f0911a129 100644 --- a/test/common/access_log/access_log_formatter_test.cc +++ b/test/common/access_log/access_log_formatter_test.cc @@ -279,14 +279,17 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_SUBJECT"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, subjectLocalCertificate()).WillRepeatedly(Return("subject")); + const std::string subject_local = "subject"; + EXPECT_CALL(*connection_info, subjectLocalCertificate()) + .WillRepeatedly(ReturnRef(subject_local)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("subject", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_LOCAL_SUBJECT"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, subjectLocalCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(*connection_info, subjectLocalCertificate()) + .WillRepeatedly(ReturnRef(EMPTY_STRING)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } @@ -298,14 +301,15 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(Return("subject")); + const std::string subject_peer = "subject"; + EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(subject_peer)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("subject", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(EMPTY_STRING)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } @@ -317,14 +321,15 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_SESSION_ID"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(Return("deadbeef")); + const std::string session_id = "deadbeef"; + EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(ReturnRef(session_id)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("deadbeef", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_SESSION_ID"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(Return("")); + EXPECT_CALL(*connection_info, sessionId()).WillRepeatedly(ReturnRef(EMPTY_STRING)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } @@ -357,14 +362,15 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_VERSION"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(Return("TLSv1.2")); + std::string tlsVersion = "TLSv1.2"; + EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(ReturnRef(tlsVersion)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("TLSv1.2", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_TLS_VERSION"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(Return("")); + EXPECT_CALL(*connection_info, tlsVersion()).WillRepeatedly(ReturnRef(EMPTY_STRING)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } @@ -399,15 +405,17 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SERIAL"); auto connection_info = std::make_shared(); + const std::string serial_number = "b8b5ecc898f2124a"; EXPECT_CALL(*connection_info, serialNumberPeerCertificate()) - .WillRepeatedly(Return("b8b5ecc898f2124a")); + .WillRepeatedly(ReturnRef(serial_number)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("b8b5ecc898f2124a", upstream_format.format(header, header, header, stream_info)); } { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SERIAL"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, serialNumberPeerCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(*connection_info, serialNumberPeerCertificate()) + .WillRepeatedly(ReturnRef(EMPTY_STRING)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } @@ -419,9 +427,9 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_ISSUER"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, issuerPeerCertificate()) - .WillRepeatedly( - Return("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); + const std::string issuer_peer = + "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; + EXPECT_CALL(*connection_info, issuerPeerCertificate()).WillRepeatedly(ReturnRef(issuer_peer)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US", upstream_format.format(header, header, header, stream_info)); @@ -429,7 +437,7 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_ISSUER"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, issuerPeerCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(*connection_info, issuerPeerCertificate()).WillRepeatedly(ReturnRef(EMPTY_STRING)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } @@ -441,9 +449,9 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, subjectPeerCertificate()) - .WillRepeatedly( - Return("CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); + const std::string subject_peer = + "CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; + EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(subject_peer)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US", upstream_format.format(header, header, header, stream_info)); @@ -451,7 +459,7 @@ TEST(AccessLogFormatterTest, streamInfoFormatter) { { StreamInfoFormatter upstream_format("DOWNSTREAM_PEER_SUBJECT"); auto connection_info = std::make_shared(); - EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(Return("")); + EXPECT_CALL(*connection_info, subjectPeerCertificate()).WillRepeatedly(ReturnRef(EMPTY_STRING)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); EXPECT_EQ("-", upstream_format.format(header, header, header, stream_info)); } diff --git a/test/common/http/codec_client_test.cc b/test/common/http/codec_client_test.cc index 8c7d63a7f6bb..1de3c04cb424 100644 --- a/test/common/http/codec_client_test.cc +++ b/test/common/http/codec_client_test.cc @@ -32,6 +32,7 @@ using testing::NiceMock; using testing::Pointee; using testing::Ref; using testing::Return; +using testing::ReturnRef; using testing::Throw; namespace Envoy { @@ -260,9 +261,9 @@ TEST_F(CodecClientTest, WatermarkPassthrough) { } TEST_F(CodecClientTest, SSLConnectionInfo) { - const auto session_id = "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; + std::string session_id = "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, sessionId()).WillByDefault(Return(session_id)); + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(session_id)); EXPECT_CALL(*connection_, ssl()).WillRepeatedly(Return(connection_info)); connection_cb_->onEvent(Network::ConnectionEvent::Connected); EXPECT_NE(nullptr, stream_info_.downstreamSslConnection()); diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 1eb5ab52e5fc..2e591a77cf92 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -932,8 +932,8 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCert) { EXPECT_CALL(*ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); std::string expected_sha("abcdefg"); EXPECT_CALL(*ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); - EXPECT_CALL(*ssl, subjectPeerCertificate()) - .WillOnce(Return("/C=US/ST=CA/L=San Francisco/OU=Lyft/CN=test.lyft.com")); + std::string peer_subject("/C=US/ST=CA/L=San Francisco/OU=Lyft/CN=test.lyft.com"); + EXPECT_CALL(*ssl, subjectPeerCertificate()).WillOnce(ReturnRef(peer_subject)); const std::vector peer_uri_sans{"test://foo.com/fe"}; EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(peer_uri_sans)); std::string expected_pem("abcde="); @@ -973,8 +973,8 @@ TEST_F(ConnectionManagerUtilityTest, MtlsSanitizeSetClientCertPeerSanEmpty) { EXPECT_CALL(*ssl, uriSanLocalCertificate()).WillOnce(Return(local_uri_sans)); std::string expected_sha("abcdefg"); EXPECT_CALL(*ssl, sha256PeerCertificateDigest()).WillOnce(ReturnRef(expected_sha)); - EXPECT_CALL(*ssl, subjectPeerCertificate()) - .WillOnce(Return("/C=US/ST=CA/L=San Francisco/OU=Lyft/CN=test.lyft.com")); + std::string peer_subject = "/C=US/ST=CA/L=San Francisco/OU=Lyft/CN=test.lyft.com"; + EXPECT_CALL(*ssl, subjectPeerCertificate()).WillOnce(ReturnRef(peer_subject)); EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(std::vector())); ON_CALL(connection_, ssl()).WillByDefault(Return(ssl)); ON_CALL(config_, forwardClientCert()) diff --git a/test/common/router/header_formatter_test.cc b/test/common/router/header_formatter_test.cc index 3cbe129caea2..34584a4420ae 100644 --- a/test/common/router/header_formatter_test.cc +++ b/test/common/router/header_formatter_test.cc @@ -156,7 +156,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalUriSanNoTls) TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubject) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(Return("subject")); + std::string subject = "subject"; + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(subject)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_LOCAL_SUBJECT", "subject"); } @@ -164,7 +165,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubject) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubjectEmpty) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(Return("")); + std::string subject; + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(subject)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_LOCAL_SUBJECT", EMPTY_STRING); } @@ -178,7 +180,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamLocalSubjectNoTls) TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionId) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, sessionId()).WillByDefault(Return("deadbeef")); + std::string session_id = "deadbeef"; + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(session_id)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_SESSION_ID", "deadbeef"); } @@ -186,7 +189,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionId) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsSessionIdEmpty) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, sessionId()).WillByDefault(Return("")); + std::string session_id; + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(session_id)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_SESSION_ID", EMPTY_STRING); } @@ -223,7 +227,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsCipherNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersion) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.2")); + std::string tls_version = "TLSv1.2"; + ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(tls_version)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_VERSION", "TLSv1.2"); } @@ -231,7 +236,7 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersion) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamTlsVersionEmpty) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("")); + ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(EMPTY_STRING)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_TLS_VERSION", EMPTY_STRING); } @@ -270,8 +275,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerFingerprintNoT TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerial) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, serialNumberPeerCertificate()) - .WillByDefault(Return("b8b5ecc898f2124a")); + const std::string serial_number = "b8b5ecc898f2124a"; + ON_CALL(*connection_info, serialNumberPeerCertificate()).WillByDefault(ReturnRef(serial_number)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_SERIAL", "b8b5ecc898f2124a"); } @@ -279,7 +284,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerial) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerialEmpty) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, serialNumberPeerCertificate()).WillByDefault(Return("")); + const std::string serial_number; + ON_CALL(*connection_info, serialNumberPeerCertificate()).WillByDefault(ReturnRef(serial_number)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_SERIAL", EMPTY_STRING); } @@ -293,9 +299,9 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSerialNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuer) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, issuerPeerCertificate()) - .WillByDefault( - Return("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); + const std::string issuer_peer = + "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; + ON_CALL(*connection_info, issuerPeerCertificate()).WillByDefault(ReturnRef(issuer_peer)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_ISSUER", "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"); @@ -304,7 +310,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuer) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuerEmpty) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, issuerPeerCertificate()).WillByDefault(Return("")); + const std::string issuer_peer; + ON_CALL(*connection_info, issuerPeerCertificate()).WillByDefault(ReturnRef(issuer_peer)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_ISSUER", EMPTY_STRING); } @@ -318,9 +325,9 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerIssuerNoTls) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubject) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, subjectPeerCertificate()) - .WillByDefault( - Return("CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); + const std::string subject_peer = + "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(subject_peer)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_SUBJECT", "CN=Test CA,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"); @@ -329,7 +336,8 @@ TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubject) { TEST_F(StreamInfoHeaderFormatterTest, TestFormatWithDownstreamPeerSubjectEmpty) { NiceMock stream_info; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(Return("")); + const std::string subject_peer; + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(subject_peer)); EXPECT_CALL(stream_info, downstreamSslConnection()).WillRepeatedly(Return(connection_info)); testFormatting(stream_info, "DOWNSTREAM_PEER_SUBJECT", EMPTY_STRING); } diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 2e8cfe69871f..3292f4ce9e87 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -3419,9 +3419,9 @@ TEST_F(RouterTest, UpstreamSSLConnection) { NiceMock encoder; Http::StreamDecoder* response_decoder = nullptr; - const auto session_id = "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; + std::string session_id = "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, sessionId()).WillByDefault(Return(session_id)); + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(session_id)); upstream_stream_info_.setDownstreamSslConnection(connection_info); expectResponseTimerCreate(); diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index a7a5b415d867..07c505c2e901 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -967,7 +967,7 @@ TEST_F(TcpProxyTest, AccessLogTlsSessionId) { const std::string tlsSessionId{ "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"}; auto mockConnectionInfo = std::make_shared(); - EXPECT_CALL(*mockConnectionInfo, sessionId()).WillOnce(Return(tlsSessionId)); + EXPECT_CALL(*mockConnectionInfo, sessionId()).WillOnce(ReturnRef(tlsSessionId)); EXPECT_CALL(filter_callbacks_.connection_, ssl()).WillRepeatedly(Return(mockConnectionInfo)); setup(1, accessLogConfig("%DOWNSTREAM_TLS_SESSION_ID%")); @@ -1016,7 +1016,7 @@ TEST_F(TcpProxyTest, AccessLogUpstreamSSLConnection) { NiceMock stream_info; const std::string session_id = "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; auto ssl_info = std::make_shared(); - EXPECT_CALL(*ssl_info, sessionId()).WillRepeatedly(Return(session_id)); + EXPECT_CALL(*ssl_info, sessionId()).WillRepeatedly(ReturnRef(session_id)); stream_info.setDownstreamSslConnection(ssl_info); EXPECT_CALL(*upstream_connections_.at(0), streamInfo()).WillRepeatedly(ReturnRef(stream_info)); diff --git a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc index 1c5575186ab3..cee4378e4a21 100644 --- a/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc +++ b/test/extensions/access_loggers/grpc/http_grpc_access_log_impl_test.cc @@ -19,6 +19,7 @@ using testing::InSequence; using testing::Invoke; using testing::NiceMock; using testing::Return; +using testing::ReturnRef; namespace Envoy { namespace Extensions { @@ -310,11 +311,15 @@ response: {} ON_CALL(*connection_info, uriSanPeerCertificate()).WillByDefault(Return(peerSans)); const std::vector localSans{"localSan1", "localSan2"}; ON_CALL(*connection_info, uriSanLocalCertificate()).WillByDefault(Return(localSans)); - ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(Return("peerSubject")); - ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(Return("localSubject")); - ON_CALL(*connection_info, sessionId()) - .WillByDefault(Return("D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B")); - ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.3")); + const std::string peerSubject = "peerSubject"; + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(peerSubject)); + const std::string localSubject = "localSubject"; + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(localSubject)); + const std::string sessionId = + "D62A523A65695219D46FE1FFE285A4C371425ACE421B110B5B8D11D3EB4D5F0B"; + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(sessionId)); + const std::string tlsVersion = "TLSv1.3"; + ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(tlsVersion)); ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2CC0)); stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; @@ -365,7 +370,12 @@ response: {} stream_info.start_time_ = SystemTime(1h); auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.2")); + const std::string empty; + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(empty)); + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(empty)); + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(empty)); + const std::string tlsVersion = "TLSv1.2"; + ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(tlsVersion)); ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; @@ -406,7 +416,12 @@ response: {} stream_info.start_time_ = SystemTime(1h); auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.1")); + const std::string empty; + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(empty)); + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(empty)); + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(empty)); + const std::string tlsVersion = "TLSv1.1"; + ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(tlsVersion)); ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; @@ -447,7 +462,12 @@ response: {} stream_info.start_time_ = SystemTime(1h); auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1")); + const std::string empty; + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(empty)); + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(empty)); + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(empty)); + const std::string tlsVersion = "TLSv1"; + ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(tlsVersion)); ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; @@ -488,7 +508,12 @@ response: {} stream_info.start_time_ = SystemTime(1h); auto connection_info = std::make_shared>(); - ON_CALL(*connection_info, tlsVersion()).WillByDefault(Return("TLSv1.4")); + const std::string empty; + ON_CALL(*connection_info, subjectPeerCertificate()).WillByDefault(ReturnRef(empty)); + ON_CALL(*connection_info, subjectLocalCertificate()).WillByDefault(ReturnRef(empty)); + ON_CALL(*connection_info, sessionId()).WillByDefault(ReturnRef(empty)); + const std::string tlsVersion = "TLSv1.4"; + ON_CALL(*connection_info, tlsVersion()).WillByDefault(ReturnRef(tlsVersion)); ON_CALL(*connection_info, ciphersuiteId()).WillByDefault(Return(0x2F)); stream_info.setDownstreamSslConnection(connection_info); stream_info.requested_server_name_ = "sni"; diff --git a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc index d04b54ad8ebe..de4ef025784e 100644 --- a/test/extensions/filters/common/ext_authz/check_request_utils_test.cc +++ b/test/extensions/filters/common/ext_authz/check_request_utils_test.cc @@ -97,6 +97,9 @@ TEST_F(CheckRequestUtilsTest, BasicTcp) { EXPECT_CALL(connection_, remoteAddress()).WillOnce(ReturnRef(addr_)); EXPECT_CALL(connection_, localAddress()).WillOnce(ReturnRef(addr_)); EXPECT_CALL(Const(connection_), ssl()).Times(2).WillRepeatedly(Return(ssl_)); + EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{"source"})); + EXPECT_CALL(*ssl_, uriSanLocalCertificate()) + .WillOnce(Return(std::vector{"destination"})); CheckRequestUtils::createTcpCheck(&net_callbacks_, request); } @@ -112,6 +115,9 @@ TEST_F(CheckRequestUtilsTest, BasicHttp) { // A client supplied EnvoyAuthPartialBody header should be ignored. Http::TestHeaderMapImpl request_headers{{Http::Headers::get().EnvoyAuthPartialBody.get(), "1"}}; + EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{"source"})); + EXPECT_CALL(*ssl_, uriSanLocalCertificate()) + .WillOnce(Return(std::vector{"destination"})); expectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, request_headers, Protobuf::Map(), @@ -129,6 +135,9 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithPartialBody) { Http::HeaderMapImpl headers_; envoy::service::auth::v2::CheckRequest request_; + EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{"source"})); + EXPECT_CALL(*ssl_, uriSanLocalCertificate()) + .WillOnce(Return(std::vector{"destination"})); expectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, headers_, Protobuf::Map(), @@ -144,6 +153,9 @@ TEST_F(CheckRequestUtilsTest, BasicHttpWithFullBody) { Http::HeaderMapImpl headers_; envoy::service::auth::v2::CheckRequest request_; + EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{"source"})); + EXPECT_CALL(*ssl_, uriSanLocalCertificate()) + .WillOnce(Return(std::vector{"destination"})); expectBasicHttp(); CheckRequestUtils::createHttpCheck(&callbacks_, headers_, Protobuf::Map(), @@ -213,11 +225,13 @@ TEST_F(CheckRequestUtilsTest, CheckAttrContextSubject) { EXPECT_CALL(*ssl_, uriSanPeerCertificate()).WillOnce(Return(std::vector{})); EXPECT_CALL(*ssl_, dnsSansPeerCertificate()).WillOnce(Return(std::vector{})); - EXPECT_CALL(*ssl_, subjectPeerCertificate()).WillOnce(Return("source")); + std::string subject_peer = "source"; + EXPECT_CALL(*ssl_, subjectPeerCertificate()).WillOnce(ReturnRef(subject_peer)); EXPECT_CALL(*ssl_, uriSanLocalCertificate()).WillOnce(Return(std::vector{})); EXPECT_CALL(*ssl_, dnsSansLocalCertificate()).WillOnce(Return(std::vector{})); - EXPECT_CALL(*ssl_, subjectLocalCertificate()).WillOnce(Return("destination")); + std::string subject_local = "destination"; + EXPECT_CALL(*ssl_, subjectLocalCertificate()).WillOnce(ReturnRef(subject_local)); callHttpCheckAndValidateRequestAttributes(); } diff --git a/test/extensions/filters/common/rbac/matchers_test.cc b/test/extensions/filters/common/rbac/matchers_test.cc index 7139ad1d1847..699b53d8b4e7 100644 --- a/test/extensions/filters/common/rbac/matchers_test.cc +++ b/test/extensions/filters/common/rbac/matchers_test.cc @@ -235,7 +235,8 @@ TEST(AuthenticatedMatcher, subjectPeerCertificate) { const std::vector sans; EXPECT_CALL(*ssl, uriSanPeerCertificate()).WillRepeatedly(Return(sans)); EXPECT_CALL(*ssl, dnsSansPeerCertificate()).WillRepeatedly(Return(sans)); - EXPECT_CALL(*ssl, subjectPeerCertificate()).WillRepeatedly(Return("bar")); + std::string peer_subject = "bar"; + EXPECT_CALL(*ssl, subjectPeerCertificate()).WillRepeatedly(ReturnRef(peer_subject)); EXPECT_CALL(Const(conn), ssl()).WillRepeatedly(Return(ssl)); envoy::config::rbac::v2::Principal_Authenticated auth; diff --git a/test/fuzz/utility.h b/test/fuzz/utility.h index 2fda4efb8de7..adc3912c0e82 100644 --- a/test/fuzz/utility.h +++ b/test/fuzz/utility.h @@ -103,6 +103,9 @@ inline test::fuzz::Headers toHeaders(const Http::HeaderMap& headers) { return fuzz_headers; } +const std::string TestSubjectPeer = + "CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US"; + inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info) { // Set mocks' default string return value to be an empty string. testing::DefaultValue::Set(EMPTY_STRING); @@ -130,8 +133,7 @@ inline TestStreamInfo fromStreamInfo(const test::fuzz::StreamInfo& stream_info) test_stream_info.downstream_remote_address_ = address; auto connection_info = std::make_shared>(); ON_CALL(*connection_info, subjectPeerCertificate()) - .WillByDefault(testing::Return( - "CN=Test Server,OU=Lyft Engineering,O=Lyft,L=San Francisco,ST=California,C=US")); + .WillByDefault(testing::ReturnRef(TestSubjectPeer)); test_stream_info.setDownstreamSslConnection(connection_info); return test_stream_info; } diff --git a/test/mocks/ssl/mocks.h b/test/mocks/ssl/mocks.h index 957cbf05c87c..041888aa99c9 100644 --- a/test/mocks/ssl/mocks.h +++ b/test/mocks/ssl/mocks.h @@ -40,21 +40,21 @@ class MockConnectionInfo : public ConnectionInfo { MOCK_CONST_METHOD0(peerCertificatePresented, bool()); MOCK_CONST_METHOD0(uriSanLocalCertificate, std::vector()); MOCK_CONST_METHOD0(sha256PeerCertificateDigest, const std::string&()); - MOCK_CONST_METHOD0(serialNumberPeerCertificate, std::string()); - MOCK_CONST_METHOD0(issuerPeerCertificate, std::string()); - MOCK_CONST_METHOD0(subjectPeerCertificate, std::string()); + MOCK_CONST_METHOD0(serialNumberPeerCertificate, const std::string&()); + MOCK_CONST_METHOD0(issuerPeerCertificate, const std::string&()); + MOCK_CONST_METHOD0(subjectPeerCertificate, const std::string&()); MOCK_CONST_METHOD0(uriSanPeerCertificate, std::vector()); - MOCK_CONST_METHOD0(subjectLocalCertificate, std::string()); + MOCK_CONST_METHOD0(subjectLocalCertificate, const std::string&()); MOCK_CONST_METHOD0(urlEncodedPemEncodedPeerCertificate, const std::string&()); MOCK_CONST_METHOD0(urlEncodedPemEncodedPeerCertificateChain, const std::string&()); MOCK_CONST_METHOD0(dnsSansPeerCertificate, std::vector()); MOCK_CONST_METHOD0(dnsSansLocalCertificate, std::vector()); MOCK_CONST_METHOD0(validFromPeerCertificate, absl::optional()); MOCK_CONST_METHOD0(expirationPeerCertificate, absl::optional()); - MOCK_CONST_METHOD0(sessionId, std::string()); + MOCK_CONST_METHOD0(sessionId, const std::string&()); MOCK_CONST_METHOD0(ciphersuiteId, uint16_t()); MOCK_CONST_METHOD0(ciphersuiteString, std::string()); - MOCK_CONST_METHOD0(tlsVersion, std::string()); + MOCK_CONST_METHOD0(tlsVersion, const std::string&()); }; class MockClientContext : public ClientContext { From d5515aed33244952786ea4b7732af9a49b26b6f6 Mon Sep 17 00:00:00 2001 From: troshko111 <931973+troshko111@users.noreply.github.com> Date: Tue, 10 Sep 2019 13:28:43 -0700 Subject: [PATCH 524/542] test: add curl features check (#8194) Add a test ensuring curl was built with the expected features. Description: Add a test ensuring curl was built with the expected features. Risk Level: Low. Testing: n/a. Docs Changes: n/a. Release Notes: n/a. Signed-off-by: Taras Roshko --- test/dependencies/BUILD | 17 +++++++++++++++++ test/dependencies/curl_test.cc | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 test/dependencies/BUILD create mode 100644 test/dependencies/curl_test.cc diff --git a/test/dependencies/BUILD b/test/dependencies/BUILD new file mode 100644 index 000000000000..2e6ae296b760 --- /dev/null +++ b/test/dependencies/BUILD @@ -0,0 +1,17 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +envoy_package() + +envoy_cc_test( + name = "curl_test", + srcs = ["curl_test.cc"], + external_deps = [ + "curl", + ], +) diff --git a/test/dependencies/curl_test.cc b/test/dependencies/curl_test.cc new file mode 100644 index 000000000000..82fdeceeec29 --- /dev/null +++ b/test/dependencies/curl_test.cc @@ -0,0 +1,32 @@ +#include "curl/curl.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Dependencies { + +TEST(CurlTest, BuiltWithExpectedFeatures) { + // Ensure built with the expected features, flags from + // https://curl.haxx.se/libcurl/c/curl_version_info.html. + curl_version_info_data* info = curl_version_info(CURLVERSION_NOW); + + EXPECT_NE(0, info->features & CURL_VERSION_ASYNCHDNS); + EXPECT_NE(0, info->ares_num); + EXPECT_NE(0, info->features & CURL_VERSION_HTTP2); + EXPECT_NE(0, info->features & CURL_VERSION_LIBZ); + EXPECT_NE(0, info->features & CURL_VERSION_IPV6); + EXPECT_NE(0, info->features & CURL_VERSION_UNIX_SOCKETS); + + EXPECT_EQ(0, info->features & CURL_VERSION_BROTLI); + EXPECT_EQ(0, info->features & CURL_VERSION_GSSAPI); + EXPECT_EQ(0, info->features & CURL_VERSION_GSSNEGOTIATE); + EXPECT_EQ(0, info->features & CURL_VERSION_KERBEROS4); + EXPECT_EQ(0, info->features & CURL_VERSION_KERBEROS5); + EXPECT_EQ(0, info->features & CURL_VERSION_NTLM); + EXPECT_EQ(0, info->features & CURL_VERSION_NTLM_WB); + EXPECT_EQ(0, info->features & CURL_VERSION_SPNEGO); + EXPECT_EQ(0, info->features & CURL_VERSION_SSL); + EXPECT_EQ(0, info->features & CURL_VERSION_SSPI); +} + +} // namespace Dependencies +} // namespace Envoy From cee989a6a5b513c4df8474efcfca5933c935787e Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Tue, 10 Sep 2019 14:15:25 -0700 Subject: [PATCH 525/542] subset lb: allow ring hash/maglev LB to work with subsets (#8030) * subset lb: allow ring hash/maglev LB to work with subsets Skip initializing the thread aware LB for a cluster when the subset load balancer is enabled. Also adds some extra checks for LB policies that are incompatible with the subset load balancer. Risk Level: low Testing: test additional checks Docs Changes: updated docs w.r.t subset lb compatibility Release Notes: n/a Fixes: #7651 Signed-off-by: Stephan Zuercher --- .../upstream/load_balancing/subsets.rst | 20 +- .../common/upstream/cluster_manager_impl.cc | 23 +- .../common/upstream/original_dst_cluster.cc | 4 - source/common/upstream/upstream_impl.cc | 12 + test/common/upstream/BUILD | 6 + .../upstream/cluster_manager_impl_test.cc | 135 ++++++++-- test/integration/BUILD | 11 + .../http_subset_lb_integration_test.cc | 234 ++++++++++++++++++ 8 files changed, 400 insertions(+), 45 deletions(-) create mode 100644 test/integration/http_subset_lb_integration_test.cc diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst b/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst index 710a72f43b40..942903ceb691 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/subsets.rst @@ -35,12 +35,20 @@ therefore, contain a definition that has the same keys as a given route in order balancing to occur. This feature can only be enabled using the V2 configuration API. Furthermore, host metadata is only -supported when using the EDS discovery type for clusters. Host metadata for subset load balancing -must be placed under the filter name ``"envoy.lb"``. Similarly, route metadata match criteria use -the ``"envoy.lb"`` filter name. Host metadata may be hierarchical (e.g., the value for a top-level -key may be a structured value or list), but the subset load balancer only compares top-level keys -and values. Therefore when using structured values, a route's match criteria will only match if an -identical structured value appears in the host's metadata. +supported when hosts are defined using +:ref:`ClusterLoadAssignments `. ClusterLoadAssignments are +available via EDS or the Cluster :ref:`load_assignment ` +field. Host metadata for subset load balancing must be placed under the filter name ``"envoy.lb"``. +Similarly, route metadata match criteria use ``"envoy.lb"`` filter name. Host metadata may be +hierarchical (e.g., the value for a top-level key may be a structured value or list), but the +subset load balancer only compares top-level keys and values. Therefore when using structured +values, a route's match criteria will only match if an identical structured value appears in the +host's metadata. + +Finally, note that subset load balancing is not available for the +:ref:`ORIGINAL_DST_LB ` or +:ref:`CLUSTER_PROVIDED ` load balancer +policies. Examples ^^^^^^^^ diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 8a1b32cb90d8..e5c06231c4ca 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -667,17 +667,22 @@ void ClusterManagerImpl::loadCluster(const envoy::api::v2::Cluster& cluster, const auto cluster_entry_it = cluster_map.find(cluster_reference.info()->name()); // If an LB is thread aware, create it here. The LB is not initialized until cluster pre-init - // finishes. + // finishes. For RingHash/Maglev don't create the LB here if subset balancing is enabled, + // because the thread_aware_lb_ field takes precedence over the subset lb). if (cluster_reference.info()->lbType() == LoadBalancerType::RingHash) { - cluster_entry_it->second->thread_aware_lb_ = std::make_unique( - cluster_reference.prioritySet(), cluster_reference.info()->stats(), - cluster_reference.info()->statsScope(), runtime_, random_, - cluster_reference.info()->lbRingHashConfig(), cluster_reference.info()->lbConfig()); + if (!cluster_reference.info()->lbSubsetInfo().isEnabled()) { + cluster_entry_it->second->thread_aware_lb_ = std::make_unique( + cluster_reference.prioritySet(), cluster_reference.info()->stats(), + cluster_reference.info()->statsScope(), runtime_, random_, + cluster_reference.info()->lbRingHashConfig(), cluster_reference.info()->lbConfig()); + } } else if (cluster_reference.info()->lbType() == LoadBalancerType::Maglev) { - cluster_entry_it->second->thread_aware_lb_ = std::make_unique( - cluster_reference.prioritySet(), cluster_reference.info()->stats(), - cluster_reference.info()->statsScope(), runtime_, random_, - cluster_reference.info()->lbConfig()); + if (!cluster_reference.info()->lbSubsetInfo().isEnabled()) { + cluster_entry_it->second->thread_aware_lb_ = std::make_unique( + cluster_reference.prioritySet(), cluster_reference.info()->stats(), + cluster_reference.info()->statsScope(), runtime_, random_, + cluster_reference.info()->lbConfig()); + } } else if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided) { cluster_entry_it->second->thread_aware_lb_ = std::move(new_cluster_pair.second); } diff --git a/source/common/upstream/original_dst_cluster.cc b/source/common/upstream/original_dst_cluster.cc index ec24c36e0e47..04b91f58f236 100644 --- a/source/common/upstream/original_dst_cluster.cc +++ b/source/common/upstream/original_dst_cluster.cc @@ -184,10 +184,6 @@ OriginalDstClusterFactory::createClusterImpl( envoy::api::v2::Cluster_LbPolicy_Name(cluster.lb_policy()), envoy::api::v2::Cluster_DiscoveryType_Name(cluster.type()))); } - if (cluster.has_lb_subset_config() && cluster.lb_subset_config().subset_selectors_size() != 0) { - throw EnvoyException( - fmt::format("cluster: cluster type 'original_dst' may not be used with lb_subset_config")); - } // TODO(mattklein123): The original DST load balancer type should be deprecated and instead // the cluster should directly supply the load balancer. This will remove diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 18aac17131d9..7c4dfa231b65 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -643,12 +643,24 @@ ClusterInfoImpl::ClusterInfoImpl( envoy::api::v2::Cluster_LbPolicy_Name(config.lb_policy()), envoy::api::v2::Cluster_DiscoveryType_Name(config.type()))); } + if (config.has_lb_subset_config()) { + throw EnvoyException( + fmt::format("cluster: LB policy {} cannot be combined with lb_subset_config", + envoy::api::v2::Cluster_LbPolicy_Name(config.lb_policy()))); + } + lb_type_ = LoadBalancerType::ClusterProvided; break; case envoy::api::v2::Cluster::MAGLEV: lb_type_ = LoadBalancerType::Maglev; break; case envoy::api::v2::Cluster::CLUSTER_PROVIDED: + if (config.has_lb_subset_config()) { + throw EnvoyException( + fmt::format("cluster: LB policy {} cannot be combined with lb_subset_config", + envoy::api::v2::Cluster_LbPolicy_Name(config.lb_policy()))); + } + lb_type_ = LoadBalancerType::ClusterProvided; break; default: diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 4e5d8b4679ba..1ac3067c7ef2 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -29,6 +29,9 @@ envoy_cc_test( envoy_cc_test( name = "cluster_manager_impl_test", srcs = ["cluster_manager_impl_test.cc"], + external_deps = [ + "abseil_optional", + ], deps = [ ":utility_lib", "//include/envoy/stats:stats_interface", @@ -43,8 +46,10 @@ envoy_cc_test( "//source/common/stats:stats_lib", "//source/common/upstream:cluster_factory_lib", "//source/common/upstream:cluster_manager_lib", + "//source/common/upstream:subset_lb_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:context_lib", + "//test/integration/clusters:custom_static_cluster", "//test/mocks/access_log:access_log_mocks", "//test/mocks/api:api_mocks", "//test/mocks/http:http_mocks", @@ -61,6 +66,7 @@ envoy_cc_test( "//test/test_common:simulated_time_system_lib", "//test/test_common:threadsafe_singleton_injector_lib", "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:cds_cc", ], ) diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index a5ef0c27c2fc..1534ba3f3b2d 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -2,6 +2,7 @@ #include #include "envoy/admin/v2alpha/config_dump.pb.h" +#include "envoy/api/v2/cds.pb.h" #include "envoy/api/v2/core/base.pb.h" #include "envoy/network/listen_socket.h" #include "envoy/upstream/upstream.h" @@ -17,10 +18,12 @@ #include "common/singleton/manager_impl.h" #include "common/upstream/cluster_factory_impl.h" #include "common/upstream/cluster_manager_impl.h" +#include "common/upstream/subset_lb.h" #include "extensions/transport_sockets/tls/context_manager_impl.h" #include "test/common/upstream/utility.h" +#include "test/integration/clusters/custom_static_cluster.h" #include "test/mocks/access_log/mocks.h" #include "test/mocks/api/mocks.h" #include "test/mocks/http/mocks.h" @@ -38,6 +41,7 @@ #include "test/test_common/threadsafe_singleton_injector.h" #include "test/test_common/utility.h" +#include "absl/strings/str_replace.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -575,14 +579,48 @@ TEST_F(ClusterManagerImplTest, OriginalDstLbRestriction2) { "'ORIGINAL_DST_LB' is allowed only with cluster type 'ORIGINAL_DST'"); } -TEST_F(ClusterManagerImplTest, SubsetLoadBalancerInitialization) { - const std::string yaml = R"EOF( +class ClusterManagerSubsetInitializationTest + : public ClusterManagerImplTest, + public testing::WithParamInterface { +public: + ClusterManagerSubsetInitializationTest() = default; + + static std::vector lbPolicies() { + int first = static_cast(envoy::api::v2::Cluster_LbPolicy_LbPolicy_MIN); + int last = static_cast(envoy::api::v2::Cluster_LbPolicy_LbPolicy_MAX); + ASSERT(first < last); + + std::vector policies; + for (int i = first; i <= last; i++) { + if (envoy::api::v2::Cluster_LbPolicy_IsValid(i)) { + auto policy = static_cast(i); + if (policy != envoy::api::v2::Cluster_LbPolicy_LOAD_BALANCING_POLICY_CONFIG) { + policies.push_back(policy); + } + } + } + return policies; + } + + static std::string paramName(const testing::TestParamInfo& info) { + const std::string& name = envoy::api::v2::Cluster_LbPolicy_Name(info.param); + return absl::StrReplaceAll(name, {{"_", ""}}); + } +}; + +// Test initialization of subset load balancer with every possible load balancer policy. +TEST_P(ClusterManagerSubsetInitializationTest, SubsetLoadBalancerInitialization) { + const std::string yamlPattern = R"EOF( static_resources: clusters: - name: cluster_1 connect_timeout: 0.250s - type: static - lb_policy: round_robin + {} + lb_policy: "{}" + lb_subset_config: + fallback_policy: ANY_ENDPOINT + subset_selectors: + - keys: [ "x" ] load_assignment: endpoints: - lb_endpoints: @@ -598,19 +636,47 @@ TEST_F(ClusterManagerImplTest, SubsetLoadBalancerInitialization) { port_value: 8001 )EOF"; - envoy::config::bootstrap::v2::Bootstrap bootstrap = parseBootstrapFromV2Yaml(yaml); - envoy::api::v2::Cluster::LbSubsetConfig* subset_config = - bootstrap.mutable_static_resources()->mutable_clusters(0)->mutable_lb_subset_config(); - subset_config->set_fallback_policy(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT); - subset_config->add_subset_selectors()->add_keys("x"); + const std::string& policy_name = envoy::api::v2::Cluster_LbPolicy_Name(GetParam()); - create(bootstrap); - checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 1 /*active*/, 0 /*warming*/); + std::string cluster_type = "type: STATIC"; + if (GetParam() == envoy::api::v2::Cluster_LbPolicy_ORIGINAL_DST_LB) { + cluster_type = "type: ORIGINAL_DST"; + } else if (GetParam() == envoy::api::v2::Cluster_LbPolicy_CLUSTER_PROVIDED) { + // This custom cluster type is registered by linking test/integration/custom/static_cluster.cc. + cluster_type = "cluster_type: { name: envoy.clusters.custom_static_with_lb }"; + } - factory_.tls_.shutdownThread(); + const std::string yaml = fmt::format(yamlPattern, cluster_type, policy_name); + + if (GetParam() == envoy::api::v2::Cluster_LbPolicy_ORIGINAL_DST_LB || + GetParam() == envoy::api::v2::Cluster_LbPolicy_CLUSTER_PROVIDED) { + EXPECT_THROW_WITH_MESSAGE( + create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, + fmt::format("cluster: LB policy {} cannot be combined with lb_subset_config", + envoy::api::v2::Cluster_LbPolicy_Name(GetParam()))); + + } else { + create(parseBootstrapFromV2Yaml(yaml)); + checkStats(1 /*added*/, 0 /*modified*/, 0 /*removed*/, 1 /*active*/, 0 /*warming*/); + + Upstream::ThreadLocalCluster* tlc = cluster_manager_->get("cluster_1"); + EXPECT_NE(nullptr, tlc); + + if (tlc) { + Upstream::LoadBalancer& lb = tlc->loadBalancer(); + EXPECT_NE(nullptr, dynamic_cast(&lb)); + } + + factory_.tls_.shutdownThread(); + } } -TEST_F(ClusterManagerImplTest, SubsetLoadBalancerRestriction) { +INSTANTIATE_TEST_SUITE_P(ClusterManagerSubsetInitializationTest, + ClusterManagerSubsetInitializationTest, + testing::ValuesIn(ClusterManagerSubsetInitializationTest::lbPolicies()), + ClusterManagerSubsetInitializationTest::paramName); + +TEST_F(ClusterManagerImplTest, SubsetLoadBalancerOriginalDstRestriction) { const std::string yaml = R"EOF( static_resources: clusters: @@ -618,17 +684,34 @@ TEST_F(ClusterManagerImplTest, SubsetLoadBalancerRestriction) { connect_timeout: 0.250s type: original_dst lb_policy: original_dst_lb + lb_subset_config: + fallback_policy: ANY_ENDPOINT + subset_selectors: + - keys: [ "x" ] )EOF"; - envoy::config::bootstrap::v2::Bootstrap bootstrap = parseBootstrapFromV2Yaml(yaml); - envoy::api::v2::Cluster::LbSubsetConfig* subset_config = - bootstrap.mutable_static_resources()->mutable_clusters(0)->mutable_lb_subset_config(); - subset_config->set_fallback_policy(envoy::api::v2::Cluster::LbSubsetConfig::ANY_ENDPOINT); - subset_config->add_subset_selectors()->add_keys("x"); + EXPECT_THROW_WITH_MESSAGE( + create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, + "cluster: LB policy ORIGINAL_DST_LB cannot be combined with lb_subset_config"); +} + +TEST_F(ClusterManagerImplTest, SubsetLoadBalancerClusterProvidedLbRestriction) { + const std::string yaml = R"EOF( +static_resources: + clusters: + - name: cluster_1 + connect_timeout: 0.250s + type: static + lb_policy: cluster_provided + lb_subset_config: + fallback_policy: ANY_ENDPOINT + subset_selectors: + - keys: [ "x" ] + )EOF"; EXPECT_THROW_WITH_MESSAGE( - create(bootstrap), EnvoyException, - "cluster: cluster type 'original_dst' may not be used with lb_subset_config"); + create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, + "cluster: LB policy CLUSTER_PROVIDED cannot be combined with lb_subset_config"); } TEST_F(ClusterManagerImplTest, SubsetLoadBalancerLocalityAware) { @@ -639,6 +722,11 @@ TEST_F(ClusterManagerImplTest, SubsetLoadBalancerLocalityAware) { connect_timeout: 0.250s type: STATIC lb_policy: ROUND_ROBIN + lb_subset_config: + fallback_policy: ANY_ENDPOINT + subset_selectors: + - keys: [ "x" ] + locality_weight_aware: true load_assignment: endpoints: - lb_endpoints: @@ -654,12 +742,7 @@ TEST_F(ClusterManagerImplTest, SubsetLoadBalancerLocalityAware) { port_value: 8001 )EOF"; - envoy::config::bootstrap::v2::Bootstrap bootstrap = parseBootstrapFromV2Yaml(yaml); - envoy::api::v2::Cluster::LbSubsetConfig* subset_config = - bootstrap.mutable_static_resources()->mutable_clusters(0)->mutable_lb_subset_config(); - subset_config->set_locality_weight_aware(true); - - EXPECT_THROW_WITH_MESSAGE(create(bootstrap), EnvoyException, + EXPECT_THROW_WITH_MESSAGE(create(parseBootstrapFromV2Yaml(yaml)), EnvoyException, "Locality weight aware subset LB requires that a " "locality_weighted_lb_config be set in cluster_1"); } diff --git a/test/integration/BUILD b/test/integration/BUILD index 70082725f8d9..fd69958d4b4f 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -252,6 +252,17 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "http_subset_lb_integration_test", + srcs = [ + "http_subset_lb_integration_test.cc", + ], + deps = [ + ":http_integration_lib", + "//test/common/upstream:utility_lib", + ], +) + envoy_cc_test( name = "http_timeout_integration_test", srcs = [ diff --git a/test/integration/http_subset_lb_integration_test.cc b/test/integration/http_subset_lb_integration_test.cc new file mode 100644 index 000000000000..427f3f5a3c64 --- /dev/null +++ b/test/integration/http_subset_lb_integration_test.cc @@ -0,0 +1,234 @@ +#include "test/integration/http_integration.h" + +#include "absl/strings/str_replace.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace { + +const std::string SUBSET_CONFIG = R"EOF( +admin: + access_log_path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 0 +static_resources: + clusters: + name: cluster_0 + lb_policy: round_robin + lb_subset_config: + subset_selectors: + - keys: [ "type" ] + load_assignment: + cluster_name: cluster_0 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + metadata: + filter_metadata: + "envoy.lb": { "type": "a" } + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + metadata: + filter_metadata: + "envoy.lb": { "type": "a" } + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + metadata: + filter_metadata: + "envoy.lb": { "type": "b" } + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + metadata: + filter_metadata: + "envoy.lb": { "type": "b" } + listeners: + name: listener_0 + address: + socket_address: + address: 127.0.0.1 + port_value: 0 + filter_chains: + filters: + name: envoy.http_connection_manager + config: + stat_prefix: config_test + http_filters: + name: envoy.router + codec_type: HTTP1 + access_log: + name: envoy.file_access_log + filter: + not_health_check_filter: {} + config: + path: /dev/null + route_config: + name: route_config_0 + virtual_hosts: + name: integration + domains: "*" + routes: + - match: + prefix: "/" + headers: + - name: "x-type" + exact_match: "a" + route: + cluster: cluster_0 + metadata_match: + filter_metadata: + "envoy.lb": { "type": "a" } + hash_policy: + - header: + header_name: "x-hash" + - match: + prefix: "/" + headers: + - name: "x-type" + exact_match: "b" + route: + cluster: cluster_0 + metadata_match: + filter_metadata: + "envoy.lb": { "type": "b" } + hash_policy: + - header: + header_name: "x-hash" + response_headers_to_add: + - header: + key: "x-host-type" + value: '%UPSTREAM_METADATA(["envoy.lb", "type"])%' + - header: + key: "x-host" + value: '%UPSTREAM_REMOTE_ADDRESS%' +)EOF"; + +} // namespace +class HttpSubsetLbIntegrationTest : public testing::TestWithParam, + public HttpIntegrationTest { +public: + // Returns all load balancer types except ORIGINAL_DST_LB and CLUSTER_PROVIDED. + static std::vector getSubsetLbTestParams() { + int first = static_cast(envoy::api::v2::Cluster_LbPolicy_LbPolicy_MIN); + int last = static_cast(envoy::api::v2::Cluster_LbPolicy_LbPolicy_MAX); + ASSERT(first < last); + + std::vector ret; + for (int i = first; i <= last; i++) { + if (!envoy::api::v2::Cluster_LbPolicy_IsValid(i)) { + continue; + } + + auto policy = static_cast(i); + + if (policy == envoy::api::v2::Cluster_LbPolicy_ORIGINAL_DST_LB || + policy == envoy::api::v2::Cluster_LbPolicy_CLUSTER_PROVIDED || + policy == envoy::api::v2::Cluster_LbPolicy_LOAD_BALANCING_POLICY_CONFIG) { + continue; + } + + ret.push_back(policy); + } + + return ret; + } + + // Converts an LbPolicy to strings suitable for test names. + static std::string + subsetLbTestParamsToString(const testing::TestParamInfo& p) { + const std::string& policy_name = envoy::api::v2::Cluster_LbPolicy_Name(p.param); + return absl::StrReplaceAll(policy_name, {{"_", ""}}); + } + + HttpSubsetLbIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, Network::Address::IpVersion::v4, + SUBSET_CONFIG) { + autonomous_upstream_ = true; + setUpstreamCount(4); + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + auto* static_resources = bootstrap.mutable_static_resources(); + for (int i = 0; i < static_resources->clusters_size(); ++i) { + auto* cluster = static_resources->mutable_clusters(i); + cluster->set_lb_policy(GetParam()); + } + }); + } + + void SetUp() override { + setDownstreamProtocol(Http::CodecClient::Type::HTTP1); + setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); + } + + // Runs a subset lb test with the given request headers, expecting the x-host-type header to + // the given type ("a" or "b"). If expect_unique_host, verifies that a single host is selected + // over n iterations (e.g. for maglev/hash-ring policies). Otherwise, expected more than one + // host to be selected over n iterations. + void runTest(Http::TestHeaderMapImpl& request_headers, const std::string expected_host_type, + const bool expect_unique_host, const int n = 10) { + std::set hosts; + for (int i = 0; i < n; i++) { + Http::TestHeaderMapImpl response_headers{{":status", "200"}}; + + // Send header only request. + IntegrationStreamDecoderPtr response = codec_client_->makeHeaderOnlyRequest(request_headers); + response->waitForEndStream(); + + // Expect a response from a host in the correct subset. + EXPECT_EQ(response->headers() + .get(Envoy::Http::LowerCaseString{"x-host-type"}) + ->value() + .getStringView(), + expected_host_type); + + hosts.emplace( + response->headers().get(Envoy::Http::LowerCaseString{"x-host"})->value().getStringView()); + } + + if (expect_unique_host) { + EXPECT_EQ(hosts.size(), 1) << "Expected a single unique host to be selected for " + << envoy::api::v2::Cluster_LbPolicy_Name(GetParam()); + } else { + EXPECT_GT(hosts.size(), 1) << "Expected multiple hosts to be selected" + << envoy::api::v2::Cluster_LbPolicy_Name(GetParam()); + } + } + + Http::TestHeaderMapImpl type_a_request_headers_{{":method", "GET"}, {":path", "/test"}, + {":scheme", "http"}, {":authority", "host"}, + {"x-type", "a"}, {"x-hash", "hash-a"}}; + Http::TestHeaderMapImpl type_b_request_headers_{{":method", "GET"}, {":path", "/test"}, + {":scheme", "http"}, {":authority", "host"}, + {"x-type", "b"}, {"x-hash", "hash-b"}}; +}; + +INSTANTIATE_TEST_SUITE_P(SubsetCompatibleLoadBalancers, HttpSubsetLbIntegrationTest, + testing::ValuesIn(HttpSubsetLbIntegrationTest::getSubsetLbTestParams()), + HttpSubsetLbIntegrationTest::subsetLbTestParamsToString); + +// Tests each subset-compatible load balancer policy with 4 hosts divided into 2 subsets. +TEST_P(HttpSubsetLbIntegrationTest, SubsetLoadBalancer) { + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + + const bool expect_unique_host = (GetParam() == envoy::api::v2::Cluster_LbPolicy_RING_HASH || + GetParam() == envoy::api::v2::Cluster_LbPolicy_MAGLEV); + + runTest(type_a_request_headers_, "a", expect_unique_host); + runTest(type_b_request_headers_, "b", expect_unique_host); +} + +} // namespace Envoy From ad10a97e16bcf9901f95b6ef5d079e12cc5c8b96 Mon Sep 17 00:00:00 2001 From: Nicolas Flacco <47160394+FAYiEKcbD0XFqF2QK2E4viAHg8rMm2VbjYKdjTg@users.noreply.github.com> Date: Tue, 10 Sep 2019 15:01:13 -0700 Subject: [PATCH 526/542] redis: add a request time metric to redis upstream (#7890) Signed-off-by: Nicolas Flacco --- .../network/redis_proxy/v2/redis_proxy.proto | 4 + .../arch_overview/other_protocols/redis.rst | 16 +++ docs/root/intro/version_history.rst | 1 + .../clusters/redis/redis_cluster.cc | 8 +- .../extensions/clusters/redis/redis_cluster.h | 2 + .../filters/network/common/redis/BUILD | 17 +++ .../filters/network/common/redis/client.h | 10 +- .../network/common/redis/client_impl.cc | 58 ++++++--- .../network/common/redis/client_impl.h | 23 +++- .../common/redis/redis_command_stats.cc | 110 ++++++++++++++++++ .../common/redis/redis_command_stats.h | 66 +++++++++++ .../filters/network/redis_proxy/BUILD | 1 + .../filters/network/redis_proxy/config.cc | 6 +- .../network/redis_proxy/conn_pool_impl.cc | 11 +- .../network/redis_proxy/conn_pool_impl.h | 4 +- .../extensions/health_checkers/redis/redis.cc | 10 +- .../extensions/health_checkers/redis/redis.h | 2 + .../clusters/redis/redis_cluster_test.cc | 4 +- .../network/common/redis/client_impl_test.cc | 15 ++- .../redis_proxy/conn_pool_impl_test.cc | 8 +- .../health_checkers/redis/redis_test.cc | 5 +- 21 files changed, 346 insertions(+), 35 deletions(-) create mode 100644 source/extensions/filters/network/common/redis/redis_command_stats.cc create mode 100644 source/extensions/filters/network/common/redis/redis_command_stats.h diff --git a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto index 35d1110df12c..78c56bb2efe6 100644 --- a/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto +++ b/api/envoy/config/filter/network/redis_proxy/v2/redis_proxy.proto @@ -88,6 +88,10 @@ message RedisProxy { // downstream unchanged. This limit defaults to 100. google.protobuf.UInt32Value max_upstream_unknown_connections = 6; + // Enable per-command statistics per upstream cluster, in addition to the filter level aggregate + // count. + bool enable_command_stats = 8; + // ReadPolicy controls how Envoy routes read commands to Redis nodes. This is currently // supported for Redis Cluster. All ReadPolicy settings except MASTER may return stale data // because replication is asynchronous and requires some delay. You need to ensure that your diff --git a/docs/root/intro/arch_overview/other_protocols/redis.rst b/docs/root/intro/arch_overview/other_protocols/redis.rst index 47e55418c679..e96cbe6a3e58 100644 --- a/docs/root/intro/arch_overview/other_protocols/redis.rst +++ b/docs/root/intro/arch_overview/other_protocols/redis.rst @@ -60,6 +60,8 @@ If passive healthchecking is desired, also configure For the purposes of passive healthchecking, connect timeouts, command timeouts, and connection close map to 5xx. All other responses from Redis are counted as a success. +.. _arch_overview_redis_cluster_support: + Redis Cluster Support (Experimental) ---------------------------------------- @@ -90,6 +92,20 @@ Every Redis cluster has its own extra statistics tree rooted at *cluster.. max_upstream_unknown_connections_reached, Counter, Total number of times that an upstream connection to an unknown host is not created after redirection having reached the connection pool's max_upstream_unknown_connections limit upstream_cx_drained, Counter, Total number of upstream connections drained of active requests before being closed + upstream_commands.upstream_rq_time, Histogram, Histogram of upstream request times for all types of requests + +.. _arch_overview_redis_cluster_command_stats: + +Per-cluster command statistics can be enabled via the setting :ref:`enable_command_stats `: + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + upstream_commands.[command].success, Counter, Total number of successful requests for a specific Redis command + upstream_commands.[command].error, Counter, Total number of failed or cancelled requests for a specific Redis command + upstream_commands.[command].total, Counter, Total number of requests for a specific Redis command (sum of success and error) + upstream_commands.[command].latency, Histogram, Latency of requests for a specific Redis command Supported commands ------------------ diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index d641e1d098db..679ea4f9e769 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -38,6 +38,7 @@ Version history * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). * performance: stats symbol table implementation (disabled by default; to test it, add "--use-fake-symbol-table 0" to the command-line arguments when starting Envoy). * rbac: added support for DNS SAN as :ref:`principal_name `. +* redis: added :ref:`enable_command_stats ` to enable :ref:`per command statistics ` for upstream clusters. * redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. * regex: introduce new :ref:`RegexMatcher ` type that provides a safe regex implementation for untrusted user input. This type is now used in all diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index dc41651b0ca1..cf6f1d9f5bad 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -168,7 +168,10 @@ RedisCluster::RedisDiscoverySession::RedisDiscoverySession( NetworkFilters::Common::Redis::Client::ClientFactory& client_factory) : parent_(parent), dispatcher_(parent.dispatcher_), resolve_timer_(parent.dispatcher_.createTimer([this]() -> void { startResolveRedis(); })), - client_factory_(client_factory), buffer_timeout_(0) {} + client_factory_(client_factory), buffer_timeout_(0), + redis_command_stats_( + NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( + parent_.info()->statsScope().symbolTable())) {} // Convert the cluster slot IP/Port response to and address, return null if the response does not // match the expected type. @@ -249,7 +252,8 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { if (!client) { client = std::make_unique(*this); client->host_ = current_host_address_; - client->client_ = client_factory_.create(host, dispatcher_, *this); + client->client_ = client_factory_.create(host, dispatcher_, *this, redis_command_stats_, + parent_.info()->statsScope()); client->client_->addConnectionCallbacks(*client); std::string auth_password = Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index f4038b7629f1..bec0ac965031 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -214,6 +214,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { uint32_t maxBufferSizeBeforeFlush() const override { return 0; } std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + bool enableCommandStats() const override { return false; } // This is effectively not in used for making the "Cluster Slots" calls. // since we call cluster slots on both the master and slaves, ANY is more appropriate here. Extensions::NetworkFilters::Common::Redis::Client::ReadPolicy readPolicy() const override { @@ -241,6 +242,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { Event::TimerPtr resolve_timer_; NetworkFilters::Common::Redis::Client::ClientFactory& client_factory_; const std::chrono::milliseconds buffer_timeout_; + NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; }; Upstream::ClusterManager& cluster_manager_; diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index 1757c2a7b9b4..336c9f2b303a 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -46,6 +46,7 @@ envoy_cc_library( hdrs = ["client.h"], deps = [ ":codec_lib", + ":redis_command_stats_lib", "//include/envoy/upstream:cluster_manager_interface", ], ) @@ -58,6 +59,7 @@ envoy_cc_library( ":client_interface", ":codec_lib", "//include/envoy/router:router_interface", + "//include/envoy/stats:timespan", "//include/envoy/thread_local:thread_local_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/buffer:buffer_lib", @@ -78,3 +80,18 @@ envoy_cc_library( ":codec_lib", ], ) + +envoy_cc_library( + name = "redis_command_stats_lib", + srcs = ["redis_command_stats.cc"], + hdrs = ["redis_command_stats.h"], + deps = [ + ":codec_interface", + ":supported_commands_lib", + "//include/envoy/stats:stats_interface", + "//include/envoy/stats:timespan", + "//source/common/common:to_lower_table_lib", + "//source/common/common:utility_lib", + "//source/common/stats:symbol_table_lib", + ], +) diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index e20df148fb82..3e5d80ce9208 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -5,6 +5,7 @@ #include "envoy/upstream/cluster_manager.h" #include "extensions/filters/network/common/redis/codec_impl.h" +#include "extensions/filters/network/common/redis/redis_command_stats.h" namespace Envoy { namespace Extensions { @@ -163,6 +164,11 @@ class Config { */ virtual uint32_t maxUpstreamUnknownConnections() const PURE; + /** + * @return when enabled, upstream cluster per-command statistics will be recorded. + */ + virtual bool enableCommandStats() const PURE; + /** * @return the read policy the proxy should use. */ @@ -184,7 +190,9 @@ class ClientFactory { * @return ClientPtr a new connection pool client. */ virtual ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config) PURE; + const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope) PURE; }; } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index 6f941d223ca0..acb7d64f771e 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -19,7 +19,8 @@ ConfigImpl::ConfigImpl( 3)), // Default timeout is 3ms. If max_buffer_size_before_flush is zero, this is not used // as the buffer is flushed on each request immediately. max_upstream_unknown_connections_( - PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)) { + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_upstream_unknown_connections, 100)), + enable_command_stats_(config.enable_command_stats()) { switch (config.read_policy()) { case envoy::config::filter::network::redis_proxy::v2:: RedisProxy_ConnPoolSettings_ReadPolicy_MASTER: @@ -48,10 +49,11 @@ ConfigImpl::ConfigImpl( ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config) { - - std::unique_ptr client( - new ClientImpl(host, dispatcher, std::move(encoder), decoder_factory, config)); + const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope) { + auto client = std::make_unique(host, dispatcher, std::move(encoder), decoder_factory, + config, redis_command_stats, scope); client->connection_ = host->createConnection(dispatcher, nullptr, nullptr).connection_; client->connection_->addConnectionCallbacks(*client); client->connection_->addReadFilter(Network::ReadFilterSharedPtr{new UpstreamReadFilter(*client)}); @@ -61,11 +63,14 @@ ClientPtr ClientImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatche } ClientImpl::ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config) + EncoderPtr&& encoder, DecoderFactory& decoder_factory, const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, Stats::Scope& scope) : host_(host), encoder_(std::move(encoder)), decoder_(decoder_factory.create(*this)), config_(config), - connect_or_op_timer_(dispatcher.createTimer([this]() -> void { onConnectOrOpTimeout(); })), - flush_timer_(dispatcher.createTimer([this]() -> void { flushBufferAndResetTimer(); })) { + connect_or_op_timer_(dispatcher.createTimer([this]() { onConnectOrOpTimeout(); })), + flush_timer_(dispatcher.createTimer([this]() { flushBufferAndResetTimer(); })), + time_source_(dispatcher.timeSource()), redis_command_stats_(redis_command_stats), + scope_(scope) { host->cluster().stats().upstream_cx_total_.inc(); host->stats().cx_total_.inc(); host->cluster().stats().upstream_cx_active_.inc(); @@ -94,7 +99,17 @@ PoolRequest* ClientImpl::makeRequest(const RespValue& request, PoolCallbacks& ca const bool empty_buffer = encoder_buffer_.length() == 0; - pending_requests_.emplace_back(*this, callbacks); + Stats::StatName command; + if (config_.enableCommandStats()) { + // Only lowercase command and get StatName if we enable command stats + command = redis_command_stats_->getCommandFromRequest(request); + redis_command_stats_->updateStatsTotal(scope_, command); + } else { + // If disabled, we use a placeholder stat name "unused" that is not used + command = redis_command_stats_->getUnusedStatName(); + } + + pending_requests_.emplace_back(*this, callbacks, command); encoder_->encode(request, encoder_buffer_); // If buffer is full, flush. If the buffer was empty before the request, start the timer. @@ -186,6 +201,14 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { ASSERT(!pending_requests_.empty()); PendingRequest& request = pending_requests_.front(); const bool canceled = request.canceled_; + + if (config_.enableCommandStats()) { + bool success = !canceled && (value->type() != Common::Redis::RespType::Error); + redis_command_stats_->updateStats(scope_, request.command_, success); + request.command_request_timer_->complete(); + } + request.aggregate_request_timer_->complete(); + PoolCallbacks& callbacks = request.callbacks_; // We need to ensure the request is popped before calling the callback, since the callback might @@ -225,8 +248,15 @@ void ClientImpl::onRespValue(RespValuePtr&& value) { putOutlierEvent(Upstream::Outlier::Result::EXT_ORIGIN_REQUEST_SUCCESS); } -ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks) - : parent_(parent), callbacks_(callbacks) { +ClientImpl::PendingRequest::PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, + Stats::StatName command) + : parent_(parent), callbacks_(callbacks), command_{command}, + aggregate_request_timer_(parent_.redis_command_stats_->createAggregateTimer( + parent_.scope_, parent_.time_source_)) { + if (parent_.config_.enableCommandStats()) { + command_request_timer_ = parent_.redis_command_stats_->createCommandTimer( + parent_.scope_, command_, parent_.time_source_); + } parent.host_->cluster().stats().upstream_rq_total_.inc(); parent.host_->stats().rq_total_.inc(); parent.host_->cluster().stats().upstream_rq_active_.inc(); @@ -248,9 +278,11 @@ void ClientImpl::PendingRequest::cancel() { ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, - Event::Dispatcher& dispatcher, const Config& config) { + Event::Dispatcher& dispatcher, const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope) { return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, - config); + config, redis_command_stats, scope); } } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index 9522482fb022..a8ab3806eb7e 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -3,6 +3,7 @@ #include #include "envoy/config/filter/network/redis_proxy/v2/redis_proxy.pb.h" +#include "envoy/stats/timespan.h" #include "envoy/thread_local/thread_local.h" #include "envoy/upstream/cluster_manager.h" @@ -49,6 +50,7 @@ class ConfigImpl : public Config { uint32_t maxUpstreamUnknownConnections() const override { return max_upstream_unknown_connections_; } + bool enableCommandStats() const override { return enable_command_stats_; } ReadPolicy readPolicy() const override { return read_policy_; } private: @@ -58,6 +60,7 @@ class ConfigImpl : public Config { const uint32_t max_buffer_size_before_flush_; const std::chrono::milliseconds buffer_flush_timeout_; const uint32_t max_upstream_unknown_connections_; + const bool enable_command_stats_; ReadPolicy read_policy_; }; @@ -65,8 +68,13 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne public: static ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, DecoderFactory& decoder_factory, - const Config& config); + const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope); + ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, + DecoderFactory& decoder_factory, const Config& config, + const RedisCommandStatsSharedPtr& redis_command_stats, Stats::Scope& scope); ~ClientImpl() override; // Client @@ -94,7 +102,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne }; struct PendingRequest : public PoolRequest { - PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks); + PendingRequest(ClientImpl& parent, PoolCallbacks& callbacks, Stats::StatName stat_name); ~PendingRequest() override; // PoolRequest @@ -102,11 +110,12 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne ClientImpl& parent_; PoolCallbacks& callbacks_; + Stats::StatName command_; bool canceled_{}; + Stats::CompletableTimespanPtr aggregate_request_timer_; + Stats::CompletableTimespanPtr command_request_timer_; }; - ClientImpl(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, EncoderPtr&& encoder, - DecoderFactory& decoder_factory, const Config& config); void onConnectOrOpTimeout(); void onData(Buffer::Instance& data); void putOutlierEvent(Upstream::Outlier::Result result); @@ -129,13 +138,17 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne Event::TimerPtr connect_or_op_timer_; bool connected_{}; Event::TimerPtr flush_timer_; + Envoy::TimeSource& time_source_; + const RedisCommandStatsSharedPtr redis_command_stats_; + Stats::Scope& scope_; }; class ClientFactoryImpl : public ClientFactory { public: // RedisProxy::ConnPool::ClientFactoryImpl ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, - const Config& config) override; + const Config& config, const RedisCommandStatsSharedPtr& redis_command_stats, + Stats::Scope& scope) override; static ClientFactoryImpl instance_; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc new file mode 100644 index 000000000000..ce11df704bac --- /dev/null +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -0,0 +1,110 @@ +#include "extensions/filters/network/common/redis/redis_command_stats.h" + +#include "extensions/filters/network/common/redis/supported_commands.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Common { +namespace Redis { + +RedisCommandStats::RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix) + : symbol_table_(symbol_table), stat_name_pool_(symbol_table_), + prefix_(stat_name_pool_.add(prefix)), + upstream_rq_time_(stat_name_pool_.add("upstream_rq_time")), + latency_(stat_name_pool_.add("latency")), total_(stat_name_pool_.add("total")), + success_(stat_name_pool_.add("success")), error_(stat_name_pool_.add("error")), + unused_metric_(stat_name_pool_.add("unused")), null_metric_(stat_name_pool_.add("null")), + unknown_metric_(stat_name_pool_.add("unknown")) { + // Note: Even if this is disabled, we track the upstream_rq_time. + // Create StatName for each Redis command. Note that we don't include Auth or Ping. + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()) { + addCommandToPool(command); + } + for (const std::string& command : + Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()) { + addCommandToPool(command); + } + for (const std::string& command : Extensions::NetworkFilters::Common::Redis::SupportedCommands:: + hashMultipleSumResultCommands()) { + addCommandToPool(command); + } + addCommandToPool(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); + addCommandToPool(Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); +} + +void RedisCommandStats::addCommandToPool(const std::string& command_string) { + Stats::StatName command = stat_name_pool_.add(command_string); + stat_name_map_[command_string] = command; +} + +Stats::Counter& RedisCommandStats::counter(Stats::Scope& scope, + const Stats::StatNameVec& stat_names) { + const Stats::SymbolTable::StoragePtr storage_ptr = symbol_table_.join(stat_names); + Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); + return scope.counterFromStatName(full_stat_name); +} + +Stats::Histogram& RedisCommandStats::histogram(Stats::Scope& scope, + const Stats::StatNameVec& stat_names) { + const Stats::SymbolTable::StoragePtr storage_ptr = symbol_table_.join(stat_names); + Stats::StatName full_stat_name = Stats::StatName(storage_ptr.get()); + return scope.histogramFromStatName(full_stat_name); +} + +Stats::CompletableTimespanPtr +RedisCommandStats::createCommandTimer(Stats::Scope& scope, Stats::StatName command, + Envoy::TimeSource& time_source) { + return std::make_unique>( + histogram(scope, {prefix_, command, latency_}), time_source); +} + +Stats::CompletableTimespanPtr +RedisCommandStats::createAggregateTimer(Stats::Scope& scope, Envoy::TimeSource& time_source) { + return std::make_unique>( + histogram(scope, {prefix_, upstream_rq_time_}), time_source); +} + +Stats::StatName RedisCommandStats::getCommandFromRequest(const RespValue& request) { + // Get command from RespValue + switch (request.type()) { + case RespType::Array: + return getCommandFromRequest(request.asArray().front()); + case RespType::Integer: + return unknown_metric_; + case RespType::Null: + return null_metric_; + default: + // Once we have a RespType::String we lowercase it and then look it up in our stat_name_map. + // If it does not exist, we return our unknown stat name. + std::string to_lower_command(request.asString()); + to_lower_table_.toLowerCase(to_lower_command); + + auto iter = stat_name_map_.find(to_lower_command); + if (iter != stat_name_map_.end()) { + return iter->second; + } else { + return unknown_metric_; + } + } +} + +void RedisCommandStats::updateStatsTotal(Stats::Scope& scope, Stats::StatName command) { + counter(scope, {prefix_, command, total_}).inc(); +} + +void RedisCommandStats::updateStats(Stats::Scope& scope, Stats::StatName command, + const bool success) { + if (success) { + counter(scope, {prefix_, command, success_}).inc(); + } else { + counter(scope, {prefix_, command, success_}).inc(); + } +} + +} // namespace Redis +} // namespace Common +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy \ No newline at end of file diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h new file mode 100644 index 000000000000..0ee2ce824c48 --- /dev/null +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include + +#include "envoy/stats/scope.h" +#include "envoy/stats/timespan.h" + +#include "common/common/to_lower_table.h" +#include "common/stats/symbol_table_impl.h" + +#include "extensions/filters/network/common/redis/codec.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Common { +namespace Redis { + +class RedisCommandStats { +public: + RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix); + + // TODO (@FAYiEKcbD0XFqF2QK2E4viAHg8rMm2VbjYKdjTg): Use Singleton to manage a single + // RedisCommandStats on the client factory so that it can be used for proxy filter, discovery and + // health check. + static std::shared_ptr + createRedisCommandStats(Stats::SymbolTable& symbol_table) { + return std::make_shared(symbol_table, "upstream_commands"); + } + + Stats::Counter& counter(Stats::Scope& scope, const Stats::StatNameVec& stat_names); + Stats::Histogram& histogram(Stats::Scope& scope, const Stats::StatNameVec& stat_names); + Stats::CompletableTimespanPtr createCommandTimer(Stats::Scope& scope, Stats::StatName command, + Envoy::TimeSource& time_source); + Stats::CompletableTimespanPtr createAggregateTimer(Stats::Scope& scope, + Envoy::TimeSource& time_source); + Stats::StatName getCommandFromRequest(const RespValue& request); + void updateStatsTotal(Stats::Scope& scope, Stats::StatName command); + void updateStats(Stats::Scope& scope, Stats::StatName command, const bool success); + Stats::StatName getUnusedStatName() { return unused_metric_; } + +private: + void addCommandToPool(const std::string& command_string); + + Stats::SymbolTable& symbol_table_; + Stats::StatNamePool stat_name_pool_; + StringMap stat_name_map_; + const Stats::StatName prefix_; + const Stats::StatName upstream_rq_time_; + const Stats::StatName latency_; + const Stats::StatName total_; + const Stats::StatName success_; + const Stats::StatName error_; + const Stats::StatName unused_metric_; + const Stats::StatName null_metric_; + const Stats::StatName unknown_metric_; + const ToLowerTable to_lower_table_; +}; +using RedisCommandStatsSharedPtr = std::shared_ptr; + +} // namespace Redis +} // namespace Common +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/network/redis_proxy/BUILD b/source/extensions/filters/network/redis_proxy/BUILD index bbc18dff95d2..2aee3f55e361 100644 --- a/source/extensions/filters/network/redis_proxy/BUILD +++ b/source/extensions/filters/network/redis_proxy/BUILD @@ -120,6 +120,7 @@ envoy_cc_library( "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", "//source/extensions/filters/network/common/redis:codec_lib", + "//source/extensions/filters/network/common/redis:redis_command_stats_lib", "//source/extensions/filters/network/redis_proxy:command_splitter_lib", "//source/extensions/filters/network/redis_proxy:conn_pool_lib", "//source/extensions/filters/network/redis_proxy:proxy_filter_lib", diff --git a/source/extensions/filters/network/redis_proxy/config.cc b/source/extensions/filters/network/redis_proxy/config.cc index 5e3e4018260d..f14176ebe274 100644 --- a/source/extensions/filters/network/redis_proxy/config.cc +++ b/source/extensions/filters/network/redis_proxy/config.cc @@ -57,15 +57,19 @@ Network::FilterFactoryCb RedisProxyFilterConfigFactory::createFilterFactoryFromP } addUniqueClusters(unique_clusters, prefix_routes.catch_all_route()); + auto redis_command_stats = + Common::Redis::RedisCommandStats::createRedisCommandStats(context.scope().symbolTable()); + Upstreams upstreams; for (auto& cluster : unique_clusters) { Stats::ScopePtr stats_scope = context.scope().createScope(fmt::format("cluster.{}.redis_cluster", cluster)); + upstreams.emplace(cluster, std::make_shared( cluster, context.clusterManager(), Common::Redis::Client::ClientFactoryImpl::instance_, context.threadLocal(), proto_config.settings(), context.api(), - std::move(stats_scope))); + std::move(stats_scope), redis_command_stats)); } auto router = diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index a097924ea734..c8449f61dc84 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -23,10 +23,12 @@ InstanceImpl::InstanceImpl( const std::string& cluster_name, Upstream::ClusterManager& cm, Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, - Api::Api& api, Stats::ScopePtr&& stats_scope) + Api::Api& api, Stats::ScopePtr&& stats_scope, + const Common::Redis::RedisCommandStatsSharedPtr& redis_command_stats) : cm_(cm), client_factory_(client_factory), tls_(tls.allocateSlot()), config_(config), - api_(api), stats_scope_(std::move(stats_scope)), redis_cluster_stats_{REDIS_CLUSTER_STATS( - POOL_COUNTER(*stats_scope_))} { + api_(api), stats_scope_(std::move(stats_scope)), + redis_command_stats_(redis_command_stats), redis_cluster_stats_{REDIS_CLUSTER_STATS( + POOL_COUNTER(*stats_scope_))} { tls_->set([this, cluster_name]( Event::Dispatcher& dispatcher) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(*this, dispatcher, cluster_name); @@ -195,7 +197,8 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared if (!client) { client = std::make_unique(*this); client->host_ = host; - client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_); + client->redis_client_ = parent_.client_factory_.create( + host, dispatcher_, parent_.config_, parent_.redis_command_stats_, *parent_.stats_scope_); client->redis_client_->addConnectionCallbacks(*client); // TODO(hyang): should the auth command and readonly command be moved to the factory method? if (!auth_password_.empty()) { diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h index 91ea51f3e752..7731943b873f 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.h +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.h @@ -51,7 +51,8 @@ class InstanceImpl : public Instance { const std::string& cluster_name, Upstream::ClusterManager& cm, Common::Redis::Client::ClientFactory& client_factory, ThreadLocal::SlotAllocator& tls, const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config, - Api::Api& api, Stats::ScopePtr&& stats_scope); + Api::Api& api, Stats::ScopePtr&& stats_scope, + const Common::Redis::RedisCommandStatsSharedPtr& redis_command_stats); // RedisProxy::ConnPool::Instance Common::Redis::Client::PoolRequest* makeRequest(const std::string& key, const Common::Redis::RespValue& request, @@ -131,6 +132,7 @@ class InstanceImpl : public Instance { Common::Redis::Client::ConfigImpl config_; Api::Api& api_; Stats::ScopePtr stats_scope_; + Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; RedisClusterStats redis_cluster_stats_; }; diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index 9f130824639c..c1107e1a489f 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -22,7 +22,11 @@ RedisHealthChecker::RedisHealthChecker( RedisHealthChecker::RedisActiveHealthCheckSession::RedisActiveHealthCheckSession( RedisHealthChecker& parent, const Upstream::HostSharedPtr& host) - : ActiveHealthCheckSession(parent, host), parent_(parent) {} + : ActiveHealthCheckSession(parent, host), parent_(parent) { + redis_command_stats_ = + Extensions::NetworkFilters::Common::Redis::RedisCommandStats::createRedisCommandStats( + parent_.cluster_.info()->statsScope().symbolTable()); +} RedisHealthChecker::RedisActiveHealthCheckSession::~RedisActiveHealthCheckSession() { ASSERT(current_request_ == nullptr); @@ -51,7 +55,9 @@ void RedisHealthChecker::RedisActiveHealthCheckSession::onEvent(Network::Connect void RedisHealthChecker::RedisActiveHealthCheckSession::onInterval() { if (!client_) { - client_ = parent_.client_factory_.create(host_, parent_.dispatcher_, *this); + client_ = + parent_.client_factory_.create(host_, parent_.dispatcher_, *this, redis_command_stats_, + parent_.cluster_.info()->statsScope()); client_->addConnectionCallbacks(*this); } diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index 4c7cc320e01f..17c8e060e716 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -81,6 +81,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + bool enableCommandStats() const override { return false; } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks void onResponse(NetworkFilters::Common::Redis::RespValuePtr&& value) override; @@ -95,6 +96,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { RedisHealthChecker& parent_; Extensions::NetworkFilters::Common::Redis::Client::ClientPtr client_; Extensions::NetworkFilters::Common::Redis::Client::PoolRequest* current_request_{}; + Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; }; enum class Type { Ping, Exists }; diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index c8b941760555..78463582049a 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -58,7 +58,9 @@ class RedisClusterTest : public testing::Test, // ClientFactory Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&, + const Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&, + Stats::Scope&) override { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index ca944879f516..8a283c67faf1 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -69,8 +69,11 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF EXPECT_CALL(*upstream_connection_, connect()); EXPECT_CALL(*upstream_connection_, noDelay(true)); + redis_command_stats_ = + Common::Redis::RedisCommandStats::createRedisCommandStats(stats_.symbolTable()); + client_ = ClientImpl::create(host_, dispatcher_, Common::Redis::EncoderPtr{encoder_}, *this, - *config_); + *config_, redis_command_stats_, stats_); EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_cx_total_.value()); EXPECT_EQ(1UL, host_->stats_.cx_total_.value()); EXPECT_EQ(false, client_->active()); @@ -107,6 +110,9 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF Network::ReadFilterSharedPtr upstream_read_filter_; std::unique_ptr config_; ClientPtr client_; + Stats::IsolatedStoreImpl stats_; + Stats::ScopePtr stats_scope_; + Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; }; TEST_F(RedisClientImplTest, BatchWithZeroBufferAndTimeout) { @@ -151,6 +157,7 @@ class ConfigBufferSizeGTSingleRequest : public Config { return std::chrono::milliseconds(1); } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + bool enableCommandStats() const override { return false; } ReadPolicy readPolicy() const override { return ReadPolicy::Master; } }; @@ -465,6 +472,7 @@ class ConfigOutlierDisabled : public Config { } ReadPolicy readPolicy() const override { return ReadPolicy::Master; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } + bool enableCommandStats() const override { return false; } }; TEST_F(RedisClientImplTest, OutlierDisabled) { @@ -871,7 +879,10 @@ TEST(RedisClientFactoryImplTest, Basic) { EXPECT_CALL(*host, createConnection_(_, _)).WillOnce(Return(conn_info)); NiceMock dispatcher; ConfigImpl config(createConnPoolSettings()); - ClientPtr client = factory.create(host, dispatcher, config); + Stats::IsolatedStoreImpl stats_; + auto redis_command_stats = + Common::Redis::RedisCommandStats::createRedisCommandStats(stats_.symbolTable()); + ClientPtr client = factory.create(host, dispatcher, config, redis_command_stats, stats_); client->close(); } diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index 9dd249ec6002..f8f03045aa8e 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -66,11 +66,13 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client max_upstream_unknown_connections_reached_.value_++; })); + auto redis_command_stats = + Common::Redis::RedisCommandStats::createRedisCommandStats(store->symbolTable()); std::unique_ptr conn_pool_impl = std::make_unique(cluster_name_, cm_, *this, tls_, Common::Redis::Client::createConnPoolSettings( 20, hashtagging, true, max_unknown_conns, read_policy_), - api_, std::move(store)); + api_, std::move(store), redis_command_stats); // Set the authentication password for this connection pool. conn_pool_impl->tls_->getTyped().auth_password_ = auth_password_; conn_pool_ = std::move(conn_pool_impl); @@ -155,7 +157,9 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client // Common::Redis::Client::ClientFactory Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, - const Common::Redis::Client::Config&) override { + const Common::Redis::Client::Config&, + const Common::Redis::RedisCommandStatsSharedPtr&, + Stats::Scope&) override { return Common::Redis::Client::ClientPtr{create_(host)}; } diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index aa18cd8d8e63..b9c215853885 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -124,7 +124,9 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, - const Extensions::NetworkFilters::Common::Redis::Client::Config&) override { + const Extensions::NetworkFilters::Common::Redis::Client::Config&, + const Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&, + Stats::Scope&) override { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } @@ -166,6 +168,7 @@ class RedisHealthCheckerTest EXPECT_EQ(session->maxBufferSizeBeforeFlush(), 0); EXPECT_EQ(session->bufferFlushTimeoutInMs(), std::chrono::milliseconds(1)); EXPECT_EQ(session->maxUpstreamUnknownConnections(), 0); + EXPECT_FALSE(session->enableCommandStats()); session->onDeferredDeleteBase(); // This must be called to pass assertions in the destructor. } From 38b926c63f347a70c933e0854ee9f31b1d2e85ce Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 10 Sep 2019 15:31:14 -0700 Subject: [PATCH 527/542] bazel: update bazel to 0.29.1 (#8198) Description: Upgrade bazel to 0.29.1 and bazel-toolchains to corresponding version. Risk Level: Low Testing: CI Docs Changes: N/A Release Notes: N/A Signed-off-by: Lizan Zhou --- .azure-pipelines/linux.yml | 7 + .bazelrc | 3 +- .bazelversion | 2 +- .circleci/config.yml | 2 +- bazel/repository_locations.bzl | 9 +- .../{bazel_0.28.1 => bazel_0.29.1}/cc/BUILD | 5 +- .../cc/armeabi_cc_toolchain_config.bzl | 0 .../cc/builtin_include_directory_paths | 14 ++ .../bazel_0.29.1}/cc/cc_toolchain_config.bzl | 155 +++++------------- .../cc/cc_wrapper.sh | 0 .../config/BUILD | 4 +- .../{bazel_0.28.1 => bazel_0.29.1}/cc/BUILD | 5 +- .../cc/armeabi_cc_toolchain_config.bzl | 0 .../cc/builtin_include_directory_paths | 12 ++ .../bazel_0.29.1}/cc/cc_toolchain_config.bzl | 155 +++++------------- .../cc/cc_wrapper.sh | 0 .../config/BUILD | 4 +- .../{bazel_0.28.1 => bazel_0.29.1}/cc/BUILD | 3 +- .../cc/armeabi_cc_toolchain_config.bzl | 0 .../cc/builtin_include_directory_paths | 14 ++ .../cc/cc_toolchain_config.bzl | 155 +++++------------- .../cc/cc_wrapper.sh | 0 .../config/BUILD | 4 +- bazel/toolchains/configs/versions.bzl | 6 +- bazel/toolchains/rbe_toolchains_config.bzl | 2 +- 25 files changed, 203 insertions(+), 358 deletions(-) rename bazel/toolchains/configs/clang/{bazel_0.28.1 => bazel_0.29.1}/cc/BUILD (95%) rename bazel/toolchains/configs/clang/{bazel_0.28.1 => bazel_0.29.1}/cc/armeabi_cc_toolchain_config.bzl (100%) create mode 100755 bazel/toolchains/configs/clang/bazel_0.29.1/cc/builtin_include_directory_paths rename bazel/toolchains/configs/{clang_libcxx/bazel_0.28.1 => clang/bazel_0.29.1}/cc/cc_toolchain_config.bzl (88%) rename bazel/toolchains/configs/clang/{bazel_0.28.1 => bazel_0.29.1}/cc/cc_wrapper.sh (100%) rename bazel/toolchains/configs/clang/{bazel_0.28.1 => bazel_0.29.1}/config/BUILD (89%) rename bazel/toolchains/configs/clang_libcxx/{bazel_0.28.1 => bazel_0.29.1}/cc/BUILD (95%) rename bazel/toolchains/configs/clang_libcxx/{bazel_0.28.1 => bazel_0.29.1}/cc/armeabi_cc_toolchain_config.bzl (100%) create mode 100755 bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/builtin_include_directory_paths rename bazel/toolchains/configs/{clang/bazel_0.28.1 => clang_libcxx/bazel_0.29.1}/cc/cc_toolchain_config.bzl (88%) rename bazel/toolchains/configs/clang_libcxx/{bazel_0.28.1 => bazel_0.29.1}/cc/cc_wrapper.sh (100%) rename bazel/toolchains/configs/clang_libcxx/{bazel_0.28.1 => bazel_0.29.1}/config/BUILD (91%) rename bazel/toolchains/configs/gcc/{bazel_0.28.1 => bazel_0.29.1}/cc/BUILD (96%) rename bazel/toolchains/configs/gcc/{bazel_0.28.1 => bazel_0.29.1}/cc/armeabi_cc_toolchain_config.bzl (100%) create mode 100755 bazel/toolchains/configs/gcc/bazel_0.29.1/cc/builtin_include_directory_paths rename bazel/toolchains/configs/gcc/{bazel_0.28.1 => bazel_0.29.1}/cc/cc_toolchain_config.bzl (88%) rename bazel/toolchains/configs/gcc/{bazel_0.28.1 => bazel_0.29.1}/cc/cc_wrapper.sh (100%) rename bazel/toolchains/configs/gcc/{bazel_0.28.1 => bazel_0.29.1}/config/BUILD (89%) diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml index f3337072613e..aa834255f303 100644 --- a/.azure-pipelines/linux.yml +++ b/.azure-pipelines/linux.yml @@ -48,6 +48,13 @@ jobs: displayName: "Check disk space at end" condition: always() + - task: PublishTestResults@2 + inputs: + testResultsFiles: '**/bazel-out/**/testlogs/**/test.xml' + testRunTitle: '$(CI_TARGET)' + searchFolder: $(Build.StagingDirectory)/tmp + condition: always() + - task: PublishBuildArtifacts@1 inputs: pathtoPublish: "$(Build.StagingDirectory)/envoy" diff --git a/.bazelrc b/.bazelrc index 5235dc6265ac..fd60cc9b0dd9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -124,13 +124,12 @@ build:remote --auth_enabled=true build:remote --experimental_inmemory_jdeps_files build:remote --experimental_inmemory_dotd_files build:remote --experimental_remote_download_outputs=toplevel -test:remote --experimental_remote_download_outputs=minimal build:remote-clang --config=remote build:remote-clang --config=rbe-toolchain-clang # Docker sandbox -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build:8246167b9d238797cbc6c03dccc9e3921c37617d +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.bazelversion b/.bazelversion index 48f7a71df4be..25939d35c738 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -0.28.1 +0.29.1 diff --git a/.circleci/config.yml b/.circleci/config.yml index 22d9de1203af..14cf64272abf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ executors: description: "A regular build executor based on ubuntu image" docker: # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here. - - image: envoyproxy/envoy-build:cd8574de783791b3353579b489222bfda74888da + - image: envoyproxy/envoy-build:cb15cc3d2010aff77d6e022ddf6d723fa8becbc0 resource_class: xlarge working_directory: /source diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 9712e2c093be..cba60a106ac2 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -4,9 +4,12 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.18.1/bazel-gazelle-0.18.1.tar.gz"], ), bazel_toolchains = dict( - sha256 = "b72e7a911436b2900b05759a1fcd735070edbd4442f0a3506ef021fdcd6e15b3", - strip_prefix = "bazel-toolchains-0.28.5", - urls = ["https://github.com/bazelbuild/bazel-toolchains/archive/0.28.5.tar.gz"], + sha256 = "ab0d8aaeaeeef413ddb03922dbdb99bbae9e1b2c157a87c77d70d45a830be5b0", + strip_prefix = "bazel-toolchains-0.29.1", + urls = [ + "https://github.com/bazelbuild/bazel-toolchains/releases/download/0.29.1/bazel-toolchains-0.29.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/0.29.1.tar.gz", + ], ), boringssl = dict( sha256 = "c712766ddc844de2a38e686e1cdd7288795e9a6fe7f699c6636f1b76703db84e", diff --git a/bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/BUILD similarity index 95% rename from bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD rename to bazel/toolchains/configs/clang/bazel_0.29.1/cc/BUILD index 015b58ead104..da902b38d578 100755 --- a/bazel/toolchains/configs/clang/bazel_0.28.1/cc/BUILD +++ b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/BUILD @@ -18,6 +18,7 @@ package(default_visibility = ["//visibility:public"]) load(":cc_toolchain_config.bzl", "cc_toolchain_config") load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") +load("@rules_cc//cc:defs.bzl", "cc_toolchain", "cc_toolchain_suite") licenses(["notice"]) # Apache 2.0 @@ -37,7 +38,7 @@ filegroup( filegroup( name = "compiler_deps", - srcs = glob(["extra_tools/**"], allow_empty = True) + [":empty"], + srcs = glob(["extra_tools/**"], allow_empty = True) + [":builtin_include_directory_paths"], ) # This is the entry point for --crosstool_top. Toolchains are found @@ -111,7 +112,7 @@ cc_toolchain_config( "-fdata-sections"], dbg_compile_flags = ["-g"], cxx_flags = ["-std=c++0x"], - link_flags = ["-fuse-ld=gold", + link_flags = ["-fuse-ld=/usr/bin/ld.gold", "-Wl,-no-as-needed", "-Wl,-z,relro,-z,now", "-B/usr/lib/llvm-8/bin", diff --git a/bazel/toolchains/configs/clang/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl similarity index 100% rename from bazel/toolchains/configs/clang/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl rename to bazel/toolchains/configs/clang/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl diff --git a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/builtin_include_directory_paths b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/builtin_include_directory_paths new file mode 100755 index 000000000000..151e3b008a2b --- /dev/null +++ b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/builtin_include_directory_paths @@ -0,0 +1,14 @@ +This file is generated by cc_configure and contains builtin include directories +that /usr/lib/llvm-8/bin/clang reported. This file is a dependency of every compilation action and +changes to it will be reflected in the action cache key. When some of these +paths change, Bazel will make sure to rerun the action, even though none of +declared action inputs or the action commandline changes. + +/usr/local/include +/usr/lib/llvm-8/lib/clang/8.0.1/include +/usr/include/x86_64-linux-gnu +/usr/include +/usr/include/c++/7.4.0 +/usr/include/x86_64-linux-gnu/c++/7.4.0 +/usr/include/c++/7.4.0/backward +/usr/include/clang/8.0.1/include diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_toolchain_config.bzl similarity index 88% rename from bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_toolchain_config.bzl rename to bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_toolchain_config.bzl index f2b12d962963..bf4d83940f89 100755 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_toolchain_config.bzl +++ b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_toolchain_config.bzl @@ -74,6 +74,12 @@ all_link_actions = [ ACTION_NAMES.cpp_link_nodeps_dynamic_library, ] +lto_index_actions = [ + ACTION_NAMES.lto_index_for_executable, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, +] + def _impl(ctx): tool_paths = [ tool_path(name = name, path = path) @@ -95,18 +101,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.compile_flags, @@ -114,18 +109,7 @@ def _impl(ctx): ] if ctx.attr.compile_flags else []), ), flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.dbg_compile_flags, @@ -134,18 +118,7 @@ def _impl(ctx): with_features = [with_feature_set(features = ["dbg"])], ), flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.opt_compile_flags, @@ -154,15 +127,7 @@ def _impl(ctx): with_features = [with_feature_set(features = ["opt"])], ), flag_set( - actions = [ - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_cpp_compile_actions + [ACTION_NAMES.lto_backend], flag_groups = ([ flag_group( flags = ctx.attr.cxx_flags, @@ -177,7 +142,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group( flags = ctx.attr.link_flags, @@ -185,7 +150,7 @@ def _impl(ctx): ] if ctx.attr.link_flags else []), ), flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group( flags = ctx.attr.opt_link_flags, @@ -215,10 +180,7 @@ def _impl(ctx): ACTION_NAMES.cpp_module_codegen, ACTION_NAMES.lto_backend, ACTION_NAMES.clif_match, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["--sysroot=%{sysroot}"], @@ -255,18 +217,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = [ flag_group( flags = ["%{user_compile_flags}"], @@ -283,18 +234,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.unfiltered_compile_flags, @@ -308,7 +248,7 @@ def _impl(ctx): name = "library_search_directories", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-L%{library_search_directories}"], @@ -328,6 +268,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_executable, ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.lto_index_for_executable, + ACTION_NAMES.lto_index_for_dynamic_library, ], flag_groups = [flag_group(flags = ["-static-libgcc"])], with_features = [ @@ -447,7 +389,7 @@ def _impl(ctx): name = "runtime_library_search_directories", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "runtime_library_search_directories", @@ -474,7 +416,7 @@ def _impl(ctx): ], ), flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "runtime_library_search_directories", @@ -502,7 +444,7 @@ def _impl(ctx): name = "fission_support", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-Wl,--gdb-index"], @@ -520,6 +462,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [flag_group(flags = ["-shared"])], ), @@ -581,10 +525,7 @@ def _impl(ctx): actions = [ ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -607,10 +548,7 @@ def _impl(ctx): ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile, ACTION_NAMES.lto_backend, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -662,7 +600,7 @@ def _impl(ctx): name = "symbol_counts", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -697,10 +635,7 @@ def _impl(ctx): ], ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, + actions = all_link_actions + lto_index_actions + [ "objc-executable", "objc++-executable", ], @@ -717,7 +652,7 @@ def _impl(ctx): name = "strip_debug_symbols", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-Wl,-S"], @@ -735,6 +670,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [ flag_group( @@ -760,7 +697,7 @@ def _impl(ctx): name = "libraries_to_link", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "libraries_to_link", @@ -847,7 +784,7 @@ def _impl(ctx): name = "user_link_flags", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["%{user_link_flags}"], @@ -885,7 +822,7 @@ def _impl(ctx): name = "linkstamps", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["%{linkstamp_paths}"], @@ -919,11 +856,7 @@ def _impl(ctx): ], ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = [flag_group(flags = ["--coverage"])], ), ], @@ -977,7 +910,10 @@ def _impl(ctx): name = "force_pic_flags", flag_sets = [ flag_set( - actions = [ACTION_NAMES.cpp_link_executable], + actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.lto_index_for_executable, + ], flag_groups = [ flag_group( flags = ["-pie"], @@ -1014,6 +950,7 @@ def _impl(ctx): ], ) + dynamic_library_linker_tool_path = tool_paths dynamic_library_linker_tool_feature = feature( name = "dynamic_library_linker_tool", flag_sets = [ @@ -1021,6 +958,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [ flag_group( @@ -1041,11 +980,7 @@ def _impl(ctx): name = "output_execpath_flags", flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-o", "%{output_execpath}"], @@ -1076,11 +1011,7 @@ def _impl(ctx): ] if ctx.attr.coverage_compile_flags else []), ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group(flags = ctx.attr.coverage_link_flags), ] if ctx.attr.coverage_link_flags else []), diff --git a/bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_wrapper.sh b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_wrapper.sh similarity index 100% rename from bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_wrapper.sh rename to bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_wrapper.sh diff --git a/bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD b/bazel/toolchains/configs/clang/bazel_0.29.1/config/BUILD similarity index 89% rename from bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD rename to bazel/toolchains/configs/clang/bazel_0.29.1/config/BUILD index 7af5e24e5de1..5fb181617f25 100644 --- a/bazel/toolchains/configs/clang/bazel_0.28.1/config/BUILD +++ b/bazel/toolchains/configs/clang/bazel_0.29.1/config/BUILD @@ -29,7 +29,7 @@ toolchain( "@bazel_tools//platforms:linux", "@bazel_tools//platforms:x86_64", ], - toolchain = "//bazel/toolchains/configs/clang/bazel_0.28.1/cc:cc-compiler-k8", + toolchain = "//bazel/toolchains/configs/clang/bazel_0.29.1/cc:cc-compiler-k8", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", ) @@ -43,7 +43,7 @@ platform( remote_execution_properties = """ properties: { name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" } properties { name: "OSFamily" diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/BUILD similarity index 95% rename from bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD rename to bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/BUILD index 8a2ac6331467..625db858205b 100755 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/BUILD +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/BUILD @@ -18,6 +18,7 @@ package(default_visibility = ["//visibility:public"]) load(":cc_toolchain_config.bzl", "cc_toolchain_config") load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") +load("@rules_cc//cc:defs.bzl", "cc_toolchain", "cc_toolchain_suite") licenses(["notice"]) # Apache 2.0 @@ -37,7 +38,7 @@ filegroup( filegroup( name = "compiler_deps", - srcs = glob(["extra_tools/**"], allow_empty = True) + [":empty"], + srcs = glob(["extra_tools/**"], allow_empty = True) + [":builtin_include_directory_paths"], ) # This is the entry point for --crosstool_top. Toolchains are found @@ -109,7 +110,7 @@ cc_toolchain_config( "-fdata-sections"], dbg_compile_flags = ["-g"], cxx_flags = ["-stdlib=libc++"], - link_flags = ["-fuse-ld=gold", + link_flags = ["-fuse-ld=/usr/bin/ld.gold", "-Wl,-no-as-needed", "-Wl,-z,relro,-z,now", "-B/usr/lib/llvm-8/bin", diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl similarity index 100% rename from bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl rename to bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/builtin_include_directory_paths b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/builtin_include_directory_paths new file mode 100755 index 000000000000..d809f97268c1 --- /dev/null +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/builtin_include_directory_paths @@ -0,0 +1,12 @@ +This file is generated by cc_configure and contains builtin include directories +that /usr/lib/llvm-8/bin/clang reported. This file is a dependency of every compilation action and +changes to it will be reflected in the action cache key. When some of these +paths change, Bazel will make sure to rerun the action, even though none of +declared action inputs or the action commandline changes. + +/usr/local/include +/usr/lib/llvm-8/lib/clang/8.0.1/include +/usr/include/x86_64-linux-gnu +/usr/include +/usr/lib/llvm-8/include/c++/v1 +/usr/include/clang/8.0.1/include diff --git a/bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_toolchain_config.bzl similarity index 88% rename from bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_toolchain_config.bzl rename to bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_toolchain_config.bzl index f2b12d962963..bf4d83940f89 100755 --- a/bazel/toolchains/configs/clang/bazel_0.28.1/cc/cc_toolchain_config.bzl +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_toolchain_config.bzl @@ -74,6 +74,12 @@ all_link_actions = [ ACTION_NAMES.cpp_link_nodeps_dynamic_library, ] +lto_index_actions = [ + ACTION_NAMES.lto_index_for_executable, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, +] + def _impl(ctx): tool_paths = [ tool_path(name = name, path = path) @@ -95,18 +101,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.compile_flags, @@ -114,18 +109,7 @@ def _impl(ctx): ] if ctx.attr.compile_flags else []), ), flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.dbg_compile_flags, @@ -134,18 +118,7 @@ def _impl(ctx): with_features = [with_feature_set(features = ["dbg"])], ), flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.opt_compile_flags, @@ -154,15 +127,7 @@ def _impl(ctx): with_features = [with_feature_set(features = ["opt"])], ), flag_set( - actions = [ - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_cpp_compile_actions + [ACTION_NAMES.lto_backend], flag_groups = ([ flag_group( flags = ctx.attr.cxx_flags, @@ -177,7 +142,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group( flags = ctx.attr.link_flags, @@ -185,7 +150,7 @@ def _impl(ctx): ] if ctx.attr.link_flags else []), ), flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group( flags = ctx.attr.opt_link_flags, @@ -215,10 +180,7 @@ def _impl(ctx): ACTION_NAMES.cpp_module_codegen, ACTION_NAMES.lto_backend, ACTION_NAMES.clif_match, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["--sysroot=%{sysroot}"], @@ -255,18 +217,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = [ flag_group( flags = ["%{user_compile_flags}"], @@ -283,18 +234,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.unfiltered_compile_flags, @@ -308,7 +248,7 @@ def _impl(ctx): name = "library_search_directories", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-L%{library_search_directories}"], @@ -328,6 +268,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_executable, ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.lto_index_for_executable, + ACTION_NAMES.lto_index_for_dynamic_library, ], flag_groups = [flag_group(flags = ["-static-libgcc"])], with_features = [ @@ -447,7 +389,7 @@ def _impl(ctx): name = "runtime_library_search_directories", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "runtime_library_search_directories", @@ -474,7 +416,7 @@ def _impl(ctx): ], ), flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "runtime_library_search_directories", @@ -502,7 +444,7 @@ def _impl(ctx): name = "fission_support", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-Wl,--gdb-index"], @@ -520,6 +462,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [flag_group(flags = ["-shared"])], ), @@ -581,10 +525,7 @@ def _impl(ctx): actions = [ ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -607,10 +548,7 @@ def _impl(ctx): ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile, ACTION_NAMES.lto_backend, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -662,7 +600,7 @@ def _impl(ctx): name = "symbol_counts", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -697,10 +635,7 @@ def _impl(ctx): ], ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, + actions = all_link_actions + lto_index_actions + [ "objc-executable", "objc++-executable", ], @@ -717,7 +652,7 @@ def _impl(ctx): name = "strip_debug_symbols", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-Wl,-S"], @@ -735,6 +670,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [ flag_group( @@ -760,7 +697,7 @@ def _impl(ctx): name = "libraries_to_link", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "libraries_to_link", @@ -847,7 +784,7 @@ def _impl(ctx): name = "user_link_flags", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["%{user_link_flags}"], @@ -885,7 +822,7 @@ def _impl(ctx): name = "linkstamps", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["%{linkstamp_paths}"], @@ -919,11 +856,7 @@ def _impl(ctx): ], ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = [flag_group(flags = ["--coverage"])], ), ], @@ -977,7 +910,10 @@ def _impl(ctx): name = "force_pic_flags", flag_sets = [ flag_set( - actions = [ACTION_NAMES.cpp_link_executable], + actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.lto_index_for_executable, + ], flag_groups = [ flag_group( flags = ["-pie"], @@ -1014,6 +950,7 @@ def _impl(ctx): ], ) + dynamic_library_linker_tool_path = tool_paths dynamic_library_linker_tool_feature = feature( name = "dynamic_library_linker_tool", flag_sets = [ @@ -1021,6 +958,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [ flag_group( @@ -1041,11 +980,7 @@ def _impl(ctx): name = "output_execpath_flags", flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-o", "%{output_execpath}"], @@ -1076,11 +1011,7 @@ def _impl(ctx): ] if ctx.attr.coverage_compile_flags else []), ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group(flags = ctx.attr.coverage_link_flags), ] if ctx.attr.coverage_link_flags else []), diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_wrapper.sh b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_wrapper.sh similarity index 100% rename from bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc/cc_wrapper.sh rename to bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_wrapper.sh diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/config/BUILD similarity index 91% rename from bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD rename to bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/config/BUILD index 385ea14041f2..3d87dd780c53 100644 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/config/BUILD +++ b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/config/BUILD @@ -29,7 +29,7 @@ toolchain( "@bazel_tools//platforms:linux", "@bazel_tools//platforms:x86_64", ], - toolchain = "//bazel/toolchains/configs/clang_libcxx/bazel_0.28.1/cc:cc-compiler-k8", + toolchain = "//bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc:cc-compiler-k8", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", ) @@ -43,7 +43,7 @@ platform( remote_execution_properties = """ properties: { name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" } properties { name: "OSFamily" diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/BUILD similarity index 96% rename from bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD rename to bazel/toolchains/configs/gcc/bazel_0.29.1/cc/BUILD index 443b34aa3eff..ae7728d61bd4 100755 --- a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/BUILD +++ b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/BUILD @@ -18,6 +18,7 @@ package(default_visibility = ["//visibility:public"]) load(":cc_toolchain_config.bzl", "cc_toolchain_config") load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") +load("@rules_cc//cc:defs.bzl", "cc_toolchain", "cc_toolchain_suite") licenses(["notice"]) # Apache 2.0 @@ -37,7 +38,7 @@ filegroup( filegroup( name = "compiler_deps", - srcs = glob(["extra_tools/**"], allow_empty = True) + [":empty"], + srcs = glob(["extra_tools/**"], allow_empty = True) + [":builtin_include_directory_paths"], ) # This is the entry point for --crosstool_top. Toolchains are found diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl similarity index 100% rename from bazel/toolchains/configs/gcc/bazel_0.28.1/cc/armeabi_cc_toolchain_config.bzl rename to bazel/toolchains/configs/gcc/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl diff --git a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/builtin_include_directory_paths b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/builtin_include_directory_paths new file mode 100755 index 000000000000..30a600ae8b06 --- /dev/null +++ b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/builtin_include_directory_paths @@ -0,0 +1,14 @@ +This file is generated by cc_configure and contains builtin include directories +that /usr/bin/gcc reported. This file is a dependency of every compilation action and +changes to it will be reflected in the action cache key. When some of these +paths change, Bazel will make sure to rerun the action, even though none of +declared action inputs or the action commandline changes. + +/usr/lib/gcc/x86_64-linux-gnu/7/include +/usr/local/include +/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed +/usr/include/x86_64-linux-gnu +/usr/include +/usr/include/c++/7 +/usr/include/x86_64-linux-gnu/c++/7 +/usr/include/c++/7/backward diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_toolchain_config.bzl similarity index 88% rename from bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_toolchain_config.bzl rename to bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_toolchain_config.bzl index f2b12d962963..bf4d83940f89 100755 --- a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_toolchain_config.bzl +++ b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_toolchain_config.bzl @@ -74,6 +74,12 @@ all_link_actions = [ ACTION_NAMES.cpp_link_nodeps_dynamic_library, ] +lto_index_actions = [ + ACTION_NAMES.lto_index_for_executable, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, +] + def _impl(ctx): tool_paths = [ tool_path(name = name, path = path) @@ -95,18 +101,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.compile_flags, @@ -114,18 +109,7 @@ def _impl(ctx): ] if ctx.attr.compile_flags else []), ), flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.dbg_compile_flags, @@ -134,18 +118,7 @@ def _impl(ctx): with_features = [with_feature_set(features = ["dbg"])], ), flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.opt_compile_flags, @@ -154,15 +127,7 @@ def _impl(ctx): with_features = [with_feature_set(features = ["opt"])], ), flag_set( - actions = [ - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_cpp_compile_actions + [ACTION_NAMES.lto_backend], flag_groups = ([ flag_group( flags = ctx.attr.cxx_flags, @@ -177,7 +142,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group( flags = ctx.attr.link_flags, @@ -185,7 +150,7 @@ def _impl(ctx): ] if ctx.attr.link_flags else []), ), flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group( flags = ctx.attr.opt_link_flags, @@ -215,10 +180,7 @@ def _impl(ctx): ACTION_NAMES.cpp_module_codegen, ACTION_NAMES.lto_backend, ACTION_NAMES.clif_match, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["--sysroot=%{sysroot}"], @@ -255,18 +217,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = [ flag_group( flags = ["%{user_compile_flags}"], @@ -283,18 +234,7 @@ def _impl(ctx): enabled = True, flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], + actions = all_compile_actions, flag_groups = ([ flag_group( flags = ctx.attr.unfiltered_compile_flags, @@ -308,7 +248,7 @@ def _impl(ctx): name = "library_search_directories", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-L%{library_search_directories}"], @@ -328,6 +268,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_executable, ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.lto_index_for_executable, + ACTION_NAMES.lto_index_for_dynamic_library, ], flag_groups = [flag_group(flags = ["-static-libgcc"])], with_features = [ @@ -447,7 +389,7 @@ def _impl(ctx): name = "runtime_library_search_directories", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "runtime_library_search_directories", @@ -474,7 +416,7 @@ def _impl(ctx): ], ), flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "runtime_library_search_directories", @@ -502,7 +444,7 @@ def _impl(ctx): name = "fission_support", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-Wl,--gdb-index"], @@ -520,6 +462,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [flag_group(flags = ["-shared"])], ), @@ -581,10 +525,7 @@ def _impl(ctx): actions = [ ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -607,10 +548,7 @@ def _impl(ctx): ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile, ACTION_NAMES.lto_backend, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + ] + all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -662,7 +600,7 @@ def _impl(ctx): name = "symbol_counts", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = [ @@ -697,10 +635,7 @@ def _impl(ctx): ], ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, + actions = all_link_actions + lto_index_actions + [ "objc-executable", "objc++-executable", ], @@ -717,7 +652,7 @@ def _impl(ctx): name = "strip_debug_symbols", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-Wl,-S"], @@ -735,6 +670,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [ flag_group( @@ -760,7 +697,7 @@ def _impl(ctx): name = "libraries_to_link", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( iterate_over = "libraries_to_link", @@ -847,7 +784,7 @@ def _impl(ctx): name = "user_link_flags", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["%{user_link_flags}"], @@ -885,7 +822,7 @@ def _impl(ctx): name = "linkstamps", flag_sets = [ flag_set( - actions = all_link_actions, + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["%{linkstamp_paths}"], @@ -919,11 +856,7 @@ def _impl(ctx): ], ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = [flag_group(flags = ["--coverage"])], ), ], @@ -977,7 +910,10 @@ def _impl(ctx): name = "force_pic_flags", flag_sets = [ flag_set( - actions = [ACTION_NAMES.cpp_link_executable], + actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.lto_index_for_executable, + ], flag_groups = [ flag_group( flags = ["-pie"], @@ -1014,6 +950,7 @@ def _impl(ctx): ], ) + dynamic_library_linker_tool_path = tool_paths dynamic_library_linker_tool_feature = feature( name = "dynamic_library_linker_tool", flag_sets = [ @@ -1021,6 +958,8 @@ def _impl(ctx): actions = [ ACTION_NAMES.cpp_link_dynamic_library, ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.lto_index_for_dynamic_library, + ACTION_NAMES.lto_index_for_nodeps_dynamic_library, ], flag_groups = [ flag_group( @@ -1041,11 +980,7 @@ def _impl(ctx): name = "output_execpath_flags", flag_sets = [ flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = [ flag_group( flags = ["-o", "%{output_execpath}"], @@ -1076,11 +1011,7 @@ def _impl(ctx): ] if ctx.attr.coverage_compile_flags else []), ), flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], + actions = all_link_actions + lto_index_actions, flag_groups = ([ flag_group(flags = ctx.attr.coverage_link_flags), ] if ctx.attr.coverage_link_flags else []), diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_wrapper.sh b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_wrapper.sh similarity index 100% rename from bazel/toolchains/configs/gcc/bazel_0.28.1/cc/cc_wrapper.sh rename to bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_wrapper.sh diff --git a/bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD b/bazel/toolchains/configs/gcc/bazel_0.29.1/config/BUILD similarity index 89% rename from bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD rename to bazel/toolchains/configs/gcc/bazel_0.29.1/config/BUILD index 4584bd320cc2..6de35eeeaf23 100644 --- a/bazel/toolchains/configs/gcc/bazel_0.28.1/config/BUILD +++ b/bazel/toolchains/configs/gcc/bazel_0.29.1/config/BUILD @@ -29,7 +29,7 @@ toolchain( "@bazel_tools//platforms:linux", "@bazel_tools//platforms:x86_64", ], - toolchain = "//bazel/toolchains/configs/gcc/bazel_0.28.1/cc:cc-compiler-k8", + toolchain = "//bazel/toolchains/configs/gcc/bazel_0.29.1/cc:cc-compiler-k8", toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", ) @@ -43,7 +43,7 @@ platform( remote_execution_properties = """ properties: { name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" + value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" } properties { name: "OSFamily" diff --git a/bazel/toolchains/configs/versions.bzl b/bazel/toolchains/configs/versions.bzl index c21f70f9cb14..bdf2a5c264ee 100644 --- a/bazel/toolchains/configs/versions.bzl +++ b/bazel/toolchains/configs/versions.bzl @@ -5,9 +5,9 @@ toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, cre toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx") toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc") _TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1,toolchain_config_spec2] -_BAZEL_TO_CONFIG_SPEC_NAMES = {"0.28.1": ["clang", "clang_libcxx", "gcc"]} -LATEST = "sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" -CONTAINER_TO_CONFIG_SPEC_NAMES = {"sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b": ["clang", "clang_libcxx", "gcc"]} +_BAZEL_TO_CONFIG_SPEC_NAMES = {"0.29.1": ["clang", "clang_libcxx", "gcc"]} +LATEST = "sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" +CONTAINER_TO_CONFIG_SPEC_NAMES = {"sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f": ["clang", "clang_libcxx", "gcc"]} _DEFAULT_TOOLCHAIN_CONFIG_SPEC = toolchain_config_spec0 TOOLCHAIN_CONFIG_AUTOGEN_SPEC = struct( bazel_to_config_spec_names_map = _BAZEL_TO_CONFIG_SPEC_NAMES, diff --git a/bazel/toolchains/rbe_toolchains_config.bzl b/bazel/toolchains/rbe_toolchains_config.bzl index 96c38c79e1f5..8820710297a7 100644 --- a/bazel/toolchains/rbe_toolchains_config.bzl +++ b/bazel/toolchains/rbe_toolchains_config.bzl @@ -4,7 +4,7 @@ load("@envoy//bazel/toolchains:configs/versions.bzl", _generated_toolchain_confi _ENVOY_BUILD_IMAGE_REGISTRY = "gcr.io" _ENVOY_BUILD_IMAGE_REPOSITORY = "envoy-ci/envoy-build" -_ENVOY_BUILD_IMAGE_DIGEST = "sha256:d1f6087fdeb6a6e5d4fd52a5dc06b15f43f49e2c20fc813bcaaa12333485a70b" +_ENVOY_BUILD_IMAGE_DIGEST = "sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" _CONFIGS_OUTPUT_BASE = "bazel/toolchains/configs" _CLANG_ENV = { From e2eb25823832d5affe602a6308c7de7ae60370bd Mon Sep 17 00:00:00 2001 From: James Forcier Date: Wed, 11 Sep 2019 08:52:00 -0400 Subject: [PATCH 528/542] upstream: Add ability to disable host selection during panic (#8024) Previously, when in a panic state, requests would be routed to all hosts. In some cases it is instead preferable to not route any requests. Add a configuration option for zone-aware load balancers which switches from routing to all hosts to no hosts. Closes #7550. Signed-off-by: James Forcier jforcier@grubhub.com Risk Level: Low Testing: 2 new unit tests written; manual testing Docs Changes: Note about new configuration option added Release Notes: added Signed-off-by: James Forcier --- api/envoy/api/v2/cds.proto | 6 + .../load_balancing/panic_threshold.rst | 19 ++- docs/root/intro/version_history.rst | 1 + source/common/upstream/load_balancer_impl.cc | 46 +++++-- source/common/upstream/load_balancer_impl.h | 4 +- .../upstream/load_balancer_impl_test.cc | 122 +++++++++++++++++- 6 files changed, 175 insertions(+), 23 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index a0095ac2aa2c..b2b71384f8c8 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -539,6 +539,12 @@ message Cluster { // * :ref:`runtime values `. // * :ref:`Zone aware routing support `. google.protobuf.UInt64Value min_cluster_size = 2; + + // If set to true, Envoy will not consider any hosts when the cluster is in :ref:`panic + // mode`. Instead, the cluster will fail all + // requests as if all hosts are unhealthy. This can help avoid potentially overwhelming a + // failing service. + bool fail_traffic_on_panic = 3; } // Configuration for :ref:`locality weighted load balancing // ` diff --git a/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst b/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst index e24022a8f07b..864ad11171e6 100644 --- a/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst +++ b/docs/root/intro/arch_overview/upstream/load_balancing/panic_threshold.rst @@ -5,13 +5,24 @@ Panic threshold During load balancing, Envoy will generally only consider available (healthy or degraded) hosts in an upstream cluster. However, if the percentage of available hosts in the cluster becomes too low, -Envoy will disregard health status and balance amongst all hosts. This is known as the *panic -threshold*. The default panic threshold is 50%. This is +Envoy will disregard health status and balance either amongst all hosts or no hosts. This is known +as the *panic threshold*. The default panic threshold is 50%. This is :ref:`configurable ` via runtime as well as in the :ref:`cluster configuration `. The panic threshold is used to avoid a situation in which host failures cascade throughout the cluster as load increases. +There are two modes Envoy can choose from when in a panic state: traffic will either be sent to all +hosts, or will be sent to no hosts (and therefore will always fail). This is configured in the +:ref:`cluster configuration `. +Choosing to fail traffic during panic scenarios can help avoid overwhelming potentially failing +upstream services, as it will reduce the load on the upstream service before all hosts have been +determined to be unhealthy. However, it eliminates the possibility of _some_ requests succeeding +even when many or all hosts in a cluster are unhealthy. This may be a good tradeoff to make if a +given service is observed to fail in an all-or-nothing pattern, as it will more quickly cut off +requests to the cluster. Conversely, if a cluster typically continues to successfully service _some_ +requests even when degraded, enabling this option is probably unhelpful. + Panic thresholds work in conjunction with priorities. If the number of available hosts in a given priority goes down, Envoy will try to shift some traffic to lower priorities. If it succeeds in finding enough available hosts in lower priorities, Envoy will disregard panic thresholds. In @@ -20,8 +31,8 @@ disregards panic thresholds and continues to distribute traffic load across prio the algorithm described :ref:`here `. However, when normalized total availability drops below 100%, Envoy assumes that there are not enough available hosts across all priority levels. It continues to distribute traffic load across priorities, -but if a given priority level's availability is below the panic threshold, traffic will go to all hosts -in that priority level regardless of their availability. +but if a given priority level's availability is below the panic threshold, traffic will go to all +(or no) hosts in that priority level regardless of their availability. The following examples explain the relationship between normalized total availability and panic threshold. It is assumed that the default value of 50% is used for the panic threshold. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 679ea4f9e769..42e8c463c0d6 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -59,6 +59,7 @@ Version history * upstream: added :ref:`an option ` that allows draining HTTP, TCP connection pools on cluster membership change. * upstream: added network filter chains to upstream connections, see :ref:`filters`. * upstream: use p2c to select hosts for least-requests load balancers if all host weights are the same, even in cases where weights are not equal to 1. +* upstream: added :ref:`fail_traffic_on_panic ` to allow failing all requests to a cluster during panic state. * zookeeper: parse responses and emit latency stats. 1.11.1 (August 13, 2019) diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 79bf87b79c44..b3172d9c9a27 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -282,7 +282,8 @@ ZoneAwareLoadBalancerBase::ZoneAwareLoadBalancerBase( routing_enabled_(PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( common_config.zone_aware_lb_config(), routing_enabled, 100, 100)), min_cluster_size_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(common_config.zone_aware_lb_config(), - min_cluster_size, 6U)) { + min_cluster_size, 6U)), + fail_traffic_on_panic_(common_config.zone_aware_lb_config().fail_traffic_on_panic()) { ASSERT(!priority_set.hostSetsPerPriority().empty()); resizePerPriorityState(); priority_set_.addPriorityUpdateCb( @@ -539,7 +540,7 @@ uint32_t ZoneAwareLoadBalancerBase::tryChooseLocalLocalityHosts(const HostSet& h return i; } -ZoneAwareLoadBalancerBase::HostsSource +absl::optional ZoneAwareLoadBalancerBase::hostSourceToUse(LoadBalancerContext* context) { auto host_set_and_source = chooseHostSet(context); @@ -549,11 +550,16 @@ ZoneAwareLoadBalancerBase::hostSourceToUse(LoadBalancerContext* context) { HostsSource hosts_source; hosts_source.priority_ = host_set.priority(); - // If the selected host set has insufficient healthy hosts, return all hosts. + // If the selected host set has insufficient healthy hosts, return all hosts (unless we should + // fail traffic on panic, in which case return no host). if (per_priority_panic_[hosts_source.priority_]) { stats_.lb_healthy_panic_.inc(); - hosts_source.source_type_ = HostsSource::SourceType::AllHosts; - return hosts_source; + if (fail_traffic_on_panic_) { + return absl::nullopt; + } else { + hosts_source.source_type_ = HostsSource::SourceType::AllHosts; + return hosts_source; + } } // If we're doing locality weighted balancing, pick locality. @@ -586,10 +592,14 @@ ZoneAwareLoadBalancerBase::hostSourceToUse(LoadBalancerContext* context) { if (isGlobalPanic(localHostSet())) { stats_.lb_local_cluster_not_ok_.inc(); - // If the local Envoy instances are in global panic, do not do locality - // based routing. - hosts_source.source_type_ = sourceType(host_availability); - return hosts_source; + // If the local Envoy instances are in global panic, and we should not fail traffic, do + // not do locality based routing. + if (fail_traffic_on_panic_) { + return absl::nullopt; + } else { + hosts_source.source_type_ = sourceType(host_availability); + return hosts_source; + } } hosts_source.source_type_ = localitySourceType(host_availability); @@ -699,8 +709,11 @@ void EdfLoadBalancerBase::refresh(uint32_t priority) { } HostConstSharedPtr EdfLoadBalancerBase::chooseHostOnce(LoadBalancerContext* context) { - const HostsSource hosts_source = hostSourceToUse(context); - auto scheduler_it = scheduler_.find(hosts_source); + const absl::optional hosts_source = hostSourceToUse(context); + if (!hosts_source) { + return nullptr; + } + auto scheduler_it = scheduler_.find(*hosts_source); // We should always have a scheduler for any return value from // hostSourceToUse() via the construction in refresh(); ASSERT(scheduler_it != scheduler_.end()); @@ -717,11 +730,11 @@ HostConstSharedPtr EdfLoadBalancerBase::chooseHostOnce(LoadBalancerContext* cont } return host; } else { - const HostVector& hosts_to_use = hostSourceToHosts(hosts_source); + const HostVector& hosts_to_use = hostSourceToHosts(*hosts_source); if (hosts_to_use.empty()) { return nullptr; } - return unweightedHostPick(hosts_to_use, hosts_source); + return unweightedHostPick(hosts_to_use, *hosts_source); } } @@ -749,7 +762,12 @@ HostConstSharedPtr LeastRequestLoadBalancer::unweightedHostPick(const HostVector } HostConstSharedPtr RandomLoadBalancer::chooseHostOnce(LoadBalancerContext* context) { - const HostVector& hosts_to_use = hostSourceToHosts(hostSourceToUse(context)); + const absl::optional hosts_source = hostSourceToUse(context); + if (!hosts_source) { + return nullptr; + } + + const HostVector& hosts_to_use = hostSourceToHosts(*hosts_source); if (hosts_to_use.empty()) { return nullptr; } diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index 98fd0c75d7d1..988955f359ab 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -223,8 +223,9 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase { /** * Pick the host source to use, doing zone aware routing when the hosts are sufficiently healthy. + * If no host is chosen (due to fail_traffic_on_panic being set), return absl::nullopt. */ - HostsSource hostSourceToUse(LoadBalancerContext* context); + absl::optional hostSourceToUse(LoadBalancerContext* context); /** * Index into priority_set via hosts source descriptor. @@ -300,6 +301,7 @@ class ZoneAwareLoadBalancerBase : public LoadBalancerBase { const uint32_t routing_enabled_; const uint64_t min_cluster_size_; + const bool fail_traffic_on_panic_; struct PerPriorityState { // The percent of requests which can be routed to the local locality. diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index e29dd9b0dcd8..a2e7366f5ece 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -539,6 +539,39 @@ TEST_P(FailoverTest, PriorityUpdatesWithLocalHostSet) { EXPECT_EQ(tertiary_host_set_.hosts_[0], lb_->chooseHost(nullptr)); } +// Test that extending the priority set with an existing LB causes the correct updates when the +// cluster is configured to disable on panic. +TEST_P(FailoverTest, PriorityUpdatesWithLocalHostSetDisableOnPanic) { + host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; + failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81")}; + common_config_.mutable_zone_aware_lb_config()->set_fail_traffic_on_panic(true); + + init(false); + // With both the primary and failover hosts unhealthy, we should select no host. + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); + + // Update the priority set with a new priority level P=2 and ensure the host + // is chosen + MockHostSet& tertiary_host_set_ = *priority_set_.getMockHostSet(2); + HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:82")})); + tertiary_host_set_.hosts_ = *hosts; + tertiary_host_set_.healthy_hosts_ = tertiary_host_set_.hosts_; + HostVector add_hosts; + add_hosts.push_back(tertiary_host_set_.hosts_[0]); + tertiary_host_set_.runCallbacks(add_hosts, {}); + EXPECT_EQ(tertiary_host_set_.hosts_[0], lb_->chooseHost(nullptr)); + + // Now add a healthy host in P=0 and make sure it is immediately selected. + host_set_.healthy_hosts_ = host_set_.hosts_; + host_set_.runCallbacks(add_hosts, {}); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); + + // Remove the healthy host and ensure we fail back over to tertiary_host_set_ + host_set_.healthy_hosts_ = {}; + host_set_.runCallbacks({}, {}); + EXPECT_EQ(tertiary_host_set_.hosts_[0], lb_->chooseHost(nullptr)); +} + // Test extending the priority set. TEST_P(FailoverTest, ExtendPrioritiesUpdatingPrioritySet) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; @@ -829,6 +862,32 @@ TEST_P(RoundRobinLoadBalancerTest, MaxUnhealthyPanic) { EXPECT_EQ(3UL, stats_.lb_healthy_panic_.value()); } +// Test that no hosts are selected when fail_traffic_on_panic is enabled. +TEST_P(RoundRobinLoadBalancerTest, MaxUnhealthyPanicDisableOnPanic) { + hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), + makeTestHost(info_, "tcp://127.0.0.1:81")}; + hostSet().hosts_ = { + makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81"), + makeTestHost(info_, "tcp://127.0.0.1:82"), makeTestHost(info_, "tcp://127.0.0.1:83"), + makeTestHost(info_, "tcp://127.0.0.1:84"), makeTestHost(info_, "tcp://127.0.0.1:85")}; + + common_config_.mutable_zone_aware_lb_config()->set_fail_traffic_on_panic(true); + + init(false); + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); + + // Take the threshold back above the panic threshold. + hostSet().healthy_hosts_ = { + makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81"), + makeTestHost(info_, "tcp://127.0.0.1:82"), makeTestHost(info_, "tcp://127.0.0.1:83")}; + hostSet().runCallbacks({}, {}); + + EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); + EXPECT_EQ(hostSet().healthy_hosts_[1], lb_->chooseHost(nullptr)); + + EXPECT_EQ(1UL, stats_.lb_healthy_panic_.value()); +} + // Ensure if the panic threshold is 0%, panic mode is disabled. TEST_P(RoundRobinLoadBalancerTest, DisablePanicMode) { hostSet().healthy_hosts_ = {}; @@ -1177,6 +1236,41 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingLocalEmpty) { EXPECT_EQ(1U, stats_.lb_local_cluster_not_ok_.value()); } +TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingLocalEmptyFailTrafficOnPanic) { + common_config_.mutable_zone_aware_lb_config()->set_fail_traffic_on_panic(true); + + if (&hostSet() == &failover_host_set_) { // P = 1 does not support zone-aware routing. + return; + } + HostVectorSharedPtr upstream_hosts(new HostVector( + {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")})); + HostVectorSharedPtr local_hosts(new HostVector({}, {})); + + HostsPerLocalitySharedPtr upstream_hosts_per_locality = makeHostsPerLocality( + {{makeTestHost(info_, "tcp://127.0.0.1:80")}, {makeTestHost(info_, "tcp://127.0.0.1:81")}}); + HostsPerLocalitySharedPtr local_hosts_per_locality = makeHostsPerLocality({{}, {}}); + + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) + .WillOnce(Return(50)) + .WillOnce(Return(50)); + EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) + .WillOnce(Return(true)); + EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 6)) + .WillOnce(Return(1)); + + hostSet().healthy_hosts_ = *upstream_hosts; + hostSet().hosts_ = *upstream_hosts; + hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; + init(true); + updateHosts(local_hosts, local_hosts_per_locality); + + // Local cluster is not OK, we'll do regular routing (and select no host, since we're in global + // panic). + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); + EXPECT_EQ(0U, stats_.lb_healthy_panic_.value()); + EXPECT_EQ(1U, stats_.lb_local_cluster_not_ok_.value()); +} + // Validate that if we have healthy host lists >= 2, but there is no local // locality included, that we skip zone aware routing and fallback. TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNoLocalLocality) { @@ -1388,20 +1482,40 @@ INSTANTIATE_TEST_SUITE_P(PrimaryOrFailover, LeastRequestLoadBalancerTest, class RandomLoadBalancerTest : public LoadBalancerTestBase { public: - RandomLoadBalancer lb_{priority_set_, nullptr, stats_, runtime_, random_, common_config_}; + void init() { + lb_.reset( + new RandomLoadBalancer(priority_set_, nullptr, stats_, runtime_, random_, common_config_)); + } + std::shared_ptr lb_; }; -TEST_P(RandomLoadBalancerTest, NoHosts) { EXPECT_EQ(nullptr, lb_.chooseHost(nullptr)); } +TEST_P(RandomLoadBalancerTest, NoHosts) { + init(); + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); +} TEST_P(RandomLoadBalancerTest, Normal) { + init(); + hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; hostSet().hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(2)); - EXPECT_EQ(hostSet().healthy_hosts_[0], lb_.chooseHost(nullptr)); + EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(3)); - EXPECT_EQ(hostSet().healthy_hosts_[1], lb_.chooseHost(nullptr)); + EXPECT_EQ(hostSet().healthy_hosts_[1], lb_->chooseHost(nullptr)); +} + +TEST_P(RandomLoadBalancerTest, FailClusterOnPanic) { + common_config_.mutable_zone_aware_lb_config()->set_fail_traffic_on_panic(true); + init(); + + hostSet().healthy_hosts_ = {}; + hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), + makeTestHost(info_, "tcp://127.0.0.1:81")}; + hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. + EXPECT_EQ(nullptr, lb_->chooseHost(nullptr)); } INSTANTIATE_TEST_SUITE_P(PrimaryOrFailover, RandomLoadBalancerTest, ::testing::Values(true, false)); From a758e8cdc543e3276814d5d4b27740b29709f4c4 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Wed, 11 Sep 2019 22:12:38 +0530 Subject: [PATCH 529/542] metrics service: flush histogram buckets (#8180) Signed-off-by: Rama Chavali --- docs/root/intro/version_history.rst | 1 + .../grpc_metrics_service_impl.cc | 44 ++++++++++++++----- .../grpc_metrics_service_impl.h | 2 +- .../metrics_service_integration_test.cc | 12 ++++- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 42e8c463c0d6..ffd12ecebe9f 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -34,6 +34,7 @@ Version history * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. +* metrics_service: added support for flushing histogram buckets. * outlier_detector: added :ref:`support for the grpc-status response header ` by mapping it to HTTP status. Guarded by envoy.reloadable_features.outlier_detection_support_for_grpc_status which defaults to true. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). * performance: stats symbol table implementation (disabled by default; to test it, add "--use-fake-symbol-table 0" to the command-line arguments when starting Envoy). diff --git a/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.cc b/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.cc index 3ecd4d2c2831..f37d2c984fd3 100644 --- a/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.cc +++ b/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.cc @@ -60,21 +60,43 @@ void MetricsServiceSink::flushGauge(const Stats::Gauge& gauge) { gauage_metric->set_value(gauge.value()); } -void MetricsServiceSink::flushHistogram(const Stats::ParentHistogram& histogram) { - io::prometheus::client::MetricFamily* metrics_family = message_.add_envoy_metrics(); - metrics_family->set_type(io::prometheus::client::MetricType::SUMMARY); - metrics_family->set_name(histogram.name()); - auto* metric = metrics_family->add_metric(); - metric->set_timestamp_ms(std::chrono::duration_cast( - time_source_.systemTime().time_since_epoch()) - .count()); - auto* summary_metric = metric->mutable_summary(); - const Stats::HistogramStatistics& hist_stats = histogram.intervalStatistics(); +void MetricsServiceSink::flushHistogram(const Stats::ParentHistogram& envoy_histogram) { + // TODO(ramaraochavali): Currently we are sending both quantile information and bucket + // information. We should make this configurable if it turns out that sending both affects + // performance. + + // Add summary information for histograms. + io::prometheus::client::MetricFamily* summary_metrics_family = message_.add_envoy_metrics(); + summary_metrics_family->set_type(io::prometheus::client::MetricType::SUMMARY); + summary_metrics_family->set_name(envoy_histogram.name()); + auto* summary_metric = summary_metrics_family->add_metric(); + summary_metric->set_timestamp_ms(std::chrono::duration_cast( + time_source_.systemTime().time_since_epoch()) + .count()); + auto* summary = summary_metric->mutable_summary(); + const Stats::HistogramStatistics& hist_stats = envoy_histogram.intervalStatistics(); for (size_t i = 0; i < hist_stats.supportedQuantiles().size(); i++) { - auto* quantile = summary_metric->add_quantile(); + auto* quantile = summary->add_quantile(); quantile->set_quantile(hist_stats.supportedQuantiles()[i]); quantile->set_value(hist_stats.computedQuantiles()[i]); } + + // Add bucket information for histograms. + io::prometheus::client::MetricFamily* histogram_metrics_family = message_.add_envoy_metrics(); + histogram_metrics_family->set_type(io::prometheus::client::MetricType::HISTOGRAM); + histogram_metrics_family->set_name(envoy_histogram.name()); + auto* histogram_metric = histogram_metrics_family->add_metric(); + histogram_metric->set_timestamp_ms(std::chrono::duration_cast( + time_source_.systemTime().time_since_epoch()) + .count()); + auto* histogram = histogram_metric->mutable_histogram(); + histogram->set_sample_count(hist_stats.sampleCount()); + histogram->set_sample_sum(hist_stats.sampleSum()); + for (size_t i = 0; i < hist_stats.supportedBuckets().size(); i++) { + auto* bucket = histogram->add_bucket(); + bucket->set_upper_bound(hist_stats.supportedBuckets()[i]); + bucket->set_cumulative_count(hist_stats.computedBuckets()[i]); + } } void MetricsServiceSink::flush(Stats::MetricSnapshot& snapshot) { diff --git a/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.h b/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.h index 14a99e92ac07..5c0473b084dc 100644 --- a/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.h +++ b/source/extensions/stat_sinks/metrics_service/grpc_metrics_service_impl.h @@ -80,7 +80,7 @@ class MetricsServiceSink : public Stats::Sink { void flushCounter(const Stats::Counter& counter); void flushGauge(const Stats::Gauge& gauge); - void flushHistogram(const Stats::ParentHistogram& histogram); + void flushHistogram(const Stats::ParentHistogram& envoy_histogram); private: GrpcMetricsStreamerSharedPtr grpc_metrics_streamer_; diff --git a/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc b/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc index 6d698a36dfc7..e19310347334 100644 --- a/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc +++ b/test/extensions/stats_sinks/metrics_service/metrics_service_integration_test.cc @@ -66,9 +66,11 @@ class MetricsServiceIntegrationTest : public Grpc::GrpcClientIntegrationParamTes ABSL_MUST_USE_RESULT AssertionResult waitForMetricsRequest() { + bool known_summary_exists = false; bool known_histogram_exists = false; bool known_counter_exists = false; bool known_gauge_exists = false; + // Sometimes stats do not come in the first flush cycle, this loop ensures that we wait till // required stats are flushed. // TODO(ramaraochavali): Figure out a more robust way to find out all required stats have been @@ -98,11 +100,18 @@ class MetricsServiceIntegrationTest : public Grpc::GrpcClientIntegrationParamTes } if (metrics_family.name() == "cluster.cluster_0.upstream_rq_time" && metrics_family.type() == ::io::prometheus::client::MetricType::SUMMARY) { - known_histogram_exists = true; + known_summary_exists = true; Stats::HistogramStatisticsImpl empty_statistics; EXPECT_EQ(metrics_family.metric(0).summary().quantile_size(), empty_statistics.supportedQuantiles().size()); } + if (metrics_family.name() == "cluster.cluster_0.upstream_rq_time" && + metrics_family.type() == ::io::prometheus::client::MetricType::HISTOGRAM) { + known_histogram_exists = true; + Stats::HistogramStatisticsImpl empty_statistics; + EXPECT_EQ(metrics_family.metric(0).histogram().bucket_size(), + empty_statistics.supportedBuckets().size()); + } ASSERT(metrics_family.metric(0).has_timestamp_ms()); if (known_counter_exists && known_gauge_exists && known_histogram_exists) { break; @@ -111,6 +120,7 @@ class MetricsServiceIntegrationTest : public Grpc::GrpcClientIntegrationParamTes } EXPECT_TRUE(known_counter_exists); EXPECT_TRUE(known_gauge_exists); + EXPECT_TRUE(known_summary_exists); EXPECT_TRUE(known_histogram_exists); return AssertionSuccess(); From eb11fdede8548316a7b24d04f4427121927bc999 Mon Sep 17 00:00:00 2001 From: Pengyuan Bian Date: Wed, 11 Sep 2019 09:53:09 -0700 Subject: [PATCH 530/542] tracing: fix random sample fraction percent (#8205) Signed-off-by: Pengyuan Bian --- .../network/http_connection_manager/config.cc | 5 +-- .../http_connection_manager/config_test.cc | 36 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 532f0e086e5d..998dce6709ab 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -282,8 +282,9 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( envoy::type::FractionalPercent random_sampling; // TODO: Random sampling historically was an integer and default to out of 10,000. We should // deprecate that and move to a straight fractional percent config. - random_sampling.set_numerator( - tracing_config.has_random_sampling() ? tracing_config.random_sampling().value() : 10000); + uint64_t random_sampling_numerator{PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT( + tracing_config, random_sampling, 10000, 10000)}; + random_sampling.set_numerator(random_sampling_numerator); random_sampling.set_denominator(envoy::type::FractionalPercent::TEN_THOUSAND); envoy::type::FractionalPercent overall_sampling; overall_sampling.set_numerator( diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index b16fbfc97ebb..3341802e659c 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -355,7 +355,7 @@ TEST_F(HttpConnectionManagerConfigTest, SamplingConfigured) { EXPECT_EQ(1, config.tracingConfig()->client_sampling_.numerator()); EXPECT_EQ(envoy::type::FractionalPercent::HUNDRED, config.tracingConfig()->client_sampling_.denominator()); - EXPECT_EQ(2, config.tracingConfig()->random_sampling_.numerator()); + EXPECT_EQ(200, config.tracingConfig()->random_sampling_.numerator()); EXPECT_EQ(envoy::type::FractionalPercent::TEN_THOUSAND, config.tracingConfig()->random_sampling_.denominator()); EXPECT_EQ(3, config.tracingConfig()->overall_sampling_.numerator()); @@ -363,6 +363,40 @@ TEST_F(HttpConnectionManagerConfigTest, SamplingConfigured) { config.tracingConfig()->overall_sampling_.denominator()); } +TEST_F(HttpConnectionManagerConfigTest, FractionalSamplingConfigured) { + const std::string yaml_string = R"EOF( + stat_prefix: ingress_http + internal_address_config: + unix_sockets: true + route_config: + name: local_route + tracing: + operation_name: ingress + client_sampling: + value: 0.1 + random_sampling: + value: 0.2 + overall_sampling: + value: 0.3 + http_filters: + - name: envoy.router + )EOF"; + + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + + EXPECT_EQ(0, config.tracingConfig()->client_sampling_.numerator()); + EXPECT_EQ(envoy::type::FractionalPercent::HUNDRED, + config.tracingConfig()->client_sampling_.denominator()); + EXPECT_EQ(20, config.tracingConfig()->random_sampling_.numerator()); + EXPECT_EQ(envoy::type::FractionalPercent::TEN_THOUSAND, + config.tracingConfig()->random_sampling_.denominator()); + EXPECT_EQ(0, config.tracingConfig()->overall_sampling_.numerator()); + EXPECT_EQ(envoy::type::FractionalPercent::HUNDRED, + config.tracingConfig()->overall_sampling_.denominator()); +} + TEST_F(HttpConnectionManagerConfigTest, UnixSocketInternalAddress) { const std::string yaml_string = R"EOF( stat_prefix: ingress_http From c78c4b42b7a8160f581f4d263e2e4e5ab2bf1ac5 Mon Sep 17 00:00:00 2001 From: antonio Date: Wed, 11 Sep 2019 09:56:44 -0700 Subject: [PATCH 531/542] stats: Add per-host memory usage test case to stats_integration_test (#8189) Signed-off-by: Antonio Vicente --- test/config/utility.cc | 4 +- test/config/utility.h | 9 +++ test/integration/BUILD | 3 + test/integration/integration.h | 3 + test/integration/stats_integration_test.cc | 79 ++++++++++++++++++---- 5 files changed, 84 insertions(+), 14 deletions(-) diff --git a/test/config/utility.cc b/test/config/utility.cc index 11efd7a79da7..c230b4c772c3 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -365,8 +365,8 @@ void ConfigHelper::finalize(const std::vector& ports) { *cluster->mutable_transport_socket(), tls_config); } } - ASSERT(port_idx == ports.size() || eds_hosts || original_dst_cluster || custom_cluster || - bootstrap_.dynamic_resources().has_cds_config()); + ASSERT(skip_port_usage_validation_ || port_idx == ports.size() || eds_hosts || + original_dst_cluster || custom_cluster || bootstrap_.dynamic_resources().has_cds_config()); if (!connect_timeout_set_) { #ifdef __APPLE__ diff --git a/test/config/utility.h b/test/config/utility.h index 0abf01c3bf60..574160651aa3 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -158,6 +158,10 @@ class ConfigHelper { // Allow a finalized configuration to be edited for generating xDS responses void applyConfigModifiers(); + // Skip validation that ensures that all upstream ports are referenced by the + // configuration generated in ConfigHelper::finalize. + void skipPortUsageValidation() { skip_port_usage_validation_ = true; } + private: // Load the first HCM struct from the first listener into a parsed proto. bool loadHttpConnectionManager( @@ -186,6 +190,11 @@ class ConfigHelper { // default). bool connect_timeout_set_{false}; + // Option to disable port usage validation for cases where the number of + // upstream ports created is expected to be larger than the number of + // upstreams in the config. + bool skip_port_usage_validation_{false}; + // A sanity check guard to make sure config is not modified after handing it to Envoy. bool finalized_{false}; }; diff --git a/test/integration/BUILD b/test/integration/BUILD index fd69958d4b4f..02a8a0101d66 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -559,6 +559,9 @@ envoy_cc_test( envoy_cc_test( name = "stats_integration_test", srcs = ["stats_integration_test.cc"], + # The symbol table cluster memory tests take a while to run specially under tsan. + # Shard it to avoid test timeout. + shard_count = 2, deps = [ ":integration_lib", "//source/common/memory:stats_lib", diff --git a/test/integration/integration.h b/test/integration/integration.h index c61dfb0fc006..7cfa3b726435 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -173,6 +173,9 @@ class BaseIntegrationTest : Logger::Loggable { void setUpstreamProtocol(FakeHttpConnection::Type protocol); // Sets fake_upstreams_count_ and alters the upstream protocol in the config_helper_ void setUpstreamCount(uint32_t count) { fake_upstreams_count_ = count; } + // Skip validation that ensures that all upstream ports are referenced by the + // configuration generated in ConfigHelper::finalize. + void skipPortUsageValidation() { config_helper_.skipPortUsageValidation(); } // Make test more deterministic by using a fixed RNG value. void setDeterministic() { deterministic_ = true; } diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index 4cfacefe2d4a..2118a0e1aaa2 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -146,9 +146,25 @@ class ClusterMemoryTestHelper : public BaseIntegrationTest { ClusterMemoryTestHelper() : BaseIntegrationTest(testing::TestWithParam::GetParam()) {} - static size_t computeMemory(int num_clusters) { + static size_t computeMemoryDelta(int initial_num_clusters, int initial_num_hosts, + int final_num_clusters, int final_num_hosts, bool allow_stats) { + // Use the same number of fake upstreams for both helpers in order to exclude memory overhead + // added by the fake upstreams. + int fake_upstreams_count = 1 + final_num_clusters * final_num_hosts; + + size_t initial_memory; + { + ClusterMemoryTestHelper helper; + helper.setUpstreamCount(fake_upstreams_count); + helper.skipPortUsageValidation(); + initial_memory = + helper.clusterMemoryHelper(initial_num_clusters, initial_num_hosts, allow_stats); + } + ClusterMemoryTestHelper helper; - return helper.clusterMemoryHelper(num_clusters, true); + helper.setUpstreamCount(fake_upstreams_count); + return helper.clusterMemoryHelper(final_num_clusters, final_num_hosts, allow_stats) - + initial_memory; } private: @@ -157,15 +173,26 @@ class ClusterMemoryTestHelper : public BaseIntegrationTest { * @param allow_stats if false, enable set_reject_all in stats_config * @return size_t the total memory allocated */ - size_t clusterMemoryHelper(int num_clusters, bool allow_stats) { + size_t clusterMemoryHelper(int num_clusters, int num_hosts, bool allow_stats) { Stats::TestUtil::MemoryTest memory_test; config_helper_.addConfigModifier([&](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { if (!allow_stats) { bootstrap.mutable_stats_config()->mutable_stats_matcher()->set_reject_all(true); } - for (int i = 1; i < num_clusters; i++) { - auto* c = bootstrap.mutable_static_resources()->add_clusters(); - c->set_name(fmt::format("cluster_{}", i)); + for (int i = 1; i < num_clusters; ++i) { + auto* cluster = bootstrap.mutable_static_resources()->add_clusters(); + cluster->set_name(fmt::format("cluster_{}", i)); + } + + for (int i = 0; i < num_clusters; ++i) { + auto* cluster = bootstrap.mutable_static_resources()->mutable_clusters(i); + for (int j = 0; j < num_hosts; ++j) { + auto* host = cluster->add_hosts(); + auto* socket_address = host->mutable_socket_address(); + socket_address->set_protocol(envoy::api::v2::core::SocketAddress::TCP); + socket_address->set_address("0.0.0.0"); + socket_address->set_port_value(80); + } } }); initialize(); @@ -194,9 +221,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // A unique instance of ClusterMemoryTest allows for multiple runs of Envoy with // differing configuration. This is necessary for measuring the memory consumption // between the different instances within the same test. - const size_t m1 = ClusterMemoryTestHelper::computeMemory(1); - const size_t m1001 = ClusterMemoryTestHelper::computeMemory(1001); - const size_t m_per_cluster = (m1001 - m1) / 1000; + const size_t m1000 = ClusterMemoryTestHelper::computeMemoryDelta(1, 0, 1001, 0, true); + const size_t m_per_cluster = (m1000) / 1000; // Note: if you are increasing this golden value because you are adding a // stat, please confirm that this will be generally useful to most Envoy @@ -245,9 +271,8 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // A unique instance of ClusterMemoryTest allows for multiple runs of Envoy with // differing configuration. This is necessary for measuring the memory consumption // between the different instances within the same test. - const size_t m1 = ClusterMemoryTestHelper::computeMemory(1); - const size_t m1001 = ClusterMemoryTestHelper::computeMemory(1001); - const size_t m_per_cluster = (m1001 - m1) / 1000; + const size_t m1000 = ClusterMemoryTestHelper::computeMemoryDelta(1, 0, 1001, 0, true); + const size_t m_per_cluster = (m1000) / 1000; // Note: if you are increasing this golden value because you are adding a // stat, please confirm that this will be generally useful to most Envoy @@ -275,5 +300,35 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { EXPECT_MEMORY_LE(m_per_cluster, 36000); } +TEST_P(ClusterMemoryTestRunner, MemoryLargeHostSizeWithStats) { + Stats::TestUtil::SymbolTableCreatorTestPeer::setUseFakeSymbolTables(false); + + // A unique instance of ClusterMemoryTest allows for multiple runs of Envoy with + // differing configuration. This is necessary for measuring the memory consumption + // between the different instances within the same test. + const size_t m1000 = ClusterMemoryTestHelper::computeMemoryDelta(1, 1, 1, 1001, true); + const size_t m_per_host = (m1000) / 1000; + + // Note: if you are increasing this golden value because you are adding a + // stat, please confirm that this will be generally useful to most Envoy + // users. Otherwise you are adding to the per-host memory overhead, which + // will be significant for Envoy installations configured to talk to large + // numbers of hosts. + // + // History of golden values: + // + // Date PR Bytes Per Host Notes + // exact upper-bound + // ---------- ----- ----------------- ----- + // 2019/09/09 8189 2739 3100 Initial per-host memory snapshot + + // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI + // 'release' builds, where we control the platform and tool-chain. So you + // will need to find the correct value only after failing CI and looking + // at the logs. + EXPECT_MEMORY_EQ(m_per_host, 2739); + EXPECT_MEMORY_LE(m_per_host, 3100); +} + } // namespace } // namespace Envoy From c9703f96ff7ce2f890debd4483b64c82bd837065 Mon Sep 17 00:00:00 2001 From: Lisa Lu Date: Wed, 11 Sep 2019 09:59:15 -0700 Subject: [PATCH 532/542] router check tool: add flag for only printing failed tests (#8160) Signed-off-by: Lisa Lu --- .../install/tools/route_table_check_tool.rst | 3 ++ docs/root/intro/version_history.rst | 1 + test/tools/router_check/router.cc | 40 ++++++++++--------- test/tools/router_check/router.h | 21 +++++++++- test/tools/router_check/router_check.cc | 4 ++ .../test/config/Weighted.golden.json | 7 ++-- .../test/config/Weighted.golden.proto.json | 4 +- .../test/config/Weighted.golden.proto.pb_text | 5 ++- .../test/config/Weighted.golden.proto.yaml | 3 +- .../router_check/test/config/Weighted.yaml | 9 +++++ test/tools/router_check/test/route_tests.sh | 21 ++++++++-- 11 files changed, 88 insertions(+), 30 deletions(-) diff --git a/docs/root/install/tools/route_table_check_tool.rst b/docs/root/install/tools/route_table_check_tool.rst index 8acb2ac34a9d..186df20b130e 100644 --- a/docs/root/install/tools/route_table_check_tool.rst +++ b/docs/root/install/tools/route_table_check_tool.rst @@ -37,6 +37,9 @@ Usage -d, --details Show detailed test execution results. The first line indicates the test name. + --only-show-failures + Displays test results for failed tests. Omits test names for passing tests if the details flag is set. + -p, --useproto Use Proto test file schema diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ffd12ecebe9f..34dbfca93522 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -51,6 +51,7 @@ Version history * router check tool: add coverage reporting & enforcement. * router check tool: add comprehensive coverage reporting. * router check tool: add deprecated field check. +* router check tool: add flag for only printing results of failed tests. * thrift_proxy: fix crashing bug on invalid transport/protocol framing * tls: added verification of IP address SAN fields in certificates against configured SANs in the * tracing: added support to the Zipkin reporter for sending list of spans as Zipkin JSON v2 and protobuf message over HTTP. diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index e8e2143f4b8f..0547c15e03be 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -100,19 +100,14 @@ RouterCheckTool::RouterCheckTool( bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_json) { Json::ObjectSharedPtr loader = Json::Factory::loadFromFile(expected_route_json, *api_); loader->validateSchema(Json::ToolSchema::routerCheckSchema()); - bool no_failures = true; for (const Json::ObjectSharedPtr& check_config : loader->asObjectArray()) { headers_finalized_ = false; ToolConfig tool_config = ToolConfig::create(check_config); tool_config.route_ = config_->route(*tool_config.headers_, tool_config.random_value_); - std::string test_name = check_config->getString("test_name", ""); - if (details_) { - std::cout << test_name << std::endl; - } + tests_.emplace_back(test_name, std::vector{}); Json::ObjectSharedPtr validate = check_config->getObject("validate"); - using checkerFunc = std::function; const std::unordered_map checkers = { {"cluster_name", @@ -128,7 +123,6 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso {"path_redirect", [this](auto&... params) -> bool { return this->compareRedirectPath(params...); }}, }; - // Call appropriate function for each match case. for (const auto& test : checkers) { if (validate->hasObject(test.first)) { @@ -142,7 +136,6 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso } } } - if (validate->hasObject("header_fields")) { for (const Json::ObjectSharedPtr& header_field : validate->getObjectArray("header_fields")) { if (!compareHeaderField(tool_config, header_field->getString("field"), @@ -151,7 +144,6 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso } } } - if (validate->hasObject("custom_header_fields")) { for (const Json::ObjectSharedPtr& header_field : validate->getObjectArray("custom_header_fields")) { @@ -162,7 +154,7 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso } } } - + printResults(); return no_failures; } @@ -183,9 +175,7 @@ bool RouterCheckTool::compareEntries(const std::string& expected_routes) { tool_config.route_ = config_->route(*tool_config.headers_, tool_config.random_value_); const std::string& test_name = check_config.test_name(); - if (details_) { - std::cout << test_name << std::endl; - } + tests_.emplace_back(test_name, std::vector{}); const envoy::RouterCheckToolSchema::ValidationAssert& validate = check_config.validate(); using checkerFunc = @@ -208,7 +198,7 @@ bool RouterCheckTool::compareEntries(const std::string& expected_routes) { } } } - + printResults(); return no_failures; } @@ -424,13 +414,24 @@ bool RouterCheckTool::compareResults(const std::string& actual, const std::strin if (expected == actual) { return true; } + tests_.back().second.emplace_back("expected: [" + expected + "], actual: [" + actual + + "], test type: " + test_type); + return false; +} +void RouterCheckTool::printResults() { // Output failure details to stdout if details_ flag is set to true - if (details_) { - std::cerr << "expected: [" << expected << "], actual: [" << actual - << "], test type: " << test_type << std::endl; + for (const auto& test_result : tests_) { + // All test names are printed if the details_ flag is true unless only_show_failures_ is also + // true. + if ((details_ && !only_show_failures_) || + (only_show_failures_ && !test_result.second.empty())) { + std::cout << test_result.first << std::endl; + for (const auto& failure : test_result.second) { + std::cerr << failure << std::endl; + } + } } - return false; } // The Mock for runtime value checks. @@ -446,6 +447,8 @@ Options::Options(int argc, char** argv) { TCLAP::CmdLine cmd("router_check_tool", ' ', "none", true); TCLAP::SwitchArg is_proto("p", "useproto", "Use Proto test file schema", cmd, false); TCLAP::SwitchArg is_detailed("d", "details", "Show detailed test execution results", cmd, false); + TCLAP::SwitchArg only_show_failures("", "only-show-failures", "Only display failing tests", cmd, + false); TCLAP::SwitchArg disable_deprecation_check("", "disable-deprecation-check", "Disable deprecated fields check", cmd, false); TCLAP::ValueArg fail_under("f", "fail-under", @@ -468,6 +471,7 @@ Options::Options(int argc, char** argv) { is_proto_ = is_proto.getValue(); is_detailed_ = is_detailed.getValue(); + only_show_failures_ = only_show_failures.getValue(); fail_under_ = fail_under.getValue(); comprehensive_coverage_ = comprehensive_coverage.getValue(); disable_deprecation_check_ = disable_deprecation_check.getValue(); diff --git a/test/tools/router_check/router.h b/test/tools/router_check/router.h index e2156afa1072..78352e86e5d5 100644 --- a/test/tools/router_check/router.h +++ b/test/tools/router_check/router.h @@ -91,6 +91,11 @@ class RouterCheckTool : Logger::Loggable { */ void setShowDetails() { details_ = true; } + /** + * Set whether to only print failing match cases. + */ + void setOnlyShowFailures() { only_show_failures_ = true; } + float coverage(bool detailed) { return detailed ? coverage_.detailedReport() : coverage_.report(); } @@ -137,6 +142,8 @@ class RouterCheckTool : Logger::Loggable { bool compareResults(const std::string& actual, const std::string& expected, const std::string& test_type); + void printResults(); + bool runtimeMock(const std::string& key, const envoy::type::FractionalPercent& default_value, uint64_t random_value); @@ -144,6 +151,12 @@ class RouterCheckTool : Logger::Loggable { bool details_{false}; + bool only_show_failures_{false}; + + // The first member of each pair is the name of the test. + // The second member is a list of any failing results for that test as strings. + std::vector>> tests_; + // TODO(hennna): Switch away from mocks following work done by @rlazarus in github issue #499. std::unique_ptr> factory_context_; std::unique_ptr config_; @@ -196,10 +209,15 @@ class Options { bool isProto() const { return is_proto_; } /** - * @return true is detailed test execution results are displayed. + * @return true if detailed test execution results are displayed. */ bool isDetailed() const { return is_detailed_; } + /** + * @return true if only test failures are displayed. + */ + bool onlyShowFailures() const { return only_show_failures_; } + /** * @return true if the deprecated field check for RouteConfiguration is disabled. */ @@ -214,6 +232,7 @@ class Options { bool comprehensive_coverage_; bool is_proto_; bool is_detailed_; + bool only_show_failures_; bool disable_deprecation_check_; }; } // namespace Envoy diff --git a/test/tools/router_check/router_check.cc b/test/tools/router_check/router_check.cc index f3856e285d8c..1792af5c5b35 100644 --- a/test/tools/router_check/router_check.cc +++ b/test/tools/router_check/router_check.cc @@ -18,6 +18,10 @@ int main(int argc, char* argv[]) { checktool.setShowDetails(); } + if (options.onlyShowFailures()) { + checktool.setOnlyShowFailures(); + } + bool is_equal = options.isProto() ? checktool.compareEntries(options.testPath()) : checktool.compareEntriesInJson(options.unlabelledTestPath()); diff --git a/test/tools/router_check/test/config/Weighted.golden.json b/test/tools/router_check/test/config/Weighted.golden.json index e12cb45bad4d..32e7adc56670 100644 --- a/test/tools/router_check/test/config/Weighted.golden.json +++ b/test/tools/router_check/test/config/Weighted.golden.json @@ -13,10 +13,11 @@ "test_name": "Test_2", "input": { ":authority": "www1.lyft.com", - ":path": "/foo", - "random_value": 115 + ":path": "/test/123", + "random_value": 115, + ":method": "GET" }, - "validate": {"cluster_name": "cluster1"} + "validate": {"cluster_name": "cluster1", "virtual_cluster_name": "test_virtual_cluster"} }, { "test_name": "Test_3", diff --git a/test/tools/router_check/test/config/Weighted.golden.proto.json b/test/tools/router_check/test/config/Weighted.golden.proto.json index 63622d1b4f8d..e48a0446928e 100644 --- a/test/tools/router_check/test/config/Weighted.golden.proto.json +++ b/test/tools/router_check/test/config/Weighted.golden.proto.json @@ -15,11 +15,11 @@ "test_name": "Test_2", "input": { "authority": "www1.lyft.com", - "path": "/foo", + "path": "/test/123", "method": "GET", "random_value": 115 }, - "validate": {"cluster_name": "cluster1"} + "validate": {"cluster_name": "cluster1", "virtual_cluster_name": "test_virtual_cluster"} }, { "test_name": "Test_3", diff --git a/test/tools/router_check/test/config/Weighted.golden.proto.pb_text b/test/tools/router_check/test/config/Weighted.golden.proto.pb_text index fa7dedcb27c5..9a9d0d8f7a8f 100644 --- a/test/tools/router_check/test/config/Weighted.golden.proto.pb_text +++ b/test/tools/router_check/test/config/Weighted.golden.proto.pb_text @@ -16,11 +16,12 @@ tests { test_name: "Test_2" input: { authority: "www1.lyft.com" - path: "/foo" + path: "/test/123" method: "GET" random_value: 115 } validate: { - cluster_name: { value: "cluster1" } + cluster_name: { value: "cluster1"} + virtual_cluster_name: { value: "test_virtual_cluster" } } } \ No newline at end of file diff --git a/test/tools/router_check/test/config/Weighted.golden.proto.yaml b/test/tools/router_check/test/config/Weighted.golden.proto.yaml index 2508111d5272..db59c022dc12 100644 --- a/test/tools/router_check/test/config/Weighted.golden.proto.yaml +++ b/test/tools/router_check/test/config/Weighted.golden.proto.yaml @@ -11,11 +11,12 @@ tests: - test_name: Test_2 input: authority: www1.lyft.com - path: "/foo" + path: "/test/123" method: GET random_value: 115 validate: cluster_name: cluster1 + virtual_cluster_name: test_virtual_cluster - test_name: Test_3 input: authority: www1.lyft.com diff --git a/test/tools/router_check/test/config/Weighted.yaml b/test/tools/router_check/test/config/Weighted.yaml index dd31345021a2..074a24b75b62 100644 --- a/test/tools/router_check/test/config/Weighted.yaml +++ b/test/tools/router_check/test/config/Weighted.yaml @@ -14,6 +14,15 @@ virtual_hosts: weight: 30 - name: cluster3 weight: 40 + virtual_clusters: + - headers: + - name: :path + safe_regex_match: + google_re2: {} + regex: ^/test/\d+$ + - name: :method + exact_match: GET + name: test_virtual_cluster - name: www2 domains: - www2.lyft.com diff --git a/test/tools/router_check/test/route_tests.sh b/test/tools/router_check/test/route_tests.sh index 1e85bea3f598..74a4eeccc0df 100755 --- a/test/tools/router_check/test/route_tests.sh +++ b/test/tools/router_check/test/route_tests.sh @@ -64,10 +64,25 @@ if [[ "${BAD_CONFIG_OUTPUT}" != *"Unable to parse"* ]]; then exit 1 fi -# Failure test case -echo "testing failure test case" +# Failure output flag test cases +echo "testing failure test cases" +# Failure test case with only details flag set FAILURE_OUTPUT=$("${PATH_BIN}" "${PATH_CONFIG}/TestRoutes.yaml" "${PATH_CONFIG}/Weighted.golden.json" "--details" 2>&1) || echo "${FAILURE_OUTPUT:-no-output}" -if [[ "${FAILURE_OUTPUT}" != *"expected: [cluster1], actual: [instant-server], test type: cluster_name"* ]]; then +if [[ "${FAILURE_OUTPUT}" != *"Test_1"*"Test_2"*"expected: [test_virtual_cluster], actual: [other], test type: virtual_cluster_name"*"expected: [cluster1], actual: [instant-server], test type: cluster_name"*"Test_3"* ]]; then + exit 1 +fi + +# Failure test case with details flag set and failures flag set +FAILURE_OUTPUT=$("${PATH_BIN}" "-c" "${PATH_CONFIG}/TestRoutes.yaml" "-t" "${PATH_CONFIG}/Weighted.golden.proto.json" "--details" "--only-show-failures" "--useproto" 2>&1) || + echo "${FAILURE_OUTPUT:-no-output}" +if [[ "${FAILURE_OUTPUT}" != *"Test_2"*"expected: [cluster1], actual: [instant-server], test type: cluster_name"* ]] || [[ "${FAILURE_OUTPUT}" == *"Test_1"* ]]; then + exit 1 +fi + +# Failure test case with details flag unset and failures flag set +FAILURE_OUTPUT=$("${PATH_BIN}" "-c" "${PATH_CONFIG}/TestRoutes.yaml" "-t" "${PATH_CONFIG}/Weighted.golden.proto.json" "--only-show-failures" "--useproto" 2>&1) || + echo "${FAILURE_OUTPUT:-no-output}" +if [[ "${FAILURE_OUTPUT}" != *"Test_2"*"expected: [cluster1], actual: [instant-server], test type: cluster_name"* ]] || [[ "${FAILURE_OUTPUT}" == *"Test_1"* ]]; then exit 1 fi From 0ee3cc37494e8bbdbe16a10d029989d904d3ced3 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Wed, 11 Sep 2019 10:12:05 -0700 Subject: [PATCH 533/542] fix link to runtime docs (#8204) Description: Looks like the runtime docs moved under operations/. The PR fixes the link. Risk Level: low Testing: existing Docs Changes: this Release Notes: n/a Signed-off-by: Derek Argueta --- source/common/protobuf/utility.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/protobuf/utility.cc b/source/common/protobuf/utility.cc index b52a716f321f..f2c94278313f 100644 --- a/source/common/protobuf/utility.cc +++ b/source/common/protobuf/utility.cc @@ -220,7 +220,7 @@ void MessageUtil::checkForUnexpectedFields(const Protobuf::Message& message, } else { const char fatal_error[] = " If continued use of this field is absolutely necessary, see " - "https://www.envoyproxy.io/docs/envoy/latest/configuration/runtime" + "https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime" "#using-runtime-overrides-for-deprecated-features for how to apply a temporary and " "highly discouraged override."; throw ProtoValidationException(err + fatal_error, message); From c5ffdda416e2d4b7a862b92bc596443c20f11a9c Mon Sep 17 00:00:00 2001 From: Xin Date: Wed, 11 Sep 2019 13:46:05 -0400 Subject: [PATCH 534/542] config: make SlotImpl detachable from its owner, and add a new runOnAllThreads interface to Slot. (#8135) See the issue in #7902, this PR is to make the SlotImpl detachable from its owner, by introducing a Booker object wraps around a SlotImpl, which bookkeeps all the on-the-fly update callbacks. And on its destruction, if there are still on-the-fly callbacks, move the SlotImpl to an deferred-delete queue, instead of destructing the SlotImpl which may cause an SEGV error. More importantly, introduce a new runOnAllThreads(ThreadLocal::UpdateCb cb) API to Slot, which requests a Slot Owner to not assume that the Slot or its owner will out-live (in Main thread) the fired on-the-fly update callbacks, and should not capture the Slot or its owner in the update_cb. Picked RDS and config-providers-framework as examples to demonstrate that this change works. {i.e., changed from the runOnAllThreads(Event::PostCb) to the new runOnAllThreads(TLS::UpdateCb) interface. } Risk Level: Medium Testing: unit test Docs Changes: N/A Release Notes: N/A [Optional Fixes #Issue] #7902 Signed-off-by: Xin Zhuang --- include/envoy/thread_local/thread_local.h | 11 +++ source/common/common/non_copyable.h | 13 ++- source/common/config/config_provider_impl.cc | 10 +++ source/common/config/config_provider_impl.h | 13 +-- source/common/router/rds_impl.cc | 8 +- .../common/thread_local/thread_local_impl.cc | 90 ++++++++++++++++++- .../common/thread_local/thread_local_impl.h | 37 +++++++- .../thread_local/thread_local_impl_test.cc | 34 +++++++ test/mocks/thread_local/mocks.h | 8 ++ 9 files changed, 203 insertions(+), 21 deletions(-) diff --git a/include/envoy/thread_local/thread_local.h b/include/envoy/thread_local/thread_local.h index 6f082a4607f5..41c77d730d19 100644 --- a/include/envoy/thread_local/thread_local.h +++ b/include/envoy/thread_local/thread_local.h @@ -74,6 +74,17 @@ class Slot { */ using InitializeCb = std::function; virtual void set(InitializeCb cb) PURE; + + /** + * UpdateCb takes the current stored data, and returns an updated/new version data. + * TLS will run the callback and replace the stored data with the returned value *in each thread*. + * + * NOTE: The update callback is not supposed to capture the Slot, or its owner. As the owner may + * be destructed in main thread before the update_cb gets called in a worker thread. + **/ + using UpdateCb = std::function; + virtual void runOnAllThreads(const UpdateCb& update_cb) PURE; + virtual void runOnAllThreads(const UpdateCb& update_cb, Event::PostCb complete_cb) PURE; }; using SlotPtr = std::unique_ptr; diff --git a/source/common/common/non_copyable.h b/source/common/common/non_copyable.h index c248a37f48e4..fb356770c3f5 100644 --- a/source/common/common/non_copyable.h +++ b/source/common/common/non_copyable.h @@ -2,14 +2,19 @@ namespace Envoy { /** - * Mixin class that makes derived classes not copyable. Like boost::noncopyable without boost. + * Mixin class that makes derived classes not copyable and not moveable. Like boost::noncopyable + * without boost. */ class NonCopyable { protected: NonCopyable() = default; -private: - NonCopyable(const NonCopyable&); - NonCopyable& operator=(const NonCopyable&); + // Non-moveable. + NonCopyable(NonCopyable&&) noexcept = delete; + NonCopyable& operator=(NonCopyable&&) noexcept = delete; + + // Non-copyable. + NonCopyable(const NonCopyable&) = delete; + NonCopyable& operator=(const NonCopyable&) = delete; }; } // namespace Envoy diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index 11cbf993e51c..5745647e2dbf 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -23,6 +23,16 @@ ConfigSubscriptionCommonBase::~ConfigSubscriptionCommonBase() { init_target_.ready(); config_provider_manager_.unbindSubscription(manager_identifier_); } + +void ConfigSubscriptionCommonBase::applyConfigUpdate(const ConfigUpdateCb& update_fn) { + tls_->runOnAllThreads([update_fn](ThreadLocal::ThreadLocalObjectSharedPtr previous) + -> ThreadLocal::ThreadLocalObjectSharedPtr { + auto prev_thread_local_config = std::dynamic_pointer_cast(previous); + prev_thread_local_config->config_ = update_fn(prev_thread_local_config->config_); + return previous; + }); +} + bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, const std::string& config_name, const std::string& version_info) { diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 4a7499c5abdb..e3a608d1ac94 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -216,19 +216,8 @@ class ConfigSubscriptionCommonBase : protected Logger::LoggablerunOnAllThreads( - [this, update_fn]() { - // NOTE: there is a known race condition between *this* subscription being teared down in - // main thread and the posted callback being executed before the destruction. See more - // details in https://github.com/envoyproxy/envoy/issues/7902 - tls_->getTyped().config_ = update_fn(getConfig()); - }, - complete_cb); - } + void applyConfigUpdate(const ConfigUpdateCb& update_fn); void setLastUpdated() { last_updated_ = time_source_.systemTime(); } diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index 64a60062a30c..af180e85738d 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -199,8 +199,12 @@ Router::ConfigConstSharedPtr RdsRouteConfigProviderImpl::config() { void RdsRouteConfigProviderImpl::onConfigUpdate() { ConfigConstSharedPtr new_config( new ConfigImpl(config_update_info_->routeConfiguration(), factory_context_, false)); - tls_->runOnAllThreads( - [this, new_config]() -> void { tls_->getTyped().config_ = new_config; }); + tls_->runOnAllThreads([new_config](ThreadLocal::ThreadLocalObjectSharedPtr previous) + -> ThreadLocal::ThreadLocalObjectSharedPtr { + auto prev_config = std::dynamic_pointer_cast(previous); + prev_config->config_ = new_config; + return previous; + }); } void RdsRouteConfigProviderImpl::validateConfig( diff --git a/source/common/thread_local/thread_local_impl.cc b/source/common/thread_local/thread_local_impl.cc index 9781db07797b..5d9f584b517e 100644 --- a/source/common/thread_local/thread_local_impl.cc +++ b/source/common/thread_local/thread_local_impl.cc @@ -27,8 +27,9 @@ SlotPtr InstanceImpl::allocateSlot() { if (free_slot_indexes_.empty()) { std::unique_ptr slot(new SlotImpl(*this, slots_.size())); - slots_.push_back(slot.get()); - return slot; + auto wrapper = std::make_unique(*this, std::move(slot)); + slots_.push_back(wrapper->slot_.get()); + return wrapper; } const uint32_t idx = free_slot_indexes_.front(); free_slot_indexes_.pop_front(); @@ -42,11 +43,64 @@ bool InstanceImpl::SlotImpl::currentThreadRegistered() { return thread_local_data_.data_.size() > index_; } +void InstanceImpl::SlotImpl::runOnAllThreads(const UpdateCb& cb) { + parent_.runOnAllThreads([this, cb]() { setThreadLocal(index_, cb(get())); }); +} + +void InstanceImpl::SlotImpl::runOnAllThreads(const UpdateCb& cb, Event::PostCb complete_cb) { + parent_.runOnAllThreads([this, cb]() { setThreadLocal(index_, cb(get())); }, complete_cb); +} + ThreadLocalObjectSharedPtr InstanceImpl::SlotImpl::get() { ASSERT(currentThreadRegistered()); return thread_local_data_.data_[index_]; } +InstanceImpl::Bookkeeper::Bookkeeper(InstanceImpl& parent, std::unique_ptr&& slot) + : parent_(parent), slot_(std::move(slot)), + ref_count_(/*not used.*/ nullptr, + [slot = slot_.get(), &parent = this->parent_](uint32_t* /* not used */) { + // On destruction, post a cleanup callback on main thread, this could happen on + // any thread. + parent.scheduleCleanup(slot); + }) {} + +ThreadLocalObjectSharedPtr InstanceImpl::Bookkeeper::get() { return slot_->get(); } + +void InstanceImpl::Bookkeeper::runOnAllThreads(const UpdateCb& cb, Event::PostCb complete_cb) { + slot_->runOnAllThreads( + [cb, ref_count = this->ref_count_](ThreadLocalObjectSharedPtr previous) { + return cb(std::move(previous)); + }, + complete_cb); +} + +void InstanceImpl::Bookkeeper::runOnAllThreads(const UpdateCb& cb) { + slot_->runOnAllThreads([cb, ref_count = this->ref_count_](ThreadLocalObjectSharedPtr previous) { + return cb(std::move(previous)); + }); +} + +bool InstanceImpl::Bookkeeper::currentThreadRegistered() { + return slot_->currentThreadRegistered(); +} + +void InstanceImpl::Bookkeeper::runOnAllThreads(Event::PostCb cb) { + // Use ref_count_ to bookkeep how many on-the-fly callback are out there. + slot_->runOnAllThreads([cb, ref_count = this->ref_count_]() { cb(); }); +} + +void InstanceImpl::Bookkeeper::runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback) { + // Use ref_count_ to bookkeep how many on-the-fly callback are out there. + slot_->runOnAllThreads([cb, main_callback, ref_count = this->ref_count_]() { cb(); }, + main_callback); +} + +void InstanceImpl::Bookkeeper::set(InitializeCb cb) { + slot_->set([cb, ref_count = this->ref_count_](Event::Dispatcher& dispatcher) + -> ThreadLocalObjectSharedPtr { return cb(dispatcher); }); +} + void InstanceImpl::registerThread(Event::Dispatcher& dispatcher, bool main_thread) { ASSERT(std::this_thread::get_id() == main_thread_id_); ASSERT(!shutdown_); @@ -61,6 +115,38 @@ void InstanceImpl::registerThread(Event::Dispatcher& dispatcher, bool main_threa } } +// Puts the slot into a deferred delete container, the slot will be destructed when its out-going +// callback reference count goes to 0. +void InstanceImpl::recycle(std::unique_ptr&& slot) { + ASSERT(std::this_thread::get_id() == main_thread_id_); + ASSERT(slot != nullptr); + auto* slot_addr = slot.get(); + deferred_deletes_.insert({slot_addr, std::move(slot)}); +} + +// Called by the Bookkeeper ref_count destructor, the SlotImpl in the deferred deletes map can be +// destructed now. +void InstanceImpl::scheduleCleanup(SlotImpl* slot) { + if (shutdown_) { + // If server is shutting down, do nothing here. + // The destruction of Bookkeeper has already transferred the SlotImpl to the deferred_deletes_ + // queue. No matter if this method is called from a Worker thread, the SlotImpl will be + // destructed on main thread when InstanceImpl destructs. + return; + } + if (std::this_thread::get_id() == main_thread_id_) { + // If called from main thread, save a callback. + ASSERT(deferred_deletes_.contains(slot)); + deferred_deletes_.erase(slot); + return; + } + main_thread_dispatcher_->post([slot, this]() { + ASSERT(deferred_deletes_.contains(slot)); + // The slot is guaranteed to be put into the deferred_deletes_ map by Bookkeeper destructor. + deferred_deletes_.erase(slot); + }); +} + void InstanceImpl::removeSlot(SlotImpl& slot) { ASSERT(std::this_thread::get_id() == main_thread_id_); diff --git a/source/common/thread_local/thread_local_impl.h b/source/common/thread_local/thread_local_impl.h index 39a0f12a3e4f..49f1889e44d7 100644 --- a/source/common/thread_local/thread_local_impl.h +++ b/source/common/thread_local/thread_local_impl.h @@ -8,6 +8,9 @@ #include "envoy/thread_local/thread_local.h" #include "common/common/logger.h" +#include "common/common/non_copyable.h" + +#include "absl/container/flat_hash_map.h" namespace Envoy { namespace ThreadLocal { @@ -15,7 +18,7 @@ namespace ThreadLocal { /** * Implementation of ThreadLocal that relies on static thread_local objects. */ -class InstanceImpl : Logger::Loggable, public Instance { +class InstanceImpl : Logger::Loggable, public NonCopyable, public Instance { public: InstanceImpl() : main_thread_id_(std::this_thread::get_id()) {} ~InstanceImpl() override; @@ -35,6 +38,8 @@ class InstanceImpl : Logger::Loggable, public Instance { // ThreadLocal::Slot ThreadLocalObjectSharedPtr get() override; bool currentThreadRegistered() override; + void runOnAllThreads(const UpdateCb& cb) override; + void runOnAllThreads(const UpdateCb& cb, Event::PostCb complete_cb) override; void runOnAllThreads(Event::PostCb cb) override { parent_.runOnAllThreads(cb); } void runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback) override { parent_.runOnAllThreads(cb, main_callback); @@ -45,17 +50,47 @@ class InstanceImpl : Logger::Loggable, public Instance { const uint64_t index_; }; + // A Wrapper of SlotImpl which on destruction returns the SlotImpl to the deferred delete queue + // (detaches it). + struct Bookkeeper : public Slot { + Bookkeeper(InstanceImpl& parent, std::unique_ptr&& slot); + ~Bookkeeper() override { parent_.recycle(std::move(slot_)); } + + // ThreadLocal::Slot + ThreadLocalObjectSharedPtr get() override; + void runOnAllThreads(const UpdateCb& cb) override; + void runOnAllThreads(const UpdateCb& cb, Event::PostCb complete_cb) override; + bool currentThreadRegistered() override; + void runOnAllThreads(Event::PostCb cb) override; + void runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback) override; + void set(InitializeCb cb) override; + + InstanceImpl& parent_; + std::unique_ptr slot_; + std::shared_ptr ref_count_; + }; + struct ThreadLocalData { Event::Dispatcher* dispatcher_{}; std::vector data_; }; + void recycle(std::unique_ptr&& slot); + // Cleanup the deferred deletes queue. + void scheduleCleanup(SlotImpl* slot); + void removeSlot(SlotImpl& slot); void runOnAllThreads(Event::PostCb cb); void runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback); static void setThreadLocal(uint32_t index, ThreadLocalObjectSharedPtr object); static thread_local ThreadLocalData thread_local_data_; + + // A indexed container for Slots that has to be deferred to delete due to out-going callbacks + // pointing to the Slot. To let the ref_count_ deleter find the SlotImpl by address, the container + // is defined as a map of SlotImpl address to the unique_ptr. + absl::flat_hash_map> deferred_deletes_; + std::vector slots_; // A list of index of freed slots. std::list free_slot_indexes_; diff --git a/test/common/thread_local/thread_local_impl_test.cc b/test/common/thread_local/thread_local_impl_test.cc index 5a678c839384..fb3bd1cf3962 100644 --- a/test/common/thread_local/thread_local_impl_test.cc +++ b/test/common/thread_local/thread_local_impl_test.cc @@ -85,6 +85,40 @@ TEST_F(ThreadLocalInstanceImplTest, All) { tls_.shutdownThread(); } +// Test that the config passed into the update callback is the previous version stored in the slot. +TEST_F(ThreadLocalInstanceImplTest, UpdateCallback) { + InSequence s; + + SlotPtr slot = tls_.allocateSlot(); + + auto newer_version = std::make_shared(); + bool update_called = false; + + TestThreadLocalObject& object_ref = setObject(*slot); + auto update_cb = [&object_ref, &update_called, + newer_version](ThreadLocalObjectSharedPtr obj) -> ThreadLocalObjectSharedPtr { + // The unit test setup have two dispatchers registered, but only one thread, this lambda will be + // called twice in the same thread. + if (!update_called) { + EXPECT_EQ(obj.get(), &object_ref); + update_called = true; + } else { + EXPECT_EQ(obj.get(), newer_version.get()); + } + + return newer_version; + }; + EXPECT_CALL(thread_dispatcher_, post(_)); + EXPECT_CALL(object_ref, onDestroy()); + EXPECT_CALL(*newer_version, onDestroy()); + slot->runOnAllThreads(update_cb); + + EXPECT_EQ(newer_version.get(), &slot->getTyped()); + + tls_.shutdownGlobalThreading(); + tls_.shutdownThread(); +} + // TODO(ramaraochavali): Run this test with real threads. The current issue in the unit // testing environment is, the post to main_dispatcher is not working as expected. diff --git a/test/mocks/thread_local/mocks.h b/test/mocks/thread_local/mocks.h index 3d7a43efaef8..a9abc6a6d562 100644 --- a/test/mocks/thread_local/mocks.h +++ b/test/mocks/thread_local/mocks.h @@ -63,6 +63,14 @@ class MockInstance : public Instance { void runOnAllThreads(Event::PostCb cb, Event::PostCb main_callback) override { parent_.runOnAllThreads(cb, main_callback); } + void runOnAllThreads(const UpdateCb& cb) override { + parent_.runOnAllThreads([cb, this]() { parent_.data_[index_] = cb(parent_.data_[index_]); }); + } + void runOnAllThreads(const UpdateCb& cb, Event::PostCb main_callback) override { + parent_.runOnAllThreads([cb, this]() { parent_.data_[index_] = cb(parent_.data_[index_]); }, + main_callback); + } + void set(InitializeCb cb) override { parent_.data_[index_] = cb(parent_.dispatcher_); } MockInstance& parent_; From da38e73ab04a0399df25856a6e05560bb86fca76 Mon Sep 17 00:00:00 2001 From: Stephan Zuercher Date: Wed, 11 Sep 2019 11:07:27 -0700 Subject: [PATCH 535/542] test: remove static config from subset lb integration test (#8203) Build the config programmatically to make future API changes less onerous. Risk Level: low (test change only) Testing: n/a Doc Changes: n/a Release Notes: n/a Signed-off-by: Stephan Zuercher --- .../http_subset_lb_integration_test.cc | 236 ++++++++---------- 1 file changed, 102 insertions(+), 134 deletions(-) diff --git a/test/integration/http_subset_lb_integration_test.cc b/test/integration/http_subset_lb_integration_test.cc index 427f3f5a3c64..10b882feef95 100644 --- a/test/integration/http_subset_lb_integration_test.cc +++ b/test/integration/http_subset_lb_integration_test.cc @@ -4,120 +4,7 @@ #include "gtest/gtest.h" namespace Envoy { -namespace { - -const std::string SUBSET_CONFIG = R"EOF( -admin: - access_log_path: /dev/null - address: - socket_address: - address: 127.0.0.1 - port_value: 0 -static_resources: - clusters: - name: cluster_0 - lb_policy: round_robin - lb_subset_config: - subset_selectors: - - keys: [ "type" ] - load_assignment: - cluster_name: cluster_0 - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 0 - metadata: - filter_metadata: - "envoy.lb": { "type": "a" } - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 0 - metadata: - filter_metadata: - "envoy.lb": { "type": "a" } - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 0 - metadata: - filter_metadata: - "envoy.lb": { "type": "b" } - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 0 - metadata: - filter_metadata: - "envoy.lb": { "type": "b" } - listeners: - name: listener_0 - address: - socket_address: - address: 127.0.0.1 - port_value: 0 - filter_chains: - filters: - name: envoy.http_connection_manager - config: - stat_prefix: config_test - http_filters: - name: envoy.router - codec_type: HTTP1 - access_log: - name: envoy.file_access_log - filter: - not_health_check_filter: {} - config: - path: /dev/null - route_config: - name: route_config_0 - virtual_hosts: - name: integration - domains: "*" - routes: - - match: - prefix: "/" - headers: - - name: "x-type" - exact_match: "a" - route: - cluster: cluster_0 - metadata_match: - filter_metadata: - "envoy.lb": { "type": "a" } - hash_policy: - - header: - header_name: "x-hash" - - match: - prefix: "/" - headers: - - name: "x-type" - exact_match: "b" - route: - cluster: cluster_0 - metadata_match: - filter_metadata: - "envoy.lb": { "type": "b" } - hash_policy: - - header: - header_name: "x-hash" - response_headers_to_add: - - header: - key: "x-host-type" - value: '%UPSTREAM_METADATA(["envoy.lb", "type"])%' - - header: - key: "x-host" - value: '%UPSTREAM_REMOTE_ADDRESS%' -)EOF"; - -} // namespace + class HttpSubsetLbIntegrationTest : public testing::TestWithParam, public HttpIntegrationTest { public: @@ -156,29 +43,101 @@ class HttpSubsetLbIntegrationTest : public testing::TestWithParamclusters_size(); ++i) { - auto* cluster = static_resources->mutable_clusters(i); - cluster->set_lb_policy(GetParam()); + auto* cluster = static_resources->mutable_clusters(0); + + cluster->set_lb_policy(GetParam()); + + // Create subsets based on type value of the "type" metadata. + cluster->mutable_lb_subset_config()->add_subset_selectors()->add_keys(type_key_); + + cluster->clear_hosts(); + + // Create a load assignment with num_hosts_ entries with metadata split evenly between type=a + // and type=b. + auto* load_assignment = cluster->mutable_load_assignment(); + load_assignment->set_cluster_name(cluster->name()); + auto* endpoints = load_assignment->add_endpoints(); + for (uint32_t i = 0; i < num_hosts_; i++) { + auto* lb_endpoint = endpoints->add_lb_endpoints(); + + // ConfigHelper will fill in ports later. + auto* endpoint = lb_endpoint->mutable_endpoint(); + auto* addr = endpoint->mutable_address()->mutable_socket_address(); + addr->set_address("127.0.0.1"); + addr->set_port_value(0); + + // Assign type metadata based on i. + auto* metadata = lb_endpoint->mutable_metadata(); + Envoy::Config::Metadata::mutableMetadataValue(*metadata, "envoy.lb", type_key_) + .set_string_value((i % 2 == 0) ? "a" : "b"); } }); + + config_helper_.addConfigModifier( + [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + hcm) { + auto* vhost = hcm.mutable_route_config()->mutable_virtual_hosts(0); + + // Report the host's type metadata and remote address on every response. + auto* resp_header = vhost->add_response_headers_to_add(); + auto* header = resp_header->mutable_header(); + header->set_key(host_type_header_); + header->set_value( + fmt::format(R"EOF(%UPSTREAM_METADATA(["envoy.lb", "{}"])%)EOF", type_key_)); + + resp_header = vhost->add_response_headers_to_add(); + header = resp_header->mutable_header(); + header->set_key(host_header_); + header->set_value("%UPSTREAM_REMOTE_ADDRESS%"); + + // Create routes for x-type=a and x-type=b headers. + vhost->clear_routes(); + configureRoute(vhost->add_routes(), "a"); + configureRoute(vhost->add_routes(), "b"); + }); } + void configureRoute(envoy::api::v2::route::Route* route, const std::string& host_type) { + auto* match = route->mutable_match(); + match->set_prefix("/"); + + // Match the x-type header against the given host_type (a/b). + auto* match_header = match->add_headers(); + match_header->set_name(type_header_); + match_header->set_exact_match(host_type); + + // Route to cluster_0, selecting metadata type=a or type=b. + auto* action = route->mutable_route(); + action->set_cluster("cluster_0"); + auto* metadata_match = action->mutable_metadata_match(); + Envoy::Config::Metadata::mutableMetadataValue(*metadata_match, "envoy.lb", type_key_) + .set_string_value(host_type); + + // Set a hash policy for hashing load balancers. + if (is_hash_lb_) { + action->add_hash_policy()->mutable_header()->set_header_name(hash_header_); + } + }; + void SetUp() override { setDownstreamProtocol(Http::CodecClient::Type::HTTP1); setUpstreamProtocol(FakeHttpConnection::Type::HTTP1); } // Runs a subset lb test with the given request headers, expecting the x-host-type header to - // the given type ("a" or "b"). If expect_unique_host, verifies that a single host is selected - // over n iterations (e.g. for maglev/hash-ring policies). Otherwise, expected more than one - // host to be selected over n iterations. + // the given type ("a" or "b"). If is_hash_lb_, verifies that a single host is selected over n + // iterations (e.g. for maglev/hash-ring policies). Otherwise, expected more than one host to be + // selected over n iterations. void runTest(Http::TestHeaderMapImpl& request_headers, const std::string expected_host_type, - const bool expect_unique_host, const int n = 10) { + const int n = 10) { std::set hosts; for (int i = 0; i < n; i++) { Http::TestHeaderMapImpl response_headers{{":status", "200"}}; @@ -189,24 +148,36 @@ class HttpSubsetLbIntegrationTest : public testing::TestWithParamheaders() - .get(Envoy::Http::LowerCaseString{"x-host-type"}) + .get(Envoy::Http::LowerCaseString{host_type_header_}) ->value() .getStringView(), expected_host_type); - hosts.emplace( - response->headers().get(Envoy::Http::LowerCaseString{"x-host"})->value().getStringView()); + // Record the upstream address. + hosts.emplace(response->headers() + .get(Envoy::Http::LowerCaseString{host_header_}) + ->value() + .getStringView()); } - if (expect_unique_host) { + if (is_hash_lb_) { EXPECT_EQ(hosts.size(), 1) << "Expected a single unique host to be selected for " << envoy::api::v2::Cluster_LbPolicy_Name(GetParam()); } else { - EXPECT_GT(hosts.size(), 1) << "Expected multiple hosts to be selected" + EXPECT_GT(hosts.size(), 1) << "Expected multiple hosts to be selected for " << envoy::api::v2::Cluster_LbPolicy_Name(GetParam()); } } + const uint32_t num_hosts_; + const bool is_hash_lb_; + + const std::string hash_header_{"x-hash"}; + const std::string host_type_header_{"x-host-type"}; + const std::string host_header_{"x-host"}; + const std::string type_header_{"x-type"}; + const std::string type_key_{"type"}; + Http::TestHeaderMapImpl type_a_request_headers_{{":method", "GET"}, {":path", "/test"}, {":scheme", "http"}, {":authority", "host"}, {"x-type", "a"}, {"x-hash", "hash-a"}}; @@ -224,11 +195,8 @@ TEST_P(HttpSubsetLbIntegrationTest, SubsetLoadBalancer) { initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); - const bool expect_unique_host = (GetParam() == envoy::api::v2::Cluster_LbPolicy_RING_HASH || - GetParam() == envoy::api::v2::Cluster_LbPolicy_MAGLEV); - - runTest(type_a_request_headers_, "a", expect_unique_host); - runTest(type_b_request_headers_, "b", expect_unique_host); + runTest(type_a_request_headers_, "a"); + runTest(type_b_request_headers_, "b"); } } // namespace Envoy From 9de651a4de6ca162d0c7fbd596b3b138c99ce05f Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Wed, 11 Sep 2019 12:28:59 -0700 Subject: [PATCH 536/542] cleanup: clarify Cluster.filters and Dispatcher::createClientConnection (#8186) Signed-off-by: Fred Douglas --- api/envoy/api/v2/cds.proto | 6 +++--- api/envoy/api/v3alpha/cds.proto | 6 +++--- include/envoy/event/dispatcher.h | 29 +++++++++++++++-------------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index b2b71384f8c8..85e9a3827c3c 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -647,9 +647,9 @@ message Cluster { // checking before removing it from the cluster. bool drain_connections_on_host_removal = 32; - // An optional list of network filters that make up the filter chain for - // outgoing connections made by the cluster. Order matters as the filters are - // processed sequentially as connection events happen. + // An (optional) network filter chain, listed in the order the filters should be applied. + // The chain will be applied to all outgoing connections that Envoy makes to the upstream + // servers of this cluster. repeated cluster.Filter filters = 40; // [#not-implemented-hide:] New mechanism for LB policy configuration. Used only if the diff --git a/api/envoy/api/v3alpha/cds.proto b/api/envoy/api/v3alpha/cds.proto index c1c18b110be0..506224a7ba46 100644 --- a/api/envoy/api/v3alpha/cds.proto +++ b/api/envoy/api/v3alpha/cds.proto @@ -634,9 +634,9 @@ message Cluster { // checking before removing it from the cluster. bool drain_connections_on_host_removal = 32; - // An optional list of network filters that make up the filter chain for - // outgoing connections made by the cluster. Order matters as the filters are - // processed sequentially as connection events happen. + // An (optional) network filter chain, listed in the order the filters should be applied. + // The chain will be applied to all outgoing connections that Envoy makes to the upstream + // servers of this cluster. repeated cluster.Filter filters = 40; } diff --git a/include/envoy/event/dispatcher.h b/include/envoy/event/dispatcher.h index 8c3a8b8a88b7..b9ae70960df2 100644 --- a/include/envoy/event/dispatcher.h +++ b/include/envoy/event/dispatcher.h @@ -59,7 +59,7 @@ class Dispatcher { virtual TimeSource& timeSource() PURE; /** - * Initialize stats for this dispatcher. Note that this can't generally be done at construction + * Initializes stats for this dispatcher. Note that this can't generally be done at construction * time, since the main and worker thread dispatchers are constructed before * ThreadLocalStoreImpl::initializeThreading. * @param scope the scope to contain the new per-dispatcher stats created here. @@ -68,12 +68,12 @@ class Dispatcher { virtual void initializeStats(Stats::Scope& scope, const std::string& prefix) PURE; /** - * Clear any items in the deferred deletion queue. + * Clears any items in the deferred deletion queue. */ virtual void clearDeferredDeleteList() PURE; /** - * Create a server connection. + * Wraps an already-accepted socket in an instance of Envoy's server Network::Connection. * @param socket supplies an open file descriptor and connection metadata to use for the * connection. Takes ownership of the socket. * @param transport_socket supplies a transport socket to be used by the connection. @@ -84,7 +84,8 @@ class Dispatcher { Network::TransportSocketPtr&& transport_socket) PURE; /** - * Create a client connection. + * Creates an instance of Envoy's Network::ClientConnection. Does NOT initiate the connection; + * the caller must then call connect() on the returned Network::ClientConnection. * @param address supplies the address to connect to. * @param source_address supplies an address to bind to or nullptr if no bind is necessary. * @param transport_socket supplies a transport socket to be used by the connection. @@ -99,7 +100,7 @@ class Dispatcher { const Network::ConnectionSocket::OptionsSharedPtr& options) PURE; /** - * Create an async DNS resolver. The resolver should only be used on the thread that runs this + * Creates an async DNS resolver. The resolver should only be used on the thread that runs this * dispatcher. * @param resolvers supplies the addresses of DNS resolvers that this resolver should use. If left * empty, it will not use any specific resolvers, but use defaults (/etc/resolv.conf) @@ -109,7 +110,7 @@ class Dispatcher { createDnsResolver(const std::vector& resolvers) PURE; /** - * Create a file event that will signal when a file is readable or writable. On UNIX systems this + * Creates a file event that will signal when a file is readable or writable. On UNIX systems this * can be used for any file like interface (files, sockets, etc.). * @param fd supplies the fd to watch. * @param cb supplies the callback to fire when the file is ready. @@ -126,7 +127,7 @@ class Dispatcher { virtual Filesystem::WatcherPtr createFilesystemWatcher() PURE; /** - * Create a listener on a specific port. + * Creates a listener on a specific port. * @param socket supplies the socket to listen on. * @param cb supplies the callbacks to invoke for listener events. * @param bind_to_port controls whether the listener binds to a transport port or not. @@ -139,7 +140,7 @@ class Dispatcher { bool hand_off_restored_destination_connections) PURE; /** - * Create a logical udp listener on a specific port. + * Creates a logical udp listener on a specific port. * @param socket supplies the socket to listen on. * @param cb supplies the udp listener callbacks to invoke for listener events. * @return Network::ListenerPtr a new listener that is owned by the caller. @@ -147,23 +148,23 @@ class Dispatcher { virtual Network::ListenerPtr createUdpListener(Network::Socket& socket, Network::UdpListenerCallbacks& cb) PURE; /** - * Allocate a timer. @see Timer for docs on how to use the timer. + * Allocates a timer. @see Timer for docs on how to use the timer. * @param cb supplies the callback to invoke when the timer fires. */ virtual Event::TimerPtr createTimer(TimerCb cb) PURE; /** - * Submit an item for deferred delete. @see DeferredDeletable. + * Submits an item for deferred delete. @see DeferredDeletable. */ virtual void deferredDelete(DeferredDeletablePtr&& to_delete) PURE; /** - * Exit the event loop. + * Exits the event loop. */ virtual void exit() PURE; /** - * Listen for a signal event. Only a single dispatcher in the process can listen for signals. + * Listens for a signal event. Only a single dispatcher in the process can listen for signals. * If more than one dispatcher calls this routine in the process the behavior is undefined. * * @param signal_num supplies the signal to listen on. @@ -173,13 +174,13 @@ class Dispatcher { virtual SignalEventPtr listenForSignal(int signal_num, SignalCb cb) PURE; /** - * Post a functor to the dispatcher. This is safe cross thread. The functor runs in the context + * Posts a functor to the dispatcher. This is safe cross thread. The functor runs in the context * of the dispatcher event loop which may be on a different thread than the caller. */ virtual void post(PostCb callback) PURE; /** - * Run the event loop. This will not return until exit() is called either from within a callback + * Runs the event loop. This will not return until exit() is called either from within a callback * or from a different thread. * @param type specifies whether to run in blocking mode (run() will not return until exit() is * called) or non-blocking mode where only active events will be executed and then From 7df47754b4e4d85d6487f61b9ae528e1d2f1c211 Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Wed, 11 Sep 2019 13:34:36 -0700 Subject: [PATCH 537/542] redis: health check is not sending the auth command on its connection (#8166) Signed-off-by: Henry Yang --- docs/root/intro/version_history.rst | 1 + include/envoy/server/health_checker_config.h | 5 + .../common/upstream/cluster_factory_impl.cc | 3 +- source/common/upstream/health_checker_impl.cc | 21 ++-- source/common/upstream/health_checker_impl.h | 12 +-- .../upstream/health_discovery_service.cc | 10 +- .../upstream/health_discovery_service.h | 3 +- .../clusters/redis/redis_cluster.cc | 20 +--- .../extensions/clusters/redis/redis_cluster.h | 11 ++- .../filters/network/common/redis/BUILD | 1 + .../filters/network/common/redis/client.h | 11 ++- .../network/common/redis/client_impl.cc | 24 ++++- .../network/common/redis/client_impl.h | 4 +- .../filters/network/redis_proxy/BUILD | 1 + .../filters/network/redis_proxy/config.h | 13 +++ .../network/redis_proxy/conn_pool_impl.cc | 27 +----- source/extensions/health_checkers/redis/BUILD | 1 + .../health_checkers/redis/config.cc | 2 +- .../extensions/health_checkers/redis/redis.cc | 12 ++- .../extensions/health_checkers/redis/redis.h | 5 +- test/common/upstream/BUILD | 1 + .../upstream/health_checker_impl_test.cc | 15 +-- .../clusters/redis/redis_cluster_test.cc | 2 +- .../network/common/redis/client_impl_test.cc | 73 +++++++++++++- .../filters/network/common/redis/mocks.h | 1 + .../redis_proxy/conn_pool_impl_test.cc | 97 +------------------ test/extensions/health_checkers/redis/BUILD | 2 + .../health_checkers/redis/config_test.cc | 14 +-- .../health_checkers/redis/redis_test.cc | 31 +++--- test/mocks/server/mocks.cc | 1 + test/mocks/server/mocks.h | 2 + 31 files changed, 224 insertions(+), 202 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 34dbfca93522..82d08ed8a443 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -41,6 +41,7 @@ Version history * rbac: added support for DNS SAN as :ref:`principal_name `. * redis: added :ref:`enable_command_stats ` to enable :ref:`per command statistics ` for upstream clusters. * redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. +* redis: fix a bug where the redis health checker ignored the upstream auth password. * regex: introduce new :ref:`RegexMatcher ` type that provides a safe regex implementation for untrusted user input. This type is now used in all configuration that processes user provided input. See :ref:`deprecated configuration details diff --git a/include/envoy/server/health_checker_config.h b/include/envoy/server/health_checker_config.h index 163a1d7c2801..638a0362adcc 100644 --- a/include/envoy/server/health_checker_config.h +++ b/include/envoy/server/health_checker_config.h @@ -43,6 +43,11 @@ class HealthCheckerFactoryContext { * messages. */ virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; + + /** + * @return Api::Api& the API used by the server. + */ + virtual Api::Api& api() PURE; }; /** diff --git a/source/common/upstream/cluster_factory_impl.cc b/source/common/upstream/cluster_factory_impl.cc index 43c6d5581977..bebf10b93326 100644 --- a/source/common/upstream/cluster_factory_impl.cc +++ b/source/common/upstream/cluster_factory_impl.cc @@ -109,7 +109,8 @@ ClusterFactoryImplBase::create(const envoy::api::v2::Cluster& cluster, } else { new_cluster_pair.first->setHealthChecker(HealthCheckerFactory::create( cluster.health_checks()[0], *new_cluster_pair.first, context.runtime(), context.random(), - context.dispatcher(), context.logManager(), context.messageValidationVisitor())); + context.dispatcher(), context.logManager(), context.messageValidationVisitor(), + context.api())); } } diff --git a/source/common/upstream/health_checker_impl.cc b/source/common/upstream/health_checker_impl.cc index 708f5e132981..ddd52756eaee 100644 --- a/source/common/upstream/health_checker_impl.cc +++ b/source/common/upstream/health_checker_impl.cc @@ -29,9 +29,11 @@ class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthChec Envoy::Runtime::RandomGenerator& random, Event::Dispatcher& dispatcher, HealthCheckEventLoggerPtr&& event_logger, - ProtobufMessage::ValidationVisitor& validation_visitor) + ProtobufMessage::ValidationVisitor& validation_visitor, + Api::Api& api) : cluster_(cluster), runtime_(runtime), random_(random), dispatcher_(dispatcher), - event_logger_(std::move(event_logger)), validation_visitor_(validation_visitor) {} + event_logger_(std::move(event_logger)), validation_visitor_(validation_visitor), api_(api) { + } Upstream::Cluster& cluster() override { return cluster_; } Envoy::Runtime::Loader& runtime() override { return runtime_; } Envoy::Runtime::RandomGenerator& random() override { return random_; } @@ -40,6 +42,7 @@ class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthChec ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { return validation_visitor_; } + Api::Api& api() override { return api_; } private: Upstream::Cluster& cluster_; @@ -48,14 +51,14 @@ class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthChec Event::Dispatcher& dispatcher_; HealthCheckEventLoggerPtr event_logger_; ProtobufMessage::ValidationVisitor& validation_visitor_; + Api::Api& api_; }; -HealthCheckerSharedPtr -HealthCheckerFactory::create(const envoy::api::v2::core::HealthCheck& health_check_config, - Upstream::Cluster& cluster, Runtime::Loader& runtime, - Runtime::RandomGenerator& random, Event::Dispatcher& dispatcher, - AccessLog::AccessLogManager& log_manager, - ProtobufMessage::ValidationVisitor& validation_visitor) { +HealthCheckerSharedPtr HealthCheckerFactory::create( + const envoy::api::v2::core::HealthCheck& health_check_config, Upstream::Cluster& cluster, + Runtime::Loader& runtime, Runtime::RandomGenerator& random, Event::Dispatcher& dispatcher, + AccessLog::AccessLogManager& log_manager, + ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api) { HealthCheckEventLoggerPtr event_logger; if (!health_check_config.event_log_path().empty()) { event_logger = std::make_unique( @@ -81,7 +84,7 @@ HealthCheckerFactory::create(const envoy::api::v2::core::HealthCheck& health_che health_check_config.custom_health_check().name()); std::unique_ptr context( new HealthCheckerFactoryContextImpl(cluster, runtime, random, dispatcher, - std::move(event_logger), validation_visitor)); + std::move(event_logger), validation_visitor, api)); return factory.createCustomHealthChecker(health_check_config, *context); } default: diff --git a/source/common/upstream/health_checker_impl.h b/source/common/upstream/health_checker_impl.h index 6ca106d947b9..c16a4393df24 100644 --- a/source/common/upstream/health_checker_impl.h +++ b/source/common/upstream/health_checker_impl.h @@ -11,6 +11,7 @@ #include "common/stream_info/stream_info_impl.h" #include "common/upstream/health_checker_base_impl.h" +#include "include/envoy/api/_virtual_includes/api_interface/envoy/api/api.h" #include "src/proto/grpc/health/v1/health.pb.h" namespace Envoy { @@ -32,12 +33,11 @@ class HealthCheckerFactory : public Logger::Loggable * @param validation_visitor message validation visitor instance. * @return a health checker. */ - static HealthCheckerSharedPtr create(const envoy::api::v2::core::HealthCheck& health_check_config, - Upstream::Cluster& cluster, Runtime::Loader& runtime, - Runtime::RandomGenerator& random, - Event::Dispatcher& dispatcher, - AccessLog::AccessLogManager& log_manager, - ProtobufMessage::ValidationVisitor& validation_visitor); + static HealthCheckerSharedPtr + create(const envoy::api::v2::core::HealthCheck& health_check_config, Upstream::Cluster& cluster, + Runtime::Loader& runtime, Runtime::RandomGenerator& random, Event::Dispatcher& dispatcher, + AccessLog::AccessLogManager& log_manager, + ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api); }; /** diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index 8fb6f44b1f15..894be7a4c9b6 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -152,7 +152,8 @@ void HdsDelegate::processMessage( info_factory_, cm_, local_info_, dispatcher_, random_, singleton_manager_, tls_, validation_visitor_, api_)); - hds_clusters_.back()->startHealthchecks(access_log_manager_, runtime_, random_, dispatcher_); + hds_clusters_.back()->startHealthchecks(access_log_manager_, runtime_, random_, dispatcher_, + api_); } } @@ -243,10 +244,11 @@ ProdClusterInfoFactory::createClusterInfo(const CreateClusterInfoParams& params) void HdsCluster::startHealthchecks(AccessLog::AccessLogManager& access_log_manager, Runtime::Loader& runtime, Runtime::RandomGenerator& random, - Event::Dispatcher& dispatcher) { + Event::Dispatcher& dispatcher, Api::Api& api) { for (auto& health_check : cluster_.health_checks()) { - health_checkers_.push_back(Upstream::HealthCheckerFactory::create( - health_check, *this, runtime, random, dispatcher, access_log_manager, validation_visitor_)); + health_checkers_.push_back( + Upstream::HealthCheckerFactory::create(health_check, *this, runtime, random, dispatcher, + access_log_manager, validation_visitor_, api)); health_checkers_.back()->start(); } } diff --git a/source/common/upstream/health_discovery_service.h b/source/common/upstream/health_discovery_service.h index b1c889a30c23..250b915896b4 100644 --- a/source/common/upstream/health_discovery_service.h +++ b/source/common/upstream/health_discovery_service.h @@ -59,7 +59,8 @@ class HdsCluster : public Cluster, Logger::Loggable { // Creates and starts healthcheckers to its endpoints void startHealthchecks(AccessLog::AccessLogManager& access_log_manager, Runtime::Loader& runtime, - Runtime::RandomGenerator& random, Event::Dispatcher& dispatcher); + Runtime::RandomGenerator& random, Event::Dispatcher& dispatcher, + Api::Api& api); std::vector healthCheckers() { return health_checkers_; }; diff --git a/source/extensions/clusters/redis/redis_cluster.cc b/source/extensions/clusters/redis/redis_cluster.cc index cf6f1d9f5bad..4906f5e0b5cd 100644 --- a/source/extensions/clusters/redis/redis_cluster.cc +++ b/source/extensions/clusters/redis/redis_cluster.cc @@ -34,7 +34,8 @@ RedisCluster::RedisCluster( : Config::Utility::translateClusterHosts(cluster.hosts())), local_info_(factory_context.localInfo()), random_(factory_context.random()), redis_discovery_session_(*this, redis_client_factory), lb_factory_(std::move(lb_factory)), - api_(api) { + auth_password_( + NetworkFilters::RedisProxy::ProtocolOptionsConfigImpl::auth_password(info(), api)) { const auto& locality_lb_endpoints = load_assignment_.endpoints(); for (const auto& locality_lb_endpoint : locality_lb_endpoints) { for (const auto& lb_endpoint : locality_lb_endpoint.lb_endpoints()) { @@ -43,13 +44,6 @@ RedisCluster::RedisCluster( *this, host.socket_address().address(), host.socket_address().port_value())); } } - - auto options = - info()->extensionProtocolOptionsTyped( - NetworkFilters::NetworkFilterNames::get().RedisProxy); - if (options) { - auth_password_datasource_ = options->auth_password_datasource(); - } } void RedisCluster::startPreInit() { @@ -253,16 +247,8 @@ void RedisCluster::RedisDiscoverySession::startResolveRedis() { client = std::make_unique(*this); client->host_ = current_host_address_; client->client_ = client_factory_.create(host, dispatcher_, *this, redis_command_stats_, - parent_.info()->statsScope()); + parent_.info()->statsScope(), parent_.auth_password_); client->client_->addConnectionCallbacks(*client); - std::string auth_password = - Envoy::Config::DataSource::read(parent_.auth_password_datasource_, true, parent_.api_); - if (!auth_password.empty()) { - // Send an AUTH command to the upstream server. - client->client_->makeRequest( - Extensions::NetworkFilters::Common::Redis::Utility::makeAuthCommand(auth_password), - null_pool_callbacks); - } } current_request_ = client->client_->makeRequest(ClusterSlotsRequest::instance_, *this); diff --git a/source/extensions/clusters/redis/redis_cluster.h b/source/extensions/clusters/redis/redis_cluster.h index bec0ac965031..aaee4f97b9cc 100644 --- a/source/extensions/clusters/redis/redis_cluster.h +++ b/source/extensions/clusters/redis/redis_cluster.h @@ -215,10 +215,12 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { std::chrono::milliseconds bufferFlushTimeoutInMs() const override { return buffer_timeout_; } uint32_t maxUpstreamUnknownConnections() const override { return 0; } bool enableCommandStats() const override { return false; } - // This is effectively not in used for making the "Cluster Slots" calls. - // since we call cluster slots on both the master and slaves, ANY is more appropriate here. + // For any readPolicy other than Master, the RedisClientFactory will send a READONLY command + // when establishing a new connection. Since we're only using this for making the "cluster + // slots" commands, the READONLY command is not relevant in this context. We're setting it to + // Master to avoid the additional READONLY command. Extensions::NetworkFilters::Common::Redis::Client::ReadPolicy readPolicy() const override { - return Extensions::NetworkFilters::Common::Redis::Client::ReadPolicy::Any; + return Extensions::NetworkFilters::Common::Redis::Client::ReadPolicy::Master; } // Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks @@ -261,8 +263,7 @@ class RedisCluster : public Upstream::BaseDynamicClusterImpl { Upstream::HostVector hosts_; Upstream::HostMap all_hosts_; - envoy::api::v2::core::DataSource auth_password_datasource_; - Api::Api& api_; + const std::string auth_password_; }; class RedisClusterFactory : public Upstream::ConfigurableClusterFactoryBase< diff --git a/source/extensions/filters/network/common/redis/BUILD b/source/extensions/filters/network/common/redis/BUILD index 336c9f2b303a..ae2702107c4a 100644 --- a/source/extensions/filters/network/common/redis/BUILD +++ b/source/extensions/filters/network/common/redis/BUILD @@ -58,6 +58,7 @@ envoy_cc_library( deps = [ ":client_interface", ":codec_lib", + ":utility_lib", "//include/envoy/router:router_interface", "//include/envoy/stats:timespan", "//include/envoy/thread_local:thread_local_interface", diff --git a/source/extensions/filters/network/common/redis/client.h b/source/extensions/filters/network/common/redis/client.h index 3e5d80ce9208..fc76c61ac44e 100644 --- a/source/extensions/filters/network/common/redis/client.h +++ b/source/extensions/filters/network/common/redis/client.h @@ -96,6 +96,12 @@ class Client : public Event::DeferredDeletable { * for some reason. */ virtual PoolRequest* makeRequest(const RespValue& request, PoolCallbacks& callbacks) PURE; + + /** + * Initialize the connection. Issue the auth command and readonly command as needed. + * @param auth password for upstream host. + */ + virtual void initialize(const std::string& auth_password) PURE; }; using ClientPtr = std::unique_ptr; @@ -187,12 +193,15 @@ class ClientFactory { * @param host supplies the upstream host. * @param dispatcher supplies the owning thread's dispatcher. * @param config supplies the connection pool configuration. + * @param redis_command_stats supplies the redis command stats. + * @param scope supplies the stats scope. + * @param auth password for upstream host. * @return ClientPtr a new connection pool client. */ virtual ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, const Config& config, const RedisCommandStatsSharedPtr& redis_command_stats, - Stats::Scope& scope) PURE; + Stats::Scope& scope, const std::string& auth_password) PURE; }; } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.cc b/source/extensions/filters/network/common/redis/client_impl.cc index acb7d64f771e..631221a39496 100644 --- a/source/extensions/filters/network/common/redis/client_impl.cc +++ b/source/extensions/filters/network/common/redis/client_impl.cc @@ -6,6 +6,9 @@ namespace NetworkFilters { namespace Common { namespace Redis { namespace Client { +namespace { +Common::Redis::Client::DoNothingPoolCallbacks null_pool_callbacks; +} // namespace ConfigImpl::ConfigImpl( const envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings& config) @@ -275,14 +278,29 @@ void ClientImpl::PendingRequest::cancel() { canceled_ = true; } +void ClientImpl::initialize(const std::string& auth_password) { + if (!auth_password.empty()) { + // Send an AUTH command to the upstream server. + makeRequest(Utility::makeAuthCommand(auth_password), null_pool_callbacks); + } + // Any connection to replica requires the READONLY command in order to perform read. + // Also the READONLY command is a no-opt for the master. + // We only need to send the READONLY command iff it's possible that the host is a replica. + if (config_.readPolicy() != Common::Redis::Client::ReadPolicy::Master) { + makeRequest(Utility::ReadOnlyRequest::instance(), null_pool_callbacks); + } +} + ClientFactoryImpl ClientFactoryImpl::instance_; ClientPtr ClientFactoryImpl::create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, const Config& config, const RedisCommandStatsSharedPtr& redis_command_stats, - Stats::Scope& scope) { - return ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, decoder_factory_, - config, redis_command_stats, scope); + Stats::Scope& scope, const std::string& auth_password) { + ClientPtr client = ClientImpl::create(host, dispatcher, EncoderPtr{new EncoderImpl()}, + decoder_factory_, config, redis_command_stats, scope); + client->initialize(auth_password); + return client; } } // namespace Client diff --git a/source/extensions/filters/network/common/redis/client_impl.h b/source/extensions/filters/network/common/redis/client_impl.h index a8ab3806eb7e..be0d1f8119be 100644 --- a/source/extensions/filters/network/common/redis/client_impl.h +++ b/source/extensions/filters/network/common/redis/client_impl.h @@ -16,6 +16,7 @@ #include "common/upstream/upstream_impl.h" #include "extensions/filters/network/common/redis/client.h" +#include "extensions/filters/network/common/redis/utility.h" namespace Envoy { namespace Extensions { @@ -85,6 +86,7 @@ class ClientImpl : public Client, public DecoderCallbacks, public Network::Conne PoolRequest* makeRequest(const RespValue& request, PoolCallbacks& callbacks) override; bool active() override { return !pending_requests_.empty(); } void flushBufferAndResetTimer(); + void initialize(const std::string& auth_password) override; private: friend class RedisClientImplTest; @@ -148,7 +150,7 @@ class ClientFactoryImpl : public ClientFactory { // RedisProxy::ConnPool::ClientFactoryImpl ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher& dispatcher, const Config& config, const RedisCommandStatsSharedPtr& redis_command_stats, - Stats::Scope& scope) override; + Stats::Scope& scope, const std::string& auth_password) override; static ClientFactoryImpl instance_; diff --git a/source/extensions/filters/network/redis_proxy/BUILD b/source/extensions/filters/network/redis_proxy/BUILD index 2aee3f55e361..7178d82679db 100644 --- a/source/extensions/filters/network/redis_proxy/BUILD +++ b/source/extensions/filters/network/redis_proxy/BUILD @@ -116,6 +116,7 @@ envoy_cc_library( hdrs = ["config.h"], deps = [ "//include/envoy/registry", + "//include/envoy/upstream:upstream_interface", "//source/common/config:filter_json_lib", "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", diff --git a/source/extensions/filters/network/redis_proxy/config.h b/source/extensions/filters/network/redis_proxy/config.h index 37d6a7c0203c..5a72b8ad9c4e 100644 --- a/source/extensions/filters/network/redis_proxy/config.h +++ b/source/extensions/filters/network/redis_proxy/config.h @@ -2,9 +2,12 @@ #include +#include "envoy/api/api.h" #include "envoy/config/filter/network/redis_proxy/v2/redis_proxy.pb.h" #include "envoy/config/filter/network/redis_proxy/v2/redis_proxy.pb.validate.h" +#include "envoy/upstream/upstream.h" +#include "common/common/empty_string.h" #include "common/config/datasource.h" #include "extensions/filters/network/common/factory_base.h" @@ -29,6 +32,16 @@ class ProtocolOptionsConfigImpl : public Upstream::ProtocolOptionsConfig { return auth_password_; } + static const std::string auth_password(const Upstream::ClusterInfoConstSharedPtr info, + Api::Api& api) { + auto options = info->extensionProtocolOptionsTyped( + NetworkFilterNames::get().RedisProxy); + if (options) { + return options->auth_password(api); + } + return EMPTY_STRING; + } + private: envoy::api::v2::core::DataSource auth_password_; }; diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index c8449f61dc84..7304ef902325 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -15,9 +15,6 @@ namespace Extensions { namespace NetworkFilters { namespace RedisProxy { namespace ConnPool { -namespace { -Common::Redis::Client::DoNothingPoolCallbacks null_pool_callbacks; -} // namespace InstanceImpl::InstanceImpl( const std::string& cluster_name, Upstream::ClusterManager& cm, @@ -56,11 +53,7 @@ InstanceImpl::ThreadLocalPool::ThreadLocalPool(InstanceImpl& parent, Event::Disp cluster_update_handle_ = parent_.cm_.addThreadLocalClusterUpdateCallbacks(*this); Upstream::ThreadLocalCluster* cluster = parent_.cm_.get(cluster_name_); if (cluster != nullptr) { - auto options = cluster->info()->extensionProtocolOptionsTyped( - NetworkFilterNames::get().RedisProxy); - if (options) { - auth_password_ = options->auth_password(parent_.api_); - } + auth_password_ = ProtocolOptionsConfigImpl::auth_password(cluster->info(), parent_.api_); onClusterAddOrUpdateNonVirtual(*cluster); } } @@ -197,22 +190,10 @@ InstanceImpl::ThreadLocalPool::threadLocalActiveClient(Upstream::HostConstShared if (!client) { client = std::make_unique(*this); client->host_ = host; - client->redis_client_ = parent_.client_factory_.create( - host, dispatcher_, parent_.config_, parent_.redis_command_stats_, *parent_.stats_scope_); + client->redis_client_ = parent_.client_factory_.create(host, dispatcher_, parent_.config_, + parent_.redis_command_stats_, + *parent_.stats_scope_, auth_password_); client->redis_client_->addConnectionCallbacks(*client); - // TODO(hyang): should the auth command and readonly command be moved to the factory method? - if (!auth_password_.empty()) { - // Send an AUTH command to the upstream server. - client->redis_client_->makeRequest(Common::Redis::Utility::makeAuthCommand(auth_password_), - null_pool_callbacks); - } - // Any connection to replica requires the READONLY command in order to perform read. - // Also the READONLY command is a no-opt for the master. - // We only need to send the READONLY command iff it's possible that the host is a replica. - if (parent_.config_.readPolicy() != Common::Redis::Client::ReadPolicy::Master) { - client->redis_client_->makeRequest(Common::Redis::Utility::ReadOnlyRequest::instance(), - null_pool_callbacks); - } } return client; } diff --git a/source/extensions/health_checkers/redis/BUILD b/source/extensions/health_checkers/redis/BUILD index 1c92f366295c..5c3c7bdffe8b 100644 --- a/source/extensions/health_checkers/redis/BUILD +++ b/source/extensions/health_checkers/redis/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( deps = [ "//source/common/upstream:health_checker_base_lib", "//source/extensions/filters/network/common/redis:client_lib", + "//source/extensions/filters/network/redis_proxy:config", "//source/extensions/filters/network/redis_proxy:conn_pool_lib", "@envoy_api//envoy/api/v2/core:health_check_cc", "@envoy_api//envoy/config/health_checker/redis/v2:redis_cc", diff --git a/source/extensions/health_checkers/redis/config.cc b/source/extensions/health_checkers/redis/config.cc index 6892781ea8dd..a3bdb6eab6de 100644 --- a/source/extensions/health_checkers/redis/config.cc +++ b/source/extensions/health_checkers/redis/config.cc @@ -17,7 +17,7 @@ Upstream::HealthCheckerSharedPtr RedisHealthCheckerFactory::createCustomHealthCh return std::make_shared( context.cluster(), config, getRedisHealthCheckConfig(config, context.messageValidationVisitor()), context.dispatcher(), - context.runtime(), context.random(), context.eventLogger(), + context.runtime(), context.random(), context.eventLogger(), context.api(), NetworkFilters::Common::Redis::Client::ClientFactoryImpl::instance_); }; diff --git a/source/extensions/health_checkers/redis/redis.cc b/source/extensions/health_checkers/redis/redis.cc index c1107e1a489f..22a7bd9be07f 100644 --- a/source/extensions/health_checkers/redis/redis.cc +++ b/source/extensions/health_checkers/redis/redis.cc @@ -9,10 +9,12 @@ RedisHealthChecker::RedisHealthChecker( const Upstream::Cluster& cluster, const envoy::api::v2::core::HealthCheck& config, const envoy::config::health_checker::redis::v2::Redis& redis_config, Event::Dispatcher& dispatcher, Runtime::Loader& runtime, Runtime::RandomGenerator& random, - Upstream::HealthCheckEventLoggerPtr&& event_logger, + Upstream::HealthCheckEventLoggerPtr&& event_logger, Api::Api& api, Extensions::NetworkFilters::Common::Redis::Client::ClientFactory& client_factory) : HealthCheckerImplBase(cluster, config, dispatcher, runtime, random, std::move(event_logger)), - client_factory_(client_factory), key_(redis_config.key()) { + client_factory_(client_factory), key_(redis_config.key()), + auth_password_(NetworkFilters::RedisProxy::ProtocolOptionsConfigImpl::auth_password( + cluster.info(), api)) { if (!key_.empty()) { type_ = Type::Exists; } else { @@ -55,9 +57,9 @@ void RedisHealthChecker::RedisActiveHealthCheckSession::onEvent(Network::Connect void RedisHealthChecker::RedisActiveHealthCheckSession::onInterval() { if (!client_) { - client_ = - parent_.client_factory_.create(host_, parent_.dispatcher_, *this, redis_command_stats_, - parent_.cluster_.info()->statsScope()); + client_ = parent_.client_factory_.create( + host_, parent_.dispatcher_, *this, redis_command_stats_, + parent_.cluster_.info()->statsScope(), parent_.auth_password_); client_->addConnectionCallbacks(*this); } diff --git a/source/extensions/health_checkers/redis/redis.h b/source/extensions/health_checkers/redis/redis.h index 17c8e060e716..5268e6e8e404 100644 --- a/source/extensions/health_checkers/redis/redis.h +++ b/source/extensions/health_checkers/redis/redis.h @@ -2,11 +2,13 @@ #include +#include "envoy/api/api.h" #include "envoy/config/health_checker/redis/v2/redis.pb.validate.h" #include "common/upstream/health_checker_base_impl.h" #include "extensions/filters/network/common/redis/client_impl.h" +#include "extensions/filters/network/redis_proxy/config.h" #include "extensions/filters/network/redis_proxy/conn_pool_impl.h" namespace Envoy { @@ -23,7 +25,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { const Upstream::Cluster& cluster, const envoy::api::v2::core::HealthCheck& config, const envoy::config::health_checker::redis::v2::Redis& redis_config, Event::Dispatcher& dispatcher, Runtime::Loader& runtime, Runtime::RandomGenerator& random, - Upstream::HealthCheckEventLoggerPtr&& event_logger, + Upstream::HealthCheckEventLoggerPtr&& event_logger, Api::Api& api, Extensions::NetworkFilters::Common::Redis::Client::ClientFactory& client_factory); static const NetworkFilters::Common::Redis::RespValue& pingHealthCheckRequest() { @@ -118,6 +120,7 @@ class RedisHealthChecker : public Upstream::HealthCheckerImplBase { Extensions::NetworkFilters::Common::Redis::Client::ClientFactory& client_factory_; Type type_; const std::string key_; + const std::string auth_password_; }; } // namespace RedisHealthChecker diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 1ac3067c7ef2..9543cadbe76c 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -125,6 +125,7 @@ envoy_cc_test( "//source/common/upstream:upstream_lib", "//test/common/http:common_lib", "//test/mocks/access_log:access_log_mocks", + "//test/mocks/api:api_mocks", "//test/mocks/network:network_mocks", "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/runtime:runtime_mocks", diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index 5087f44d37de..b14b942a0d0c 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -18,6 +18,7 @@ #include "test/common/http/common.h" #include "test/common/upstream/utility.h" #include "test/mocks/access_log/mocks.h" +#include "test/mocks/api/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/protobuf/mocks.h" #include "test/mocks/runtime/mocks.h" @@ -62,10 +63,11 @@ TEST(HealthCheckerFactoryTest, GrpcHealthCheckHTTP2NotConfiguredException) { Event::MockDispatcher dispatcher; AccessLog::MockAccessLogManager log_manager; NiceMock validation_visitor; + Api::MockApi api; EXPECT_THROW_WITH_MESSAGE( HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, runtime, random, - dispatcher, log_manager, validation_visitor), + dispatcher, log_manager, validation_visitor, api), EnvoyException, "fake_cluster cluster must support HTTP/2 for gRPC healthchecking"); } @@ -80,12 +82,13 @@ TEST(HealthCheckerFactoryTest, CreateGrpc) { Event::MockDispatcher dispatcher; AccessLog::MockAccessLogManager log_manager; NiceMock validation_visitor; + Api::MockApi api; - EXPECT_NE(nullptr, - dynamic_cast( - HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, runtime, - random, dispatcher, log_manager, validation_visitor) - .get())); + EXPECT_NE(nullptr, dynamic_cast( + HealthCheckerFactory::create(createGrpcHealthCheckConfig(), cluster, + runtime, random, dispatcher, log_manager, + validation_visitor, api) + .get())); } class TestHttpHealthCheckerImpl : public HttpHealthCheckerImpl { diff --git a/test/extensions/clusters/redis/redis_cluster_test.cc b/test/extensions/clusters/redis/redis_cluster_test.cc index 78463582049a..1f51829ced2e 100644 --- a/test/extensions/clusters/redis/redis_cluster_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_test.cc @@ -60,7 +60,7 @@ class RedisClusterTest : public testing::Test, create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, const Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&, - Stats::Scope&) override { + Stats::Scope&, const std::string&) override { EXPECT_EQ(22120, host->address()->ip()->port()); return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{ create_(host->address()->asString())}; diff --git a/test/extensions/filters/network/common/redis/client_impl_test.cc b/test/extensions/filters/network/common/redis/client_impl_test.cc index 8a283c67faf1..54c67ba9a5bf 100644 --- a/test/extensions/filters/network/common/redis/client_impl_test.cc +++ b/test/extensions/filters/network/common/redis/client_impl_test.cc @@ -5,6 +5,7 @@ #include "common/upstream/upstream_impl.h" #include "extensions/filters/network/common/redis/client_impl.h" +#include "extensions/filters/network/common/redis/utility.h" #include "test/extensions/filters/network/common/redis/mocks.h" #include "test/extensions/filters/network/common/redis/test_utils.h" @@ -15,6 +16,7 @@ #include "gtest/gtest.h" using testing::_; +using testing::Eq; using testing::InSequence; using testing::Invoke; using testing::Ref; @@ -98,6 +100,28 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF client_impl->onRespValue(std::move(response1)); } + void testInitializeReadPolicy( + envoy::config::filter::network::redis_proxy::v2::RedisProxy::ConnPoolSettings::ReadPolicy + read_policy) { + InSequence s; + + setup(std::make_unique(createConnPoolSettings(20, true, true, 100, read_policy))); + + Common::Redis::RespValue readonly_request = Utility::ReadOnlyRequest::instance(); + EXPECT_CALL(*encoder_, encode(Eq(readonly_request), _)); + EXPECT_CALL(*flush_timer_, enabled()).WillOnce(Return(false)); + client_->initialize(auth_password_); + + EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_rq_total_.value()); + EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_rq_active_.value()); + EXPECT_EQ(1UL, host_->stats_.rq_total_.value()); + EXPECT_EQ(1UL, host_->stats_.rq_active_.value()); + + EXPECT_CALL(*upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); + EXPECT_CALL(*connect_or_op_timer_, disableTimer()); + client_->close(); + } + const std::string cluster_name_{"foo"}; std::shared_ptr host_{new NiceMock()}; Event::MockDispatcher dispatcher_; @@ -113,6 +137,7 @@ class RedisClientImplTest : public testing::Test, public Common::Redis::DecoderF Stats::IsolatedStoreImpl stats_; Stats::ScopePtr stats_scope_; Common::Redis::RedisCommandStatsSharedPtr redis_command_stats_; + std::string auth_password_; }; TEST_F(RedisClientImplTest, BatchWithZeroBufferAndTimeout) { @@ -259,6 +284,8 @@ TEST_F(RedisClientImplTest, Basic) { setup(); + client_->initialize(auth_password_); + Common::Redis::RespValue request1; MockPoolCallbacks callbacks1; EXPECT_CALL(*encoder_, encode(Ref(request1), _)); @@ -304,6 +331,47 @@ TEST_F(RedisClientImplTest, Basic) { client_->close(); } +TEST_F(RedisClientImplTest, InitializedWithAuthPassword) { + InSequence s; + + setup(); + + auth_password_ = "testing password"; + Common::Redis::RespValue auth_request = Utility::makeAuthCommand(auth_password_); + EXPECT_CALL(*encoder_, encode(Eq(auth_request), _)); + EXPECT_CALL(*flush_timer_, enabled()).WillOnce(Return(false)); + client_->initialize(auth_password_); + + EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_rq_total_.value()); + EXPECT_EQ(1UL, host_->cluster_.stats_.upstream_rq_active_.value()); + EXPECT_EQ(1UL, host_->stats_.rq_total_.value()); + EXPECT_EQ(1UL, host_->stats_.rq_active_.value()); + + EXPECT_CALL(*upstream_connection_, close(Network::ConnectionCloseType::NoFlush)); + EXPECT_CALL(*connect_or_op_timer_, disableTimer()); + client_->close(); +} + +TEST_F(RedisClientImplTest, InitializedWithPreferMasterReadPolicy) { + testInitializeReadPolicy(envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_MASTER); +} + +TEST_F(RedisClientImplTest, InitializedWithReplicaReadPolicy) { + testInitializeReadPolicy(envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_REPLICA); +} + +TEST_F(RedisClientImplTest, InitializedWithPreferReplicaReadPolicy) { + testInitializeReadPolicy(envoy::config::filter::network::redis_proxy::v2:: + RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_REPLICA); +} + +TEST_F(RedisClientImplTest, InitializedWithAnyReadPolicy) { + testInitializeReadPolicy( + envoy::config::filter::network::redis_proxy::v2::RedisProxy_ConnPoolSettings_ReadPolicy_ANY); +} + TEST_F(RedisClientImplTest, Cancel) { InSequence s; @@ -882,10 +950,11 @@ TEST(RedisClientFactoryImplTest, Basic) { Stats::IsolatedStoreImpl stats_; auto redis_command_stats = Common::Redis::RedisCommandStats::createRedisCommandStats(stats_.symbolTable()); - ClientPtr client = factory.create(host, dispatcher, config, redis_command_stats, stats_); + const std::string auth_password; + ClientPtr client = + factory.create(host, dispatcher, config, redis_command_stats, stats_, auth_password); client->close(); } - } // namespace Client } // namespace Redis } // namespace Common diff --git a/test/extensions/filters/network/common/redis/mocks.h b/test/extensions/filters/network/common/redis/mocks.h index 859ea03cf42b..a44e41ef63e9 100644 --- a/test/extensions/filters/network/common/redis/mocks.h +++ b/test/extensions/filters/network/common/redis/mocks.h @@ -73,6 +73,7 @@ class MockClient : public Client { MOCK_METHOD0(close, void()); MOCK_METHOD2(makeRequest, PoolRequest*(const Common::Redis::RespValue& request, PoolCallbacks& callbacks)); + MOCK_METHOD1(initialize, void(const std::string& password)); std::list callbacks_; }; diff --git a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc index f8f03045aa8e..262e8d9bcd81 100644 --- a/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/conn_pool_impl_test.cc @@ -96,9 +96,6 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client Common::Redis::RespValue value; Common::Redis::Client::MockPoolCallbacks callbacks; Common::Redis::Client::MockPoolRequest active_request; - if (create_client && !auth_password_.empty()) { - EXPECT_CALL(*client_, makeRequest(_, _)).WillOnce(Return(nullptr)); - } EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, address()) .WillRepeatedly(Return(test_address_)); EXPECT_CALL(*client_, makeRequest(Ref(value), Ref(callbacks))) @@ -159,7 +156,8 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr host, Event::Dispatcher&, const Common::Redis::Client::Config&, const Common::Redis::RedisCommandStatsSharedPtr&, - Stats::Scope&) override { + Stats::Scope&, const std::string& password) override { + EXPECT_EQ(auth_password_, password); return Common::Redis::Client::ClientPtr{create_(host)}; } @@ -189,10 +187,6 @@ class RedisConnPoolImplTest : public testing::Test, public Common::Redis::Client return cm_.thread_local_cluster_.lb_.host_; })); EXPECT_CALL(*this, create_(_)).WillOnce(Return(client)); - EXPECT_CALL( - *client, - makeRequest(Eq(NetworkFilters::Common::Redis::Utility::ReadOnlyRequest::instance()), _)) - .WillOnce(Return(&readonly_request)); EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, address()) .WillRepeatedly(Return(test_address_)); EXPECT_CALL(*client, makeRequest(Ref(value), Ref(callbacks))).WillOnce(Return(&active_request)); @@ -251,40 +245,6 @@ TEST_F(RedisConnPoolImplTest, Basic) { tls_.shutdownThread(); }; -TEST_F(RedisConnPoolImplTest, BasicWithAuthPassword) { - InSequence s; - - auth_password_ = "testing password"; - setup(); - - Common::Redis::RespValue value; - Common::Redis::Client::MockPoolRequest auth_request, active_request; - Common::Redis::Client::MockPoolCallbacks callbacks; - Common::Redis::Client::MockClient* client = new NiceMock(); - - EXPECT_CALL(cm_.thread_local_cluster_.lb_, chooseHost(_)) - .WillOnce(Invoke([&](Upstream::LoadBalancerContext* context) -> Upstream::HostConstSharedPtr { - EXPECT_EQ(context->computeHashKey().value(), MurmurHash::murmurHash2_64("hash_key")); - EXPECT_EQ(context->metadataMatchCriteria(), nullptr); - EXPECT_EQ(context->downstreamConnection(), nullptr); - return cm_.thread_local_cluster_.lb_.host_; - })); - EXPECT_CALL(*this, create_(_)).WillOnce(Return(client)); - EXPECT_CALL( - *client, - makeRequest(Eq(NetworkFilters::Common::Redis::Utility::makeAuthCommand(auth_password_)), _)) - .WillOnce(Return(&auth_request)); - EXPECT_CALL(*cm_.thread_local_cluster_.lb_.host_, address()) - .WillRepeatedly(Return(test_address_)); - EXPECT_CALL(*client, makeRequest(Ref(value), Ref(callbacks))).WillOnce(Return(&active_request)); - Common::Redis::Client::PoolRequest* request = - conn_pool_->makeRequest("hash_key", value, callbacks); - EXPECT_EQ(&active_request, request); - - EXPECT_CALL(*client, close()); - tls_.shutdownThread(); -}; - TEST_F(RedisConnPoolImplTest, BasicWithReadPolicy) { testReadPolicy(envoy::config::filter::network::redis_proxy::v2:: RedisProxy_ConnPoolSettings_ReadPolicy_PREFER_MASTER, @@ -602,59 +562,6 @@ TEST_F(RedisConnPoolImplTest, MakeRequestToHostWithZeroMaxUnknownUpstreamConnect tls_.shutdownThread(); } -TEST_F(RedisConnPoolImplTest, MakeRequestToHostWithAuthPassword) { - InSequence s; - - auth_password_ = "superduperpassword"; - setup(false); - - Common::Redis::RespValue value; - Common::Redis::Client::MockPoolRequest auth_request1, active_request1; - Common::Redis::Client::MockPoolRequest auth_request2, active_request2; - Common::Redis::Client::MockPoolCallbacks callbacks1; - Common::Redis::Client::MockPoolCallbacks callbacks2; - Common::Redis::Client::MockClient* client1 = new NiceMock(); - Common::Redis::Client::MockClient* client2 = new NiceMock(); - Upstream::HostConstSharedPtr host1; - Upstream::HostConstSharedPtr host2; - - // There is no cluster yet, so makeRequestToHost() should fail. - EXPECT_EQ(nullptr, conn_pool_->makeRequestToHost("10.0.0.1:3000", value, callbacks1)); - // Add the cluster now. - update_callbacks_->onClusterAddOrUpdate(cm_.thread_local_cluster_); - - EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host1), Return(client1))); - EXPECT_CALL( - *client1, - makeRequest(Eq(NetworkFilters::Common::Redis::Utility::makeAuthCommand(auth_password_)), _)) - .WillOnce(Return(&auth_request1)); - EXPECT_CALL(*client1, makeRequest(Ref(value), Ref(callbacks1))) - .WillOnce(Return(&active_request1)); - Common::Redis::Client::PoolRequest* request1 = - conn_pool_->makeRequestToHost("10.0.0.1:3000", value, callbacks1); - EXPECT_EQ(&active_request1, request1); - EXPECT_EQ(host1->address()->asString(), "10.0.0.1:3000"); - - // IPv6 address returned from Redis server will not have square brackets - // around it, while Envoy represents Address::Ipv6Instance addresses with square brackets around - // the address. - EXPECT_CALL(*this, create_(_)).WillOnce(DoAll(SaveArg<0>(&host2), Return(client2))); - EXPECT_CALL( - *client2, - makeRequest(Eq(NetworkFilters::Common::Redis::Utility::makeAuthCommand(auth_password_)), _)) - .WillOnce(Return(&auth_request2)); - EXPECT_CALL(*client2, makeRequest(Ref(value), Ref(callbacks2))) - .WillOnce(Return(&active_request2)); - Common::Redis::Client::PoolRequest* request2 = - conn_pool_->makeRequestToHost("2001:470:813B:0:0:0:0:1:3333", value, callbacks2); - EXPECT_EQ(&active_request2, request2); - EXPECT_EQ(host2->address()->asString(), "[2001:470:813b::1]:3333"); - - EXPECT_CALL(*client2, close()); - EXPECT_CALL(*client1, close()); - tls_.shutdownThread(); -} - // This test forces the creation of 2 hosts (one with an IPv4 address, and the other with an IPv6 // address) and pending requests using makeRequestToHost(). After their creation, "new" hosts are // discovered, and the original hosts are put aside to drain. The test then verifies the drain diff --git a/test/extensions/health_checkers/redis/BUILD b/test/extensions/health_checkers/redis/BUILD index 015f73f5e4f4..8f38c2dfe32d 100644 --- a/test/extensions/health_checkers/redis/BUILD +++ b/test/extensions/health_checkers/redis/BUILD @@ -16,6 +16,7 @@ envoy_extension_cc_test( srcs = ["redis_test.cc"], extension_name = "envoy.health_checkers.redis", deps = [ + "//source/common/api:api_lib", "//source/extensions/health_checkers/redis", "//source/extensions/health_checkers/redis:utility", "//test/common/upstream:utility_lib", @@ -25,6 +26,7 @@ envoy_extension_cc_test( "//test/mocks/network:network_mocks", "//test/mocks/runtime:runtime_mocks", "//test/mocks/upstream:upstream_mocks", + "//test/test_common:utility_lib", ], ) diff --git a/test/extensions/health_checkers/redis/config_test.cc b/test/extensions/health_checkers/redis/config_test.cc index 2dcb9a439127..8b38426fc652 100644 --- a/test/extensions/health_checkers/redis/config_test.cc +++ b/test/extensions/health_checkers/redis/config_test.cc @@ -106,12 +106,14 @@ TEST(HealthCheckerFactoryTest, CreateRedisViaUpstreamHealthCheckerFactory) { Runtime::MockRandomGenerator random; Event::MockDispatcher dispatcher; AccessLog::MockAccessLogManager log_manager; - - EXPECT_NE(nullptr, dynamic_cast( - Upstream::HealthCheckerFactory::create( - Upstream::parseHealthCheckFromV2Yaml(yaml), cluster, runtime, random, - dispatcher, log_manager, ProtobufMessage::getStrictValidationVisitor()) - .get())); + NiceMock api; + + EXPECT_NE(nullptr, + dynamic_cast( + Upstream::HealthCheckerFactory::create( + Upstream::parseHealthCheckFromV2Yaml(yaml), cluster, runtime, random, + dispatcher, log_manager, ProtobufMessage::getStrictValidationVisitor(), api) + .get())); } } // namespace } // namespace RedisHealthChecker diff --git a/test/extensions/health_checkers/redis/redis_test.cc b/test/extensions/health_checkers/redis/redis_test.cc index b9c215853885..c55616220361 100644 --- a/test/extensions/health_checkers/redis/redis_test.cc +++ b/test/extensions/health_checkers/redis/redis_test.cc @@ -1,5 +1,7 @@ #include +#include "envoy/api/api.h" + #include "extensions/health_checkers/redis/redis.h" #include "extensions/health_checkers/redis/utility.h" @@ -29,7 +31,7 @@ class RedisHealthCheckerTest public: RedisHealthCheckerTest() : cluster_(new NiceMock()), - event_logger_(new Upstream::MockHealthCheckEventLogger()) {} + event_logger_(new Upstream::MockHealthCheckEventLogger()), api_(Api::createApiForTest()) {} void setup() { const std::string yaml = R"EOF( @@ -48,9 +50,9 @@ class RedisHealthCheckerTest const auto& redis_config = getRedisHealthCheckConfig( health_check_config, ProtobufMessage::getStrictValidationVisitor()); - health_checker_.reset( - new RedisHealthChecker(*cluster_, health_check_config, redis_config, dispatcher_, runtime_, - random_, Upstream::HealthCheckEventLoggerPtr(event_logger_), *this)); + health_checker_.reset(new RedisHealthChecker( + *cluster_, health_check_config, redis_config, dispatcher_, runtime_, random_, + Upstream::HealthCheckEventLoggerPtr(event_logger_), *api_, *this)); } void setupAlwaysLogHealthCheckFailures() { @@ -71,9 +73,9 @@ class RedisHealthCheckerTest const auto& redis_config = getRedisHealthCheckConfig( health_check_config, ProtobufMessage::getStrictValidationVisitor()); - health_checker_.reset( - new RedisHealthChecker(*cluster_, health_check_config, redis_config, dispatcher_, runtime_, - random_, Upstream::HealthCheckEventLoggerPtr(event_logger_), *this)); + health_checker_.reset(new RedisHealthChecker( + *cluster_, health_check_config, redis_config, dispatcher_, runtime_, random_, + Upstream::HealthCheckEventLoggerPtr(event_logger_), *api_, *this)); } void setupExistsHealthcheck() { @@ -94,9 +96,9 @@ class RedisHealthCheckerTest const auto& redis_config = getRedisHealthCheckConfig( health_check_config, ProtobufMessage::getStrictValidationVisitor()); - health_checker_.reset( - new RedisHealthChecker(*cluster_, health_check_config, redis_config, dispatcher_, runtime_, - random_, Upstream::HealthCheckEventLoggerPtr(event_logger_), *this)); + health_checker_.reset(new RedisHealthChecker( + *cluster_, health_check_config, redis_config, dispatcher_, runtime_, random_, + Upstream::HealthCheckEventLoggerPtr(event_logger_), *api_, *this)); } void setupDontReuseConnection() { @@ -117,16 +119,16 @@ class RedisHealthCheckerTest const auto& redis_config = getRedisHealthCheckConfig( health_check_config, ProtobufMessage::getStrictValidationVisitor()); - health_checker_.reset( - new RedisHealthChecker(*cluster_, health_check_config, redis_config, dispatcher_, runtime_, - random_, Upstream::HealthCheckEventLoggerPtr(event_logger_), *this)); + health_checker_.reset(new RedisHealthChecker( + *cluster_, health_check_config, redis_config, dispatcher_, runtime_, random_, + Upstream::HealthCheckEventLoggerPtr(event_logger_), *api_, *this)); } Extensions::NetworkFilters::Common::Redis::Client::ClientPtr create(Upstream::HostConstSharedPtr, Event::Dispatcher&, const Extensions::NetworkFilters::Common::Redis::Client::Config&, const Extensions::NetworkFilters::Common::Redis::RedisCommandStatsSharedPtr&, - Stats::Scope&) override { + Stats::Scope&, const std::string&) override { return Extensions::NetworkFilters::Common::Redis::Client::ClientPtr{create_()}; } @@ -183,6 +185,7 @@ class RedisHealthCheckerTest Extensions::NetworkFilters::Common::Redis::Client::MockPoolRequest pool_request_; Extensions::NetworkFilters::Common::Redis::Client::PoolCallbacks* pool_callbacks_{}; std::shared_ptr health_checker_; + Api::ApiPtr api_; }; TEST_F(RedisHealthCheckerTest, PingAndVariousFailures) { diff --git a/test/mocks/server/mocks.cc b/test/mocks/server/mocks.cc index e97b45e7e8a7..92ba5f5456dc 100644 --- a/test/mocks/server/mocks.cc +++ b/test/mocks/server/mocks.cc @@ -223,6 +223,7 @@ MockHealthCheckerFactoryContext::MockHealthCheckerFactoryContext() { ON_CALL(*this, eventLogger_()).WillByDefault(Return(event_logger_)); ON_CALL(*this, messageValidationVisitor()) .WillByDefault(ReturnRef(ProtobufMessage::getStrictValidationVisitor())); + ON_CALL(*this, api()).WillByDefault(ReturnRef(api_)); } MockHealthCheckerFactoryContext::~MockHealthCheckerFactoryContext() = default; diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index fd0ee9b27ef6..de7d678c383b 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -546,6 +546,7 @@ class MockHealthCheckerFactoryContext : public virtual HealthCheckerFactoryConte MOCK_METHOD0(runtime, Envoy::Runtime::Loader&()); MOCK_METHOD0(eventLogger_, Upstream::HealthCheckEventLogger*()); MOCK_METHOD0(messageValidationVisitor, ProtobufMessage::ValidationVisitor&()); + MOCK_METHOD0(api, Api::Api&()); Upstream::HealthCheckEventLoggerPtr eventLogger() override { return Upstream::HealthCheckEventLoggerPtr(eventLogger_()); } @@ -555,6 +556,7 @@ class MockHealthCheckerFactoryContext : public virtual HealthCheckerFactoryConte testing::NiceMock random_; testing::NiceMock runtime_; testing::NiceMock* event_logger_{}; + testing::NiceMock api_{}; }; } // namespace Configuration From 73ad41ae0f36e998d6761763fb84a8d7d085d13a Mon Sep 17 00:00:00 2001 From: Nicolas Flacco <47160394+FAYiEKcbD0XFqF2QK2E4viAHg8rMm2VbjYKdjTg@users.noreply.github.com> Date: Wed, 11 Sep 2019 13:36:05 -0700 Subject: [PATCH 538/542] redis: mirroring should work when default value is zero, not just greater than zero (#8089) Signed-off-by: Nicolas Flacco --- .../filters/network/redis_proxy/router_impl.cc | 8 +++++--- .../filters/network/redis_proxy/router_impl.h | 2 +- .../network/redis_proxy/router_impl_test.cc | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/source/extensions/filters/network/redis_proxy/router_impl.cc b/source/extensions/filters/network/redis_proxy/router_impl.cc index 6cfa59022041..1cb86f68762e 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.cc +++ b/source/extensions/filters/network/redis_proxy/router_impl.cc @@ -10,7 +10,9 @@ MirrorPolicyImpl::MirrorPolicyImpl(const envoy::config::filter::network::redis_p const ConnPool::InstanceSharedPtr upstream, Runtime::Loader& runtime) : runtime_key_(config.runtime_fraction().runtime_key()), - default_value_(config.runtime_fraction().default_value()), + default_value_(config.has_runtime_fraction() ? absl::optional( + config.runtime_fraction().default_value()) + : absl::nullopt), exclude_read_commands_(config.exclude_read_commands()), upstream_(upstream), runtime_(runtime) {} @@ -23,8 +25,8 @@ bool MirrorPolicyImpl::shouldMirror(const std::string& command) const { return false; } - if (default_value_.numerator() > 0) { - return runtime_.snapshot().featureEnabled(runtime_key_, default_value_); + if (default_value_.has_value()) { + return runtime_.snapshot().featureEnabled(runtime_key_, default_value_.value()); } return true; diff --git a/source/extensions/filters/network/redis_proxy/router_impl.h b/source/extensions/filters/network/redis_proxy/router_impl.h index 26963adb1898..fdacf760891b 100644 --- a/source/extensions/filters/network/redis_proxy/router_impl.h +++ b/source/extensions/filters/network/redis_proxy/router_impl.h @@ -37,7 +37,7 @@ class MirrorPolicyImpl : public MirrorPolicy { private: const std::string runtime_key_; - const envoy::type::FractionalPercent default_value_; + const absl::optional default_value_; const bool exclude_read_commands_; ConnPool::InstanceSharedPtr upstream_; Runtime::Loader& runtime_; diff --git a/test/extensions/filters/network/redis_proxy/router_impl_test.cc b/test/extensions/filters/network/redis_proxy/router_impl_test.cc index 28f66bbd871b..b4d12c41942a 100644 --- a/test/extensions/filters/network/redis_proxy/router_impl_test.cc +++ b/test/extensions/filters/network/redis_proxy/router_impl_test.cc @@ -225,6 +225,22 @@ TEST(MirrorPolicyImplTest, ExcludeReadCommands) { EXPECT_EQ(true, policy.shouldMirror("set")); } +TEST(MirrorPolicyImplTest, DefaultValueZero) { + envoy::config::filter::network::redis_proxy::v2::RedisProxy::PrefixRoutes::Route:: + RequestMirrorPolicy config; + auto* runtime_fraction = config.mutable_runtime_fraction(); + auto* percentage = runtime_fraction->mutable_default_value(); + percentage->set_numerator(0); + percentage->set_denominator(envoy::type::FractionalPercent::HUNDRED); + auto upstream = std::make_shared(); + NiceMock runtime; + + MirrorPolicyImpl policy(config, upstream, runtime); + + EXPECT_EQ(false, policy.shouldMirror("get")); + EXPECT_EQ(false, policy.shouldMirror("set")); +} + TEST(MirrorPolicyImplTest, DeterminedByRuntimeFraction) { envoy::config::filter::network::redis_proxy::v2::RedisProxy::PrefixRoutes::Route:: RequestMirrorPolicy config; From d0e1db6b2ea424a228ac83585f80bf7c2ad2dcc4 Mon Sep 17 00:00:00 2001 From: htuch Date: Thu, 12 Sep 2019 02:33:58 +0300 Subject: [PATCH 539/542] tools: regularize pip/venv for format_python_tools.py. (#8176) As well as being a nice cleanup, this fixes some issues I had with local Docker use of fix_format as a non-root user. Signed-off-by: Harvey Tuch --- tools/format_python_tools.py | 2 +- tools/format_python_tools.sh | 14 +++----------- tools/shell_utils.sh | 10 ++++++---- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/tools/format_python_tools.py b/tools/format_python_tools.py index 8b2cacc321bb..9d56bcc9d965 100644 --- a/tools/format_python_tools.py +++ b/tools/format_python_tools.py @@ -39,7 +39,7 @@ def validateFormat(fix=False): successful_update_files = set() for python_file in collectFiles(): reformatted_source, encoding, changed = FormatFile(python_file, - style_config='.style.yapf', + style_config='tools/.style.yapf', in_place=fix, print_diff=not fix) if not fix: diff --git a/tools/format_python_tools.sh b/tools/format_python_tools.sh index 6749ade47071..56587489a73a 100755 --- a/tools/format_python_tools.sh +++ b/tools/format_python_tools.sh @@ -1,19 +1,11 @@ #!/bin/bash -set -e - -VENV_DIR="pyformat" -SCRIPTPATH=$(realpath "$(dirname $0)") -. $SCRIPTPATH/shell_utils.sh -cd "$SCRIPTPATH" +. tools/shell_utils.sh -source_venv "$VENV_DIR" -echo "Installing requirements..." -pip3 -q install --upgrade pip -pip3 -q install -r requirements.txt +set -e echo "Running Python format check..." -python3 format_python_tools.py $1 +python_venv format_python_tools $1 echo "Running Python3 flake8 check..." python3 -m flake8 --version diff --git a/tools/shell_utils.sh b/tools/shell_utils.sh index bb5a14c25e9e..560087615afc 100644 --- a/tools/shell_utils.sh +++ b/tools/shell_utils.sh @@ -1,6 +1,6 @@ source_venv() { VENV_DIR=$1 - if [[ "$VIRTUAL_ENV" == "" ]]; then + if [[ "${VIRTUAL_ENV}" == "" ]]; then if [[ ! -d "${VENV_DIR}"/venv ]]; then virtualenv "${VENV_DIR}"/venv --no-site-packages --python=python3 fi @@ -14,10 +14,12 @@ python_venv() { SCRIPT_DIR=$(realpath "$(dirname "$0")") BUILD_DIR=build_tools - VENV_DIR="$BUILD_DIR/$1" + PY_NAME="$1" + VENV_DIR="${BUILD_DIR}/${PY_NAME}" - source_venv "$VENV_DIR" + source_venv "${VENV_DIR}" pip install -r "${SCRIPT_DIR}"/requirements.txt - python3 "${SCRIPT_DIR}/$1.py" $* + shift + python3 "${SCRIPT_DIR}/${PY_NAME}.py" $* } From 90ed92f8285ea215ef51a35b93609f1bb9cc1e5b Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Thu, 12 Sep 2019 12:32:27 -0400 Subject: [PATCH 540/542] absl: Absl hash hook in a couple of places rather than hash functors (#8179) Signed-off-by: Joshua Marantz --- source/common/common/utility.h | 7 ---- source/common/stats/symbol_table_impl.h | 43 ++++++++++++--------- source/common/stats/tag_producer_impl.h | 4 +- test/common/stats/BUILD | 1 + test/common/stats/symbol_table_impl_test.cc | 21 ++++++++++ 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/source/common/common/utility.h b/source/common/common/utility.h index 9a74512da743..883bd5051970 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -500,13 +500,6 @@ template class IntervalSetImpl : public IntervalSet { std::set intervals_; // Intervals do not overlap or abut. }; -/** - * Hashing functor for use with unordered_map and unordered_set with string_view as a key. - */ -struct StringViewHash { - std::size_t operator()(const absl::string_view& k) const { return HashUtil::xxHash64(k); } -}; - /** * Hashing functor for use with enum class types. * This is needed for GCC 5.X; newer versions of GCC, as well as clang7, provide native hashing diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index dbbb633dcbe8..b83da159c035 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -229,7 +229,7 @@ class SymbolTableImpl : public SymbolTable { // Bitmap implementation. // The encode map stores both the symbol and the ref count of that symbol. // Using absl::string_view lets us only store the complete string once, in the decode map. - using EncodeMap = absl::flat_hash_map; + using EncodeMap = absl::flat_hash_map; using DecodeMap = absl::flat_hash_map; EncodeMap encode_map_ GUARDED_BY(lock_); DecodeMap decode_map_ GUARDED_BY(lock_); @@ -317,16 +317,31 @@ class StatName { // for lookup in a cache, and then on a miss need to store the data directly. StatName(const StatName& src, SymbolTable::Storage memory); + /** + * Defines default hash function so StatName can be used as a key in an absl + * hash-table without specifying a functor. See + * https://abseil.io/docs/cpp/guides/hash for details. + */ + template friend H AbslHashValue(H h, StatName stat_name) { + if (stat_name.empty()) { + return H::combine(std::move(h), absl::string_view()); + } + + // Casts the raw data as a string_view. Note that this string_view will not + // be in human-readable form, but it will be compatible with a string-view + // hasher. + const char* cdata = reinterpret_cast(stat_name.data()); + absl::string_view data_as_string_view = absl::string_view(cdata, stat_name.dataSize()); + return H::combine(std::move(h), data_as_string_view); + } + /** * Note that this hash function will return a different hash than that of * the elaborated string. * * @return uint64_t a hash of the underlying representation. */ - uint64_t hash() const { - const char* cdata = reinterpret_cast(data()); - return HashUtil::xxHash64(absl::string_view(cdata, dataSize())); - } + uint64_t hash() const { return absl::Hash()(*this); } bool operator==(const StatName& rhs) const { const uint64_t sz = dataSize(); @@ -339,6 +354,9 @@ class StatName { * overhead for the size itself. */ uint64_t dataSize() const { + if (size_and_data_ == nullptr) { + return 0; + } return size_and_data_[0] | (static_cast(size_and_data_[1]) << 8); } @@ -523,22 +541,11 @@ class StatNameList { SymbolTable::StoragePtr storage_; }; -// Helper class for constructing hash-tables with StatName keys. -struct StatNameHash { - size_t operator()(const StatName& a) const { return a.hash(); } -}; - -// Helper class for constructing hash-tables with StatName keys. -struct StatNameCompare { - bool operator()(const StatName& a, const StatName& b) const { return a == b; } -}; - // Value-templatized hash-map with StatName key. -template -using StatNameHashMap = absl::flat_hash_map; +template using StatNameHashMap = absl::flat_hash_map; // Hash-set of StatNames -using StatNameHashSet = absl::flat_hash_set; +using StatNameHashSet = absl::flat_hash_set; // Helper class for sorting StatNames. struct StatNameLessThan { diff --git a/source/common/stats/tag_producer_impl.h b/source/common/stats/tag_producer_impl.h index 6a4fcb07a6ef..e18f111e9238 100644 --- a/source/common/stats/tag_producer_impl.h +++ b/source/common/stats/tag_producer_impl.h @@ -17,6 +17,7 @@ #include "common/config/well_known_names.h" #include "common/protobuf/protobuf.h" +#include "absl/container/flat_hash_map.h" #include "absl/strings/string_view.h" namespace Envoy { @@ -97,8 +98,7 @@ class TagProducerImpl : public TagProducer { // Maps a prefix word extracted out of a regex to a vector of TagExtractors. Note that // the storage for the prefix string is owned by the TagExtractor, which, depending on // implementation, may need make a copy of the prefix. - std::unordered_map, StringViewHash> - tag_extractor_prefix_map_; + absl::flat_hash_map> tag_extractor_prefix_map_; std::vector default_tags_; }; diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index ab725c0a2608..b4b528194103 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -84,6 +84,7 @@ envoy_cc_test( envoy_cc_test( name = "symbol_table_impl_test", srcs = ["symbol_table_impl_test.cc"], + external_deps = ["abseil_hash_testing"], deps = [ ":stat_test_utility_lib", "//source/common/common:mutex_tracer_lib", diff --git a/test/common/stats/symbol_table_impl_test.cc b/test/common/stats/symbol_table_impl_test.cc index d6a533b092a5..4917caeca936 100644 --- a/test/common/stats/symbol_table_impl_test.cc +++ b/test/common/stats/symbol_table_impl_test.cc @@ -10,6 +10,7 @@ #include "test/test_common/logging.h" #include "test/test_common/utility.h" +#include "absl/hash/hash_testing.h" #include "absl/synchronization/blocking_counter.h" #include "gtest/gtest.h" @@ -565,6 +566,26 @@ TEST_P(StatNameTest, StatNameSet) { EXPECT_NE(dynamic2.data(), dynamic.data()); } +TEST_P(StatNameTest, StatNameEmptyEquivalent) { + StatName empty1; + StatName empty2 = makeStat(""); + StatName non_empty = makeStat("a"); + EXPECT_EQ(empty1, empty2); + EXPECT_EQ(empty1.hash(), empty2.hash()); + EXPECT_NE(empty1, non_empty); + EXPECT_NE(empty2, non_empty); + EXPECT_NE(empty1.hash(), non_empty.hash()); + EXPECT_NE(empty2.hash(), non_empty.hash()); +} + +TEST_P(StatNameTest, SupportsAbslHash) { + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly({ + StatName(), + makeStat(""), + makeStat("hello.world"), + })); +} + // Tests the memory savings realized from using symbol tables with 1k // clusters. This test shows the memory drops from almost 8M to less than // 2M. Note that only SymbolTableImpl is tested for memory consumption, From 555131514ed96ab75af45598a776b72358c16aaa Mon Sep 17 00:00:00 2001 From: Daniel Grimm Date: Thu, 12 Sep 2019 18:38:41 +0200 Subject: [PATCH 541/542] Update dependency: jwt_verify_lib (#8212) Signed-off-by: Daniel Grimm --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index cba60a106ac2..ab0285435934 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -161,10 +161,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/msgpack/msgpack-c/releases/download/cpp-3.2.0/msgpack-3.2.0.tar.gz"], ), com_github_google_jwt_verify = dict( - sha256 = "8ab9a0b3f8b7eab5f1cd059920e81fdc138cd4ee657c1412af891652929885c5", - strip_prefix = "jwt_verify_lib-6356535ae83a3f1820b6b06dae80cd6a0a03e7f2", - # 2019-07-01 - urls = ["https://github.com/google/jwt_verify_lib/archive/6356535ae83a3f1820b6b06dae80cd6a0a03e7f2.tar.gz"], + sha256 = "2d57d336239d5fe36a03849ddbea1bff09a1720e1c4a46bbb9743c71732b0d43", + strip_prefix = "jwt_verify_lib-0f14d43f20381cfae0469cb2309b2e220c0f0ea3", + # 2019-07-08 + urls = ["https://github.com/google/jwt_verify_lib/archive/0f14d43f20381cfae0469cb2309b2e220c0f0ea3.tar.gz"], ), com_github_nodejs_http_parser = dict( sha256 = "ef26268c54c8084d17654ba2ed5140bffeffd2a040a895ffb22a6cca3f6c613f", From 36cf26b9d6ac32f5671c24b92f9ea22ab70bc6c1 Mon Sep 17 00:00:00 2001 From: James Forcier Date: Thu, 12 Sep 2019 14:49:07 -0400 Subject: [PATCH 542/542] upstream: add failure percentage-based outlier detection (#8130) Description: Add a new outlier detection mode which compares each host's rate of request failure to a configured fixed threshold. Risk Level: Low Testing: 2 new unit tests added. Docs Changes: New mode and config options described. Release Notes: white_check_mark Fixes #8105 Signed-off-by: James Forcier --- .../api/v2/cluster/outlier_detection.proto | 30 ++ .../v2alpha/outlier_detection_event.proto | 12 + .../cluster_manager/cluster_runtime.rst | 25 ++ .../cluster_manager/cluster_stats.rst | 4 + .../intro/arch_overview/upstream/outlier.rst | 27 ++ docs/root/intro/version_history.rst | 1 + .../common/upstream/outlier_detection_impl.cc | 116 ++++++- .../common/upstream/outlier_detection_impl.h | 18 +- .../upstream/outlier_detection_impl_test.cc | 296 +++++++++++++++++- 9 files changed, 508 insertions(+), 21 deletions(-) diff --git a/api/envoy/api/v2/cluster/outlier_detection.proto b/api/envoy/api/v2/cluster/outlier_detection.proto index 72cf038cb034..d457c8165f49 100644 --- a/api/envoy/api/v2/cluster/outlier_detection.proto +++ b/api/envoy/api/v2/cluster/outlier_detection.proto @@ -111,4 +111,34 @@ message OutlierDetection { // is set to true. google.protobuf.UInt32Value enforcing_local_origin_success_rate = 15 [(validate.rules).uint32.lte = 100]; + + // The failure percentage to use when determining failure percentage-based outlier detection. If + // the failure percentage of a given host is greater than or equal to this value, it will be + // ejected. Defaults to 85. + google.protobuf.UInt32Value failure_percentage_threshold = 16 [(validate.rules).uint32.lte = 100]; + + // The % chance that a host will be actually ejected when an outlier status is detected through + // failure percentage statistics. This setting can be used to disable ejection or to ramp it up + // slowly. Defaults to 0. + // + // [#next-major-version: setting this without setting failure_percentage_threshold should be + // invalid in v4.] + google.protobuf.UInt32Value enforcing_failure_percentage = 17 [(validate.rules).uint32.lte = 100]; + + // The % chance that a host will be actually ejected when an outlier status is detected through + // local-origin failure percentage statistics. This setting can be used to disable ejection or to + // ramp it up slowly. Defaults to 0. + google.protobuf.UInt32Value enforcing_failure_percentage_local_origin = 18 + [(validate.rules).uint32.lte = 100]; + + // The minimum number of hosts in a cluster in order to perform failure percentage-based ejection. + // If the total number of hosts in the cluster is less than this value, failure percentage-based + // ejection will not be performed. Defaults to 5. + google.protobuf.UInt32Value failure_percentage_minimum_hosts = 19; + + // The minimum number of total requests that must be collected in one interval (as defined by the + // interval duration above) to perform failure percentage-based ejection for this host. If the + // volume is lower than this setting, failure percentage-based ejection will not be performed for + // this host. Defaults to 50. + google.protobuf.UInt32Value failure_percentage_request_volume = 20; } diff --git a/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto b/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto index 836eeec42837..1273f84d6df2 100644 --- a/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto +++ b/api/envoy/data/cluster/v2alpha/outlier_detection_event.proto @@ -39,6 +39,7 @@ message OutlierDetectionEvent { option (validate.required) = true; OutlierEjectSuccessRate eject_success_rate_event = 9; OutlierEjectConsecutive eject_consecutive_event = 10; + OutlierEjectFailurePercentage eject_failure_percentage_event = 11; } } @@ -75,6 +76,12 @@ enum OutlierEjectionType { // is set to *true*. // See :ref:`Cluster outlier detection ` documentation for SUCCESS_RATE_LOCAL_ORIGIN = 4; + // Runs over aggregated success rate statistics from every host in cluster and selects hosts for + // which ratio of failed replies is above configured value. + FAILURE_PERCENTAGE = 5; + // Runs over aggregated success rate statistics for local origin failures from every host in + // cluster and selects hosts for which ratio of failed replies is above configured value. + FAILURE_PERCENTAGE_LOCAL_ORIGIN = 6; } // Represents possible action applied to upstream host @@ -97,3 +104,8 @@ message OutlierEjectSuccessRate { message OutlierEjectConsecutive { } + +message OutlierEjectFailurePercentage { + // Host's success rate at the time of the ejection event on a 0-100 range. + uint32 host_success_rate = 1 [(validate.rules).uint32.lte = 100]; +} diff --git a/docs/root/configuration/upstream/cluster_manager/cluster_runtime.rst b/docs/root/configuration/upstream/cluster_manager/cluster_runtime.rst index a64750cf6625..195d025c24bc 100644 --- a/docs/root/configuration/upstream/cluster_manager/cluster_runtime.rst +++ b/docs/root/configuration/upstream/cluster_manager/cluster_runtime.rst @@ -102,6 +102,31 @@ outlier_detection.success_rate_stdev_factor ` setting in outlier detection +outlier_detection.enforcing_failure_percentage + :ref:`enforcing_failure_percentage + ` + setting in outlier detection + +outlier_detection.enforcing_failure_percentage_local_origin + :ref:`enforcing_failure_percentage_local_origin + ` + setting in outlier detection + +outlier_detection.failure_percentage_request_volume + :ref:`failure_percentage_request_volume + ` + setting in outlier detection + +outlier_detection.failure_percentage_minimum_hosts + :ref:`failure_percentage_minimum_hosts + ` + setting in outlier detection + +outlier_detection.failure_percentage_threshold + :ref:`failure_percentage_threshold + ` + setting in outlier detection + Core ---- diff --git a/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst b/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst index f4f1890baf8d..57bd438109b6 100644 --- a/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst +++ b/docs/root/configuration/upstream/cluster_manager/cluster_stats.rst @@ -141,6 +141,10 @@ statistics will be rooted at *cluster..outlier_detection.* and contain the ejections_detected_consecutive_local_origin_failure, Counter, Number of detected consecutive local origin failure ejections (even if unenforced) ejections_enforced_local_origin_success_rate, Counter, Number of enforced success rate outlier ejections for locally originated failures ejections_detected_local_origin_success_rate, Counter, Number of detected success rate outlier ejections for locally originated failures (even if unenforced) + ejections_enforced_failure_percentage, Counter, Number of enforced failure percentage outlier ejections. Exact meaning of this counter depends on :ref:`outlier_detection.split_external_local_origin_errors` config item. Refer to :ref:`Outlier Detection documentation` for details. + ejections_detected_failure_percentage, Counter, Number of detected failure percentage outlier ejections (even if unenforced). Exact meaning of this counter depends on :ref:`outlier_detection.split_external_local_origin_errors` config item. Refer to :ref:`Outlier Detection documentation` for details. + ejections_enforced_failure_percentage_local_origin, Counter, Number of enforced failure percentage outlier ejections for locally originated failures + ejections_detected_failure_percentage_local_origin, Counter, Number of detected failure percentage outlier ejections for locally originated failures (even if unenforced) ejections_total, Counter, Deprecated. Number of ejections due to any outlier type (even if unenforced) ejections_consecutive_5xx, Counter, Deprecated. Number of consecutive 5xx ejections (even if unenforced) diff --git a/docs/root/intro/arch_overview/upstream/outlier.rst b/docs/root/intro/arch_overview/upstream/outlier.rst index 5b3c392b6a70..6743fba99147 100644 --- a/docs/root/intro/arch_overview/upstream/outlier.rst +++ b/docs/root/intro/arch_overview/upstream/outlier.rst @@ -145,6 +145,33 @@ Most configuration items, namely types of errors, but :ref:`outlier_detection.enforcing_success_rate` applies to externally originated errors only and :ref:`outlier_detection.enforcing_local_origin_success_rate` applies to locally originated errors only. +.. _arch_overview_outlier_detection_failure_percentage: + +Failure Percentage +^^^^^^^^^^^^^^^^^^ + +Failure Percentage based outlier ejection functions similarly to the success rate detecion type, in +that it relies on success rate data from each host in a cluster. However, rather than compare those +values to the mean success rate of the cluster as a whole, they are compared to a flat +user-configured threshold. This threshold is configured via the +:ref:`outlier_detection.failure_percentage_threshold` +field. + +The other configuration fields for failure percentage based ejection are similar to the fields for +success rate ejection. Failure percentage based ejection also obeys +:ref:`outlier_detection.split_external_local_origin_errors`; +the enforcement percentages for externally- and locally-originated errors are controlled by +:ref:`outlier_detection.enforcing_failure_percentage` +and +:ref:`outlier_detection.enforcing_failure_percentage_local_origin`, +respectively. As with success rate detection, detection will not be performed for a host if its +request volume over the aggregation interval is less than the +:ref:`outlier_detection.failure_percentage_request_volume` +value. Detection also will not be performed for a cluster if the number of hosts with the minimum +required request volume in an interval is less than the +:ref:`outlier_detection.failure_percentage_minimum_hosts` +value. + .. _arch_overview_outlier_detection_grpc: gRPC diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 82d08ed8a443..11e887e225e0 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -61,6 +61,7 @@ Version history * tracing: added :ref:`max_path_tag_length ` to support customizing the length of the request path included in the extracted `http.url ` tag. * upstream: added :ref:`an option ` that allows draining HTTP, TCP connection pools on cluster membership change. * upstream: added network filter chains to upstream connections, see :ref:`filters`. +* upstream: added new :ref:`failure-percentage based outlier detection` mode. * upstream: use p2c to select hosts for least-requests load balancers if all host weights are the same, even in cases where weights are not equal to 1. * upstream: added :ref:`fail_traffic_on_panic ` to allow failing all requests to a cluster during panic state. * zookeeper: parse responses and emit latency stats. diff --git a/source/common/upstream/outlier_detection_impl.cc b/source/common/upstream/outlier_detection_impl.cc index 0d0efa147366..436a2ef28633 100644 --- a/source/common/upstream/outlier_detection_impl.cc +++ b/source/common/upstream/outlier_detection_impl.cc @@ -222,12 +222,22 @@ DetectorConfig::DetectorConfig(const envoy::api::v2::cluster::OutlierDetection& PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, success_rate_request_volume, 100))), success_rate_stdev_factor_(static_cast( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, success_rate_stdev_factor, 1900))), + failure_percentage_threshold_(static_cast( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, failure_percentage_threshold, 85))), + failure_percentage_minimum_hosts_(static_cast( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, failure_percentage_minimum_hosts, 5))), + failure_percentage_request_volume_(static_cast( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, failure_percentage_request_volume, 50))), enforcing_consecutive_5xx_(static_cast( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_consecutive_5xx, 100))), enforcing_consecutive_gateway_failure_(static_cast( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_consecutive_gateway_failure, 0))), enforcing_success_rate_(static_cast( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_success_rate, 100))), + enforcing_failure_percentage_(static_cast( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_failure_percentage, 0))), + enforcing_failure_percentage_local_origin_(static_cast( + PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, enforcing_failure_percentage_local_origin, 0))), split_external_local_origin_errors_(config.split_external_local_origin_errors()), consecutive_local_origin_failure_(static_cast( PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, consecutive_local_origin_failure, 5))), @@ -355,6 +365,13 @@ bool DetectorImpl::enforceEjection(envoy::data::cluster::v2alpha::OutlierEjectio return runtime_.snapshot().featureEnabled( "outlier_detection.enforcing_local_origin_success_rate", config_.enforcingLocalOriginSuccessRate()); + case envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE: + return runtime_.snapshot().featureEnabled("outlier_detection.enforcing_failure_percentage", + config_.enforcingFailurePercentage()); + case envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE_LOCAL_ORIGIN: + return runtime_.snapshot().featureEnabled( + "outlier_detection.enforcing_failure_percentage_local_origin", + config_.enforcingFailurePercentageLocalOrigin()); default: // Checked by schema. NOT_REACHED_GCOVR_EXCL_LINE; @@ -382,6 +399,12 @@ void DetectorImpl::updateEnforcedEjectionStats( case envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN: stats_.ejections_enforced_local_origin_success_rate_.inc(); break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE: + stats_.ejections_enforced_failure_percentage_.inc(); + break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE_LOCAL_ORIGIN: + stats_.ejections_enforced_local_origin_failure_percentage_.inc(); + break; default: // Checked by schema. NOT_REACHED_GCOVR_EXCL_LINE; @@ -406,6 +429,12 @@ void DetectorImpl::updateDetectedEjectionStats( case envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN: stats_.ejections_detected_local_origin_success_rate_.inc(); break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE: + stats_.ejections_detected_failure_percentage_.inc(); + break; + case envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE_LOCAL_ORIGIN: + stats_.ejections_detected_local_origin_failure_percentage_.inc(); + break; default: // Checked by schema. NOT_REACHED_GCOVR_EXCL_LINE; @@ -556,32 +585,55 @@ void DetectorImpl::processSuccessRateEjections( "outlier_detection.success_rate_minimum_hosts", config_.successRateMinimumHosts()); uint64_t success_rate_request_volume = runtime_.snapshot().getInteger( "outlier_detection.success_rate_request_volume", config_.successRateRequestVolume()); + uint64_t failure_percentage_minimum_hosts = + runtime_.snapshot().getInteger("outlier_detection.failure_percentage_minimum_hosts", + config_.failurePercentageMinimumHosts()); + uint64_t failure_percentage_request_volume = + runtime_.snapshot().getInteger("outlier_detection.failure_percentage_request_volume", + config_.failurePercentageRequestVolume()); + std::vector valid_success_rate_hosts; + std::vector valid_failure_percentage_hosts; double success_rate_sum = 0; // Reset the Detector's success rate mean and stdev. getSRNums(monitor_type) = {-1, -1}; // Exit early if there are not enough hosts. - if (host_monitors_.size() < success_rate_minimum_hosts) { + if (host_monitors_.size() < success_rate_minimum_hosts && + host_monitors_.size() < failure_percentage_minimum_hosts) { return; } // reserve upper bound of vector size to avoid reallocation. valid_success_rate_hosts.reserve(host_monitors_.size()); + valid_failure_percentage_hosts.reserve(host_monitors_.size()); for (const auto& host : host_monitors_) { // Don't do work if the host is already ejected. if (!host.first->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)) { - absl::optional host_success_rate = host.second->getSRMonitor(monitor_type) - .successRateAccumulator() - .getSuccessRate(success_rate_request_volume); - - if (host_success_rate) { - valid_success_rate_hosts.emplace_back( - HostSuccessRatePair(host.first, host_success_rate.value())); - success_rate_sum += host_success_rate.value(); - host.second->successRate(monitor_type, host_success_rate.value()); + absl::optional> host_success_rate_and_volume = + host.second->getSRMonitor(monitor_type) + .successRateAccumulator() + .getSuccessRateAndVolume(); + + if (!host_success_rate_and_volume) { + continue; + } + double success_rate = host_success_rate_and_volume.value().first; + double request_volume = host_success_rate_and_volume.value().second; + + if (request_volume >= + std::min(success_rate_request_volume, failure_percentage_request_volume)) { + host.second->successRate(monitor_type, success_rate); + } + + if (request_volume >= success_rate_request_volume) { + valid_success_rate_hosts.emplace_back(HostSuccessRatePair(host.first, success_rate)); + success_rate_sum += success_rate; + } + if (request_volume >= failure_percentage_request_volume) { + valid_failure_percentage_hosts.emplace_back(HostSuccessRatePair(host.first, success_rate)); } } } @@ -607,6 +659,28 @@ void DetectorImpl::processSuccessRateEjections( } } } + + if (!valid_failure_percentage_hosts.empty() && + valid_failure_percentage_hosts.size() >= failure_percentage_minimum_hosts) { + const double failure_percentage_threshold = runtime_.snapshot().getInteger( + "outlier_detection.failure_percentage_threshold", config_.failurePercentageThreshold()); + + for (const auto& host_success_rate_pair : valid_failure_percentage_hosts) { + if ((100.0 - host_success_rate_pair.success_rate_) >= failure_percentage_threshold) { + // We should eject. + + // The ejection type returned by the SuccessRateMonitor's getEjectionType() will be a + // SUCCESS_RATE type, so we need to figure it out for ourselves. + const envoy::data::cluster::v2alpha::OutlierEjectionType type = + (monitor_type == DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin) + ? envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE + : envoy::data::cluster::v2alpha::OutlierEjectionType:: + FAILURE_PERCENTAGE_LOCAL_ORIGIN; + updateDetectedEjectionStats(type); + ejectHost(host_success_rate_pair.host_, type); + } + } + } } void DetectorImpl::onIntervalTimer() { @@ -660,6 +734,15 @@ void EventLoggerImpl::logEject(const HostDescriptionConstSharedPtr& host, Detect detector.successRateEjectionThreshold(monitor_type)); event.mutable_eject_success_rate_event()->set_host_success_rate( host->outlierDetector().successRate(monitor_type)); + } else if ((type == envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE) || + (type == envoy::data::cluster::v2alpha::OutlierEjectionType:: + FAILURE_PERCENTAGE_LOCAL_ORIGIN)) { + const DetectorHostMonitor::SuccessRateMonitorType monitor_type = + (type == envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE) + ? DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin + : DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin; + event.mutable_eject_failure_percentage_event()->set_host_success_rate( + host->outlierDetector().successRate(monitor_type)); } else { event.mutable_eject_consecutive_event(); } @@ -707,14 +790,15 @@ SuccessRateAccumulatorBucket* SuccessRateAccumulator::updateCurrentWriter() { return current_success_rate_bucket_.get(); } -absl::optional -SuccessRateAccumulator::getSuccessRate(uint64_t success_rate_request_volume) { - if (backup_success_rate_bucket_->total_request_counter_ < success_rate_request_volume) { - return {}; +absl::optional> SuccessRateAccumulator::getSuccessRateAndVolume() { + if (!backup_success_rate_bucket_->total_request_counter_) { + return absl::nullopt; } - return {backup_success_rate_bucket_->success_request_counter_ * 100.0 / - backup_success_rate_bucket_->total_request_counter_}; + double success_rate = backup_success_rate_bucket_->success_request_counter_ * 100.0 / + backup_success_rate_bucket_->total_request_counter_; + + return {{success_rate, backup_success_rate_bucket_->total_request_counter_}}; } } // namespace Outlier diff --git a/source/common/upstream/outlier_detection_impl.h b/source/common/upstream/outlier_detection_impl.h index 2b703c2adba4..633cd52f16cb 100644 --- a/source/common/upstream/outlier_detection_impl.h +++ b/source/common/upstream/outlier_detection_impl.h @@ -92,7 +92,7 @@ class SuccessRateAccumulator { * @return a valid absl::optional with the success rate. If there were not enough * requests, an invalid absl::optional is returned. */ - absl::optional getSuccessRate(uint64_t success_rate_request_volume); + absl::optional> getSuccessRateAndVolume(); private: std::unique_ptr current_success_rate_bucket_; @@ -214,13 +214,17 @@ class DetectorHostMonitorImpl : public DetectorHostMonitor { COUNTER(ejections_detected_consecutive_5xx) \ COUNTER(ejections_detected_consecutive_gateway_failure) \ COUNTER(ejections_detected_success_rate) \ + COUNTER(ejections_detected_failure_percentage) \ COUNTER(ejections_enforced_consecutive_5xx) \ COUNTER(ejections_enforced_consecutive_gateway_failure) \ COUNTER(ejections_enforced_success_rate) \ + COUNTER(ejections_enforced_failure_percentage) \ COUNTER(ejections_detected_consecutive_local_origin_failure) \ COUNTER(ejections_enforced_consecutive_local_origin_failure) \ COUNTER(ejections_detected_local_origin_success_rate) \ COUNTER(ejections_enforced_local_origin_success_rate) \ + COUNTER(ejections_detected_local_origin_failure_percentage) \ + COUNTER(ejections_enforced_local_origin_failure_percentage) \ COUNTER(ejections_enforced_total) \ COUNTER(ejections_overflow) \ COUNTER(ejections_success_rate) \ @@ -249,11 +253,18 @@ class DetectorConfig { uint64_t successRateMinimumHosts() const { return success_rate_minimum_hosts_; } uint64_t successRateRequestVolume() const { return success_rate_request_volume_; } uint64_t successRateStdevFactor() const { return success_rate_stdev_factor_; } + uint64_t failurePercentageThreshold() const { return failure_percentage_threshold_; } + uint64_t failurePercentageMinimumHosts() const { return failure_percentage_minimum_hosts_; } + uint64_t failurePercentageRequestVolume() const { return failure_percentage_request_volume_; } uint64_t enforcingConsecutive5xx() const { return enforcing_consecutive_5xx_; } uint64_t enforcingConsecutiveGatewayFailure() const { return enforcing_consecutive_gateway_failure_; } uint64_t enforcingSuccessRate() const { return enforcing_success_rate_; } + uint64_t enforcingFailurePercentage() const { return enforcing_failure_percentage_; } + uint64_t enforcingFailurePercentageLocalOrigin() const { + return enforcing_failure_percentage_local_origin_; + } bool splitExternalLocalOriginErrors() const { return split_external_local_origin_errors_; } uint64_t consecutiveLocalOriginFailure() const { return consecutive_local_origin_failure_; } uint64_t enforcingConsecutiveLocalOriginFailure() const { @@ -270,9 +281,14 @@ class DetectorConfig { const uint64_t success_rate_minimum_hosts_; const uint64_t success_rate_request_volume_; const uint64_t success_rate_stdev_factor_; + const uint64_t failure_percentage_threshold_; + const uint64_t failure_percentage_minimum_hosts_; + const uint64_t failure_percentage_request_volume_; const uint64_t enforcing_consecutive_5xx_; const uint64_t enforcing_consecutive_gateway_failure_; const uint64_t enforcing_success_rate_; + const uint64_t enforcing_failure_percentage_; + const uint64_t enforcing_failure_percentage_local_origin_; const bool split_external_local_origin_errors_; const uint64_t consecutive_local_origin_failure_; const uint64_t enforcing_consecutive_local_origin_failure_; diff --git a/test/common/upstream/outlier_detection_impl_test.cc b/test/common/upstream/outlier_detection_impl_test.cc index 8c8c51062031..1df588abbc6f 100644 --- a/test/common/upstream/outlier_detection_impl_test.cc +++ b/test/common/upstream/outlier_detection_impl_test.cc @@ -129,6 +129,9 @@ enforcing_success_rate: 20 success_rate_minimum_hosts: 50 success_rate_request_volume: 200 success_rate_stdev_factor: 3000 +failure_percentage_minimum_hosts: 10 +failure_percentage_request_volume: 25 +failure_percentage_threshold: 70 )EOF"; envoy::api::v2::cluster::OutlierDetection outlier_detection; @@ -148,6 +151,11 @@ success_rate_stdev_factor: 3000 EXPECT_EQ(50UL, detector->config().successRateMinimumHosts()); EXPECT_EQ(200UL, detector->config().successRateRequestVolume()); EXPECT_EQ(3000UL, detector->config().successRateStdevFactor()); + EXPECT_EQ(0UL, detector->config().enforcingFailurePercentage()); + EXPECT_EQ(0UL, detector->config().enforcingFailurePercentageLocalOrigin()); + EXPECT_EQ(10UL, detector->config().failurePercentageMinimumHosts()); + EXPECT_EQ(25UL, detector->config().failurePercentageRequestVolume()); + EXPECT_EQ(70UL, detector->config().failurePercentageThreshold()); } TEST_F(OutlierDetectorImplTest, DestroyWithActive) { @@ -902,9 +910,11 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateExternalOrigin) { time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); + // The success rate should be *calculated* since the minimum request volume was met for failure + // percentage ejection, but the host should not be ejected. EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); - EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( - DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(50UL, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); EXPECT_EQ(-1, detector->successRateAverage( DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); EXPECT_EQ(-1, detector->successRateEjectionThreshold( @@ -1032,9 +1042,11 @@ TEST_F(OutlierDetectorImplTest, BasicFlowSuccessRateLocalOrigin) { time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); interval_timer_->invokeCallback(); + // The success rate should be *calculated* since the minimum request volume was met for failure + // percentage ejection, but the host should not be ejected. EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); - EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( - DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(50UL, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); EXPECT_EQ(-1, detector->successRateAverage(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); EXPECT_EQ(-1, detector->successRateEjectionThreshold( @@ -1056,6 +1068,252 @@ TEST_F(OutlierDetectorImplTest, EmptySuccessRate) { interval_timer_->invokeCallback(); } +TEST_F(OutlierDetectorImplTest, BasicFlowFailurePercentageExternalOrigin) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({ + "tcp://127.0.0.1:80", + "tcp://127.0.0.1:81", + "tcp://127.0.0.1:82", + "tcp://127.0.0.1:83", + "tcp://127.0.0.1:84", + }); + + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + std::shared_ptr detector(DetectorImpl::create( + cluster_, empty_outlier_detection_, dispatcher_, runtime_, time_system_, event_logger_)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + // Turn off 5xx detection and SR detection to test failure percentage detection in isolation. + ON_CALL(runtime_.snapshot_, featureEnabled("outlier_detection.enforcing_consecutive_5xx", 100)) + .WillByDefault(Return(false)); + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_consecutive_gateway_failure", 100)) + .WillByDefault(Return(false)); + ON_CALL(runtime_.snapshot_, featureEnabled("outlier_detection.enforcing_success_rate", 100)) + .WillByDefault(Return(false)); + // Now turn on failure percentage detection. + ON_CALL(runtime_.snapshot_, featureEnabled("outlier_detection.enforcing_failure_percentage", 0)) + .WillByDefault(Return(true)); + // Expect non-enforcing logging to happen every time the consecutive_5xx_ counter + // gets saturated (every 5 times). + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[3]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, false)) + .Times(50); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[3]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE, + false)) + .Times(50); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, false)) + .Times(60); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE, + false)) + .Times(60); + + // Cause a failure percentage error on one host. First 3 hosts have perfect failure percentage; + // fourth host has failure percentage slightly below threshold; fifth has failure percentage + // slightly above threshold. + loadRq(hosts_, 50, 200); + loadRq(hosts_[3], 250, 503); + loadRq(hosts_[4], 300, 503); + + time_system_.setMonotonicTime(std::chrono::milliseconds(10000)); + EXPECT_CALL(checker_, check(hosts_[4])); + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE, + true)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.success_rate_stdev_factor", 1900)) + .WillByDefault(Return(1900)); + interval_timer_->invokeCallback(); + EXPECT_FLOAT_EQ(100.0 * (50.0 / 300.0), + hosts_[3]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_FLOAT_EQ(100.0 * (50.0 / 350.0), + hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + // Make sure that local origin success rate monitor is not affected + EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(-1, + detector->successRateAverage(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(-1, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_FALSE(hosts_[3]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Interval that doesn't bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(19999)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + interval_timer_->invokeCallback(); + EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Interval that does bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(50001)); + EXPECT_CALL(checker_, check(hosts_[4])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[4]))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + interval_timer_->invokeCallback(); + EXPECT_FALSE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + + // Expect non-enforcing logging to happen every time the consecutive_5xx_ counter + // gets saturated (every 5 times). + EXPECT_CALL(*event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_5XX, false)) + .Times(5); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_GATEWAY_FAILURE, + false)) + .Times(5); + + // Give 4 hosts enough request volume but not to the 5th. Should not cause an ejection. + loadRq(hosts_, 25, 200); + loadRq(hosts_[4], 25, 503); + + time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + interval_timer_->invokeCallback(); + // The success rate should be *calculated* since the minimum request volume was met for failure + // percentage ejection, but the host should not be ejected. + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + EXPECT_EQ(50UL, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(-1, detector->successRateAverage( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(-1, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); +} + +TEST_F(OutlierDetectorImplTest, BasicFlowFailurePercentageLocalOrigin) { + EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); + addHosts({ + "tcp://127.0.0.1:80", + "tcp://127.0.0.1:81", + "tcp://127.0.0.1:82", + "tcp://127.0.0.1:83", + "tcp://127.0.0.1:84", + }); + + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + std::shared_ptr detector(DetectorImpl::create( + cluster_, outlier_detection_split_, dispatcher_, runtime_, time_system_, event_logger_)); + detector->addChangedStateCb([&](HostSharedPtr host) -> void { checker_.check(host); }); + + // Turn off 5xx detection and SR detection to test failure percentage detection in isolation. + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_consecutive_local_origin_failure", 100)) + .WillByDefault(Return(false)); + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_local_origin_success_rate", 100)) + .WillByDefault(Return(false)); + // Now turn on failure percentage detection. + ON_CALL(runtime_.snapshot_, + featureEnabled("outlier_detection.enforcing_failure_percentage_local_origin", 0)) + .WillByDefault(Return(true)); + // Expect non-enforcing logging to happen every time the consecutive_ counter + // gets saturated (every 5 times). + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE, + false)) + .Times(40); + // Cause a failure percentage error on one host. First 4 of the hosts have perfect failure + // percentage. + loadRq(hosts_, 200, Result::LOCAL_ORIGIN_CONNECT_SUCCESS); + loadRq(hosts_[4], 200, Result::LOCAL_ORIGIN_CONNECT_FAILED); + + time_system_.setMonotonicTime(std::chrono::milliseconds(10000)); + EXPECT_CALL(checker_, check(hosts_[4])); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE_LOCAL_ORIGIN, + true)); + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::SUCCESS_RATE_LOCAL_ORIGIN, + false)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + ON_CALL(runtime_.snapshot_, getInteger("outlier_detection.failure_percentage_threshold", 85)) + .WillByDefault(Return(40)); + interval_timer_->invokeCallback(); + EXPECT_EQ(50, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(90, + detector->successRateAverage(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(52, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + // Make sure that external origin success rate monitor is not affected + EXPECT_EQ(-1, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(-1, detector->successRateAverage( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_EQ(-1, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)); + EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Interval that doesn't bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(19999)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + interval_timer_->invokeCallback(); + EXPECT_TRUE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(1UL, outlier_detection_ejections_active_.value()); + + // Interval that does bring the host back in. + time_system_.setMonotonicTime(std::chrono::milliseconds(50001)); + EXPECT_CALL(checker_, check(hosts_[4])); + EXPECT_CALL(*event_logger_, + logUneject(std::static_pointer_cast(hosts_[4]))); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + interval_timer_->invokeCallback(); + EXPECT_FALSE(hosts_[4]->healthFlagGet(Host::HealthFlag::FAILED_OUTLIER_CHECK)); + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + + // Expect non-enforcing logging to happen every time the consecutive_ counter + // gets saturated (every 5 times). + EXPECT_CALL( + *event_logger_, + logEject(std::static_pointer_cast(hosts_[4]), _, + envoy::data::cluster::v2alpha::OutlierEjectionType::CONSECUTIVE_LOCAL_ORIGIN_FAILURE, + false)) + .Times(5); + + // Give 4 hosts enough request volume but not to the 5th. Should not cause an ejection. + loadRq(hosts_, 25, Result::LOCAL_ORIGIN_CONNECT_SUCCESS); + loadRq(hosts_[4], 25, Result::LOCAL_ORIGIN_CONNECT_FAILED); + + time_system_.setMonotonicTime(std::chrono::milliseconds(60001)); + EXPECT_CALL(*interval_timer_, enableTimer(std::chrono::milliseconds(10000), _)); + interval_timer_->invokeCallback(); + // The success rate should be *calculated* since the minimum request volume was met for failure + // percentage ejection, but the host should not be ejected. + EXPECT_EQ(0UL, outlier_detection_ejections_active_.value()); + EXPECT_EQ(50UL, hosts_[4]->outlierDetector().successRate( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(-1, + detector->successRateAverage(DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); + EXPECT_EQ(-1, detector->successRateEjectionThreshold( + DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)); +} + TEST_F(OutlierDetectorImplTest, RemoveWhileEjected) { EXPECT_CALL(cluster_.prioritySet(), addMemberUpdateCb(_)); addHosts({"tcp://127.0.0.1:80"}); @@ -1345,6 +1603,36 @@ TEST(OutlierDetectionEventLoggerImplTest, All) { .WillOnce(SaveArg<0>(&log4)); event_logger.logUneject(host); Json::Factory::loadFromString(log4); + + StringViewSaver log5; + EXPECT_CALL(host->outlier_detector_, lastUnejectionTime()).WillOnce(ReturnRef(monotonic_time)); + EXPECT_CALL(host->outlier_detector_, + successRate(DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)) + .WillOnce(Return(0)); + EXPECT_CALL(*file, + write(absl::string_view( + "{\"type\":\"FAILURE_PERCENTAGE\",\"cluster_name\":\"fake_cluster\"," + "\"upstream_url\":\"10.0.0.1:443\",\"action\":\"EJECT\"," + "\"num_ejections\":0,\"enforced\":false,\"eject_failure_percentage_event\":{" + "\"host_success_rate\":0},\"timestamp\":\"2018-12-18T09:00:00Z\"," + "\"secs_since_last_action\":\"30\"}\n"))) + .WillOnce(SaveArg<0>(&log5)); + event_logger.logEject(host, detector, + envoy::data::cluster::v2alpha::OutlierEjectionType::FAILURE_PERCENTAGE, + false); + Json::Factory::loadFromString(log5); + + StringViewSaver log6; + EXPECT_CALL(host->outlier_detector_, lastEjectionTime()).WillOnce(ReturnRef(monotonic_time)); + EXPECT_CALL(*file, + write(absl::string_view( + "{\"type\":\"CONSECUTIVE_5XX\",\"cluster_name\":\"fake_cluster\"," + "\"upstream_url\":\"10.0.0.1:443\",\"action\":\"UNEJECT\"," + "\"num_ejections\":0,\"enforced\":false,\"timestamp\":\"2018-12-18T09:00:00Z\"," + "\"secs_since_last_action\":\"30\"}\n"))) + .WillOnce(SaveArg<0>(&log6)); + event_logger.logUneject(host); + Json::Factory::loadFromString(log6); } TEST(OutlierUtility, SRThreshold) {